diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py index d5e4a7cf66..a4009af5d3 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/cli.py @@ -13,7 +13,8 @@ import httpx import typer -from nemo_platform_ext.cli.commands.services._process import ( +from nemo_platform_ext.cli.core.help_formatter import create_typer_app +from nemo_platform_ext.local.process import ( ForegroundInstanceError, InstanceAlreadyRunningError, InstanceDescriptor, @@ -24,7 +25,6 @@ check_port_available_for_start, compute_scope, format_port_conflict, - get_create_time, instance_log_bytes, is_instance_alive, list_instances, @@ -37,7 +37,7 @@ stop_instance, write_descriptor, ) -from nemo_platform_ext.cli.core.help_formatter import create_typer_app +from nmp.platform_runner.config import DEFAULT_LOCAL_SERVICES_BIND_HOST, PlatformAppConfig logger = logging.getLogger(__name__) @@ -45,7 +45,6 @@ _HEALTH_TIMEOUT_SECONDS = 60 _HEALTH_POLL_INTERVAL = 2.0 -_DEFAULT_HOST = "127.0.0.1" _DEFAULT_PORT = 8080 _DEFAULT_STOP_TIMEOUT = 30.0 @@ -60,7 +59,7 @@ def services_callback(ctx: typer.Context) -> None: for info in running: desc = info.descriptor assert desc is not None - typer.echo(f"\nRunning: {info.scope} (pid {desc.pid}, {desc.host}:{desc.port}, {desc.mode})") + typer.echo(f"\nRunning: {info.scope} (pid {desc.pid}, {desc.config.host}:{desc.config.port}, {desc.mode})") def _require_services_extra() -> None: @@ -92,7 +91,7 @@ def _parse_csv_option(value: str | None) -> list[str] | None: def _wait_for_healthy( host: str, port: int, - timeout: int = _HEALTH_TIMEOUT_SECONDS, + timeout: float = _HEALTH_TIMEOUT_SECONDS, poll_interval: float = _HEALTH_POLL_INTERVAL, ) -> bool: """Poll the platform status endpoint until it responds or timeout.""" @@ -124,15 +123,8 @@ def _effective_base_dir() -> str | None: def _find_sole_running_scope(base_dir: Path | None) -> str: - """Find the scope of the single running instance for this working directory. - - When the user runs ``restart`` without ``--instance`` or ``--port``, we - can't know which scope to target because the scope includes the port. - This function scans all running instances whose scope starts with the - same git-root hash prefix. If exactly one matches, return it. - Otherwise fall back to the default scope (hash-DEFAULT_PORT). - """ - prefix = compute_scope(port=0, instance_name=None).rsplit("-", 1)[0] + """Return the only running scope for this working directory, or the default scope.""" + prefix = compute_scope(port=0).rsplit("-", 1)[0] running = [i for i in list_instances(base_dir=base_dir) if i.alive and i.scope.startswith(prefix + "-")] if len(running) == 1: return running[0].scope @@ -213,7 +205,7 @@ def run_services( str | None, typer.Option("--config", help="Path to a platform configuration YAML file."), ] = None, - host: Annotated[str, typer.Option("--host", help="Host to bind to.")] = _DEFAULT_HOST, + host: Annotated[str, typer.Option("--host", help="Host to bind to.")] = DEFAULT_LOCAL_SERVICES_BIND_HOST, port: Annotated[int, typer.Option("--port", help="Port to bind to.")] = _DEFAULT_PORT, instance: Annotated[ str | None, @@ -226,7 +218,7 @@ def run_services( _require_services_extra() _warn_bind_all(host) - scope = compute_scope(port=port, instance_name=instance) + scope = compute_scope(port=port, explicit_scope=instance) base_dir_str = _effective_base_dir() base_dir = Path(base_dir_str) if base_dir_str else None @@ -242,20 +234,23 @@ def run_services( # "foreground", which protects interactive ``run`` sessions from being # killed by ``stop``. mode = "background" if os.environ.get("_NMP_LAUNCH_MODE") == "background" else "foreground" - - desc = InstanceDescriptor( - pid=os.getpid(), - scope=scope, - host=host, - port=port, - mode=mode, - create_time=get_create_time(os.getpid()), + platform_config = PlatformAppConfig( services=_parse_csv_option(services), - controllers=_parse_csv_option(controllers), service_group=service_group, + controllers=_parse_csv_option(controllers), controller_group=controller_group, sidecars=_parse_csv_option(sidecars), config_path=config, + scope=scope, + host=host, + port=port, + state_root=base_dir, + ) + + desc = InstanceDescriptor.from_config( + platform_config, + mode=mode, + pid=os.getpid(), ) write_descriptor(desc, base_dir=base_dir) @@ -269,14 +264,7 @@ def _cleanup() -> None: from nmp.platform_runner.run import run_platform run_platform( - services=_parse_csv_option(services), - service_group=service_group, - controllers=_parse_csv_option(controllers), - controller_group=controller_group, - sidecars=_parse_csv_option(sidecars), - config_path=config, - host=host, - port=port, + config=platform_config, on_shutdown=_cleanup, ) @@ -327,7 +315,7 @@ def start_services( str | None, typer.Option("--config", help="Path to a platform configuration YAML file."), ] = None, - host: Annotated[str, typer.Option("--host", help="Host to bind to.")] = _DEFAULT_HOST, + host: Annotated[str, typer.Option("--host", help="Host to bind to.")] = DEFAULT_LOCAL_SERVICES_BIND_HOST, port: Annotated[int, typer.Option("--port", help="Port to bind to.")] = _DEFAULT_PORT, instance: Annotated[ str | None, @@ -351,7 +339,7 @@ def start_services( raise typer.BadParameter("Cannot combine --controllers with --controller-group.") _warn_bind_all(host) - scope = compute_scope(port=port, instance_name=instance) + scope = compute_scope(port=port, explicit_scope=instance) base_dir_str = _effective_base_dir() base_dir = Path(base_dir_str) if base_dir_str else None @@ -360,20 +348,22 @@ def start_services( _ensure_port_available(host, port, scope, base_dir=base_dir) - typer.echo("Starting platform services...") - proc = start_background( - scope=scope, + platform_config = PlatformAppConfig( services=_parse_csv_option(services), service_group=service_group, controllers=_parse_csv_option(controllers), controller_group=controller_group, sidecars=_parse_csv_option(sidecars), config_path=config, + scope=scope, host=host, port=port, - base_dir=base_dir, + state_root=base_dir, ) + typer.echo("Starting platform services...") + proc = start_background(platform_config) + if not _wait_for_healthy(host, port): exit_code = proc.poll() if exit_code is not None: @@ -426,7 +416,7 @@ def stop_services_cmd( nemo services stop nemo services stop --timeout 60 """ - scope = compute_scope(port=port, instance_name=instance) + scope = compute_scope(port=port, explicit_scope=instance) base_dir_str = _effective_base_dir() base_dir = Path(base_dir_str) if base_dir_str else None @@ -496,7 +486,10 @@ def restart_services( ] = None, host: Annotated[ str | None, - typer.Option("--host", help="Host to bind to. Defaults to previous value or 127.0.0.1."), + typer.Option( + "--host", + help=f"Host to bind to. Defaults to previous value or {DEFAULT_LOCAL_SERVICES_BIND_HOST}.", + ), ] = None, port: Annotated[ int | None, @@ -529,8 +522,8 @@ def restart_services( base_dir = Path(base_dir_str) if base_dir_str else None if instance is not None or port is not None: - effective_port = port if port is not None else _DEFAULT_PORT - scope = compute_scope(port=effective_port, instance_name=instance) + effective_scope_port = port if port is not None else _DEFAULT_PORT + scope = compute_scope(port=effective_scope_port, explicit_scope=instance) else: scope = _find_sole_running_scope(base_dir) @@ -547,37 +540,47 @@ def restart_services( # appropriate even for foreground targets. stop_instance(scope, base_dir=base_dir, force=True) - effective_services = _parse_csv_option(services) if services is not None else (prev.services if prev else None) - effective_service_group = service_group if service_group is not None else (prev.service_group if prev else None) - effective_controllers = ( - _parse_csv_option(controllers) if controllers is not None else (prev.controllers if prev else None) + previous_config = prev.config if prev else None + effective_services = _parse_csv_option(services) if services is not None else None + if services is None and previous_config is not None: + effective_services = previous_config.services + effective_service_group = service_group if service_group is not None else None + if service_group is None and previous_config is not None: + effective_service_group = previous_config.service_group + effective_controllers = _parse_csv_option(controllers) if controllers is not None else None + if controllers is None and previous_config is not None: + effective_controllers = previous_config.controllers + effective_controller_group = controller_group if controller_group is not None else None + if controller_group is None and previous_config is not None: + effective_controller_group = previous_config.controller_group + effective_sidecars = _parse_csv_option(sidecars) if sidecars is not None else None + if sidecars is None and previous_config is not None: + effective_sidecars = previous_config.sidecars + effective_config = config if config is not None else (previous_config.config_path if previous_config else None) + effective_host = ( + host if host is not None else (previous_config.host if previous_config else DEFAULT_LOCAL_SERVICES_BIND_HOST) ) - effective_controller_group = ( - controller_group if controller_group is not None else (prev.controller_group if prev else None) - ) - effective_sidecars = _parse_csv_option(sidecars) if sidecars is not None else (prev.sidecars if prev else None) - effective_config = config if config is not None else (prev.config_path if prev else None) - effective_host = host if host is not None else (prev.host if prev else _DEFAULT_HOST) - effective_port = port if port is not None else (prev.port if prev else _DEFAULT_PORT) + effective_port = port if port is not None else (previous_config.port if previous_config else _DEFAULT_PORT) _warn_bind_all(effective_host) _ensure_port_available(effective_host, effective_port, scope, base_dir=base_dir) - - typer.echo("Starting platform services...") - proc = start_background( - scope=scope, + platform_config = PlatformAppConfig( services=effective_services, service_group=effective_service_group, controllers=effective_controllers, controller_group=effective_controller_group, sidecars=effective_sidecars, config_path=effective_config, + scope=scope, host=effective_host, port=effective_port, - base_dir=base_dir, + state_root=base_dir, ) + typer.echo("Starting platform services...") + proc = start_background(platform_config) + if not _wait_for_healthy(effective_host, effective_port): exit_code = proc.poll() if exit_code is not None: @@ -613,7 +616,7 @@ def status_services( ] = _DEFAULT_PORT, ) -> None: """Show status of the platform services instance for this scope.""" - scope = compute_scope(port=port, instance_name=instance) + scope = compute_scope(port=port, explicit_scope=instance) base_dir_str = _effective_base_dir() base_dir = Path(base_dir_str) if base_dir_str else None @@ -641,13 +644,13 @@ def status_services( except ValueError: uptime = "unknown" - healthy = _wait_for_healthy(desc.host, desc.port, timeout=3, poll_interval=0.5) + healthy = _wait_for_healthy(desc.config.host, desc.config.port, timeout=3, poll_interval=0.5) health_str = "healthy" if healthy else "unhealthy" - typer.echo(f"Scope: {desc.scope}") + typer.echo(f"Scope: {desc.config.scope}") typer.echo(f"PID: {desc.pid}") typer.echo(f"Mode: {desc.mode}") - typer.echo(f"Address: {desc.host}:{desc.port}") + typer.echo(f"Address: {desc.config.host}:{desc.config.port}") typer.echo(f"Uptime: {uptime}") typer.echo(f"Health: {health_str}") log = log_path_for(scope, base_dir=base_dir) @@ -668,7 +671,7 @@ def _print_instance_table(instances: list[InstanceInfo]) -> None: pid = addr = mode = "-" if info.descriptor: pid = str(info.descriptor.pid) - addr = f"{info.descriptor.host}:{info.descriptor.port}" + addr = f"{info.descriptor.config.host}:{info.descriptor.config.port}" mode = info.descriptor.mode typer.echo(f"{info.scope:<25} {status:<10} {pid:<10} {addr:<25} {mode:<12}") @@ -872,7 +875,7 @@ def logs_services( nemo services logs --path nemo services logs -n 100 """ - scope = compute_scope(port=port, instance_name=instance) + scope = compute_scope(port=port, explicit_scope=instance) base_dir_str = _effective_base_dir() base_dir = Path(base_dir_str) if base_dir_str else None diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py index 1c52e750b1..626d56bd07 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/setup.py @@ -30,20 +30,12 @@ from nemo_platform_plugin.secrets.client import SecretsClient from nemo_platform_plugin.secrets.types import PlatformSecretCreateRequest, PlatformSecretUpdateRequest from nmp.common.config import nmp_user_data_dir +from nmp.platform_runner.config import DEFAULT_LOCAL_SERVICES_BIND_HOST, PlatformAppConfig from pydantic import SecretStr from rich import box from rich.console import Console from rich.panel import Panel -from nemo_platform_ext.cli.commands.services._process import ( - DEFAULT_SERVICES_BIND_HOST, - check_port_available_for_start, - compute_scope, - format_port_conflict, - log_path_for, - start_background, - stop_instance, -) from nemo_platform_ext.cli.commands.skills import registry as skills_registry from nemo_platform_ext.cli.commands.skills.base import Scope, Skill from nemo_platform_ext.cli.commands.skills.registry import get_installer, load_skills @@ -51,6 +43,14 @@ from nemo_platform_ext.cli.core.errors import handle_errors from nemo_platform_ext.config.config import Config from nemo_platform_ext.config.models import ConfigFile, ConfigParams, LocalServicesConfig +from nemo_platform_ext.local.process import ( + check_port_available_for_start, + compute_scope, + format_port_conflict, + log_path_for, + start_background, + stop_instance, +) from nemo_platform_ext.ui.prompts import ( UserCancelled, is_interactive, @@ -635,8 +635,10 @@ def _start_services_background(base_url: str, data_dir: str | None = None) -> su exported it). """ port = _resolve_services_port(base_url) - scope = compute_scope(port=port) - return start_background(scope=scope, port=port, data_dir=data_dir) + return start_background( + PlatformAppConfig(scope=compute_scope(port=port), port=port), + data_dir=data_dir, + ) def _last_startup_service(log_path: Path | None) -> str: @@ -686,16 +688,13 @@ def _kill_existing_services(base_url: str) -> None: Delegates to the shared process lifecycle module. """ - port = _resolve_services_port(base_url) - scope = compute_scope(port=port) - stop_instance(scope, timeout=2.0, force=True) + stop_instance(compute_scope(port=_resolve_services_port(base_url)), timeout=2.0, force=True) def _ensure_port_available_for_start(base_url: str) -> None: """Fail fast when the services port cannot be bound.""" port = _resolve_services_port(base_url) - scope = compute_scope(port=port) - conflict = check_port_available_for_start(DEFAULT_SERVICES_BIND_HOST, port, scope) + conflict = check_port_available_for_start(DEFAULT_LOCAL_SERVICES_BIND_HOST, port, compute_scope(port=port)) if conflict is None: return lines = format_port_conflict(conflict) @@ -777,8 +776,7 @@ def _maybe_start_services( _ensure_port_available_for_start(base_url) proc = _start_services_background(base_url, data_dir=data_dir) - port = _resolve_services_port(base_url) - log = log_path_for(compute_scope(port=port)) + log = log_path_for(compute_scope(port=_resolve_services_port(base_url))) if not _wait_for_platform(base_url, timeout=timeout, log_path=log): exit_code = proc.poll() diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/local/_service_child.py b/packages/nemo_platform_ext/src/nemo_platform_ext/local/_service_child.py new file mode 100644 index 0000000000..9cecdeec28 --- /dev/null +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/local/_service_child.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Child entrypoint for SDK-started local services daemons.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from nemo_platform_ext.local.services import ServiceRunConfig, run_services + + +def main(argv: list[str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + if len(args) != 1: + sys.stderr.write("usage: python -m nemo_platform_ext.local._service_child \n") + return 2 + request_path = Path(args[0]) + try: + payload = json.loads(request_path.read_text(encoding="utf-8")) + finally: + request_path.unlink(missing_ok=True) + run_services(ServiceRunConfig(**payload), _mode="daemon") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/_process.py b/packages/nemo_platform_ext/src/nemo_platform_ext/local/process.py similarity index 76% rename from packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/_process.py rename to packages/nemo_platform_ext/src/nemo_platform_ext/local/process.py index 8ce8cda699..7ee0cd4a9b 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/services/_process.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/local/process.py @@ -3,8 +3,13 @@ """Local process lifecycle for ``nemo services``. -Uses per-instance scoped directories under ``$XDG_STATE_HOME/nmp/instances/`` -with flock-based liveness tracking. Each instance directory contains: +In this module, "instance" is a local services process/resource, and "scope" +is the stable key used for that instance's lock, descriptor, socket, and log +paths. The CLI exposes this key as ``--instance`` for compatibility, but +internal code should use "scope" when referring to the key. + +Uses per-scope directories under ``$XDG_STATE_HOME/nmp/instances/`` +with flock-based liveness tracking. Each scope directory contains: - ``services.lock`` -- exclusive flock held for the process lifetime - ``instance.json`` -- descriptor with PID, port, services, etc. @@ -23,7 +28,6 @@ import json import logging import os -import re import shutil import signal import socket @@ -34,10 +38,16 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path -from typing import Literal +from typing import Literal, Self import psutil -from pydantic import BaseModel, Field +from nmp.platform_runner.config import ( + DEFAULT_LOCAL_SERVICES_BIND_HOST, + PlatformAppConfig, + default_state_root, + validate_scope, +) +from pydantic import BaseModel, Field, model_validator logger = logging.getLogger(__name__) @@ -45,10 +55,10 @@ DESCRIPTOR_FILENAME = "instance.json" LOG_FILENAME = "services.log" -DEFAULT_SERVICES_BIND_HOST = "127.0.0.1" SUGGESTED_ALT_PORT = 9090 _SIGTERM_POLL_INTERVAL = 0.25 +_SIGKILL_WAIT_TIMEOUT = 5.0 _DEFAULT_STOP_TIMEOUT = 30.0 @@ -62,10 +72,7 @@ def _pause(seconds: float) -> None: def _base_state_dir() -> Path: - xdg = os.environ.get("XDG_STATE_HOME") - if xdg: - return Path(xdg) / "nmp" - return Path.home() / ".local" / "state" / "nmp" + return default_state_root() def _instances_dir(*, base_dir: Path | None = None) -> Path: @@ -73,7 +80,7 @@ def _instances_dir(*, base_dir: Path | None = None) -> Path: def _find_git_root() -> str: - """Walk up from cwd looking for a ``.git`` directory. Falls back to cwd.""" + """Walk up from cwd looking for a ``.git`` directory. Falls back to cwd.""" cur = Path.cwd().resolve() for parent in (cur, *cur.parents): if (parent / ".git").exists(): @@ -84,24 +91,19 @@ def _find_git_root() -> str: _scope_prefix_cache: str | None = None -_SCOPE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") - - -def _validate_scope(scope: str) -> str: - """Ensure *scope* is safe to use as a directory name.""" - if not _SCOPE_RE.fullmatch(scope): - raise ValueError(f"Invalid instance scope: {scope!r}") - return scope +def compute_scope(*, port: int, explicit_scope: str | None = None) -> str: + """Compute the local services scope. + The default scope is ``sha1(git_toplevel_or_cwd)[:8]-``. Including + the port is intentional: it lets two local services instances from the same + checkout use different TCP ports without sharing a lock, descriptor, or log + directory. -def compute_scope(*, port: int, instance_name: str | None = None) -> str: - """Compute a scope identifier for this working directory + port. - - Default: ``sha1(git_toplevel_or_cwd)[:8]-``. - Override with an explicit *instance_name*. + Explicit scopes are validated and returned as-is, so they do not encode the + port. Callers that pass an explicit scope own its uniqueness. """ - if instance_name: - return _validate_scope(instance_name) + if explicit_scope: + return validate_scope(explicit_scope) global _scope_prefix_cache # noqa: PLW0603 if _scope_prefix_cache is None: root = _find_git_root() @@ -110,7 +112,8 @@ def compute_scope(*, port: int, instance_name: str | None = None) -> str: def instance_dir(scope: str, *, base_dir: Path | None = None) -> Path: - d = _instances_dir(base_dir=base_dir) / _validate_scope(scope) + """Return the state directory for *scope*, creating it if needed.""" + d = _instances_dir(base_dir=base_dir) / validate_scope(scope) d.mkdir(parents=True, exist_ok=True) return d @@ -222,7 +225,7 @@ def _instance_owns_listener( desc = read_descriptor(scope, base_dir=base_dir) if desc is None: return False - return desc.port == port and _normalize_bind_host(desc.host) == _normalize_bind_host(host) + return desc.config.port == port and _normalize_bind_host(desc.config.host) == _normalize_bind_host(host) def is_port_bindable(host: str, port: int) -> bool: @@ -272,8 +275,9 @@ def format_port_conflict(err: PortConflict) -> list[str]: Message text depends on ``err.kind`` (foreign process vs NeMo instance). """ if err.kind == "nemo_instance": + owner = f" '{err.scope}'" if err.scope else "" return [ - f"Port {err.port} is in use by a NeMo Platform instance for this directory.", + f"Port {err.port} is in use by NeMo Platform instance{owner}.", "Stop it first with: nemo services stop", "Or restart with: nemo services restart", ] @@ -292,23 +296,39 @@ def format_port_conflict(err: PortConflict) -> list[str]: class InstanceDescriptor(BaseModel): pid: int - scope: str - host: str = "127.0.0.1" - port: int = 8080 - mode: Literal["foreground", "background"] = "background" + config: PlatformAppConfig = Field(default_factory=PlatformAppConfig) + transport: Literal["tcp", "uds"] = "tcp" + mode: Literal["foreground", "background", "daemon"] = "background" create_time: float = 0.0 started_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) - services: list[str] | None = None - controllers: list[str] | None = None - service_group: str | None = None - controller_group: str | None = None - sidecars: list[str] | None = None - config_path: str | None = None - log_path: str | None = None + + @model_validator(mode="after") + def _validate_client_transport(self) -> Self: + if self.transport == "uds" and self.config.socket_path is None: + raise ValueError("UDS client transport requires config.socket_path") + return self + + @classmethod + def from_config( + cls, + config: PlatformAppConfig, + *, + mode: Literal["foreground", "background", "daemon"], + transport: Literal["uds", "tcp"] = "tcp", + pid: int | None = None, + ) -> Self: + resolved_pid = os.getpid() if pid is None else pid + return cls( + pid=resolved_pid, + config=config, + transport=transport, + mode=mode, + create_time=get_create_time(resolved_pid), + ) def write_descriptor(desc: InstanceDescriptor, *, base_dir: Path | None = None) -> Path: - d = instance_dir(desc.scope, base_dir=base_dir) + d = instance_dir(desc.config.scope, base_dir=base_dir) path = d / DESCRIPTOR_FILENAME payload = desc.model_dump() fd, tmp = tempfile.mkstemp(dir=str(d), suffix=".tmp") @@ -335,10 +355,19 @@ def read_descriptor(scope: str, *, base_dir: Path | None = None) -> InstanceDesc return None try: data = json.loads(path.read_text()) - return InstanceDescriptor.model_validate(data) + desc = InstanceDescriptor.model_validate(data) except (json.JSONDecodeError, KeyError, TypeError, ValueError): logger.debug("Corrupt descriptor at %s, ignoring", path, exc_info=True) return None + if desc.config.scope != scope: + logger.debug( + "Descriptor at %s has scope=%r but lives under %r, ignoring", + path, + desc.config.scope, + scope, + ) + return None + return desc def remove_descriptor(scope: str, *, base_dir: Path | None = None) -> None: @@ -351,24 +380,24 @@ def remove_descriptor(scope: str, *, base_dir: Path | None = None) -> None: def _scope_dir(scope: str, *, base_dir: Path | None = None) -> Path: - return _instances_dir(base_dir=base_dir) / _validate_scope(scope) + return _instances_dir(base_dir=base_dir) / validate_scope(scope) def _is_log_file(path: Path) -> bool: return path.name == LOG_FILENAME or path.name.startswith(f"{LOG_FILENAME}.") -def _iter_log_files(scope_dir: Path): - if not scope_dir.is_dir(): +def _iter_log_files(scope_dir_path: Path): + if not scope_dir_path.is_dir(): return - for path in scope_dir.iterdir(): + for path in scope_dir_path.iterdir(): if path.is_file() and _is_log_file(path): yield path -def _has_preservable_logs(scope_dir: Path) -> bool: - """Return True if *scope_dir* contains non-empty service log files.""" - return any(path.stat().st_size > 0 for path in _iter_log_files(scope_dir)) +def _has_preservable_logs(scope_dir_path: Path) -> bool: + """Return True if *scope_dir_path* contains non-empty service log files.""" + return any(path.stat().st_size > 0 for path in _iter_log_files(scope_dir_path)) def is_removable_ghost( @@ -377,17 +406,17 @@ def is_removable_ghost( base_dir: Path | None = None, descriptor: InstanceDescriptor | None = None, ) -> bool: - """True when a dead scope dir has no descriptor and no non-empty logs.""" + """True when a dead scope directory has no descriptor and no non-empty logs.""" if is_instance_alive(scope, base_dir=base_dir): return False if descriptor is not None: return False - scope_dir = _scope_dir(scope, base_dir=base_dir) - if not scope_dir.is_dir(): + scope_dir_path = _scope_dir(scope, base_dir=base_dir) + if not scope_dir_path.is_dir(): return False - if (scope_dir / DESCRIPTOR_FILENAME).exists(): + if (scope_dir_path / DESCRIPTOR_FILENAME).exists(): return False - return not _has_preservable_logs(scope_dir) + return not _has_preservable_logs(scope_dir_path) # --------------------------------------------------------------------------- @@ -425,7 +454,7 @@ class InstanceInfo: def list_instances(*, base_dir: Path | None = None) -> list[InstanceInfo]: - """Scan all instance directories and return their status. + """Scan all scope directories and return their status. Side effects: - Removes stale descriptors for dead instances. @@ -448,7 +477,7 @@ def list_instances(*, base_dir: Path | None = None) -> list[InstanceInfo]: try: shutil.rmtree(child) except OSError: - logger.debug("Could not remove ghost instance dir %s", child, exc_info=True) + logger.debug("Could not remove ghost scope directory %s", child, exc_info=True) else: continue results.append(InstanceInfo(scope=scope, alive=alive, descriptor=desc)) @@ -456,28 +485,28 @@ def list_instances(*, base_dir: Path | None = None) -> list[InstanceInfo]: def remove_instance(scope: str, *, base_dir: Path | None = None) -> bool: - """Remove an instance scope directory. + """Remove a scope directory. - Returns False if the scope did not exist or could not be removed. + Returns False if the scope directory did not exist or could not be removed. """ - scope = _validate_scope(scope) + scope = validate_scope(scope) if is_instance_alive(scope, base_dir=base_dir): raise InstanceStillRunningError(scope) - scope_dir = _scope_dir(scope, base_dir=base_dir) - if not scope_dir.is_dir(): + scope_dir_path = _scope_dir(scope, base_dir=base_dir) + if not scope_dir_path.is_dir(): return False with contextlib.suppress(OSError): - shutil.rmtree(scope_dir) - return not scope_dir.is_dir() + shutil.rmtree(scope_dir_path) + return not scope_dir_path.is_dir() def list_stopped_scopes(*, base_dir: Path | None = None) -> list[str]: - """Return scope names for instances that are not alive.""" + """Return scopes for instances that are not alive.""" return [info.scope for info in list_instances(base_dir=base_dir) if not info.alive] def prune_instances(*, base_dir: Path | None = None) -> list[str]: - """Remove all stopped instance directories. Returns removed scope names.""" + """Remove all stopped scope directories. Returns removed scopes.""" removed: list[str] = [] for scope in list_stopped_scopes(base_dir=base_dir): if remove_instance(scope, base_dir=base_dir): @@ -497,11 +526,15 @@ def instance_log_bytes(scope: str, *, base_dir: Path | None = None) -> int: def rotate_log(scope: str, *, base_dir: Path | None = None) -> Path: """Rotate the existing log and return the path for the new one.""" - d = instance_dir(scope, base_dir=base_dir) - log_path = d / LOG_FILENAME + return rotate_log_path(log_path_for(scope, base_dir=base_dir)) + + +def rotate_log_path(log_path: Path) -> Path: + """Rotate the existing log at *log_path* and return the path for the new one.""" + log_path.parent.mkdir(parents=True, exist_ok=True) if log_path.exists() and log_path.stat().st_size > 0: ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") - rotated = d / f"{LOG_FILENAME}.{ts}" + rotated = log_path.with_name(f"{log_path.name}.{ts}") log_path.rename(rotated) return log_path @@ -633,6 +666,10 @@ def stop_instance( return StopResult(stopped_pids=[], swept_children=swept) except OSError: logger.debug("Failed to send SIGKILL to pid %d", pid, exc_info=True) + if not _wait_for_pid_exit(pid, timeout=_SIGKILL_WAIT_TIMEOUT): + logger.warning("PID %d is still alive after SIGKILL; preserving descriptor", pid) + swept = _sweep_orphans(children) if children else [] + return StopResult(stopped_pids=[], swept_children=swept) swept = _sweep_orphans(children) if children else [] @@ -640,13 +677,21 @@ def stop_instance( return StopResult(stopped_pids=[pid], swept_children=swept) +def _wait_for_pid_exit(pid: int, *, timeout: float) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not _pid_alive(pid): + return True + _pause(_SIGTERM_POLL_INTERVAL) + return not _pid_alive(pid) + + def _pid_alive(pid: int) -> bool: try: - os.kill(pid, 0) - return True - except ProcessLookupError: + return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE + except psutil.NoSuchProcess: return False - except PermissionError: + except psutil.AccessDenied: return True except OSError: return False @@ -658,17 +703,8 @@ def _pid_alive(pid: int) -> bool: def start_background( + config: PlatformAppConfig | None = None, *, - scope: str, - services: list[str] | None = None, - service_group: str | None = None, - controllers: list[str] | None = None, - controller_group: str | None = None, - sidecars: list[str] | None = None, - config_path: str | None = None, - host: str = DEFAULT_SERVICES_BIND_HOST, - port: int = 8080, - base_dir: Path | None = None, data_dir: str | None = None, ) -> subprocess.Popen: """Launch ``nemo services run`` as a detached background subprocess. @@ -676,31 +712,32 @@ def start_background( The child acquires the flock and writes its own descriptor. The parent returns the ``Popen`` handle for health polling. """ - log_file_path = rotate_log(scope, base_dir=base_dir) + config = config or PlatformAppConfig(host=DEFAULT_LOCAL_SERVICES_BIND_HOST) + log_file_path = rotate_log_path(config.log_file_path()) log_file = open(log_file_path, "a") # noqa: SIM115 nemo_bin = str(Path(sys.executable).parent / "nemo") args: list[str] = [nemo_bin, "services", "run"] - if services: - args += ["--services", ",".join(services)] - if service_group: - args += ["--service-group", service_group] - if controllers: - args += ["--controllers", ",".join(controllers)] - if controller_group: - args += ["--controller-group", controller_group] - if sidecars: - args += ["--sidecars", ",".join(sidecars)] - if config_path: - args += ["--config", config_path] - args += ["--host", host, "--port", str(port)] - args += ["--instance", scope] + if config.services: + args += ["--services", ",".join(config.services)] + if config.service_group: + args += ["--service-group", config.service_group] + if config.controllers: + args += ["--controllers", ",".join(config.controllers)] + if config.controller_group: + args += ["--controller-group", config.controller_group] + if config.sidecars: + args += ["--sidecars", ",".join(config.sidecars)] + if config.config_path: + args += ["--config", config.config_path] + args += ["--host", config.host, "--port", str(config.port)] + args += ["--instance", config.scope] env = os.environ.copy() if data_dir and "NMP_DATA_DIR" not in env: env["NMP_DATA_DIR"] = data_dir - if base_dir: - env["_NMP_STATE_DIR"] = str(base_dir) + if config.state_root is not None: + env["_NMP_STATE_DIR"] = str(config.state_root) # Tell the child ``run`` process it was launched by ``start`` so it # records mode="background" in its descriptor. This is internal # parent-to-child signaling -- not a public API surface -- following the diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/local/services.py b/packages/nemo_platform_ext/src/nemo_platform_ext/local/services.py new file mode 100644 index 0000000000..d115fe06ec --- /dev/null +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/local/services.py @@ -0,0 +1,728 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Programmatic local lifecycle API for NeMo Platform services.""" + +from __future__ import annotations + +import asyncio +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import time +from collections.abc import MutableMapping, Sequence +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import Any, Literal, Protocol, Self, runtime_checkable + +from nemo_platform import AsyncNeMoPlatform, NeMoPlatform +from nemo_platform_ext.local import process +from nemo_platform_ext.local.transport import ( + EMBEDDED_BASE_URL, + UDS_BASE_URL, + build_async_asgi_http_client, + build_async_http_client, + build_sync_asgi_http_client, + build_sync_http_client, + probe_status, + tcp_base_url, + wait_for_status, + wait_for_status_async, +) +from nmp.platform_runner.config import ( + DEFAULT_SCOPE, + PlatformAppConfig, + default_runtime_root, + default_state_root, + validate_scope, +) + +_AF_UNIX_PATH_MAX_BYTES = 103 if sys.platform.startswith(("darwin", "freebsd", "openbsd", "netbsd")) else 107 + + +class ServicesError(RuntimeError): + """Base class for local services lifecycle errors.""" + + +class ServicesExtraRequiredError(ServicesError): + """Raised when local service dependencies are not installed.""" + + +class ServicesAlreadyRunningError(ServicesError): + """Raised when a requested local instance is already running.""" + + +class ServicesNotRunningError(ServicesError): + """Raised when a requested local instance is not running.""" + + +class ServicesPortInUseError(ServicesError): + """Raised when TCP startup targets an unavailable port.""" + + +class ServicesStartupTimeoutError(ServicesError): + """Raised when startup does not become healthy before the timeout.""" + + +class ServicesStartupExitedError(ServicesError): + """Raised when a daemon child exits before becoming healthy.""" + + +class ServicesSocketStaleError(ServicesError): + """Raised when a stale socket cannot be removed.""" + + +def _as_tuple(value: Sequence[str] | None) -> tuple[str, ...] | None: + if value is None: + return None + return tuple(value) + + +def _optional_str(value: str | Path | None) -> str | None: + if value is None: + return None + return str(value) + + +def _optional_list(value: Sequence[str] | None) -> list[str] | None: + if value is None: + return None + return list(value) + + +class ServiceMode(StrEnum): + EMBEDDED = "embedded" + DAEMON = "daemon" + + +@dataclass(frozen=True) +class StartServicesResult: + requested: list[str] + started: list[str] + already_active: list[str] + active: list[str] + + +@runtime_checkable +class LocalServiceHandle(Protocol): + """Shared lifecycle/client contract for local services handles.""" + + def is_running(self) -> bool: ... + + def wait_until_ready(self, timeout: float | None = None) -> None: ... + + async def wait_until_ready_async(self, timeout: float | None = None) -> None: ... + + def client(self, **kwargs: Any) -> NeMoPlatform: ... + + def async_client(self, **kwargs: Any) -> AsyncNeMoPlatform: ... + + def start_services(self, service_names: Sequence[str]) -> StartServicesResult: ... + + async def start_services_async(self, service_names: Sequence[str]) -> StartServicesResult: ... + + def stop(self, *, timeout: float = 30.0, force: bool = False) -> process.StopResult: ... + + async def stop_async(self, *, timeout: float = 30.0, force: bool = False) -> process.StopResult: ... + + +@dataclass +class ServiceRunConfig: + services: Sequence[str] | None = None + service_group: str | None = None + controllers: Sequence[str] | None = None + controller_group: str | None = None + sidecars: Sequence[str] | None = None + config_path: str | Path | None = None + transport: Literal["uds", "tcp"] = "uds" + socket_path: str | Path | None = None + http_gateway: Literal["enabled", "disabled"] = "disabled" + http_gateway_host: str = "127.0.0.1" + http_gateway_port: int | None = None + host: str = "127.0.0.1" + port: int = 8080 + scope: str = DEFAULT_SCOPE + state_dir: str | Path | None = None + runtime_dir: str | Path | None = None + data_dir: str | Path | None = None + readiness_timeout: float = 60.0 + readiness_poll_interval: float = 0.5 + mode: ServiceMode | str = ServiceMode.DAEMON + + def __post_init__(self) -> None: + self.services = _as_tuple(self.services) + self.controllers = _as_tuple(self.controllers) + self.sidecars = _as_tuple(self.sidecars) + try: + self.mode = ServiceMode(self.mode) + except ValueError as error: + raise ValueError("mode must be 'embedded' or 'daemon'") from error + + if self.services and self.service_group: + raise ValueError("services cannot be combined with service_group") + if self.controllers and self.controller_group: + raise ValueError("controllers cannot be combined with controller_group") + if self.transport not in {"uds", "tcp"}: + raise ValueError("transport must be 'uds' or 'tcp'") + if self.http_gateway not in {"enabled", "disabled"}: + raise ValueError("http_gateway must be 'enabled' or 'disabled'") + if self.http_gateway == "enabled" and self.transport != "uds": + raise ValueError("gateway can only be enabled for UDS transport") + if self.readiness_timeout <= 0: + raise ValueError("readiness_timeout must be greater than 0") + if self.readiness_poll_interval <= 0: + raise ValueError("readiness_poll_interval must be greater than 0") + self.scope = validate_scope(self.scope) + + @property + def state_root(self) -> Path: + return Path(self.state_dir).expanduser() if self.state_dir is not None else default_state_root() + + @property + def runtime_root(self) -> Path: + return Path(self.runtime_dir).expanduser() if self.runtime_dir is not None else default_runtime_root() + + @property + def resolved_socket_path(self) -> Path | None: + if self.socket_path is not None: + socket_path = Path(self.socket_path).expanduser() + elif self.transport == "uds": + socket_path = PlatformAppConfig( + scope=self.scope, + runtime_root=self.runtime_root, + ).socket_file_path() + else: + return None + if not socket_path.is_absolute(): + raise ValueError(f"UDS socket path must be absolute: {socket_path}") + return socket_path + + def to_platform_app_config(self) -> PlatformAppConfig: + return PlatformAppConfig( + services=self.services, + service_group=self.service_group, + controllers=self.controllers, + controller_group=self.controller_group, + sidecars=self.sidecars, + config_path=_optional_str(self.config_path), + scope=self.scope, + host=self.host, + port=self.port, + socket_path=_optional_str(self.resolved_socket_path), + state_root=_optional_str(self.state_root), + runtime_root=_optional_str(self.runtime_dir), + ) + + def to_child_payload(self) -> dict[str, object]: + return { + "mode": ServiceMode(self.mode).value, + "services": _optional_list(self.services), + "service_group": self.service_group, + "controllers": _optional_list(self.controllers), + "controller_group": self.controller_group, + "sidecars": _optional_list(self.sidecars), + "config_path": _optional_str(self.config_path), + "transport": self.transport, + "socket_path": _optional_str(self.socket_path), + "http_gateway": self.http_gateway, + "http_gateway_host": self.http_gateway_host, + "http_gateway_port": self.http_gateway_port, + "host": self.host, + "port": self.port, + "scope": self.scope, + "state_dir": _optional_str(self.state_dir), + "runtime_dir": _optional_str(self.runtime_dir), + "data_dir": _optional_str(self.data_dir), + "readiness_timeout": self.readiness_timeout, + "readiness_poll_interval": self.readiness_poll_interval, + } + + +@dataclass(frozen=True) +class DaemonServiceHandle: + scope: str + transport: Literal["uds", "tcp"] + socket_path: Path | None + gateway_base_url: str | None + host: str + port: int + pid: int | None + mode: Literal["foreground", "daemon"] + log_path: Path | None + state_dir: Path | None + runtime_dir: Path | None + + @classmethod + def from_descriptor(cls, desc: process.InstanceDescriptor) -> Self: + socket_path = Path(desc.config.socket_path) if desc.config.socket_path else None + runtime_dir = desc.config.runtime_dir() if socket_path else None + return cls( + scope=desc.config.scope, + transport=desc.transport, + socket_path=socket_path, + gateway_base_url=None, + host=desc.config.host, + port=desc.config.port, + pid=desc.pid, + mode="daemon" if desc.mode == "daemon" else "foreground", + log_path=desc.config.log_file_path(), + state_dir=desc.config.state_dir(), + runtime_dir=runtime_dir, + ) + + @classmethod + def from_config( + cls, + config: ServiceRunConfig, + *, + pid: int | None = None, + ) -> Self: + app_config = config.to_platform_app_config() + socket_path = config.resolved_socket_path + runtime_dir = app_config.runtime_dir() if socket_path else None + return cls( + scope=config.scope, + transport=config.transport, + socket_path=socket_path, + gateway_base_url=None, + host=config.host, + port=config.port, + pid=pid, + mode="daemon", + log_path=app_config.log_file_path(), + state_dir=app_config.state_dir(), + runtime_dir=runtime_dir, + ) + + @property + def base_url(self) -> str: + if self.transport == "uds": + return UDS_BASE_URL + return tcp_base_url(self.host, self.port) + + def _state_root(self) -> Path | None: + if self.state_dir is None: + return None + if self.state_dir.parent.name == "instances": + return self.state_dir.parent.parent + return self.state_dir.parent + + def is_running(self) -> bool: + state_root = self._state_root() + return process.is_instance_alive(self.scope, base_dir=state_root) + + def wait_until_ready(self, timeout: float | None = None) -> None: + if not wait_for_status( + base_url=self.base_url, + socket_path=self.socket_path if self.transport == "uds" else None, + timeout=60.0 if timeout is None else timeout, + ): + raise ServicesStartupTimeoutError(f"Timed out waiting for services instance {self.scope!r}") + + async def wait_until_ready_async(self, timeout: float | None = None) -> None: + if not await wait_for_status_async( + base_url=self.base_url, + socket_path=self.socket_path if self.transport == "uds" else None, + timeout=60.0 if timeout is None else timeout, + ): + raise ServicesStartupTimeoutError(f"Timed out waiting for services instance {self.scope!r}") + + def stop(self, *, timeout: float = 30.0, force: bool = False) -> process.StopResult: + state_root = self._state_root() + return process.stop_instance(self.scope, base_dir=state_root, timeout=timeout, force=force) + + async def stop_async(self, *, timeout: float = 30.0, force: bool = False) -> process.StopResult: + return await asyncio.to_thread(self.stop, timeout=timeout, force=force) + + def start_services(self, service_names: Sequence[str]) -> StartServicesResult: + raise ServicesError("Staged service start is not implemented for daemon mode yet") + + async def start_services_async(self, service_names: Sequence[str]) -> StartServicesResult: + return await asyncio.to_thread(self.start_services, service_names) + + def client(self, **kwargs: Any) -> NeMoPlatform: + if self.transport == "uds": + if self.socket_path is None: + raise ServicesError("UDS service handle is missing socket_path") + kwargs.setdefault("http_client", build_sync_http_client(self.socket_path)) + kwargs.setdefault("base_url", self.base_url) + return NeMoPlatform(**kwargs) + + def async_client(self, **kwargs: Any) -> AsyncNeMoPlatform: + if self.transport == "uds": + if self.socket_path is None: + raise ServicesError("UDS service handle is missing socket_path") + kwargs.setdefault("http_client", build_async_http_client(self.socket_path)) + kwargs.setdefault("base_url", self.base_url) + return AsyncNeMoPlatform(**kwargs) + + +@dataclass(frozen=True) +class EmbeddedServiceHandle: + app: Any + runtime: object + + def is_running(self) -> bool: + return True + + def wait_until_ready(self, timeout: float | None = None) -> None: + return None + + async def wait_until_ready_async(self, timeout: float | None = None) -> None: + return None + + def client(self, **kwargs: Any) -> NeMoPlatform: + kwargs.setdefault("http_client", build_sync_asgi_http_client(self.app)) + kwargs.setdefault("base_url", EMBEDDED_BASE_URL) + return NeMoPlatform(**kwargs) + + def async_client(self, **kwargs: Any) -> AsyncNeMoPlatform: + kwargs.setdefault("http_client", build_async_asgi_http_client(self.app)) + kwargs.setdefault("base_url", EMBEDDED_BASE_URL) + return AsyncNeMoPlatform(**kwargs) + + def start_services(self, service_names: Sequence[str]) -> StartServicesResult: + raise ServicesError("Staged service start is not implemented for embedded mode yet") + + async def start_services_async(self, service_names: Sequence[str]) -> StartServicesResult: + return await asyncio.to_thread(self.start_services, service_names) + + def stop(self, *, timeout: float = 30.0, force: bool = False) -> process.StopResult: + return process.StopResult(stopped_pids=[], swept_children=[]) + + async def stop_async(self, *, timeout: float = 30.0, force: bool = False) -> process.StopResult: + return self.stop(timeout=timeout, force=force) + + +def require_services_extra() -> None: + if importlib.util.find_spec("pyleak") is not None: + return + raise ServicesExtraRequiredError("Install service dependencies with `pip install 'nemo-platform[all]'`.") + + +def _validate_socket_path_length(socket_path: Path) -> None: + encoded_length = len(os.fsencode(socket_path)) + if encoded_length > _AF_UNIX_PATH_MAX_BYTES: + raise ValueError( + "UDS socket path is too long for AF_UNIX " + f"({encoded_length} bytes; maximum is {_AF_UNIX_PATH_MAX_BYTES} bytes): {socket_path}" + ) + + +def _validated_socket_path(config: ServiceRunConfig) -> Path | None: + socket_path = config.resolved_socket_path + if socket_path is None: + return None + _validate_socket_path_length(socket_path) + return socket_path + + +def _prepare_socket(config: ServiceRunConfig) -> Path | None: + socket_path = _validated_socket_path(config) + if socket_path is None: + return None + socket_path.parent.mkdir(parents=True, exist_ok=True) + if not socket_path.exists(): + return socket_path + if probe_status(base_url=UDS_BASE_URL, socket_path=socket_path, timeout=0.5): + raise ServicesAlreadyRunningError(f"UDS socket is live at {socket_path}") + try: + socket_path.unlink() + except OSError as error: + raise ServicesSocketStaleError(f"Could not remove stale socket at {socket_path}") from error + return socket_path + + +def _check_tcp_available(config: ServiceRunConfig) -> None: + conflict = process.check_port_available_for_start( + config.host, + config.port, + config.scope, + base_dir=config.state_root, + ) + if conflict is not None: + raise ServicesPortInUseError("\n".join(process.format_port_conflict(conflict))) + + +def _write_run_request(config: ServiceRunConfig) -> Path: + state_dir = config.to_platform_app_config().state_dir(create=True) + fd, tmp = tempfile.mkstemp(dir=state_dir, suffix=".json") + path = Path(tmp) + try: + with os.fdopen(fd, "w", encoding="utf-8") as file: + fd = -1 + json.dump(config.to_child_payload(), file, indent=2) + file.write("\n") + except BaseException: + if fd >= 0: + os.close(fd) + fd = -1 + path.unlink(missing_ok=True) + raise + finally: + if fd >= 0: + os.close(fd) + return path + + +def _terminate_startup_process(proc: subprocess.Popen) -> None: + if proc.poll() is not None: + return + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +def serve_embedded_app(app: Any, cfg: ServiceRunConfig, socket_path: Path | None) -> None: + import uvicorn + + if socket_path is not None: + from nmp.platform_runner.server import _run_server_on_bound_sockets + + _run_server_on_bound_sockets(app, host=cfg.host, port=cfg.port, socket_path=str(socket_path)) + else: + uvicorn.run(app, host=cfg.host, port=cfg.port, log_config=None) + + +def run_services( + config: ServiceRunConfig | None = None, + *, + _mode: Literal["foreground", "daemon"] = "foreground", + env: MutableMapping[str, str] | None = None, +) -> None: + cfg = config or ServiceRunConfig() + app_config = cfg.to_platform_app_config() + require_services_extra() + if cfg.http_gateway == "enabled": + raise ServicesError("HTTP gateway support is not implemented yet") + if process.is_instance_alive(cfg.scope, base_dir=cfg.state_root): + raise ServicesAlreadyRunningError(f"Instance {cfg.scope!r} is already running") + _check_tcp_available(cfg) + lock_fd = process.acquire_lock(cfg.scope, base_dir=cfg.state_root) + original_data_dir = os.environ.get("NMP_DATA_DIR") + try: + socket_path = _prepare_socket(cfg) + app_config.log_file_path(create_parent=True) + if cfg.data_dir is not None and "NMP_DATA_DIR" not in os.environ: + os.environ["NMP_DATA_DIR"] = str(cfg.data_dir) + desc = process.InstanceDescriptor.from_config( + app_config, + mode=_mode, + transport=cfg.transport, + ) + process.write_descriptor(desc, base_dir=cfg.state_root) + embedded_handle = start_embedded_services(cfg, env=env) + serve_embedded_app(embedded_handle.app, cfg, socket_path) + finally: + try: + process.remove_descriptor(cfg.scope, base_dir=cfg.state_root) + finally: + if original_data_dir is None: + os.environ.pop("NMP_DATA_DIR", None) + else: + os.environ["NMP_DATA_DIR"] = original_data_dir + os.close(lock_fd) + + +def daemonize_services(config: ServiceRunConfig | None = None) -> DaemonServiceHandle: + cfg = config or ServiceRunConfig() + app_config = cfg.to_platform_app_config() + require_services_extra() + if cfg.http_gateway == "enabled": + raise ServicesError("HTTP gateway support is not implemented yet") + if process.is_instance_alive(cfg.scope, base_dir=cfg.state_root): + raise ServicesAlreadyRunningError(f"Instance {cfg.scope!r} is already running") + _check_tcp_available(cfg) + socket_path = _validated_socket_path(cfg) + if ( + socket_path is not None + and socket_path.exists() + and probe_status(base_url=UDS_BASE_URL, socket_path=socket_path, timeout=0.5) + ): + raise ServicesAlreadyRunningError(f"UDS socket is live at {socket_path}") + + request_path = _write_run_request(cfg) + log_path = process.rotate_log_path(app_config.log_file_path()) + log_file = open(log_path, "a") # noqa: SIM115 + env = os.environ.copy() + if cfg.data_dir is not None and "NMP_DATA_DIR" not in env: + env["NMP_DATA_DIR"] = str(cfg.data_dir) + proc: subprocess.Popen | None = None + ownership_transferred = False + try: + try: + child_module = f"{__package__}._service_child" + proc = subprocess.Popen( + [sys.executable, "-m", child_module, str(request_path)], + stdout=log_file, + stderr=subprocess.STDOUT, + env=env, + start_new_session=True, + stdin=subprocess.DEVNULL, + close_fds=True, + ) + finally: + log_file.close() + assert proc is not None + handle = DaemonServiceHandle.from_config(cfg, pid=proc.pid) + deadline = time.monotonic() + cfg.readiness_timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + if proc.poll() is not None: + raise ServicesStartupExitedError(f"Services daemon exited with code {proc.returncode}; log: {log_path}") + if probe_status( + base_url=handle.base_url, + socket_path=handle.socket_path if handle.transport == "uds" else None, + timeout=remaining, + ): + ownership_transferred = True + return handle + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(cfg.readiness_poll_interval, remaining)) + raise ServicesStartupTimeoutError(f"Timed out waiting for services daemon {cfg.scope!r}; log: {log_path}") + finally: + if proc is not None and not ownership_transferred: + _terminate_startup_process(proc) + + +async def daemonize_services_async(config: ServiceRunConfig | None = None) -> DaemonServiceHandle: + return await asyncio.to_thread(daemonize_services, config) + + +def start_embedded_services( + config: ServiceRunConfig | None = None, + *, + env: MutableMapping[str, str] | None = None, +) -> EmbeddedServiceHandle: + """Start platform services in the current process. + + Args: + env: Environment mapping passed to :func:`build_platform_app`. + Defaults to ``None`` which writes to ``os.environ``. Tests can + pass an empty dict to avoid polluting the process environment. + """ + cfg = config or ServiceRunConfig(mode=ServiceMode.EMBEDDED) + from nmp.platform_runner.server import build_platform_app + + app = build_platform_app( + config=cfg.to_platform_app_config(), + env=env, + ) + runtime = getattr(app.state, "platform_runtime", None) + return EmbeddedServiceHandle(app=app, runtime=runtime) + + +async def start_embedded_services_async(config: ServiceRunConfig | None = None) -> EmbeddedServiceHandle: + return start_embedded_services(config) + + +def get_service_handle(config: ServiceRunConfig | None = None) -> DaemonServiceHandle | None: + cfg = config or ServiceRunConfig() + desc = process.read_descriptor(cfg.scope, base_dir=cfg.state_root) + if desc is None or not process.is_instance_alive(cfg.scope, base_dir=cfg.state_root): + return None + return DaemonServiceHandle.from_descriptor(desc) + + +def list_service_handles(state_dir: str | Path | None = None) -> list[DaemonServiceHandle]: + state_root = Path(state_dir).expanduser() if state_dir is not None else default_state_root() + handles: list[DaemonServiceHandle] = [] + for info in process.list_instances(base_dir=state_root): + if info.descriptor is not None and info.alive: + handles.append(DaemonServiceHandle.from_descriptor(info.descriptor)) + return handles + + +def ensure_services( + config: ServiceRunConfig | None = None, + *, + daemonize: bool | None = None, +) -> LocalServiceHandle: + cfg = config or ServiceRunConfig() + if cfg.mode is ServiceMode.EMBEDDED: + return start_embedded_services(cfg) + + handle = get_service_handle(cfg) + if handle is not None: + return handle + if daemonize is False: + raise ServicesNotRunningError(f"Instance {cfg.scope!r} is not running") + return daemonize_services(cfg) + + +async def ensure_services_async( + config: ServiceRunConfig | None = None, + *, + daemonize: bool | None = None, +) -> LocalServiceHandle: + cfg = config or ServiceRunConfig() + if cfg.mode is ServiceMode.EMBEDDED: + return await start_embedded_services_async(cfg) + + handle = get_service_handle(cfg) + if handle is not None: + return handle + if daemonize is False: + raise ServicesNotRunningError(f"Instance {cfg.scope!r} is not running") + return await daemonize_services_async(cfg) + + +def connect_services( + config: ServiceRunConfig | None = None, + *, + daemonize: bool | None = None, + start_if_needed: bool = True, + **client_kwargs: Any, +) -> NeMoPlatform: + cfg = config or ServiceRunConfig() + if not start_if_needed and cfg.mode is ServiceMode.DAEMON and get_service_handle(cfg) is None: + raise ServicesNotRunningError(f"Instance {cfg.scope!r} is not running") + handle = ensure_services(cfg, daemonize=daemonize) + return handle.client(**client_kwargs) + + +async def connect_services_async( + config: ServiceRunConfig | None = None, + *, + daemonize: bool | None = None, + start_if_needed: bool = True, + **client_kwargs: Any, +) -> AsyncNeMoPlatform: + cfg = config or ServiceRunConfig() + if not start_if_needed and cfg.mode is ServiceMode.DAEMON and get_service_handle(cfg) is None: + raise ServicesNotRunningError(f"Instance {cfg.scope!r} is not running") + handle = await ensure_services_async(cfg, daemonize=daemonize) + return handle.async_client(**client_kwargs) + + +def stop_services( + config: ServiceRunConfig | None = None, + *, + timeout: float = 30.0, + force: bool = False, +) -> process.StopResult: + cfg = config or ServiceRunConfig() + handle = get_service_handle(cfg) + if handle is None: + raise ServicesNotRunningError(f"Instance {cfg.scope!r} is not running") + return handle.stop(timeout=timeout, force=force) + + +async def stop_services_async( + config: ServiceRunConfig | None = None, + *, + timeout: float = 30.0, + force: bool = False, +) -> process.StopResult: + return await asyncio.to_thread(stop_services, config, timeout=timeout, force=force) diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/local/transport.py b/packages/nemo_platform_ext/src/nemo_platform_ext/local/transport.py new file mode 100644 index 0000000000..da3c08344a --- /dev/null +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/local/transport.py @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Local service transport helpers for TCP and Unix domain sockets.""" + +from __future__ import annotations + +import asyncio +import time +from pathlib import Path +from typing import Any, TypeAlias + +import httpx +from fastapi.testclient import TestClient +from nmp.common.platform_endpoint import UDS_BASE_URL + +HttpxTimeout: TypeAlias = float | httpx.Timeout | None +_DEFAULT_TIMEOUT: float = 5.0 +EMBEDDED_BASE_URL = "http://nemo-platform.local" + +__all__ = [ + "EMBEDDED_BASE_URL", + "UDS_BASE_URL", + "build_async_asgi_http_client", + "build_async_http_client", + "build_sync_asgi_http_client", + "build_sync_http_client", + "probe_status", + "probe_status_async", + "tcp_base_url", + "wait_for_status", + "wait_for_status_async", +] + + +def build_sync_asgi_http_client(app: Any, *, timeout: HttpxTimeout = _DEFAULT_TIMEOUT) -> Any: + _ = timeout + return TestClient( + app, + base_url=EMBEDDED_BASE_URL, + follow_redirects=True, + ) + + +def build_async_asgi_http_client(app: Any, *, timeout: HttpxTimeout = _DEFAULT_TIMEOUT) -> httpx.AsyncClient: + return httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url=EMBEDDED_BASE_URL, + follow_redirects=True, + timeout=timeout, + ) + + +def build_sync_http_client(socket_path: Path, *, timeout: HttpxTimeout = _DEFAULT_TIMEOUT) -> httpx.Client: + return httpx.Client( + transport=httpx.HTTPTransport(uds=str(socket_path)), + follow_redirects=True, + timeout=timeout, + ) + + +def build_async_http_client(socket_path: Path, *, timeout: HttpxTimeout = _DEFAULT_TIMEOUT) -> httpx.AsyncClient: + return httpx.AsyncClient( + transport=httpx.AsyncHTTPTransport(uds=str(socket_path)), + follow_redirects=True, + timeout=timeout, + ) + + +def tcp_base_url(host: str, port: int) -> str: + connect_host = "localhost" if host in {"0.0.0.0", "::"} else host # noqa: S104 + normalized = connect_host.strip("[]") + url_host = f"[{normalized}]" if ":" in normalized else normalized + return str(httpx.URL(scheme="http", host=url_host, port=port)) + + +def probe_status( + *, + base_url: str, + socket_path: Path | None = None, + timeout: float = 2.0, +) -> bool: + client = ( + build_sync_http_client(socket_path, timeout=timeout) + if socket_path is not None + else httpx.Client(timeout=timeout) + ) + try: + response = client.get(f"{base_url.rstrip('/')}/status") + return response.status_code == 200 + except httpx.RequestError: + return False + finally: + client.close() + + +async def probe_status_async( + *, + base_url: str, + socket_path: Path | None = None, + timeout: float = 2.0, +) -> bool: + client = ( + build_async_http_client(socket_path, timeout=timeout) + if socket_path is not None + else httpx.AsyncClient(timeout=timeout) + ) + try: + response = await client.get(f"{base_url.rstrip('/')}/status") + return response.status_code == 200 + except httpx.RequestError: + return False + finally: + await client.aclose() + + +def wait_for_status( + *, + base_url: str, + socket_path: Path | None = None, + timeout: float = 60.0, + poll_interval: float = 0.5, +) -> bool: + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + if probe_status(base_url=base_url, socket_path=socket_path, timeout=remaining): + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep(min(poll_interval, remaining)) + + +async def wait_for_status_async( + *, + base_url: str, + socket_path: Path | None = None, + timeout: float = 60.0, + poll_interval: float = 0.5, +) -> bool: + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + if await probe_status_async(base_url=base_url, socket_path=socket_path, timeout=remaining): + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + await asyncio.sleep(min(poll_interval, remaining)) diff --git a/packages/nemo_platform_ext/tests/cli/commands/conftest.py b/packages/nemo_platform_ext/tests/cli/commands/conftest.py index 24df2fb6c0..ff6593b338 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/conftest.py +++ b/packages/nemo_platform_ext/tests/cli/commands/conftest.py @@ -3,7 +3,7 @@ from __future__ import annotations -import nemo_platform_ext.cli.commands.services._process as _process_mod +import nemo_platform_ext.local.process as _process_mod import pytest diff --git a/packages/nemo_platform_ext/tests/cli/commands/test_services.py b/packages/nemo_platform_ext/tests/cli/commands/test_services.py index 1a78c52a62..7c939fcb8f 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/test_services.py +++ b/packages/nemo_platform_ext/tests/cli/commands/test_services.py @@ -13,11 +13,11 @@ import socket from pathlib import Path from types import ModuleType -from unittest.mock import ANY, MagicMock, patch +from unittest.mock import MagicMock, patch import pytest from nemo_platform_ext.cli.app import app -from nemo_platform_ext.cli.commands.services._process import ( +from nemo_platform_ext.local.process import ( ForegroundInstanceError, InstanceDescriptor, StopResult, @@ -26,16 +26,17 @@ read_descriptor, write_descriptor, ) +from nmp.platform_runner.config import PlatformAppConfig from typer.testing import CliRunner runner = CliRunner() -_PROCESS_MODULE = "nemo_platform_ext.cli.commands.services._process" +_PROCESS_MODULE = "nemo_platform_ext.local.process" _CLI_MODULE = "nemo_platform_ext.cli.commands.services.cli" def _seed_stopped_scope(base_dir: Path, scope: str, *, log_content: str = "x\n") -> Path: - """Create a stopped instance directory with service logs.""" + """Create a stopped scope directory with service logs.""" d = instance_dir(scope, base_dir=base_dir) (d / "services.log").write_text(log_content) return d @@ -118,17 +119,18 @@ def test_run_invokes_runner(base_dir: Path): ) assert result.exit_code == 0, result.stderr - mock_run_platform.assert_called_once_with( - services=["auth", "entities"], - service_group=None, - controllers=["jobs", "models"], - controller_group=None, - sidecars=None, - config_path=None, - host="127.0.0.1", - port=9000, - on_shutdown=ANY, - ) + mock_run_platform.assert_called_once() + _, kwargs = mock_run_platform.call_args + config = kwargs["config"] + assert config.services == ["auth", "entities"] + assert config.service_group is None + assert config.controllers == ["jobs", "models"] + assert config.controller_group is None + assert config.sidecars is None + assert config.config_path is None + assert config.host == "127.0.0.1" + assert config.port == 9000 + assert kwargs["on_shutdown"] is not None def test_run_refuses_when_already_running(base_dir: Path): @@ -163,7 +165,7 @@ def test_run_writes_descriptor(base_dir: Path): desc = read_descriptor("desc-test", base_dir=base_dir) assert desc is not None assert desc.mode == "foreground" - assert desc.port == 9999 + assert desc.config.port == 9999 def test_run_records_background_mode_when_launched_by_start(base_dir: Path): @@ -382,7 +384,7 @@ def test_restart_errors_when_no_prior_instance(self, base_dir: Path): ["services", "restart", "--instance", "ghost"], ) assert result.exit_code == 1 - assert "No instance found" in result.stderr + assert "No instance found for scope" in result.stderr assert "nemo services start" in result.stderr def test_restart_stops_and_starts(self, base_dir: Path): @@ -390,9 +392,7 @@ def test_restart_stops_and_starts(self, base_dir: Path): fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=1.0, ) @@ -429,9 +429,7 @@ def test_restart_exits_early_when_port_occupied_by_foreign_process(self, base_di desc = InstanceDescriptor( pid=99999, - scope=scope, - host="127.0.0.1", - port=port, + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=port), mode="background", create_time=1.0, ) @@ -459,13 +457,15 @@ def test_restart_preserves_previous_args(self, base_dir: Path): fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=9000, + config=PlatformAppConfig( + scope=scope, + services=["entities", "models"], + controllers=["jobs"], + host="127.0.0.1", + port=9000, + ), mode="background", create_time=1.0, - services=["entities", "models"], - controllers=["jobs"], ) write_descriptor(desc, base_dir=base_dir) @@ -488,11 +488,12 @@ def test_restart_preserves_previous_args(self, base_dir: Path): os.close(fd) assert result.exit_code == 0 - _, kwargs = mock_start.call_args - assert kwargs["services"] == ["entities", "models"] - assert kwargs["controllers"] == ["jobs"] - assert kwargs["host"] == "127.0.0.1" - assert kwargs["port"] == 9000 + args, _kwargs = mock_start.call_args + config = args[0] + assert config.services == ["entities", "models"] + assert config.controllers == ["jobs"] + assert config.host == "127.0.0.1" + assert config.port == 9000 # --------------------------------------------------------------------------- @@ -504,16 +505,14 @@ class TestServicesStatus: def test_not_running(self, base_dir: Path): result = runner.invoke(app, ["services", "status", "--instance", "none"]) assert result.exit_code == 0 - assert "No running instance" in result.stdout + assert "No running instance for scope" in result.stdout def test_running_instance(self, base_dir: Path): scope = "status-test" fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="foreground", create_time=1.0, ) @@ -548,9 +547,7 @@ def test_lists_running_instance(self, base_dir: Path): fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=1.0, ) @@ -594,9 +591,7 @@ def test_mixed_running_and_stopped(self, base_dir: Path): write_descriptor( InstanceDescriptor( pid=os.getpid(), - scope=running_scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=running_scope), mode="background", create_time=1.0, ), @@ -658,7 +653,7 @@ def test_rm_requires_scope(self, base_dir: Path): def test_rm_rejects_invalid_scope(self, base_dir: Path): result = runner.invoke(app, ["services", "rm", "../escape"]) assert result.exit_code == 1 - assert "Invalid instance scope" in result.stderr + assert "Invalid scope" in result.stderr def test_rm_rejects_conflicting_scope_args(self, base_dir: Path): result = runner.invoke(app, ["services", "rm", "scope-a", "--instance", "scope-b"]) @@ -747,7 +742,7 @@ def test_default_host_is_loopback(base_dir: Path): assert result.exit_code == 0, result.stderr _, kwargs = mock_run_platform.call_args - assert kwargs["host"] == "127.0.0.1" + assert kwargs["config"].host == "127.0.0.1" def test_bind_all_warning(base_dir: Path): diff --git a/packages/nemo_platform_ext/tests/cli/commands/test_services_lifecycle.py b/packages/nemo_platform_ext/tests/cli/commands/test_services_lifecycle.py index fd45922a1c..ad382426c4 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/test_services_lifecycle.py +++ b/packages/nemo_platform_ext/tests/cli/commands/test_services_lifecycle.py @@ -27,7 +27,7 @@ import pytest from nemo_platform_ext.cli.app import app -from nemo_platform_ext.cli.commands.services._process import ( +from nemo_platform_ext.local.process import ( InstanceDescriptor, PortConflict, acquire_lock, @@ -43,6 +43,7 @@ stop_instance, write_descriptor, ) +from nmp.platform_runner.config import PlatformAppConfig from typer.testing import CliRunner _runner = CliRunner() @@ -70,18 +71,14 @@ import psutil as _psutil desc = { "pid": os.getpid(), - "scope": scope, - "host": "127.0.0.1", - "port": 8080, + "config": { + "scope": scope, + "host": "127.0.0.1", + "port": 8080, + }, "mode": "background", "create_time": _psutil.Process(os.getpid()).create_time(), "started_at": "test", - "services": None, - "controllers": None, - "service_group": None, - "controller_group": None, - "sidecars": None, - "config_path": None, "log_path": None, } desc_path = os.path.join(inst_dir, "instance.json") @@ -291,9 +288,7 @@ def test_stale_descriptor_with_reused_pid(self, tmp_path: Path) -> None: # Write a descriptor with the sleeper's PID but wrong create_time desc = InstanceDescriptor( pid=sleeper.pid, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=0.0, # intentionally wrong ) @@ -363,7 +358,7 @@ def test_log_preserved_across_restart(self, tmp_path: Path) -> None: log = d / "services.log" log.write_text("first boot log content\n") - from nemo_platform_ext.cli.commands.services._process import rotate_log + from nemo_platform_ext.local.process import rotate_log new_log = rotate_log(scope, base_dir=base_dir) new_log.write_text("second boot log content\n") @@ -399,15 +394,15 @@ def test_log_preserved_across_restart(self, tmp_path: Path) -> None: import psutil as _psutil desc = { "pid": os.getpid(), - "scope": scope, - "host": "127.0.0.1", - "port": port, + "config": { + "scope": scope, + "host": "127.0.0.1", + "port": port, + }, "mode": "background", "create_time": _psutil.Process(os.getpid()).create_time(), "started_at": "test", - "services": None, "controllers": None, - "service_group": None, "controller_group": None, - "sidecars": None, "config_path": None, "log_path": None, + "log_path": None, } desc_path = os.path.join(inst_dir, "instance.json") with open(desc_path, "w") as f: @@ -523,7 +518,7 @@ def test_stop_after_health_check(self, tmp_path: Path) -> None: assert is_instance_alive(scope, base_dir=base_dir) desc = read_descriptor(scope, base_dir=base_dir) assert desc is not None - assert desc.port == port + assert desc.config.port == port result = stop_instance(scope, base_dir=base_dir, timeout=5.0) assert proc.pid in result.stopped_pids @@ -536,7 +531,7 @@ def test_stop_after_health_check(self, tmp_path: Path) -> None: class TestInstanceCleanup: - """Integration tests for rm/prune and post-stop instance directories.""" + """Integration tests for rm/prune and post-stop scope directories.""" def test_stop_leaves_record_until_rm(self, tmp_path: Path, monkeypatch) -> None: base_dir = tmp_path / "state" @@ -668,9 +663,7 @@ def test_check_port_returns_nemo_instance_when_lock_held_and_port_blocked(self, write_descriptor( InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=port, + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=port), mode="background", create_time=1.0, ), @@ -704,9 +697,7 @@ def test_check_port_returns_foreign_when_alive_instance_uses_different_port(self write_descriptor( InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=nemo_port, + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=nemo_port), mode="background", create_time=1.0, ), diff --git a/packages/nemo_platform_ext/tests/cli/commands/test_services_process.py b/packages/nemo_platform_ext/tests/cli/commands/test_services_process.py index fa18e47a65..26478c20d5 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/test_services_process.py +++ b/packages/nemo_platform_ext/tests/cli/commands/test_services_process.py @@ -6,6 +6,7 @@ from __future__ import annotations import os +import signal import subprocess import sys import time @@ -14,7 +15,8 @@ import psutil import pytest -from nemo_platform_ext.cli.commands.services._process import ( +from nemo_platform_ext.local import process as process_module +from nemo_platform_ext.local.process import ( ForegroundInstanceError, InstanceAlreadyRunningError, InstanceDescriptor, @@ -40,6 +42,7 @@ validate_pid, write_descriptor, ) +from nmp.platform_runner.config import DEFAULT_LOCAL_SERVICES_BIND_HOST, PlatformAppConfig @pytest.fixture() @@ -48,67 +51,23 @@ def base_dir(tmp_path: Path) -> Path: # --------------------------------------------------------------------------- -# Scope computation +# Scope resolution # --------------------------------------------------------------------------- class TestComputeScope: - def test_explicit_instance_name(self) -> None: - assert compute_scope(port=8080, instance_name="myapp") == "myapp" + def test_explicit_scope(self) -> None: + assert compute_scope(port=1234, explicit_scope="myapp") == "myapp" - def test_default_scope_includes_port(self) -> None: - scope = compute_scope(port=9090) - assert scope.endswith("-9090") - - def test_default_scope_is_deterministic(self) -> None: - a = compute_scope(port=8080) - b = compute_scope(port=8080) - assert a == b - - def test_different_ports_different_scopes(self) -> None: - a = compute_scope(port=8080) - b = compute_scope(port=9090) - assert a != b - - def test_hash_prefix_is_8_chars(self) -> None: + def test_default_scope_is_stable_for_port(self) -> None: scope = compute_scope(port=8080) - prefix = scope.rsplit("-", 1)[0] - assert len(prefix) == 8 - - def test_git_failure_falls_back_to_cwd(self) -> None: - import nemo_platform_ext.cli.commands.services._process as proc_mod - - proc_mod._scope_prefix_cache = None - try: - with patch.object(proc_mod, "_find_git_root", return_value="/no/git/here"): - scope = compute_scope(port=8080) - assert scope.endswith("-8080") - assert len(scope.rsplit("-", 1)[0]) == 8 - finally: - proc_mod._scope_prefix_cache = None - def test_different_git_roots_produce_different_prefixes(self) -> None: - """Two different working directories (worktrees) produce distinct scopes.""" - import nemo_platform_ext.cli.commands.services._process as proc_mod - - with patch.object(proc_mod, "_find_git_root", return_value="/workspace/project-a"): - scope_a = compute_scope(port=8080) - - proc_mod._scope_prefix_cache = None - - with patch.object(proc_mod, "_find_git_root", return_value="/workspace/project-b"): - scope_b = compute_scope(port=8080) - - assert scope_a != scope_b - assert scope_a.endswith("-8080") - assert scope_b.endswith("-8080") - prefix_a = scope_a.rsplit("-", 1)[0] - prefix_b = scope_b.rsplit("-", 1)[0] - assert prefix_a != prefix_b + assert scope == compute_scope(port=8080) + assert scope.endswith("-8080") # --------------------------------------------------------------------------- -# Instance directory +# Scope directory # --------------------------------------------------------------------------- @@ -168,26 +127,27 @@ class TestDescriptorRoundTrip: def test_write_and_read(self, base_dir: Path) -> None: desc = InstanceDescriptor( pid=12345, - scope="test-8080", - host="127.0.0.1", - port=8080, + config=PlatformAppConfig( + scope="test-8080", + services=["entities", "models"], + controllers=["jobs"], + host="127.0.0.1", + ), mode="background", create_time=1000.0, - services=["entities", "models"], - controllers=["jobs"], ) write_descriptor(desc, base_dir=base_dir) recovered = read_descriptor("test-8080", base_dir=base_dir) assert recovered is not None assert recovered.pid == 12345 - assert recovered.scope == "test-8080" - assert recovered.host == "127.0.0.1" - assert recovered.port == 8080 + assert recovered.config.scope == "test-8080" + assert recovered.config.host == "127.0.0.1" + assert recovered.config.port == 8080 assert recovered.mode == "background" assert recovered.create_time == 1000.0 - assert recovered.services == ["entities", "models"] - assert recovered.controllers == ["jobs"] + assert recovered.config.services == ["entities", "models"] + assert recovered.config.controllers == ["jobs"] def test_read_missing_returns_none(self, base_dir: Path) -> None: assert read_descriptor("no-such-scope", base_dir=base_dir) is None @@ -200,9 +160,7 @@ def test_read_corrupt_returns_none(self, base_dir: Path) -> None: def test_remove_descriptor(self, base_dir: Path) -> None: desc = InstanceDescriptor( pid=1, - scope="rm-test", - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope="rm-test"), mode="background", create_time=1.0, ) @@ -245,9 +203,7 @@ def test_lists_alive_instance(self, base_dir: Path) -> None: fd = acquire_lock("alive-one", base_dir=base_dir) desc = InstanceDescriptor( pid=os.getpid(), - scope="alive-one", - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope="alive-one"), mode="foreground", create_time=1.0, ) @@ -265,9 +221,7 @@ def test_cleans_up_dead_descriptor(self, base_dir: Path) -> None: d = instance_dir("dead-scope", base_dir=base_dir) desc = InstanceDescriptor( pid=999999, - scope="dead-scope", - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope="dead-scope"), mode="background", create_time=1.0, ) @@ -281,9 +235,7 @@ def test_stale_descriptor_with_logs_stays_listed(self, base_dir: Path) -> None: d = instance_dir("dead-with-logs", base_dir=base_dir) desc = InstanceDescriptor( pid=999999, - scope="dead-with-logs", - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope="dead-with-logs"), mode="background", create_time=1.0, ) @@ -370,7 +322,7 @@ def test_refuses_running_instance(self, base_dir: Path) -> None: os.close(fd) def test_rejects_invalid_scope(self, base_dir: Path) -> None: - with pytest.raises(ValueError, match="Invalid instance scope"): + with pytest.raises(ValueError, match="Invalid scope"): remove_instance("../escape", base_dir=base_dir) def test_returns_false_when_rmtree_fails(self, base_dir: Path) -> None: @@ -378,7 +330,7 @@ def test_returns_false_when_rmtree_fails(self, base_dir: Path) -> None: (d / "services.log").write_text("logs\n") with patch( - "nemo_platform_ext.cli.commands.services._process.shutil.rmtree", + "nemo_platform_ext.local.process.shutil.rmtree", side_effect=OSError("permission denied"), ): assert remove_instance("rmtree-fail", base_dir=base_dir) is False @@ -489,9 +441,7 @@ def test_stops_running_process(self, base_dir: Path) -> None: fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=proc.pid, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=psutil.Process(proc.pid).create_time(), ) @@ -512,9 +462,7 @@ def test_cleans_up_stale_descriptor(self, base_dir: Path) -> None: scope = "stale" desc = InstanceDescriptor( pid=999999999, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=0.0, ) @@ -529,9 +477,7 @@ def test_refuses_to_stop_foreground_instance(self, base_dir: Path) -> None: try: desc = InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="foreground", create_time=1.0, ) @@ -555,9 +501,7 @@ def test_force_stops_foreground_instance(self, base_dir: Path) -> None: try: desc = InstanceDescriptor( pid=proc.pid, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="foreground", create_time=psutil.Process(proc.pid).create_time(), ) @@ -571,6 +515,34 @@ def test_force_stops_foreground_instance(self, base_dir: Path) -> None: proc.kill() proc.wait(timeout=5) + def test_preserves_descriptor_when_sigkill_does_not_stop_parent(self, base_dir: Path, monkeypatch) -> None: + scope = "sigkill-still-alive" + desc = InstanceDescriptor( + pid=12345, + config=PlatformAppConfig(scope=scope), + mode="background", + create_time=1.0, + ) + write_descriptor(desc, base_dir=base_dir) + kill_signals: list[int] = [] + + def fake_kill(_pid: int, sig: int) -> None: + kill_signals.append(sig) + + monkeypatch.setattr(process_module, "validate_pid", lambda _pid, _create_time: True) + monkeypatch.setattr(process_module, "_pid_alive", lambda _pid: True) + monkeypatch.setattr(process_module, "_snapshot_children", lambda _pid: [object()]) + monkeypatch.setattr(process_module, "_sweep_orphans", lambda _children: [222]) + monkeypatch.setattr(process_module, "_SIGKILL_WAIT_TIMEOUT", 0.0) + monkeypatch.setattr(process_module.os, "kill", fake_kill) + + result = stop_instance(scope, base_dir=base_dir, timeout=0.0) + + assert kill_signals == [signal.SIGTERM, signal.SIGKILL] + assert result.stopped_pids == [] + assert result.swept_children == [222] + assert read_descriptor(scope, base_dir=base_dir) is not None + # --------------------------------------------------------------------------- # start_background @@ -578,21 +550,49 @@ def test_force_stops_foreground_instance(self, base_dir: Path) -> None: class TestStartBackground: + def test_uses_default_platform_app_config(self, tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path)) + mock_proc = MagicMock() + mock_proc.pid = 99998 + captured_args: list[str] = [] + captured_env: dict[str, str] = {} + + def fake_popen(args, **kwargs): + captured_args.extend(args) + captured_env.update(kwargs["env"]) + return mock_proc + + with patch( + "nemo_platform_ext.local.process.subprocess.Popen", + side_effect=fake_popen, + ): + proc = start_background() + + assert proc.pid == 99998 + assert captured_args[captured_args.index("--instance") + 1] == "default" + assert captured_args[captured_args.index("--host") + 1] == "127.0.0.1" + assert captured_args[captured_args.index("--port") + 1] == "8080" + assert captured_env["XDG_STATE_HOME"] == str(tmp_path) + assert "_NMP_STATE_DIR" not in captured_env + assert (tmp_path / "nmp" / "instances" / "default" / "services.log").exists() + def test_launches_detached_subprocess(self, base_dir: Path) -> None: mock_proc = MagicMock() mock_proc.pid = 99999 with patch( - "nemo_platform_ext.cli.commands.services._process.subprocess.Popen", + "nemo_platform_ext.local.process.subprocess.Popen", return_value=mock_proc, ) as mock_popen: proc = start_background( - scope="bg-test", - services=["entities", "models"], - controllers=["jobs"], - host="127.0.0.1", - port=8080, - base_dir=base_dir, + PlatformAppConfig( + scope="bg-test", + services=["entities", "models"], + controllers=["jobs"], + host="127.0.0.1", + port=8080, + state_root=base_dir, + ), ) assert proc.pid == 99999 @@ -612,13 +612,16 @@ def fake_popen(args, **kwargs): return mock_proc with patch( - "nemo_platform_ext.cli.commands.services._process.subprocess.Popen", + "nemo_platform_ext.local.process.subprocess.Popen", side_effect=fake_popen, ): start_background( - scope="data-dir-test", + PlatformAppConfig( + scope="data-dir-test", + host=DEFAULT_LOCAL_SERVICES_BIND_HOST, + state_root=base_dir, + ), data_dir="/chosen/data/dir", - base_dir=base_dir, ) assert captured_env.get("NMP_DATA_DIR") == "/chosen/data/dir" @@ -634,13 +637,16 @@ def fake_popen(args, **kwargs): return mock_proc with patch( - "nemo_platform_ext.cli.commands.services._process.subprocess.Popen", + "nemo_platform_ext.local.process.subprocess.Popen", side_effect=fake_popen, ): start_background( - scope="shell-env-test", + PlatformAppConfig( + scope="shell-env-test", + host=DEFAULT_LOCAL_SERVICES_BIND_HOST, + state_root=base_dir, + ), data_dir="/chosen/data/dir", - base_dir=base_dir, ) assert captured_env.get("NMP_DATA_DIR") == "/shell/wins" @@ -654,16 +660,22 @@ def test_rotates_log_before_start(self, base_dir: Path) -> None: mock_proc.pid = 5555 with patch( - "nemo_platform_ext.cli.commands.services._process.subprocess.Popen", + "nemo_platform_ext.local.process.subprocess.Popen", return_value=mock_proc, ): - start_background(scope="rotate-test", base_dir=base_dir) + start_background( + PlatformAppConfig( + scope="rotate-test", + host=DEFAULT_LOCAL_SERVICES_BIND_HOST, + state_root=base_dir, + ), + ) rotated = list(d.glob("services.log.*")) assert len(rotated) == 1 assert rotated[0].read_text() == "old log content\n" - def test_forwards_instance_scope_to_child(self, base_dir: Path) -> None: + def test_forwards_scope_to_child(self, base_dir: Path) -> None: mock_proc = MagicMock() mock_proc.pid = 7777 captured_args: list[str] = [] @@ -673,20 +685,22 @@ def fake_popen(args, **kwargs): return mock_proc with patch( - "nemo_platform_ext.cli.commands.services._process.subprocess.Popen", + "nemo_platform_ext.local.process.subprocess.Popen", side_effect=fake_popen, ): start_background( - scope="custom-scope", - services=["entities"], - host="127.0.0.1", - port=9090, - base_dir=base_dir, + PlatformAppConfig( + scope="custom-key", + services=["entities"], + host="127.0.0.1", + port=9090, + state_root=base_dir, + ), ) assert "--instance" in captured_args idx = captured_args.index("--instance") - assert captured_args[idx + 1] == "custom-scope" + assert captured_args[idx + 1] == "custom-key" def test_sets_launch_mode_background_in_child_env(self, base_dir: Path) -> None: mock_proc = MagicMock() @@ -698,10 +712,16 @@ def fake_popen(args, **kwargs): return mock_proc with patch( - "nemo_platform_ext.cli.commands.services._process.subprocess.Popen", + "nemo_platform_ext.local.process.subprocess.Popen", side_effect=fake_popen, ): - start_background(scope="mode-test", base_dir=base_dir) + start_background( + PlatformAppConfig( + scope="mode-test", + host=DEFAULT_LOCAL_SERVICES_BIND_HOST, + state_root=base_dir, + ), + ) assert captured_env.get("_NMP_LAUNCH_MODE") == "background" @@ -846,9 +866,7 @@ def test_sweeps_surviving_children(self, base_dir: Path) -> None: fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=parent.pid, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=psutil.Process(parent.pid).create_time(), ) @@ -883,9 +901,7 @@ def test_swept_children_empty_when_no_children(self, base_dir: Path) -> None: fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=proc.pid, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=psutil.Process(proc.pid).create_time(), ) @@ -917,9 +933,7 @@ def test_restart_path_sweeps_children(self, base_dir: Path) -> None: fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=parent.pid, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="foreground", create_time=psutil.Process(parent.pid).create_time(), ) diff --git a/packages/nemo_platform_ext/tests/cli/commands/test_setup.py b/packages/nemo_platform_ext/tests/cli/commands/test_setup.py index a911b1a3a1..bfc66f71e9 100644 --- a/packages/nemo_platform_ext/tests/cli/commands/test_setup.py +++ b/packages/nemo_platform_ext/tests/cli/commands/test_setup.py @@ -14,7 +14,6 @@ import typer from click.exceptions import Exit as ClickExit from nemo_platform.resources.inference.providers import ProvidersResource -from nemo_platform_ext.cli.commands.services._process import PortConflict from nemo_platform_ext.cli.commands.setup import ( _AGENT_API_READINESS_POLL_INTERVAL, _AGENT_DEPLOY_POLL_INTERVAL, @@ -77,6 +76,7 @@ Context, ContextDefinition, ) +from nemo_platform_ext.local.process import PortConflict from nemo_platform_plugin.client.errors import NotFoundError from nemo_platform_plugin.secrets.types import PlatformSecretCreateRequest, PlatformSecretUpdateRequest @@ -644,9 +644,10 @@ def test_start_services_background_forwards_data_dir(self): mock_start.return_value = MagicMock(pid=42) _start_services_background("http://localhost:9090", data_dir="/chosen/data/dir") mock_start.assert_called_once() - _, kwargs = mock_start.call_args + args, kwargs = mock_start.call_args + config = args[0] assert kwargs["data_dir"] == "/chosen/data/dir" - assert kwargs["port"] == 9090 + assert config.port == 9090 def test_auto_mode_skips_prompt_and_uses_persisted(self, tmp_path, monkeypatch): """`--auto` must not prompt but should still honor any persisted data dir.""" diff --git a/packages/nemo_platform_ext/tests/local/test_config_environment.py b/packages/nemo_platform_ext/tests/local/test_config_environment.py new file mode 100644 index 0000000000..8255b3b6be --- /dev/null +++ b/packages/nemo_platform_ext/tests/local/test_config_environment.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for configuration and environment resolution. + +These tests exercise the real ``apply_run_environment`` code path with +actual YAML config files, verifying that environment variables are set +correctly for different host, port, and base_url scenarios. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from nmp.platform_runner.config import ( + ResolvedRunConfiguration, + apply_run_environment, + default_config_path, +) + + +def _resolved( + *, + services: set[str] | None = None, + controllers: set[str] | None = None, + sidecars: set[str] | None = None, + host: str = "127.0.0.1", + port: int = 8080, + config_path: str | None = None, + socket_path: str | None = None, +) -> ResolvedRunConfiguration: + return ResolvedRunConfiguration( + services=services or set(), + controllers=controllers or set(), + sidecars=sidecars or set(), + host=host, + port=port, + config_path=config_path or default_config_path(), + socket_path=socket_path, + available_services={}, + available_controllers={}, + ) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_apply_run_environment_preserves_external_base_url() -> None: + """Pre-set NMP_BASE_URL (e.g. from k8s/Helm) must not be overwritten.""" + env: dict[str, str] = {"NMP_BASE_URL": "https://platform.k8s.internal:443"} + config = _resolved(host="0.0.0.0", port=9090) + + apply_run_environment(config, env=env) + + assert env["NMP_BASE_URL"] == "https://platform.k8s.internal:443" + + +@pytest.mark.integration +def test_apply_run_environment_wildcard_host_becomes_loopback(tmp_path: Path) -> None: + """A wildcard bind host (0.0.0.0) in the config file should resolve to + 127.0.0.1 for the base URL, using the actual bind port.""" + config_file = tmp_path / "config.yaml" + config_file.write_text("platform:\n base_url: http://0.0.0.0:8080\n") + + env: dict[str, str] = {} + config = _resolved(host="0.0.0.0", port=9090, config_path=str(config_file)) + + apply_run_environment(config, env=env) + + assert env["NMP_BASE_URL"] == "http://127.0.0.1:9090" + assert env["NMP_SERVICE_HOST"] == "127.0.0.1" + assert env["NMP_SERVICE_PORT"] == "9090" + + +@pytest.mark.integration +def test_apply_run_environment_ipv6_literal_bracketed(tmp_path: Path) -> None: + """An IPv6 config base_url should produce a bracketed host in the resolved URL.""" + config_file = tmp_path / "config.yaml" + config_file.write_text("platform:\n base_url: http://[::1]:8080\n") + + env: dict[str, str] = {} + config = _resolved(host="::1", port=9090, config_path=str(config_file)) + + apply_run_environment(config, env=env) + + assert env["NMP_BASE_URL"] == "http://[::1]:9090" + + +@pytest.mark.integration +def test_config_file_base_url_malformed_yaml_falls_back(tmp_path: Path) -> None: + """A corrupt config file should fall back to the bind-derived URL.""" + config_file = tmp_path / "config.yaml" + config_file.write_text("{{{{not valid yaml at all") + + env: dict[str, str] = {} + config = _resolved(host="127.0.0.1", port=7777, config_path=str(config_file)) + + apply_run_environment(config, env=env) + + # Falls back to bind-derived: http://: + assert env["NMP_BASE_URL"] == "http://127.0.0.1:7777" + + +@pytest.mark.integration +def test_config_file_missing_falls_back(tmp_path: Path) -> None: + """A missing config file should fall back to the bind-derived URL.""" + env: dict[str, str] = {} + config = _resolved(host="127.0.0.1", port=5555, config_path=str(tmp_path / "nonexistent.yaml")) + + apply_run_environment(config, env=env) + + assert env["NMP_BASE_URL"] == "http://127.0.0.1:5555" + + +@pytest.mark.integration +def test_apply_run_environment_clears_empty_service_lists() -> None: + """When services/controllers/sidecars are empty sets, their env vars + should be removed (popped) rather than set to empty strings.""" + env: dict[str, str] = { + "NMP_SERVICES": "old-service", + "NMP_CONTROLLERS": "old-controller", + "NMP_SIDECARS": "old-sidecar", + } + config = _resolved(services=set(), controllers=set(), sidecars=set()) + + apply_run_environment(config, env=env) + + assert "NMP_SERVICES" not in env + assert "NMP_CONTROLLERS" not in env + assert "NMP_SIDECARS" not in env + + +@pytest.mark.integration +def test_apply_run_environment_sets_service_lists() -> None: + """Non-empty service/controller/sidecar sets should be written as + comma-separated, sorted env var values.""" + env: dict[str, str] = {} + config = _resolved( + services={"models", "auth", "secrets"}, + controllers={"beta-controller"}, + sidecars={"adapters"}, + ) + + apply_run_environment(config, env=env) + + assert env["NMP_SERVICES"] == "auth,models,secrets" + assert env["NMP_CONTROLLERS"] == "beta-controller" + assert env["NMP_SIDECARS"] == "adapters" + + +@pytest.mark.integration +def test_apply_run_environment_uds_transport_uses_unix_base_url() -> None: + """When a socket_path is set (UDS transport), the base URL should use + the ``unix://`` scheme.""" + env: dict[str, str] = {} + config = _resolved(socket_path="/tmp/nemo.sock") + + apply_run_environment(config, env=env) + + assert env["NMP_BASE_URL"] == "unix:///tmp/nemo.sock" + + +@pytest.mark.integration +def test_apply_run_environment_preserves_external_host_and_port() -> None: + """Pre-set NMP_SERVICE_HOST and NMP_SERVICE_PORT should not be overwritten.""" + env: dict[str, str] = { + "NMP_SERVICE_HOST": "10.0.0.1", + "NMP_SERVICE_PORT": "443", + } + config = _resolved(host="0.0.0.0", port=9090) + + apply_run_environment(config, env=env) + + assert env["NMP_SERVICE_HOST"] == "10.0.0.1" + assert env["NMP_SERVICE_PORT"] == "443" + + +@pytest.mark.integration +def test_apply_run_environment_ipv6_wildcard_becomes_loopback(tmp_path: Path) -> None: + """The IPv6 wildcard ``::`` should resolve to ``::1`` for internal clients.""" + # Use a config file without platform.base_url so the bind host drives the URL. + config_file = tmp_path / "config.yaml" + config_file.write_text("platform:\n seed_on_startup: false\n") + + env: dict[str, str] = {} + config = _resolved(host="::", port=8080, config_path=str(config_file)) + + apply_run_environment(config, env=env) + + assert env["NMP_SERVICE_HOST"] == "::1" + # Base URL should have bracketed IPv6. + assert env["NMP_BASE_URL"] == "http://[::1]:8080" diff --git a/packages/nemo_platform_ext/tests/local/test_daemon_lifecycle.py b/packages/nemo_platform_ext/tests/local/test_daemon_lifecycle.py new file mode 100644 index 0000000000..e1a3c5c7e1 --- /dev/null +++ b/packages/nemo_platform_ext/tests/local/test_daemon_lifecycle.py @@ -0,0 +1,503 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for daemon subprocess lifecycle. + +These tests spawn REAL child processes via ``daemonize_services()``, exercise +real lock acquisition, descriptor file I/O, HTTP readiness probing, and +graceful shutdown via ``stop_instance()``. Nothing is monkeypatched away — +the child runs a real uvicorn server with the ``hello-world`` service. + +Requirements: +- All packages installed (``uv sync --all-packages``) so entry-point + discovery finds hello-world. +- ``pyleak`` importable (from the ``[all]`` extra). +""" + +from __future__ import annotations + +import os +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import psutil +import pytest +from nemo_platform_ext.local import process, services +from nemo_platform_ext.local.process import ForegroundInstanceError +from nemo_platform_ext.local.services import ( + ServiceRunConfig, + ServicesAlreadyRunningError, + ServicesStartupExitedError, +) +from nmp.platform_runner.config import PlatformAppConfig + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _free_tcp_port() -> int: + """Bind to port 0, let the OS pick, then release and return the port number.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _daemon_config( + tmp_path: Path, + *, + scope: str = "integ-daemon", + port: int | None = None, +) -> ServiceRunConfig: + """Build a ServiceRunConfig that is fully isolated under ``tmp_path``.""" + return ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + controllers=(), + sidecars=(), + transport="tcp", + host="127.0.0.1", + port=port or _free_tcp_port(), + scope=scope, + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + readiness_timeout=30.0, + readiness_poll_interval=0.3, + ) + + +def _ensure_stopped(cfg: ServiceRunConfig) -> None: + """Best-effort cleanup: stop any instance left running by a test.""" + try: + process.stop_instance(cfg.scope, base_dir=cfg.state_root, timeout=10, force=True) + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_daemonize_services_spawns_child_that_becomes_ready(tmp_path: Path) -> None: + """Spawn a real daemon subprocess, verify readiness via HTTP, then + gracefully shut down with ``stop_instance``.""" + cfg = _daemon_config(tmp_path) + handle = None + try: + handle = services.daemonize_services(cfg) + + # -- The handle should report the child's PID and transport details. + assert handle.pid is not None + assert handle.port == cfg.port + assert handle.transport == "tcp" + + # -- The lock file should be held by the child. + assert process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + + # -- The descriptor should have been written by the child. + desc = process.read_descriptor(cfg.scope, base_dir=cfg.state_root) + assert desc is not None + assert desc.pid == handle.pid + assert desc.mode == "daemon" + assert "hello-world" in (desc.config.services or []) + + # -- The child should still be running and respond to /status. + assert services.probe_status(base_url=f"http://127.0.0.1:{cfg.port}", timeout=5.0) + + # -- Graceful shutdown. + result = process.stop_instance(cfg.scope, base_dir=cfg.state_root, timeout=15) + assert handle.pid in result.stopped_pids + + # -- After stop, the lock should be released and the descriptor removed. + assert not process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + assert process.read_descriptor(cfg.scope, base_dir=cfg.state_root) is None + finally: + _ensure_stopped(cfg) + + +@pytest.mark.integration +def test_daemonize_services_child_exit_before_readiness(tmp_path: Path) -> None: + """When the child exits before becoming ready, ``daemonize_services`` + should raise ``ServicesStartupExitedError`` with the log path.""" + # Spawn a child that will exit immediately: give it a bogus service name + # that will fail validation in resolve_run_configuration. + bad_cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("nonexistent-service-xyz",), + controllers=(), + sidecars=(), + transport="tcp", + host="127.0.0.1", + port=_free_tcp_port(), + scope="integ-early-exit", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + readiness_timeout=15.0, + readiness_poll_interval=0.2, + ) + with pytest.raises(ServicesStartupExitedError, match="exited with code"): + services.daemonize_services(bad_cfg) + + # -- The lock should not be held after the failed startup. + assert not process.is_instance_alive(bad_cfg.scope, base_dir=bad_cfg.state_root) + + +@pytest.mark.integration +def test_stale_socket_cleanup_after_process_crash(tmp_path: Path) -> None: + """If a previous daemon crashed and left a UDS socket file, a new daemon + startup should clean it up and succeed.""" + scope = "stale" + # Use a short temp directory to stay within AF_UNIX path limits (103 bytes on macOS). + short_tmp = Path(tempfile.mkdtemp(prefix="nemo-")) + runtime_dir = short_tmp / "run" + + # Create a stale UDS socket file (no process listening). + socket_dir = runtime_dir / scope + socket_dir.mkdir(parents=True, exist_ok=True) + stale_socket = socket_dir / "nemo-platform.sock" + # Bind a real UDS socket to create the file, then close immediately. + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(str(stale_socket)) + assert stale_socket.exists() + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + controllers=(), + sidecars=(), + transport="uds", + host="127.0.0.1", + port=_free_tcp_port(), + scope=scope, + state_dir=short_tmp / "state", + runtime_dir=runtime_dir, + readiness_timeout=30.0, + readiness_poll_interval=0.3, + ) + try: + services.daemonize_services(cfg) + + # -- The daemon should be ready. + assert process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + + # -- The stale socket should have been replaced with the new one. + assert stale_socket.exists() + finally: + _ensure_stopped(cfg) + import shutil + + shutil.rmtree(short_tmp, ignore_errors=True) + + +@pytest.mark.integration +def test_concurrent_daemonize_rejects_duplicate_instance(tmp_path: Path) -> None: + """Starting a second daemon with the same instance scope should fail + with ``ServicesAlreadyRunningError`` while the first is running.""" + cfg = _daemon_config(tmp_path, scope="integ-dup") + try: + services.daemonize_services(cfg) + assert process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + + # -- A second daemonize with the same scope should fail. + dup_cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + controllers=(), + sidecars=(), + transport="tcp", + host="127.0.0.1", + port=_free_tcp_port(), # Different port, same scope. + scope="integ-dup", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + readiness_timeout=5.0, + readiness_poll_interval=0.2, + ) + with pytest.raises(ServicesAlreadyRunningError): + services.daemonize_services(dup_cfg) + + # -- Original instance should still be alive. + assert process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + finally: + _ensure_stopped(cfg) + + +@pytest.mark.integration +def test_stop_instance_escalates_sigterm_to_sigkill(tmp_path: Path) -> None: + """If the daemon child ignores SIGTERM, ``stop_instance`` should escalate + to SIGKILL after the timeout and successfully terminate the process.""" + # Instead of using daemonize_services (which starts a uvicorn server that + # handles SIGTERM), we manually simulate a daemon process that ignores SIGTERM + # using the process module primitives directly. + scope = "integ-sigkill" + state_dir = tmp_path / "state" + + # Spawn a child process that ignores SIGTERM. + child = subprocess.Popen( + [ + sys.executable, + "-c", + "import signal, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "open('/dev/null', 'w'); time.sleep(300)", + ], + start_new_session=True, + ) + try: + # Write a descriptor so stop_instance can find the process. + desc = process.InstanceDescriptor( + pid=child.pid, + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=0, state_root=state_dir), + transport="tcp", + mode="daemon", + create_time=psutil.Process(child.pid).create_time(), + ) + process.write_descriptor(desc, base_dir=state_dir) + + # Also create a lock file the process "holds" — but since it's a + # different process, we simulate by NOT acquiring a real flock (the + # test exercises PID-based stop, not flock-based liveness). + + # Stop with a very short timeout so it escalates quickly. + result = process.stop_instance(scope, base_dir=state_dir, timeout=1.0, force=True) + assert child.pid in result.stopped_pids + + # The child should be dead now. + child.wait(timeout=5) + assert child.returncode is not None + finally: + try: + child.kill() + child.wait(timeout=3) + except Exception: + pass + + +@pytest.mark.integration +def test_daemonize_services_cleans_up_on_child_exception(tmp_path: Path) -> None: + """When the child process crashes during init (e.g. corrupted request JSON), + the parent detects the exit, raises, and the lock is not left held.""" + scope = "integ-crash" + state_dir = tmp_path / "state" + instance_dir = process.instance_dir(scope, base_dir=state_dir) + + # Write a corrupted request file that will make _service_child crash + # during JSON deserialization. + fd, tmp_req = tempfile.mkstemp(dir=str(instance_dir), suffix=".json") + os.write(fd, b"NOT VALID JSON {{{") + os.close(fd) + + log_path = process.log_path_for(scope, base_dir=state_dir) + log_file = open(log_path, "a") # noqa: SIM115 + child_module = "nemo_platform_ext.local._service_child" + proc = subprocess.Popen( + [sys.executable, "-m", child_module, tmp_req], + stdout=log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + stdin=subprocess.DEVNULL, + close_fds=True, + ) + log_file.close() + + # Wait for the child to exit (it should crash quickly on bad JSON). + proc.wait(timeout=10) + assert proc.returncode != 0 + + # The lock should not be held — the child never acquired it. + assert not process.is_instance_alive(scope, base_dir=state_dir) + + # The request file should have been cleaned up by _service_child. + assert not Path(tmp_req).exists() + + +# --------------------------------------------------------------------------- +# Priority 2: Process Lifecycle & Cleanup +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_stop_instance_sweeps_orphaned_children(tmp_path: Path) -> None: + """When a daemon parent is stopped, any grandchild processes that survive + should be swept by ``_sweep_orphans``.""" + scope = "integ-orphans" + state_dir = tmp_path / "state" + + # Spawn a parent that spawns a long-lived grandchild, then sleeps. + parent = subprocess.Popen( + [ + sys.executable, + "-c", + "import subprocess, sys, time; " + "gc = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(300)']); " + "time.sleep(300)", + ], + start_new_session=True, + ) + try: + # Give the parent time to spawn the grandchild. + time.sleep(0.5) + grandchildren = psutil.Process(parent.pid).children(recursive=True) + assert len(grandchildren) >= 1, "grandchild was not spawned" + + desc = process.InstanceDescriptor( + pid=parent.pid, + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=0, state_root=state_dir), + transport="tcp", + mode="daemon", + create_time=psutil.Process(parent.pid).create_time(), + ) + process.write_descriptor(desc, base_dir=state_dir) + + result = process.stop_instance(scope, base_dir=state_dir, timeout=10, force=True) + assert parent.pid in result.stopped_pids + assert len(result.swept_children) >= 1 + + # Both parent and grandchild should be dead. + parent.wait(timeout=5) + for gc in grandchildren: + gc.wait(timeout=5) + finally: + try: + parent.kill() + parent.wait(timeout=3) + except Exception: + pass + for gc in grandchildren: + try: + gc.kill() + gc.wait(timeout=3) + except Exception: + pass + + +@pytest.mark.integration +def test_stop_instance_foreground_mode_requires_force(tmp_path: Path) -> None: + """Stopping a foreground-mode instance without ``force=True`` should raise + ``ForegroundInstanceError``. With ``force=True`` it should proceed.""" + scope = "integ-foreground" + state_dir = tmp_path / "state" + + child = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(300)"], + start_new_session=True, + ) + try: + desc = process.InstanceDescriptor( + pid=child.pid, + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=0, state_root=state_dir), + transport="tcp", + mode="foreground", + create_time=psutil.Process(child.pid).create_time(), + ) + process.write_descriptor(desc, base_dir=state_dir) + + # Without force, should raise. + with pytest.raises(ForegroundInstanceError): + process.stop_instance(scope, base_dir=state_dir, timeout=5) + + # Process should still be alive after the rejected stop. + assert child.poll() is None + + # With force, should succeed. + result = process.stop_instance(scope, base_dir=state_dir, timeout=5, force=True) + assert child.pid in result.stopped_pids + child.wait(timeout=5) + finally: + try: + child.kill() + child.wait(timeout=3) + except Exception: + pass + + +@pytest.mark.integration +def test_is_instance_alive_with_stale_lock(tmp_path: Path) -> None: + """If the lock file exists but no process holds the flock, + ``is_instance_alive`` should return False.""" + scope = "integ-stale-lock" + state_dir = tmp_path / "state" + + # Create the lock file without holding a flock on it. + inst_dir = process.instance_dir(scope, base_dir=state_dir) + lock_path = inst_dir / process.LOCK_FILENAME + lock_path.touch() + + assert not process.is_instance_alive(scope, base_dir=state_dir) + + +@pytest.mark.integration +def test_is_instance_alive_with_held_lock(tmp_path: Path) -> None: + """If a process holds the flock, ``is_instance_alive`` should return True.""" + scope = "integ-held-lock" + state_dir = tmp_path / "state" + + fd = process.acquire_lock(scope, base_dir=state_dir) + try: + assert process.is_instance_alive(scope, base_dir=state_dir) + finally: + os.close(fd) + + # After releasing the fd (which releases the flock), should be false. + assert not process.is_instance_alive(scope, base_dir=state_dir) + + +@pytest.mark.integration +def test_validate_pid_detects_recycled_process(tmp_path: Path) -> None: + """After a process dies, ``validate_pid`` should return False if the PID is + reused by a different process (detected via create_time mismatch).""" + # Spawn and immediately kill a short-lived process to get a PID + create_time. + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(300)"]) + pid = child.pid + create_time = psutil.Process(pid).create_time() + + # The PID is alive and create_time matches. + assert process.validate_pid(pid, create_time) + + # Kill it. + child.kill() + child.wait(timeout=5) + + # Now validate_pid should return False — the process is dead. + assert not process.validate_pid(pid, create_time) + + # Even with a wildly wrong create_time, should be False for a dead PID. + assert not process.validate_pid(pid, 0.0) + + +@pytest.mark.integration +def test_rotate_log_preserves_existing_content(tmp_path: Path) -> None: + """``rotate_log`` should rename the existing log and return the path for + the new (empty) log. The old content must be preserved.""" + scope = "integ-rotate" + state_dir = tmp_path / "state" + + # Write initial log content. + log_path = process.log_path_for(scope, base_dir=state_dir) + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text("original log content\n") + + # Rotate. + new_log = process.rotate_log(scope, base_dir=state_dir) + assert new_log == log_path + assert not log_path.exists() # Original was renamed. + + # Find the rotated file. + rotated_files = [f for f in log_path.parent.iterdir() if f.name.startswith("services.log.")] + assert len(rotated_files) == 1 + assert rotated_files[0].read_text() == "original log content\n" + + # Write new content, rotate again. + log_path.write_text("second run\n") + process.rotate_log(scope, base_dir=state_dir) + + rotated_files = sorted(f for f in log_path.parent.iterdir() if f.name.startswith("services.log.")) + assert len(rotated_files) == 2 + contents = {f.read_text() for f in rotated_files} + assert "original log content\n" in contents + assert "second run\n" in contents diff --git a/packages/nemo_platform_ext/tests/local/test_health_child.py b/packages/nemo_platform_ext/tests/local/test_health_child.py new file mode 100644 index 0000000000..04a975d2f0 --- /dev/null +++ b/packages/nemo_platform_ext/tests/local/test_health_child.py @@ -0,0 +1,340 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for health/readiness probing, lifespan, and child process module. + +Covers Priorities 5 (lifespan), 6 (health), and 7 (child process) from the +integration test plan. +""" + +from __future__ import annotations + +import json +import os +import socket +import subprocess +import sys +import tempfile +import threading +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from nemo_platform_ext.local import process +from nemo_platform_ext.local.services import ServiceRunConfig +from nemo_platform_ext.local.transport import probe_status, wait_for_status + + +def _free_tcp_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +# --------------------------------------------------------------------------- +# Priority 5: Multi-Service Startup & Lifespan +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_create_app_starts_and_joins_controller_threads() -> None: + """A controller registered via ``create_app`` should have its thread + started during lifespan and stopped on exit.""" + started = threading.Event() + stopped = threading.Event() + + def controller_run(stop_signal: threading.Event) -> None: + started.set() + stop_signal.wait(timeout=5.0) + stopped.set() + + with ( + patch("nmp.platform_runner.server.get_platform_config") as mock_pc, + patch("nmp.platform_runner.server.get_auth_config") as mock_ac, + patch("nmp.common.auth.middleware.get_auth_config") as mock_ac2, + ): + mock_pc.return_value = MagicMock(seed_on_startup=False, redirect_root_to_studio=False) + mock_ac.return_value = MagicMock(enabled=False, policy_decision_point_provider="embedded") + mock_ac2.return_value = MagicMock(enabled=False, policy_decision_point_provider="embedded") + + from nmp.platform_runner.server import create_app + + app = create_app(services=[], controller_run_funcs={"test-ctrl": controller_run}) + + from fastapi.testclient import TestClient + + with TestClient(app): + assert started.wait(timeout=2.0), "controller thread did not start" + + assert stopped.wait(timeout=2.0), "controller thread did not stop after lifespan exit" + + +@pytest.mark.integration +def test_create_app_controller_thread_join_timeout() -> None: + """A controller that ignores the stop signal should not hang shutdown — + ``thread.join(timeout=5)`` should return even if the controller is still running.""" + started = threading.Event() + + def stubborn_controller(stop_signal: threading.Event) -> None: + started.set() + # Ignore stop_signal — simulate a controller that hangs. + import time + + time.sleep(300) + + with ( + patch("nmp.platform_runner.server.get_platform_config") as mock_pc, + patch("nmp.platform_runner.server.get_auth_config") as mock_ac, + patch("nmp.common.auth.middleware.get_auth_config") as mock_ac2, + ): + mock_pc.return_value = MagicMock(seed_on_startup=False, redirect_root_to_studio=False) + mock_ac.return_value = MagicMock(enabled=False, policy_decision_point_provider="embedded") + mock_ac2.return_value = MagicMock(enabled=False, policy_decision_point_provider="embedded") + + from nmp.platform_runner.server import create_app + + app = create_app(services=[], controller_run_funcs={"stubborn": stubborn_controller}) + + from fastapi.testclient import TestClient + + # The TestClient __exit__ triggers lifespan exit, which calls thread.join(timeout=5). + # This should NOT hang forever — the 5s timeout should let shutdown proceed. + with TestClient(app): + assert started.wait(timeout=2.0), "controller thread did not start" + + # If we got here, shutdown didn't hang. The stubborn thread is still running + # but as a daemon thread it will be cleaned up when the test process exits. + + +@pytest.mark.integration +def test_lifespan_cleanup_runs_on_app_shutdown() -> None: + """``close_shared_http_clients`` should be called during lifespan teardown.""" + cleanup_called = threading.Event() + + with ( + patch("nmp.platform_runner.server.get_platform_config") as mock_pc, + patch("nmp.platform_runner.server.get_auth_config") as mock_ac, + patch("nmp.common.auth.middleware.get_auth_config") as mock_ac2, + patch("nmp.platform_runner.server.close_shared_http_clients") as mock_close, + ): + mock_pc.return_value = MagicMock(seed_on_startup=False, redirect_root_to_studio=False) + mock_ac.return_value = MagicMock(enabled=False, policy_decision_point_provider="embedded") + mock_ac2.return_value = MagicMock(enabled=False, policy_decision_point_provider="embedded") + + async def fake_close(): + cleanup_called.set() + + mock_close.side_effect = fake_close + + from nmp.platform_runner.server import create_app + + app = create_app(services=[]) + + from fastapi.testclient import TestClient + + with TestClient(app): + pass + + assert cleanup_called.is_set(), "close_shared_http_clients was not called during shutdown" + + +# --------------------------------------------------------------------------- +# Priority 6: Health & Readiness +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_wait_for_status_retries_on_transient_errors(tmp_path: Path) -> None: + """``wait_for_status`` should retry on connection refused and eventually + return True once the server starts responding.""" + from nemo_platform_ext.local import services + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + controllers=(), + sidecars=(), + transport="tcp", + host="127.0.0.1", + port=_free_tcp_port(), + scope="integ-wait-retry", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + readiness_timeout=30.0, + readiness_poll_interval=0.3, + ) + + # Start the daemon — wait_for_status should retry until it's ready. + services.daemonize_services(cfg) + try: + # The daemon is already ready (daemonize_services waits for readiness). + # Verify wait_for_status succeeds with a fresh probe. + assert wait_for_status( + base_url=f"http://127.0.0.1:{cfg.port}", + timeout=5.0, + poll_interval=0.2, + ) + finally: + process.stop_instance(cfg.scope, base_dir=cfg.state_root, timeout=10, force=True) + + +@pytest.mark.integration +def test_wait_for_status_times_out_on_no_server() -> None: + """``wait_for_status`` should return False when no server is listening.""" + port = _free_tcp_port() + result = wait_for_status( + base_url=f"http://127.0.0.1:{port}", + timeout=1.0, + poll_interval=0.2, + ) + assert result is False + + +@pytest.mark.integration +def test_probe_status_with_missing_uds_socket() -> None: + """Probing a non-existent UDS socket should return False.""" + result = probe_status( + base_url="http+unix:///nonexistent/path/nemo.sock", + socket_path=Path("/nonexistent/path/nemo.sock"), + timeout=1.0, + ) + assert result is False + + +@pytest.mark.integration +def test_probe_status_against_real_daemon(tmp_path: Path) -> None: + """``probe_status`` should return True against a running daemon.""" + from nemo_platform_ext.local import services + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + controllers=(), + sidecars=(), + transport="tcp", + host="127.0.0.1", + port=_free_tcp_port(), + scope="integ-probe-real", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + readiness_timeout=30.0, + readiness_poll_interval=0.3, + ) + services.daemonize_services(cfg) + try: + assert probe_status(base_url=f"http://127.0.0.1:{cfg.port}", timeout=5.0) + finally: + process.stop_instance(cfg.scope, base_dir=cfg.state_root, timeout=10, force=True) + + +# --------------------------------------------------------------------------- +# Priority 7: Child Process Module +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_service_child_loads_config_and_starts(tmp_path: Path) -> None: + """Write valid JSON config, run ``_service_child`` in a subprocess, + verify it starts and accepts HTTP connections.""" + port = _free_tcp_port() + state_dir = tmp_path / "state" + runtime_dir = tmp_path / "runtime" + scope = "integ-child-real" + + payload = ServiceRunConfig( + mode="daemon", + services=("hello-world",), + controllers=(), + sidecars=(), + transport="tcp", + host="127.0.0.1", + port=port, + scope=scope, + state_dir=str(state_dir), + runtime_dir=str(runtime_dir), + ).to_child_payload() + + # Write the request file the way daemonize_services does. + instance_dir = process.instance_dir(scope, base_dir=state_dir) + fd, req_path = tempfile.mkstemp(dir=str(instance_dir), suffix=".json") + os.write(fd, (json.dumps(payload) + "\n").encode()) + os.close(fd) + + log_path = process.log_path_for(scope, base_dir=state_dir) + log_file = open(log_path, "a") # noqa: SIM115 + subprocess.Popen( + [sys.executable, "-m", "nemo_platform_ext.local._service_child", req_path], + stdout=log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + stdin=subprocess.DEVNULL, + close_fds=True, + ) + log_file.close() + + try: + # Wait for the child to become ready. + assert wait_for_status( + base_url=f"http://127.0.0.1:{port}", + timeout=30.0, + poll_interval=0.3, + ), "child process did not become ready" + + # The request file should have been cleaned up. + assert not Path(req_path).exists() + + # The child should have acquired the lock and written a descriptor. + assert process.is_instance_alive(scope, base_dir=state_dir) + desc = process.read_descriptor(scope, base_dir=state_dir) + assert desc is not None + assert desc.mode == "daemon" + finally: + process.stop_instance(scope, base_dir=state_dir, timeout=10, force=True) + + +@pytest.mark.integration +def test_service_child_corrupted_payload(tmp_path: Path) -> None: + """Bad JSON in the request file should cause the child to exit non-zero.""" + scope = "integ-child-bad" + state_dir = tmp_path / "state" + instance_dir = process.instance_dir(scope, base_dir=state_dir) + + fd, req_path = tempfile.mkstemp(dir=str(instance_dir), suffix=".json") + os.write(fd, b"<<>>") + os.close(fd) + + proc = subprocess.Popen( + [sys.executable, "-m", "nemo_platform_ext.local._service_child", req_path], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + start_new_session=True, + stdin=subprocess.DEVNULL, + ) + proc.wait(timeout=15) + assert proc.returncode != 0 + + +@pytest.mark.integration +def test_service_child_cleans_up_request_file(tmp_path: Path) -> None: + """The request file should be unlinked even when the child crashes.""" + scope = "integ-child-cleanup" + state_dir = tmp_path / "state" + instance_dir = process.instance_dir(scope, base_dir=state_dir) + + fd, req_path = tempfile.mkstemp(dir=str(instance_dir), suffix=".json") + os.write(fd, b"<<>>") + os.close(fd) + + assert Path(req_path).exists() + + proc = subprocess.Popen( + [sys.executable, "-m", "nemo_platform_ext.local._service_child", req_path], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + start_new_session=True, + stdin=subprocess.DEVNULL, + ) + proc.wait(timeout=15) + + # The request file should have been cleaned up regardless of the error. + assert not Path(req_path).exists() diff --git a/packages/nemo_platform_ext/tests/local/test_port_socket.py b/packages/nemo_platform_ext/tests/local/test_port_socket.py new file mode 100644 index 0000000000..59804c6615 --- /dev/null +++ b/packages/nemo_platform_ext/tests/local/test_port_socket.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for TCP/UDS port and socket management. + +These tests exercise real port binding, socket creation, and conflict +detection using actual OS resources. +""" + +from __future__ import annotations + +import socket +import tempfile +from pathlib import Path + +import pytest +from nemo_platform_ext.local import process, services +from nemo_platform_ext.local.services import ( + ServiceRunConfig, + ServicesPortInUseError, +) +from nmp.platform_runner.config import PlatformAppConfig + + +def _free_tcp_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_tcp_port_conflict_with_foreign_process(tmp_path: Path) -> None: + """When a foreign (non-NeMo) process holds a port, ``_check_tcp_available`` + should raise ``ServicesPortInUseError`` with a helpful suggestion.""" + # Bind a TCP port and hold it open. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as blocker: + blocker.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + blocker.bind(("127.0.0.1", 0)) + blocker.listen(1) + port = blocker.getsockname()[1] + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + transport="tcp", + host="127.0.0.1", + port=port, + scope="integ-port-foreign", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + + with pytest.raises(ServicesPortInUseError, match="already in use by another process"): + services._check_tcp_available(cfg) + + +@pytest.mark.integration +def test_tcp_port_conflict_with_nemo_instance(tmp_path: Path) -> None: + """When a NeMo instance holds a port, the error should distinguish it + from a foreign process.""" + scope = "integ-port-nemo" + state_dir = tmp_path / "state" + + # Bind a port and also create a descriptor matching the scope/host/port. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as blocker: + blocker.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + blocker.bind(("127.0.0.1", 0)) + blocker.listen(1) + port = blocker.getsockname()[1] + + # Create a live lock and descriptor so it looks like a NeMo instance. + lock_fd = process.acquire_lock(scope, base_dir=state_dir) + try: + desc = process.InstanceDescriptor( + pid=1, # Dummy PID — the flock is what matters. + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=port), + transport="tcp", + mode="daemon", + create_time=0.0, + ) + process.write_descriptor(desc, base_dir=state_dir) + + conflict = process.check_port_available_for_start("127.0.0.1", port, scope, base_dir=state_dir) + assert conflict is not None + assert conflict.kind == "nemo_instance" + assert conflict.port == port + + lines = process.format_port_conflict(conflict) + assert any("NeMo Platform" in line for line in lines) + finally: + import os + + os.close(lock_fd) + + +@pytest.mark.integration +def test_tcp_port_available_when_free(tmp_path: Path) -> None: + """When a port is free, ``check_port_available_for_start`` returns None.""" + port = _free_tcp_port() + conflict = process.check_port_available_for_start("127.0.0.1", port, "integ-free", base_dir=tmp_path / "state") + assert conflict is None + + +@pytest.mark.integration +def test_uds_socket_path_max_validation() -> None: + """A socket path exceeding AF_UNIX_PATH_MAX should raise ValueError.""" + # Build a path that is exactly one byte over the limit. + max_bytes = services._AF_UNIX_PATH_MAX_BYTES + # Create a path that exceeds the limit. + long_path = "/" + "x" * max_bytes # len("/") + max_bytes > max_bytes + assert len(long_path.encode()) > max_bytes + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + transport="uds", + socket_path=long_path, + scope="integ-long-sock", + state_dir="/tmp/state", + runtime_dir="/tmp/run", + ) + # _validated_socket_path calls _validate_socket_path_length internally. + with pytest.raises(ValueError, match="too long for AF_UNIX"): + services._validated_socket_path(cfg) + + +@pytest.mark.integration +def test_prepare_socket_removes_stale_socket(tmp_path: Path) -> None: + """``_prepare_socket`` should remove a stale (unreachable) socket file + and allow a new daemon to bind.""" + scope = "stale2" + short_tmp = Path(tempfile.mkdtemp(prefix="nemo-")) + runtime_dir = short_tmp / "run" + + # Create a stale socket file. + socket_dir = runtime_dir / scope + socket_dir.mkdir(parents=True, exist_ok=True) + stale_socket = socket_dir / "nemo-platform.sock" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(str(stale_socket)) + assert stale_socket.exists() + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + transport="uds", + scope=scope, + state_dir=short_tmp / "state", + runtime_dir=runtime_dir, + ) + + # _prepare_socket should probe, find it stale, remove it, and return the path. + result = services._prepare_socket(cfg) + assert result is not None + # The stale socket should have been removed (the new server hasn't bound yet). + assert not stale_socket.exists() + + import shutil + + shutil.rmtree(short_tmp, ignore_errors=True) diff --git a/packages/nemo_platform_ext/tests/local/test_services.py b/packages/nemo_platform_ext/tests/local/test_services.py new file mode 100644 index 0000000000..1964276ba0 --- /dev/null +++ b/packages/nemo_platform_ext/tests/local/test_services.py @@ -0,0 +1,1039 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import os +import socket +import subprocess +import sys +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from nemo_platform_ext.local import _service_child, services +from nemo_platform_ext.local.process import ( + DESCRIPTOR_FILENAME, + InstanceDescriptor, +) +from nemo_platform_ext.local.services import ServiceRunConfig +from nemo_platform_ext.local.transport import UDS_BASE_URL +from nmp.platform_runner.config import ( + PlatformAppConfig, + default_runtime_root, + default_state_root, + validate_scope, +) + + +def _allow_tmp_path_socket_paths(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(services, "_AF_UNIX_PATH_MAX_BYTES", 4096) + + +def _free_tcp_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _embedded_handle() -> services.EmbeddedServiceHandle: + return services.EmbeddedServiceHandle(app=object(), runtime=object()) + + +def test_service_run_config_normalizes_lists_to_tuples() -> None: + cfg = ServiceRunConfig(services=["entities", "models"], controllers=["jobs"]) + + assert cfg.services == ("entities", "models") + assert cfg.controllers == ("jobs",) + + +def test_service_run_config_converts_to_platform_app_config(tmp_path: Path) -> None: + cfg = ServiceRunConfig( + services=["entities", "models"], + controllers=[], + sidecars=["adapters"], + config_path=tmp_path / "local.yaml", + socket_path=tmp_path / "nemo.sock", + mode="embedded", + ) + + app_config = cfg.to_platform_app_config() + + assert app_config.services == ("entities", "models") + assert app_config.controllers == () + assert app_config.sidecars == ("adapters",) + assert app_config.config_path == str(tmp_path / "local.yaml") + assert app_config.socket_path == str(tmp_path / "nemo.sock") + assert app_config.runtime_root is None + assert app_config.runtime_dir() == tmp_path + assert app_config.host == "127.0.0.1" + assert app_config.port == 8080 + + +def test_instance_descriptor_converts_from_service_run_config( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + cfg = ServiceRunConfig( + services=["entities", "models"], + controllers=[], + sidecars=["adapters"], + config_path=tmp_path / "local.yaml", + socket_path=tmp_path / "nemo.sock", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + ) + monkeypatch.setattr(services.process, "get_create_time", lambda _pid: 123.0) + app_config = cfg.to_platform_app_config() + app_config.log_path = str(tmp_path / "nemo.log") + + desc = InstanceDescriptor.from_config( + app_config, + pid=4242, + mode="daemon", + transport=cfg.transport, + ) + + assert desc.pid == 4242 + assert desc.config.scope == "default" + assert desc.config.host == "127.0.0.1" + assert desc.config.port == 8080 + assert desc.transport == "uds" + assert desc.config.socket_path == str(tmp_path / "nemo.sock") + assert desc.config.state_root == str(tmp_path / "state") + assert desc.config.runtime_root == str(tmp_path / "run") + assert desc.config.state_dir() == tmp_path / "state" / "instances" / "default" + assert desc.config.runtime_dir() == tmp_path / "run" / "default" + assert desc.mode == "daemon" + assert desc.create_time == 123.0 + assert desc.config.services == ("entities", "models") + assert desc.config.controllers == () + assert desc.config.sidecars == ("adapters",) + assert desc.config.config_path == str(tmp_path / "local.yaml") + assert desc.config.log_path == str(tmp_path / "nemo.log") + assert desc.config.log_file_path() == tmp_path / "nemo.log" + payload = desc.model_dump() + assert "services" not in payload + assert "host" not in payload + assert "state_dir" not in payload + assert "runtime_dir" not in payload + assert "log_path" not in payload + assert payload["config"]["services"] == ("entities", "models") + assert payload["config"]["socket_path"] == str(tmp_path / "nemo.sock") + assert payload["config"]["state_root"] == str(tmp_path / "state") + assert payload["config"]["runtime_root"] == str(tmp_path / "run") + assert payload["config"]["log_path"] == str(tmp_path / "nemo.log") + + +def test_service_mode_enum_values() -> None: + assert services.ServiceMode.EMBEDDED.value == "embedded" + assert services.ServiceMode.DAEMON.value == "daemon" + + +def test_service_run_config_defaults_to_daemon_mode() -> None: + cfg = ServiceRunConfig() + + assert cfg.mode is services.ServiceMode.DAEMON + + +def test_service_run_config_accepts_mode_strings() -> None: + cfg = ServiceRunConfig(mode="embedded") + + assert cfg.mode is services.ServiceMode.EMBEDDED + + +def test_service_run_config_rejects_unknown_mode() -> None: + with pytest.raises(ValueError, match="mode must be 'embedded' or 'daemon'"): + ServiceRunConfig(mode="foreground") + + +def test_embedded_and_daemon_handles_implement_local_service_handle(tmp_path: Path) -> None: + embedded = services.EmbeddedServiceHandle(app=object(), runtime=object()) + daemon = services.DaemonServiceHandle( + scope="dev", + transport="tcp", + socket_path=None, + gateway_base_url=None, + host="127.0.0.1", + port=8080, + pid=123, + mode="daemon", + log_path=None, + state_dir=tmp_path / "state" / "instances" / "dev", + runtime_dir=None, + ) + + assert isinstance(embedded, services.LocalServiceHandle) + assert isinstance(daemon, services.LocalServiceHandle) + + +def test_start_services_result_is_shared_result_type() -> None: + result = services.StartServicesResult( + requested=["jobs"], + started=["auth", "jobs"], + already_active=[], + active=["secrets", "auth", "jobs"], + ) + + assert result.requested == ["jobs"] + assert result.started == ["auth", "jobs"] + assert result.active == ["secrets", "auth", "jobs"] + + +def test_service_run_config_rejects_services_with_service_group() -> None: + with pytest.raises(ValueError, match="services cannot be combined with service_group"): + ServiceRunConfig(services=("entities",), service_group="all") + + +def test_service_run_config_defaults_to_named_uds_instance() -> None: + cfg = ServiceRunConfig() + + assert cfg.transport == "uds" + assert cfg.http_gateway == "disabled" + assert cfg.scope == "default" + assert cfg.socket_path is None + + +@pytest.mark.parametrize("instance", ["has space", "../bad"]) +def test_service_run_config_rejects_invalid_scope_names(instance: str) -> None: + with pytest.raises(ValueError, match="scope"): + ServiceRunConfig(scope=instance) + + +def test_service_run_config_rejects_gateway_for_tcp_transport() -> None: + with pytest.raises(ValueError, match="gateway.*UDS"): + ServiceRunConfig(transport="tcp", http_gateway="enabled") + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("readiness_timeout", 0.0, "readiness_timeout"), + ("readiness_timeout", -1.0, "readiness_timeout"), + ("readiness_poll_interval", 0.0, "readiness_poll_interval"), + ("readiness_poll_interval", -1.0, "readiness_poll_interval"), + ], +) +def test_service_run_config_rejects_non_positive_readiness_values(field: str, value: float, message: str) -> None: + with pytest.raises(ValueError, match=message): + if field == "readiness_timeout": + ServiceRunConfig(readiness_timeout=value) + else: + ServiceRunConfig(readiness_poll_interval=value) + + +def test_process_paths_follow_existing_nmp_state_convention(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + + assert default_state_root() == tmp_path / "state" / "nmp" + assert default_runtime_root() == tmp_path / "state" / "nmp" / "run" + assert ( + PlatformAppConfig(scope="dev").socket_file_path() + == tmp_path / "state" / "nmp" / "run" / "dev" / "nemo-platform.sock" + ) + assert validate_scope("dev_1-2") == "dev_1-2" + + +def test_resolved_socket_path_rejects_relative_explicit_path() -> None: + cfg = ServiceRunConfig(socket_path="relative.sock") + + with pytest.raises(ValueError, match="UDS socket path must be absolute"): + _ = cfg.resolved_socket_path + + +def test_resolved_socket_path_rejects_relative_runtime_dir() -> None: + cfg = ServiceRunConfig(runtime_dir="relative-run") + + with pytest.raises(ValueError, match="runtime root must be absolute"): + _ = cfg.resolved_socket_path + + +def test_resolved_socket_path_rejects_relative_socket_path_with_tcp_client() -> None: + cfg = ServiceRunConfig(transport="tcp", socket_path="relative.sock") + + with pytest.raises(ValueError, match="UDS socket path must be absolute"): + _ = cfg.resolved_socket_path + + +def test_tcp_client_can_still_configure_uds_listener(tmp_path: Path) -> None: + cfg = ServiceRunConfig(transport="tcp", socket_path=tmp_path / "nemo.sock") + + app_config = cfg.to_platform_app_config() + + assert app_config.socket_path == str(tmp_path / "nemo.sock") + assert app_config.runtime_dir() == tmp_path + + +def test_instance_descriptor_rejects_uds_client_without_socket_path() -> None: + with pytest.raises(ValueError, match="UDS client transport requires config.socket_path"): + InstanceDescriptor(pid=1, config=PlatformAppConfig(scope="dev"), transport="uds") + + +def test_prepare_socket_rejects_long_generated_path_before_creating_runtime_dir( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(services, "_AF_UNIX_PATH_MAX_BYTES", 1, raising=False) + runtime_root = tmp_path / "runtime" + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=runtime_root) + + with pytest.raises(ValueError, match="UDS socket path is too long.*AF_UNIX"): + services._prepare_socket(cfg) + + assert not runtime_root.exists() + + +def test_validate_socket_path_length_reserves_trailing_nul(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(services, "_AF_UNIX_PATH_MAX_BYTES", 3, raising=False) + + services._validate_socket_path_length(Path("abc")) + with pytest.raises(ValueError, match=r"4 bytes; maximum is 3 bytes"): + services._validate_socket_path_length(Path("abcd")) + + +def test_prepare_socket_rejects_long_explicit_path_before_filesystem_or_probe( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(services, "_AF_UNIX_PATH_MAX_BYTES", 1, raising=False) + socket_parent = tmp_path / "explicit" + cfg = ServiceRunConfig(socket_path=socket_parent / "nemo-platform.sock") + + with patch("nemo_platform_ext.local.services.probe_status") as probe_status: + with pytest.raises(ValueError, match="UDS socket path is too long.*AF_UNIX"): + services._prepare_socket(cfg) + + probe_status.assert_not_called() + assert not socket_parent.exists() + + +def test_run_services_prepares_socket_after_acquiring_lock(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", port=_free_tcp_port(), state_dir=tmp_path / "state", runtime_dir=tmp_path / "run" + ) + events: list[str] = [] + real_acquire_lock = services.process.acquire_lock + + def acquire_lock(scope: str, *, base_dir: Path | None = None) -> int: + events.append("lock") + return real_acquire_lock(scope, base_dir=base_dir) + + def prepare_socket(config: ServiceRunConfig) -> Path | None: + events.append("prepare") + lock_path = ( + services.process.instance_dir(config.scope, base_dir=config.state_root) / services.process.LOCK_FILENAME + ) + assert lock_path.exists() + return config.resolved_socket_path + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services.process.acquire_lock", side_effect=acquire_lock), + patch("nemo_platform_ext.local.services._prepare_socket", side_effect=prepare_socket), + patch("nemo_platform_ext.local.services.start_embedded_services", return_value=_embedded_handle()), + patch("nemo_platform_ext.local.services.serve_embedded_app"), + ): + services.run_services(cfg) + + assert events == ["lock", "prepare"] + + +def test_run_services_cleans_lock_when_socket_prepare_fails(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", port=_free_tcp_port(), state_dir=tmp_path / "state", runtime_dir=tmp_path / "run" + ) + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch( + "nemo_platform_ext.local.services._prepare_socket", + side_effect=services.ServicesSocketStaleError("boom"), + ), + ): + with pytest.raises(services.ServicesSocketStaleError, match="boom"): + services.run_services(cfg) + + assert not services.process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + + +def test_run_services_restores_env_and_closes_lock_when_descriptor_cleanup_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + port=_free_tcp_port(), + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + data_dir=tmp_path / "data", + ) + monkeypatch.delenv("NMP_DATA_DIR", raising=False) + real_acquire_lock = services.process.acquire_lock + locked_fd: int | None = None + + def acquire_lock(scope: str, *, base_dir: Path | None = None) -> int: + nonlocal locked_fd + locked_fd = real_acquire_lock(scope, base_dir=base_dir) + return locked_fd + + real_close = os.close + closed_fds: list[int] = [] + + def close(fd: int) -> None: + closed_fds.append(fd) + real_close(fd) + + monkeypatch.setattr(os, "close", close) + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services.process.acquire_lock", side_effect=acquire_lock), + patch( + "nemo_platform_ext.local.services.process.remove_descriptor", + side_effect=RuntimeError("descriptor cleanup failed"), + ), + patch("nemo_platform_ext.local.services.start_embedded_services", return_value=_embedded_handle()), + patch("nemo_platform_ext.local.services.serve_embedded_app"), + ): + with pytest.raises(RuntimeError, match="descriptor cleanup failed"): + services.run_services(cfg) + + assert "NMP_DATA_DIR" not in os.environ + assert locked_fd is not None + assert locked_fd in closed_fds + assert closed_fds[-1] == locked_fd + + +def test_run_services_foreground_serves_embedded_app(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + mode=services.ServiceMode.EMBEDDED, + scope="dev", + port=_free_tcp_port(), + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + ) + app = object() + handle = services.EmbeddedServiceHandle(app=app, runtime=object()) + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services.start_embedded_services", return_value=handle) as start_embedded, + patch("nemo_platform_ext.local.services.serve_embedded_app") as serve_embedded, + ): + services.run_services(cfg) + + start_embedded.assert_called_once_with(cfg, env=None) + serve_embedded.assert_called_once() + assert serve_embedded.call_args.args[0] is app + + +def test_daemon_service_handle_uds_client_uses_socket_transport(tmp_path: Path) -> None: + socket_path = tmp_path / "nemo-platform.sock" + handle = services.DaemonServiceHandle( + scope="dev", + transport="uds", + socket_path=socket_path, + gateway_base_url="http://127.0.0.1:9999", + host="127.0.0.1", + port=8080, + pid=123, + mode="daemon", + log_path=None, + state_dir=tmp_path / "state" / "instances" / "dev", + runtime_dir=tmp_path, + ) + + client = handle.client() + try: + assert str(client.base_url).rstrip("/") == UDS_BASE_URL + assert handle.gateway_base_url == "http://127.0.0.1:9999" + finally: + client.close() + + +def test_embedded_handle_async_client_uses_asgi_transport() -> None: + app = MagicMock() + runtime = MagicMock() + http_client = object() + client_value = object() + handle = services.EmbeddedServiceHandle(app=app, runtime=runtime) + + with ( + patch( + "nemo_platform_ext.local.services.build_async_asgi_http_client", return_value=http_client + ) as build_client, + patch("nemo_platform_ext.local.services.AsyncNeMoPlatform", return_value=client_value) as platform_cls, + ): + client = handle.async_client(access_token="test-token") + + build_client.assert_called_once_with(app) + platform_cls.assert_called_once_with( + access_token="test-token", + http_client=http_client, + base_url=services.EMBEDDED_BASE_URL, + ) + assert client is client_value + + +def test_ensure_services_dispatches_to_embedded_mode() -> None: + cfg = ServiceRunConfig(mode=services.ServiceMode.EMBEDDED) + embedded_handle = MagicMock(spec=services.EmbeddedServiceHandle) + + with patch("nemo_platform_ext.local.services.start_embedded_services", return_value=embedded_handle): + handle = services.ensure_services(cfg) + + assert handle is embedded_handle + + +def test_ensure_services_dispatches_to_daemon_mode() -> None: + cfg = ServiceRunConfig(mode=services.ServiceMode.DAEMON) + daemon_handle = MagicMock(spec=services.DaemonServiceHandle) + + with ( + patch("nemo_platform_ext.local.services.get_service_handle", return_value=None), + patch("nemo_platform_ext.local.services.daemonize_services", return_value=daemon_handle), + ): + handle = services.ensure_services(cfg) + + assert handle is daemon_handle + + +def test_connect_services_uses_selected_mode_handle_client() -> None: + cfg = ServiceRunConfig(mode=services.ServiceMode.EMBEDDED) + handle = MagicMock(spec=services.EmbeddedServiceHandle) + client = object() + handle.client.return_value = client + + with patch("nemo_platform_ext.local.services.ensure_services", return_value=handle): + result = services.connect_services(cfg, access_token="test") + + assert result is client + handle.client.assert_called_once_with(access_token="test") + + +@pytest.mark.parametrize("mode", [services.ServiceMode.EMBEDDED, services.ServiceMode.DAEMON]) +def test_ensure_services_returns_handle_with_parity_methods(mode: services.ServiceMode) -> None: + cfg = ServiceRunConfig(mode=mode) + if mode is services.ServiceMode.EMBEDDED: + handle = MagicMock(spec=services.EmbeddedServiceHandle) + patch_target = "nemo_platform_ext.local.services.start_embedded_services" + else: + handle = MagicMock(spec=services.DaemonServiceHandle) + patch_target = "nemo_platform_ext.local.services.daemonize_services" + + with ( + patch("nemo_platform_ext.local.services.get_service_handle", return_value=None), + patch(patch_target, return_value=handle), + ): + result = services.ensure_services(cfg) + + assert result is handle + for method_name in ( + "is_running", + "wait_until_ready", + "wait_until_ready_async", + "client", + "async_client", + "start_services", + "start_services_async", + "stop", + "stop_async", + ): + assert hasattr(result, method_name), method_name + + +def test_daemonize_services_starts_child_module(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + service_group="all", + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + readiness_timeout=0.1, + readiness_poll_interval=0.01, + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services._check_tcp_available"), + patch("nemo_platform_ext.local.services.probe_status", return_value=True), + patch("nemo_platform_ext.local.services.subprocess.Popen", return_value=proc) as popen, + ): + handle = services.daemonize_services(cfg) + + args = popen.call_args.args[0] + assert args[:3] == [sys.executable, "-m", f"{services.__package__}._service_child"] + request_path = Path(args[3]) + assert request_path.parent == tmp_path / "state" / "instances" / "dev" + assert request_path.suffix == ".json" + assert request_path.name != "run-request.json" + assert handle.transport == "uds" + assert handle.socket_path == tmp_path / "run" / "dev" / "nemo-platform.sock" + assert handle.pid == 4242 + proc.terminate.assert_not_called() + proc.kill.assert_not_called() + + +def test_daemonize_services_leaves_socket_preparation_to_child(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + readiness_timeout=0.1, + readiness_poll_interval=0.01, + ) + socket_path = cfg.resolved_socket_path + assert socket_path is not None + socket_path.parent.mkdir(parents=True) + socket_path.write_text("stale", encoding="utf-8") + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services._prepare_socket", side_effect=AssertionError("parent prepared socket")), + patch("nemo_platform_ext.local.services.probe_status", side_effect=[False, True]), + patch("nemo_platform_ext.local.services.subprocess.Popen", return_value=proc), + ): + handle = services.daemonize_services(cfg) + + assert handle.socket_path == socket_path + assert socket_path.read_text(encoding="utf-8") == "stale" + + +def test_write_run_request_writes_complete_payload_when_os_write_is_short( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + real_write = os.write + + def short_write(fd: int, data: bytes) -> int: + return real_write(fd, data[: max(1, len(data) // 2)]) + + monkeypatch.setattr(services.os, "write", short_write) + + request_path = services._write_run_request(cfg) + + expected_payload = json.dumps(cfg.to_child_payload(), indent=2) + "\n" + assert request_path.read_text(encoding="utf-8") == expected_payload + + +def test_service_child_unlinks_request_after_read(tmp_path: Path) -> None: + request_path = tmp_path / "run-request.json" + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + request_path.write_text(json.dumps(cfg.to_child_payload()), encoding="utf-8") + + with patch("nemo_platform_ext.local._service_child.run_services") as run_services: + result = _service_child.main([str(request_path)]) + + assert result == 0 + assert not request_path.exists() + child_cfg = run_services.call_args.args[0] + assert child_cfg.scope == "dev" + assert child_cfg.state_dir == str(tmp_path / "state") + assert child_cfg.runtime_dir == str(tmp_path / "run") + assert run_services.call_args.kwargs == {"_mode": "daemon"} + + +def test_daemonize_services_terminates_child_on_timeout(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + readiness_timeout=0.01, + readiness_poll_interval=0.001, + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services.probe_status", return_value=False), + patch("nemo_platform_ext.local.services.subprocess.Popen", return_value=proc), + ): + with pytest.raises(services.ServicesStartupTimeoutError): + services.daemonize_services(cfg) + + proc.terminate.assert_called_once_with() + proc.kill.assert_not_called() + + +def test_daemonize_services_bounds_probe_and_sleep_by_remaining_deadline( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + readiness_timeout=5.0, + readiness_poll_interval=10.0, + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services.probe_status", return_value=False) as probe_status, + patch("nemo_platform_ext.local.services.subprocess.Popen", return_value=proc), + patch("nemo_platform_ext.local.services.time.monotonic", side_effect=[0.0, 4.0, 4.5, 5.0]), + patch("nemo_platform_ext.local.services.time.sleep") as sleep, + ): + with pytest.raises(services.ServicesStartupTimeoutError): + services.daemonize_services(cfg) + + assert probe_status.call_args.kwargs["timeout"] == pytest.approx(1.0) + sleep.assert_called_once() + assert sleep.call_args.args[0] == pytest.approx(0.5) + proc.terminate.assert_called_once_with() + + +def test_daemonize_services_terminates_child_on_handle_error(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + transport="tcp", + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services._check_tcp_available"), + patch("nemo_platform_ext.local.services.subprocess.Popen", return_value=proc), + patch( + "nemo_platform_ext.local.services.DaemonServiceHandle.from_config", + side_effect=RuntimeError("handle failed"), + ), + ): + with pytest.raises(RuntimeError, match="handle failed"): + services.daemonize_services(cfg) + + proc.terminate.assert_called_once_with() + proc.kill.assert_not_called() + + +def test_daemonize_services_terminates_child_on_probe_error(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + transport="tcp", + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services._check_tcp_available"), + patch("nemo_platform_ext.local.services.probe_status", side_effect=RuntimeError("probe failed")), + patch("nemo_platform_ext.local.services.subprocess.Popen", return_value=proc), + ): + with pytest.raises(RuntimeError, match="probe failed"): + services.daemonize_services(cfg) + + proc.terminate.assert_called_once_with() + proc.kill.assert_not_called() + + +def test_daemonize_services_terminates_child_on_sleep_interruption( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + transport="tcp", + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services._check_tcp_available"), + patch("nemo_platform_ext.local.services.probe_status", return_value=False), + patch("nemo_platform_ext.local.services.subprocess.Popen", return_value=proc), + patch("nemo_platform_ext.local.services.time.sleep", side_effect=KeyboardInterrupt), + ): + with pytest.raises(KeyboardInterrupt): + services.daemonize_services(cfg) + + proc.terminate.assert_called_once_with() + proc.kill.assert_not_called() + + +def test_daemonize_services_kills_child_when_terminate_times_out( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + transport="tcp", + readiness_timeout=0.01, + readiness_poll_interval=0.001, + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + proc.wait.side_effect = [subprocess.TimeoutExpired("nemo services", 5), None] + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services._check_tcp_available"), + patch("nemo_platform_ext.local.services.probe_status", return_value=False), + patch("nemo_platform_ext.local.services.subprocess.Popen", return_value=proc), + ): + with pytest.raises(services.ServicesStartupTimeoutError): + services.daemonize_services(cfg) + + proc.terminate.assert_called_once_with() + proc.kill.assert_called_once_with() + + +async def test_daemonize_services_async_uses_thread(tmp_path: Path) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + handle = MagicMock() + + with patch("nemo_platform_ext.local.services.asyncio.to_thread", new=AsyncMock(return_value=handle)) as to_thread: + result = await services.daemonize_services_async(cfg) + + assert result is handle + to_thread.assert_awaited_once_with(services.daemonize_services, cfg) + + +def test_run_services_serves_embedded_app_with_socket_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + services=["entities"], + controllers=["jobs"], + scope="dev", + port=_free_tcp_port(), + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + ) + handle = _embedded_handle() + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services.start_embedded_services", return_value=handle) as start_embedded, + patch("nemo_platform_ext.local.services.serve_embedded_app") as serve_embedded, + ): + services.run_services(cfg, _mode="daemon") + + start_embedded.assert_called_once_with(cfg, env=None) + serve_embedded.assert_called_once_with(handle.app, cfg, tmp_path / "run" / "dev" / "nemo-platform.sock") + assert not (tmp_path / "state" / "instances" / "dev" / DESCRIPTOR_FILENAME).exists() + + +def test_serve_embedded_app_with_socket_path_listens_on_tcp_and_uds(tmp_path: Path) -> None: + cfg = ServiceRunConfig(transport="tcp", host="127.0.0.1", port=9090) + app = object() + socket_path = tmp_path / "nemo.sock" + + with patch("nmp.platform_runner.server._run_server_on_bound_sockets") as run_bound_sockets: + services.serve_embedded_app(app, cfg, socket_path) + + run_bound_sockets.assert_called_once_with(app, host="127.0.0.1", port=9090, socket_path=str(socket_path)) + + +def test_run_services_cleans_lock_when_log_path_resolution_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", port=_free_tcp_port(), state_dir=tmp_path / "state", runtime_dir=tmp_path / "run" + ) + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch.object(PlatformAppConfig, "log_file_path", side_effect=RuntimeError("boom")), + ): + with pytest.raises(RuntimeError, match="boom"): + services.run_services(cfg) + + assert not services.process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + + +def test_run_services_restores_data_dir_and_lock_when_descriptor_write_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + port=_free_tcp_port(), + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + data_dir=tmp_path / "data", + ) + monkeypatch.delenv("NMP_DATA_DIR", raising=False) + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services.process.write_descriptor", side_effect=RuntimeError("boom")), + ): + with pytest.raises(RuntimeError, match="boom"): + services.run_services(cfg) + + assert "NMP_DATA_DIR" not in os.environ + assert not services.process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + + +def test_run_services_restores_existing_data_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", port=_free_tcp_port(), state_dir=tmp_path / "state", runtime_dir=tmp_path / "run" + ) + monkeypatch.setenv("NMP_DATA_DIR", "/shell/data") + + with ( + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services.start_embedded_services", return_value=_embedded_handle()), + patch("nemo_platform_ext.local.services.serve_embedded_app"), + ): + services.run_services(cfg) + + assert os.environ["NMP_DATA_DIR"] == "/shell/data" + + +def test_daemon_service_handle_tcp_client_uses_tcp_base_url(tmp_path: Path) -> None: + handle = services.DaemonServiceHandle( + scope="dev", + transport="tcp", + socket_path=None, + gateway_base_url=None, + host="0.0.0.0", + port=9090, + pid=123, + mode="daemon", + log_path=None, + state_dir=tmp_path / "state" / "instances" / "dev", + runtime_dir=None, + ) + + with patch("nemo_platform_ext.local.services.NeMoPlatform") as sdk: + handle.client(timeout=12) + + sdk.assert_called_once_with(timeout=12, base_url="http://localhost:9090") + + +def test_daemon_service_handle_uds_client_requires_socket_path(tmp_path: Path) -> None: + handle = services.DaemonServiceHandle( + scope="dev", + transport="uds", + socket_path=None, + gateway_base_url=None, + host="127.0.0.1", + port=8080, + pid=123, + mode="daemon", + log_path=None, + state_dir=tmp_path / "state" / "instances" / "dev", + runtime_dir=tmp_path / "run", + ) + + with pytest.raises(services.ServicesError, match="missing socket_path"): + handle.client() + + +def test_ensure_services_returns_existing_handle(tmp_path: Path) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + handle = MagicMock() + + with ( + patch("nemo_platform_ext.local.services.get_service_handle", return_value=handle), + patch("nemo_platform_ext.local.services.daemonize_services") as daemonize, + ): + result = services.ensure_services(cfg) + + assert result is handle + daemonize.assert_not_called() + + +def test_connect_services_respects_start_if_needed_false(tmp_path: Path) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + + with patch("nemo_platform_ext.local.services.get_service_handle", return_value=None): + with pytest.raises(services.ServicesNotRunningError, match="not running"): + services.connect_services(cfg, start_if_needed=False) + + +def test_stop_services_delegates_to_handle(tmp_path: Path) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + handle = MagicMock() + stop_result = MagicMock() + handle.stop.return_value = stop_result + + with patch("nemo_platform_ext.local.services.get_service_handle", return_value=handle): + result = services.stop_services(cfg, timeout=3.0, force=True) + + assert result is stop_result + handle.stop.assert_called_once_with(timeout=3.0, force=True) + + +def test_get_service_handle_returns_none_without_live_descriptor(tmp_path: Path) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + + with patch("nemo_platform_ext.local.services.process.read_descriptor", return_value=None): + assert services.get_service_handle(cfg) is None + + +def test_list_service_handles_filters_dead_or_descriptorless_instances(tmp_path: Path) -> None: + live_desc = InstanceDescriptor( + pid=123, + transport="tcp", + config=PlatformAppConfig(scope="live", state_root=tmp_path / "state"), + mode="daemon", + ) + infos = [ + MagicMock(descriptor=live_desc, alive=True), + MagicMock(descriptor=None, alive=True), + MagicMock(descriptor=live_desc, alive=False), + ] + + with patch("nemo_platform_ext.local.services.process.list_instances", return_value=infos): + handles = services.list_service_handles(tmp_path / "state") + + assert [handle.scope for handle in handles] == ["live"] + + +def test_get_service_handle_reads_live_descriptor(tmp_path: Path) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + state_dir = tmp_path / "state" / "instances" / "dev" + state_dir.mkdir(parents=True) + desc = InstanceDescriptor( + pid=123, + config=PlatformAppConfig( + scope="dev", + socket_path=str(tmp_path / "run" / "dev" / "nemo-platform.sock"), + state_root=tmp_path / "state", + runtime_root=tmp_path / "run", + ), + transport="uds", + mode="daemon", + ) + (state_dir / DESCRIPTOR_FILENAME).write_text(desc.model_dump_json(), encoding="utf-8") + + with patch("nemo_platform_ext.local.process.is_instance_alive", return_value=True): + handle = services.get_service_handle(cfg) + + assert handle is not None + assert handle.scope == "dev" + assert handle.transport == "uds" diff --git a/packages/nemo_platform_ext/tests/local/test_services_contract.py b/packages/nemo_platform_ext/tests/local/test_services_contract.py new file mode 100644 index 0000000000..acfdc227ca --- /dev/null +++ b/packages/nemo_platform_ext/tests/local/test_services_contract.py @@ -0,0 +1,334 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import threading +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from nemo_platform_ext.local import services +from nemo_platform_ext.local.process import StopResult +from nemo_platform_ext.local.services import ServiceRunConfig +from nmp.platform_runner.config import PlatformAppConfig + + +@dataclass(frozen=True) +class ModeContractCase: + mode: services.ServiceMode + launcher_patch: str + existing_handle_patch_value: object | None + + +@dataclass +class ContractHandle: + mode: services.ServiceMode + calls: list[tuple[str, object]] = field(default_factory=list) + + def is_running(self) -> bool: + self.calls.append(("is_running", None)) + return True + + def wait_until_ready(self, timeout: float | None = None) -> None: + self.calls.append(("wait_until_ready", timeout)) + + async def wait_until_ready_async(self, timeout: float | None = None) -> None: + self.calls.append(("wait_until_ready_async", timeout)) + + def client(self, **kwargs: object) -> tuple[str, services.ServiceMode, dict[str, object]]: + self.calls.append(("client", kwargs)) + return ("client", self.mode, kwargs) + + def async_client(self, **kwargs: object) -> tuple[str, services.ServiceMode, dict[str, object]]: + self.calls.append(("async_client", kwargs)) + return ("async_client", self.mode, kwargs) + + def start_services(self, service_names: list[str] | tuple[str, ...]) -> services.StartServicesResult: + requested = list(service_names) + self.calls.append(("start_services", requested)) + return services.StartServicesResult( + requested=requested, + started=["auth", *requested], + already_active=[], + active=["secrets", "auth", *requested], + ) + + async def start_services_async(self, service_names: list[str] | tuple[str, ...]) -> services.StartServicesResult: + requested = list(service_names) + self.calls.append(("start_services_async", requested)) + return services.StartServicesResult( + requested=requested, + started=["auth", *requested], + already_active=[], + active=["secrets", "auth", *requested], + ) + + def stop(self, *, timeout: float = 30.0, force: bool = False) -> StopResult: + self.calls.append(("stop", {"timeout": timeout, "force": force})) + return StopResult(stopped_pids=[], swept_children=[]) + + async def stop_async(self, *, timeout: float = 30.0, force: bool = False) -> StopResult: + self.calls.append(("stop_async", {"timeout": timeout, "force": force})) + return StopResult(stopped_pids=[], swept_children=[]) + + +MODE_CONTRACT_CASES = [ + ModeContractCase( + mode=services.ServiceMode.EMBEDDED, + launcher_patch="nemo_platform_ext.local.services.start_embedded_services", + existing_handle_patch_value=None, + ), + ModeContractCase( + mode=services.ServiceMode.DAEMON, + launcher_patch="nemo_platform_ext.local.services.daemonize_services", + existing_handle_patch_value=None, + ), +] + + +@pytest.fixture(params=MODE_CONTRACT_CASES, ids=lambda case: case.mode.value) +def mode_case(request: pytest.FixtureRequest) -> ModeContractCase: + return request.param + + +def _config_for(case: ModeContractCase, tmp_path: Path) -> ServiceRunConfig: + return ServiceRunConfig( + mode=case.mode, + services=("secrets",), + scope=f"{case.mode.value}-contract", + state_dir=tmp_path / case.mode.value / "state", + runtime_dir=tmp_path / case.mode.value / "runtime", + ) + + +def test_contract_ensure_services_returns_running_mode_handle( + mode_case: ModeContractCase, + tmp_path: Path, +) -> None: + cfg = _config_for(mode_case, tmp_path) + handle = ContractHandle(mode_case.mode) + + with ( + patch( + "nemo_platform_ext.local.services.get_service_handle", return_value=mode_case.existing_handle_patch_value + ), + patch(mode_case.launcher_patch, return_value=handle), + ): + result = services.ensure_services(cfg) + + assert result is handle + assert result.is_running() is True + assert result.calls == [("is_running", None)] + + +def test_contract_connect_services_returns_client_from_selected_mode( + mode_case: ModeContractCase, + tmp_path: Path, +) -> None: + cfg = _config_for(mode_case, tmp_path) + handle = ContractHandle(mode_case.mode) + + with patch("nemo_platform_ext.local.services.ensure_services", return_value=handle): + client = services.connect_services(cfg, api_key="test-key") + + assert client == ("client", mode_case.mode, {"api_key": "test-key"}) + assert handle.calls == [("client", {"api_key": "test-key"})] + + +@pytest.mark.asyncio +async def test_contract_handle_lifecycle_methods_have_same_semantics( + mode_case: ModeContractCase, +) -> None: + handle = ContractHandle(mode_case.mode) + + handle.wait_until_ready(timeout=1.5) + await handle.wait_until_ready_async(timeout=2.5) + sync_start = handle.start_services(["jobs"]) + async_start = await handle.start_services_async(["jobs"]) + stop_result = handle.stop(timeout=3.0, force=True) + async_stop_result = await handle.stop_async(timeout=4.0, force=False) + + assert sync_start == services.StartServicesResult( + requested=["jobs"], + started=["auth", "jobs"], + already_active=[], + active=["secrets", "auth", "jobs"], + ) + assert async_start == sync_start + assert stop_result == StopResult(stopped_pids=[], swept_children=[]) + assert async_stop_result == StopResult(stopped_pids=[], swept_children=[]) + assert handle.calls == [ + ("wait_until_ready", 1.5), + ("wait_until_ready_async", 2.5), + ("start_services", ["jobs"]), + ("start_services_async", ["jobs"]), + ("stop", {"timeout": 3.0, "force": True}), + ("stop_async", {"timeout": 4.0, "force": False}), + ] + + +def test_contract_real_handles_report_same_staged_start_status_before_staged_start_lands(tmp_path: Path) -> None: + embedded = services.EmbeddedServiceHandle(app=object(), runtime=object()) + daemon = services.DaemonServiceHandle( + scope="daemon-contract", + transport="tcp", + socket_path=None, + gateway_base_url=None, + host="127.0.0.1", + port=8080, + pid=None, + mode="daemon", + log_path=None, + state_dir=tmp_path / "state", + runtime_dir=None, + ) + + for handle in (embedded, daemon): + with pytest.raises(services.ServicesError, match="Staged service start is not implemented"): + handle.start_services(["jobs"]) + + +def test_contract_embedded_and_daemon_child_both_delegate_models_to_platform_builder( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, object]] = [] + + def fake_build_platform_app( + config: PlatformAppConfig | None = None, + *, + env: object = None, + http_client: object = None, + ) -> MagicMock: + calls.append({"config": config, "env": env, "http_client": http_client}) + return MagicMock() + + def service_config(mode: services.ServiceMode) -> ServiceRunConfig: + return ServiceRunConfig( + mode=mode, + services=("models",), + controllers=(), + transport="tcp", + scope="sc-test", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + + with patch("nmp.platform_runner.server.build_platform_app", side_effect=fake_build_platform_app): + services.start_embedded_services(service_config(services.ServiceMode.EMBEDDED)) + + with ( + patch("nmp.platform_runner.server.build_platform_app", side_effect=fake_build_platform_app), + patch("nemo_platform_ext.local.services.require_services_extra"), + patch("nemo_platform_ext.local.services.process.is_instance_alive", return_value=False), + patch("nemo_platform_ext.local.services._check_tcp_available"), + patch("nemo_platform_ext.local.services.process.acquire_lock", return_value=123), + patch("nemo_platform_ext.local.services.process.log_path_for", return_value=tmp_path / "nemo.log"), + patch("nemo_platform_ext.local.services.process.write_descriptor"), + patch("nemo_platform_ext.local.services.process.remove_descriptor"), + patch("nemo_platform_ext.local.services.serve_embedded_app"), + patch("nemo_platform_ext.local.services.os.close"), + ): + services.run_services(service_config(services.ServiceMode.DAEMON), _mode="daemon") + + configs: list[PlatformAppConfig] = [] + for call in calls: + config = call["config"] + assert isinstance(config, PlatformAppConfig) + configs.append(config) + assert [config.services for config in configs] == [("models",), ("models",)] + assert [config.controllers for config in configs] == [(), ()] + assert [config.sidecars for config in configs] == [None, None] + + +def _sidecar_with_events(started: threading.Event, stopped: threading.Event) -> Callable[[threading.Event], None]: + def run(stop_signal: threading.Event) -> None: + started.set() + stop_signal.wait(timeout=5.0) + stopped.set() + + return run + + +def _patch_runner_registry( + monkeypatch: pytest.MonkeyPatch, + *, + sidecar_run_func: Callable[[threading.Event], None], +) -> None: + """Patch the platform runner registry so only a dummy 'models' service + and a test sidecar are available, avoiding real service imports.""" + from nmp.common.config import AuthConfig + from nmp.common.config.base import OIDCConfig + from nmp.common.service import Service + from nmp.platform_runner import config as runner_config + from nmp.platform_runner import registry, server + + class _DummyService(Service): + def __init__(self) -> None: + super().__init__(name="models", module_name="test.contract") + + def get_routers(self): + return [] + + dummy_services: dict[str, Service] = {"models": _DummyService()} + dummy_sidecars: dict[str, Callable] = {"adapters": sidecar_run_func} + + monkeypatch.setattr(runner_config, "get_available_services", lambda: dummy_services) + monkeypatch.setattr(runner_config, "get_available_controllers", lambda: {}) + monkeypatch.setattr( + runner_config, + "get_service_groups", + lambda _available: {"all": ["models"], "core": ["models"], "api": []}, + ) + monkeypatch.setattr(runner_config, "get_controller_groups", lambda _available: {"all": [], "core": []}) + monkeypatch.setattr(runner_config, "get_default_controllers", lambda _groups: []) + monkeypatch.setattr(runner_config, "AVAILABLE_SIDECARS", dummy_sidecars) + monkeypatch.setattr(registry, "AVAILABLE_SIDECARS", dummy_sidecars) + monkeypatch.setattr(server, "AVAILABLE_SIDECARS", dummy_sidecars, raising=False) + monkeypatch.setattr(server, "order_services_by_dependencies", lambda svc: svc) + + auth_cfg = AuthConfig( + enabled=False, + policy_decision_point_base_url="http://localhost:8181", + oidc=OIDCConfig(enabled=False), + ) + monkeypatch.setattr(server, "get_auth_config", lambda: auth_cfg) + monkeypatch.setattr("nmp.common.auth.middleware.get_auth_config", lambda: auth_cfg) + platform_cfg = MagicMock() + platform_cfg.seed_on_startup = False + platform_cfg.redirect_root_to_studio = False + monkeypatch.setattr(server, "get_platform_config", lambda: platform_cfg) + + +def test_embedded_mode_starts_sidecar_thread_via_full_resolution_path( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """End-to-end: start_embedded_services(models) resolves the adapters sidecar + and the sidecar thread actually runs when the app lifespan starts.""" + started = threading.Event() + stopped = threading.Event() + + _patch_runner_registry(monkeypatch, sidecar_run_func=_sidecar_with_events(started, stopped)) + + cfg = ServiceRunConfig( + mode=services.ServiceMode.EMBEDDED, + services=("models",), + controllers=(), + transport="tcp", + scope="sidecar-e2e", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + handle = services.start_embedded_services(cfg, env={}) + + from fastapi.testclient import TestClient + + with TestClient(handle.app) as client: + assert started.wait(timeout=2.0), "sidecar thread did not start" + assert client.get("/").status_code == 200 + + assert stopped.wait(timeout=2.0), "sidecar thread did not stop" diff --git a/packages/nemo_platform_ext/tests/local/test_sidecar_integration.py b/packages/nemo_platform_ext/tests/local/test_sidecar_integration.py new file mode 100644 index 0000000000..33cadabaa7 --- /dev/null +++ b/packages/nemo_platform_ext/tests/local/test_sidecar_integration.py @@ -0,0 +1,231 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for sidecar lifecycle in embedded and daemon modes. + +These tests let the real ``build_platform_app`` → ``resolve_run_configuration`` → +``create_app`` chain run with a lightweight test sidecar registered in the +platform runner registry. They verify that sidecar threads actually start and +stop during the FastAPI app lifespan, covering the full resolution path without +mocking away the core wiring. +""" + +from __future__ import annotations + +import threading +from collections.abc import Callable +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from nemo_platform_ext.local import services +from nemo_platform_ext.local.services import ServiceRunConfig +from nmp.common.config import AuthConfig +from nmp.common.config.base import OIDCConfig +from nmp.common.service import Service +from nmp.platform_runner import config as runner_config +from nmp.platform_runner import registry, server + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class _DummyService(Service): + """Minimal service that registers no routers.""" + + def __init__(self, name: str = "models") -> None: + super().__init__(name=name, module_name="test.sidecar_integration") + + def get_routers(self): + return [] + + +def _sidecar_with_events(started: threading.Event, stopped: threading.Event) -> Callable[[threading.Event], None]: + """Return a sidecar ``run(stop_signal)`` that signals start/stop via events.""" + + def run(stop_signal: threading.Event) -> None: + started.set() + stop_signal.wait(timeout=5.0) + stopped.set() + + return run + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def sidecar_events() -> tuple[threading.Event, threading.Event]: + return threading.Event(), threading.Event() + + +@pytest.fixture +def patched_registry( + monkeypatch: pytest.MonkeyPatch, + sidecar_events: tuple[threading.Event, threading.Event], +) -> tuple[threading.Event, threading.Event]: + """Patch the platform runner registry with a dummy models service and a + test sidecar, plus minimal auth/platform config stubs.""" + started, stopped = sidecar_events + dummy_services: dict[str, Service] = {"models": _DummyService()} + dummy_sidecars: dict[str, Callable] = {"adapters": _sidecar_with_events(started, stopped)} + + monkeypatch.setattr(runner_config, "get_available_services", lambda: dummy_services) + monkeypatch.setattr(runner_config, "get_available_controllers", lambda: {}) + monkeypatch.setattr( + runner_config, + "get_service_groups", + lambda _available: {"all": ["models"], "core": ["models"], "api": []}, + ) + monkeypatch.setattr(runner_config, "get_controller_groups", lambda _available: {"all": [], "core": []}) + monkeypatch.setattr(runner_config, "get_default_controllers", lambda _groups: []) + monkeypatch.setattr(runner_config, "AVAILABLE_SIDECARS", dummy_sidecars) + monkeypatch.setattr(registry, "AVAILABLE_SIDECARS", dummy_sidecars) + monkeypatch.setattr(server, "AVAILABLE_SIDECARS", dummy_sidecars, raising=False) + monkeypatch.setattr(server, "order_services_by_dependencies", lambda svc: svc) + + auth_cfg = AuthConfig( + enabled=False, + policy_decision_point_base_url="http://localhost:8181", + oidc=OIDCConfig(enabled=False), + ) + monkeypatch.setattr(server, "get_auth_config", lambda: auth_cfg) + monkeypatch.setattr("nmp.common.auth.middleware.get_auth_config", lambda: auth_cfg) + platform_cfg = MagicMock() + platform_cfg.seed_on_startup = False + platform_cfg.redirect_root_to_studio = False + monkeypatch.setattr(server, "get_platform_config", lambda: platform_cfg) + + return started, stopped + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_embedded_sidecar_auto_resolved_from_service_dependency( + patched_registry: tuple[threading.Event, threading.Event], + tmp_path: Path, +) -> None: + """start_embedded_services(models) auto-resolves the adapters sidecar via + SERVICE_SIDECAR_DEPENDENCIES and starts it during app lifespan.""" + started, stopped = patched_registry + + cfg = ServiceRunConfig( + mode=services.ServiceMode.EMBEDDED, + services=("models",), + controllers=(), + # sidecars=None triggers auto-resolution + transport="tcp", + scope="integ-embedded-auto", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + handle = services.start_embedded_services(cfg, env={}) + + from fastapi.testclient import TestClient + + with TestClient(handle.app) as client: + assert started.wait(timeout=2.0), "sidecar thread did not start" + assert client.get("/").status_code == 200 + + assert stopped.wait(timeout=2.0), "sidecar thread did not stop after lifespan exit" + + +@pytest.mark.integration +def test_embedded_explicit_sidecar_without_services( + patched_registry: tuple[threading.Event, threading.Event], + tmp_path: Path, +) -> None: + """An explicitly requested sidecar runs even when no services are selected.""" + started, stopped = patched_registry + + cfg = ServiceRunConfig( + mode=services.ServiceMode.EMBEDDED, + services=(), + controllers=(), + sidecars=("adapters",), + transport="tcp", + scope="integ-embedded-explicit", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + handle = services.start_embedded_services(cfg, env={}) + + from fastapi.testclient import TestClient + + with TestClient(handle.app) as client: + assert started.wait(timeout=2.0), "sidecar thread did not start" + assert client.get("/").status_code == 200 + + assert stopped.wait(timeout=2.0), "sidecar thread did not stop after lifespan exit" + + +@pytest.mark.integration +def test_run_services_daemon_mode_starts_sidecar_in_process( + patched_registry: tuple[threading.Event, threading.Event], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """run_services(_mode='daemon') exercises the daemon code path in-process. + It calls start_embedded_services then serve_embedded_app. We intercept + serve_embedded_app to capture the app and exercise its lifespan, proving + the daemon path wires sidecars identically to embedded mode.""" + started, stopped = patched_registry + captured_app = {} + + def fake_serve(app, cfg, socket_path): + captured_app["app"] = app + + monkeypatch.setattr("nemo_platform_ext.local.services.require_services_extra", lambda: None) + monkeypatch.setattr("nemo_platform_ext.local.services.process.is_instance_alive", lambda *a, **kw: False) + monkeypatch.setattr("nemo_platform_ext.local.services._check_tcp_available", lambda *a: None) + monkeypatch.setattr("nemo_platform_ext.local.services.process.acquire_lock", lambda *a, **kw: 123) + monkeypatch.setattr("nemo_platform_ext.local.services.process.log_path_for", lambda *a, **kw: tmp_path / "nemo.log") + monkeypatch.setattr("nemo_platform_ext.local.services.process.write_descriptor", lambda *a, **kw: None) + monkeypatch.setattr("nemo_platform_ext.local.services.process.remove_descriptor", lambda *a, **kw: None) + monkeypatch.setattr("nemo_platform_ext.local.services.serve_embedded_app", fake_serve) + monkeypatch.setattr("nemo_platform_ext.local.services.os.close", lambda fd: None) + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("models",), + controllers=(), + transport="tcp", + scope="integ-daemon", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + services.run_services(cfg, _mode="daemon", env={}) + + assert "app" in captured_app, "serve_embedded_app was not called" + + from fastapi.testclient import TestClient + + with TestClient(captured_app["app"]) as client: + assert started.wait(timeout=2.0), "sidecar thread did not start in daemon mode" + assert client.get("/").status_code == 200 + + assert stopped.wait(timeout=2.0), "sidecar thread did not stop after lifespan exit" + + +@pytest.mark.integration +def test_embedded_rejects_unknown_sidecar_name(tmp_path: Path, patched_registry) -> None: + """Requesting a sidecar not in the registry raises ValueError with a clear message.""" + cfg = ServiceRunConfig( + mode=services.ServiceMode.EMBEDDED, + services=(), + controllers=(), + sidecars=("nonexistent",), + transport="tcp", + scope="integ-unknown", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + with pytest.raises(ValueError, match="Unknown sidecars: nonexistent"): + services.start_embedded_services(cfg, env={}) diff --git a/packages/nemo_platform_ext/tests/local/test_transport.py b/packages/nemo_platform_ext/tests/local/test_transport.py new file mode 100644 index 0000000000..7f22074890 --- /dev/null +++ b/packages/nemo_platform_ext/tests/local/test_transport.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from fastapi import FastAPI +from nemo_platform_ext.local import transport + + +def _assert_timeout_values(timeout: httpx.Timeout, expected: float | None) -> None: + assert timeout.connect == expected + assert timeout.read == expected + assert timeout.write == expected + assert timeout.pool == expected + + +def test_build_sync_http_client_uses_finite_default_timeout(tmp_path) -> None: + client = transport.build_sync_http_client(tmp_path / "nemo.sock") + try: + _assert_timeout_values(client.timeout, 5.0) + finally: + client.close() + + +def test_build_sync_http_client_preserves_explicit_timeout_values(tmp_path) -> None: + no_timeout_client = transport.build_sync_http_client(tmp_path / "nemo.sock", timeout=None) + finite_timeout_client = transport.build_sync_http_client(tmp_path / "nemo.sock", timeout=12.0) + try: + _assert_timeout_values(no_timeout_client.timeout, None) + _assert_timeout_values(finite_timeout_client.timeout, 12.0) + finally: + no_timeout_client.close() + finite_timeout_client.close() + + +@pytest.mark.asyncio +async def test_build_async_http_client_uses_finite_default_timeout(tmp_path) -> None: + client = transport.build_async_http_client(tmp_path / "nemo.sock") + try: + _assert_timeout_values(client.timeout, 5.0) + finally: + await client.aclose() + + +@pytest.mark.asyncio +async def test_build_async_http_client_preserves_explicit_timeout_values(tmp_path) -> None: + no_timeout_client = transport.build_async_http_client(tmp_path / "nemo.sock", timeout=None) + finite_timeout_client = transport.build_async_http_client(tmp_path / "nemo.sock", timeout=12.0) + try: + _assert_timeout_values(no_timeout_client.timeout, None) + _assert_timeout_values(finite_timeout_client.timeout, 12.0) + finally: + await no_timeout_client.aclose() + await finite_timeout_client.aclose() + + +def test_build_sync_asgi_http_client_reaches_app() -> None: + app = FastAPI() + + @app.get("/status") + async def status() -> dict[str, str]: + return {"status": "healthy"} + + client = transport.build_sync_asgi_http_client(app) + try: + response = client.get("http://nemo-platform.local/status") + finally: + client.close() + + assert response.status_code == 200 + assert response.json() == {"status": "healthy"} + + +@pytest.mark.asyncio +async def test_build_async_asgi_http_client_reaches_app() -> None: + app = FastAPI() + + @app.get("/status") + async def status() -> dict[str, str]: + return {"status": "healthy"} + + client = transport.build_async_asgi_http_client(app) + try: + response = await client.get("http://nemo-platform.local/status") + finally: + await client.aclose() + + assert response.status_code == 200 + assert response.json() == {"status": "healthy"} + + +def test_wait_for_status_bounds_probe_and_sleep_by_remaining_deadline() -> None: + with ( + patch("nemo_platform_ext.local.transport.probe_status", return_value=False) as probe_status, + patch("nemo_platform_ext.local.transport.time.monotonic", side_effect=[0.0, 4.0, 4.5, 5.0]), + patch("nemo_platform_ext.local.transport.time.sleep") as sleep, + ): + result = transport.wait_for_status(base_url="http://127.0.0.1:8080", timeout=5.0, poll_interval=10.0) + + assert result is False + assert probe_status.call_args.kwargs["timeout"] == pytest.approx(1.0) + sleep.assert_called_once() + assert sleep.call_args.args[0] == pytest.approx(0.5) + + +@pytest.mark.asyncio +async def test_wait_for_status_async_bounds_probe_and_sleep_by_remaining_deadline() -> None: + with ( + patch( + "nemo_platform_ext.local.transport.probe_status_async", new=AsyncMock(return_value=False) + ) as probe_status, + patch("nemo_platform_ext.local.transport.time.monotonic", side_effect=[0.0, 4.0, 4.5, 5.0]), + patch("nemo_platform_ext.local.transport.asyncio.sleep", new=AsyncMock()) as sleep, + ): + result = await transport.wait_for_status_async( + base_url="http://127.0.0.1:8080", timeout=5.0, poll_interval=10.0 + ) + + assert result is False + assert probe_status.await_args.kwargs["timeout"] == pytest.approx(1.0) + sleep.assert_awaited_once() + assert sleep.await_args.args[0] == pytest.approx(0.5) + + +def test_probe_status_returns_true_for_status_200() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:8080/status" + return httpx.Response(200) + + with patch("nemo_platform_ext.local.transport.httpx.Client") as client_factory: + client = client_factory.return_value + client.get.side_effect = lambda url: handler(httpx.Request("GET", url)) + assert transport.probe_status(base_url="http://127.0.0.1:8080") is True + client.close.assert_called_once_with() + + +def test_probe_status_returns_false_for_request_error() -> None: + with patch("nemo_platform_ext.local.transport.httpx.Client") as client_factory: + client = client_factory.return_value + client.get.side_effect = httpx.ConnectError("boom") + assert transport.probe_status(base_url="http://127.0.0.1:8080") is False + client.close.assert_called_once_with() + + +@pytest.mark.asyncio +async def test_probe_status_async_returns_false_for_request_error() -> None: + with patch("nemo_platform_ext.local.transport.httpx.AsyncClient") as client_factory: + client = client_factory.return_value + client.get = AsyncMock(side_effect=httpx.ConnectError("boom")) + client.aclose = AsyncMock() + assert await transport.probe_status_async(base_url="http://127.0.0.1:8080") is False + client.aclose.assert_awaited_once_with() + + +def test_wait_for_status_returns_true_without_sleep_when_probe_succeeds() -> None: + with ( + patch("nemo_platform_ext.local.transport.probe_status", return_value=True) as probe_mock, + patch("nemo_platform_ext.local.transport.time.sleep") as sleep, + ): + assert transport.wait_for_status(base_url="http://127.0.0.1:8080", timeout=5.0) is True + + probe_mock.assert_called_once() + sleep.assert_not_called() diff --git a/packages/nmp_common/src/nmp/common/auth/client.py b/packages/nmp_common/src/nmp/common/auth/client.py index 4a75948359..67ddb90d17 100644 --- a/packages/nmp_common/src/nmp/common/auth/client.py +++ b/packages/nmp_common/src/nmp/common/auth/client.py @@ -11,6 +11,7 @@ import httpx from nmp.common.config import AuthConfig +from nmp.common.platform_endpoint import parse_platform_endpoint from pydantic import BaseModel, Field from .authz_format import validate_permission_strings, validate_runtime_authorize_scopes @@ -69,6 +70,10 @@ def policy_decision_point_base_url(self) -> Optional[str]: """Policy Decision Point (PDP) base URL for permission checks.""" return self.config.policy_decision_point_base_url + def _new_pdp_http_client(self) -> httpx.AsyncClient: + endpoint = parse_platform_endpoint(self.config.policy_decision_point_base_url) + return endpoint.async_http_client(timeout=self.config.policy_decision_point_request_timeout_seconds) + @property def _pdp_request_headers(self) -> dict[str, str]: """Headers sent with every PDP HTTP request. @@ -154,9 +159,7 @@ async def authorize_request( if client: response = await client.post(auth_url, json={"input": auth_input}, headers=pdp_headers) else: - async with httpx.AsyncClient( - timeout=self.config.policy_decision_point_request_timeout_seconds - ) as temp_client: + async with self._new_pdp_http_client() as temp_client: response = await temp_client.post(auth_url, json={"input": auth_input}, headers=pdp_headers) response.raise_for_status() @@ -222,7 +225,7 @@ async def create_model( client = self.http_client should_close = False if client is None: - client = httpx.AsyncClient(timeout=self.config.policy_decision_point_request_timeout_seconds) + client = self._new_pdp_http_client() should_close = True try: @@ -401,6 +404,9 @@ async def wait_role( if not self.auth_enabled: return True + if not self.policy_decision_point_base_url: + raise RuntimeError("Policy Decision Point URL not configured for role checks") + if poll_interval is None: poll_interval = self.config.propagation_poll_interval_seconds start_time = asyncio.get_event_loop().time() @@ -408,11 +414,7 @@ async def wait_role( # Use provided http_client, instance http_client (from middleware), or create a new one # See architecture/docs/http-client-injection.md for injection patterns. - client = ( - http_client - or self.http_client - or httpx.AsyncClient(timeout=self.config.policy_decision_point_request_timeout_seconds) - ) + client = http_client or self.http_client or self._new_pdp_http_client() should_close = http_client is None and self.http_client is None try: diff --git a/packages/nmp_common/src/nmp/common/auth/middleware.py b/packages/nmp_common/src/nmp/common/auth/middleware.py index ed4bc11b38..348b6477b8 100644 --- a/packages/nmp_common/src/nmp/common/auth/middleware.py +++ b/packages/nmp_common/src/nmp/common/auth/middleware.py @@ -10,6 +10,7 @@ from fastapi import Request, Response from nmp.common.config import AuthConfig, get_auth_config from nmp.common.observability.context import get_app_ctx +from nmp.common.platform_endpoint import parse_platform_endpoint from starlette.middleware.base import BaseHTTPMiddleware from starlette.responses import JSONResponse from starlette.types import ASGIApp @@ -55,8 +56,8 @@ def _embedded_pdp_base_url_hint(config: AuthConfig) -> str: return "" base = (config.policy_decision_point_base_url or "").strip() return ( - " For embedded PDP, auth.policy_decision_point_base_url must be the HTTP origin where " - "this process serves /apis/auth (same as platform base_url / NMP_BASE_URL). " + " For embedded PDP, auth.policy_decision_point_base_url must be the typed endpoint where " + "this process serves /apis/auth (same as platform base_url / NMP_BASE_URL; HTTP(S) or unix://). " f"Absolute PDP URLs ignore the injected ASGI client base_url. Current auth.policy_decision_point_base_url={base!r}." ) @@ -166,7 +167,8 @@ def _get_client(self, request: Request) -> httpx.AsyncClient: An async HTTP client configured with auth.policy_decision_point_request_timeout_seconds """ if self._client is None: - self._client = httpx.AsyncClient(timeout=self.config.policy_decision_point_request_timeout_seconds) + endpoint = parse_platform_endpoint(self.config.policy_decision_point_base_url) + self._client = endpoint.async_http_client(timeout=self.config.policy_decision_point_request_timeout_seconds) return self._client def _update_auth_context(self, principal: Principal) -> None: diff --git a/packages/nmp_common/src/nmp/common/config/base.py b/packages/nmp_common/src/nmp/common/config/base.py index 1bb654c6da..badb052cf1 100644 --- a/packages/nmp_common/src/nmp/common/config/base.py +++ b/packages/nmp_common/src/nmp/common/config/base.py @@ -317,9 +317,14 @@ class AuthConfig(create_service_config_class("auth")): # ty: ignore[unsupported ) def get_pdp_url(self, entrypoint: str) -> str: + # Import lazily to avoid a module cycle: platform_endpoint imports + # PlatformConfig from nmp.common.config, which is defined in this file. + from nmp.common.platform_endpoint import parse_platform_endpoint + + endpoint = parse_platform_endpoint(self.policy_decision_point_base_url) if self.policy_decision_point_provider == "opa": - return f"{self.policy_decision_point_base_url}/v1/data/authz/{entrypoint}" - return f"{self.policy_decision_point_base_url}/apis/auth/v2/authz/{entrypoint}" + return f"{endpoint.connect_base_url}/v1/data/authz/{entrypoint}" + return f"{endpoint.connect_base_url}/apis/auth/v2/authz/{entrypoint}" @property def auth_url(self) -> str: diff --git a/packages/nmp_common/src/nmp/common/platform_endpoint.py b/packages/nmp_common/src/nmp/common/platform_endpoint.py new file mode 100644 index 0000000000..c03cb121ef --- /dev/null +++ b/packages/nmp_common/src/nmp/common/platform_endpoint.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Typed platform endpoint resolution for HTTP(S) and Unix domain sockets.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +import httpx +from httpx._types import TimeoutTypes +from nmp.common.config import PlatformConfig + +UDS_BASE_URL = "http://nemo-platform.local" + + +@dataclass(frozen=True) +class PlatformEndpoint: + connect_base_url: str + socket_path: Path | None + transport: Literal["tcp", "uds"] + + def sync_http_client(self, *, timeout: TimeoutTypes | None = None) -> httpx.Client: + if self.transport == "uds": + if self.socket_path is None: + raise ValueError("UDS endpoint is missing a socket path") + transport = httpx.HTTPTransport(uds=str(self.socket_path)) + if timeout is None: + return httpx.Client(transport=transport, follow_redirects=True) + return httpx.Client(transport=transport, follow_redirects=True, timeout=timeout) + if timeout is None: + return httpx.Client(follow_redirects=True) + return httpx.Client(follow_redirects=True, timeout=timeout) + + def async_http_client(self, *, timeout: TimeoutTypes | None = None) -> httpx.AsyncClient: + if self.transport == "uds": + if self.socket_path is None: + raise ValueError("UDS endpoint is missing a socket path") + transport = httpx.AsyncHTTPTransport(uds=str(self.socket_path)) + if timeout is None: + return httpx.AsyncClient(transport=transport, follow_redirects=True) + return httpx.AsyncClient(transport=transport, follow_redirects=True, timeout=timeout) + if timeout is None: + return httpx.AsyncClient(follow_redirects=True) + return httpx.AsyncClient(follow_redirects=True, timeout=timeout) + + +def resolve_platform_endpoint(platform_config: PlatformConfig | None = None) -> PlatformEndpoint: + """Resolve the default platform endpoint from ``NMP_BASE_URL`` / config.""" + + if platform_config is None: + from nmp.common.config import Configuration + + platform_config = Configuration.get_platform_config() + return parse_platform_endpoint(platform_config.base_url) + + +def resolve_service_endpoint(service_name: str, platform_config: PlatformConfig | None = None) -> PlatformEndpoint: + """Resolve a service endpoint using ``NMP__URL`` before ``NMP_BASE_URL``.""" + + if platform_config is None: + from nmp.common.config import Configuration + + platform_config = Configuration.get_platform_config() + env_name = f"NMP_{service_name.upper().replace('-', '_')}_URL" + endpoint = os.environ.get(env_name) or platform_config.get_service_url(service_name) + return parse_platform_endpoint(endpoint) + + +def parse_platform_endpoint(endpoint: str) -> PlatformEndpoint: + """Parse an HTTP(S) or ``unix://`` endpoint into a typed transport model.""" + + if endpoint.startswith(("http://", "https://")): + try: + parsed = httpx.URL(endpoint) + except httpx.InvalidURL as error: + raise ValueError(f"Invalid platform endpoint URL {endpoint!r}") from error + if not parsed.host: + raise ValueError(f"HTTP(S) platform endpoint must include a host, got {endpoint!r}") + return PlatformEndpoint(connect_base_url=endpoint.rstrip("/"), socket_path=None, transport="tcp") + if endpoint.startswith("unix://"): + socket_path = _parse_unix_socket_path(endpoint) + return PlatformEndpoint(connect_base_url=UDS_BASE_URL, socket_path=socket_path, transport="uds") + if endpoint.startswith("/"): + raise ValueError(f"Raw socket paths are not valid endpoint URLs; use unix://{endpoint}") + raise ValueError(f"Unsupported platform endpoint URL {endpoint!r}; expected http://, https://, or unix://") + + +def _parse_unix_socket_path(endpoint: str) -> Path: + raw_path = endpoint.removeprefix("unix://") + if not raw_path.startswith("/"): + raise ValueError(f"UDS endpoint must use an absolute socket path, got {endpoint!r}") + return Path(raw_path) diff --git a/packages/nmp_common/src/nmp/common/sdk_factory.py b/packages/nmp_common/src/nmp/common/sdk_factory.py index 37076d3f82..a34a3daa94 100644 --- a/packages/nmp_common/src/nmp/common/sdk_factory.py +++ b/packages/nmp_common/src/nmp/common/sdk_factory.py @@ -14,6 +14,7 @@ from nmp.common.http_clients import shared_async_http_client, shared_sync_http_client from nmp.common.observability import MARK_INTERNAL_REQUEST_HEADERS from nmp.common.observability.otel import get_otel_headers +from nmp.common.platform_endpoint import PlatformEndpoint, resolve_platform_endpoint, resolve_service_endpoint logger = logging.getLogger(__name__) PlatformSDKT = TypeVar("PlatformSDKT", NeMoPlatform, AsyncNeMoPlatform) @@ -61,7 +62,18 @@ def resolve_platform_request_url( return request_url api_name = match.group(1) - service_url = httpx.URL(platform_config.get_service_url(api_name)) + svc_endpoint = resolve_service_endpoint(api_name, platform_config) + if svc_endpoint.transport == "uds": + logger.debug( + "Routing URL to UDS service", + extra={ + "service": api_name, + "path": request_url.path, + "transport": svc_endpoint.transport, + }, + ) + return request_url.copy_with(scheme="http", host="nemo-platform.local", port=None) + service_url = httpx.URL(svc_endpoint.connect_base_url) routed_url = request_url.copy_with( scheme=service_url.scheme, host=service_url.host, @@ -120,6 +132,30 @@ def with_options_preserving_request_router(base_sdk: PlatformSDKT, **kwargs: Any return scoped_sdk +def _sync_http_client_for_endpoint( + endpoint: PlatformEndpoint, + http_client: httpx.Client | None, +) -> httpx.Client: + if http_client is not None: + return http_client + if endpoint.transport == "uds": + return endpoint.sync_http_client() + return shared_sync_http_client() + + +def _async_http_client_for_endpoint( + endpoint: PlatformEndpoint, + http_client: httpx.AsyncClient | None, +) -> httpx.AsyncClient: + if http_client is not None: + return http_client + if _test_http_client is not None: + return _test_http_client + if endpoint.transport == "uds": + return endpoint.async_http_client() + return shared_async_http_client() + + def _get_default_headers( as_service: str | None = None, internal: bool = False, on_behalf_of: str | Principal | None = None ) -> dict[str, str]: @@ -206,9 +242,10 @@ def get_platform_sdk( Configured NeMoPlatform SDK instance. """ headers = _get_default_headers(as_service, internal, on_behalf_of) + endpoint = resolve_platform_endpoint() sdk = NeMoPlatform( - base_url=base_url or _base_url_from_config(), - http_client=http_client or shared_sync_http_client(), + base_url=base_url or endpoint.connect_base_url, + http_client=_sync_http_client_for_endpoint(endpoint, http_client), default_headers=headers if headers else None, ) return attach_platform_request_router(sdk) @@ -295,13 +332,14 @@ def get_async_platform_sdk( Configured AsyncNeMoPlatform SDK instance. """ headers = _get_default_headers(as_service, internal, on_behalf_of) + endpoint = resolve_platform_endpoint() # Use explicitly provided http_client (from DependencyProvider) or fall back to # module-level _test_http_client for backward compatibility with direct callers. - effective_client = http_client or _test_http_client or shared_async_http_client() + effective_client = _async_http_client_for_endpoint(endpoint, http_client) sdk = AsyncNeMoPlatform( - base_url=base_url or _base_url_from_config(), + base_url=base_url or endpoint.connect_base_url, http_client=effective_client, default_headers=headers if headers else None, ) diff --git a/packages/nmp_common/src/nmp/common/service/api/health.py b/packages/nmp_common/src/nmp/common/service/api/health.py index 9d5714096e..95722c2902 100644 --- a/packages/nmp_common/src/nmp/common/service/api/health.py +++ b/packages/nmp_common/src/nmp/common/service/api/health.py @@ -6,15 +6,58 @@ import logging import threading import time +from collections.abc import Mapping from typing import Any import httpx from nmp.common.config import PlatformConfig from nmp.common.observability import MARK_INTERNAL_REQUEST_HEADERS +from nmp.common.platform_endpoint import resolve_service_endpoint logger = logging.getLogger(__name__) +def _status_names(values: object) -> set[str]: + if not isinstance(values, list): + return set() + + names: set[str] = set() + for value in values: + if isinstance(value, Mapping): + name = value.get("name") + else: + name = getattr(value, "name", value) + if isinstance(name, str): + names.add(name) + return names + + +def service_ready_state_from_status(data: object, service_name: str) -> bool | None: + """Return service readiness from a platform /status payload. + + ``True`` means the service is ready or absent from this platform deployment. + ``False`` means the service is explicitly present but not ready. + ``None`` means the payload shape is unusable and should be retried. + """ + if not isinstance(data, Mapping): + return None + + services = data.get("services") or {} + if not isinstance(services, Mapping): + return None + + ready = _status_names(services.get("ready") or []) + if service_name in ready: + return True + + not_ready = _status_names(services.get("not_ready") or []) + if service_name in not_ready: + return False + + # Service is absent from this deployment — treat as ready so callers don't block. + return True + + async def async_wait_for_service_ready( platform_config: PlatformConfig, service_name: str, @@ -40,10 +83,11 @@ async def async_wait_for_service_ready( """ import asyncio - status_url = f"{platform_config.get_service_url(service_name).rstrip('/')}/status" + endpoint = resolve_service_endpoint(service_name, platform_config) + status_url = f"{endpoint.connect_base_url.rstrip('/')}/status" own_client = http_client is None if http_client is None: - http_client = httpx.AsyncClient(timeout=2.0) + http_client = endpoint.async_http_client(timeout=2.0) logger.debug("Waiting for service to be ready", extra={"service": service_name, "url": status_url}) @@ -56,9 +100,9 @@ async def async_wait_for_service_ready( headers=MARK_INTERNAL_REQUEST_HEADERS, ) if response.status_code == 200: - data: dict[str, Any] = response.json() - ready = (data.get("services") or {}).get("ready") or [] - if service_name in ready: + data: Any = response.json() + ready = service_ready_state_from_status(data, service_name) + if ready is True: logger.info("Service is ready", extra={"service": service_name}) return True except (httpx.RequestError, ValueError) as e: @@ -97,23 +141,16 @@ async def async_wait_for_dependencies( Returns: True if all dependencies became ready, False if any timed out. """ - own_client = http_client is None - if own_client: - http_client = httpx.AsyncClient(timeout=2.0) - try: - for dep in dependency_names: - if not await async_wait_for_service_ready( - platform_config, - dep, - timeout=timeout_per_service, - poll_interval=poll_interval, - http_client=http_client, - ): - return False - return True - finally: - if own_client and http_client is not None: - await http_client.aclose() + for dep in dependency_names: + if not await async_wait_for_service_ready( + platform_config, + dep, + timeout=timeout_per_service, + poll_interval=poll_interval, + http_client=http_client, + ): + return False + return True def wait_for_service_ready( @@ -141,31 +178,36 @@ def wait_for_service_ready( True if the service became ready, False if timeout or stop signal. """ start_time = time.time() - status_url = f"{platform_config.get_service_url(service_name).rstrip('/')}/status" + endpoint = resolve_service_endpoint(service_name, platform_config) + status_url = f"{endpoint.connect_base_url.rstrip('/')}/status" + http_client = endpoint.sync_http_client(timeout=2.0) logger.info( "Waiting for service to be ready", extra={"service": service_name, "url": status_url}, ) - while not stop_signal.is_set() and (time.time() - start_time) < timeout: - try: - response = httpx.get(status_url, timeout=2.0, headers=MARK_INTERNAL_REQUEST_HEADERS) - if response.status_code == 200: - data: dict[str, Any] = response.json() - ready = (data.get("services") or {}).get("ready") or [] - if service_name in ready: - logger.debug( - "Service is ready", - extra={"service": service_name, "url": status_url}, - ) - return True - except (httpx.RequestError, ValueError) as e: - logger.debug( - "Status check failed, will retry", - extra={"service": service_name, "url": status_url, "error": str(e)}, - ) - time.sleep(poll_interval) + try: + while not stop_signal.is_set() and (time.time() - start_time) < timeout: + try: + response = http_client.get(status_url, headers=MARK_INTERNAL_REQUEST_HEADERS) + if response.status_code == 200: + data: Any = response.json() + ready = service_ready_state_from_status(data, service_name) + if ready is True: + logger.debug( + "Service is ready", + extra={"service": service_name, "url": status_url}, + ) + return True + except (httpx.RequestError, ValueError) as e: + logger.debug( + "Status check failed, will retry", + extra={"service": service_name, "url": status_url, "error": str(e)}, + ) + time.sleep(poll_interval) + finally: + http_client.close() if stop_signal.is_set(): logger.debug("Stop signal received while waiting for service") diff --git a/packages/nmp_common/src/nmp/common/service/base.py b/packages/nmp_common/src/nmp/common/service/base.py index f1d59d573c..8a1c076fc3 100644 --- a/packages/nmp_common/src/nmp/common/service/base.py +++ b/packages/nmp_common/src/nmp/common/service/base.py @@ -20,6 +20,7 @@ from nmp.common.config import Configuration, PlatformConfig, ServiceConfig from nmp.common.controller import Controller from nmp.common.entities.client import EntityClient +from nmp.common.platform_endpoint import resolve_service_endpoint logger = logging.getLogger(__name__) @@ -545,39 +546,37 @@ async def startup(self) -> None: import time from nmp.common.observability import MARK_INTERNAL_REQUEST_HEADERS + from nmp.common.service.api.health import service_ready_state_from_status - status_url = f"{self.platform_config.get_service_url(service_name).rstrip('/')}/status" - client = self._dependency_provider.get_http_client() + endpoint = resolve_service_endpoint(service_name, self.platform_config) + status_url = f"{endpoint.connect_base_url.rstrip('/')}/status" + own_client = endpoint.transport == "uds" + client = endpoint.async_http_client(timeout=2.0) if own_client else self._dependency_provider.get_http_client() logger.debug("Waiting for service to be ready", extra={"service": service_name, "url": status_url}) start_time = time.time() - while (time.time() - start_time) < timeout: - try: - response = await client.get(status_url, timeout=2.0, headers=MARK_INTERNAL_REQUEST_HEADERS) - if response.status_code == 200: - data = response.json() - services = data.get("services") or {} - ready = services.get("ready") or [] - if service_name in ready: - logger.debug("Service is ready", extra={"service": service_name}) - return True - # If the service isn't in any list (ready/not_ready), it's not - # part of this deployment — skip waiting rather than timing out. - not_ready = services.get("not_ready") or [] - not_ready_names = [ - n.get("name", n) if isinstance(n, dict) else getattr(n, "name", n) for n in not_ready - ] - if service_name not in ready and service_name not in not_ready_names: - logger.debug( - "Dependency not present in platform, skipping wait", - extra={"service": service_name}, - ) - return True - # Service is in not_ready; keep polling - except httpx.RequestError: - pass - await asyncio.sleep(poll_interval) + try: + while (time.time() - start_time) < timeout: + try: + response = await client.get(status_url, timeout=2.0, headers=MARK_INTERNAL_REQUEST_HEADERS) + if response.status_code == 200: + try: + data = response.json() + except ValueError: + data = None + ready = service_ready_state_from_status(data, service_name) + if ready is True: + logger.debug("Service is ready", extra={"service": service_name}) + return True + # ``False`` means the service is explicitly not_ready; keep polling. + # ``None`` means the status payload shape was unusable; retry. + except httpx.RequestError: + pass + await asyncio.sleep(poll_interval) + finally: + if own_client: + await client.aclose() logger.warning("Timeout waiting for service to be ready", extra={"service": service_name, "timeout": timeout}) return False diff --git a/packages/nmp_common/tests/auth/test_client.py b/packages/nmp_common/tests/auth/test_client.py index 724c5fd9fc..1ac5e82075 100644 --- a/packages/nmp_common/tests/auth/test_client.py +++ b/packages/nmp_common/tests/auth/test_client.py @@ -228,6 +228,48 @@ async def test_authorize_request_sends_delegate_claims(self, auth_config, princi assert body["on_behalf_of_principal_id"] == "user@example.com" +class TestWaitRole: + @pytest.mark.asyncio + async def test_wait_role_requires_pdp_url_before_creating_client(self, principal): + auth_config = AuthConfig(enabled=True, policy_decision_point_base_url="") + auth_client = AuthClient(principal=principal, config=auth_config) + + with patch.object(auth_client, "_new_pdp_http_client") as new_client: + with pytest.raises(RuntimeError, match="Policy Decision Point URL not configured"): + await auth_client.wait_role( + "user@example.com", + "test-workspace", + "Viewer", + timeout=0.01, + poll_interval=0.01, + ) + + new_client.assert_not_called() + + @pytest.mark.asyncio + async def test_wait_role_reuses_provided_http_client(self, auth_config, principal): + mock_http_client = httpx.AsyncClient() + mock_response = MagicMock() + mock_response.json.return_value = {"result": {"has_role": True}} + mock_response.raise_for_status = MagicMock() + + with ( + patch.object(mock_http_client, "post", new_callable=AsyncMock, return_value=mock_response), + patch.object(mock_http_client, "aclose", new_callable=AsyncMock) as close, + ): + auth_client = AuthClient(principal=principal, config=auth_config) + assert await auth_client.wait_role( + "user@example.com", + "test-workspace", + "Viewer", + timeout=0.01, + poll_interval=0.01, + http_client=mock_http_client, + ) + + close.assert_not_called() + + class TestOnBehalfOfHasPermissions: """Tests for the on_behalf_of_has_permissions method.""" diff --git a/packages/nmp_common/tests/nmp_common/test_common_config.py b/packages/nmp_common/tests/nmp_common/test_common_config.py index a0bcac90c5..047a419827 100644 --- a/packages/nmp_common/tests/nmp_common/test_common_config.py +++ b/packages/nmp_common/tests/nmp_common/test_common_config.py @@ -5,6 +5,7 @@ import pytest from nmp.common.config import ( + AuthConfig, CommonServiceConfig, Configuration, DatabaseConfig, @@ -12,6 +13,7 @@ get_common_service_config, get_platform_config, ) +from nmp.common.platform_endpoint import UDS_BASE_URL class TestPlatformConfig: @@ -174,6 +176,29 @@ def test_create_service_pattern(self): assert match.group(1) == "my" +class TestAuthConfig: + """Tests for shared auth configuration helpers.""" + + def test_get_pdp_url_for_embedded_provider_normalizes_base_url(self): + config = AuthConfig(policy_decision_point_base_url="http://localhost:8080/") + + assert config.get_pdp_url("allow") == "http://localhost:8080/apis/auth/v2/authz/allow" + assert config.auth_url == "http://localhost:8080/apis/auth/v2/authz/allow" + + def test_get_pdp_url_for_opa_provider_normalizes_base_url(self): + config = AuthConfig( + policy_decision_point_base_url="http://opa:8181/", + policy_decision_point_provider="opa", + ) + + assert config.get_pdp_url("has_permissions") == "http://opa:8181/v1/data/authz/has_permissions" + + def test_get_pdp_url_for_uds_endpoint_uses_connect_base_url(self): + config = AuthConfig(policy_decision_point_base_url="unix:///tmp/nemo-platform.sock") + + assert config.get_pdp_url("allow") == f"{UDS_BASE_URL}/apis/auth/v2/authz/allow" + + class TestCommonServiceConfig: """Tests for CommonServiceConfig.""" diff --git a/packages/nmp_common/tests/nmp_common/test_common_service.py b/packages/nmp_common/tests/nmp_common/test_common_service.py index 7d29d156e3..45d1bcce6f 100644 --- a/packages/nmp_common/tests/nmp_common/test_common_service.py +++ b/packages/nmp_common/tests/nmp_common/test_common_service.py @@ -3,10 +3,16 @@ """Tests for nmp.common.service module.""" +import asyncio +import threading from typing import List +from unittest.mock import AsyncMock, patch +import httpx import pytest from fastapi import APIRouter, FastAPI +from fastapi.testclient import TestClient +from nmp.common.config import PlatformConfig from nmp.common.service import DependencyProvider, RouterConfig, Service @@ -27,8 +33,8 @@ def _route_paths(app: FastAPI) -> set[str]: class MockService(Service): """Mock implementation of Service for testing.""" - def __init__(self): - super().__init__(name="test-service", module_name="nmp.test") + def __init__(self, dependency_provider: DependencyProvider | None = None): + super().__init__(name="test-service", module_name="nmp.test", dependency_provider=dependency_provider) def get_routers(self) -> List[RouterConfig]: router = APIRouter() @@ -158,6 +164,51 @@ async def test_service_is_ready_default(self): service = MockService() assert await service.is_ready() is True + @pytest.mark.asyncio + async def test_wait_for_service_ready_retries_malformed_status_payloads(self): + """Test malformed 200 /status payloads are retried.""" + requests: list[httpx.Request] = [] + responses = [ + httpx.Response(status_code=200, content=b"{"), + httpx.Response(status_code=200, json=["not", "a", "mapping"]), + httpx.Response(status_code=200, json={"services": {"ready": ["entities"]}}), + ] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return responses.pop(0) + + provider = DependencyProvider() + provider._platform_config = PlatformConfig(base_url="http://platform.local") + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + provider._http_client = client + service = MockService(dependency_provider=provider) + + ready = await service.wait_for_service_ready("entities", timeout=1.0, poll_interval=0) + + assert ready is True + assert len(requests) == 3 + + @pytest.mark.asyncio + async def test_wait_for_service_ready_skips_service_absent_from_status(self): + """Test dependencies absent from /status are treated as not part of this deployment.""" + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json={"services": {"ready": ["entities"], "not_ready": []}}) + + provider = DependencyProvider() + provider._platform_config = PlatformConfig(base_url="http://platform.local") + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + provider._http_client = client + service = MockService(dependency_provider=provider) + + ready = await service.wait_for_service_ready("models", timeout=1.0, poll_interval=0) + + assert ready is True + assert len(requests) == 1 + class TestDependencyProvider: """Tests for DependencyProvider class.""" @@ -183,3 +234,46 @@ def test_service_has_provider(self): service = MockService() assert service.dependency_provider is not None assert isinstance(service.dependency_provider, DependencyProvider) + + +class LifecycleService(MockService): + def __init__(self): + super().__init__() + self.events: list[str] = [] + self.started = threading.Event() + + async def on_startup(self) -> None: + self.events.append("on_startup") + + async def startup(self) -> None: + self.events.append("startup") + self.started.set() + await asyncio.Event().wait() + + async def on_shutdown(self) -> None: + self.events.append("on_shutdown") + await super().on_shutdown() + + +def test_service_lifespan_runs_startup_task_and_shutdown_cleanup() -> None: + service = LifecycleService() + + with TestClient(service.app) as client: + assert service.started.wait(timeout=1.0) + assert client.get("/test").json() == {"message": "test"} + + assert service.events == ["on_startup", "startup", "on_shutdown"] + assert service._startup_background_tasks + assert service._startup_background_tasks[0].cancelled() + + +@pytest.mark.asyncio +async def test_wait_for_dependencies_returns_false_when_dependency_times_out() -> None: + service = MockService() + service._dependencies = ["entities", "auth"] + + with patch.object(service, "wait_for_service_ready", new=AsyncMock(side_effect=[True, False])) as wait: + ready = await service._wait_for_dependencies(timeout=0.01) + + assert ready is False + assert [call.args[0] for call in wait.await_args_list] == ["entities", "auth"] diff --git a/packages/nmp_common/tests/nmp_common/test_dependency_provider.py b/packages/nmp_common/tests/nmp_common/test_dependency_provider.py new file mode 100644 index 0000000000..efffb0e41a --- /dev/null +++ b/packages/nmp_common/tests/nmp_common/test_dependency_provider.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import FastAPI +from nmp.common.service import DependencyProvider +from nmp.common.service.dependencies import get_entity_client, get_platform_config, get_sdk_client + + +def test_get_http_client_caches_default_client() -> None: + provider = DependencyProvider() + client = MagicMock() + + with patch("nmp.common.service.base.DefaultAsyncHttpxClient", return_value=client) as factory: + first = provider.get_http_client() + second = provider.get_http_client() + + assert first is client + assert second is client + factory.assert_called_once_with() + + +def test_get_sdk_client_caches_request_sdk_and_creates_fresh_service_sdk() -> None: + provider = DependencyProvider() + request_sdk = MagicMock(name="request_sdk") + service_sdk = MagicMock(name="service_sdk") + + with patch("nmp.common.sdk_factory.get_async_platform_sdk", side_effect=[request_sdk, service_sdk]) as factory: + assert provider.get_sdk_client() is request_sdk + assert provider.get_sdk_client() is request_sdk + assert provider.get_sdk_client(as_service="jobs") is service_sdk + + assert factory.call_args_list[0].kwargs == {"http_client": None} + assert factory.call_args_list[1].kwargs == { + "as_service": "jobs", + "internal": True, + "http_client": None, + } + + +def test_setup_dependencies_registers_fastapi_overrides() -> None: + provider = DependencyProvider() + app = FastAPI() + service = MagicMock() + service._service_config = None + + provider.setup_dependencies(app, service) + + assert app.dependency_overrides[get_sdk_client] == provider.get_request_scoped_sdk + assert app.dependency_overrides[get_entity_client] == provider.get_entity_client + assert app.dependency_overrides[get_platform_config] == provider.get_platform_config + + +@pytest.mark.asyncio +async def test_close_closes_managed_clients_and_clears_references() -> None: + provider = DependencyProvider() + http_client = MagicMock() + http_client.aclose = AsyncMock() + sdk = MagicMock() + sdk.close = AsyncMock() + provider._http_client = http_client + provider._sdk_client = sdk + + await provider.close() + + http_client.aclose.assert_awaited_once_with() + sdk.close.assert_awaited_once_with() + assert provider._http_client is None + assert provider._sdk_client is None + + +def test_get_entity_client_as_service_uses_fresh_service_sdk() -> None: + provider = DependencyProvider() + sdk = MagicMock(name="service_sdk") + entities_api = MagicMock(name="entities_api") + entity_client = MagicMock(name="entity_client") + + with ( + patch.object(provider, "get_sdk_client", return_value=sdk) as get_sdk, + patch("nemo_platform.resources.entities.AsyncEntitiesResource", return_value=entities_api) as resource, + patch("nmp.common.entities.client.EntityClient", return_value=entity_client) as client_factory, + ): + result = provider.get_entity_client(as_service="models") + + assert result is entity_client + get_sdk.assert_called_once_with(as_service="models") + resource.assert_called_once_with(sdk) + client_factory.assert_called_once_with(entities_api) diff --git a/packages/nmp_common/tests/nmp_common/test_service_health.py b/packages/nmp_common/tests/nmp_common/test_service_health.py new file mode 100644 index 0000000000..773979ec7c --- /dev/null +++ b/packages/nmp_common/tests/nmp_common/test_service_health.py @@ -0,0 +1,156 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import threading +from unittest.mock import MagicMock, patch + +import httpx +import pytest +from nmp.common.config import PlatformConfig +from nmp.common.observability import MARK_INTERNAL_REQUEST_HEADERS +from nmp.common.service.api.health import ( + async_wait_for_dependencies, + async_wait_for_service_ready, + wait_for_service_ready, +) + + +@pytest.mark.asyncio +async def test_async_wait_for_service_ready_skips_service_absent_from_status() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json={"services": {"ready": ["entities"], "not_ready": []}}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + ready = await async_wait_for_service_ready( + PlatformConfig(base_url="http://platform.local"), + "models", + timeout=1.0, + poll_interval=0, + http_client=client, + ) + + assert ready is True + assert len(requests) == 1 + assert str(requests[0].url) == "http://platform.local/status" + for key, value in MARK_INTERNAL_REQUEST_HEADERS.items(): + assert requests[0].headers[key] == value + + +@pytest.mark.asyncio +async def test_async_wait_for_service_ready_waits_for_explicitly_not_ready_service() -> None: + responses = [ + httpx.Response(200, json={"services": {"ready": ["entities"], "not_ready": [{"name": "models"}]}}), + httpx.Response(200, json={"services": {"ready": ["entities", "models"], "not_ready": []}}), + ] + + def handler(_request: httpx.Request) -> httpx.Response: + return responses.pop(0) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + ready = await async_wait_for_service_ready( + PlatformConfig(base_url="http://platform.local"), + "models", + timeout=1.0, + poll_interval=0, + http_client=client, + ) + + assert ready is True + assert responses == [] + + +@pytest.mark.asyncio +async def test_async_wait_for_service_ready_retries_intermediate_failures() -> None: + calls = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + if calls == 1: + raise httpx.ConnectError("connection refused", request=request) + if calls == 2: + return httpx.Response(503) + if calls == 3: + return httpx.Response(200, content=b"{") + if calls == 4: + return httpx.Response(200, json=["not", "a", "mapping"]) + return httpx.Response(200, json={"services": {"ready": ["auth"], "not_ready": []}}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + ready = await async_wait_for_service_ready( + PlatformConfig(base_url="http://platform.local"), + "auth", + timeout=1.0, + poll_interval=0, + http_client=client, + ) + + assert ready is True + assert calls == 5 + + +@pytest.mark.asyncio +async def test_async_wait_for_dependencies_stops_after_unready_dependency() -> None: + with patch( + "nmp.common.service.api.health.async_wait_for_service_ready", + side_effect=[True, False], + ) as wait: + ready = await async_wait_for_dependencies( + PlatformConfig(base_url="http://platform.local"), + ["entities", "auth", "files"], + timeout_per_service=0.01, + poll_interval=0, + ) + + assert ready is False + assert [call.args[1] for call in wait.await_args_list] == ["entities", "auth"] + + +def test_wait_for_service_ready_skips_service_absent_from_status() -> None: + client = MagicMock() + response = MagicMock() + response.status_code = 200 + response.json.return_value = {"services": {"ready": ["entities"], "not_ready": []}} + client.get.return_value = response + + endpoint = MagicMock() + endpoint.connect_base_url = "http://platform.local" + endpoint.sync_http_client.return_value = client + + with patch("nmp.common.service.api.health.resolve_service_endpoint", return_value=endpoint): + ready = wait_for_service_ready( + PlatformConfig(base_url="http://platform.local"), + "models", + threading.Event(), + timeout=1.0, + poll_interval=0, + ) + + assert ready is True + client.get.assert_called_once_with("http://platform.local/status", headers=MARK_INTERNAL_REQUEST_HEADERS) + client.close.assert_called_once_with() + + +def test_wait_for_service_ready_returns_false_when_stop_signal_is_set() -> None: + stop_signal = threading.Event() + stop_signal.set() + client = MagicMock() + endpoint = MagicMock() + endpoint.connect_base_url = "http://platform.local" + endpoint.sync_http_client.return_value = client + + with patch("nmp.common.service.api.health.resolve_service_endpoint", return_value=endpoint): + ready = wait_for_service_ready( + PlatformConfig(base_url="http://platform.local"), + "entities", + stop_signal, + timeout=1.0, + poll_interval=0, + ) + + assert ready is False + client.get.assert_not_called() + client.close.assert_called_once_with() diff --git a/packages/nmp_common/tests/sdk_factory/test_sdk.py b/packages/nmp_common/tests/sdk_factory/test_sdk.py index efb24ba120..7c87c0631b 100644 --- a/packages/nmp_common/tests/sdk_factory/test_sdk.py +++ b/packages/nmp_common/tests/sdk_factory/test_sdk.py @@ -119,6 +119,15 @@ def test_get_platform_sdk_routes_local_service_path_to_process_listener(monkeypa assert prepared.path == "/apis/auth/v2/authz/allow" +def test_get_platform_sdk_uses_uds_endpoint_from_base_url(): + config = PlatformConfig(base_url="unix:///tmp/nemo-platform.sock") # type: ignore[abstract] + + with patch("nmp.common.sdk_factory.Configuration.get_platform_config", return_value=config): + sdk = get_platform_sdk() + + assert sdk.base_url == "http://nemo-platform.local" + + def test_get_platform_sdk_with_service_principal(): """Test get_platform_sdk with as_service parameter.""" sdk = get_platform_sdk(as_service="my-service") @@ -159,6 +168,15 @@ def test_get_async_platform_sdk(): assert str(sdk.base_url).rstrip("/") == str(expected).rstrip("/") +def test_get_async_platform_sdk_uses_uds_endpoint_from_base_url(): + config = PlatformConfig(base_url="unix:///tmp/nemo-platform.sock") # type: ignore[abstract] + + with patch("nmp.common.sdk_factory.Configuration.get_platform_config", return_value=config): + sdk = get_async_platform_sdk() + + assert str(sdk.base_url).rstrip("/") == "http://nemo-platform.local" + + def test_get_async_platform_sdk_with_service_principal(): """Test get_async_platform_sdk with as_service parameter.""" sdk = get_async_platform_sdk(as_service="async-service") @@ -590,6 +608,24 @@ def test_get_platform_sdk_routes_entities_path_to_entities_service( assert "/apis/entities/v2/workspaces" in str(prepared.path) +def test_get_platform_sdk_routes_service_path_to_env_override( + monkeypatch: pytest.MonkeyPatch, + platform_config_with_service_discovery, +): + monkeypatch.setenv("NMP_ENTITIES_URL", "http://entities-env:9090") + with patch( + "nmp.common.sdk_factory.Configuration.get_platform_config", + return_value=platform_config_with_service_discovery, + ): + sdk = get_platform_sdk() + request_url = "http://platform:8080/apis/entities/v2/workspaces" + prepared = sdk._prepare_url(request_url) + + assert prepared.host == "entities-env" + assert prepared.port == 9090 + assert prepared.scheme == "http" + + def test_get_platform_sdk_routes_jobs_path_to_jobs_service( platform_config_with_service_discovery, ): diff --git a/packages/nmp_common/tests/test_platform_endpoint.py b/packages/nmp_common/tests/test_platform_endpoint.py new file mode 100644 index 0000000000..0be4476362 --- /dev/null +++ b/packages/nmp_common/tests/test_platform_endpoint.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path +from unittest.mock import patch + +import pytest +from nmp.common.config import PlatformConfig +from nmp.common.platform_endpoint import UDS_BASE_URL, parse_platform_endpoint, resolve_service_endpoint + + +def test_parse_tcp_endpoint() -> None: + endpoint = parse_platform_endpoint("http://127.0.0.1:8080/") + + assert endpoint.transport == "tcp" + assert endpoint.connect_base_url == "http://127.0.0.1:8080" + assert endpoint.socket_path is None + + +@pytest.mark.parametrize("url", ["http://", "https://"]) +def test_parse_rejects_hostless_http_endpoint(url: str) -> None: + with pytest.raises(ValueError, match="must include a host"): + parse_platform_endpoint(url) + + +def test_parse_https_endpoint_preserves_normalized_connect_base_url() -> None: + endpoint = parse_platform_endpoint("https://platform.example.com/api/") + + assert endpoint.transport == "tcp" + assert endpoint.connect_base_url == "https://platform.example.com/api" + assert endpoint.socket_path is None + + +def test_parse_uds_endpoint() -> None: + endpoint = parse_platform_endpoint("unix:///tmp/nemo-platform.sock") + + assert endpoint.transport == "uds" + assert endpoint.connect_base_url == UDS_BASE_URL + assert endpoint.socket_path == Path("/tmp/nemo-platform.sock") + + +def test_parse_rejects_raw_socket_path() -> None: + with pytest.raises(ValueError, match="use unix:///tmp/nemo-platform.sock"): + parse_platform_endpoint("/tmp/nemo-platform.sock") + + +def test_parse_rejects_relative_uds_socket_path() -> None: + with pytest.raises(ValueError, match="absolute socket path"): + parse_platform_endpoint("unix://relative.sock") + + +def test_resolve_service_endpoint_uses_service_specific_url(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NMP_SECRETS_URL", "unix:///tmp/secrets.sock") + config = PlatformConfig(base_url="http://platform:8080") + + endpoint = resolve_service_endpoint("secrets", config) + + assert endpoint.transport == "uds" + assert endpoint.socket_path == Path("/tmp/secrets.sock") + + +def test_resolve_service_endpoint_falls_back_to_base_url(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("NMP_SECRETS_URL", raising=False) + config = PlatformConfig(base_url="http://platform:8080") + + endpoint = resolve_service_endpoint("secrets", config) + + assert endpoint.transport == "tcp" + assert endpoint.connect_base_url == "http://platform:8080" + + +def test_endpoint_env_family_is_not_part_of_contract(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NMP_SECRETS_ENDPOINT", "unix:///tmp/secrets.sock") + monkeypatch.delenv("NMP_SECRETS_URL", raising=False) + config = PlatformConfig(base_url="http://platform:8080") + + endpoint = resolve_service_endpoint("secrets", config) + + assert endpoint.transport == "tcp" + assert endpoint.connect_base_url == "http://platform:8080" + + +def test_sync_http_client_omits_unset_timeout() -> None: + endpoint = parse_platform_endpoint("http://127.0.0.1:8080") + + with patch("nmp.common.platform_endpoint.httpx.Client") as client: + endpoint.sync_http_client() + + client.assert_called_once_with(follow_redirects=True) + + +def test_sync_http_client_passes_explicit_timeout() -> None: + endpoint = parse_platform_endpoint("http://127.0.0.1:8080") + + with patch("nmp.common.platform_endpoint.httpx.Client") as client: + endpoint.sync_http_client(timeout=2.0) + + client.assert_called_once_with(follow_redirects=True, timeout=2.0) + + +def test_uds_sync_http_client_keeps_transport_and_omits_unset_timeout() -> None: + endpoint = parse_platform_endpoint("unix:///tmp/nemo-platform.sock") + + with patch("nmp.common.platform_endpoint.httpx.Client") as client: + endpoint.sync_http_client() + + kwargs = client.call_args.kwargs + assert kwargs["follow_redirects"] is True + assert "transport" in kwargs + assert "timeout" not in kwargs + + +def test_async_http_client_omits_unset_timeout() -> None: + endpoint = parse_platform_endpoint("http://127.0.0.1:8080") + + with patch("nmp.common.platform_endpoint.httpx.AsyncClient") as client: + endpoint.async_http_client() + + client.assert_called_once_with(follow_redirects=True) + + +def test_async_http_client_passes_explicit_timeout() -> None: + endpoint = parse_platform_endpoint("http://127.0.0.1:8080") + + with patch("nmp.common.platform_endpoint.httpx.AsyncClient") as client: + endpoint.async_http_client(timeout=2.0) + + client.assert_called_once_with(follow_redirects=True, timeout=2.0) + + +def test_uds_async_http_client_keeps_transport_and_omits_unset_timeout() -> None: + endpoint = parse_platform_endpoint("unix:///tmp/nemo-platform.sock") + + with patch("nmp.common.platform_endpoint.httpx.AsyncClient") as client: + endpoint.async_http_client() + + kwargs = client.call_args.kwargs + assert kwargs["follow_redirects"] is True + assert "transport" in kwargs + assert "timeout" not in kwargs diff --git a/packages/nmp_platform_runner/src/nmp/platform_runner/config.py b/packages/nmp_platform_runner/src/nmp/platform_runner/config.py index 977f9bba45..06c9acc189 100644 --- a/packages/nmp_platform_runner/src/nmp/platform_runner/config.py +++ b/packages/nmp_platform_runner/src/nmp/platform_runner/config.py @@ -6,9 +6,11 @@ from __future__ import annotations import os -from collections.abc import MutableMapping +import re +from collections.abc import MutableMapping, Sequence from dataclasses import dataclass, field from importlib.resources import files +from pathlib import Path from urllib.parse import urlparse from nmp.common.config import ( @@ -22,6 +24,7 @@ from nmp.platform_runner.loader import ControllerRunFunc from nmp.platform_runner.registry import ( AVAILABLE_SIDECARS, + SERVICE_SIDECAR_DEPENDENCIES, get_available_controllers, get_available_services, get_controller_groups, @@ -29,6 +32,103 @@ get_service_groups, ) +DEFAULT_SCOPE = "default" +DEFAULT_PLATFORM_BIND_HOST = "0.0.0.0" +DEFAULT_LOCAL_SERVICES_BIND_HOST = "127.0.0.1" + +_SCOPE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_INSTANCES_DIRNAME = "instances" +_SOCKET_FILENAME = "nemo-platform.sock" +_LOG_FILENAME = "services.log" + + +@dataclass +class PlatformAppConfig: + """Service selection and listener binding for a platform app/server.""" + + services: Sequence[str] | None = None + service_group: str | None = None + controllers: Sequence[str] | None = None + controller_group: str | None = None + sidecars: Sequence[str] | None = None + config_path: str | None = None + scope: str = DEFAULT_SCOPE + host: str = DEFAULT_PLATFORM_BIND_HOST + port: int = 8080 + socket_path: str | Path | None = None + state_root: str | Path | None = None + runtime_root: str | Path | None = None + log_path: str | Path | None = None + + def __post_init__(self) -> None: + self.scope = validate_scope(self.scope) + self.socket_path = _resolve_socket_path(self.socket_path) + self.state_root = _resolve_absolute_path(self.state_root, "state root") + self.runtime_root = _resolve_absolute_path(self.runtime_root, "runtime root") + self.log_path = _resolve_absolute_path(self.log_path, "log path") + + @property + def state_root_path(self) -> Path: + return Path(self.state_root) if self.state_root is not None else default_state_root() + + @property + def runtime_root_path(self) -> Path: + return Path(self.runtime_root) if self.runtime_root is not None else default_runtime_root() + + def state_dir(self, *, create: bool = False) -> Path: + path = self.state_root_path / _INSTANCES_DIRNAME / self.scope + if create: + path.mkdir(parents=True, exist_ok=True) + return path + + def runtime_dir(self, *, create: bool = False) -> Path: + if self.runtime_root is None and self.socket_path is not None: + path = Path(self.socket_path).parent + else: + path = self.runtime_root_path / self.scope + if create: + path.mkdir(parents=True, exist_ok=True) + return path + + def socket_file_path(self) -> Path: + if self.socket_path is not None: + return Path(self.socket_path) + return self.runtime_dir() / _SOCKET_FILENAME + + def log_file_path(self, *, create_parent: bool = False) -> Path: + path = Path(self.log_path) if self.log_path is not None else self.state_dir() / _LOG_FILENAME + if create_parent: + path.parent.mkdir(parents=True, exist_ok=True) + return path + + +def validate_scope(scope: str) -> str: + """Ensure *scope* is safe to use for local state and socket paths.""" + if not _SCOPE_RE.fullmatch(scope): + raise ValueError(f"Invalid scope: {scope!r}") + return scope + + +def default_state_root() -> Path: + """Return the local services state root.""" + xdg = os.environ.get("XDG_STATE_HOME") + if xdg: + return Path(xdg) / "nmp" + return Path.home() / ".local" / "state" / "nmp" + + +def default_runtime_root() -> Path: + """Return the local services runtime root for sockets and volatile metadata.""" + return default_state_root() / "run" + + +def _sidecars_for_services(service_names: set[str]) -> set[str]: + selected: set[str] = set() + for service_name in service_names: + selected.update(SERVICE_SIDECAR_DEPENDENCIES.get(service_name, set())) + return selected + + _IPV4_LOOPBACK = "127.0.0.1" _IPV6_LOOPBACK = "::1" _IPV4_WILDCARDS = frozenset({"0.0.0.0"}) @@ -43,6 +143,7 @@ class ResolvedRunConfiguration: host: str port: int config_path: str + socket_path: str | None = None available_services: dict[str, str | Service] = field(default_factory=dict) available_controllers: dict[str, str | ControllerRunFunc] = field(default_factory=dict) @@ -56,22 +157,15 @@ def default_config_path() -> str: def resolve_run_configuration( - *, - services: list[str] | None = None, - service_group: str | None = None, - controllers: list[str] | None = None, - controller_group: str | None = None, - sidecars: list[str] | None = None, - config_path: str | None = None, - host: str = "0.0.0.0", - port: int = 8080, + config: PlatformAppConfig | None = None, ) -> ResolvedRunConfiguration: - """Resolve and validate platform run arguments. + """Resolve and validate platform run configuration. Group selectors are convenience shortcuts for callers that are not also naming specific services or controllers. Mixing the two is ambiguous, so these combinations fail fast instead of silently ignoring the group. """ + config = config or PlatformAppConfig() available_services = get_available_services() available_controllers = get_available_controllers() available_sidecars = AVAILABLE_SIDECARS @@ -79,30 +173,30 @@ def resolve_run_configuration( controller_groups = get_controller_groups(available_controllers) default_controllers = set(get_default_controllers(controller_groups)) - selected_services = set(services or []) - selected_controllers = set(controllers or []) - selected_sidecars = set(sidecars or []) + selected_services = set(config.services or []) + selected_controllers = set(config.controllers or []) + selected_sidecars = set(config.sidecars or []) # Explicit selections and group selectors are mutually exclusive. The old # entrypoint rejected these combinations, and keeping that behavior avoids a # confusing silent-ignore UX for callers. - if service_group and selected_services: + if config.service_group and selected_services: raise ValueError("--services cannot be combined with --service-group") - if controller_group and selected_controllers: + if config.controller_group and selected_controllers: raise ValueError("--controllers cannot be combined with --controller-group") - if service_group and not selected_services: - if service_group not in service_groups: + if config.service_group and not selected_services: + if config.service_group not in service_groups: valid_groups = ", ".join(sorted(service_groups)) - raise ValueError(f"Unknown service group: {service_group}. Available groups: {valid_groups}") - selected_services.update(service_groups[service_group]) + raise ValueError(f"Unknown service group: {config.service_group}. Available groups: {valid_groups}") + selected_services.update(service_groups[config.service_group]) - if controller_group and not selected_controllers: - if controller_group not in controller_groups: + if config.controller_group and not selected_controllers: + if config.controller_group not in controller_groups: valid_groups = ", ".join(sorted(controller_groups)) - raise ValueError(f"Unknown controller group: {controller_group}. Available groups: {valid_groups}") - selected_controllers.update(controller_groups[controller_group]) + raise ValueError(f"Unknown controller group: {config.controller_group}. Available groups: {valid_groups}") + selected_controllers.update(controller_groups[config.controller_group]) invalid_services = selected_services - set(available_services) if invalid_services: @@ -116,30 +210,53 @@ def resolve_run_configuration( requested = ", ".join(sorted(invalid_controllers)) raise ValueError(f"Unknown controllers: {requested}. Available controllers: {available}") + if not selected_services and not selected_controllers and not selected_sidecars: + # No explicit selection means "run the platform": start the default + # service group plus the default controller set. + selected_services.update(service_groups["all"]) + selected_controllers.update(default_controllers) + + selected_sidecars.update(_sidecars_for_services(selected_services)) + invalid_sidecars = selected_sidecars - set(available_sidecars) if invalid_sidecars: available = ", ".join(sorted(available_sidecars)) requested = ", ".join(sorted(invalid_sidecars)) raise ValueError(f"Unknown sidecars: {requested}. Available sidecars: {available}") - if not selected_services and not selected_controllers and not selected_sidecars: - # No explicit selection means "run the platform": start the default - # service group plus the default controller set. - selected_services.update(service_groups["all"]) - selected_controllers.update(default_controllers) + resolved_socket_path = _resolve_socket_path(config.socket_path) return ResolvedRunConfiguration( services=selected_services, controllers=selected_controllers, sidecars=selected_sidecars, - host=host, - port=port, - config_path=config_path or default_config_path(), + host=config.host, + port=config.port, + config_path=config.config_path or default_config_path(), + socket_path=resolved_socket_path, available_services=available_services, available_controllers=available_controllers, ) +def _resolve_socket_path(socket_path: str | Path | None) -> str | None: + if socket_path is None: + return None + path = Path(socket_path).expanduser() + if not path.is_absolute(): + raise ValueError(f"UDS socket path must be absolute: {socket_path}") + return str(path) + + +def _resolve_absolute_path(path_value: str | Path | None, label: str) -> str | None: + if path_value is None: + return None + path = Path(path_value).expanduser() + if not path.is_absolute(): + raise ValueError(f"{label} must be absolute: {path_value}") + return str(path) + + def apply_run_environment( config: ResolvedRunConfiguration, env: MutableMapping[str, str] | None = None, @@ -181,14 +298,17 @@ def apply_run_environment( connect_host = _connect_host_for_internal_clients(config.host) effective_host = env.setdefault("NMP_SERVICE_HOST", connect_host) effective_port = env.setdefault("NMP_SERVICE_PORT", str(config.port)) - config_base_url_parts = _config_file_base_url_parts(config.config_path) - if config_base_url_parts is not None: - scheme, config_host = config_base_url_parts - host_for_url = _bracket_ipv6(_connect_host_for_internal_clients(config_host)) - default_base_url = f"{scheme}://{host_for_url}:{effective_port}" + if config.socket_path: + default_base_url = f"unix://{config.socket_path}" else: - host_for_url = _bracket_ipv6(effective_host) - default_base_url = f"http://{host_for_url}:{effective_port}" + config_base_url_parts = _config_file_base_url_parts(config.config_path) + if config_base_url_parts is not None: + scheme, config_host = config_base_url_parts + host_for_url = _bracket_ipv6(_connect_host_for_internal_clients(config_host)) + default_base_url = f"{scheme}://{host_for_url}:{effective_port}" + else: + host_for_url = _bracket_ipv6(effective_host) + default_base_url = f"http://{host_for_url}:{effective_port}" base_url = env.setdefault("NMP_BASE_URL", default_base_url) # Embedded PDP is usually served from the same platform process, so its # self-call origin must stay aligned with the resolved base URL. Deployed diff --git a/packages/nmp_platform_runner/src/nmp/platform_runner/registry.py b/packages/nmp_platform_runner/src/nmp/platform_runner/registry.py index bb4b0f4100..ab8bcefa45 100644 --- a/packages/nmp_platform_runner/src/nmp/platform_runner/registry.py +++ b/packages/nmp_platform_runner/src/nmp/platform_runner/registry.py @@ -39,6 +39,10 @@ "adapters": "nmp.core.models.sidecars.adapters.main:run", } +SERVICE_SIDECAR_DEPENDENCIES: dict[str, set[str]] = { + "models": {"adapters"}, +} + CORE_SERVICES = [ "auth", "models", diff --git a/packages/nmp_platform_runner/src/nmp/platform_runner/run.py b/packages/nmp_platform_runner/src/nmp/platform_runner/run.py index 0dbad175ed..d3a54eac98 100644 --- a/packages/nmp_platform_runner/src/nmp/platform_runner/run.py +++ b/packages/nmp_platform_runner/src/nmp/platform_runner/run.py @@ -17,7 +17,11 @@ from nmp.common.observability import initialize_obs, setup_global_instrumentations from nmp.common.observability.otel import settings as otel_settings from nmp.common.service import CircularDependencyError, Service -from nmp.platform_runner.config import apply_run_environment, resolve_run_configuration +from nmp.platform_runner.config import ( + PlatformAppConfig, + apply_run_environment, + resolve_run_configuration, +) from nmp.platform_runner.health import get_platform_resource_attributes from nmp.platform_runner.loader import ( ControllerRunFunc, @@ -72,15 +76,8 @@ def run_controllers_in_threads( def run_platform( + config: PlatformAppConfig | None = None, *, - services: list[str] | None = None, - service_group: str | None = None, - controllers: list[str] | None = None, - controller_group: str | None = None, - sidecars: list[str] | None = None, - config_path: str | None = None, - host: str = "0.0.0.0", - port: int = 8080, reload_app_factory: str | None = None, on_shutdown: Callable[[], object] | None = None, ) -> None: @@ -88,16 +85,7 @@ def run_platform( t_total = time.perf_counter() t0 = time.perf_counter() - resolved = resolve_run_configuration( - services=services, - service_group=service_group, - controllers=controllers, - controller_group=controller_group, - sidecars=sidecars, - config_path=config_path, - host=host, - port=port, - ) + resolved = resolve_run_configuration(config) apply_run_environment(resolved) _startup_phase("resolve_config", t0) @@ -163,7 +151,7 @@ def signal_handler(signum: int, _frame: object) -> None: controller_threads.extend(run_controllers_in_threads(controller_run_funcs, controller_stop_signal)) if sidecar_run_funcs: controller_threads.extend(run_controllers_in_threads(sidecar_run_funcs, controller_stop_signal)) - run_server(service_instances, host=resolved.host, port=resolved.port) + run_server(service_instances, host=resolved.host, port=resolved.port, socket_path=resolved.socket_path) except ValueError as error: logger.error("Configuration error: %s", error) raise SystemExit(1) from error diff --git a/packages/nmp_platform_runner/src/nmp/platform_runner/server.py b/packages/nmp_platform_runner/src/nmp/platform_runner/server.py index daac4f6182..eef0e55d2c 100644 --- a/packages/nmp_platform_runner/src/nmp/platform_runner/server.py +++ b/packages/nmp_platform_runner/src/nmp/platform_runner/server.py @@ -10,6 +10,7 @@ import logging import os import threading +from collections.abc import Callable, Mapping, MutableMapping from contextlib import asynccontextmanager from typing import cast @@ -25,6 +26,7 @@ from nmp.common.observability.context import create_app_context_dependency from nmp.common.pyleak import detect_blocking from nmp.common.service import Service +from nmp.platform_runner.config import PlatformAppConfig from nmp.platform_runner.health import ReadinessCheck, create_platform_health_router, get_platform_resource_attributes from nmp.platform_runner.loader import ( ControllerRunFunc, @@ -32,7 +34,12 @@ load_service, order_services_by_dependencies, ) -from nmp.platform_runner.registry import get_available_controllers, get_available_services, get_openapi_service_names +from nmp.platform_runner.registry import ( + AVAILABLE_SIDECARS, + get_available_controllers, + get_available_services, + get_openapi_service_names, +) from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request from starlette.responses import RedirectResponse, Response @@ -272,12 +279,85 @@ async def root_handler() -> Response: return app -def run_server(services: list[Service] | None = None, host: str = "0.0.0.0", port: int = 8080) -> None: +def _load_run_functions( + names: list[str], + registry: Mapping[str, str | Callable[[threading.Event], object]], +) -> dict[str, Callable[[threading.Event], object]]: + run_funcs: dict[str, Callable[[threading.Event], object]] = {} + for name in names: + value = registry[name] + if isinstance(value, str): + run_funcs[name] = load_controller_run_func(name, value) + else: + run_funcs[name] = value + return run_funcs + + +def build_platform_app( + config: PlatformAppConfig | None = None, + *, + http_client: httpx.AsyncClient | None = None, + env: MutableMapping[str, str] | None = None, +) -> FastAPI: + """Build a platform FastAPI app without starting uvicorn. + + Args: + config: App-build selection and bind configuration. Prefer this over + individual service/controller/sidecar keyword arguments for new + callers. + env: Environment mapping passed to :func:`apply_run_environment`. + Defaults to ``None`` which writes to ``os.environ``. Tests can + pass an empty dict to avoid polluting the process environment. + """ + from nmp.platform_runner.config import apply_run_environment, resolve_run_configuration + + resolved = resolve_run_configuration(config) + apply_run_environment(resolved, env=env) + + service_instances = [] + for service_name in sorted(resolved.services): + service_value = resolved.available_services[service_name] + service_instances.append( + service_value if isinstance(service_value, Service) else load_service(service_name, service_value) + ) + service_instances = order_services_by_dependencies(service_instances) + + collisions = resolved.controllers & resolved.sidecars + if collisions: + raise ValueError(f"Controller/sidecar name collision: {', '.join(sorted(collisions))}") + + controller_run_funcs = _load_run_functions(sorted(resolved.controllers), resolved.available_controllers) + sidecar_run_funcs = _load_run_functions(sorted(resolved.sidecars), AVAILABLE_SIDECARS) + controller_run_funcs.update(sidecar_run_funcs) + + return create_app(service_instances, controller_run_funcs=controller_run_funcs, http_client=http_client) + + +def run_server( + services: list[Service] | None = None, + host: str = "0.0.0.0", + port: int = 8080, + socket_path: str | None = None, +) -> None: """Run the platform API server.""" preflight_embedded_auth_policy_wasm(get_auth_config()) app = create_app(services or []) setup_fastapi_instrumentations(app) - uvicorn.run(app, host=host, port=port, log_config=None) + if socket_path: + _run_server_on_bound_sockets(app, host=host, port=port, socket_path=socket_path) + else: + uvicorn.run(app, host=host, port=port, log_config=None) + + +def _run_server_on_bound_sockets(app: FastAPI, *, host: str, port: int, socket_path: str) -> None: + tcp_config = uvicorn.Config(app, host=host, port=port, log_config=None) + uds_config = uvicorn.Config(app, uds=socket_path, log_config=None) + sockets = [tcp_config.bind_socket(), uds_config.bind_socket()] + try: + asyncio.run(uvicorn.Server(tcp_config).serve(sockets=sockets)) + finally: + for sock in sockets: + sock.close() def run_server_with_reload(app_factory: str, host: str = "0.0.0.0", port: int = 8080) -> None: diff --git a/packages/nmp_platform_runner/tests/test_config.py b/packages/nmp_platform_runner/tests/test_config.py index 2318957569..738ae21fb8 100644 --- a/packages/nmp_platform_runner/tests/test_config.py +++ b/packages/nmp_platform_runner/tests/test_config.py @@ -6,6 +6,8 @@ import pytest from nmp.platform_runner import registry from nmp.platform_runner.config import ( + DEFAULT_PLATFORM_BIND_HOST, + PlatformAppConfig, ResolvedRunConfiguration, apply_run_environment, default_config_path, @@ -26,6 +28,7 @@ def _make_config( sidecars: set[str] | None = None, host: str = "0.0.0.0", port: int = 8080, + socket_path: str | None = None, config_path: str = "/nonexistent/nmp-test-config.yaml", ) -> ResolvedRunConfiguration: return ResolvedRunConfiguration( @@ -34,14 +37,13 @@ def _make_config( sidecars=sidecars if sidecars is not None else set(), host=host, port=port, + socket_path=socket_path, config_path=config_path, ) def resolve(**kwargs): - params = {} - params.update(kwargs) - return resolve_run_configuration(**params) + return resolve_run_configuration(PlatformAppConfig(**kwargs)) def test_default_config_path_points_to_bundled_local_config(): @@ -50,9 +52,80 @@ def test_default_config_path_points_to_bundled_local_config(): assert path.endswith(("nmp/platform_runner/config/local.yaml", "nemo_platform/services/runner/config/local.yaml")) +def test_platform_app_config_keeps_sequence_fields_simple(): + config = PlatformAppConfig( + services=["models"], + controllers=[], + sidecars=["adapters"], + ) + + assert config.services == ["models"] + assert config.controllers == [] + assert config.sidecars == ["adapters"] + + +def test_platform_app_config_derives_instance_paths_from_roots(tmp_path: Path): + config = PlatformAppConfig(scope="dev", state_root=tmp_path / "state", runtime_root=tmp_path / "run") + + assert config.state_dir() == tmp_path / "state" / "instances" / "dev" + assert config.runtime_dir() == tmp_path / "run" / "dev" + assert config.socket_file_path() == tmp_path / "run" / "dev" / "nemo-platform.sock" + assert config.log_file_path() == tmp_path / "state" / "instances" / "dev" / "services.log" + + +def test_platform_app_config_runtime_dir_defaults_to_explicit_socket_parent(tmp_path: Path): + config = PlatformAppConfig(socket_path=tmp_path / "custom.sock") + + assert config.runtime_dir() == tmp_path + assert config.socket_file_path() == tmp_path / "custom.sock" + + +def test_platform_app_config_uses_explicit_log_path(tmp_path: Path): + config = PlatformAppConfig(state_root=tmp_path / "state", log_path=tmp_path / "logs" / "nemo.log") + + assert config.log_file_path() == tmp_path / "logs" / "nemo.log" + + +def test_platform_app_config_rejects_relative_socket_path(): + with pytest.raises(ValueError, match="UDS socket path must be absolute"): + PlatformAppConfig(socket_path="relative/path") + + +def test_platform_app_config_rejects_relative_state_root(): + with pytest.raises(ValueError, match="state root must be absolute"): + PlatformAppConfig(state_root="relative/path") + + +def test_platform_app_config_rejects_relative_runtime_root(): + with pytest.raises(ValueError, match="runtime root must be absolute"): + PlatformAppConfig(runtime_root="relative/path") + + +def test_platform_app_config_rejects_relative_log_path(): + with pytest.raises(ValueError, match="log path must be absolute"): + PlatformAppConfig(log_path="relative/path") + + +def test_resolve_run_configuration_accepts_platform_app_config(): + resolved = resolve_run_configuration( + PlatformAppConfig( + services=["auth"], + controllers=[], + host="127.0.0.1", + port=9090, + ) + ) + + assert resolved.services == {"auth"} + assert resolved.controllers == set() + assert resolved.host == "127.0.0.1" + assert resolved.port == 9090 + + def test_no_arguments_defaults_to_all_services_and_default_controllers(): resolved = resolve() + assert resolved.host == DEFAULT_PLATFORM_BIND_HOST assert resolved.services.issuperset( { "auth", @@ -112,6 +185,17 @@ def test_extra_services_are_available_for_resolution(): resolve(services=["custom-service"]) +def test_resolve_rejects_relative_socket_path(): + with pytest.raises(ValueError, match="UDS socket path must be absolute"): + resolve(socket_path="relative.sock") + + +def test_resolve_preserves_absolute_socket_path(): + resolved = resolve(socket_path="/tmp/nemo-platform.sock") + + assert resolved.socket_path == "/tmp/nemo-platform.sock" + + # --------------------------------------------------------------------------- # Topology regression tests for apply_run_environment # @@ -146,6 +230,11 @@ def test_config_file_gateway_base_url_seeds_base_url(self, tmp_path: Path): assert env["NMP_BASE_URL"] == "https://nemo-gateway:8080" assert env["NMP_SERVICE_HOST"] == "127.0.0.1" + def test_sets_uds_base_url_when_socket_path_is_present(self): + env: dict[str, str] = {} + apply_run_environment(_make_config(socket_path="/tmp/nemo-platform.sock"), env=env) + assert env["NMP_BASE_URL"] == "unix:///tmp/nemo-platform.sock" + def test_sets_embedded_pdp_base_url_from_base_url(self): env: dict[str, str] = {} apply_run_environment(_make_config(host="0.0.0.0", port=9090), env=env) diff --git a/packages/nmp_platform_runner/tests/test_health.py b/packages/nmp_platform_runner/tests/test_health.py new file mode 100644 index 0000000000..f83cc964e5 --- /dev/null +++ b/packages/nmp_platform_runner/tests/test_health.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from nmp.common.controller.controller_manager import ControllerManager +from nmp.common.service import RouterConfig, Service +from nmp.platform_runner.health import create_platform_health_router + + +class ProbeService(Service): + def __init__(self, name: str, *, ready: bool = True) -> None: + super().__init__(name=name, module_name=f"nmp.{name}") + self.ready = ready + + def get_routers(self) -> list[RouterConfig]: + return [] + + async def is_ready(self) -> bool: + return self.ready + + +@pytest.fixture(autouse=True) +def reset_controller_manager() -> None: + ControllerManager._instance = None + yield + ControllerManager._instance = None + + +def _client_for(services: list[Service]) -> TestClient: + app = FastAPI() + app.include_router(create_platform_health_router(services)) + return TestClient(app) + + +def test_status_and_ready_are_healthy_when_no_services_are_running() -> None: + client = _client_for([]) + + status_response = client.get("/status") + ready_response = client.get("/health/ready") + + assert status_response.status_code == 200 + assert status_response.json() == { + "status": "healthy", + "services": {"ready": [], "not_ready": []}, + "controllers": {"healthy": True, "status": {}}, + } + assert ready_response.status_code == 200 + assert ready_response.json() == {"status": "ready"} + + +def test_status_only_reports_services_registered_with_runner() -> None: + registered = ProbeService("entities", ready=True) + not_started = ProbeService("models", ready=False) + client = _client_for([registered]) + + response = client.get("/status") + + assert response.status_code == 200 + payload = response.json() + assert payload["status"] == "healthy" + assert payload["services"] == {"ready": ["entities"], "not_ready": []} + assert not_started.name not in payload["services"]["ready"] + assert not_started.name not in [service["name"] for service in payload["services"]["not_ready"]] + assert client.get("/health/ready").status_code == 200 + + +def test_status_remains_healthy_when_new_service_is_registered_after_it_is_ready() -> None: + entities = ProbeService("entities", ready=True) + models = ProbeService("models", ready=False) + services: list[Service] = [entities] + client = _client_for(services) + + assert client.get("/status").json()["status"] == "healthy" + + models.ready = True + services.append(models) + + response = client.get("/status") + + assert response.status_code == 200 + assert response.json()["status"] == "healthy" + assert response.json()["services"] == {"ready": ["entities", "models"], "not_ready": []} + assert client.get("/health/ready").status_code == 200 + + +def test_registered_not_ready_service_degrades_status_and_blocks_readiness() -> None: + entities = ProbeService("entities", ready=True) + models = ProbeService("models", ready=False) + client = _client_for([entities, models]) + + status_response = client.get("/status") + ready_response = client.get("/health/ready") + + assert status_response.status_code == 200 + assert status_response.json()["status"] == "degraded" + assert status_response.json()["services"] == { + "ready": ["entities"], + "not_ready": [{"name": "models", "message": ""}], + } + assert ready_response.status_code == 503 + assert ready_response.json() == {"detail": {"status": "not_ready"}} diff --git a/packages/nmp_platform_runner/tests/test_run.py b/packages/nmp_platform_runner/tests/test_run.py index f5cb9d4617..def553aadc 100644 --- a/packages/nmp_platform_runner/tests/test_run.py +++ b/packages/nmp_platform_runner/tests/test_run.py @@ -65,7 +65,7 @@ def test_run_platform_marks_loaded_services_local_before_starting_controllers(mo ) services = [_StubService("jobs"), _StubService("entities")] - monkeypatch.setattr(runner, "resolve_run_configuration", lambda **_: resolved) + monkeypatch.setattr(runner, "resolve_run_configuration", lambda *_args, **_kwargs: resolved) monkeypatch.setattr(runner, "apply_run_environment", lambda config: None) monkeypatch.setattr(runner, "initialize_obs", lambda *, resource_attributes: None) monkeypatch.setattr(runner, "setup_global_instrumentations", lambda: None) @@ -76,7 +76,7 @@ def test_run_platform_marks_loaded_services_local_before_starting_controllers(mo lambda names, registry, kind: {"jobs": lambda stop_signal: None} if kind == "controller" else {}, ) monkeypatch.setattr(runner, "_display_banner", lambda **_: None) - monkeypatch.setattr(runner, "run_server", lambda services, host, port: None) + monkeypatch.setattr(runner, "run_server", lambda services, host, port, socket_path=None: None) monkeypatch.setattr(runner.signal, "signal", lambda *args: None) def capture_controller_start( diff --git a/packages/nmp_platform_runner/tests/test_server.py b/packages/nmp_platform_runner/tests/test_server.py index f752d1c51a..2087de75f1 100644 --- a/packages/nmp_platform_runner/tests/test_server.py +++ b/packages/nmp_platform_runner/tests/test_server.py @@ -3,6 +3,7 @@ import asyncio import builtins +import os import sys import threading import time @@ -15,9 +16,33 @@ from nmp.common.config import AuthConfig, Configuration from nmp.common.config.base import OIDCConfig from nmp.common.service import Service +from nmp.platform_runner import config as runner_config from nmp.platform_runner import server from nmp.platform_runner.health import ReadinessCheck, create_platform_health_router +_RUN_ENV_KEYS = ( + "NMP_CONFIG_FILE_PATH", + "NMP_SERVICE_HOST", + "NMP_SERVICE_PORT", + "NMP_BASE_URL", + "NMP_AUTH_POLICY_DECISION_POINT_BASE_URL", + "NMP_SERVICES", + "NMP_CONTROLLERS", + "NMP_SIDECARS", +) + + +@pytest.fixture(autouse=True) +def restore_platform_runner_env(): + original_env = {key: os.environ.get(key) for key in _RUN_ENV_KEYS} + yield + for key, value in original_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + Configuration.clear_cache() + def _make_auth_config(*, enabled: bool) -> AuthConfig: return AuthConfig( @@ -180,6 +205,59 @@ def test_create_app_mounted_services_drive_sdk_local_routing_without_services_en Configuration.clear_cache() +def test_build_platform_app_returns_app_without_running_uvicorn(monkeypatch): + plugin_service = PluginService() + captured: dict[str, object] = {} + + monkeypatch.setattr(runner_config, "get_available_services", lambda: {"agents": plugin_service}) + monkeypatch.setattr(runner_config, "get_available_controllers", lambda: {}) + monkeypatch.setattr(runner_config, "get_controller_groups", lambda _controllers: {"all": [], "core": []}) + monkeypatch.setattr(server, "order_services_by_dependencies", lambda services: services) + + def fake_create_app(services, controller_run_funcs=None, http_client=None): + captured["services"] = services + captured["controller_run_funcs"] = controller_run_funcs + captured["http_client"] = http_client + return FastAPI() + + monkeypatch.setattr(server, "create_app", fake_create_app) + + app = server.build_platform_app(runner_config.PlatformAppConfig(services=["agents"], controllers=[]), env={}) + + assert isinstance(app, FastAPI) + assert captured["services"] == [plugin_service] + assert captured["controller_run_funcs"] == {} + assert captured["http_client"] is None + + +def test_build_platform_app_accepts_platform_app_config(monkeypatch): + plugin_service = PluginService() + captured: dict[str, object] = {} + + monkeypatch.setattr(runner_config, "get_available_services", lambda: {"agents": plugin_service}) + monkeypatch.setattr(runner_config, "get_available_controllers", lambda: {}) + monkeypatch.setattr(runner_config, "get_controller_groups", lambda _controllers: {"all": [], "core": []}) + monkeypatch.setattr(server, "order_services_by_dependencies", lambda services: services) + + def fake_create_app(services, controller_run_funcs=None, http_client=None): + captured["services"] = services + captured["controller_run_funcs"] = controller_run_funcs + captured["http_client"] = http_client + return FastAPI() + + monkeypatch.setattr(server, "create_app", fake_create_app) + + app = server.build_platform_app( + config=runner_config.PlatformAppConfig(services=("agents",), controllers=()), + env={}, + ) + + assert isinstance(app, FastAPI) + assert captured["services"] == [plugin_service] + assert captured["controller_run_funcs"] == {} + assert captured["http_client"] is None + + def test_embedded_auth_preflight_invokes_policy_wasm_helper(monkeypatch): calls: list[bool] = [] auth_cfg = AuthConfig( @@ -235,6 +313,25 @@ def test_run_server_runs_embedded_auth_preflight(): uvicorn_run.assert_called_once() +def test_run_server_can_bind_tcp_and_unix_domain_socket(): + auth_cfg = _make_auth_config(enabled=True) + with ( + patch("nmp.platform_runner.server.get_auth_config", return_value=auth_cfg), + patch("nmp.platform_runner.server.preflight_embedded_auth_policy_wasm"), + patch("nmp.platform_runner.server.create_app", return_value=FastAPI()), + patch("nmp.platform_runner.server.setup_fastapi_instrumentations"), + patch("nmp.platform_runner.server._run_server_on_bound_sockets") as run_bound_sockets, + ): + server.run_server(services=[], host="127.0.0.1", port=9999, socket_path="/tmp/nemo-platform.sock") + + run_bound_sockets.assert_called_once() + assert run_bound_sockets.call_args.kwargs == { + "host": "127.0.0.1", + "port": 9999, + "socket_path": "/tmp/nemo-platform.sock", + } + + def test_create_default_app_raises_for_unknown_service_from_env(monkeypatch): monkeypatch.setattr(server, "_obs_initialized", True) monkeypatch.setenv("NMP_SERVICES", "missing-service") diff --git a/packages/nmp_platform_runner/tests/test_sidecars.py b/packages/nmp_platform_runner/tests/test_sidecars.py new file mode 100644 index 0000000000..e3950b3a93 --- /dev/null +++ b/packages/nmp_platform_runner/tests/test_sidecars.py @@ -0,0 +1,213 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import threading +from collections.abc import Callable +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from nmp.common.config import AuthConfig +from nmp.common.config.base import OIDCConfig +from nmp.common.service import Service +from nmp.platform_runner import config as runner_config +from nmp.platform_runner import server + + +class DummyService(Service): + def __init__(self, name: str = "models") -> None: + super().__init__(name=name, module_name="test.sidecars") + + def get_routers(self): + return [] + + +def _dummy_sidecar(_stop_signal: threading.Event) -> None: + return None + + +def _patch_runner_discovery( + monkeypatch: pytest.MonkeyPatch, + *, + services: dict[str, Service] | None = None, + controllers: dict[str, Callable[[threading.Event], object]] | None = None, + sidecars: dict[str, Callable[[threading.Event], object]] | None = None, +) -> None: + services = services if services is not None else {"models": DummyService("models")} + controllers = controllers if controllers is not None else {} + sidecars = sidecars if sidecars is not None else {"adapters": _dummy_sidecar} + + monkeypatch.setattr(runner_config, "get_available_services", lambda: services) + monkeypatch.setattr(runner_config, "get_available_controllers", lambda: controllers) + monkeypatch.setattr( + runner_config, + "get_service_groups", + lambda _available: {"all": list(services), "core": list(services), "api": []}, + ) + monkeypatch.setattr( + runner_config, "get_controller_groups", lambda _available: {"all": list(controllers), "core": list(controllers)} + ) + monkeypatch.setattr(runner_config, "get_default_controllers", lambda _groups: list(controllers)) + monkeypatch.setattr(runner_config, "AVAILABLE_SIDECARS", sidecars) + monkeypatch.setattr("nmp.platform_runner.registry.AVAILABLE_SIDECARS", sidecars) + monkeypatch.setattr(server, "AVAILABLE_SIDECARS", sidecars, raising=False) + monkeypatch.setattr(server, "order_services_by_dependencies", lambda service_instances: service_instances) + + +def test_models_service_resolves_adapters_sidecar_dependency(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_runner_discovery(monkeypatch) + + resolved = runner_config.resolve_run_configuration( + runner_config.PlatformAppConfig(services=["models"], controllers=[]) + ) + + assert resolved.services == {"models"} + assert resolved.sidecars == {"adapters"} + + +def test_explicit_sidecar_can_run_without_services(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_runner_discovery(monkeypatch) + + resolved = runner_config.resolve_run_configuration( + runner_config.PlatformAppConfig(services=[], controllers=[], sidecars=["adapters"]) + ) + + assert resolved.services == set() + assert resolved.controllers == set() + assert resolved.sidecars == {"adapters"} + + +def _auth_config(enabled: bool = False) -> AuthConfig: + return AuthConfig( + enabled=enabled, + policy_decision_point_base_url="http://localhost:8181", + oidc=OIDCConfig(enabled=False), + ) + + +def _sidecar_with_events(started: threading.Event, stopped: threading.Event) -> Callable[[threading.Event], None]: + def run(stop_signal: threading.Event) -> None: + started.set() + stop_signal.wait(timeout=5.0) + stopped.set() + + return run + + +def test_create_app_starts_and_stops_dummy_sidecar_with_lifespan() -> None: + started = threading.Event() + stopped = threading.Event() + + with ( + patch("nmp.platform_runner.server.get_platform_config") as platform_config, + patch("nmp.platform_runner.server.get_auth_config", return_value=_auth_config(False)), + patch("nmp.common.auth.middleware.get_auth_config", return_value=_auth_config(False)), + ): + platform_config.return_value.seed_on_startup = False + platform_config.return_value.redirect_root_to_studio = False + app = server.create_app( + services=[], + controller_run_funcs={"adapters": _sidecar_with_events(started, stopped)}, + ) + from fastapi.testclient import TestClient + + with TestClient(app) as client: + assert started.wait(timeout=1.0) + assert client.get("/").status_code == 200 + + assert stopped.wait(timeout=1.0) + + +def test_build_platform_app_loads_dependent_sidecar_into_lifespan(monkeypatch: pytest.MonkeyPatch) -> None: + started = threading.Event() + stopped = threading.Event() + _patch_runner_discovery(monkeypatch, sidecars={"adapters": _sidecar_with_events(started, stopped)}) + + with ( + patch("nmp.platform_runner.server.get_platform_config") as platform_config, + patch("nmp.platform_runner.server.get_auth_config", return_value=_auth_config(False)), + patch("nmp.common.auth.middleware.get_auth_config", return_value=_auth_config(False)), + ): + platform_config.return_value.seed_on_startup = False + platform_config.return_value.redirect_root_to_studio = False + app = server.build_platform_app(runner_config.PlatformAppConfig(services=["models"], controllers=[]), env={}) + from fastapi.testclient import TestClient + + with TestClient(app) as client: + assert started.wait(timeout=1.0) + assert client.get("/").status_code == 200 + + assert stopped.wait(timeout=1.0) + + +def test_build_platform_app_rejects_controller_sidecar_name_collision(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_runner_discovery( + monkeypatch, + controllers={"adapters": _dummy_sidecar}, + sidecars={"adapters": _dummy_sidecar}, + ) + + with pytest.raises(ValueError, match="Controller/sidecar name collision: adapters"): + server.build_platform_app( + runner_config.PlatformAppConfig(controllers=["adapters"], sidecars=["adapters"]), + env={}, + ) + + +def test_real_adapters_sidecar_entrypoint_starts_and_stops_with_required_env( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + from nmp.core.models.sidecars.adapters import main as adapters_main + + started = threading.Event() + stopped = threading.Event() + manager = MagicMock() + + class FakeLoop: + def __init__(self, *_args: object, **_kwargs: object) -> None: + pass + + def start(self) -> None: + started.set() + + def stop(self) -> None: + stopped.set() + + def join(self) -> None: + return None + + lora_dir = tmp_path / "loras" + monkeypatch.setenv("NIM_PEFT_SOURCE", str(lora_dir)) + monkeypatch.setenv("NMP_MODEL_ENTITY_WORKSPACE", "default") + monkeypatch.setenv("NMP_MODEL_ENTITY_NAME", "test-model") + monkeypatch.setenv("NIM_PEFT_REFRESH_INTERVAL", "30") + monkeypatch.delenv("VLLM_ENDPOINT", raising=False) + + monkeypatch.setattr(adapters_main, "get_platform_config", lambda: MagicMock(base_url="http://platform.local")) + monkeypatch.setattr(adapters_main, "get_platform_sdk", lambda **_kwargs: MagicMock()) + monkeypatch.setattr(adapters_main.asyncio, "new_event_loop", lambda: MagicMock()) + monkeypatch.setattr(adapters_main, "Loop", FakeLoop) + monkeypatch.setattr(adapters_main, "TimedLoopWaiter", lambda *_args, **_kwargs: object()) + monkeypatch.setattr(adapters_main.ControllerManager, "get_instance", classmethod(lambda _cls: manager)) + + stop_signal = threading.Event() + thread = threading.Thread(target=adapters_main.run, args=(stop_signal,), daemon=True) + try: + thread.start() + + assert started.wait(timeout=1.0) + manager.register.assert_called_once() + assert manager.register.call_args.args[0] == "adapters_controller" + + stop_signal.set() + thread.join(timeout=1.0) + + assert not thread.is_alive() + assert stopped.is_set() + finally: + stop_signal.set() + thread.join(timeout=1.0) + adapters_main.adapters_controller_monitored = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py index 4df852bb8e..8a7e3022bd 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/cli.py @@ -13,7 +13,8 @@ import httpx import typer -from nemo_platform.cli.commands.services._process import ( +from nemo_platform.cli.core.help_formatter import create_typer_app +from nemo_platform.local.process import ( ForegroundInstanceError, InstanceAlreadyRunningError, InstanceDescriptor, @@ -24,7 +25,6 @@ check_port_available_for_start, compute_scope, format_port_conflict, - get_create_time, instance_log_bytes, is_instance_alive, list_instances, @@ -37,7 +37,7 @@ stop_instance, write_descriptor, ) -from nemo_platform.cli.core.help_formatter import create_typer_app +from nmp.platform_runner.config import DEFAULT_LOCAL_SERVICES_BIND_HOST, PlatformAppConfig logger = logging.getLogger(__name__) @@ -45,7 +45,6 @@ _HEALTH_TIMEOUT_SECONDS = 60 _HEALTH_POLL_INTERVAL = 2.0 -_DEFAULT_HOST = "127.0.0.1" _DEFAULT_PORT = 8080 _DEFAULT_STOP_TIMEOUT = 30.0 @@ -60,7 +59,7 @@ def services_callback(ctx: typer.Context) -> None: for info in running: desc = info.descriptor assert desc is not None - typer.echo(f"\nRunning: {info.scope} (pid {desc.pid}, {desc.host}:{desc.port}, {desc.mode})") + typer.echo(f"\nRunning: {info.scope} (pid {desc.pid}, {desc.config.host}:{desc.config.port}, {desc.mode})") def _require_services_extra() -> None: @@ -92,7 +91,7 @@ def _parse_csv_option(value: str | None) -> list[str] | None: def _wait_for_healthy( host: str, port: int, - timeout: int = _HEALTH_TIMEOUT_SECONDS, + timeout: float = _HEALTH_TIMEOUT_SECONDS, poll_interval: float = _HEALTH_POLL_INTERVAL, ) -> bool: """Poll the platform status endpoint until it responds or timeout.""" @@ -124,15 +123,8 @@ def _effective_base_dir() -> str | None: def _find_sole_running_scope(base_dir: Path | None) -> str: - """Find the scope of the single running instance for this working directory. - - When the user runs ``restart`` without ``--instance`` or ``--port``, we - can't know which scope to target because the scope includes the port. - This function scans all running instances whose scope starts with the - same git-root hash prefix. If exactly one matches, return it. - Otherwise fall back to the default scope (hash-DEFAULT_PORT). - """ - prefix = compute_scope(port=0, instance_name=None).rsplit("-", 1)[0] + """Return the only running scope for this working directory, or the default scope.""" + prefix = compute_scope(port=0).rsplit("-", 1)[0] running = [i for i in list_instances(base_dir=base_dir) if i.alive and i.scope.startswith(prefix + "-")] if len(running) == 1: return running[0].scope @@ -213,7 +205,7 @@ def run_services( str | None, typer.Option("--config", help="Path to a platform configuration YAML file."), ] = None, - host: Annotated[str, typer.Option("--host", help="Host to bind to.")] = _DEFAULT_HOST, + host: Annotated[str, typer.Option("--host", help="Host to bind to.")] = DEFAULT_LOCAL_SERVICES_BIND_HOST, port: Annotated[int, typer.Option("--port", help="Port to bind to.")] = _DEFAULT_PORT, instance: Annotated[ str | None, @@ -226,7 +218,7 @@ def run_services( _require_services_extra() _warn_bind_all(host) - scope = compute_scope(port=port, instance_name=instance) + scope = compute_scope(port=port, explicit_scope=instance) base_dir_str = _effective_base_dir() base_dir = Path(base_dir_str) if base_dir_str else None @@ -242,20 +234,23 @@ def run_services( # "foreground", which protects interactive ``run`` sessions from being # killed by ``stop``. mode = "background" if os.environ.get("_NMP_LAUNCH_MODE") == "background" else "foreground" - - desc = InstanceDescriptor( - pid=os.getpid(), - scope=scope, - host=host, - port=port, - mode=mode, - create_time=get_create_time(os.getpid()), + platform_config = PlatformAppConfig( services=_parse_csv_option(services), - controllers=_parse_csv_option(controllers), service_group=service_group, + controllers=_parse_csv_option(controllers), controller_group=controller_group, sidecars=_parse_csv_option(sidecars), config_path=config, + scope=scope, + host=host, + port=port, + state_root=base_dir, + ) + + desc = InstanceDescriptor.from_config( + platform_config, + mode=mode, + pid=os.getpid(), ) write_descriptor(desc, base_dir=base_dir) @@ -269,14 +264,7 @@ def _cleanup() -> None: from nmp.platform_runner.run import run_platform run_platform( - services=_parse_csv_option(services), - service_group=service_group, - controllers=_parse_csv_option(controllers), - controller_group=controller_group, - sidecars=_parse_csv_option(sidecars), - config_path=config, - host=host, - port=port, + config=platform_config, on_shutdown=_cleanup, ) @@ -327,7 +315,7 @@ def start_services( str | None, typer.Option("--config", help="Path to a platform configuration YAML file."), ] = None, - host: Annotated[str, typer.Option("--host", help="Host to bind to.")] = _DEFAULT_HOST, + host: Annotated[str, typer.Option("--host", help="Host to bind to.")] = DEFAULT_LOCAL_SERVICES_BIND_HOST, port: Annotated[int, typer.Option("--port", help="Port to bind to.")] = _DEFAULT_PORT, instance: Annotated[ str | None, @@ -351,7 +339,7 @@ def start_services( raise typer.BadParameter("Cannot combine --controllers with --controller-group.") _warn_bind_all(host) - scope = compute_scope(port=port, instance_name=instance) + scope = compute_scope(port=port, explicit_scope=instance) base_dir_str = _effective_base_dir() base_dir = Path(base_dir_str) if base_dir_str else None @@ -360,20 +348,22 @@ def start_services( _ensure_port_available(host, port, scope, base_dir=base_dir) - typer.echo("Starting platform services...") - proc = start_background( - scope=scope, + platform_config = PlatformAppConfig( services=_parse_csv_option(services), service_group=service_group, controllers=_parse_csv_option(controllers), controller_group=controller_group, sidecars=_parse_csv_option(sidecars), config_path=config, + scope=scope, host=host, port=port, - base_dir=base_dir, + state_root=base_dir, ) + typer.echo("Starting platform services...") + proc = start_background(platform_config) + if not _wait_for_healthy(host, port): exit_code = proc.poll() if exit_code is not None: @@ -426,7 +416,7 @@ def stop_services_cmd( nemo services stop nemo services stop --timeout 60 """ - scope = compute_scope(port=port, instance_name=instance) + scope = compute_scope(port=port, explicit_scope=instance) base_dir_str = _effective_base_dir() base_dir = Path(base_dir_str) if base_dir_str else None @@ -496,7 +486,10 @@ def restart_services( ] = None, host: Annotated[ str | None, - typer.Option("--host", help="Host to bind to. Defaults to previous value or 127.0.0.1."), + typer.Option( + "--host", + help=f"Host to bind to. Defaults to previous value or {DEFAULT_LOCAL_SERVICES_BIND_HOST}.", + ), ] = None, port: Annotated[ int | None, @@ -529,8 +522,8 @@ def restart_services( base_dir = Path(base_dir_str) if base_dir_str else None if instance is not None or port is not None: - effective_port = port if port is not None else _DEFAULT_PORT - scope = compute_scope(port=effective_port, instance_name=instance) + effective_scope_port = port if port is not None else _DEFAULT_PORT + scope = compute_scope(port=effective_scope_port, explicit_scope=instance) else: scope = _find_sole_running_scope(base_dir) @@ -547,37 +540,47 @@ def restart_services( # appropriate even for foreground targets. stop_instance(scope, base_dir=base_dir, force=True) - effective_services = _parse_csv_option(services) if services is not None else (prev.services if prev else None) - effective_service_group = service_group if service_group is not None else (prev.service_group if prev else None) - effective_controllers = ( - _parse_csv_option(controllers) if controllers is not None else (prev.controllers if prev else None) + previous_config = prev.config if prev else None + effective_services = _parse_csv_option(services) if services is not None else None + if services is None and previous_config is not None: + effective_services = previous_config.services + effective_service_group = service_group if service_group is not None else None + if service_group is None and previous_config is not None: + effective_service_group = previous_config.service_group + effective_controllers = _parse_csv_option(controllers) if controllers is not None else None + if controllers is None and previous_config is not None: + effective_controllers = previous_config.controllers + effective_controller_group = controller_group if controller_group is not None else None + if controller_group is None and previous_config is not None: + effective_controller_group = previous_config.controller_group + effective_sidecars = _parse_csv_option(sidecars) if sidecars is not None else None + if sidecars is None and previous_config is not None: + effective_sidecars = previous_config.sidecars + effective_config = config if config is not None else (previous_config.config_path if previous_config else None) + effective_host = ( + host if host is not None else (previous_config.host if previous_config else DEFAULT_LOCAL_SERVICES_BIND_HOST) ) - effective_controller_group = ( - controller_group if controller_group is not None else (prev.controller_group if prev else None) - ) - effective_sidecars = _parse_csv_option(sidecars) if sidecars is not None else (prev.sidecars if prev else None) - effective_config = config if config is not None else (prev.config_path if prev else None) - effective_host = host if host is not None else (prev.host if prev else _DEFAULT_HOST) - effective_port = port if port is not None else (prev.port if prev else _DEFAULT_PORT) + effective_port = port if port is not None else (previous_config.port if previous_config else _DEFAULT_PORT) _warn_bind_all(effective_host) _ensure_port_available(effective_host, effective_port, scope, base_dir=base_dir) - - typer.echo("Starting platform services...") - proc = start_background( - scope=scope, + platform_config = PlatformAppConfig( services=effective_services, service_group=effective_service_group, controllers=effective_controllers, controller_group=effective_controller_group, sidecars=effective_sidecars, config_path=effective_config, + scope=scope, host=effective_host, port=effective_port, - base_dir=base_dir, + state_root=base_dir, ) + typer.echo("Starting platform services...") + proc = start_background(platform_config) + if not _wait_for_healthy(effective_host, effective_port): exit_code = proc.poll() if exit_code is not None: @@ -613,7 +616,7 @@ def status_services( ] = _DEFAULT_PORT, ) -> None: """Show status of the platform services instance for this scope.""" - scope = compute_scope(port=port, instance_name=instance) + scope = compute_scope(port=port, explicit_scope=instance) base_dir_str = _effective_base_dir() base_dir = Path(base_dir_str) if base_dir_str else None @@ -641,13 +644,13 @@ def status_services( except ValueError: uptime = "unknown" - healthy = _wait_for_healthy(desc.host, desc.port, timeout=3, poll_interval=0.5) + healthy = _wait_for_healthy(desc.config.host, desc.config.port, timeout=3, poll_interval=0.5) health_str = "healthy" if healthy else "unhealthy" - typer.echo(f"Scope: {desc.scope}") + typer.echo(f"Scope: {desc.config.scope}") typer.echo(f"PID: {desc.pid}") typer.echo(f"Mode: {desc.mode}") - typer.echo(f"Address: {desc.host}:{desc.port}") + typer.echo(f"Address: {desc.config.host}:{desc.config.port}") typer.echo(f"Uptime: {uptime}") typer.echo(f"Health: {health_str}") log = log_path_for(scope, base_dir=base_dir) @@ -668,7 +671,7 @@ def _print_instance_table(instances: list[InstanceInfo]) -> None: pid = addr = mode = "-" if info.descriptor: pid = str(info.descriptor.pid) - addr = f"{info.descriptor.host}:{info.descriptor.port}" + addr = f"{info.descriptor.config.host}:{info.descriptor.config.port}" mode = info.descriptor.mode typer.echo(f"{info.scope:<25} {status:<10} {pid:<10} {addr:<25} {mode:<12}") @@ -872,7 +875,7 @@ def logs_services( nemo services logs --path nemo services logs -n 100 """ - scope = compute_scope(port=port, instance_name=instance) + scope = compute_scope(port=port, explicit_scope=instance) base_dir_str = _effective_base_dir() base_dir = Path(base_dir_str) if base_dir_str else None diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py index 29081aa867..78849e4181 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/setup.py @@ -30,20 +30,12 @@ from nemo_platform_plugin.secrets.client import SecretsClient from nemo_platform_plugin.secrets.types import PlatformSecretCreateRequest, PlatformSecretUpdateRequest from nmp.common.config import nmp_user_data_dir +from nmp.platform_runner.config import DEFAULT_LOCAL_SERVICES_BIND_HOST, PlatformAppConfig from pydantic import SecretStr from rich import box from rich.console import Console from rich.panel import Panel -from nemo_platform.cli.commands.services._process import ( - DEFAULT_SERVICES_BIND_HOST, - check_port_available_for_start, - compute_scope, - format_port_conflict, - log_path_for, - start_background, - stop_instance, -) from nemo_platform.cli.commands.skills import registry as skills_registry from nemo_platform.cli.commands.skills.base import Scope, Skill from nemo_platform.cli.commands.skills.registry import get_installer, load_skills @@ -51,6 +43,14 @@ from nemo_platform.cli.core.errors import handle_errors from nemo_platform.config.config import Config from nemo_platform.config.models import ConfigFile, ConfigParams, LocalServicesConfig +from nemo_platform.local.process import ( + check_port_available_for_start, + compute_scope, + format_port_conflict, + log_path_for, + start_background, + stop_instance, +) from nemo_platform.ui.prompts import ( UserCancelled, is_interactive, @@ -635,8 +635,10 @@ def _start_services_background(base_url: str, data_dir: str | None = None) -> su exported it). """ port = _resolve_services_port(base_url) - scope = compute_scope(port=port) - return start_background(scope=scope, port=port, data_dir=data_dir) + return start_background( + PlatformAppConfig(scope=compute_scope(port=port), port=port), + data_dir=data_dir, + ) def _last_startup_service(log_path: Path | None) -> str: @@ -686,16 +688,13 @@ def _kill_existing_services(base_url: str) -> None: Delegates to the shared process lifecycle module. """ - port = _resolve_services_port(base_url) - scope = compute_scope(port=port) - stop_instance(scope, timeout=2.0, force=True) + stop_instance(compute_scope(port=_resolve_services_port(base_url)), timeout=2.0, force=True) def _ensure_port_available_for_start(base_url: str) -> None: """Fail fast when the services port cannot be bound.""" port = _resolve_services_port(base_url) - scope = compute_scope(port=port) - conflict = check_port_available_for_start(DEFAULT_SERVICES_BIND_HOST, port, scope) + conflict = check_port_available_for_start(DEFAULT_LOCAL_SERVICES_BIND_HOST, port, compute_scope(port=port)) if conflict is None: return lines = format_port_conflict(conflict) @@ -777,8 +776,7 @@ def _maybe_start_services( _ensure_port_available_for_start(base_url) proc = _start_services_background(base_url, data_dir=data_dir) - port = _resolve_services_port(base_url) - log = log_path_for(compute_scope(port=port)) + log = log_path_for(compute_scope(port=_resolve_services_port(base_url))) if not _wait_for_platform(base_url, timeout=timeout, log_path=log): exit_code = proc.poll() diff --git a/sdk/python/nemo-platform/src/nemo_platform/local/_service_child.py b/sdk/python/nemo-platform/src/nemo_platform/local/_service_child.py new file mode 100644 index 0000000000..1ff27df811 --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/local/_service_child.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Child entrypoint for SDK-started local services daemons.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from nemo_platform.local.services import ServiceRunConfig, run_services + + +def main(argv: list[str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + if len(args) != 1: + sys.stderr.write("usage: python -m nemo_platform.local._service_child \n") + return 2 + request_path = Path(args[0]) + try: + payload = json.loads(request_path.read_text(encoding="utf-8")) + finally: + request_path.unlink(missing_ok=True) + run_services(ServiceRunConfig(**payload), _mode="daemon") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/_process.py b/sdk/python/nemo-platform/src/nemo_platform/local/process.py similarity index 76% rename from sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/_process.py rename to sdk/python/nemo-platform/src/nemo_platform/local/process.py index 8ce8cda699..7ee0cd4a9b 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/services/_process.py +++ b/sdk/python/nemo-platform/src/nemo_platform/local/process.py @@ -3,8 +3,13 @@ """Local process lifecycle for ``nemo services``. -Uses per-instance scoped directories under ``$XDG_STATE_HOME/nmp/instances/`` -with flock-based liveness tracking. Each instance directory contains: +In this module, "instance" is a local services process/resource, and "scope" +is the stable key used for that instance's lock, descriptor, socket, and log +paths. The CLI exposes this key as ``--instance`` for compatibility, but +internal code should use "scope" when referring to the key. + +Uses per-scope directories under ``$XDG_STATE_HOME/nmp/instances/`` +with flock-based liveness tracking. Each scope directory contains: - ``services.lock`` -- exclusive flock held for the process lifetime - ``instance.json`` -- descriptor with PID, port, services, etc. @@ -23,7 +28,6 @@ import json import logging import os -import re import shutil import signal import socket @@ -34,10 +38,16 @@ from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path -from typing import Literal +from typing import Literal, Self import psutil -from pydantic import BaseModel, Field +from nmp.platform_runner.config import ( + DEFAULT_LOCAL_SERVICES_BIND_HOST, + PlatformAppConfig, + default_state_root, + validate_scope, +) +from pydantic import BaseModel, Field, model_validator logger = logging.getLogger(__name__) @@ -45,10 +55,10 @@ DESCRIPTOR_FILENAME = "instance.json" LOG_FILENAME = "services.log" -DEFAULT_SERVICES_BIND_HOST = "127.0.0.1" SUGGESTED_ALT_PORT = 9090 _SIGTERM_POLL_INTERVAL = 0.25 +_SIGKILL_WAIT_TIMEOUT = 5.0 _DEFAULT_STOP_TIMEOUT = 30.0 @@ -62,10 +72,7 @@ def _pause(seconds: float) -> None: def _base_state_dir() -> Path: - xdg = os.environ.get("XDG_STATE_HOME") - if xdg: - return Path(xdg) / "nmp" - return Path.home() / ".local" / "state" / "nmp" + return default_state_root() def _instances_dir(*, base_dir: Path | None = None) -> Path: @@ -73,7 +80,7 @@ def _instances_dir(*, base_dir: Path | None = None) -> Path: def _find_git_root() -> str: - """Walk up from cwd looking for a ``.git`` directory. Falls back to cwd.""" + """Walk up from cwd looking for a ``.git`` directory. Falls back to cwd.""" cur = Path.cwd().resolve() for parent in (cur, *cur.parents): if (parent / ".git").exists(): @@ -84,24 +91,19 @@ def _find_git_root() -> str: _scope_prefix_cache: str | None = None -_SCOPE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") - - -def _validate_scope(scope: str) -> str: - """Ensure *scope* is safe to use as a directory name.""" - if not _SCOPE_RE.fullmatch(scope): - raise ValueError(f"Invalid instance scope: {scope!r}") - return scope +def compute_scope(*, port: int, explicit_scope: str | None = None) -> str: + """Compute the local services scope. + The default scope is ``sha1(git_toplevel_or_cwd)[:8]-``. Including + the port is intentional: it lets two local services instances from the same + checkout use different TCP ports without sharing a lock, descriptor, or log + directory. -def compute_scope(*, port: int, instance_name: str | None = None) -> str: - """Compute a scope identifier for this working directory + port. - - Default: ``sha1(git_toplevel_or_cwd)[:8]-``. - Override with an explicit *instance_name*. + Explicit scopes are validated and returned as-is, so they do not encode the + port. Callers that pass an explicit scope own its uniqueness. """ - if instance_name: - return _validate_scope(instance_name) + if explicit_scope: + return validate_scope(explicit_scope) global _scope_prefix_cache # noqa: PLW0603 if _scope_prefix_cache is None: root = _find_git_root() @@ -110,7 +112,8 @@ def compute_scope(*, port: int, instance_name: str | None = None) -> str: def instance_dir(scope: str, *, base_dir: Path | None = None) -> Path: - d = _instances_dir(base_dir=base_dir) / _validate_scope(scope) + """Return the state directory for *scope*, creating it if needed.""" + d = _instances_dir(base_dir=base_dir) / validate_scope(scope) d.mkdir(parents=True, exist_ok=True) return d @@ -222,7 +225,7 @@ def _instance_owns_listener( desc = read_descriptor(scope, base_dir=base_dir) if desc is None: return False - return desc.port == port and _normalize_bind_host(desc.host) == _normalize_bind_host(host) + return desc.config.port == port and _normalize_bind_host(desc.config.host) == _normalize_bind_host(host) def is_port_bindable(host: str, port: int) -> bool: @@ -272,8 +275,9 @@ def format_port_conflict(err: PortConflict) -> list[str]: Message text depends on ``err.kind`` (foreign process vs NeMo instance). """ if err.kind == "nemo_instance": + owner = f" '{err.scope}'" if err.scope else "" return [ - f"Port {err.port} is in use by a NeMo Platform instance for this directory.", + f"Port {err.port} is in use by NeMo Platform instance{owner}.", "Stop it first with: nemo services stop", "Or restart with: nemo services restart", ] @@ -292,23 +296,39 @@ def format_port_conflict(err: PortConflict) -> list[str]: class InstanceDescriptor(BaseModel): pid: int - scope: str - host: str = "127.0.0.1" - port: int = 8080 - mode: Literal["foreground", "background"] = "background" + config: PlatformAppConfig = Field(default_factory=PlatformAppConfig) + transport: Literal["tcp", "uds"] = "tcp" + mode: Literal["foreground", "background", "daemon"] = "background" create_time: float = 0.0 started_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) - services: list[str] | None = None - controllers: list[str] | None = None - service_group: str | None = None - controller_group: str | None = None - sidecars: list[str] | None = None - config_path: str | None = None - log_path: str | None = None + + @model_validator(mode="after") + def _validate_client_transport(self) -> Self: + if self.transport == "uds" and self.config.socket_path is None: + raise ValueError("UDS client transport requires config.socket_path") + return self + + @classmethod + def from_config( + cls, + config: PlatformAppConfig, + *, + mode: Literal["foreground", "background", "daemon"], + transport: Literal["uds", "tcp"] = "tcp", + pid: int | None = None, + ) -> Self: + resolved_pid = os.getpid() if pid is None else pid + return cls( + pid=resolved_pid, + config=config, + transport=transport, + mode=mode, + create_time=get_create_time(resolved_pid), + ) def write_descriptor(desc: InstanceDescriptor, *, base_dir: Path | None = None) -> Path: - d = instance_dir(desc.scope, base_dir=base_dir) + d = instance_dir(desc.config.scope, base_dir=base_dir) path = d / DESCRIPTOR_FILENAME payload = desc.model_dump() fd, tmp = tempfile.mkstemp(dir=str(d), suffix=".tmp") @@ -335,10 +355,19 @@ def read_descriptor(scope: str, *, base_dir: Path | None = None) -> InstanceDesc return None try: data = json.loads(path.read_text()) - return InstanceDescriptor.model_validate(data) + desc = InstanceDescriptor.model_validate(data) except (json.JSONDecodeError, KeyError, TypeError, ValueError): logger.debug("Corrupt descriptor at %s, ignoring", path, exc_info=True) return None + if desc.config.scope != scope: + logger.debug( + "Descriptor at %s has scope=%r but lives under %r, ignoring", + path, + desc.config.scope, + scope, + ) + return None + return desc def remove_descriptor(scope: str, *, base_dir: Path | None = None) -> None: @@ -351,24 +380,24 @@ def remove_descriptor(scope: str, *, base_dir: Path | None = None) -> None: def _scope_dir(scope: str, *, base_dir: Path | None = None) -> Path: - return _instances_dir(base_dir=base_dir) / _validate_scope(scope) + return _instances_dir(base_dir=base_dir) / validate_scope(scope) def _is_log_file(path: Path) -> bool: return path.name == LOG_FILENAME or path.name.startswith(f"{LOG_FILENAME}.") -def _iter_log_files(scope_dir: Path): - if not scope_dir.is_dir(): +def _iter_log_files(scope_dir_path: Path): + if not scope_dir_path.is_dir(): return - for path in scope_dir.iterdir(): + for path in scope_dir_path.iterdir(): if path.is_file() and _is_log_file(path): yield path -def _has_preservable_logs(scope_dir: Path) -> bool: - """Return True if *scope_dir* contains non-empty service log files.""" - return any(path.stat().st_size > 0 for path in _iter_log_files(scope_dir)) +def _has_preservable_logs(scope_dir_path: Path) -> bool: + """Return True if *scope_dir_path* contains non-empty service log files.""" + return any(path.stat().st_size > 0 for path in _iter_log_files(scope_dir_path)) def is_removable_ghost( @@ -377,17 +406,17 @@ def is_removable_ghost( base_dir: Path | None = None, descriptor: InstanceDescriptor | None = None, ) -> bool: - """True when a dead scope dir has no descriptor and no non-empty logs.""" + """True when a dead scope directory has no descriptor and no non-empty logs.""" if is_instance_alive(scope, base_dir=base_dir): return False if descriptor is not None: return False - scope_dir = _scope_dir(scope, base_dir=base_dir) - if not scope_dir.is_dir(): + scope_dir_path = _scope_dir(scope, base_dir=base_dir) + if not scope_dir_path.is_dir(): return False - if (scope_dir / DESCRIPTOR_FILENAME).exists(): + if (scope_dir_path / DESCRIPTOR_FILENAME).exists(): return False - return not _has_preservable_logs(scope_dir) + return not _has_preservable_logs(scope_dir_path) # --------------------------------------------------------------------------- @@ -425,7 +454,7 @@ class InstanceInfo: def list_instances(*, base_dir: Path | None = None) -> list[InstanceInfo]: - """Scan all instance directories and return their status. + """Scan all scope directories and return their status. Side effects: - Removes stale descriptors for dead instances. @@ -448,7 +477,7 @@ def list_instances(*, base_dir: Path | None = None) -> list[InstanceInfo]: try: shutil.rmtree(child) except OSError: - logger.debug("Could not remove ghost instance dir %s", child, exc_info=True) + logger.debug("Could not remove ghost scope directory %s", child, exc_info=True) else: continue results.append(InstanceInfo(scope=scope, alive=alive, descriptor=desc)) @@ -456,28 +485,28 @@ def list_instances(*, base_dir: Path | None = None) -> list[InstanceInfo]: def remove_instance(scope: str, *, base_dir: Path | None = None) -> bool: - """Remove an instance scope directory. + """Remove a scope directory. - Returns False if the scope did not exist or could not be removed. + Returns False if the scope directory did not exist or could not be removed. """ - scope = _validate_scope(scope) + scope = validate_scope(scope) if is_instance_alive(scope, base_dir=base_dir): raise InstanceStillRunningError(scope) - scope_dir = _scope_dir(scope, base_dir=base_dir) - if not scope_dir.is_dir(): + scope_dir_path = _scope_dir(scope, base_dir=base_dir) + if not scope_dir_path.is_dir(): return False with contextlib.suppress(OSError): - shutil.rmtree(scope_dir) - return not scope_dir.is_dir() + shutil.rmtree(scope_dir_path) + return not scope_dir_path.is_dir() def list_stopped_scopes(*, base_dir: Path | None = None) -> list[str]: - """Return scope names for instances that are not alive.""" + """Return scopes for instances that are not alive.""" return [info.scope for info in list_instances(base_dir=base_dir) if not info.alive] def prune_instances(*, base_dir: Path | None = None) -> list[str]: - """Remove all stopped instance directories. Returns removed scope names.""" + """Remove all stopped scope directories. Returns removed scopes.""" removed: list[str] = [] for scope in list_stopped_scopes(base_dir=base_dir): if remove_instance(scope, base_dir=base_dir): @@ -497,11 +526,15 @@ def instance_log_bytes(scope: str, *, base_dir: Path | None = None) -> int: def rotate_log(scope: str, *, base_dir: Path | None = None) -> Path: """Rotate the existing log and return the path for the new one.""" - d = instance_dir(scope, base_dir=base_dir) - log_path = d / LOG_FILENAME + return rotate_log_path(log_path_for(scope, base_dir=base_dir)) + + +def rotate_log_path(log_path: Path) -> Path: + """Rotate the existing log at *log_path* and return the path for the new one.""" + log_path.parent.mkdir(parents=True, exist_ok=True) if log_path.exists() and log_path.stat().st_size > 0: ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") - rotated = d / f"{LOG_FILENAME}.{ts}" + rotated = log_path.with_name(f"{log_path.name}.{ts}") log_path.rename(rotated) return log_path @@ -633,6 +666,10 @@ def stop_instance( return StopResult(stopped_pids=[], swept_children=swept) except OSError: logger.debug("Failed to send SIGKILL to pid %d", pid, exc_info=True) + if not _wait_for_pid_exit(pid, timeout=_SIGKILL_WAIT_TIMEOUT): + logger.warning("PID %d is still alive after SIGKILL; preserving descriptor", pid) + swept = _sweep_orphans(children) if children else [] + return StopResult(stopped_pids=[], swept_children=swept) swept = _sweep_orphans(children) if children else [] @@ -640,13 +677,21 @@ def stop_instance( return StopResult(stopped_pids=[pid], swept_children=swept) +def _wait_for_pid_exit(pid: int, *, timeout: float) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not _pid_alive(pid): + return True + _pause(_SIGTERM_POLL_INTERVAL) + return not _pid_alive(pid) + + def _pid_alive(pid: int) -> bool: try: - os.kill(pid, 0) - return True - except ProcessLookupError: + return psutil.Process(pid).status() != psutil.STATUS_ZOMBIE + except psutil.NoSuchProcess: return False - except PermissionError: + except psutil.AccessDenied: return True except OSError: return False @@ -658,17 +703,8 @@ def _pid_alive(pid: int) -> bool: def start_background( + config: PlatformAppConfig | None = None, *, - scope: str, - services: list[str] | None = None, - service_group: str | None = None, - controllers: list[str] | None = None, - controller_group: str | None = None, - sidecars: list[str] | None = None, - config_path: str | None = None, - host: str = DEFAULT_SERVICES_BIND_HOST, - port: int = 8080, - base_dir: Path | None = None, data_dir: str | None = None, ) -> subprocess.Popen: """Launch ``nemo services run`` as a detached background subprocess. @@ -676,31 +712,32 @@ def start_background( The child acquires the flock and writes its own descriptor. The parent returns the ``Popen`` handle for health polling. """ - log_file_path = rotate_log(scope, base_dir=base_dir) + config = config or PlatformAppConfig(host=DEFAULT_LOCAL_SERVICES_BIND_HOST) + log_file_path = rotate_log_path(config.log_file_path()) log_file = open(log_file_path, "a") # noqa: SIM115 nemo_bin = str(Path(sys.executable).parent / "nemo") args: list[str] = [nemo_bin, "services", "run"] - if services: - args += ["--services", ",".join(services)] - if service_group: - args += ["--service-group", service_group] - if controllers: - args += ["--controllers", ",".join(controllers)] - if controller_group: - args += ["--controller-group", controller_group] - if sidecars: - args += ["--sidecars", ",".join(sidecars)] - if config_path: - args += ["--config", config_path] - args += ["--host", host, "--port", str(port)] - args += ["--instance", scope] + if config.services: + args += ["--services", ",".join(config.services)] + if config.service_group: + args += ["--service-group", config.service_group] + if config.controllers: + args += ["--controllers", ",".join(config.controllers)] + if config.controller_group: + args += ["--controller-group", config.controller_group] + if config.sidecars: + args += ["--sidecars", ",".join(config.sidecars)] + if config.config_path: + args += ["--config", config.config_path] + args += ["--host", config.host, "--port", str(config.port)] + args += ["--instance", config.scope] env = os.environ.copy() if data_dir and "NMP_DATA_DIR" not in env: env["NMP_DATA_DIR"] = data_dir - if base_dir: - env["_NMP_STATE_DIR"] = str(base_dir) + if config.state_root is not None: + env["_NMP_STATE_DIR"] = str(config.state_root) # Tell the child ``run`` process it was launched by ``start`` so it # records mode="background" in its descriptor. This is internal # parent-to-child signaling -- not a public API surface -- following the diff --git a/sdk/python/nemo-platform/src/nemo_platform/local/services.py b/sdk/python/nemo-platform/src/nemo_platform/local/services.py new file mode 100644 index 0000000000..bd904c9cfd --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/local/services.py @@ -0,0 +1,728 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Programmatic local lifecycle API for NeMo Platform services.""" + +from __future__ import annotations + +import asyncio +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import time +from collections.abc import MutableMapping, Sequence +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path +from typing import Any, Literal, Protocol, Self, runtime_checkable + +from nemo_platform import AsyncNeMoPlatform, NeMoPlatform +from nemo_platform.local import process +from nemo_platform.local.transport import ( + EMBEDDED_BASE_URL, + UDS_BASE_URL, + build_async_asgi_http_client, + build_async_http_client, + build_sync_asgi_http_client, + build_sync_http_client, + probe_status, + tcp_base_url, + wait_for_status, + wait_for_status_async, +) +from nmp.platform_runner.config import ( + DEFAULT_SCOPE, + PlatformAppConfig, + default_runtime_root, + default_state_root, + validate_scope, +) + +_AF_UNIX_PATH_MAX_BYTES = 103 if sys.platform.startswith(("darwin", "freebsd", "openbsd", "netbsd")) else 107 + + +class ServicesError(RuntimeError): + """Base class for local services lifecycle errors.""" + + +class ServicesExtraRequiredError(ServicesError): + """Raised when local service dependencies are not installed.""" + + +class ServicesAlreadyRunningError(ServicesError): + """Raised when a requested local instance is already running.""" + + +class ServicesNotRunningError(ServicesError): + """Raised when a requested local instance is not running.""" + + +class ServicesPortInUseError(ServicesError): + """Raised when TCP startup targets an unavailable port.""" + + +class ServicesStartupTimeoutError(ServicesError): + """Raised when startup does not become healthy before the timeout.""" + + +class ServicesStartupExitedError(ServicesError): + """Raised when a daemon child exits before becoming healthy.""" + + +class ServicesSocketStaleError(ServicesError): + """Raised when a stale socket cannot be removed.""" + + +def _as_tuple(value: Sequence[str] | None) -> tuple[str, ...] | None: + if value is None: + return None + return tuple(value) + + +def _optional_str(value: str | Path | None) -> str | None: + if value is None: + return None + return str(value) + + +def _optional_list(value: Sequence[str] | None) -> list[str] | None: + if value is None: + return None + return list(value) + + +class ServiceMode(StrEnum): + EMBEDDED = "embedded" + DAEMON = "daemon" + + +@dataclass(frozen=True) +class StartServicesResult: + requested: list[str] + started: list[str] + already_active: list[str] + active: list[str] + + +@runtime_checkable +class LocalServiceHandle(Protocol): + """Shared lifecycle/client contract for local services handles.""" + + def is_running(self) -> bool: ... + + def wait_until_ready(self, timeout: float | None = None) -> None: ... + + async def wait_until_ready_async(self, timeout: float | None = None) -> None: ... + + def client(self, **kwargs: Any) -> NeMoPlatform: ... + + def async_client(self, **kwargs: Any) -> AsyncNeMoPlatform: ... + + def start_services(self, service_names: Sequence[str]) -> StartServicesResult: ... + + async def start_services_async(self, service_names: Sequence[str]) -> StartServicesResult: ... + + def stop(self, *, timeout: float = 30.0, force: bool = False) -> process.StopResult: ... + + async def stop_async(self, *, timeout: float = 30.0, force: bool = False) -> process.StopResult: ... + + +@dataclass +class ServiceRunConfig: + services: Sequence[str] | None = None + service_group: str | None = None + controllers: Sequence[str] | None = None + controller_group: str | None = None + sidecars: Sequence[str] | None = None + config_path: str | Path | None = None + transport: Literal["uds", "tcp"] = "uds" + socket_path: str | Path | None = None + http_gateway: Literal["enabled", "disabled"] = "disabled" + http_gateway_host: str = "127.0.0.1" + http_gateway_port: int | None = None + host: str = "127.0.0.1" + port: int = 8080 + scope: str = DEFAULT_SCOPE + state_dir: str | Path | None = None + runtime_dir: str | Path | None = None + data_dir: str | Path | None = None + readiness_timeout: float = 60.0 + readiness_poll_interval: float = 0.5 + mode: ServiceMode | str = ServiceMode.DAEMON + + def __post_init__(self) -> None: + self.services = _as_tuple(self.services) + self.controllers = _as_tuple(self.controllers) + self.sidecars = _as_tuple(self.sidecars) + try: + self.mode = ServiceMode(self.mode) + except ValueError as error: + raise ValueError("mode must be 'embedded' or 'daemon'") from error + + if self.services and self.service_group: + raise ValueError("services cannot be combined with service_group") + if self.controllers and self.controller_group: + raise ValueError("controllers cannot be combined with controller_group") + if self.transport not in {"uds", "tcp"}: + raise ValueError("transport must be 'uds' or 'tcp'") + if self.http_gateway not in {"enabled", "disabled"}: + raise ValueError("http_gateway must be 'enabled' or 'disabled'") + if self.http_gateway == "enabled" and self.transport != "uds": + raise ValueError("gateway can only be enabled for UDS transport") + if self.readiness_timeout <= 0: + raise ValueError("readiness_timeout must be greater than 0") + if self.readiness_poll_interval <= 0: + raise ValueError("readiness_poll_interval must be greater than 0") + self.scope = validate_scope(self.scope) + + @property + def state_root(self) -> Path: + return Path(self.state_dir).expanduser() if self.state_dir is not None else default_state_root() + + @property + def runtime_root(self) -> Path: + return Path(self.runtime_dir).expanduser() if self.runtime_dir is not None else default_runtime_root() + + @property + def resolved_socket_path(self) -> Path | None: + if self.socket_path is not None: + socket_path = Path(self.socket_path).expanduser() + elif self.transport == "uds": + socket_path = PlatformAppConfig( + scope=self.scope, + runtime_root=self.runtime_root, + ).socket_file_path() + else: + return None + if not socket_path.is_absolute(): + raise ValueError(f"UDS socket path must be absolute: {socket_path}") + return socket_path + + def to_platform_app_config(self) -> PlatformAppConfig: + return PlatformAppConfig( + services=self.services, + service_group=self.service_group, + controllers=self.controllers, + controller_group=self.controller_group, + sidecars=self.sidecars, + config_path=_optional_str(self.config_path), + scope=self.scope, + host=self.host, + port=self.port, + socket_path=_optional_str(self.resolved_socket_path), + state_root=_optional_str(self.state_root), + runtime_root=_optional_str(self.runtime_dir), + ) + + def to_child_payload(self) -> dict[str, object]: + return { + "mode": ServiceMode(self.mode).value, + "services": _optional_list(self.services), + "service_group": self.service_group, + "controllers": _optional_list(self.controllers), + "controller_group": self.controller_group, + "sidecars": _optional_list(self.sidecars), + "config_path": _optional_str(self.config_path), + "transport": self.transport, + "socket_path": _optional_str(self.socket_path), + "http_gateway": self.http_gateway, + "http_gateway_host": self.http_gateway_host, + "http_gateway_port": self.http_gateway_port, + "host": self.host, + "port": self.port, + "scope": self.scope, + "state_dir": _optional_str(self.state_dir), + "runtime_dir": _optional_str(self.runtime_dir), + "data_dir": _optional_str(self.data_dir), + "readiness_timeout": self.readiness_timeout, + "readiness_poll_interval": self.readiness_poll_interval, + } + + +@dataclass(frozen=True) +class DaemonServiceHandle: + scope: str + transport: Literal["uds", "tcp"] + socket_path: Path | None + gateway_base_url: str | None + host: str + port: int + pid: int | None + mode: Literal["foreground", "daemon"] + log_path: Path | None + state_dir: Path | None + runtime_dir: Path | None + + @classmethod + def from_descriptor(cls, desc: process.InstanceDescriptor) -> Self: + socket_path = Path(desc.config.socket_path) if desc.config.socket_path else None + runtime_dir = desc.config.runtime_dir() if socket_path else None + return cls( + scope=desc.config.scope, + transport=desc.transport, + socket_path=socket_path, + gateway_base_url=None, + host=desc.config.host, + port=desc.config.port, + pid=desc.pid, + mode="daemon" if desc.mode == "daemon" else "foreground", + log_path=desc.config.log_file_path(), + state_dir=desc.config.state_dir(), + runtime_dir=runtime_dir, + ) + + @classmethod + def from_config( + cls, + config: ServiceRunConfig, + *, + pid: int | None = None, + ) -> Self: + app_config = config.to_platform_app_config() + socket_path = config.resolved_socket_path + runtime_dir = app_config.runtime_dir() if socket_path else None + return cls( + scope=config.scope, + transport=config.transport, + socket_path=socket_path, + gateway_base_url=None, + host=config.host, + port=config.port, + pid=pid, + mode="daemon", + log_path=app_config.log_file_path(), + state_dir=app_config.state_dir(), + runtime_dir=runtime_dir, + ) + + @property + def base_url(self) -> str: + if self.transport == "uds": + return UDS_BASE_URL + return tcp_base_url(self.host, self.port) + + def _state_root(self) -> Path | None: + if self.state_dir is None: + return None + if self.state_dir.parent.name == "instances": + return self.state_dir.parent.parent + return self.state_dir.parent + + def is_running(self) -> bool: + state_root = self._state_root() + return process.is_instance_alive(self.scope, base_dir=state_root) + + def wait_until_ready(self, timeout: float | None = None) -> None: + if not wait_for_status( + base_url=self.base_url, + socket_path=self.socket_path if self.transport == "uds" else None, + timeout=60.0 if timeout is None else timeout, + ): + raise ServicesStartupTimeoutError(f"Timed out waiting for services instance {self.scope!r}") + + async def wait_until_ready_async(self, timeout: float | None = None) -> None: + if not await wait_for_status_async( + base_url=self.base_url, + socket_path=self.socket_path if self.transport == "uds" else None, + timeout=60.0 if timeout is None else timeout, + ): + raise ServicesStartupTimeoutError(f"Timed out waiting for services instance {self.scope!r}") + + def stop(self, *, timeout: float = 30.0, force: bool = False) -> process.StopResult: + state_root = self._state_root() + return process.stop_instance(self.scope, base_dir=state_root, timeout=timeout, force=force) + + async def stop_async(self, *, timeout: float = 30.0, force: bool = False) -> process.StopResult: + return await asyncio.to_thread(self.stop, timeout=timeout, force=force) + + def start_services(self, service_names: Sequence[str]) -> StartServicesResult: + raise ServicesError("Staged service start is not implemented for daemon mode yet") + + async def start_services_async(self, service_names: Sequence[str]) -> StartServicesResult: + return await asyncio.to_thread(self.start_services, service_names) + + def client(self, **kwargs: Any) -> NeMoPlatform: + if self.transport == "uds": + if self.socket_path is None: + raise ServicesError("UDS service handle is missing socket_path") + kwargs.setdefault("http_client", build_sync_http_client(self.socket_path)) + kwargs.setdefault("base_url", self.base_url) + return NeMoPlatform(**kwargs) + + def async_client(self, **kwargs: Any) -> AsyncNeMoPlatform: + if self.transport == "uds": + if self.socket_path is None: + raise ServicesError("UDS service handle is missing socket_path") + kwargs.setdefault("http_client", build_async_http_client(self.socket_path)) + kwargs.setdefault("base_url", self.base_url) + return AsyncNeMoPlatform(**kwargs) + + +@dataclass(frozen=True) +class EmbeddedServiceHandle: + app: Any + runtime: object + + def is_running(self) -> bool: + return True + + def wait_until_ready(self, timeout: float | None = None) -> None: + return None + + async def wait_until_ready_async(self, timeout: float | None = None) -> None: + return None + + def client(self, **kwargs: Any) -> NeMoPlatform: + kwargs.setdefault("http_client", build_sync_asgi_http_client(self.app)) + kwargs.setdefault("base_url", EMBEDDED_BASE_URL) + return NeMoPlatform(**kwargs) + + def async_client(self, **kwargs: Any) -> AsyncNeMoPlatform: + kwargs.setdefault("http_client", build_async_asgi_http_client(self.app)) + kwargs.setdefault("base_url", EMBEDDED_BASE_URL) + return AsyncNeMoPlatform(**kwargs) + + def start_services(self, service_names: Sequence[str]) -> StartServicesResult: + raise ServicesError("Staged service start is not implemented for embedded mode yet") + + async def start_services_async(self, service_names: Sequence[str]) -> StartServicesResult: + return await asyncio.to_thread(self.start_services, service_names) + + def stop(self, *, timeout: float = 30.0, force: bool = False) -> process.StopResult: + return process.StopResult(stopped_pids=[], swept_children=[]) + + async def stop_async(self, *, timeout: float = 30.0, force: bool = False) -> process.StopResult: + return self.stop(timeout=timeout, force=force) + + +def require_services_extra() -> None: + if importlib.util.find_spec("pyleak") is not None: + return + raise ServicesExtraRequiredError("Install service dependencies with `pip install 'nemo-platform[all]'`.") + + +def _validate_socket_path_length(socket_path: Path) -> None: + encoded_length = len(os.fsencode(socket_path)) + if encoded_length > _AF_UNIX_PATH_MAX_BYTES: + raise ValueError( + "UDS socket path is too long for AF_UNIX " + f"({encoded_length} bytes; maximum is {_AF_UNIX_PATH_MAX_BYTES} bytes): {socket_path}" + ) + + +def _validated_socket_path(config: ServiceRunConfig) -> Path | None: + socket_path = config.resolved_socket_path + if socket_path is None: + return None + _validate_socket_path_length(socket_path) + return socket_path + + +def _prepare_socket(config: ServiceRunConfig) -> Path | None: + socket_path = _validated_socket_path(config) + if socket_path is None: + return None + socket_path.parent.mkdir(parents=True, exist_ok=True) + if not socket_path.exists(): + return socket_path + if probe_status(base_url=UDS_BASE_URL, socket_path=socket_path, timeout=0.5): + raise ServicesAlreadyRunningError(f"UDS socket is live at {socket_path}") + try: + socket_path.unlink() + except OSError as error: + raise ServicesSocketStaleError(f"Could not remove stale socket at {socket_path}") from error + return socket_path + + +def _check_tcp_available(config: ServiceRunConfig) -> None: + conflict = process.check_port_available_for_start( + config.host, + config.port, + config.scope, + base_dir=config.state_root, + ) + if conflict is not None: + raise ServicesPortInUseError("\n".join(process.format_port_conflict(conflict))) + + +def _write_run_request(config: ServiceRunConfig) -> Path: + state_dir = config.to_platform_app_config().state_dir(create=True) + fd, tmp = tempfile.mkstemp(dir=state_dir, suffix=".json") + path = Path(tmp) + try: + with os.fdopen(fd, "w", encoding="utf-8") as file: + fd = -1 + json.dump(config.to_child_payload(), file, indent=2) + file.write("\n") + except BaseException: + if fd >= 0: + os.close(fd) + fd = -1 + path.unlink(missing_ok=True) + raise + finally: + if fd >= 0: + os.close(fd) + return path + + +def _terminate_startup_process(proc: subprocess.Popen) -> None: + if proc.poll() is not None: + return + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +def serve_embedded_app(app: Any, cfg: ServiceRunConfig, socket_path: Path | None) -> None: + import uvicorn + + if socket_path is not None: + from nmp.platform_runner.server import _run_server_on_bound_sockets + + _run_server_on_bound_sockets(app, host=cfg.host, port=cfg.port, socket_path=str(socket_path)) + else: + uvicorn.run(app, host=cfg.host, port=cfg.port, log_config=None) + + +def run_services( + config: ServiceRunConfig | None = None, + *, + _mode: Literal["foreground", "daemon"] = "foreground", + env: MutableMapping[str, str] | None = None, +) -> None: + cfg = config or ServiceRunConfig() + app_config = cfg.to_platform_app_config() + require_services_extra() + if cfg.http_gateway == "enabled": + raise ServicesError("HTTP gateway support is not implemented yet") + if process.is_instance_alive(cfg.scope, base_dir=cfg.state_root): + raise ServicesAlreadyRunningError(f"Instance {cfg.scope!r} is already running") + _check_tcp_available(cfg) + lock_fd = process.acquire_lock(cfg.scope, base_dir=cfg.state_root) + original_data_dir = os.environ.get("NMP_DATA_DIR") + try: + socket_path = _prepare_socket(cfg) + app_config.log_file_path(create_parent=True) + if cfg.data_dir is not None and "NMP_DATA_DIR" not in os.environ: + os.environ["NMP_DATA_DIR"] = str(cfg.data_dir) + desc = process.InstanceDescriptor.from_config( + app_config, + mode=_mode, + transport=cfg.transport, + ) + process.write_descriptor(desc, base_dir=cfg.state_root) + embedded_handle = start_embedded_services(cfg, env=env) + serve_embedded_app(embedded_handle.app, cfg, socket_path) + finally: + try: + process.remove_descriptor(cfg.scope, base_dir=cfg.state_root) + finally: + if original_data_dir is None: + os.environ.pop("NMP_DATA_DIR", None) + else: + os.environ["NMP_DATA_DIR"] = original_data_dir + os.close(lock_fd) + + +def daemonize_services(config: ServiceRunConfig | None = None) -> DaemonServiceHandle: + cfg = config or ServiceRunConfig() + app_config = cfg.to_platform_app_config() + require_services_extra() + if cfg.http_gateway == "enabled": + raise ServicesError("HTTP gateway support is not implemented yet") + if process.is_instance_alive(cfg.scope, base_dir=cfg.state_root): + raise ServicesAlreadyRunningError(f"Instance {cfg.scope!r} is already running") + _check_tcp_available(cfg) + socket_path = _validated_socket_path(cfg) + if ( + socket_path is not None + and socket_path.exists() + and probe_status(base_url=UDS_BASE_URL, socket_path=socket_path, timeout=0.5) + ): + raise ServicesAlreadyRunningError(f"UDS socket is live at {socket_path}") + + request_path = _write_run_request(cfg) + log_path = process.rotate_log_path(app_config.log_file_path()) + log_file = open(log_path, "a") # noqa: SIM115 + env = os.environ.copy() + if cfg.data_dir is not None and "NMP_DATA_DIR" not in env: + env["NMP_DATA_DIR"] = str(cfg.data_dir) + proc: subprocess.Popen | None = None + ownership_transferred = False + try: + try: + child_module = f"{__package__}._service_child" + proc = subprocess.Popen( + [sys.executable, "-m", child_module, str(request_path)], + stdout=log_file, + stderr=subprocess.STDOUT, + env=env, + start_new_session=True, + stdin=subprocess.DEVNULL, + close_fds=True, + ) + finally: + log_file.close() + assert proc is not None + handle = DaemonServiceHandle.from_config(cfg, pid=proc.pid) + deadline = time.monotonic() + cfg.readiness_timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + if proc.poll() is not None: + raise ServicesStartupExitedError(f"Services daemon exited with code {proc.returncode}; log: {log_path}") + if probe_status( + base_url=handle.base_url, + socket_path=handle.socket_path if handle.transport == "uds" else None, + timeout=remaining, + ): + ownership_transferred = True + return handle + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(cfg.readiness_poll_interval, remaining)) + raise ServicesStartupTimeoutError(f"Timed out waiting for services daemon {cfg.scope!r}; log: {log_path}") + finally: + if proc is not None and not ownership_transferred: + _terminate_startup_process(proc) + + +async def daemonize_services_async(config: ServiceRunConfig | None = None) -> DaemonServiceHandle: + return await asyncio.to_thread(daemonize_services, config) + + +def start_embedded_services( + config: ServiceRunConfig | None = None, + *, + env: MutableMapping[str, str] | None = None, +) -> EmbeddedServiceHandle: + """Start platform services in the current process. + + Args: + env: Environment mapping passed to :func:`build_platform_app`. + Defaults to ``None`` which writes to ``os.environ``. Tests can + pass an empty dict to avoid polluting the process environment. + """ + cfg = config or ServiceRunConfig(mode=ServiceMode.EMBEDDED) + from nmp.platform_runner.server import build_platform_app + + app = build_platform_app( + config=cfg.to_platform_app_config(), + env=env, + ) + runtime = getattr(app.state, "platform_runtime", None) + return EmbeddedServiceHandle(app=app, runtime=runtime) + + +async def start_embedded_services_async(config: ServiceRunConfig | None = None) -> EmbeddedServiceHandle: + return start_embedded_services(config) + + +def get_service_handle(config: ServiceRunConfig | None = None) -> DaemonServiceHandle | None: + cfg = config or ServiceRunConfig() + desc = process.read_descriptor(cfg.scope, base_dir=cfg.state_root) + if desc is None or not process.is_instance_alive(cfg.scope, base_dir=cfg.state_root): + return None + return DaemonServiceHandle.from_descriptor(desc) + + +def list_service_handles(state_dir: str | Path | None = None) -> list[DaemonServiceHandle]: + state_root = Path(state_dir).expanduser() if state_dir is not None else default_state_root() + handles: list[DaemonServiceHandle] = [] + for info in process.list_instances(base_dir=state_root): + if info.descriptor is not None and info.alive: + handles.append(DaemonServiceHandle.from_descriptor(info.descriptor)) + return handles + + +def ensure_services( + config: ServiceRunConfig | None = None, + *, + daemonize: bool | None = None, +) -> LocalServiceHandle: + cfg = config or ServiceRunConfig() + if cfg.mode is ServiceMode.EMBEDDED: + return start_embedded_services(cfg) + + handle = get_service_handle(cfg) + if handle is not None: + return handle + if daemonize is False: + raise ServicesNotRunningError(f"Instance {cfg.scope!r} is not running") + return daemonize_services(cfg) + + +async def ensure_services_async( + config: ServiceRunConfig | None = None, + *, + daemonize: bool | None = None, +) -> LocalServiceHandle: + cfg = config or ServiceRunConfig() + if cfg.mode is ServiceMode.EMBEDDED: + return await start_embedded_services_async(cfg) + + handle = get_service_handle(cfg) + if handle is not None: + return handle + if daemonize is False: + raise ServicesNotRunningError(f"Instance {cfg.scope!r} is not running") + return await daemonize_services_async(cfg) + + +def connect_services( + config: ServiceRunConfig | None = None, + *, + daemonize: bool | None = None, + start_if_needed: bool = True, + **client_kwargs: Any, +) -> NeMoPlatform: + cfg = config or ServiceRunConfig() + if not start_if_needed and cfg.mode is ServiceMode.DAEMON and get_service_handle(cfg) is None: + raise ServicesNotRunningError(f"Instance {cfg.scope!r} is not running") + handle = ensure_services(cfg, daemonize=daemonize) + return handle.client(**client_kwargs) + + +async def connect_services_async( + config: ServiceRunConfig | None = None, + *, + daemonize: bool | None = None, + start_if_needed: bool = True, + **client_kwargs: Any, +) -> AsyncNeMoPlatform: + cfg = config or ServiceRunConfig() + if not start_if_needed and cfg.mode is ServiceMode.DAEMON and get_service_handle(cfg) is None: + raise ServicesNotRunningError(f"Instance {cfg.scope!r} is not running") + handle = await ensure_services_async(cfg, daemonize=daemonize) + return handle.async_client(**client_kwargs) + + +def stop_services( + config: ServiceRunConfig | None = None, + *, + timeout: float = 30.0, + force: bool = False, +) -> process.StopResult: + cfg = config or ServiceRunConfig() + handle = get_service_handle(cfg) + if handle is None: + raise ServicesNotRunningError(f"Instance {cfg.scope!r} is not running") + return handle.stop(timeout=timeout, force=force) + + +async def stop_services_async( + config: ServiceRunConfig | None = None, + *, + timeout: float = 30.0, + force: bool = False, +) -> process.StopResult: + return await asyncio.to_thread(stop_services, config, timeout=timeout, force=force) diff --git a/sdk/python/nemo-platform/src/nemo_platform/local/transport.py b/sdk/python/nemo-platform/src/nemo_platform/local/transport.py new file mode 100644 index 0000000000..da3c08344a --- /dev/null +++ b/sdk/python/nemo-platform/src/nemo_platform/local/transport.py @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Local service transport helpers for TCP and Unix domain sockets.""" + +from __future__ import annotations + +import asyncio +import time +from pathlib import Path +from typing import Any, TypeAlias + +import httpx +from fastapi.testclient import TestClient +from nmp.common.platform_endpoint import UDS_BASE_URL + +HttpxTimeout: TypeAlias = float | httpx.Timeout | None +_DEFAULT_TIMEOUT: float = 5.0 +EMBEDDED_BASE_URL = "http://nemo-platform.local" + +__all__ = [ + "EMBEDDED_BASE_URL", + "UDS_BASE_URL", + "build_async_asgi_http_client", + "build_async_http_client", + "build_sync_asgi_http_client", + "build_sync_http_client", + "probe_status", + "probe_status_async", + "tcp_base_url", + "wait_for_status", + "wait_for_status_async", +] + + +def build_sync_asgi_http_client(app: Any, *, timeout: HttpxTimeout = _DEFAULT_TIMEOUT) -> Any: + _ = timeout + return TestClient( + app, + base_url=EMBEDDED_BASE_URL, + follow_redirects=True, + ) + + +def build_async_asgi_http_client(app: Any, *, timeout: HttpxTimeout = _DEFAULT_TIMEOUT) -> httpx.AsyncClient: + return httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url=EMBEDDED_BASE_URL, + follow_redirects=True, + timeout=timeout, + ) + + +def build_sync_http_client(socket_path: Path, *, timeout: HttpxTimeout = _DEFAULT_TIMEOUT) -> httpx.Client: + return httpx.Client( + transport=httpx.HTTPTransport(uds=str(socket_path)), + follow_redirects=True, + timeout=timeout, + ) + + +def build_async_http_client(socket_path: Path, *, timeout: HttpxTimeout = _DEFAULT_TIMEOUT) -> httpx.AsyncClient: + return httpx.AsyncClient( + transport=httpx.AsyncHTTPTransport(uds=str(socket_path)), + follow_redirects=True, + timeout=timeout, + ) + + +def tcp_base_url(host: str, port: int) -> str: + connect_host = "localhost" if host in {"0.0.0.0", "::"} else host # noqa: S104 + normalized = connect_host.strip("[]") + url_host = f"[{normalized}]" if ":" in normalized else normalized + return str(httpx.URL(scheme="http", host=url_host, port=port)) + + +def probe_status( + *, + base_url: str, + socket_path: Path | None = None, + timeout: float = 2.0, +) -> bool: + client = ( + build_sync_http_client(socket_path, timeout=timeout) + if socket_path is not None + else httpx.Client(timeout=timeout) + ) + try: + response = client.get(f"{base_url.rstrip('/')}/status") + return response.status_code == 200 + except httpx.RequestError: + return False + finally: + client.close() + + +async def probe_status_async( + *, + base_url: str, + socket_path: Path | None = None, + timeout: float = 2.0, +) -> bool: + client = ( + build_async_http_client(socket_path, timeout=timeout) + if socket_path is not None + else httpx.AsyncClient(timeout=timeout) + ) + try: + response = await client.get(f"{base_url.rstrip('/')}/status") + return response.status_code == 200 + except httpx.RequestError: + return False + finally: + await client.aclose() + + +def wait_for_status( + *, + base_url: str, + socket_path: Path | None = None, + timeout: float = 60.0, + poll_interval: float = 0.5, +) -> bool: + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + if probe_status(base_url=base_url, socket_path=socket_path, timeout=remaining): + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep(min(poll_interval, remaining)) + + +async def wait_for_status_async( + *, + base_url: str, + socket_path: Path | None = None, + timeout: float = 60.0, + poll_interval: float = 0.5, +) -> bool: + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + if await probe_status_async(base_url=base_url, socket_path=socket_path, timeout=remaining): + return True + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + await asyncio.sleep(min(poll_interval, remaining)) diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/conftest.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/conftest.py index a5889a0b4a..bc1a86a21c 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/conftest.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/conftest.py @@ -3,7 +3,7 @@ from __future__ import annotations -import nemo_platform.cli.commands.services._process as _process_mod +import nemo_platform.local.process as _process_mod import pytest diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services.py index a33ed5213a..ee18d2a9ee 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services.py @@ -13,11 +13,11 @@ import socket from pathlib import Path from types import ModuleType -from unittest.mock import ANY, MagicMock, patch +from unittest.mock import MagicMock, patch import pytest from nemo_platform.cli.app import app -from nemo_platform.cli.commands.services._process import ( +from nemo_platform.local.process import ( ForegroundInstanceError, InstanceDescriptor, StopResult, @@ -26,16 +26,17 @@ read_descriptor, write_descriptor, ) +from nmp.platform_runner.config import PlatformAppConfig from typer.testing import CliRunner runner = CliRunner() -_PROCESS_MODULE = "nemo_platform.cli.commands.services._process" +_PROCESS_MODULE = "nemo_platform.local.process" _CLI_MODULE = "nemo_platform.cli.commands.services.cli" def _seed_stopped_scope(base_dir: Path, scope: str, *, log_content: str = "x\n") -> Path: - """Create a stopped instance directory with service logs.""" + """Create a stopped scope directory with service logs.""" d = instance_dir(scope, base_dir=base_dir) (d / "services.log").write_text(log_content) return d @@ -118,17 +119,18 @@ def test_run_invokes_runner(base_dir: Path): ) assert result.exit_code == 0, result.stderr - mock_run_platform.assert_called_once_with( - services=["auth", "entities"], - service_group=None, - controllers=["jobs", "models"], - controller_group=None, - sidecars=None, - config_path=None, - host="127.0.0.1", - port=9000, - on_shutdown=ANY, - ) + mock_run_platform.assert_called_once() + _, kwargs = mock_run_platform.call_args + config = kwargs["config"] + assert config.services == ["auth", "entities"] + assert config.service_group is None + assert config.controllers == ["jobs", "models"] + assert config.controller_group is None + assert config.sidecars is None + assert config.config_path is None + assert config.host == "127.0.0.1" + assert config.port == 9000 + assert kwargs["on_shutdown"] is not None def test_run_refuses_when_already_running(base_dir: Path): @@ -163,7 +165,7 @@ def test_run_writes_descriptor(base_dir: Path): desc = read_descriptor("desc-test", base_dir=base_dir) assert desc is not None assert desc.mode == "foreground" - assert desc.port == 9999 + assert desc.config.port == 9999 def test_run_records_background_mode_when_launched_by_start(base_dir: Path): @@ -382,7 +384,7 @@ def test_restart_errors_when_no_prior_instance(self, base_dir: Path): ["services", "restart", "--instance", "ghost"], ) assert result.exit_code == 1 - assert "No instance found" in result.stderr + assert "No instance found for scope" in result.stderr assert "nemo services start" in result.stderr def test_restart_stops_and_starts(self, base_dir: Path): @@ -390,9 +392,7 @@ def test_restart_stops_and_starts(self, base_dir: Path): fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=1.0, ) @@ -429,9 +429,7 @@ def test_restart_exits_early_when_port_occupied_by_foreign_process(self, base_di desc = InstanceDescriptor( pid=99999, - scope=scope, - host="127.0.0.1", - port=port, + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=port), mode="background", create_time=1.0, ) @@ -459,13 +457,15 @@ def test_restart_preserves_previous_args(self, base_dir: Path): fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=9000, + config=PlatformAppConfig( + scope=scope, + services=["entities", "models"], + controllers=["jobs"], + host="127.0.0.1", + port=9000, + ), mode="background", create_time=1.0, - services=["entities", "models"], - controllers=["jobs"], ) write_descriptor(desc, base_dir=base_dir) @@ -488,11 +488,12 @@ def test_restart_preserves_previous_args(self, base_dir: Path): os.close(fd) assert result.exit_code == 0 - _, kwargs = mock_start.call_args - assert kwargs["services"] == ["entities", "models"] - assert kwargs["controllers"] == ["jobs"] - assert kwargs["host"] == "127.0.0.1" - assert kwargs["port"] == 9000 + args, _kwargs = mock_start.call_args + config = args[0] + assert config.services == ["entities", "models"] + assert config.controllers == ["jobs"] + assert config.host == "127.0.0.1" + assert config.port == 9000 # --------------------------------------------------------------------------- @@ -504,16 +505,14 @@ class TestServicesStatus: def test_not_running(self, base_dir: Path): result = runner.invoke(app, ["services", "status", "--instance", "none"]) assert result.exit_code == 0 - assert "No running instance" in result.stdout + assert "No running instance for scope" in result.stdout def test_running_instance(self, base_dir: Path): scope = "status-test" fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="foreground", create_time=1.0, ) @@ -548,9 +547,7 @@ def test_lists_running_instance(self, base_dir: Path): fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=1.0, ) @@ -594,9 +591,7 @@ def test_mixed_running_and_stopped(self, base_dir: Path): write_descriptor( InstanceDescriptor( pid=os.getpid(), - scope=running_scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=running_scope), mode="background", create_time=1.0, ), @@ -658,7 +653,7 @@ def test_rm_requires_scope(self, base_dir: Path): def test_rm_rejects_invalid_scope(self, base_dir: Path): result = runner.invoke(app, ["services", "rm", "../escape"]) assert result.exit_code == 1 - assert "Invalid instance scope" in result.stderr + assert "Invalid scope" in result.stderr def test_rm_rejects_conflicting_scope_args(self, base_dir: Path): result = runner.invoke(app, ["services", "rm", "scope-a", "--instance", "scope-b"]) @@ -747,7 +742,7 @@ def test_default_host_is_loopback(base_dir: Path): assert result.exit_code == 0, result.stderr _, kwargs = mock_run_platform.call_args - assert kwargs["host"] == "127.0.0.1" + assert kwargs["config"].host == "127.0.0.1" def test_bind_all_warning(base_dir: Path): diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_lifecycle.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_lifecycle.py index 2d035e1355..1ee89693bf 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_lifecycle.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_lifecycle.py @@ -27,7 +27,7 @@ import pytest from nemo_platform.cli.app import app -from nemo_platform.cli.commands.services._process import ( +from nemo_platform.local.process import ( InstanceDescriptor, PortConflict, acquire_lock, @@ -43,6 +43,7 @@ stop_instance, write_descriptor, ) +from nmp.platform_runner.config import PlatformAppConfig from typer.testing import CliRunner _runner = CliRunner() @@ -70,18 +71,14 @@ import psutil as _psutil desc = { "pid": os.getpid(), - "scope": scope, - "host": "127.0.0.1", - "port": 8080, + "config": { + "scope": scope, + "host": "127.0.0.1", + "port": 8080, + }, "mode": "background", "create_time": _psutil.Process(os.getpid()).create_time(), "started_at": "test", - "services": None, - "controllers": None, - "service_group": None, - "controller_group": None, - "sidecars": None, - "config_path": None, "log_path": None, } desc_path = os.path.join(inst_dir, "instance.json") @@ -291,9 +288,7 @@ def test_stale_descriptor_with_reused_pid(self, tmp_path: Path) -> None: # Write a descriptor with the sleeper's PID but wrong create_time desc = InstanceDescriptor( pid=sleeper.pid, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=0.0, # intentionally wrong ) @@ -363,7 +358,7 @@ def test_log_preserved_across_restart(self, tmp_path: Path) -> None: log = d / "services.log" log.write_text("first boot log content\n") - from nemo_platform.cli.commands.services._process import rotate_log + from nemo_platform.local.process import rotate_log new_log = rotate_log(scope, base_dir=base_dir) new_log.write_text("second boot log content\n") @@ -399,15 +394,15 @@ def test_log_preserved_across_restart(self, tmp_path: Path) -> None: import psutil as _psutil desc = { "pid": os.getpid(), - "scope": scope, - "host": "127.0.0.1", - "port": port, + "config": { + "scope": scope, + "host": "127.0.0.1", + "port": port, + }, "mode": "background", "create_time": _psutil.Process(os.getpid()).create_time(), "started_at": "test", - "services": None, "controllers": None, - "service_group": None, "controller_group": None, - "sidecars": None, "config_path": None, "log_path": None, + "log_path": None, } desc_path = os.path.join(inst_dir, "instance.json") with open(desc_path, "w") as f: @@ -523,7 +518,7 @@ def test_stop_after_health_check(self, tmp_path: Path) -> None: assert is_instance_alive(scope, base_dir=base_dir) desc = read_descriptor(scope, base_dir=base_dir) assert desc is not None - assert desc.port == port + assert desc.config.port == port result = stop_instance(scope, base_dir=base_dir, timeout=5.0) assert proc.pid in result.stopped_pids @@ -536,7 +531,7 @@ def test_stop_after_health_check(self, tmp_path: Path) -> None: class TestInstanceCleanup: - """Integration tests for rm/prune and post-stop instance directories.""" + """Integration tests for rm/prune and post-stop scope directories.""" def test_stop_leaves_record_until_rm(self, tmp_path: Path, monkeypatch) -> None: base_dir = tmp_path / "state" @@ -668,9 +663,7 @@ def test_check_port_returns_nemo_instance_when_lock_held_and_port_blocked(self, write_descriptor( InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=port, + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=port), mode="background", create_time=1.0, ), @@ -704,9 +697,7 @@ def test_check_port_returns_foreign_when_alive_instance_uses_different_port(self write_descriptor( InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=nemo_port, + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=nemo_port), mode="background", create_time=1.0, ), diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_process.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_process.py index ae86b6b9f7..6008aed5c9 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_process.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_services_process.py @@ -6,6 +6,7 @@ from __future__ import annotations import os +import signal import subprocess import sys import time @@ -14,7 +15,8 @@ import psutil import pytest -from nemo_platform.cli.commands.services._process import ( +from nemo_platform.local import process as process_module +from nemo_platform.local.process import ( ForegroundInstanceError, InstanceAlreadyRunningError, InstanceDescriptor, @@ -40,6 +42,7 @@ validate_pid, write_descriptor, ) +from nmp.platform_runner.config import DEFAULT_LOCAL_SERVICES_BIND_HOST, PlatformAppConfig @pytest.fixture() @@ -48,67 +51,23 @@ def base_dir(tmp_path: Path) -> Path: # --------------------------------------------------------------------------- -# Scope computation +# Scope resolution # --------------------------------------------------------------------------- class TestComputeScope: - def test_explicit_instance_name(self) -> None: - assert compute_scope(port=8080, instance_name="myapp") == "myapp" + def test_explicit_scope(self) -> None: + assert compute_scope(port=1234, explicit_scope="myapp") == "myapp" - def test_default_scope_includes_port(self) -> None: - scope = compute_scope(port=9090) - assert scope.endswith("-9090") - - def test_default_scope_is_deterministic(self) -> None: - a = compute_scope(port=8080) - b = compute_scope(port=8080) - assert a == b - - def test_different_ports_different_scopes(self) -> None: - a = compute_scope(port=8080) - b = compute_scope(port=9090) - assert a != b - - def test_hash_prefix_is_8_chars(self) -> None: + def test_default_scope_is_stable_for_port(self) -> None: scope = compute_scope(port=8080) - prefix = scope.rsplit("-", 1)[0] - assert len(prefix) == 8 - - def test_git_failure_falls_back_to_cwd(self) -> None: - import nemo_platform.cli.commands.services._process as proc_mod - - proc_mod._scope_prefix_cache = None - try: - with patch.object(proc_mod, "_find_git_root", return_value="/no/git/here"): - scope = compute_scope(port=8080) - assert scope.endswith("-8080") - assert len(scope.rsplit("-", 1)[0]) == 8 - finally: - proc_mod._scope_prefix_cache = None - def test_different_git_roots_produce_different_prefixes(self) -> None: - """Two different working directories (worktrees) produce distinct scopes.""" - import nemo_platform.cli.commands.services._process as proc_mod - - with patch.object(proc_mod, "_find_git_root", return_value="/workspace/project-a"): - scope_a = compute_scope(port=8080) - - proc_mod._scope_prefix_cache = None - - with patch.object(proc_mod, "_find_git_root", return_value="/workspace/project-b"): - scope_b = compute_scope(port=8080) - - assert scope_a != scope_b - assert scope_a.endswith("-8080") - assert scope_b.endswith("-8080") - prefix_a = scope_a.rsplit("-", 1)[0] - prefix_b = scope_b.rsplit("-", 1)[0] - assert prefix_a != prefix_b + assert scope == compute_scope(port=8080) + assert scope.endswith("-8080") # --------------------------------------------------------------------------- -# Instance directory +# Scope directory # --------------------------------------------------------------------------- @@ -168,26 +127,27 @@ class TestDescriptorRoundTrip: def test_write_and_read(self, base_dir: Path) -> None: desc = InstanceDescriptor( pid=12345, - scope="test-8080", - host="127.0.0.1", - port=8080, + config=PlatformAppConfig( + scope="test-8080", + services=["entities", "models"], + controllers=["jobs"], + host="127.0.0.1", + ), mode="background", create_time=1000.0, - services=["entities", "models"], - controllers=["jobs"], ) write_descriptor(desc, base_dir=base_dir) recovered = read_descriptor("test-8080", base_dir=base_dir) assert recovered is not None assert recovered.pid == 12345 - assert recovered.scope == "test-8080" - assert recovered.host == "127.0.0.1" - assert recovered.port == 8080 + assert recovered.config.scope == "test-8080" + assert recovered.config.host == "127.0.0.1" + assert recovered.config.port == 8080 assert recovered.mode == "background" assert recovered.create_time == 1000.0 - assert recovered.services == ["entities", "models"] - assert recovered.controllers == ["jobs"] + assert recovered.config.services == ["entities", "models"] + assert recovered.config.controllers == ["jobs"] def test_read_missing_returns_none(self, base_dir: Path) -> None: assert read_descriptor("no-such-scope", base_dir=base_dir) is None @@ -200,9 +160,7 @@ def test_read_corrupt_returns_none(self, base_dir: Path) -> None: def test_remove_descriptor(self, base_dir: Path) -> None: desc = InstanceDescriptor( pid=1, - scope="rm-test", - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope="rm-test"), mode="background", create_time=1.0, ) @@ -245,9 +203,7 @@ def test_lists_alive_instance(self, base_dir: Path) -> None: fd = acquire_lock("alive-one", base_dir=base_dir) desc = InstanceDescriptor( pid=os.getpid(), - scope="alive-one", - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope="alive-one"), mode="foreground", create_time=1.0, ) @@ -265,9 +221,7 @@ def test_cleans_up_dead_descriptor(self, base_dir: Path) -> None: d = instance_dir("dead-scope", base_dir=base_dir) desc = InstanceDescriptor( pid=999999, - scope="dead-scope", - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope="dead-scope"), mode="background", create_time=1.0, ) @@ -281,9 +235,7 @@ def test_stale_descriptor_with_logs_stays_listed(self, base_dir: Path) -> None: d = instance_dir("dead-with-logs", base_dir=base_dir) desc = InstanceDescriptor( pid=999999, - scope="dead-with-logs", - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope="dead-with-logs"), mode="background", create_time=1.0, ) @@ -370,7 +322,7 @@ def test_refuses_running_instance(self, base_dir: Path) -> None: os.close(fd) def test_rejects_invalid_scope(self, base_dir: Path) -> None: - with pytest.raises(ValueError, match="Invalid instance scope"): + with pytest.raises(ValueError, match="Invalid scope"): remove_instance("../escape", base_dir=base_dir) def test_returns_false_when_rmtree_fails(self, base_dir: Path) -> None: @@ -378,7 +330,7 @@ def test_returns_false_when_rmtree_fails(self, base_dir: Path) -> None: (d / "services.log").write_text("logs\n") with patch( - "nemo_platform.cli.commands.services._process.shutil.rmtree", + "nemo_platform.local.process.shutil.rmtree", side_effect=OSError("permission denied"), ): assert remove_instance("rmtree-fail", base_dir=base_dir) is False @@ -489,9 +441,7 @@ def test_stops_running_process(self, base_dir: Path) -> None: fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=proc.pid, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=psutil.Process(proc.pid).create_time(), ) @@ -512,9 +462,7 @@ def test_cleans_up_stale_descriptor(self, base_dir: Path) -> None: scope = "stale" desc = InstanceDescriptor( pid=999999999, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=0.0, ) @@ -529,9 +477,7 @@ def test_refuses_to_stop_foreground_instance(self, base_dir: Path) -> None: try: desc = InstanceDescriptor( pid=os.getpid(), - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="foreground", create_time=1.0, ) @@ -555,9 +501,7 @@ def test_force_stops_foreground_instance(self, base_dir: Path) -> None: try: desc = InstanceDescriptor( pid=proc.pid, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="foreground", create_time=psutil.Process(proc.pid).create_time(), ) @@ -571,6 +515,34 @@ def test_force_stops_foreground_instance(self, base_dir: Path) -> None: proc.kill() proc.wait(timeout=5) + def test_preserves_descriptor_when_sigkill_does_not_stop_parent(self, base_dir: Path, monkeypatch) -> None: + scope = "sigkill-still-alive" + desc = InstanceDescriptor( + pid=12345, + config=PlatformAppConfig(scope=scope), + mode="background", + create_time=1.0, + ) + write_descriptor(desc, base_dir=base_dir) + kill_signals: list[int] = [] + + def fake_kill(_pid: int, sig: int) -> None: + kill_signals.append(sig) + + monkeypatch.setattr(process_module, "validate_pid", lambda _pid, _create_time: True) + monkeypatch.setattr(process_module, "_pid_alive", lambda _pid: True) + monkeypatch.setattr(process_module, "_snapshot_children", lambda _pid: [object()]) + monkeypatch.setattr(process_module, "_sweep_orphans", lambda _children: [222]) + monkeypatch.setattr(process_module, "_SIGKILL_WAIT_TIMEOUT", 0.0) + monkeypatch.setattr(process_module.os, "kill", fake_kill) + + result = stop_instance(scope, base_dir=base_dir, timeout=0.0) + + assert kill_signals == [signal.SIGTERM, signal.SIGKILL] + assert result.stopped_pids == [] + assert result.swept_children == [222] + assert read_descriptor(scope, base_dir=base_dir) is not None + # --------------------------------------------------------------------------- # start_background @@ -578,21 +550,49 @@ def test_force_stops_foreground_instance(self, base_dir: Path) -> None: class TestStartBackground: + def test_uses_default_platform_app_config(self, tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path)) + mock_proc = MagicMock() + mock_proc.pid = 99998 + captured_args: list[str] = [] + captured_env: dict[str, str] = {} + + def fake_popen(args, **kwargs): + captured_args.extend(args) + captured_env.update(kwargs["env"]) + return mock_proc + + with patch( + "nemo_platform.local.process.subprocess.Popen", + side_effect=fake_popen, + ): + proc = start_background() + + assert proc.pid == 99998 + assert captured_args[captured_args.index("--instance") + 1] == "default" + assert captured_args[captured_args.index("--host") + 1] == "127.0.0.1" + assert captured_args[captured_args.index("--port") + 1] == "8080" + assert captured_env["XDG_STATE_HOME"] == str(tmp_path) + assert "_NMP_STATE_DIR" not in captured_env + assert (tmp_path / "nmp" / "instances" / "default" / "services.log").exists() + def test_launches_detached_subprocess(self, base_dir: Path) -> None: mock_proc = MagicMock() mock_proc.pid = 99999 with patch( - "nemo_platform.cli.commands.services._process.subprocess.Popen", + "nemo_platform.local.process.subprocess.Popen", return_value=mock_proc, ) as mock_popen: proc = start_background( - scope="bg-test", - services=["entities", "models"], - controllers=["jobs"], - host="127.0.0.1", - port=8080, - base_dir=base_dir, + PlatformAppConfig( + scope="bg-test", + services=["entities", "models"], + controllers=["jobs"], + host="127.0.0.1", + port=8080, + state_root=base_dir, + ), ) assert proc.pid == 99999 @@ -612,13 +612,16 @@ def fake_popen(args, **kwargs): return mock_proc with patch( - "nemo_platform.cli.commands.services._process.subprocess.Popen", + "nemo_platform.local.process.subprocess.Popen", side_effect=fake_popen, ): start_background( - scope="data-dir-test", + PlatformAppConfig( + scope="data-dir-test", + host=DEFAULT_LOCAL_SERVICES_BIND_HOST, + state_root=base_dir, + ), data_dir="/chosen/data/dir", - base_dir=base_dir, ) assert captured_env.get("NMP_DATA_DIR") == "/chosen/data/dir" @@ -634,13 +637,16 @@ def fake_popen(args, **kwargs): return mock_proc with patch( - "nemo_platform.cli.commands.services._process.subprocess.Popen", + "nemo_platform.local.process.subprocess.Popen", side_effect=fake_popen, ): start_background( - scope="shell-env-test", + PlatformAppConfig( + scope="shell-env-test", + host=DEFAULT_LOCAL_SERVICES_BIND_HOST, + state_root=base_dir, + ), data_dir="/chosen/data/dir", - base_dir=base_dir, ) assert captured_env.get("NMP_DATA_DIR") == "/shell/wins" @@ -654,16 +660,22 @@ def test_rotates_log_before_start(self, base_dir: Path) -> None: mock_proc.pid = 5555 with patch( - "nemo_platform.cli.commands.services._process.subprocess.Popen", + "nemo_platform.local.process.subprocess.Popen", return_value=mock_proc, ): - start_background(scope="rotate-test", base_dir=base_dir) + start_background( + PlatformAppConfig( + scope="rotate-test", + host=DEFAULT_LOCAL_SERVICES_BIND_HOST, + state_root=base_dir, + ), + ) rotated = list(d.glob("services.log.*")) assert len(rotated) == 1 assert rotated[0].read_text() == "old log content\n" - def test_forwards_instance_scope_to_child(self, base_dir: Path) -> None: + def test_forwards_scope_to_child(self, base_dir: Path) -> None: mock_proc = MagicMock() mock_proc.pid = 7777 captured_args: list[str] = [] @@ -673,20 +685,22 @@ def fake_popen(args, **kwargs): return mock_proc with patch( - "nemo_platform.cli.commands.services._process.subprocess.Popen", + "nemo_platform.local.process.subprocess.Popen", side_effect=fake_popen, ): start_background( - scope="custom-scope", - services=["entities"], - host="127.0.0.1", - port=9090, - base_dir=base_dir, + PlatformAppConfig( + scope="custom-key", + services=["entities"], + host="127.0.0.1", + port=9090, + state_root=base_dir, + ), ) assert "--instance" in captured_args idx = captured_args.index("--instance") - assert captured_args[idx + 1] == "custom-scope" + assert captured_args[idx + 1] == "custom-key" def test_sets_launch_mode_background_in_child_env(self, base_dir: Path) -> None: mock_proc = MagicMock() @@ -698,10 +712,16 @@ def fake_popen(args, **kwargs): return mock_proc with patch( - "nemo_platform.cli.commands.services._process.subprocess.Popen", + "nemo_platform.local.process.subprocess.Popen", side_effect=fake_popen, ): - start_background(scope="mode-test", base_dir=base_dir) + start_background( + PlatformAppConfig( + scope="mode-test", + host=DEFAULT_LOCAL_SERVICES_BIND_HOST, + state_root=base_dir, + ), + ) assert captured_env.get("_NMP_LAUNCH_MODE") == "background" @@ -846,9 +866,7 @@ def test_sweeps_surviving_children(self, base_dir: Path) -> None: fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=parent.pid, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=psutil.Process(parent.pid).create_time(), ) @@ -883,9 +901,7 @@ def test_swept_children_empty_when_no_children(self, base_dir: Path) -> None: fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=proc.pid, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="background", create_time=psutil.Process(proc.pid).create_time(), ) @@ -917,9 +933,7 @@ def test_restart_path_sweeps_children(self, base_dir: Path) -> None: fd = acquire_lock(scope, base_dir=base_dir) desc = InstanceDescriptor( pid=parent.pid, - scope=scope, - host="127.0.0.1", - port=8080, + config=PlatformAppConfig(scope=scope), mode="foreground", create_time=psutil.Process(parent.pid).create_time(), ) diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_setup.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_setup.py index 0998f9e6f5..84a99ee90d 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_setup.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/commands/test_setup.py @@ -14,7 +14,6 @@ import typer from click.exceptions import Exit as ClickExit from nemo_platform.resources.inference.providers import ProvidersResource -from nemo_platform.cli.commands.services._process import PortConflict from nemo_platform.cli.commands.setup import ( _AGENT_API_READINESS_POLL_INTERVAL, _AGENT_DEPLOY_POLL_INTERVAL, @@ -77,6 +76,7 @@ Context, ContextDefinition, ) +from nemo_platform.local.process import PortConflict from nemo_platform_plugin.client.errors import NotFoundError from nemo_platform_plugin.secrets.types import PlatformSecretCreateRequest, PlatformSecretUpdateRequest @@ -644,9 +644,10 @@ def test_start_services_background_forwards_data_dir(self): mock_start.return_value = MagicMock(pid=42) _start_services_background("http://localhost:9090", data_dir="/chosen/data/dir") mock_start.assert_called_once() - _, kwargs = mock_start.call_args + args, kwargs = mock_start.call_args + config = args[0] assert kwargs["data_dir"] == "/chosen/data/dir" - assert kwargs["port"] == 9090 + assert config.port == 9090 def test_auto_mode_skips_prompt_and_uses_persisted(self, tmp_path, monkeypatch): """`--auto` must not prompt but should still honor any persisted data dir.""" diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/__init__.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/__init__.py new file mode 100644 index 0000000000..1275d78dff --- /dev/null +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/__init__.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_config_environment.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_config_environment.py new file mode 100644 index 0000000000..8255b3b6be --- /dev/null +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_config_environment.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for configuration and environment resolution. + +These tests exercise the real ``apply_run_environment`` code path with +actual YAML config files, verifying that environment variables are set +correctly for different host, port, and base_url scenarios. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from nmp.platform_runner.config import ( + ResolvedRunConfiguration, + apply_run_environment, + default_config_path, +) + + +def _resolved( + *, + services: set[str] | None = None, + controllers: set[str] | None = None, + sidecars: set[str] | None = None, + host: str = "127.0.0.1", + port: int = 8080, + config_path: str | None = None, + socket_path: str | None = None, +) -> ResolvedRunConfiguration: + return ResolvedRunConfiguration( + services=services or set(), + controllers=controllers or set(), + sidecars=sidecars or set(), + host=host, + port=port, + config_path=config_path or default_config_path(), + socket_path=socket_path, + available_services={}, + available_controllers={}, + ) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_apply_run_environment_preserves_external_base_url() -> None: + """Pre-set NMP_BASE_URL (e.g. from k8s/Helm) must not be overwritten.""" + env: dict[str, str] = {"NMP_BASE_URL": "https://platform.k8s.internal:443"} + config = _resolved(host="0.0.0.0", port=9090) + + apply_run_environment(config, env=env) + + assert env["NMP_BASE_URL"] == "https://platform.k8s.internal:443" + + +@pytest.mark.integration +def test_apply_run_environment_wildcard_host_becomes_loopback(tmp_path: Path) -> None: + """A wildcard bind host (0.0.0.0) in the config file should resolve to + 127.0.0.1 for the base URL, using the actual bind port.""" + config_file = tmp_path / "config.yaml" + config_file.write_text("platform:\n base_url: http://0.0.0.0:8080\n") + + env: dict[str, str] = {} + config = _resolved(host="0.0.0.0", port=9090, config_path=str(config_file)) + + apply_run_environment(config, env=env) + + assert env["NMP_BASE_URL"] == "http://127.0.0.1:9090" + assert env["NMP_SERVICE_HOST"] == "127.0.0.1" + assert env["NMP_SERVICE_PORT"] == "9090" + + +@pytest.mark.integration +def test_apply_run_environment_ipv6_literal_bracketed(tmp_path: Path) -> None: + """An IPv6 config base_url should produce a bracketed host in the resolved URL.""" + config_file = tmp_path / "config.yaml" + config_file.write_text("platform:\n base_url: http://[::1]:8080\n") + + env: dict[str, str] = {} + config = _resolved(host="::1", port=9090, config_path=str(config_file)) + + apply_run_environment(config, env=env) + + assert env["NMP_BASE_URL"] == "http://[::1]:9090" + + +@pytest.mark.integration +def test_config_file_base_url_malformed_yaml_falls_back(tmp_path: Path) -> None: + """A corrupt config file should fall back to the bind-derived URL.""" + config_file = tmp_path / "config.yaml" + config_file.write_text("{{{{not valid yaml at all") + + env: dict[str, str] = {} + config = _resolved(host="127.0.0.1", port=7777, config_path=str(config_file)) + + apply_run_environment(config, env=env) + + # Falls back to bind-derived: http://: + assert env["NMP_BASE_URL"] == "http://127.0.0.1:7777" + + +@pytest.mark.integration +def test_config_file_missing_falls_back(tmp_path: Path) -> None: + """A missing config file should fall back to the bind-derived URL.""" + env: dict[str, str] = {} + config = _resolved(host="127.0.0.1", port=5555, config_path=str(tmp_path / "nonexistent.yaml")) + + apply_run_environment(config, env=env) + + assert env["NMP_BASE_URL"] == "http://127.0.0.1:5555" + + +@pytest.mark.integration +def test_apply_run_environment_clears_empty_service_lists() -> None: + """When services/controllers/sidecars are empty sets, their env vars + should be removed (popped) rather than set to empty strings.""" + env: dict[str, str] = { + "NMP_SERVICES": "old-service", + "NMP_CONTROLLERS": "old-controller", + "NMP_SIDECARS": "old-sidecar", + } + config = _resolved(services=set(), controllers=set(), sidecars=set()) + + apply_run_environment(config, env=env) + + assert "NMP_SERVICES" not in env + assert "NMP_CONTROLLERS" not in env + assert "NMP_SIDECARS" not in env + + +@pytest.mark.integration +def test_apply_run_environment_sets_service_lists() -> None: + """Non-empty service/controller/sidecar sets should be written as + comma-separated, sorted env var values.""" + env: dict[str, str] = {} + config = _resolved( + services={"models", "auth", "secrets"}, + controllers={"beta-controller"}, + sidecars={"adapters"}, + ) + + apply_run_environment(config, env=env) + + assert env["NMP_SERVICES"] == "auth,models,secrets" + assert env["NMP_CONTROLLERS"] == "beta-controller" + assert env["NMP_SIDECARS"] == "adapters" + + +@pytest.mark.integration +def test_apply_run_environment_uds_transport_uses_unix_base_url() -> None: + """When a socket_path is set (UDS transport), the base URL should use + the ``unix://`` scheme.""" + env: dict[str, str] = {} + config = _resolved(socket_path="/tmp/nemo.sock") + + apply_run_environment(config, env=env) + + assert env["NMP_BASE_URL"] == "unix:///tmp/nemo.sock" + + +@pytest.mark.integration +def test_apply_run_environment_preserves_external_host_and_port() -> None: + """Pre-set NMP_SERVICE_HOST and NMP_SERVICE_PORT should not be overwritten.""" + env: dict[str, str] = { + "NMP_SERVICE_HOST": "10.0.0.1", + "NMP_SERVICE_PORT": "443", + } + config = _resolved(host="0.0.0.0", port=9090) + + apply_run_environment(config, env=env) + + assert env["NMP_SERVICE_HOST"] == "10.0.0.1" + assert env["NMP_SERVICE_PORT"] == "443" + + +@pytest.mark.integration +def test_apply_run_environment_ipv6_wildcard_becomes_loopback(tmp_path: Path) -> None: + """The IPv6 wildcard ``::`` should resolve to ``::1`` for internal clients.""" + # Use a config file without platform.base_url so the bind host drives the URL. + config_file = tmp_path / "config.yaml" + config_file.write_text("platform:\n seed_on_startup: false\n") + + env: dict[str, str] = {} + config = _resolved(host="::", port=8080, config_path=str(config_file)) + + apply_run_environment(config, env=env) + + assert env["NMP_SERVICE_HOST"] == "::1" + # Base URL should have bracketed IPv6. + assert env["NMP_BASE_URL"] == "http://[::1]:8080" diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_daemon_lifecycle.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_daemon_lifecycle.py new file mode 100644 index 0000000000..d38ac7c8eb --- /dev/null +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_daemon_lifecycle.py @@ -0,0 +1,503 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for daemon subprocess lifecycle. + +These tests spawn REAL child processes via ``daemonize_services()``, exercise +real lock acquisition, descriptor file I/O, HTTP readiness probing, and +graceful shutdown via ``stop_instance()``. Nothing is monkeypatched away — +the child runs a real uvicorn server with the ``hello-world`` service. + +Requirements: +- All packages installed (``uv sync --all-packages``) so entry-point + discovery finds hello-world. +- ``pyleak`` importable (from the ``[all]`` extra). +""" + +from __future__ import annotations + +import os +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import psutil +import pytest +from nemo_platform.local import process, services +from nemo_platform.local.process import ForegroundInstanceError +from nemo_platform.local.services import ( + ServiceRunConfig, + ServicesAlreadyRunningError, + ServicesStartupExitedError, +) +from nmp.platform_runner.config import PlatformAppConfig + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _free_tcp_port() -> int: + """Bind to port 0, let the OS pick, then release and return the port number.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _daemon_config( + tmp_path: Path, + *, + scope: str = "integ-daemon", + port: int | None = None, +) -> ServiceRunConfig: + """Build a ServiceRunConfig that is fully isolated under ``tmp_path``.""" + return ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + controllers=(), + sidecars=(), + transport="tcp", + host="127.0.0.1", + port=port or _free_tcp_port(), + scope=scope, + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + readiness_timeout=30.0, + readiness_poll_interval=0.3, + ) + + +def _ensure_stopped(cfg: ServiceRunConfig) -> None: + """Best-effort cleanup: stop any instance left running by a test.""" + try: + process.stop_instance(cfg.scope, base_dir=cfg.state_root, timeout=10, force=True) + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_daemonize_services_spawns_child_that_becomes_ready(tmp_path: Path) -> None: + """Spawn a real daemon subprocess, verify readiness via HTTP, then + gracefully shut down with ``stop_instance``.""" + cfg = _daemon_config(tmp_path) + handle = None + try: + handle = services.daemonize_services(cfg) + + # -- The handle should report the child's PID and transport details. + assert handle.pid is not None + assert handle.port == cfg.port + assert handle.transport == "tcp" + + # -- The lock file should be held by the child. + assert process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + + # -- The descriptor should have been written by the child. + desc = process.read_descriptor(cfg.scope, base_dir=cfg.state_root) + assert desc is not None + assert desc.pid == handle.pid + assert desc.mode == "daemon" + assert "hello-world" in (desc.config.services or []) + + # -- The child should still be running and respond to /status. + assert services.probe_status(base_url=f"http://127.0.0.1:{cfg.port}", timeout=5.0) + + # -- Graceful shutdown. + result = process.stop_instance(cfg.scope, base_dir=cfg.state_root, timeout=15) + assert handle.pid in result.stopped_pids + + # -- After stop, the lock should be released and the descriptor removed. + assert not process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + assert process.read_descriptor(cfg.scope, base_dir=cfg.state_root) is None + finally: + _ensure_stopped(cfg) + + +@pytest.mark.integration +def test_daemonize_services_child_exit_before_readiness(tmp_path: Path) -> None: + """When the child exits before becoming ready, ``daemonize_services`` + should raise ``ServicesStartupExitedError`` with the log path.""" + # Spawn a child that will exit immediately: give it a bogus service name + # that will fail validation in resolve_run_configuration. + bad_cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("nonexistent-service-xyz",), + controllers=(), + sidecars=(), + transport="tcp", + host="127.0.0.1", + port=_free_tcp_port(), + scope="integ-early-exit", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + readiness_timeout=15.0, + readiness_poll_interval=0.2, + ) + with pytest.raises(ServicesStartupExitedError, match="exited with code"): + services.daemonize_services(bad_cfg) + + # -- The lock should not be held after the failed startup. + assert not process.is_instance_alive(bad_cfg.scope, base_dir=bad_cfg.state_root) + + +@pytest.mark.integration +def test_stale_socket_cleanup_after_process_crash(tmp_path: Path) -> None: + """If a previous daemon crashed and left a UDS socket file, a new daemon + startup should clean it up and succeed.""" + scope = "stale" + # Use a short temp directory to stay within AF_UNIX path limits (103 bytes on macOS). + short_tmp = Path(tempfile.mkdtemp(prefix="nemo-")) + runtime_dir = short_tmp / "run" + + # Create a stale UDS socket file (no process listening). + socket_dir = runtime_dir / scope + socket_dir.mkdir(parents=True, exist_ok=True) + stale_socket = socket_dir / "nemo-platform.sock" + # Bind a real UDS socket to create the file, then close immediately. + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(str(stale_socket)) + assert stale_socket.exists() + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + controllers=(), + sidecars=(), + transport="uds", + host="127.0.0.1", + port=_free_tcp_port(), + scope=scope, + state_dir=short_tmp / "state", + runtime_dir=runtime_dir, + readiness_timeout=30.0, + readiness_poll_interval=0.3, + ) + try: + services.daemonize_services(cfg) + + # -- The daemon should be ready. + assert process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + + # -- The stale socket should have been replaced with the new one. + assert stale_socket.exists() + finally: + _ensure_stopped(cfg) + import shutil + + shutil.rmtree(short_tmp, ignore_errors=True) + + +@pytest.mark.integration +def test_concurrent_daemonize_rejects_duplicate_instance(tmp_path: Path) -> None: + """Starting a second daemon with the same instance scope should fail + with ``ServicesAlreadyRunningError`` while the first is running.""" + cfg = _daemon_config(tmp_path, scope="integ-dup") + try: + services.daemonize_services(cfg) + assert process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + + # -- A second daemonize with the same scope should fail. + dup_cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + controllers=(), + sidecars=(), + transport="tcp", + host="127.0.0.1", + port=_free_tcp_port(), # Different port, same scope. + scope="integ-dup", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + readiness_timeout=5.0, + readiness_poll_interval=0.2, + ) + with pytest.raises(ServicesAlreadyRunningError): + services.daemonize_services(dup_cfg) + + # -- Original instance should still be alive. + assert process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + finally: + _ensure_stopped(cfg) + + +@pytest.mark.integration +def test_stop_instance_escalates_sigterm_to_sigkill(tmp_path: Path) -> None: + """If the daemon child ignores SIGTERM, ``stop_instance`` should escalate + to SIGKILL after the timeout and successfully terminate the process.""" + # Instead of using daemonize_services (which starts a uvicorn server that + # handles SIGTERM), we manually simulate a daemon process that ignores SIGTERM + # using the process module primitives directly. + scope = "integ-sigkill" + state_dir = tmp_path / "state" + + # Spawn a child process that ignores SIGTERM. + child = subprocess.Popen( + [ + sys.executable, + "-c", + "import signal, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "open('/dev/null', 'w'); time.sleep(300)", + ], + start_new_session=True, + ) + try: + # Write a descriptor so stop_instance can find the process. + desc = process.InstanceDescriptor( + pid=child.pid, + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=0, state_root=state_dir), + transport="tcp", + mode="daemon", + create_time=psutil.Process(child.pid).create_time(), + ) + process.write_descriptor(desc, base_dir=state_dir) + + # Also create a lock file the process "holds" — but since it's a + # different process, we simulate by NOT acquiring a real flock (the + # test exercises PID-based stop, not flock-based liveness). + + # Stop with a very short timeout so it escalates quickly. + result = process.stop_instance(scope, base_dir=state_dir, timeout=1.0, force=True) + assert child.pid in result.stopped_pids + + # The child should be dead now. + child.wait(timeout=5) + assert child.returncode is not None + finally: + try: + child.kill() + child.wait(timeout=3) + except Exception: + pass + + +@pytest.mark.integration +def test_daemonize_services_cleans_up_on_child_exception(tmp_path: Path) -> None: + """When the child process crashes during init (e.g. corrupted request JSON), + the parent detects the exit, raises, and the lock is not left held.""" + scope = "integ-crash" + state_dir = tmp_path / "state" + instance_dir = process.instance_dir(scope, base_dir=state_dir) + + # Write a corrupted request file that will make _service_child crash + # during JSON deserialization. + fd, tmp_req = tempfile.mkstemp(dir=str(instance_dir), suffix=".json") + os.write(fd, b"NOT VALID JSON {{{") + os.close(fd) + + log_path = process.log_path_for(scope, base_dir=state_dir) + log_file = open(log_path, "a") # noqa: SIM115 + child_module = "nemo_platform.local._service_child" + proc = subprocess.Popen( + [sys.executable, "-m", child_module, tmp_req], + stdout=log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + stdin=subprocess.DEVNULL, + close_fds=True, + ) + log_file.close() + + # Wait for the child to exit (it should crash quickly on bad JSON). + proc.wait(timeout=10) + assert proc.returncode != 0 + + # The lock should not be held — the child never acquired it. + assert not process.is_instance_alive(scope, base_dir=state_dir) + + # The request file should have been cleaned up by _service_child. + assert not Path(tmp_req).exists() + + +# --------------------------------------------------------------------------- +# Priority 2: Process Lifecycle & Cleanup +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_stop_instance_sweeps_orphaned_children(tmp_path: Path) -> None: + """When a daemon parent is stopped, any grandchild processes that survive + should be swept by ``_sweep_orphans``.""" + scope = "integ-orphans" + state_dir = tmp_path / "state" + + # Spawn a parent that spawns a long-lived grandchild, then sleeps. + parent = subprocess.Popen( + [ + sys.executable, + "-c", + "import subprocess, sys, time; " + "gc = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(300)']); " + "time.sleep(300)", + ], + start_new_session=True, + ) + try: + # Give the parent time to spawn the grandchild. + time.sleep(0.5) + grandchildren = psutil.Process(parent.pid).children(recursive=True) + assert len(grandchildren) >= 1, "grandchild was not spawned" + + desc = process.InstanceDescriptor( + pid=parent.pid, + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=0, state_root=state_dir), + transport="tcp", + mode="daemon", + create_time=psutil.Process(parent.pid).create_time(), + ) + process.write_descriptor(desc, base_dir=state_dir) + + result = process.stop_instance(scope, base_dir=state_dir, timeout=10, force=True) + assert parent.pid in result.stopped_pids + assert len(result.swept_children) >= 1 + + # Both parent and grandchild should be dead. + parent.wait(timeout=5) + for gc in grandchildren: + gc.wait(timeout=5) + finally: + try: + parent.kill() + parent.wait(timeout=3) + except Exception: + pass + for gc in grandchildren: + try: + gc.kill() + gc.wait(timeout=3) + except Exception: + pass + + +@pytest.mark.integration +def test_stop_instance_foreground_mode_requires_force(tmp_path: Path) -> None: + """Stopping a foreground-mode instance without ``force=True`` should raise + ``ForegroundInstanceError``. With ``force=True`` it should proceed.""" + scope = "integ-foreground" + state_dir = tmp_path / "state" + + child = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(300)"], + start_new_session=True, + ) + try: + desc = process.InstanceDescriptor( + pid=child.pid, + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=0, state_root=state_dir), + transport="tcp", + mode="foreground", + create_time=psutil.Process(child.pid).create_time(), + ) + process.write_descriptor(desc, base_dir=state_dir) + + # Without force, should raise. + with pytest.raises(ForegroundInstanceError): + process.stop_instance(scope, base_dir=state_dir, timeout=5) + + # Process should still be alive after the rejected stop. + assert child.poll() is None + + # With force, should succeed. + result = process.stop_instance(scope, base_dir=state_dir, timeout=5, force=True) + assert child.pid in result.stopped_pids + child.wait(timeout=5) + finally: + try: + child.kill() + child.wait(timeout=3) + except Exception: + pass + + +@pytest.mark.integration +def test_is_instance_alive_with_stale_lock(tmp_path: Path) -> None: + """If the lock file exists but no process holds the flock, + ``is_instance_alive`` should return False.""" + scope = "integ-stale-lock" + state_dir = tmp_path / "state" + + # Create the lock file without holding a flock on it. + inst_dir = process.instance_dir(scope, base_dir=state_dir) + lock_path = inst_dir / process.LOCK_FILENAME + lock_path.touch() + + assert not process.is_instance_alive(scope, base_dir=state_dir) + + +@pytest.mark.integration +def test_is_instance_alive_with_held_lock(tmp_path: Path) -> None: + """If a process holds the flock, ``is_instance_alive`` should return True.""" + scope = "integ-held-lock" + state_dir = tmp_path / "state" + + fd = process.acquire_lock(scope, base_dir=state_dir) + try: + assert process.is_instance_alive(scope, base_dir=state_dir) + finally: + os.close(fd) + + # After releasing the fd (which releases the flock), should be false. + assert not process.is_instance_alive(scope, base_dir=state_dir) + + +@pytest.mark.integration +def test_validate_pid_detects_recycled_process(tmp_path: Path) -> None: + """After a process dies, ``validate_pid`` should return False if the PID is + reused by a different process (detected via create_time mismatch).""" + # Spawn and immediately kill a short-lived process to get a PID + create_time. + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(300)"]) + pid = child.pid + create_time = psutil.Process(pid).create_time() + + # The PID is alive and create_time matches. + assert process.validate_pid(pid, create_time) + + # Kill it. + child.kill() + child.wait(timeout=5) + + # Now validate_pid should return False — the process is dead. + assert not process.validate_pid(pid, create_time) + + # Even with a wildly wrong create_time, should be False for a dead PID. + assert not process.validate_pid(pid, 0.0) + + +@pytest.mark.integration +def test_rotate_log_preserves_existing_content(tmp_path: Path) -> None: + """``rotate_log`` should rename the existing log and return the path for + the new (empty) log. The old content must be preserved.""" + scope = "integ-rotate" + state_dir = tmp_path / "state" + + # Write initial log content. + log_path = process.log_path_for(scope, base_dir=state_dir) + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text("original log content\n") + + # Rotate. + new_log = process.rotate_log(scope, base_dir=state_dir) + assert new_log == log_path + assert not log_path.exists() # Original was renamed. + + # Find the rotated file. + rotated_files = [f for f in log_path.parent.iterdir() if f.name.startswith("services.log.")] + assert len(rotated_files) == 1 + assert rotated_files[0].read_text() == "original log content\n" + + # Write new content, rotate again. + log_path.write_text("second run\n") + process.rotate_log(scope, base_dir=state_dir) + + rotated_files = sorted(f for f in log_path.parent.iterdir() if f.name.startswith("services.log.")) + assert len(rotated_files) == 2 + contents = {f.read_text() for f in rotated_files} + assert "original log content\n" in contents + assert "second run\n" in contents diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_health_child.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_health_child.py new file mode 100644 index 0000000000..683e7b3720 --- /dev/null +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_health_child.py @@ -0,0 +1,340 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for health/readiness probing, lifespan, and child process module. + +Covers Priorities 5 (lifespan), 6 (health), and 7 (child process) from the +integration test plan. +""" + +from __future__ import annotations + +import json +import os +import socket +import subprocess +import sys +import tempfile +import threading +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from nemo_platform.local import process +from nemo_platform.local.services import ServiceRunConfig +from nemo_platform.local.transport import probe_status, wait_for_status + + +def _free_tcp_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +# --------------------------------------------------------------------------- +# Priority 5: Multi-Service Startup & Lifespan +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_create_app_starts_and_joins_controller_threads() -> None: + """A controller registered via ``create_app`` should have its thread + started during lifespan and stopped on exit.""" + started = threading.Event() + stopped = threading.Event() + + def controller_run(stop_signal: threading.Event) -> None: + started.set() + stop_signal.wait(timeout=5.0) + stopped.set() + + with ( + patch("nmp.platform_runner.server.get_platform_config") as mock_pc, + patch("nmp.platform_runner.server.get_auth_config") as mock_ac, + patch("nmp.common.auth.middleware.get_auth_config") as mock_ac2, + ): + mock_pc.return_value = MagicMock(seed_on_startup=False, redirect_root_to_studio=False) + mock_ac.return_value = MagicMock(enabled=False, policy_decision_point_provider="embedded") + mock_ac2.return_value = MagicMock(enabled=False, policy_decision_point_provider="embedded") + + from nmp.platform_runner.server import create_app + + app = create_app(services=[], controller_run_funcs={"test-ctrl": controller_run}) + + from fastapi.testclient import TestClient + + with TestClient(app): + assert started.wait(timeout=2.0), "controller thread did not start" + + assert stopped.wait(timeout=2.0), "controller thread did not stop after lifespan exit" + + +@pytest.mark.integration +def test_create_app_controller_thread_join_timeout() -> None: + """A controller that ignores the stop signal should not hang shutdown — + ``thread.join(timeout=5)`` should return even if the controller is still running.""" + started = threading.Event() + + def stubborn_controller(stop_signal: threading.Event) -> None: + started.set() + # Ignore stop_signal — simulate a controller that hangs. + import time + + time.sleep(300) + + with ( + patch("nmp.platform_runner.server.get_platform_config") as mock_pc, + patch("nmp.platform_runner.server.get_auth_config") as mock_ac, + patch("nmp.common.auth.middleware.get_auth_config") as mock_ac2, + ): + mock_pc.return_value = MagicMock(seed_on_startup=False, redirect_root_to_studio=False) + mock_ac.return_value = MagicMock(enabled=False, policy_decision_point_provider="embedded") + mock_ac2.return_value = MagicMock(enabled=False, policy_decision_point_provider="embedded") + + from nmp.platform_runner.server import create_app + + app = create_app(services=[], controller_run_funcs={"stubborn": stubborn_controller}) + + from fastapi.testclient import TestClient + + # The TestClient __exit__ triggers lifespan exit, which calls thread.join(timeout=5). + # This should NOT hang forever — the 5s timeout should let shutdown proceed. + with TestClient(app): + assert started.wait(timeout=2.0), "controller thread did not start" + + # If we got here, shutdown didn't hang. The stubborn thread is still running + # but as a daemon thread it will be cleaned up when the test process exits. + + +@pytest.mark.integration +def test_lifespan_cleanup_runs_on_app_shutdown() -> None: + """``close_shared_http_clients`` should be called during lifespan teardown.""" + cleanup_called = threading.Event() + + with ( + patch("nmp.platform_runner.server.get_platform_config") as mock_pc, + patch("nmp.platform_runner.server.get_auth_config") as mock_ac, + patch("nmp.common.auth.middleware.get_auth_config") as mock_ac2, + patch("nmp.platform_runner.server.close_shared_http_clients") as mock_close, + ): + mock_pc.return_value = MagicMock(seed_on_startup=False, redirect_root_to_studio=False) + mock_ac.return_value = MagicMock(enabled=False, policy_decision_point_provider="embedded") + mock_ac2.return_value = MagicMock(enabled=False, policy_decision_point_provider="embedded") + + async def fake_close(): + cleanup_called.set() + + mock_close.side_effect = fake_close + + from nmp.platform_runner.server import create_app + + app = create_app(services=[]) + + from fastapi.testclient import TestClient + + with TestClient(app): + pass + + assert cleanup_called.is_set(), "close_shared_http_clients was not called during shutdown" + + +# --------------------------------------------------------------------------- +# Priority 6: Health & Readiness +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_wait_for_status_retries_on_transient_errors(tmp_path: Path) -> None: + """``wait_for_status`` should retry on connection refused and eventually + return True once the server starts responding.""" + from nemo_platform.local import services + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + controllers=(), + sidecars=(), + transport="tcp", + host="127.0.0.1", + port=_free_tcp_port(), + scope="integ-wait-retry", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + readiness_timeout=30.0, + readiness_poll_interval=0.3, + ) + + # Start the daemon — wait_for_status should retry until it's ready. + services.daemonize_services(cfg) + try: + # The daemon is already ready (daemonize_services waits for readiness). + # Verify wait_for_status succeeds with a fresh probe. + assert wait_for_status( + base_url=f"http://127.0.0.1:{cfg.port}", + timeout=5.0, + poll_interval=0.2, + ) + finally: + process.stop_instance(cfg.scope, base_dir=cfg.state_root, timeout=10, force=True) + + +@pytest.mark.integration +def test_wait_for_status_times_out_on_no_server() -> None: + """``wait_for_status`` should return False when no server is listening.""" + port = _free_tcp_port() + result = wait_for_status( + base_url=f"http://127.0.0.1:{port}", + timeout=1.0, + poll_interval=0.2, + ) + assert result is False + + +@pytest.mark.integration +def test_probe_status_with_missing_uds_socket() -> None: + """Probing a non-existent UDS socket should return False.""" + result = probe_status( + base_url="http+unix:///nonexistent/path/nemo.sock", + socket_path=Path("/nonexistent/path/nemo.sock"), + timeout=1.0, + ) + assert result is False + + +@pytest.mark.integration +def test_probe_status_against_real_daemon(tmp_path: Path) -> None: + """``probe_status`` should return True against a running daemon.""" + from nemo_platform.local import services + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + controllers=(), + sidecars=(), + transport="tcp", + host="127.0.0.1", + port=_free_tcp_port(), + scope="integ-probe-real", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + readiness_timeout=30.0, + readiness_poll_interval=0.3, + ) + services.daemonize_services(cfg) + try: + assert probe_status(base_url=f"http://127.0.0.1:{cfg.port}", timeout=5.0) + finally: + process.stop_instance(cfg.scope, base_dir=cfg.state_root, timeout=10, force=True) + + +# --------------------------------------------------------------------------- +# Priority 7: Child Process Module +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_service_child_loads_config_and_starts(tmp_path: Path) -> None: + """Write valid JSON config, run ``_service_child`` in a subprocess, + verify it starts and accepts HTTP connections.""" + port = _free_tcp_port() + state_dir = tmp_path / "state" + runtime_dir = tmp_path / "runtime" + scope = "integ-child-real" + + payload = ServiceRunConfig( + mode="daemon", + services=("hello-world",), + controllers=(), + sidecars=(), + transport="tcp", + host="127.0.0.1", + port=port, + scope=scope, + state_dir=str(state_dir), + runtime_dir=str(runtime_dir), + ).to_child_payload() + + # Write the request file the way daemonize_services does. + instance_dir = process.instance_dir(scope, base_dir=state_dir) + fd, req_path = tempfile.mkstemp(dir=str(instance_dir), suffix=".json") + os.write(fd, (json.dumps(payload) + "\n").encode()) + os.close(fd) + + log_path = process.log_path_for(scope, base_dir=state_dir) + log_file = open(log_path, "a") # noqa: SIM115 + subprocess.Popen( + [sys.executable, "-m", "nemo_platform.local._service_child", req_path], + stdout=log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + stdin=subprocess.DEVNULL, + close_fds=True, + ) + log_file.close() + + try: + # Wait for the child to become ready. + assert wait_for_status( + base_url=f"http://127.0.0.1:{port}", + timeout=30.0, + poll_interval=0.3, + ), "child process did not become ready" + + # The request file should have been cleaned up. + assert not Path(req_path).exists() + + # The child should have acquired the lock and written a descriptor. + assert process.is_instance_alive(scope, base_dir=state_dir) + desc = process.read_descriptor(scope, base_dir=state_dir) + assert desc is not None + assert desc.mode == "daemon" + finally: + process.stop_instance(scope, base_dir=state_dir, timeout=10, force=True) + + +@pytest.mark.integration +def test_service_child_corrupted_payload(tmp_path: Path) -> None: + """Bad JSON in the request file should cause the child to exit non-zero.""" + scope = "integ-child-bad" + state_dir = tmp_path / "state" + instance_dir = process.instance_dir(scope, base_dir=state_dir) + + fd, req_path = tempfile.mkstemp(dir=str(instance_dir), suffix=".json") + os.write(fd, b"<<>>") + os.close(fd) + + proc = subprocess.Popen( + [sys.executable, "-m", "nemo_platform.local._service_child", req_path], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + start_new_session=True, + stdin=subprocess.DEVNULL, + ) + proc.wait(timeout=15) + assert proc.returncode != 0 + + +@pytest.mark.integration +def test_service_child_cleans_up_request_file(tmp_path: Path) -> None: + """The request file should be unlinked even when the child crashes.""" + scope = "integ-child-cleanup" + state_dir = tmp_path / "state" + instance_dir = process.instance_dir(scope, base_dir=state_dir) + + fd, req_path = tempfile.mkstemp(dir=str(instance_dir), suffix=".json") + os.write(fd, b"<<>>") + os.close(fd) + + assert Path(req_path).exists() + + proc = subprocess.Popen( + [sys.executable, "-m", "nemo_platform.local._service_child", req_path], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + start_new_session=True, + stdin=subprocess.DEVNULL, + ) + proc.wait(timeout=15) + + # The request file should have been cleaned up regardless of the error. + assert not Path(req_path).exists() diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_port_socket.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_port_socket.py new file mode 100644 index 0000000000..243b65b24d --- /dev/null +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_port_socket.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for TCP/UDS port and socket management. + +These tests exercise real port binding, socket creation, and conflict +detection using actual OS resources. +""" + +from __future__ import annotations + +import socket +import tempfile +from pathlib import Path + +import pytest +from nemo_platform.local import process, services +from nemo_platform.local.services import ( + ServiceRunConfig, + ServicesPortInUseError, +) +from nmp.platform_runner.config import PlatformAppConfig + + +def _free_tcp_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_tcp_port_conflict_with_foreign_process(tmp_path: Path) -> None: + """When a foreign (non-NeMo) process holds a port, ``_check_tcp_available`` + should raise ``ServicesPortInUseError`` with a helpful suggestion.""" + # Bind a TCP port and hold it open. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as blocker: + blocker.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + blocker.bind(("127.0.0.1", 0)) + blocker.listen(1) + port = blocker.getsockname()[1] + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + transport="tcp", + host="127.0.0.1", + port=port, + scope="integ-port-foreign", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + + with pytest.raises(ServicesPortInUseError, match="already in use by another process"): + services._check_tcp_available(cfg) + + +@pytest.mark.integration +def test_tcp_port_conflict_with_nemo_instance(tmp_path: Path) -> None: + """When a NeMo instance holds a port, the error should distinguish it + from a foreign process.""" + scope = "integ-port-nemo" + state_dir = tmp_path / "state" + + # Bind a port and also create a descriptor matching the scope/host/port. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as blocker: + blocker.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + blocker.bind(("127.0.0.1", 0)) + blocker.listen(1) + port = blocker.getsockname()[1] + + # Create a live lock and descriptor so it looks like a NeMo instance. + lock_fd = process.acquire_lock(scope, base_dir=state_dir) + try: + desc = process.InstanceDescriptor( + pid=1, # Dummy PID — the flock is what matters. + config=PlatformAppConfig(scope=scope, host="127.0.0.1", port=port), + transport="tcp", + mode="daemon", + create_time=0.0, + ) + process.write_descriptor(desc, base_dir=state_dir) + + conflict = process.check_port_available_for_start("127.0.0.1", port, scope, base_dir=state_dir) + assert conflict is not None + assert conflict.kind == "nemo_instance" + assert conflict.port == port + + lines = process.format_port_conflict(conflict) + assert any("NeMo Platform" in line for line in lines) + finally: + import os + + os.close(lock_fd) + + +@pytest.mark.integration +def test_tcp_port_available_when_free(tmp_path: Path) -> None: + """When a port is free, ``check_port_available_for_start`` returns None.""" + port = _free_tcp_port() + conflict = process.check_port_available_for_start("127.0.0.1", port, "integ-free", base_dir=tmp_path / "state") + assert conflict is None + + +@pytest.mark.integration +def test_uds_socket_path_max_validation() -> None: + """A socket path exceeding AF_UNIX_PATH_MAX should raise ValueError.""" + # Build a path that is exactly one byte over the limit. + max_bytes = services._AF_UNIX_PATH_MAX_BYTES + # Create a path that exceeds the limit. + long_path = "/" + "x" * max_bytes # len("/") + max_bytes > max_bytes + assert len(long_path.encode()) > max_bytes + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + transport="uds", + socket_path=long_path, + scope="integ-long-sock", + state_dir="/tmp/state", + runtime_dir="/tmp/run", + ) + # _validated_socket_path calls _validate_socket_path_length internally. + with pytest.raises(ValueError, match="too long for AF_UNIX"): + services._validated_socket_path(cfg) + + +@pytest.mark.integration +def test_prepare_socket_removes_stale_socket(tmp_path: Path) -> None: + """``_prepare_socket`` should remove a stale (unreachable) socket file + and allow a new daemon to bind.""" + scope = "stale2" + short_tmp = Path(tempfile.mkdtemp(prefix="nemo-")) + runtime_dir = short_tmp / "run" + + # Create a stale socket file. + socket_dir = runtime_dir / scope + socket_dir.mkdir(parents=True, exist_ok=True) + stale_socket = socket_dir / "nemo-platform.sock" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.bind(str(stale_socket)) + assert stale_socket.exists() + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("hello-world",), + transport="uds", + scope=scope, + state_dir=short_tmp / "state", + runtime_dir=runtime_dir, + ) + + # _prepare_socket should probe, find it stale, remove it, and return the path. + result = services._prepare_socket(cfg) + assert result is not None + # The stale socket should have been removed (the new server hasn't bound yet). + assert not stale_socket.exists() + + import shutil + + shutil.rmtree(short_tmp, ignore_errors=True) diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_services.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_services.py new file mode 100644 index 0000000000..e2b2bca695 --- /dev/null +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_services.py @@ -0,0 +1,1039 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import os +import socket +import subprocess +import sys +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from nemo_platform.local import _service_child, services +from nemo_platform.local.process import ( + DESCRIPTOR_FILENAME, + InstanceDescriptor, +) +from nemo_platform.local.services import ServiceRunConfig +from nemo_platform.local.transport import UDS_BASE_URL +from nmp.platform_runner.config import ( + PlatformAppConfig, + default_runtime_root, + default_state_root, + validate_scope, +) + + +def _allow_tmp_path_socket_paths(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(services, "_AF_UNIX_PATH_MAX_BYTES", 4096) + + +def _free_tcp_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _embedded_handle() -> services.EmbeddedServiceHandle: + return services.EmbeddedServiceHandle(app=object(), runtime=object()) + + +def test_service_run_config_normalizes_lists_to_tuples() -> None: + cfg = ServiceRunConfig(services=["entities", "models"], controllers=["jobs"]) + + assert cfg.services == ("entities", "models") + assert cfg.controllers == ("jobs",) + + +def test_service_run_config_converts_to_platform_app_config(tmp_path: Path) -> None: + cfg = ServiceRunConfig( + services=["entities", "models"], + controllers=[], + sidecars=["adapters"], + config_path=tmp_path / "local.yaml", + socket_path=tmp_path / "nemo.sock", + mode="embedded", + ) + + app_config = cfg.to_platform_app_config() + + assert app_config.services == ("entities", "models") + assert app_config.controllers == () + assert app_config.sidecars == ("adapters",) + assert app_config.config_path == str(tmp_path / "local.yaml") + assert app_config.socket_path == str(tmp_path / "nemo.sock") + assert app_config.runtime_root is None + assert app_config.runtime_dir() == tmp_path + assert app_config.host == "127.0.0.1" + assert app_config.port == 8080 + + +def test_instance_descriptor_converts_from_service_run_config( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + cfg = ServiceRunConfig( + services=["entities", "models"], + controllers=[], + sidecars=["adapters"], + config_path=tmp_path / "local.yaml", + socket_path=tmp_path / "nemo.sock", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + ) + monkeypatch.setattr(services.process, "get_create_time", lambda _pid: 123.0) + app_config = cfg.to_platform_app_config() + app_config.log_path = str(tmp_path / "nemo.log") + + desc = InstanceDescriptor.from_config( + app_config, + pid=4242, + mode="daemon", + transport=cfg.transport, + ) + + assert desc.pid == 4242 + assert desc.config.scope == "default" + assert desc.config.host == "127.0.0.1" + assert desc.config.port == 8080 + assert desc.transport == "uds" + assert desc.config.socket_path == str(tmp_path / "nemo.sock") + assert desc.config.state_root == str(tmp_path / "state") + assert desc.config.runtime_root == str(tmp_path / "run") + assert desc.config.state_dir() == tmp_path / "state" / "instances" / "default" + assert desc.config.runtime_dir() == tmp_path / "run" / "default" + assert desc.mode == "daemon" + assert desc.create_time == 123.0 + assert desc.config.services == ("entities", "models") + assert desc.config.controllers == () + assert desc.config.sidecars == ("adapters",) + assert desc.config.config_path == str(tmp_path / "local.yaml") + assert desc.config.log_path == str(tmp_path / "nemo.log") + assert desc.config.log_file_path() == tmp_path / "nemo.log" + payload = desc.model_dump() + assert "services" not in payload + assert "host" not in payload + assert "state_dir" not in payload + assert "runtime_dir" not in payload + assert "log_path" not in payload + assert payload["config"]["services"] == ("entities", "models") + assert payload["config"]["socket_path"] == str(tmp_path / "nemo.sock") + assert payload["config"]["state_root"] == str(tmp_path / "state") + assert payload["config"]["runtime_root"] == str(tmp_path / "run") + assert payload["config"]["log_path"] == str(tmp_path / "nemo.log") + + +def test_service_mode_enum_values() -> None: + assert services.ServiceMode.EMBEDDED.value == "embedded" + assert services.ServiceMode.DAEMON.value == "daemon" + + +def test_service_run_config_defaults_to_daemon_mode() -> None: + cfg = ServiceRunConfig() + + assert cfg.mode is services.ServiceMode.DAEMON + + +def test_service_run_config_accepts_mode_strings() -> None: + cfg = ServiceRunConfig(mode="embedded") + + assert cfg.mode is services.ServiceMode.EMBEDDED + + +def test_service_run_config_rejects_unknown_mode() -> None: + with pytest.raises(ValueError, match="mode must be 'embedded' or 'daemon'"): + ServiceRunConfig(mode="foreground") + + +def test_embedded_and_daemon_handles_implement_local_service_handle(tmp_path: Path) -> None: + embedded = services.EmbeddedServiceHandle(app=object(), runtime=object()) + daemon = services.DaemonServiceHandle( + scope="dev", + transport="tcp", + socket_path=None, + gateway_base_url=None, + host="127.0.0.1", + port=8080, + pid=123, + mode="daemon", + log_path=None, + state_dir=tmp_path / "state" / "instances" / "dev", + runtime_dir=None, + ) + + assert isinstance(embedded, services.LocalServiceHandle) + assert isinstance(daemon, services.LocalServiceHandle) + + +def test_start_services_result_is_shared_result_type() -> None: + result = services.StartServicesResult( + requested=["jobs"], + started=["auth", "jobs"], + already_active=[], + active=["secrets", "auth", "jobs"], + ) + + assert result.requested == ["jobs"] + assert result.started == ["auth", "jobs"] + assert result.active == ["secrets", "auth", "jobs"] + + +def test_service_run_config_rejects_services_with_service_group() -> None: + with pytest.raises(ValueError, match="services cannot be combined with service_group"): + ServiceRunConfig(services=("entities",), service_group="all") + + +def test_service_run_config_defaults_to_named_uds_instance() -> None: + cfg = ServiceRunConfig() + + assert cfg.transport == "uds" + assert cfg.http_gateway == "disabled" + assert cfg.scope == "default" + assert cfg.socket_path is None + + +@pytest.mark.parametrize("instance", ["has space", "../bad"]) +def test_service_run_config_rejects_invalid_scope_names(instance: str) -> None: + with pytest.raises(ValueError, match="scope"): + ServiceRunConfig(scope=instance) + + +def test_service_run_config_rejects_gateway_for_tcp_transport() -> None: + with pytest.raises(ValueError, match="gateway.*UDS"): + ServiceRunConfig(transport="tcp", http_gateway="enabled") + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("readiness_timeout", 0.0, "readiness_timeout"), + ("readiness_timeout", -1.0, "readiness_timeout"), + ("readiness_poll_interval", 0.0, "readiness_poll_interval"), + ("readiness_poll_interval", -1.0, "readiness_poll_interval"), + ], +) +def test_service_run_config_rejects_non_positive_readiness_values(field: str, value: float, message: str) -> None: + with pytest.raises(ValueError, match=message): + if field == "readiness_timeout": + ServiceRunConfig(readiness_timeout=value) + else: + ServiceRunConfig(readiness_poll_interval=value) + + +def test_process_paths_follow_existing_nmp_state_convention(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + + assert default_state_root() == tmp_path / "state" / "nmp" + assert default_runtime_root() == tmp_path / "state" / "nmp" / "run" + assert ( + PlatformAppConfig(scope="dev").socket_file_path() + == tmp_path / "state" / "nmp" / "run" / "dev" / "nemo-platform.sock" + ) + assert validate_scope("dev_1-2") == "dev_1-2" + + +def test_resolved_socket_path_rejects_relative_explicit_path() -> None: + cfg = ServiceRunConfig(socket_path="relative.sock") + + with pytest.raises(ValueError, match="UDS socket path must be absolute"): + _ = cfg.resolved_socket_path + + +def test_resolved_socket_path_rejects_relative_runtime_dir() -> None: + cfg = ServiceRunConfig(runtime_dir="relative-run") + + with pytest.raises(ValueError, match="runtime root must be absolute"): + _ = cfg.resolved_socket_path + + +def test_resolved_socket_path_rejects_relative_socket_path_with_tcp_client() -> None: + cfg = ServiceRunConfig(transport="tcp", socket_path="relative.sock") + + with pytest.raises(ValueError, match="UDS socket path must be absolute"): + _ = cfg.resolved_socket_path + + +def test_tcp_client_can_still_configure_uds_listener(tmp_path: Path) -> None: + cfg = ServiceRunConfig(transport="tcp", socket_path=tmp_path / "nemo.sock") + + app_config = cfg.to_platform_app_config() + + assert app_config.socket_path == str(tmp_path / "nemo.sock") + assert app_config.runtime_dir() == tmp_path + + +def test_instance_descriptor_rejects_uds_client_without_socket_path() -> None: + with pytest.raises(ValueError, match="UDS client transport requires config.socket_path"): + InstanceDescriptor(pid=1, config=PlatformAppConfig(scope="dev"), transport="uds") + + +def test_prepare_socket_rejects_long_generated_path_before_creating_runtime_dir( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(services, "_AF_UNIX_PATH_MAX_BYTES", 1, raising=False) + runtime_root = tmp_path / "runtime" + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=runtime_root) + + with pytest.raises(ValueError, match="UDS socket path is too long.*AF_UNIX"): + services._prepare_socket(cfg) + + assert not runtime_root.exists() + + +def test_validate_socket_path_length_reserves_trailing_nul(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(services, "_AF_UNIX_PATH_MAX_BYTES", 3, raising=False) + + services._validate_socket_path_length(Path("abc")) + with pytest.raises(ValueError, match=r"4 bytes; maximum is 3 bytes"): + services._validate_socket_path_length(Path("abcd")) + + +def test_prepare_socket_rejects_long_explicit_path_before_filesystem_or_probe( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(services, "_AF_UNIX_PATH_MAX_BYTES", 1, raising=False) + socket_parent = tmp_path / "explicit" + cfg = ServiceRunConfig(socket_path=socket_parent / "nemo-platform.sock") + + with patch("nemo_platform.local.services.probe_status") as probe_status: + with pytest.raises(ValueError, match="UDS socket path is too long.*AF_UNIX"): + services._prepare_socket(cfg) + + probe_status.assert_not_called() + assert not socket_parent.exists() + + +def test_run_services_prepares_socket_after_acquiring_lock(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", port=_free_tcp_port(), state_dir=tmp_path / "state", runtime_dir=tmp_path / "run" + ) + events: list[str] = [] + real_acquire_lock = services.process.acquire_lock + + def acquire_lock(scope: str, *, base_dir: Path | None = None) -> int: + events.append("lock") + return real_acquire_lock(scope, base_dir=base_dir) + + def prepare_socket(config: ServiceRunConfig) -> Path | None: + events.append("prepare") + lock_path = ( + services.process.instance_dir(config.scope, base_dir=config.state_root) / services.process.LOCK_FILENAME + ) + assert lock_path.exists() + return config.resolved_socket_path + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services.process.acquire_lock", side_effect=acquire_lock), + patch("nemo_platform.local.services._prepare_socket", side_effect=prepare_socket), + patch("nemo_platform.local.services.start_embedded_services", return_value=_embedded_handle()), + patch("nemo_platform.local.services.serve_embedded_app"), + ): + services.run_services(cfg) + + assert events == ["lock", "prepare"] + + +def test_run_services_cleans_lock_when_socket_prepare_fails(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", port=_free_tcp_port(), state_dir=tmp_path / "state", runtime_dir=tmp_path / "run" + ) + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch( + "nemo_platform.local.services._prepare_socket", + side_effect=services.ServicesSocketStaleError("boom"), + ), + ): + with pytest.raises(services.ServicesSocketStaleError, match="boom"): + services.run_services(cfg) + + assert not services.process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + + +def test_run_services_restores_env_and_closes_lock_when_descriptor_cleanup_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + port=_free_tcp_port(), + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + data_dir=tmp_path / "data", + ) + monkeypatch.delenv("NMP_DATA_DIR", raising=False) + real_acquire_lock = services.process.acquire_lock + locked_fd: int | None = None + + def acquire_lock(scope: str, *, base_dir: Path | None = None) -> int: + nonlocal locked_fd + locked_fd = real_acquire_lock(scope, base_dir=base_dir) + return locked_fd + + real_close = os.close + closed_fds: list[int] = [] + + def close(fd: int) -> None: + closed_fds.append(fd) + real_close(fd) + + monkeypatch.setattr(os, "close", close) + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services.process.acquire_lock", side_effect=acquire_lock), + patch( + "nemo_platform.local.services.process.remove_descriptor", + side_effect=RuntimeError("descriptor cleanup failed"), + ), + patch("nemo_platform.local.services.start_embedded_services", return_value=_embedded_handle()), + patch("nemo_platform.local.services.serve_embedded_app"), + ): + with pytest.raises(RuntimeError, match="descriptor cleanup failed"): + services.run_services(cfg) + + assert "NMP_DATA_DIR" not in os.environ + assert locked_fd is not None + assert locked_fd in closed_fds + assert closed_fds[-1] == locked_fd + + +def test_run_services_foreground_serves_embedded_app(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + mode=services.ServiceMode.EMBEDDED, + scope="dev", + port=_free_tcp_port(), + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + ) + app = object() + handle = services.EmbeddedServiceHandle(app=app, runtime=object()) + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services.start_embedded_services", return_value=handle) as start_embedded, + patch("nemo_platform.local.services.serve_embedded_app") as serve_embedded, + ): + services.run_services(cfg) + + start_embedded.assert_called_once_with(cfg, env=None) + serve_embedded.assert_called_once() + assert serve_embedded.call_args.args[0] is app + + +def test_daemon_service_handle_uds_client_uses_socket_transport(tmp_path: Path) -> None: + socket_path = tmp_path / "nemo-platform.sock" + handle = services.DaemonServiceHandle( + scope="dev", + transport="uds", + socket_path=socket_path, + gateway_base_url="http://127.0.0.1:9999", + host="127.0.0.1", + port=8080, + pid=123, + mode="daemon", + log_path=None, + state_dir=tmp_path / "state" / "instances" / "dev", + runtime_dir=tmp_path, + ) + + client = handle.client() + try: + assert str(client.base_url).rstrip("/") == UDS_BASE_URL + assert handle.gateway_base_url == "http://127.0.0.1:9999" + finally: + client.close() + + +def test_embedded_handle_async_client_uses_asgi_transport() -> None: + app = MagicMock() + runtime = MagicMock() + http_client = object() + client_value = object() + handle = services.EmbeddedServiceHandle(app=app, runtime=runtime) + + with ( + patch( + "nemo_platform.local.services.build_async_asgi_http_client", return_value=http_client + ) as build_client, + patch("nemo_platform.local.services.AsyncNeMoPlatform", return_value=client_value) as platform_cls, + ): + client = handle.async_client(access_token="test-token") + + build_client.assert_called_once_with(app) + platform_cls.assert_called_once_with( + access_token="test-token", + http_client=http_client, + base_url=services.EMBEDDED_BASE_URL, + ) + assert client is client_value + + +def test_ensure_services_dispatches_to_embedded_mode() -> None: + cfg = ServiceRunConfig(mode=services.ServiceMode.EMBEDDED) + embedded_handle = MagicMock(spec=services.EmbeddedServiceHandle) + + with patch("nemo_platform.local.services.start_embedded_services", return_value=embedded_handle): + handle = services.ensure_services(cfg) + + assert handle is embedded_handle + + +def test_ensure_services_dispatches_to_daemon_mode() -> None: + cfg = ServiceRunConfig(mode=services.ServiceMode.DAEMON) + daemon_handle = MagicMock(spec=services.DaemonServiceHandle) + + with ( + patch("nemo_platform.local.services.get_service_handle", return_value=None), + patch("nemo_platform.local.services.daemonize_services", return_value=daemon_handle), + ): + handle = services.ensure_services(cfg) + + assert handle is daemon_handle + + +def test_connect_services_uses_selected_mode_handle_client() -> None: + cfg = ServiceRunConfig(mode=services.ServiceMode.EMBEDDED) + handle = MagicMock(spec=services.EmbeddedServiceHandle) + client = object() + handle.client.return_value = client + + with patch("nemo_platform.local.services.ensure_services", return_value=handle): + result = services.connect_services(cfg, access_token="test") + + assert result is client + handle.client.assert_called_once_with(access_token="test") + + +@pytest.mark.parametrize("mode", [services.ServiceMode.EMBEDDED, services.ServiceMode.DAEMON]) +def test_ensure_services_returns_handle_with_parity_methods(mode: services.ServiceMode) -> None: + cfg = ServiceRunConfig(mode=mode) + if mode is services.ServiceMode.EMBEDDED: + handle = MagicMock(spec=services.EmbeddedServiceHandle) + patch_target = "nemo_platform.local.services.start_embedded_services" + else: + handle = MagicMock(spec=services.DaemonServiceHandle) + patch_target = "nemo_platform.local.services.daemonize_services" + + with ( + patch("nemo_platform.local.services.get_service_handle", return_value=None), + patch(patch_target, return_value=handle), + ): + result = services.ensure_services(cfg) + + assert result is handle + for method_name in ( + "is_running", + "wait_until_ready", + "wait_until_ready_async", + "client", + "async_client", + "start_services", + "start_services_async", + "stop", + "stop_async", + ): + assert hasattr(result, method_name), method_name + + +def test_daemonize_services_starts_child_module(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + service_group="all", + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + readiness_timeout=0.1, + readiness_poll_interval=0.01, + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services._check_tcp_available"), + patch("nemo_platform.local.services.probe_status", return_value=True), + patch("nemo_platform.local.services.subprocess.Popen", return_value=proc) as popen, + ): + handle = services.daemonize_services(cfg) + + args = popen.call_args.args[0] + assert args[:3] == [sys.executable, "-m", f"{services.__package__}._service_child"] + request_path = Path(args[3]) + assert request_path.parent == tmp_path / "state" / "instances" / "dev" + assert request_path.suffix == ".json" + assert request_path.name != "run-request.json" + assert handle.transport == "uds" + assert handle.socket_path == tmp_path / "run" / "dev" / "nemo-platform.sock" + assert handle.pid == 4242 + proc.terminate.assert_not_called() + proc.kill.assert_not_called() + + +def test_daemonize_services_leaves_socket_preparation_to_child(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + readiness_timeout=0.1, + readiness_poll_interval=0.01, + ) + socket_path = cfg.resolved_socket_path + assert socket_path is not None + socket_path.parent.mkdir(parents=True) + socket_path.write_text("stale", encoding="utf-8") + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services._prepare_socket", side_effect=AssertionError("parent prepared socket")), + patch("nemo_platform.local.services.probe_status", side_effect=[False, True]), + patch("nemo_platform.local.services.subprocess.Popen", return_value=proc), + ): + handle = services.daemonize_services(cfg) + + assert handle.socket_path == socket_path + assert socket_path.read_text(encoding="utf-8") == "stale" + + +def test_write_run_request_writes_complete_payload_when_os_write_is_short( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + real_write = os.write + + def short_write(fd: int, data: bytes) -> int: + return real_write(fd, data[: max(1, len(data) // 2)]) + + monkeypatch.setattr(services.os, "write", short_write) + + request_path = services._write_run_request(cfg) + + expected_payload = json.dumps(cfg.to_child_payload(), indent=2) + "\n" + assert request_path.read_text(encoding="utf-8") == expected_payload + + +def test_service_child_unlinks_request_after_read(tmp_path: Path) -> None: + request_path = tmp_path / "run-request.json" + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + request_path.write_text(json.dumps(cfg.to_child_payload()), encoding="utf-8") + + with patch("nemo_platform.local._service_child.run_services") as run_services: + result = _service_child.main([str(request_path)]) + + assert result == 0 + assert not request_path.exists() + child_cfg = run_services.call_args.args[0] + assert child_cfg.scope == "dev" + assert child_cfg.state_dir == str(tmp_path / "state") + assert child_cfg.runtime_dir == str(tmp_path / "run") + assert run_services.call_args.kwargs == {"_mode": "daemon"} + + +def test_daemonize_services_terminates_child_on_timeout(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + readiness_timeout=0.01, + readiness_poll_interval=0.001, + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services.probe_status", return_value=False), + patch("nemo_platform.local.services.subprocess.Popen", return_value=proc), + ): + with pytest.raises(services.ServicesStartupTimeoutError): + services.daemonize_services(cfg) + + proc.terminate.assert_called_once_with() + proc.kill.assert_not_called() + + +def test_daemonize_services_bounds_probe_and_sleep_by_remaining_deadline( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + readiness_timeout=5.0, + readiness_poll_interval=10.0, + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services.probe_status", return_value=False) as probe_status, + patch("nemo_platform.local.services.subprocess.Popen", return_value=proc), + patch("nemo_platform.local.services.time.monotonic", side_effect=[0.0, 4.0, 4.5, 5.0]), + patch("nemo_platform.local.services.time.sleep") as sleep, + ): + with pytest.raises(services.ServicesStartupTimeoutError): + services.daemonize_services(cfg) + + assert probe_status.call_args.kwargs["timeout"] == pytest.approx(1.0) + sleep.assert_called_once() + assert sleep.call_args.args[0] == pytest.approx(0.5) + proc.terminate.assert_called_once_with() + + +def test_daemonize_services_terminates_child_on_handle_error(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + transport="tcp", + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services._check_tcp_available"), + patch("nemo_platform.local.services.subprocess.Popen", return_value=proc), + patch( + "nemo_platform.local.services.DaemonServiceHandle.from_config", + side_effect=RuntimeError("handle failed"), + ), + ): + with pytest.raises(RuntimeError, match="handle failed"): + services.daemonize_services(cfg) + + proc.terminate.assert_called_once_with() + proc.kill.assert_not_called() + + +def test_daemonize_services_terminates_child_on_probe_error(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + transport="tcp", + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services._check_tcp_available"), + patch("nemo_platform.local.services.probe_status", side_effect=RuntimeError("probe failed")), + patch("nemo_platform.local.services.subprocess.Popen", return_value=proc), + ): + with pytest.raises(RuntimeError, match="probe failed"): + services.daemonize_services(cfg) + + proc.terminate.assert_called_once_with() + proc.kill.assert_not_called() + + +def test_daemonize_services_terminates_child_on_sleep_interruption( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + transport="tcp", + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services._check_tcp_available"), + patch("nemo_platform.local.services.probe_status", return_value=False), + patch("nemo_platform.local.services.subprocess.Popen", return_value=proc), + patch("nemo_platform.local.services.time.sleep", side_effect=KeyboardInterrupt), + ): + with pytest.raises(KeyboardInterrupt): + services.daemonize_services(cfg) + + proc.terminate.assert_called_once_with() + proc.kill.assert_not_called() + + +def test_daemonize_services_kills_child_when_terminate_times_out( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + transport="tcp", + readiness_timeout=0.01, + readiness_poll_interval=0.001, + ) + proc = MagicMock() + proc.pid = 4242 + proc.poll.return_value = None + proc.wait.side_effect = [subprocess.TimeoutExpired("nemo services", 5), None] + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services._check_tcp_available"), + patch("nemo_platform.local.services.probe_status", return_value=False), + patch("nemo_platform.local.services.subprocess.Popen", return_value=proc), + ): + with pytest.raises(services.ServicesStartupTimeoutError): + services.daemonize_services(cfg) + + proc.terminate.assert_called_once_with() + proc.kill.assert_called_once_with() + + +async def test_daemonize_services_async_uses_thread(tmp_path: Path) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + handle = MagicMock() + + with patch("nemo_platform.local.services.asyncio.to_thread", new=AsyncMock(return_value=handle)) as to_thread: + result = await services.daemonize_services_async(cfg) + + assert result is handle + to_thread.assert_awaited_once_with(services.daemonize_services, cfg) + + +def test_run_services_serves_embedded_app_with_socket_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + services=["entities"], + controllers=["jobs"], + scope="dev", + port=_free_tcp_port(), + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + ) + handle = _embedded_handle() + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services.start_embedded_services", return_value=handle) as start_embedded, + patch("nemo_platform.local.services.serve_embedded_app") as serve_embedded, + ): + services.run_services(cfg, _mode="daemon") + + start_embedded.assert_called_once_with(cfg, env=None) + serve_embedded.assert_called_once_with(handle.app, cfg, tmp_path / "run" / "dev" / "nemo-platform.sock") + assert not (tmp_path / "state" / "instances" / "dev" / DESCRIPTOR_FILENAME).exists() + + +def test_serve_embedded_app_with_socket_path_listens_on_tcp_and_uds(tmp_path: Path) -> None: + cfg = ServiceRunConfig(transport="tcp", host="127.0.0.1", port=9090) + app = object() + socket_path = tmp_path / "nemo.sock" + + with patch("nmp.platform_runner.server._run_server_on_bound_sockets") as run_bound_sockets: + services.serve_embedded_app(app, cfg, socket_path) + + run_bound_sockets.assert_called_once_with(app, host="127.0.0.1", port=9090, socket_path=str(socket_path)) + + +def test_run_services_cleans_lock_when_log_path_resolution_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", port=_free_tcp_port(), state_dir=tmp_path / "state", runtime_dir=tmp_path / "run" + ) + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch.object(PlatformAppConfig, "log_file_path", side_effect=RuntimeError("boom")), + ): + with pytest.raises(RuntimeError, match="boom"): + services.run_services(cfg) + + assert not services.process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + + +def test_run_services_restores_data_dir_and_lock_when_descriptor_write_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", + port=_free_tcp_port(), + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "run", + data_dir=tmp_path / "data", + ) + monkeypatch.delenv("NMP_DATA_DIR", raising=False) + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services.process.write_descriptor", side_effect=RuntimeError("boom")), + ): + with pytest.raises(RuntimeError, match="boom"): + services.run_services(cfg) + + assert "NMP_DATA_DIR" not in os.environ + assert not services.process.is_instance_alive(cfg.scope, base_dir=cfg.state_root) + + +def test_run_services_restores_existing_data_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _allow_tmp_path_socket_paths(monkeypatch) + cfg = ServiceRunConfig( + scope="dev", port=_free_tcp_port(), state_dir=tmp_path / "state", runtime_dir=tmp_path / "run" + ) + monkeypatch.setenv("NMP_DATA_DIR", "/shell/data") + + with ( + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services.start_embedded_services", return_value=_embedded_handle()), + patch("nemo_platform.local.services.serve_embedded_app"), + ): + services.run_services(cfg) + + assert os.environ["NMP_DATA_DIR"] == "/shell/data" + + +def test_daemon_service_handle_tcp_client_uses_tcp_base_url(tmp_path: Path) -> None: + handle = services.DaemonServiceHandle( + scope="dev", + transport="tcp", + socket_path=None, + gateway_base_url=None, + host="0.0.0.0", + port=9090, + pid=123, + mode="daemon", + log_path=None, + state_dir=tmp_path / "state" / "instances" / "dev", + runtime_dir=None, + ) + + with patch("nemo_platform.local.services.NeMoPlatform") as sdk: + handle.client(timeout=12) + + sdk.assert_called_once_with(timeout=12, base_url="http://localhost:9090") + + +def test_daemon_service_handle_uds_client_requires_socket_path(tmp_path: Path) -> None: + handle = services.DaemonServiceHandle( + scope="dev", + transport="uds", + socket_path=None, + gateway_base_url=None, + host="127.0.0.1", + port=8080, + pid=123, + mode="daemon", + log_path=None, + state_dir=tmp_path / "state" / "instances" / "dev", + runtime_dir=tmp_path / "run", + ) + + with pytest.raises(services.ServicesError, match="missing socket_path"): + handle.client() + + +def test_ensure_services_returns_existing_handle(tmp_path: Path) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + handle = MagicMock() + + with ( + patch("nemo_platform.local.services.get_service_handle", return_value=handle), + patch("nemo_platform.local.services.daemonize_services") as daemonize, + ): + result = services.ensure_services(cfg) + + assert result is handle + daemonize.assert_not_called() + + +def test_connect_services_respects_start_if_needed_false(tmp_path: Path) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + + with patch("nemo_platform.local.services.get_service_handle", return_value=None): + with pytest.raises(services.ServicesNotRunningError, match="not running"): + services.connect_services(cfg, start_if_needed=False) + + +def test_stop_services_delegates_to_handle(tmp_path: Path) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + handle = MagicMock() + stop_result = MagicMock() + handle.stop.return_value = stop_result + + with patch("nemo_platform.local.services.get_service_handle", return_value=handle): + result = services.stop_services(cfg, timeout=3.0, force=True) + + assert result is stop_result + handle.stop.assert_called_once_with(timeout=3.0, force=True) + + +def test_get_service_handle_returns_none_without_live_descriptor(tmp_path: Path) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + + with patch("nemo_platform.local.services.process.read_descriptor", return_value=None): + assert services.get_service_handle(cfg) is None + + +def test_list_service_handles_filters_dead_or_descriptorless_instances(tmp_path: Path) -> None: + live_desc = InstanceDescriptor( + pid=123, + transport="tcp", + config=PlatformAppConfig(scope="live", state_root=tmp_path / "state"), + mode="daemon", + ) + infos = [ + MagicMock(descriptor=live_desc, alive=True), + MagicMock(descriptor=None, alive=True), + MagicMock(descriptor=live_desc, alive=False), + ] + + with patch("nemo_platform.local.services.process.list_instances", return_value=infos): + handles = services.list_service_handles(tmp_path / "state") + + assert [handle.scope for handle in handles] == ["live"] + + +def test_get_service_handle_reads_live_descriptor(tmp_path: Path) -> None: + cfg = ServiceRunConfig(scope="dev", state_dir=tmp_path / "state", runtime_dir=tmp_path / "run") + state_dir = tmp_path / "state" / "instances" / "dev" + state_dir.mkdir(parents=True) + desc = InstanceDescriptor( + pid=123, + config=PlatformAppConfig( + scope="dev", + socket_path=str(tmp_path / "run" / "dev" / "nemo-platform.sock"), + state_root=tmp_path / "state", + runtime_root=tmp_path / "run", + ), + transport="uds", + mode="daemon", + ) + (state_dir / DESCRIPTOR_FILENAME).write_text(desc.model_dump_json(), encoding="utf-8") + + with patch("nemo_platform.local.process.is_instance_alive", return_value=True): + handle = services.get_service_handle(cfg) + + assert handle is not None + assert handle.scope == "dev" + assert handle.transport == "uds" diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_services_contract.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_services_contract.py new file mode 100644 index 0000000000..747a3b0850 --- /dev/null +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_services_contract.py @@ -0,0 +1,334 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import threading +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from nemo_platform.local import services +from nemo_platform.local.process import StopResult +from nemo_platform.local.services import ServiceRunConfig +from nmp.platform_runner.config import PlatformAppConfig + + +@dataclass(frozen=True) +class ModeContractCase: + mode: services.ServiceMode + launcher_patch: str + existing_handle_patch_value: object | None + + +@dataclass +class ContractHandle: + mode: services.ServiceMode + calls: list[tuple[str, object]] = field(default_factory=list) + + def is_running(self) -> bool: + self.calls.append(("is_running", None)) + return True + + def wait_until_ready(self, timeout: float | None = None) -> None: + self.calls.append(("wait_until_ready", timeout)) + + async def wait_until_ready_async(self, timeout: float | None = None) -> None: + self.calls.append(("wait_until_ready_async", timeout)) + + def client(self, **kwargs: object) -> tuple[str, services.ServiceMode, dict[str, object]]: + self.calls.append(("client", kwargs)) + return ("client", self.mode, kwargs) + + def async_client(self, **kwargs: object) -> tuple[str, services.ServiceMode, dict[str, object]]: + self.calls.append(("async_client", kwargs)) + return ("async_client", self.mode, kwargs) + + def start_services(self, service_names: list[str] | tuple[str, ...]) -> services.StartServicesResult: + requested = list(service_names) + self.calls.append(("start_services", requested)) + return services.StartServicesResult( + requested=requested, + started=["auth", *requested], + already_active=[], + active=["secrets", "auth", *requested], + ) + + async def start_services_async(self, service_names: list[str] | tuple[str, ...]) -> services.StartServicesResult: + requested = list(service_names) + self.calls.append(("start_services_async", requested)) + return services.StartServicesResult( + requested=requested, + started=["auth", *requested], + already_active=[], + active=["secrets", "auth", *requested], + ) + + def stop(self, *, timeout: float = 30.0, force: bool = False) -> StopResult: + self.calls.append(("stop", {"timeout": timeout, "force": force})) + return StopResult(stopped_pids=[], swept_children=[]) + + async def stop_async(self, *, timeout: float = 30.0, force: bool = False) -> StopResult: + self.calls.append(("stop_async", {"timeout": timeout, "force": force})) + return StopResult(stopped_pids=[], swept_children=[]) + + +MODE_CONTRACT_CASES = [ + ModeContractCase( + mode=services.ServiceMode.EMBEDDED, + launcher_patch="nemo_platform.local.services.start_embedded_services", + existing_handle_patch_value=None, + ), + ModeContractCase( + mode=services.ServiceMode.DAEMON, + launcher_patch="nemo_platform.local.services.daemonize_services", + existing_handle_patch_value=None, + ), +] + + +@pytest.fixture(params=MODE_CONTRACT_CASES, ids=lambda case: case.mode.value) +def mode_case(request: pytest.FixtureRequest) -> ModeContractCase: + return request.param + + +def _config_for(case: ModeContractCase, tmp_path: Path) -> ServiceRunConfig: + return ServiceRunConfig( + mode=case.mode, + services=("secrets",), + scope=f"{case.mode.value}-contract", + state_dir=tmp_path / case.mode.value / "state", + runtime_dir=tmp_path / case.mode.value / "runtime", + ) + + +def test_contract_ensure_services_returns_running_mode_handle( + mode_case: ModeContractCase, + tmp_path: Path, +) -> None: + cfg = _config_for(mode_case, tmp_path) + handle = ContractHandle(mode_case.mode) + + with ( + patch( + "nemo_platform.local.services.get_service_handle", return_value=mode_case.existing_handle_patch_value + ), + patch(mode_case.launcher_patch, return_value=handle), + ): + result = services.ensure_services(cfg) + + assert result is handle + assert result.is_running() is True + assert result.calls == [("is_running", None)] + + +def test_contract_connect_services_returns_client_from_selected_mode( + mode_case: ModeContractCase, + tmp_path: Path, +) -> None: + cfg = _config_for(mode_case, tmp_path) + handle = ContractHandle(mode_case.mode) + + with patch("nemo_platform.local.services.ensure_services", return_value=handle): + client = services.connect_services(cfg, api_key="test-key") + + assert client == ("client", mode_case.mode, {"api_key": "test-key"}) + assert handle.calls == [("client", {"api_key": "test-key"})] + + +@pytest.mark.asyncio +async def test_contract_handle_lifecycle_methods_have_same_semantics( + mode_case: ModeContractCase, +) -> None: + handle = ContractHandle(mode_case.mode) + + handle.wait_until_ready(timeout=1.5) + await handle.wait_until_ready_async(timeout=2.5) + sync_start = handle.start_services(["jobs"]) + async_start = await handle.start_services_async(["jobs"]) + stop_result = handle.stop(timeout=3.0, force=True) + async_stop_result = await handle.stop_async(timeout=4.0, force=False) + + assert sync_start == services.StartServicesResult( + requested=["jobs"], + started=["auth", "jobs"], + already_active=[], + active=["secrets", "auth", "jobs"], + ) + assert async_start == sync_start + assert stop_result == StopResult(stopped_pids=[], swept_children=[]) + assert async_stop_result == StopResult(stopped_pids=[], swept_children=[]) + assert handle.calls == [ + ("wait_until_ready", 1.5), + ("wait_until_ready_async", 2.5), + ("start_services", ["jobs"]), + ("start_services_async", ["jobs"]), + ("stop", {"timeout": 3.0, "force": True}), + ("stop_async", {"timeout": 4.0, "force": False}), + ] + + +def test_contract_real_handles_report_same_staged_start_status_before_staged_start_lands(tmp_path: Path) -> None: + embedded = services.EmbeddedServiceHandle(app=object(), runtime=object()) + daemon = services.DaemonServiceHandle( + scope="daemon-contract", + transport="tcp", + socket_path=None, + gateway_base_url=None, + host="127.0.0.1", + port=8080, + pid=None, + mode="daemon", + log_path=None, + state_dir=tmp_path / "state", + runtime_dir=None, + ) + + for handle in (embedded, daemon): + with pytest.raises(services.ServicesError, match="Staged service start is not implemented"): + handle.start_services(["jobs"]) + + +def test_contract_embedded_and_daemon_child_both_delegate_models_to_platform_builder( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, object]] = [] + + def fake_build_platform_app( + config: PlatformAppConfig | None = None, + *, + env: object = None, + http_client: object = None, + ) -> MagicMock: + calls.append({"config": config, "env": env, "http_client": http_client}) + return MagicMock() + + def service_config(mode: services.ServiceMode) -> ServiceRunConfig: + return ServiceRunConfig( + mode=mode, + services=("models",), + controllers=(), + transport="tcp", + scope="sc-test", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + + with patch("nmp.platform_runner.server.build_platform_app", side_effect=fake_build_platform_app): + services.start_embedded_services(service_config(services.ServiceMode.EMBEDDED)) + + with ( + patch("nmp.platform_runner.server.build_platform_app", side_effect=fake_build_platform_app), + patch("nemo_platform.local.services.require_services_extra"), + patch("nemo_platform.local.services.process.is_instance_alive", return_value=False), + patch("nemo_platform.local.services._check_tcp_available"), + patch("nemo_platform.local.services.process.acquire_lock", return_value=123), + patch("nemo_platform.local.services.process.log_path_for", return_value=tmp_path / "nemo.log"), + patch("nemo_platform.local.services.process.write_descriptor"), + patch("nemo_platform.local.services.process.remove_descriptor"), + patch("nemo_platform.local.services.serve_embedded_app"), + patch("nemo_platform.local.services.os.close"), + ): + services.run_services(service_config(services.ServiceMode.DAEMON), _mode="daemon") + + configs: list[PlatformAppConfig] = [] + for call in calls: + config = call["config"] + assert isinstance(config, PlatformAppConfig) + configs.append(config) + assert [config.services for config in configs] == [("models",), ("models",)] + assert [config.controllers for config in configs] == [(), ()] + assert [config.sidecars for config in configs] == [None, None] + + +def _sidecar_with_events(started: threading.Event, stopped: threading.Event) -> Callable[[threading.Event], None]: + def run(stop_signal: threading.Event) -> None: + started.set() + stop_signal.wait(timeout=5.0) + stopped.set() + + return run + + +def _patch_runner_registry( + monkeypatch: pytest.MonkeyPatch, + *, + sidecar_run_func: Callable[[threading.Event], None], +) -> None: + """Patch the platform runner registry so only a dummy 'models' service + and a test sidecar are available, avoiding real service imports.""" + from nmp.common.config import AuthConfig + from nmp.common.config.base import OIDCConfig + from nmp.common.service import Service + from nmp.platform_runner import config as runner_config + from nmp.platform_runner import registry, server + + class _DummyService(Service): + def __init__(self) -> None: + super().__init__(name="models", module_name="test.contract") + + def get_routers(self): + return [] + + dummy_services: dict[str, Service] = {"models": _DummyService()} + dummy_sidecars: dict[str, Callable] = {"adapters": sidecar_run_func} + + monkeypatch.setattr(runner_config, "get_available_services", lambda: dummy_services) + monkeypatch.setattr(runner_config, "get_available_controllers", lambda: {}) + monkeypatch.setattr( + runner_config, + "get_service_groups", + lambda _available: {"all": ["models"], "core": ["models"], "api": []}, + ) + monkeypatch.setattr(runner_config, "get_controller_groups", lambda _available: {"all": [], "core": []}) + monkeypatch.setattr(runner_config, "get_default_controllers", lambda _groups: []) + monkeypatch.setattr(runner_config, "AVAILABLE_SIDECARS", dummy_sidecars) + monkeypatch.setattr(registry, "AVAILABLE_SIDECARS", dummy_sidecars) + monkeypatch.setattr(server, "AVAILABLE_SIDECARS", dummy_sidecars, raising=False) + monkeypatch.setattr(server, "order_services_by_dependencies", lambda svc: svc) + + auth_cfg = AuthConfig( + enabled=False, + policy_decision_point_base_url="http://localhost:8181", + oidc=OIDCConfig(enabled=False), + ) + monkeypatch.setattr(server, "get_auth_config", lambda: auth_cfg) + monkeypatch.setattr("nmp.common.auth.middleware.get_auth_config", lambda: auth_cfg) + platform_cfg = MagicMock() + platform_cfg.seed_on_startup = False + platform_cfg.redirect_root_to_studio = False + monkeypatch.setattr(server, "get_platform_config", lambda: platform_cfg) + + +def test_embedded_mode_starts_sidecar_thread_via_full_resolution_path( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """End-to-end: start_embedded_services(models) resolves the adapters sidecar + and the sidecar thread actually runs when the app lifespan starts.""" + started = threading.Event() + stopped = threading.Event() + + _patch_runner_registry(monkeypatch, sidecar_run_func=_sidecar_with_events(started, stopped)) + + cfg = ServiceRunConfig( + mode=services.ServiceMode.EMBEDDED, + services=("models",), + controllers=(), + transport="tcp", + scope="sidecar-e2e", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + handle = services.start_embedded_services(cfg, env={}) + + from fastapi.testclient import TestClient + + with TestClient(handle.app) as client: + assert started.wait(timeout=2.0), "sidecar thread did not start" + assert client.get("/").status_code == 200 + + assert stopped.wait(timeout=2.0), "sidecar thread did not stop" diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_sidecar_integration.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_sidecar_integration.py new file mode 100644 index 0000000000..ae813ac3f4 --- /dev/null +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_sidecar_integration.py @@ -0,0 +1,231 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integration tests for sidecar lifecycle in embedded and daemon modes. + +These tests let the real ``build_platform_app`` → ``resolve_run_configuration`` → +``create_app`` chain run with a lightweight test sidecar registered in the +platform runner registry. They verify that sidecar threads actually start and +stop during the FastAPI app lifespan, covering the full resolution path without +mocking away the core wiring. +""" + +from __future__ import annotations + +import threading +from collections.abc import Callable +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from nemo_platform.local import services +from nemo_platform.local.services import ServiceRunConfig +from nmp.common.config import AuthConfig +from nmp.common.config.base import OIDCConfig +from nmp.common.service import Service +from nmp.platform_runner import config as runner_config +from nmp.platform_runner import registry, server + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class _DummyService(Service): + """Minimal service that registers no routers.""" + + def __init__(self, name: str = "models") -> None: + super().__init__(name=name, module_name="test.sidecar_integration") + + def get_routers(self): + return [] + + +def _sidecar_with_events(started: threading.Event, stopped: threading.Event) -> Callable[[threading.Event], None]: + """Return a sidecar ``run(stop_signal)`` that signals start/stop via events.""" + + def run(stop_signal: threading.Event) -> None: + started.set() + stop_signal.wait(timeout=5.0) + stopped.set() + + return run + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def sidecar_events() -> tuple[threading.Event, threading.Event]: + return threading.Event(), threading.Event() + + +@pytest.fixture +def patched_registry( + monkeypatch: pytest.MonkeyPatch, + sidecar_events: tuple[threading.Event, threading.Event], +) -> tuple[threading.Event, threading.Event]: + """Patch the platform runner registry with a dummy models service and a + test sidecar, plus minimal auth/platform config stubs.""" + started, stopped = sidecar_events + dummy_services: dict[str, Service] = {"models": _DummyService()} + dummy_sidecars: dict[str, Callable] = {"adapters": _sidecar_with_events(started, stopped)} + + monkeypatch.setattr(runner_config, "get_available_services", lambda: dummy_services) + monkeypatch.setattr(runner_config, "get_available_controllers", lambda: {}) + monkeypatch.setattr( + runner_config, + "get_service_groups", + lambda _available: {"all": ["models"], "core": ["models"], "api": []}, + ) + monkeypatch.setattr(runner_config, "get_controller_groups", lambda _available: {"all": [], "core": []}) + monkeypatch.setattr(runner_config, "get_default_controllers", lambda _groups: []) + monkeypatch.setattr(runner_config, "AVAILABLE_SIDECARS", dummy_sidecars) + monkeypatch.setattr(registry, "AVAILABLE_SIDECARS", dummy_sidecars) + monkeypatch.setattr(server, "AVAILABLE_SIDECARS", dummy_sidecars, raising=False) + monkeypatch.setattr(server, "order_services_by_dependencies", lambda svc: svc) + + auth_cfg = AuthConfig( + enabled=False, + policy_decision_point_base_url="http://localhost:8181", + oidc=OIDCConfig(enabled=False), + ) + monkeypatch.setattr(server, "get_auth_config", lambda: auth_cfg) + monkeypatch.setattr("nmp.common.auth.middleware.get_auth_config", lambda: auth_cfg) + platform_cfg = MagicMock() + platform_cfg.seed_on_startup = False + platform_cfg.redirect_root_to_studio = False + monkeypatch.setattr(server, "get_platform_config", lambda: platform_cfg) + + return started, stopped + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +def test_embedded_sidecar_auto_resolved_from_service_dependency( + patched_registry: tuple[threading.Event, threading.Event], + tmp_path: Path, +) -> None: + """start_embedded_services(models) auto-resolves the adapters sidecar via + SERVICE_SIDECAR_DEPENDENCIES and starts it during app lifespan.""" + started, stopped = patched_registry + + cfg = ServiceRunConfig( + mode=services.ServiceMode.EMBEDDED, + services=("models",), + controllers=(), + # sidecars=None triggers auto-resolution + transport="tcp", + scope="integ-embedded-auto", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + handle = services.start_embedded_services(cfg, env={}) + + from fastapi.testclient import TestClient + + with TestClient(handle.app) as client: + assert started.wait(timeout=2.0), "sidecar thread did not start" + assert client.get("/").status_code == 200 + + assert stopped.wait(timeout=2.0), "sidecar thread did not stop after lifespan exit" + + +@pytest.mark.integration +def test_embedded_explicit_sidecar_without_services( + patched_registry: tuple[threading.Event, threading.Event], + tmp_path: Path, +) -> None: + """An explicitly requested sidecar runs even when no services are selected.""" + started, stopped = patched_registry + + cfg = ServiceRunConfig( + mode=services.ServiceMode.EMBEDDED, + services=(), + controllers=(), + sidecars=("adapters",), + transport="tcp", + scope="integ-embedded-explicit", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + handle = services.start_embedded_services(cfg, env={}) + + from fastapi.testclient import TestClient + + with TestClient(handle.app) as client: + assert started.wait(timeout=2.0), "sidecar thread did not start" + assert client.get("/").status_code == 200 + + assert stopped.wait(timeout=2.0), "sidecar thread did not stop after lifespan exit" + + +@pytest.mark.integration +def test_run_services_daemon_mode_starts_sidecar_in_process( + patched_registry: tuple[threading.Event, threading.Event], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """run_services(_mode='daemon') exercises the daemon code path in-process. + It calls start_embedded_services then serve_embedded_app. We intercept + serve_embedded_app to capture the app and exercise its lifespan, proving + the daemon path wires sidecars identically to embedded mode.""" + started, stopped = patched_registry + captured_app = {} + + def fake_serve(app, cfg, socket_path): + captured_app["app"] = app + + monkeypatch.setattr("nemo_platform.local.services.require_services_extra", lambda: None) + monkeypatch.setattr("nemo_platform.local.services.process.is_instance_alive", lambda *a, **kw: False) + monkeypatch.setattr("nemo_platform.local.services._check_tcp_available", lambda *a: None) + monkeypatch.setattr("nemo_platform.local.services.process.acquire_lock", lambda *a, **kw: 123) + monkeypatch.setattr("nemo_platform.local.services.process.log_path_for", lambda *a, **kw: tmp_path / "nemo.log") + monkeypatch.setattr("nemo_platform.local.services.process.write_descriptor", lambda *a, **kw: None) + monkeypatch.setattr("nemo_platform.local.services.process.remove_descriptor", lambda *a, **kw: None) + monkeypatch.setattr("nemo_platform.local.services.serve_embedded_app", fake_serve) + monkeypatch.setattr("nemo_platform.local.services.os.close", lambda fd: None) + + cfg = ServiceRunConfig( + mode=services.ServiceMode.DAEMON, + services=("models",), + controllers=(), + transport="tcp", + scope="integ-daemon", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + services.run_services(cfg, _mode="daemon", env={}) + + assert "app" in captured_app, "serve_embedded_app was not called" + + from fastapi.testclient import TestClient + + with TestClient(captured_app["app"]) as client: + assert started.wait(timeout=2.0), "sidecar thread did not start in daemon mode" + assert client.get("/").status_code == 200 + + assert stopped.wait(timeout=2.0), "sidecar thread did not stop after lifespan exit" + + +@pytest.mark.integration +def test_embedded_rejects_unknown_sidecar_name(tmp_path: Path, patched_registry) -> None: + """Requesting a sidecar not in the registry raises ValueError with a clear message.""" + cfg = ServiceRunConfig( + mode=services.ServiceMode.EMBEDDED, + services=(), + controllers=(), + sidecars=("nonexistent",), + transport="tcp", + scope="integ-unknown", + state_dir=tmp_path / "state", + runtime_dir=tmp_path / "runtime", + ) + with pytest.raises(ValueError, match="Unknown sidecars: nonexistent"): + services.start_embedded_services(cfg, env={}) diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_transport.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_transport.py new file mode 100644 index 0000000000..79251dece3 --- /dev/null +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/local/test_transport.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from fastapi import FastAPI +from nemo_platform.local import transport + + +def _assert_timeout_values(timeout: httpx.Timeout, expected: float | None) -> None: + assert timeout.connect == expected + assert timeout.read == expected + assert timeout.write == expected + assert timeout.pool == expected + + +def test_build_sync_http_client_uses_finite_default_timeout(tmp_path) -> None: + client = transport.build_sync_http_client(tmp_path / "nemo.sock") + try: + _assert_timeout_values(client.timeout, 5.0) + finally: + client.close() + + +def test_build_sync_http_client_preserves_explicit_timeout_values(tmp_path) -> None: + no_timeout_client = transport.build_sync_http_client(tmp_path / "nemo.sock", timeout=None) + finite_timeout_client = transport.build_sync_http_client(tmp_path / "nemo.sock", timeout=12.0) + try: + _assert_timeout_values(no_timeout_client.timeout, None) + _assert_timeout_values(finite_timeout_client.timeout, 12.0) + finally: + no_timeout_client.close() + finite_timeout_client.close() + + +@pytest.mark.asyncio +async def test_build_async_http_client_uses_finite_default_timeout(tmp_path) -> None: + client = transport.build_async_http_client(tmp_path / "nemo.sock") + try: + _assert_timeout_values(client.timeout, 5.0) + finally: + await client.aclose() + + +@pytest.mark.asyncio +async def test_build_async_http_client_preserves_explicit_timeout_values(tmp_path) -> None: + no_timeout_client = transport.build_async_http_client(tmp_path / "nemo.sock", timeout=None) + finite_timeout_client = transport.build_async_http_client(tmp_path / "nemo.sock", timeout=12.0) + try: + _assert_timeout_values(no_timeout_client.timeout, None) + _assert_timeout_values(finite_timeout_client.timeout, 12.0) + finally: + await no_timeout_client.aclose() + await finite_timeout_client.aclose() + + +def test_build_sync_asgi_http_client_reaches_app() -> None: + app = FastAPI() + + @app.get("/status") + async def status() -> dict[str, str]: + return {"status": "healthy"} + + client = transport.build_sync_asgi_http_client(app) + try: + response = client.get("http://nemo-platform.local/status") + finally: + client.close() + + assert response.status_code == 200 + assert response.json() == {"status": "healthy"} + + +@pytest.mark.asyncio +async def test_build_async_asgi_http_client_reaches_app() -> None: + app = FastAPI() + + @app.get("/status") + async def status() -> dict[str, str]: + return {"status": "healthy"} + + client = transport.build_async_asgi_http_client(app) + try: + response = await client.get("http://nemo-platform.local/status") + finally: + await client.aclose() + + assert response.status_code == 200 + assert response.json() == {"status": "healthy"} + + +def test_wait_for_status_bounds_probe_and_sleep_by_remaining_deadline() -> None: + with ( + patch("nemo_platform.local.transport.probe_status", return_value=False) as probe_status, + patch("nemo_platform.local.transport.time.monotonic", side_effect=[0.0, 4.0, 4.5, 5.0]), + patch("nemo_platform.local.transport.time.sleep") as sleep, + ): + result = transport.wait_for_status(base_url="http://127.0.0.1:8080", timeout=5.0, poll_interval=10.0) + + assert result is False + assert probe_status.call_args.kwargs["timeout"] == pytest.approx(1.0) + sleep.assert_called_once() + assert sleep.call_args.args[0] == pytest.approx(0.5) + + +@pytest.mark.asyncio +async def test_wait_for_status_async_bounds_probe_and_sleep_by_remaining_deadline() -> None: + with ( + patch( + "nemo_platform.local.transport.probe_status_async", new=AsyncMock(return_value=False) + ) as probe_status, + patch("nemo_platform.local.transport.time.monotonic", side_effect=[0.0, 4.0, 4.5, 5.0]), + patch("nemo_platform.local.transport.asyncio.sleep", new=AsyncMock()) as sleep, + ): + result = await transport.wait_for_status_async( + base_url="http://127.0.0.1:8080", timeout=5.0, poll_interval=10.0 + ) + + assert result is False + assert probe_status.await_args.kwargs["timeout"] == pytest.approx(1.0) + sleep.assert_awaited_once() + assert sleep.await_args.args[0] == pytest.approx(0.5) + + +def test_probe_status_returns_true_for_status_200() -> None: + def handler(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "http://127.0.0.1:8080/status" + return httpx.Response(200) + + with patch("nemo_platform.local.transport.httpx.Client") as client_factory: + client = client_factory.return_value + client.get.side_effect = lambda url: handler(httpx.Request("GET", url)) + assert transport.probe_status(base_url="http://127.0.0.1:8080") is True + client.close.assert_called_once_with() + + +def test_probe_status_returns_false_for_request_error() -> None: + with patch("nemo_platform.local.transport.httpx.Client") as client_factory: + client = client_factory.return_value + client.get.side_effect = httpx.ConnectError("boom") + assert transport.probe_status(base_url="http://127.0.0.1:8080") is False + client.close.assert_called_once_with() + + +@pytest.mark.asyncio +async def test_probe_status_async_returns_false_for_request_error() -> None: + with patch("nemo_platform.local.transport.httpx.AsyncClient") as client_factory: + client = client_factory.return_value + client.get = AsyncMock(side_effect=httpx.ConnectError("boom")) + client.aclose = AsyncMock() + assert await transport.probe_status_async(base_url="http://127.0.0.1:8080") is False + client.aclose.assert_awaited_once_with() + + +def test_wait_for_status_returns_true_without_sleep_when_probe_succeeds() -> None: + with ( + patch("nemo_platform.local.transport.probe_status", return_value=True) as probe_mock, + patch("nemo_platform.local.transport.time.sleep") as sleep, + ): + assert transport.wait_for_status(base_url="http://127.0.0.1:8080", timeout=5.0) is True + + probe_mock.assert_called_once() + sleep.assert_not_called() diff --git a/services/core/jobs/jobs-launcher/cmd/otel.go b/services/core/jobs/jobs-launcher/cmd/otel.go index 92b03cdabc..1e3664a2e7 100644 --- a/services/core/jobs/jobs-launcher/cmd/otel.go +++ b/services/core/jobs/jobs-launcher/cmd/otel.go @@ -9,12 +9,15 @@ import ( "fmt" "log/slog" "net/http" + "net/url" "os" + "strconv" + "strings" "sync/atomic" "time" + "github.com/NVIDIA-NeMo/nemo-platform/services/core/jobs/jobs-launcher/nmpclient" "go.opentelemetry.io/contrib/bridges/otelslog" - "go.opentelemetry.io/contrib/exporters/autoexport" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" "go.opentelemetry.io/otel/exporters/stdout/stdoutlog" @@ -24,14 +27,22 @@ import ( ) const ( - name = "nmp.nvidia.com/nemo-platform/jobs-launcher" - NEMO_JOB_WORKSPACE = "NEMO_JOB_WORKSPACE" - NEMO_JOB_ID_ENV = "NEMO_JOB_ID" - NEMO_JOB_ATTEMPT_ID_ENV = "NEMO_JOB_ATTEMPT_ID" - NEMO_JOB_STEP_NAME_ENV = "NEMO_JOB_STEP" - NEMO_JOB_TASK_ID_ENV = "NEMO_JOB_TASK" - nmpJobLogsEndpointEnv = "NMP_JOB_LOGS_ENDPOINT" - otlpHTTPLogExportTimeout = 10 * time.Second + name = "nmp.nvidia.com/nemo-platform/jobs-launcher" + NEMO_JOB_WORKSPACE = "NEMO_JOB_WORKSPACE" + NEMO_JOB_ID_ENV = "NEMO_JOB_ID" + NEMO_JOB_ATTEMPT_ID_ENV = "NEMO_JOB_ATTEMPT_ID" + NEMO_JOB_STEP_NAME_ENV = "NEMO_JOB_STEP" + NEMO_JOB_TASK_ID_ENV = "NEMO_JOB_TASK" + + launcherLogsExporterEnv = "NMP_JOB_LAUNCHER_LOGS_EXPORTER" + launcherOTLPLogsEndpointEnv = "NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT" + launcherOTLPLogsHeadersEnv = "NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS" + launcherOTLPLogsProtocolEnv = "NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL" + launcherOTLPLogsTimeoutEnv = "NMP_JOB_LAUNCHER_OTLP_LOGS_TIMEOUT" + launcherOTLPLogsCompressEnv = "NMP_JOB_LAUNCHER_OTLP_LOGS_COMPRESSION" + launcherOTLPHTTPProto = "http/protobuf" + defaultLauncherLogsExporter = "console" + otlpHTTPLogExportTimeout = 10 * time.Second ) var ( @@ -111,26 +122,64 @@ func newLoggerProvider(ctx context.Context, res *resource.Resource) (*log.Logger } func newLogExporter(ctx context.Context) (log.Exporter, error) { - if endpoint := os.Getenv(nmpJobLogsEndpointEnv); endpoint != "" { - if os.Getenv(workloadIdentityTokenFileEnv) != "" { - tokenSource, err := newOTLPLogWorkloadAuthTokenSource(ctx) - if err != nil { - return nil, fmt.Errorf("configure workload identity auth for OTLP logs: %w", err) - } - return newRefreshableAuthLogExporter(ctx, endpoint, tokenSource, otlpHTTPLogExporter) + switch strings.ToLower(strings.TrimSpace(os.Getenv(launcherLogsExporterEnv))) { + case "", defaultLauncherLogsExporter, "stdout": + return stdoutlog.New() + case "none": + return noopLogExporter{}, nil + case "otlp": + return newLauncherOTLPLogExporter(ctx) + default: + return nil, fmt.Errorf("unsupported %s value %q", launcherLogsExporterEnv, os.Getenv(launcherLogsExporterEnv)) + } +} + +func newLauncherOTLPLogExporter(ctx context.Context) (log.Exporter, error) { + protocol := strings.TrimSpace(os.Getenv(launcherOTLPLogsProtocolEnv)) + if protocol == "" { + protocol = launcherOTLPHTTPProto + } + if protocol != launcherOTLPHTTPProto { + return nil, fmt.Errorf("%s must be %q, got %q", launcherOTLPLogsProtocolEnv, launcherOTLPHTTPProto, protocol) + } + + endpointURL := strings.TrimSpace(os.Getenv(launcherOTLPLogsEndpointEnv)) + if endpointURL == "" { + return nil, fmt.Errorf("%s is required when %s=otlp", launcherOTLPLogsEndpointEnv, launcherLogsExporterEnv) + } + + // If a workload identity token file is available, use a refreshable auth + // transport that exchanges the projected SA token for platform credentials. + if os.Getenv(workloadIdentityTokenFileEnv) != "" { + tokenSource, err := newOTLPLogWorkloadAuthTokenSource(ctx) + if err != nil { + return nil, fmt.Errorf("configure workload identity auth for OTLP logs: %w", err) } - return otlploghttp.New(ctx, otlploghttp.WithEndpointURL(endpoint)) + return newRefreshableAuthLogExporter(ctx, endpointURL, tokenSource, otlpHTTPLogExporter) } - return autoexport.NewLogExporter( - ctx, - // Default to a stdout log exporter if autoexport fails to configure one. - autoexport.WithFallbackLogExporter( - func(ctx context.Context) (log.Exporter, error) { - return stdoutlog.New() - }, - ), - ) + options := []otlploghttp.Option{ + otlploghttp.WithEndpointURL(endpointURL), + otlploghttp.WithHeaders(parseLauncherOTLPHeaders()), + } + + timeout, timeoutSet, err := parseLauncherOTLPTimeout() + if err != nil { + return nil, err + } + if httpClient := launcherOTLPHTTPClient(timeout, timeoutSet); httpClient != nil { + options = append(options, otlploghttp.WithHTTPClient(httpClient)) + } + if timeoutSet { + options = append(options, otlploghttp.WithTimeout(timeout)) + } + if compression, ok, err := parseLauncherOTLPCompression(); err != nil { + return nil, err + } else if ok { + options = append(options, otlploghttp.WithCompression(compression)) + } + + return otlploghttp.New(ctx, options...) } type authHeaderSource interface { @@ -214,3 +263,77 @@ func (t *authHeaderTransport) baseTransport() http.RoundTripper { func otlpHTTPLogExporter(ctx context.Context, opts ...otlploghttp.Option) (log.Exporter, error) { return otlploghttp.New(ctx, opts...) } + +func launcherOTLPHTTPClient(timeout time.Duration, timeoutSet bool) *http.Client { + endpoint, err := nmpclient.ResolvePlatformEndpointFromEnv() + if err != nil || endpoint.Transport != nmpclient.TransportUDS { + return nil + } + httpClient := endpoint.HTTPClient() + if timeoutSet { + httpClient.Timeout = timeout + } + return httpClient +} + +func parseLauncherOTLPHeaders() map[string]string { + raw := strings.TrimSpace(os.Getenv(launcherOTLPLogsHeadersEnv)) + if raw == "" { + return nil + } + headers := map[string]string{} + for _, item := range strings.Split(raw, ",") { + key, value, ok := strings.Cut(strings.TrimSpace(item), "=") + if !ok || key == "" { + continue + } + if decoded, err := url.PathUnescape(value); err == nil { + value = decoded + } + headers[key] = value + } + return headers +} + +func parseLauncherOTLPTimeout() (time.Duration, bool, error) { + raw := strings.TrimSpace(os.Getenv(launcherOTLPLogsTimeoutEnv)) + if raw == "" { + return 0, false, nil + } + duration, err := time.ParseDuration(raw) + if err == nil { + return duration, true, nil + } + milliseconds, intErr := strconv.Atoi(raw) + if intErr == nil { + return time.Duration(milliseconds) * time.Millisecond, true, nil + } + return 0, false, fmt.Errorf("invalid %s value %q: %w", launcherOTLPLogsTimeoutEnv, raw, err) +} + +func parseLauncherOTLPCompression() (otlploghttp.Compression, bool, error) { + switch strings.ToLower(strings.TrimSpace(os.Getenv(launcherOTLPLogsCompressEnv))) { + case "": + return otlploghttp.NoCompression, false, nil + case "none": + return otlploghttp.NoCompression, true, nil + case "gzip": + return otlploghttp.GzipCompression, true, nil + default: + return otlploghttp.NoCompression, false, fmt.Errorf("unsupported %s value %q", launcherOTLPLogsCompressEnv, os.Getenv(launcherOTLPLogsCompressEnv)) + } +} + +type noopLogExporter struct{} + +func (noopLogExporter) Export(context.Context, []log.Record) error { + return nil +} + +func (noopLogExporter) Shutdown(context.Context) error { + return nil +} + +func (noopLogExporter) ForceFlush(context.Context) error { + return nil +} diff --git a/services/core/jobs/jobs-launcher/cmd/otel_test.go b/services/core/jobs/jobs-launcher/cmd/otel_test.go new file mode 100644 index 0000000000..8cd1b5b9c8 --- /dev/null +++ b/services/core/jobs/jobs-launcher/cmd/otel_test.go @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "testing" + "time" +) + +func TestLauncherOTLPHTTPClientSetsUDSTimeout(t *testing.T) { + t.Setenv("NMP_BASE_URL", "unix:///tmp/nemo-platform.sock") + + client := launcherOTLPHTTPClient(250*time.Millisecond, true) + if client == nil { + t.Fatal("expected UDS HTTP client") + } + if client.Timeout != 250*time.Millisecond { + t.Fatalf("expected UDS HTTP client timeout 250ms, got %s", client.Timeout) + } +} + +func TestLauncherOTLPHTTPClientLeavesUDSTimeoutUnset(t *testing.T) { + t.Setenv("NMP_BASE_URL", "unix:///tmp/nemo-platform.sock") + + client := launcherOTLPHTTPClient(250*time.Millisecond, false) + if client == nil { + t.Fatal("expected UDS HTTP client") + } + if client.Timeout != 0 { + t.Fatalf("expected UDS HTTP client timeout to remain unset, got %s", client.Timeout) + } +} + +func TestLauncherOTLPHTTPClientSkipsTCP(t *testing.T) { + t.Setenv("NMP_BASE_URL", "http://127.0.0.1:8080") + + if client := launcherOTLPHTTPClient(250*time.Millisecond, true); client != nil { + t.Fatal("expected no custom HTTP client for TCP endpoint") + } +} diff --git a/services/core/jobs/jobs-launcher/cmd/run.go b/services/core/jobs/jobs-launcher/cmd/run.go index f5ac028767..eef947cec3 100644 --- a/services/core/jobs/jobs-launcher/cmd/run.go +++ b/services/core/jobs/jobs-launcher/cmd/run.go @@ -10,17 +10,22 @@ import ( "fmt" "io" "log/slog" + "net/http" + "net/url" "os" "os/exec" "os/signal" "strings" "sync" "syscall" + "time" "github.com/NVIDIA-NeMo/nemo-platform/services/core/jobs/jobs-launcher/nmpclient" "github.com/spf13/cobra" ) +const secretFetchTimeout = 30 * time.Second + var runCmd = &cobra.Command{ Use: "run [args...]", Short: "Run a subprocess and tail its logs", @@ -108,11 +113,27 @@ func parseSecretReferences(secretsEnv string) ([]secretReference, error) { // fetchSecrets retrieves secrets using the NeMo Platform API client and returns them as environment variables func fetchSecrets(apiBaseURL string, principal *nmpclient.Principal, secretRefs []secretReference) ([]string, error) { + return fetchSecretsWithClient(nmpclient.NewSecretClient(apiBaseURL, principal), secretRefs) +} + +func fetchSecretsWithEndpoint(endpoint nmpclient.Endpoint, principal *nmpclient.Principal, secretRefs []secretReference) ([]string, error) { + return fetchSecretsWithClient( + nmpclient.NewSecretClientWithHTTPClient(endpoint.ConnectBaseURL, principal, secretEndpointHTTPClient(endpoint)), + secretRefs, + ) +} + +func secretEndpointHTTPClient(endpoint nmpclient.Endpoint) *http.Client { + httpClient := *endpoint.HTTPClient() + httpClient.Timeout = secretFetchTimeout + return &httpClient +} + +func fetchSecretsWithClient(client nmpclient.SecretClient, secretRefs []secretReference) ([]string, error) { if len(secretRefs) == 0 { return nil, nil } - client := nmpclient.NewSecretClient(apiBaseURL, principal) envVars := make([]string, 0, len(secretRefs)) for _, ref := range secretRefs { @@ -131,6 +152,19 @@ func fetchSecrets(apiBaseURL string, principal *nmpclient.Principal, secretRefs return envVars, nil } +func workloadEnvFromParent() []string { + env := os.Environ() + filtered := make([]string, 0, len(env)) + for _, item := range env { + key, _, _ := strings.Cut(item, "=") + if strings.HasPrefix(key, "NMP_JOB_LAUNCHER_") { + continue + } + filtered = append(filtered, item) + } + return filtered +} + // runExecWithStdin sets up OTEL and runs the specified command with stdin func runExecWithStdin(args []string) (exitCode int, err error) { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) @@ -148,6 +182,30 @@ func runExecWithStdin(args []string) (exitCode int, err error) { return runExec(args, os.Stdin) } +func configureOTELHeadersFromWorkloadToken() { + token := os.Getenv("NEMO_WORKLOAD_TOKEN") + if token == "" { + return + } + + const headersEnv = launcherOTLPLogsHeadersEnv + headers := os.Getenv(headersEnv) + for _, item := range strings.Split(headers, ",") { + key, _, _ := strings.Cut(strings.TrimSpace(item), "=") + if strings.EqualFold(key, "authorization") { + return + } + } + + authHeader := "Authorization=" + url.PathEscape("Bearer "+token) + if headers == "" { + os.Setenv(headersEnv, authHeader) + return + } + os.Setenv(headersEnv, headers+","+authHeader) +} + + // runExec runs the specified command with arguments, injecting secrets as environment variables if specified func runExec(args []string, stdinReader io.Reader) (int, error) { // Command and arguments @@ -160,8 +218,8 @@ func runExec(args []string, stdinReader io.Reader) (int, error) { // Prepare the subprocess cmd := exec.Command(cmdName, cmdArgs...) - // Inherit parent environment - cmd.Env = os.Environ() + // Inherit parent environment, excluding launcher-private control variables. + cmd.Env = workloadEnvFromParent() // Parse and fetch secrets if NEMO_JOB_SECRETS is set secretsEnv := os.Getenv("NEMO_JOB_SECRETS") @@ -173,18 +231,16 @@ func runExec(args []string, stdinReader io.Reader) (int, error) { } if len(secretRefs) > 0 { - // Get API configuration from environment - apiBaseURL := os.Getenv("NMP_SECRETS_URL") - - if apiBaseURL == "" { - logger.Printf("Error: NMP_SECRETS_URL environment variable is required when NEMO_JOB_SECRETS is set\n") - return 1, fmt.Errorf("NMP_SECRETS_URL is not set") + secretEndpoint, err := nmpclient.ResolveServiceEndpointFromEnv("secrets") + if err != nil { + logger.Printf("Error: NMP_SECRETS_URL or NMP_BASE_URL is required when NEMO_JOB_SECRETS is set: %v\n", err) + return 1, fmt.Errorf("secrets endpoint is not configured: %w", err) } // Build auth context from NMP_PRINCIPAL JSON env var set by the jobs controller principal := nmpclient.PrincipalFromEnv() - secretEnvVars, err := fetchSecrets(apiBaseURL, principal, secretRefs) + secretEnvVars, err := fetchSecretsWithEndpoint(secretEndpoint, principal, secretRefs) if err != nil { logger.Printf("Error fetching secrets: %v\n", err) return 1, err diff --git a/services/core/jobs/jobs-launcher/cmd/run_test.go b/services/core/jobs/jobs-launcher/cmd/run_test.go index f1e696a8aa..4e6fa4a420 100644 --- a/services/core/jobs/jobs-launcher/cmd/run_test.go +++ b/services/core/jobs/jobs-launcher/cmd/run_test.go @@ -153,6 +153,7 @@ func TestRunExecWithSecrets(t *testing.T) { name string secretsEnv string apiURL string + baseURL string principalJSON string expectedExitCode int expectError bool @@ -173,6 +174,14 @@ func TestRunExecWithSecrets(t *testing.T) { expectedExitCode: 0, expectError: false, }, + { + name: "uses_base_url_fallback", + secretsEnv: "TEST_SECRET=default/test-secret", + baseURL: mockServer.URL, + principalJSON: `{"id":"test-principal"}`, + expectedExitCode: 0, + expectError: false, + }, { name: "missing_api_url", secretsEnv: "TEST_SECRET=default/test-secret", @@ -197,6 +206,7 @@ func TestRunExecWithSecrets(t *testing.T) { origEnvVars := map[string]envVarState{ "NEMO_JOB_SECRETS": getEnvState("NEMO_JOB_SECRETS"), "NMP_SECRETS_URL": getEnvState("NMP_SECRETS_URL"), + "NMP_BASE_URL": getEnvState("NMP_BASE_URL"), "NMP_PRINCIPAL": getEnvState("NMP_PRINCIPAL"), } defer restoreEnvVars(origEnvVars) @@ -210,6 +220,11 @@ func TestRunExecWithSecrets(t *testing.T) { } else { os.Unsetenv("NMP_SECRETS_URL") } + if tc.baseURL != "" { + os.Setenv("NMP_BASE_URL", tc.baseURL) + } else { + os.Unsetenv("NMP_BASE_URL") + } if tc.principalJSON != "" { os.Setenv("NMP_PRINCIPAL", tc.principalJSON) } else { @@ -258,6 +273,7 @@ func TestRunExecWithSecretsNotFound(t *testing.T) { origEnvVars := map[string]envVarState{ "NEMO_JOB_SECRETS": getEnvState("NEMO_JOB_SECRETS"), "NMP_SECRETS_URL": getEnvState("NMP_SECRETS_URL"), + "NMP_BASE_URL": getEnvState("NMP_BASE_URL"), "NMP_PRINCIPAL": getEnvState("NMP_PRINCIPAL"), } defer restoreEnvVars(origEnvVars) @@ -265,6 +281,7 @@ func TestRunExecWithSecretsNotFound(t *testing.T) { // Set test environment variables os.Setenv("NEMO_JOB_SECRETS", "NONEXISTENT_SECRET=default/nonexistent") os.Setenv("NMP_SECRETS_URL", mockServer.URL) + os.Unsetenv("NMP_BASE_URL") os.Setenv("NMP_PRINCIPAL", `{"id":"test-principal"}`) exitCode, err := runExec([]string{"echo", "test"}, nil) @@ -282,6 +299,37 @@ func TestRunExecWithSecretsNotFound(t *testing.T) { } } +func TestSecretEndpointHTTPClientSetsBoundedTimeout(t *testing.T) { + originalDefaultTimeout := http.DefaultClient.Timeout + + tcpEndpoint, err := nmpclient.ParseEndpoint("http://127.0.0.1:8080") + if err != nil { + t.Fatalf("ParseEndpoint returned error: %v", err) + } + tcpClient := secretEndpointHTTPClient(tcpEndpoint) + if tcpClient == http.DefaultClient { + t.Fatal("expected bounded TCP client to avoid mutating http.DefaultClient") + } + if tcpClient.Timeout != secretFetchTimeout { + t.Fatalf("expected TCP client timeout %s, got %s", secretFetchTimeout, tcpClient.Timeout) + } + if http.DefaultClient.Timeout != originalDefaultTimeout { + t.Fatalf("expected http.DefaultClient timeout to remain %s, got %s", originalDefaultTimeout, http.DefaultClient.Timeout) + } + + udsEndpoint, err := nmpclient.ParseEndpoint("unix:///tmp/nemo-platform.sock") + if err != nil { + t.Fatalf("ParseEndpoint returned error: %v", err) + } + udsClient := secretEndpointHTTPClient(udsEndpoint) + if udsClient.Timeout != secretFetchTimeout { + t.Fatalf("expected UDS client timeout %s, got %s", secretFetchTimeout, udsClient.Timeout) + } + if udsClient.Transport == nil { + t.Fatal("expected UDS client to preserve custom transport") + } +} + func TestRunExecWithoutSecrets(t *testing.T) { // Ensure no secrets environment variables are set origSecretsEnv, wasSet := os.LookupEnv("NEMO_JOB_SECRETS") @@ -306,6 +354,90 @@ func TestRunExecWithoutSecrets(t *testing.T) { } } +func TestConfigureOTELHeadersFromWorkloadToken(t *testing.T) { + testCases := []struct { + name string + token string + existingHeaders string + expectedHeaders string + }{ + { + name: "adds_authorization_header", + token: "token.with-symbols_123", + expectedHeaders: "Authorization=Bearer%20token.with-symbols_123", + }, + { + name: "preserves_existing_headers", + token: "abc.def", + existingHeaders: "X-NMP-Principal-Id=nemo-user", + expectedHeaders: "X-NMP-Principal-Id=nemo-user,Authorization=Bearer%20abc.def", + }, + { + name: "keeps_existing_authorization_header", + token: "abc.def", + existingHeaders: "authorization=Bearer+explicit", + expectedHeaders: "authorization=Bearer+explicit", + }, + { + name: "does_nothing_without_token", + existingHeaders: "X-Test=value", + expectedHeaders: "X-Test=value", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + origEnvVars := map[string]envVarState{ + "NEMO_WORKLOAD_TOKEN": getEnvState("NEMO_WORKLOAD_TOKEN"), + launcherOTLPLogsHeadersEnv: getEnvState(launcherOTLPLogsHeadersEnv), + "OTEL_EXPORTER_OTLP_LOGS_HEADERS": getEnvState("OTEL_EXPORTER_OTLP_LOGS_HEADERS"), + } + defer restoreEnvVars(origEnvVars) + + if tc.token != "" { + os.Setenv("NEMO_WORKLOAD_TOKEN", tc.token) + } else { + os.Unsetenv("NEMO_WORKLOAD_TOKEN") + } + if tc.existingHeaders != "" { + os.Setenv(launcherOTLPLogsHeadersEnv, tc.existingHeaders) + } else { + os.Unsetenv(launcherOTLPLogsHeadersEnv) + } + os.Setenv("OTEL_EXPORTER_OTLP_LOGS_HEADERS", "user-owned=value") + + configureOTELHeadersFromWorkloadToken() + + got := os.Getenv(launcherOTLPLogsHeadersEnv) + if got != tc.expectedHeaders { + t.Errorf("Expected launcher OTLP headers %q, got %q", tc.expectedHeaders, got) + } + if got := os.Getenv("OTEL_EXPORTER_OTLP_LOGS_HEADERS"); got != "user-owned=value" { + t.Errorf("Expected user OTEL headers to be preserved, got %q", got) + } + }) + } +} + +func TestWorkloadEnvFromParentFiltersLauncherPrivateVars(t *testing.T) { + origEnvVars := map[string]envVarState{ + "NMP_JOB_LAUNCHER_LOGS_EXPORTER": getEnvState("NMP_JOB_LAUNCHER_LOGS_EXPORTER"), + "OTEL_LOGS_EXPORTER": getEnvState("OTEL_LOGS_EXPORTER"), + } + defer restoreEnvVars(origEnvVars) + + os.Setenv("NMP_JOB_LAUNCHER_LOGS_EXPORTER", "otlp") + os.Setenv("OTEL_LOGS_EXPORTER", "otlp") + + env := strings.Join(workloadEnvFromParent(), "\n") + if strings.Contains(env, "NMP_JOB_LAUNCHER_LOGS_EXPORTER=") { + t.Fatal("expected launcher-private env var to be filtered") + } + if !strings.Contains(env, "OTEL_LOGS_EXPORTER=otlp") { + t.Fatal("expected user OTEL env var to be preserved") + } +} + func TestParseSecretReferences(t *testing.T) { testCases := []struct { name string diff --git a/services/core/jobs/jobs-launcher/cmd/workload_auth_test.go b/services/core/jobs/jobs-launcher/cmd/workload_auth_test.go index cb4a72d74f..d05a8623fc 100644 --- a/services/core/jobs/jobs-launcher/cmd/workload_auth_test.go +++ b/services/core/jobs/jobs-launcher/cmd/workload_auth_test.go @@ -149,7 +149,8 @@ func TestNewLogExporterCachesWorkloadAuthAcrossExports(t *testing.T) { t.Setenv(nmpBaseURLEnv, server.URL) t.Setenv(workloadIdentityTokenFileEnv, subjectTokenPath) - t.Setenv(nmpJobLogsEndpointEnv, server.URL+"/v1/logs") + t.Setenv(launcherLogsExporterEnv, "otlp") + t.Setenv(launcherOTLPLogsEndpointEnv, server.URL+"/v1/logs") exporter, err := newLogExporter(context.Background()) if err != nil { @@ -352,7 +353,8 @@ func TestNewLogExporterPropagatesInitialWorkloadAuthFailure(t *testing.T) { t.Setenv(nmpBaseURLEnv, server.URL) t.Setenv(workloadIdentityTokenFileEnv, subjectTokenPath) - t.Setenv(nmpJobLogsEndpointEnv, server.URL+"/v1/logs") + t.Setenv(launcherLogsExporterEnv, "otlp") + t.Setenv(launcherOTLPLogsEndpointEnv, server.URL+"/v1/logs") _, err := newLogExporter(context.Background()) if err == nil { diff --git a/services/core/jobs/jobs-launcher/go.mod b/services/core/jobs/jobs-launcher/go.mod index 5f1dd590bb..eaaaf83b03 100644 --- a/services/core/jobs/jobs-launcher/go.mod +++ b/services/core/jobs/jobs-launcher/go.mod @@ -15,7 +15,6 @@ require ( ) require ( - github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/go-logr/logr v1.4.3 // indirect diff --git a/services/core/jobs/jobs-launcher/go.sum b/services/core/jobs/jobs-launcher/go.sum index f9b7cb2248..46143c776c 100644 --- a/services/core/jobs/jobs-launcher/go.sum +++ b/services/core/jobs/jobs-launcher/go.sum @@ -1,5 +1,3 @@ -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -22,12 +20,6 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF2 github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= diff --git a/services/core/jobs/jobs-launcher/nmpclient/client.go b/services/core/jobs/jobs-launcher/nmpclient/client.go index 041457213f..b95aa247d3 100644 --- a/services/core/jobs/jobs-launcher/nmpclient/client.go +++ b/services/core/jobs/jobs-launcher/nmpclient/client.go @@ -67,13 +67,24 @@ type secretClient struct { } func NewSecretClient(apiBaseURL string, principal *Principal) SecretClient { + return NewSecretClientWithHTTPClient(apiBaseURL, principal, http.DefaultClient) +} + +func NewSecretClientWithHTTPClient(apiBaseURL string, principal *Principal, httpClient *http.Client) SecretClient { + if httpClient == nil { + httpClient = http.DefaultClient + } return &secretClient{ - httpClient: http.DefaultClient, + httpClient: httpClient, principal: principal, apiBaseURL: apiBaseURL, } } +func NewSecretClientForEndpoint(endpoint Endpoint, principal *Principal) SecretClient { + return NewSecretClientWithHTTPClient(endpoint.ConnectBaseURL, principal, endpoint.HTTPClient()) +} + func (c *secretClient) GetSecret(workspaceID, secretName string) (*PlatformSecretAccessResponse, error) { secretURL := getSecretURL(c.apiBaseURL, workspaceID, secretName) diff --git a/services/core/jobs/jobs-launcher/nmpclient/client_test.go b/services/core/jobs/jobs-launcher/nmpclient/client_test.go index bcdca23379..4a56517202 100644 --- a/services/core/jobs/jobs-launcher/nmpclient/client_test.go +++ b/services/core/jobs/jobs-launcher/nmpclient/client_test.go @@ -5,6 +5,7 @@ package nmpclient import ( "fmt" + "net" "net/http" "net/http/httptest" "os" @@ -162,6 +163,55 @@ func TestSecretClient_GetSecret(t *testing.T) { } } +func TestSecretClient_GetSecretOverUDS(t *testing.T) { + socketFile, err := os.CreateTemp("", "nmp-*.sock") + if err != nil { + t.Fatalf("failed to create temp socket path: %v", err) + } + socketPath := socketFile.Name() + socketFile.Close() + os.Remove(socketPath) + listener, err := net.Listen("unix", socketPath) + if err != nil { + t.Fatalf("failed to listen on unix socket: %v", err) + } + + server := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/apis/secrets/v2/workspaces/default/secrets/api-key/access" { + t.Errorf("unexpected path: %s", r.URL.Path) + http.Error(w, "unexpected path", http.StatusNotFound) + return + } + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, `{"value":"secret-over-uds"}`) + }), + } + defer server.Close() + t.Cleanup(func() { + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + t.Errorf("failed to remove unix socket path: %v", err) + } + }) + go func() { + _ = server.Serve(listener) + }() + + endpoint, err := ParseEndpoint("unix://" + socketPath) + if err != nil { + t.Fatalf("ParseEndpoint returned error: %v", err) + } + client := NewSecretClientForEndpoint(endpoint, &Principal{ID: "test-principal"}) + + secret, err := client.GetSecret("default", "api-key") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if secret.Value != "secret-over-uds" { + t.Fatalf("unexpected secret value: %s", secret.Value) + } +} + func TestSecretClient_AuthHeaders(t *testing.T) { testCases := []struct { name string diff --git a/services/core/jobs/jobs-launcher/nmpclient/endpoint.go b/services/core/jobs/jobs-launcher/nmpclient/endpoint.go new file mode 100644 index 0000000000..67beea265b --- /dev/null +++ b/services/core/jobs/jobs-launcher/nmpclient/endpoint.go @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package nmpclient + +import ( + "context" + "fmt" + "net" + "net/http" + "net/url" + "os" + "strings" +) + +const UDSBaseURL = "http://nemo-platform.local" + +type Transport string + +const ( + TransportTCP Transport = "tcp" + TransportUDS Transport = "uds" +) + +type Endpoint struct { + ConnectBaseURL string + SocketPath string + Transport Transport +} + +func ParseEndpoint(raw string) (Endpoint, error) { + if raw == "" { + return Endpoint{}, fmt.Errorf("platform endpoint URL is not configured") + } + if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") { + parsed, err := url.Parse(raw) + if err != nil || parsed.Host == "" { + return Endpoint{}, fmt.Errorf("invalid platform endpoint URL %q", raw) + } + return Endpoint{ + ConnectBaseURL: strings.TrimRight(raw, "/"), + Transport: TransportTCP, + }, nil + } + if strings.HasPrefix(raw, "unix://") { + socketPath := strings.TrimPrefix(raw, "unix://") + if !strings.HasPrefix(socketPath, "/") { + return Endpoint{}, fmt.Errorf("UDS endpoint must use an absolute socket path, got %q", raw) + } + return Endpoint{ + ConnectBaseURL: UDSBaseURL, + SocketPath: socketPath, + Transport: TransportUDS, + }, nil + } + if strings.HasPrefix(raw, "/") { + return Endpoint{}, fmt.Errorf("raw socket paths are not valid endpoint URLs; use unix://%s", raw) + } + return Endpoint{}, fmt.Errorf("unsupported platform endpoint URL %q; expected http://, https://, or unix://", raw) +} + +func ResolvePlatformEndpointFromEnv() (Endpoint, error) { + return ParseEndpoint(os.Getenv("NMP_BASE_URL")) +} + +func ResolveServiceEndpointFromEnv(service string) (Endpoint, error) { + if serviceEnv := os.Getenv(serviceURLEnvName(service)); serviceEnv != "" { + return ParseEndpoint(serviceEnv) + } + return ResolvePlatformEndpointFromEnv() +} + +func serviceURLEnvName(service string) string { + normalized := strings.ToUpper(strings.ReplaceAll(service, "-", "_")) + return "NMP_" + normalized + "_URL" +} + +func (e Endpoint) HTTPClient() *http.Client { + if e.Transport != TransportUDS { + return http.DefaultClient + } + transport := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", e.SocketPath) + }, + } + return &http.Client{Transport: transport} +} diff --git a/services/core/jobs/jobs-launcher/nmpclient/endpoint_test.go b/services/core/jobs/jobs-launcher/nmpclient/endpoint_test.go new file mode 100644 index 0000000000..c892ae61ab --- /dev/null +++ b/services/core/jobs/jobs-launcher/nmpclient/endpoint_test.go @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package nmpclient + +import ( + "fmt" + "net" + "net/http" + "os" + "testing" +) + +func TestParseEndpointTCP(t *testing.T) { + endpoint, err := ParseEndpoint("http://127.0.0.1:8080/") + if err != nil { + t.Fatalf("ParseEndpoint returned error: %v", err) + } + if endpoint.ConnectBaseURL != "http://127.0.0.1:8080" { + t.Fatalf("unexpected ConnectBaseURL: %s", endpoint.ConnectBaseURL) + } + if endpoint.Transport != TransportTCP { + t.Fatalf("unexpected transport: %s", endpoint.Transport) + } +} + +func TestParseEndpointUDS(t *testing.T) { + endpoint, err := ParseEndpoint("unix:///tmp/nemo-platform.sock") + if err != nil { + t.Fatalf("ParseEndpoint returned error: %v", err) + } + if endpoint.ConnectBaseURL != UDSBaseURL { + t.Fatalf("unexpected ConnectBaseURL: %s", endpoint.ConnectBaseURL) + } + if endpoint.SocketPath != "/tmp/nemo-platform.sock" { + t.Fatalf("unexpected socket path: %s", endpoint.SocketPath) + } + if endpoint.Transport != TransportUDS { + t.Fatalf("unexpected transport: %s", endpoint.Transport) + } +} + +func TestParseEndpointRejectsRawSocketPath(t *testing.T) { + if _, err := ParseEndpoint("/tmp/nemo-platform.sock"); err == nil { + t.Fatal("expected raw socket path to be rejected") + } +} + +func TestResolveServiceEndpointPrefersServiceURL(t *testing.T) { + t.Setenv("NMP_BASE_URL", "http://platform:8080") + t.Setenv("NMP_SECRETS_URL", "unix:///tmp/secrets.sock") + + endpoint, err := ResolveServiceEndpointFromEnv("secrets") + if err != nil { + t.Fatalf("ResolveServiceEndpointFromEnv returned error: %v", err) + } + if endpoint.Transport != TransportUDS { + t.Fatalf("expected UDS endpoint, got %s", endpoint.Transport) + } + if endpoint.SocketPath != "/tmp/secrets.sock" { + t.Fatalf("unexpected socket path: %s", endpoint.SocketPath) + } +} + +func TestEndpointContractDoesNotReadEndpointEnvFamily(t *testing.T) { + t.Setenv("NMP_PLATFORM_ENDPOINT", "unix:///tmp/platform.sock") + t.Setenv("NMP_SECRETS_ENDPOINT", "unix:///tmp/secrets.sock") + + if _, err := ResolvePlatformEndpointFromEnv(); err == nil { + t.Fatal("expected missing NMP_BASE_URL to fail") + } + if _, err := ResolveServiceEndpointFromEnv("secrets"); err == nil { + t.Fatal("expected missing NMP_SECRETS_URL and NMP_BASE_URL to fail") + } +} + +func TestUDSHTTPClient(t *testing.T) { + socketFile, err := os.CreateTemp("", "nmp-*.sock") + if err != nil { + t.Fatalf("failed to create temp socket path: %v", err) + } + socketPath := socketFile.Name() + socketFile.Close() + os.Remove(socketPath) + listener, err := net.Listen("unix", socketPath) + if err != nil { + t.Fatalf("failed to listen on unix socket: %v", err) + } + + receivedPath := make(chan string, 1) + server := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedPath <- r.URL.Path + if r.URL.Path != "/status" { + http.Error(w, "unexpected path", http.StatusNotFound) + return + } + fmt.Fprintln(w, `{"ok":true}`) + }), + } + defer server.Close() + defer os.Remove(socketPath) + go func() { + _ = server.Serve(listener) + }() + + endpoint, err := ParseEndpoint("unix://" + socketPath) + if err != nil { + t.Fatalf("ParseEndpoint returned error: %v", err) + } + resp, err := endpoint.HTTPClient().Get(endpoint.ConnectBaseURL + "/status") + if err != nil { + t.Fatalf("UDS request failed: %v", err) + } + defer resp.Body.Close() + if path := <-receivedPath; path != "/status" { + t.Fatalf("unexpected path: %s", path) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("unexpected status: %d", resp.StatusCode) + } +} diff --git a/services/core/jobs/src/nmp/core/jobs/controllers/backends/base.py b/services/core/jobs/src/nmp/core/jobs/controllers/backends/base.py index 64432ba840..e689ef64f5 100644 --- a/services/core/jobs/src/nmp/core/jobs/controllers/backends/base.py +++ b/services/core/jobs/src/nmp/core/jobs/controllers/backends/base.py @@ -7,8 +7,9 @@ import logging from abc import ABC, abstractmethod from collections.abc import Iterable +from dataclasses import dataclass from enum import Enum -from typing import Generic, Optional, TypeVar +from typing import Generic, Literal, Optional, TypeVar from urllib.parse import SplitResult, urlsplit from nemo_platform import NeMoPlatform @@ -40,6 +41,7 @@ PERSISTENT_JOB_STORAGE_PATH_ENVVAR, TASK_CONFIG_ENVVAR, ) +from nmp.common.platform_endpoint import parse_platform_endpoint from nmp.common.sdk_factory import get_entity_parts from nmp.core.jobs.app.providers import ComputeResources from pydantic import BaseModel, model_validator @@ -155,6 +157,31 @@ def get_workload_identity_token_audience() -> str: return "nemo-platform" +NMP_JOB_LAUNCHER_LOGS_EXPORTER_ENVVAR = "NMP_JOB_LAUNCHER_LOGS_EXPORTER" +NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR = "NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT" +NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS_ENVVAR = "NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS" +NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL_ENVVAR = "NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL" +NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH_ENVVAR = "NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH" +NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT_ENVVAR = "NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT" +NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL = "http/protobuf" + + +@dataclass(frozen=True) +class OtlpLogsEndpointConfig: + endpoint: str + transport: Literal["tcp", "uds"] + socket_path: str | None = None + + def to_env(self) -> dict[str, str]: + env = { + NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR: self.endpoint, + NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT_ENVVAR: self.transport, + } + if self.socket_path is not None: + env[NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH_ENVVAR] = self.socket_path + return env + + class JobUpdate(BaseModel): status: PlatformJobStatus status_details: dict | None = None @@ -531,14 +558,41 @@ def get_logs_endpoint_from_fileset( Returns: Full OTLP logs endpoint URL with appropriate loopback address applied. """ - # Job telemetry is emitted from a separate process/container/pod. When Files - # runs in-process with the API server, local service URLs are not necessarily - # routable from job runtime networks, so fall back through the same - # workload-facing base URL used for job SDK env vars. - base_url = platform_config.service_discovery.get("files") or _job_runtime_base_url(platform_config) + return get_logs_endpoint_config_from_fileset( + platform_config, + workspace, + fileset_id, + loopback_address=loopback_address, + ).endpoint + + +def get_logs_endpoint_config_from_fileset( + platform_config: PlatformConfig, workspace: str, fileset_id: str, loopback_address: str | None = None +) -> OtlpLogsEndpointConfig: + """Get OTLP logs endpoint config, preserving transport metadata for local UDS runtimes. + + Job telemetry is emitted from a separate process/container/pod. When Files + runs in-process with the API server, local service URLs are not necessarily + routable from job runtime networks, so fall back through the same + workload-facing base URL used for job SDK env vars. + """ + # Check service_discovery for a files-specific URL first, then fall back to + # the job runtime base URL (service_discovery["platform"] / base_url). + files_discovery_url = platform_config.service_discovery.get("files") + if files_discovery_url: + platform_endpoint = parse_platform_endpoint(files_discovery_url) + else: + runtime_base = _job_runtime_base_url(platform_config) + platform_endpoint = parse_platform_endpoint(runtime_base) + + base_url = platform_endpoint.connect_base_url # Use configured loopback_address, or fall back to automatic detection effective_override = loopback_address or platform_config.loopback_address or determine_loopback_override() base_url = _replace_loopback_address(base_url, effective_override) - return f"{base_url}/apis/files/v2/workspaces/{workspace}/filesets/{fileset_id}/otlp/v1/logs" + return OtlpLogsEndpointConfig( + endpoint=f"{base_url}/apis/files/v2/workspaces/{workspace}/filesets/{fileset_id}/otlp/v1/logs", + transport=platform_endpoint.transport, + socket_path=str(platform_endpoint.socket_path) if platform_endpoint.socket_path is not None else None, + ) diff --git a/services/core/jobs/src/nmp/core/jobs/controllers/backends/docker.py b/services/core/jobs/src/nmp/core/jobs/controllers/backends/docker.py index b3eefe3640..c0a81ce1b1 100644 --- a/services/core/jobs/src/nmp/core/jobs/controllers/backends/docker.py +++ b/services/core/jobs/src/nmp/core/jobs/controllers/backends/docker.py @@ -89,7 +89,11 @@ GPUExecutionProvider, ) from nmp.core.jobs.controllers.backends.base import ( - JOB_LOGS_ENDPOINT_ENVVAR, + NMP_JOB_LAUNCHER_LOGS_EXPORTER_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL, + NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL_ENVVAR, WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR, WORKLOAD_IDENTITY_TOKEN_FILE_PATH, WORKLOAD_IDENTITY_VOLUME_PATH, @@ -951,12 +955,14 @@ def schedule_single_container( EPHEMERAL_TASK_STORAGE_PATH_ENVVAR: DEFAULT_TASK_STORAGE_PATH, CONFIG_TASK_STORAGE_PATH_ENVVAR: DEFAULT_CONFIG_STORAGE_PATH, NEMO_JOB_STEP_CONFIG_FILE_PATH_ENVVAR: DEFAULT_NEMO_JOB_STEP_CONFIG_FILE_PATH, - # Endpoint used by jobs-launcher to upload task stdout/stderr logs. - JOB_LOGS_ENDPOINT_ENVVAR: get_logs_endpoint_from_fileset( + # Private env vars for jobs-launcher to export captured logs. + NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR: get_logs_endpoint_from_fileset( platform_config, step.workspace, step.fileset, ), + NMP_JOB_LAUNCHER_LOGS_EXPORTER_ENVVAR: "otlp", + NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL_ENVVAR: NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL, # Inject secret environment variable mappings for the jobs-launcher to fetch NEMO_JOB_SECRETS_ENVVAR: self.get_secrets_environment_variable_for_injection(step), } @@ -973,6 +979,8 @@ def schedule_single_container( env_var_dict = principal.get_env_var() for name, value in env_var_dict.items(): env[name] = value + # Also set launcher OTLP headers for authenticated platform log export. + env[NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS_ENVVAR] = principal.get_otlp_headers_value() step_config_json = json.dumps(step.step_spec.config) diff --git a/services/core/jobs/src/nmp/core/jobs/controllers/backends/kubernetes/common.py b/services/core/jobs/src/nmp/core/jobs/controllers/backends/kubernetes/common.py index d6def96124..08a66bae89 100644 --- a/services/core/jobs/src/nmp/core/jobs/controllers/backends/kubernetes/common.py +++ b/services/core/jobs/src/nmp/core/jobs/controllers/backends/kubernetes/common.py @@ -78,7 +78,11 @@ ) from nmp.core.jobs.app.providers import ComputeResources, ContainerSpec from nmp.core.jobs.controllers.backends.base import ( - JOB_LOGS_ENDPOINT_ENVVAR, + NMP_JOB_LAUNCHER_LOGS_EXPORTER_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL, + NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL_ENVVAR, WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR, WORKLOAD_IDENTITY_TOKEN_FILE_PATH, WORKLOAD_IDENTITY_VOLUME_NAME, @@ -1001,13 +1005,15 @@ def create_pod_template_spec( ), client.V1EnvVar(name=EPHEMERAL_TASK_STORAGE_PATH_ENVVAR, value=DEFAULT_TASK_STORAGE_PATH), client.V1EnvVar( - name=JOB_LOGS_ENDPOINT_ENVVAR, + name=NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR, value=get_logs_endpoint_from_fileset( platform_config, step.workspace, step.fileset, ), ), + client.V1EnvVar(name=NMP_JOB_LAUNCHER_LOGS_EXPORTER_ENVVAR, value="otlp"), + client.V1EnvVar(name=NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL_ENVVAR, value=NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL), client.V1EnvVar(name=NEMO_JOB_SECRETS_ENVVAR, value=secret_env_var_str), ] ) @@ -1022,6 +1028,10 @@ def create_pod_template_spec( env_var_dict = principal.get_env_var() for name, value in env_var_dict.items(): env.append(client.V1EnvVar(name=name, value=value)) + # Also set launcher OTLP headers for authenticated platform log export. + env.append( + client.V1EnvVar(name=NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS_ENVVAR, value=principal.get_otlp_headers_value()) + ) # Thread through shared platform envvars to the job shared_envvars = get_job_runtime_shared_envvars(platform_config) diff --git a/services/core/jobs/src/nmp/core/jobs/controllers/backends/subprocess.py b/services/core/jobs/src/nmp/core/jobs/controllers/backends/subprocess.py index 6b71ad0d95..8ed810439d 100644 --- a/services/core/jobs/src/nmp/core/jobs/controllers/backends/subprocess.py +++ b/services/core/jobs/src/nmp/core/jobs/controllers/backends/subprocess.py @@ -38,10 +38,14 @@ from nmp.core.jobs.app.providers import SubprocessExecutionProvider from nmp.core.jobs.app.schemas import BaseExecutionProfile from nmp.core.jobs.controllers.backends.base import ( + NMP_JOB_LAUNCHER_LOGS_EXPORTER_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL, + NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL_ENVVAR, JobBackend, JobExecutionProfileConfig, JobUpdate, - get_logs_endpoint_from_fileset, + get_logs_endpoint_config_from_fileset, ) from nmp.core.jobs.controllers.backends.subprocess_runtime import ( SubprocessOtelLogger, @@ -428,6 +432,9 @@ def _prepare_runtime(self, step: PlatformJobStepWithContext) -> tuple[dict[str, log_path.touch() platform_config = get_platform_config() + otlp_logs_endpoint = get_logs_endpoint_config_from_fileset( + platform_config, step.workspace, step.fileset, loopback_address="localhost" + ) env = {name: value for name, value in os.environ.items() if name in SUBPROCESS_INHERITED_ENV_ALLOWLIST} env.update(self._execution_profile_config.env) env.update( @@ -442,17 +449,12 @@ def _prepare_runtime(self, step: PlatformJobStepWithContext) -> tuple[dict[str, CONFIG_TASK_STORAGE_PATH_ENVVAR: str(config_dir), PERSISTENT_JOB_STORAGE_PATH_ENVVAR: str(persistent_dir), NEMO_JOB_STEP_CONFIG_FILE_PATH_ENVVAR: str(config_path), - "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT": get_logs_endpoint_from_fileset( - platform_config, - step.workspace, - step.fileset, - loopback_address="localhost", - ), - "OTEL_LOGS_EXPORTER": "otlp", - "OTEL_SERVICE_NAME": "nmp-job-task", + NMP_JOB_LAUNCHER_LOGS_EXPORTER_ENVVAR: "otlp", + NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL_ENVVAR: NMP_JOB_LAUNCHER_OTLP_LOGS_PROTOCOL, NEMO_JOB_SECRETS_ENVVAR: self.get_secrets_environment_variable_for_injection(step), } ) + env.update(otlp_logs_endpoint.to_env()) if spec and spec.environment: for envvar in spec.environment: @@ -479,7 +481,7 @@ def _prepare_runtime(self, step: PlatformJobStepWithContext) -> tuple[dict[str, auth_context = AuthContext.model_validate(step.auth_context.model_dump(mode="python", exclude_none=True)) principal = auth_context.to_principal() env.update(principal.get_env_var()) - env["OTEL_EXPORTER_OTLP_LOGS_HEADERS"] = principal.get_otlp_headers_value() + env[NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS_ENVVAR] = principal.get_otlp_headers_value() inject_secret_env_vars(env) return env, task_id, work_dir, log_path, persistent_dir diff --git a/services/core/jobs/src/nmp/core/jobs/controllers/backends/subprocess_runtime.py b/services/core/jobs/src/nmp/core/jobs/controllers/backends/subprocess_runtime.py index 6d3c2ee931..5865d8b3e3 100644 --- a/services/core/jobs/src/nmp/core/jobs/controllers/backends/subprocess_runtime.py +++ b/services/core/jobs/src/nmp/core/jobs/controllers/backends/subprocess_runtime.py @@ -6,15 +6,18 @@ import json import logging import threading +from collections.abc import Mapping from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from time import time_ns -from typing import IO +from typing import IO, Any from urllib.error import HTTPError, URLError from urllib.parse import quote, unquote, urlparse from urllib.request import Request, urlopen +import httpx +import requests from nmp.common.auth.models import NMP_PRINCIPAL_ENVVAR, Principal from nmp.common.jobs.constants import NEMO_JOB_SECRETS_ENVVAR from opentelemetry._logs import Logger @@ -26,6 +29,11 @@ logger = logging.getLogger(__name__) +NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR = "NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT" +NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS_ENVVAR = "NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS" +NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH_ENVVAR = "NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH" +NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT_ENVVAR = "NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT" + @dataclass(frozen=True) class SecretReference: @@ -55,6 +63,79 @@ def close(self) -> None: self.provider.shutdown() +class _UnixSocketOTLPSession(requests.Session): + def __init__(self, socket_path: str) -> None: + super().__init__() + self._client = httpx.Client( + transport=httpx.HTTPTransport(uds=socket_path), + follow_redirects=True, + ) + + def request( + self, + method: Any, + url: Any, + params: Any = None, + data: Any = None, + headers: Any = None, + cookies: Any = None, + files: Any = None, + auth: Any = None, + timeout: Any = None, + allow_redirects: bool = True, + proxies: Any = None, + hooks: Any = None, + stream: Any = None, + verify: Any = None, + cert: Any = None, + json: Any = None, + ) -> requests.Response: + try: + response = self._client.request( + str(method), + str(url), + params=params, + content=data, + headers=_merge_headers(self.headers, headers), + json=json, + timeout=timeout, + follow_redirects=allow_redirects, + ) + except httpx.TimeoutException as error: + raise requests.exceptions.Timeout(str(error)) from error + except httpx.TransportError as error: + raise requests.exceptions.ConnectionError(str(error)) from error + except httpx.HTTPError as error: + raise requests.exceptions.RequestException(str(error)) from error + return _to_requests_response(response) + + def close(self) -> None: + self._client.close() + super().close() + + +def _merge_headers(base_headers: Mapping[Any, Any], extra_headers: Any) -> dict[str, str]: + headers = {str(key): str(value) for key, value in base_headers.items()} + if extra_headers is None: + return headers + if isinstance(extra_headers, Mapping): + headers.update({str(key): str(value) for key, value in extra_headers.items()}) + return headers + headers.update({str(key): str(value) for key, value in extra_headers}) + return headers + + +def _to_requests_response(response: httpx.Response) -> requests.Response: + requests_response = requests.Response() + requests_response.status_code = response.status_code + requests_response.reason = response.reason_phrase + requests_response.url = str(response.url) + requests_response.headers.update(response.headers) + requests_response._content = response.content + requests_response.encoding = response.encoding + return requests_response + + def parse_secret_references(secrets_env: str) -> list[SecretReference]: if not secrets_env: return [] @@ -155,11 +236,11 @@ def create_otel_logger( step: str, task_id: str, ) -> SubprocessOtelLogger | None: - endpoint = env.get("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT") or env.get("OTEL_EXPORTER_OTLP_ENDPOINT") + endpoint = env.get(NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR) if not endpoint: return None - headers = _parse_otel_headers(env.get("OTEL_EXPORTER_OTLP_LOGS_HEADERS", "")) + headers = _parse_otel_headers(env.get(NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS_ENVVAR, "")) resource = Resource.create( { "workspace": workspace, @@ -170,13 +251,28 @@ def create_otel_logger( } ) logger_provider = LoggerProvider(resource=resource) - logger_provider.add_log_record_processor( - BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint, headers=headers or None)) - ) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(_build_otlp_log_exporter(env, endpoint, headers))) logger.info("Created local OTEL logger", extra={"endpoint": endpoint, "job": job, "step": step}) return SubprocessOtelLogger(logger_provider.get_logger("nmp.jobs.subprocess"), logger_provider) +def _build_otlp_log_exporter(env: dict[str, str], endpoint: str, headers: dict[str, str]) -> OTLPLogExporter: + transport = env.get(NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT_ENVVAR, "tcp") + if transport == "tcp": + return OTLPLogExporter(endpoint=endpoint, headers=headers or None) + if transport != "uds": + raise ValueError(f"unsupported OTLP logs transport: {transport!r}") + + socket_path = env.get(NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH_ENVVAR) + if not socket_path: + raise ValueError(f"{NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH_ENVVAR} is required for UDS OTLP logs") + return OTLPLogExporter( + endpoint=endpoint, + headers=headers or None, + session=_UnixSocketOTLPSession(socket_path), + ) + + def _parse_otel_headers(headers_env: str) -> dict[str, str]: if not headers_env: return {} diff --git a/services/core/jobs/tests/controllers/test_base.py b/services/core/jobs/tests/controllers/test_base.py index d5ba6f2ace..8ccefa73d3 100644 --- a/services/core/jobs/tests/controllers/test_base.py +++ b/services/core/jobs/tests/controllers/test_base.py @@ -18,12 +18,16 @@ from nmp.core.jobs.app.schemas import PlatformJobStepSpec, StepLifecycle from nmp.core.jobs.controllers.backends.base import ( JOB_LOGS_ENDPOINT_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT_ENVVAR, WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR, JobExecutionProfileConfig, _contains_loopback_address, _replace_loopback_address, find_reserved_managed_job_environment_variable_names, get_job_runtime_shared_envvars, + get_logs_endpoint_config_from_fileset, get_logs_endpoint_from_fileset, get_workload_identity_token_audience, resolve_task_image, @@ -341,6 +345,38 @@ def test_service_discovery_files_with_loopback_replacement(self): "http://host.docker.internal:3000/apis/files/v2/workspaces/default/filesets/job-logs-123/otlp/v1/logs" ) + def test_uds_files_url_uses_placeholder_http_origin(self): + """UDS files URL is normalized to a valid HTTP URL for OTLP request construction.""" + config = PlatformConfig( # type: ignore[abstract] + service_discovery={"files": "unix:///tmp/nemo-platform.sock"}, + loopback_address="host.docker.internal", + ) + + result = get_logs_endpoint_from_fileset(config, workspace="default", fileset_id="job-logs-123") + + assert result == ( + "http://nemo-platform.local/apis/files/v2/workspaces/default/filesets/job-logs-123/otlp/v1/logs" + ) + + def test_uds_files_url_preserves_transport_metadata(self): + config = PlatformConfig( # type: ignore[abstract] + service_discovery={"files": "unix:///tmp/nemo-platform.sock"}, + loopback_address="host.docker.internal", + ) + + result = get_logs_endpoint_config_from_fileset(config, workspace="default", fileset_id="job-logs-123") + + assert result.endpoint == ( + "http://nemo-platform.local/apis/files/v2/workspaces/default/filesets/job-logs-123/otlp/v1/logs" + ) + assert result.transport == "uds" + assert result.socket_path == "/tmp/nemo-platform.sock" + assert result.to_env() == { + NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR: result.endpoint, + NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT_ENVVAR: "uds", + NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH_ENVVAR: "/tmp/nemo-platform.sock", + } + class TestGetJobRuntimeSharedEnvvars: def test_uses_service_discovery_gateway_urls_for_job_runtimes(self): diff --git a/services/core/jobs/tests/controllers/test_docker_backend.py b/services/core/jobs/tests/controllers/test_docker_backend.py index 08102731d7..4a52fd2564 100644 --- a/services/core/jobs/tests/controllers/test_docker_backend.py +++ b/services/core/jobs/tests/controllers/test_docker_backend.py @@ -53,7 +53,7 @@ PlatformJobStepSpec, ) from nmp.core.jobs.controllers.backends.base import ( - JOB_LOGS_ENDPOINT_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR, WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR, WORKLOAD_IDENTITY_TOKEN_FILE_PATH, WORKLOAD_IDENTITY_VOLUME_PATH, @@ -769,7 +769,7 @@ def test_docker_job_uses_service_discovery_urls_for_job_runtime(mock_nmp_client, assert env_vars["NMP_FILES_URL"] == "https://nemo-gateway:8080" assert env_vars["NMP_MODELS_URL"] == "https://nemo-gateway:8080" assert env_vars["NMP_SECRETS_URL"] == "https://nemo-gateway:8080" - assert env_vars[JOB_LOGS_ENDPOINT_ENVVAR].startswith("https://nemo-gateway:8080/apis/files/") + assert env_vars[NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR].startswith("https://nemo-gateway:8080/apis/files/") def test_docker_job_execution_profile_config_rejects_reserved_env_vars(): @@ -2471,12 +2471,11 @@ def test_job_step_with_auth_context(): def test_docker_job_schedule_with_auth_context(docker_job, docker_client_mock, test_job_step_with_auth_context): - """Test that scheduling sets NMP_PRINCIPAL without injecting OTEL log headers. + """Test that scheduling sets NMP_PRINCIPAL and launcher OTLP headers when auth_context is present. Verifies GitLab issue #3390 Gap 2: job tasks should run with the creating - user's auth context, propagated via the NMP_PRINCIPAL environment variable. - Job log upload auth is handled by jobs-launcher workload token exchange, - not by globally scoped OTEL header environment variables. + user's auth context, propagated via the NMP_PRINCIPAL environment variable + and private launcher OTLP headers for authenticated telemetry export. """ step_spec = test_job_step_with_auth_context.step_spec executor_config = step_spec.executor @@ -2505,12 +2504,16 @@ def test_docker_job_schedule_with_auth_context(docker_job, docker_client_mock, t "groups": ["engineering", "ml-team"], } - assert env[JOB_LOGS_ENDPOINT_ENVVAR].endswith( - "/apis/files/v2/workspaces/default/filesets/test-logs-fileset/otlp/v1/logs" - ) + # Verify launcher OTLP headers env var is set for authenticated telemetry + assert "NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS" in env + otlp_headers = env["NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS"] + # URL-encoded: @ -> %40, , -> %2C + assert "X-NMP-Principal-Id=creator%40example.com" in otlp_headers + assert "X-NMP-Principal-Email=creator%40example.com" in otlp_headers + assert "X-NMP-Principal-Groups=engineering%2Cml-team" in otlp_headers + + # Verify no globally scoped OTEL header environment variables are set assert "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT" not in env - assert "OTEL_EXPORTER_OTLP_PROTOCOL" not in env - assert "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL" not in env assert "OTEL_LOGS_EXPORTER" not in env assert "OTEL_SERVICE_NAME" not in env assert "OTEL_EXPORTER_OTLP_LOGS_HEADERS" not in env @@ -2534,7 +2537,7 @@ def test_docker_job_schedule_without_auth_context(docker_job, docker_client_mock # Verify auth env vars are NOT set env = kwargs["environment"] assert NMP_PRINCIPAL_ENVVAR not in env - assert "OTEL_EXPORTER_OTLP_LOGS_HEADERS" not in env + assert "NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS" not in env def test_docker_job_schedule_with_auth_context_empty_groups(): diff --git a/services/core/jobs/tests/controllers/test_kubernetes_backend.py b/services/core/jobs/tests/controllers/test_kubernetes_backend.py index 728bb0354c..6f82e6a21c 100644 --- a/services/core/jobs/tests/controllers/test_kubernetes_backend.py +++ b/services/core/jobs/tests/controllers/test_kubernetes_backend.py @@ -38,7 +38,7 @@ PlatformJobStepSpec, ) from nmp.core.jobs.controllers.backends.base import ( - JOB_LOGS_ENDPOINT_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR, WORKLOAD_IDENTITY_TOKEN_FILE_ENVVAR, WORKLOAD_IDENTITY_TOKEN_FILE_PATH, WORKLOAD_IDENTITY_VOLUME_NAME, @@ -884,7 +884,7 @@ def test_kubernetes_job_uses_service_discovery_urls_for_job_runtime( assert env_vars["NMP_FILES_URL"] == "https://nemo-gateway:8080" assert env_vars["NMP_MODELS_URL"] == "https://nemo-gateway:8080" assert env_vars["NMP_SECRETS_URL"] == "https://nemo-gateway:8080" - assert env_vars[JOB_LOGS_ENDPOINT_ENVVAR].startswith("https://nemo-gateway:8080/apis/files/") + assert env_vars[NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR].startswith("https://nemo-gateway:8080/apis/files/") def test_kubernetes_job_execution_profile_config_rejects_reserved_env_vars(): @@ -2145,12 +2145,11 @@ def test_step_pending_with_auth_context() -> PlatformJobStepWithContext: def test_kubernetes_job_schedule_with_auth_context( kubernetes_job, cpu_execution_provider, test_step_pending_with_auth_context ): - """Test that scheduling sets NMP_PRINCIPAL without injecting OTEL log headers. + """Test that scheduling sets NMP_PRINCIPAL and launcher OTLP headers when auth_context is present. Verifies GitLab issue #3390 Gap 2: job tasks should run with the creating - user's auth context, propagated via the NMP_PRINCIPAL environment variable. - Job log upload auth is handled by jobs-launcher workload token exchange, - not by globally scoped OTEL header environment variables. + user's auth context, propagated via the NMP_PRINCIPAL environment variable + and private launcher OTLP headers for authenticated telemetry export. """ import json @@ -2184,12 +2183,16 @@ def test_kubernetes_job_schedule_with_auth_context( "groups": ["engineering", "ml-team"], } - assert env_vars[JOB_LOGS_ENDPOINT_ENVVAR].endswith( - "/apis/files/v2/workspaces/default/filesets/test-logs-fileset/otlp/v1/logs" - ) + # Verify launcher OTLP headers env var is set for authenticated telemetry + assert "NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS" in env_vars + otlp_headers = env_vars["NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS"] + # URL-encoded: @ -> %40, , -> %2C + assert "X-NMP-Principal-Id=creator%40example.com" in otlp_headers + assert "X-NMP-Principal-Email=creator%40example.com" in otlp_headers + assert "X-NMP-Principal-Groups=engineering%2Cml-team" in otlp_headers + + # Verify no globally scoped OTEL header environment variables are set assert "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT" not in env_var_names - assert "OTEL_EXPORTER_OTLP_PROTOCOL" not in env_var_names - assert "OTEL_EXPORTER_OTLP_LOGS_PROTOCOL" not in env_var_names assert "OTEL_LOGS_EXPORTER" not in env_var_names assert "OTEL_SERVICE_NAME" not in env_var_names assert "OTEL_EXPORTER_OTLP_LOGS_HEADERS" not in env_var_names @@ -2217,7 +2220,7 @@ def test_kubernetes_job_schedule_without_auth_context(kubernetes_job, cpu_execut # Verify auth env vars are NOT set assert NMP_PRINCIPAL_ENVVAR not in env_vars - assert "OTEL_EXPORTER_OTLP_LOGS_HEADERS" not in env_vars + assert "NMP_JOB_LAUNCHER_OTLP_LOGS_HEADERS" not in env_vars def test_cleanup_steps_with_multi_step_job_only_first_step_complete(kubernetes_job): diff --git a/services/core/jobs/tests/controllers/test_subprocess_backend.py b/services/core/jobs/tests/controllers/test_subprocess_backend.py index a32d8d220b..9aed7313bb 100644 --- a/services/core/jobs/tests/controllers/test_subprocess_backend.py +++ b/services/core/jobs/tests/controllers/test_subprocess_backend.py @@ -8,8 +8,14 @@ from types import SimpleNamespace from unittest.mock import patch +from nmp.common.config import PlatformConfig from nmp.common.jobs.schemas import PlatformJobStatus from nmp.core.jobs.app.providers import SubprocessExecutionProvider +from nmp.core.jobs.controllers.backends.base import ( + NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT_ENVVAR, +) from nmp.core.jobs.controllers.backends.subprocess import ( SubprocessJobBackend, SubprocessJobExecutionProfileConfig, @@ -157,6 +163,38 @@ def test_schedule_uses_allowlisted_host_environment(mock_nmp_client, tmp_path, m assert metadata.process.wait(timeout=5) == 0 +def test_schedule_preserves_uds_otlp_metadata_in_runtime_env(mock_nmp_client, tmp_path, test_step_pending): + platform_config = PlatformConfig( # type: ignore[abstract] + service_discovery={"files": "unix:///tmp/nemo-platform.sock"}, + loopback_address=None, + ) + backend = _subprocess_backend(mock_nmp_client, tmp_path, platform_config) + step = _step_with_command(test_step_pending, ["/bin/sh", "-c", "true"]) + captured_env = {} + + def fake_create_otel_logger(*, env, **_kwargs): + captured_env.update(env) + return None + + with ( + patch("nmp.core.jobs.controllers.backends.subprocess.get_platform_config", return_value=platform_config), + patch("nmp.core.jobs.controllers.backends.subprocess.create_otel_logger", side_effect=fake_create_otel_logger), + ): + update = backend.schedule(step.step_spec.executor, step) + + assert update.status == PlatformJobStatus.PENDING + metadata = backend._process_registry.get( + SubprocessProcessKey(step.workspace, step.job, str(step.attempt_id), step.name) + ) + assert metadata is not None + assert metadata.process.wait(timeout=5) == 0 + assert captured_env[NMP_JOB_LAUNCHER_OTLP_LOGS_ENDPOINT_ENVVAR] == ( + "http://nemo-platform.local/apis/files/v2/workspaces/default/filesets/test-logs-fileset/otlp/v1/logs" + ) + assert captured_env[NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT_ENVVAR] == "uds" + assert captured_env[NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH_ENVVAR] == "/tmp/nemo-platform.sock" + + def test_schedule_terminates_process_when_post_popen_setup_fails( mock_nmp_client, tmp_path, mock_platform_config, test_step_pending ): diff --git a/services/core/jobs/tests/controllers/test_subprocess_runtime.py b/services/core/jobs/tests/controllers/test_subprocess_runtime.py index 2b6a6b7ff6..3f3e7fad61 100644 --- a/services/core/jobs/tests/controllers/test_subprocess_runtime.py +++ b/services/core/jobs/tests/controllers/test_subprocess_runtime.py @@ -11,7 +11,11 @@ from nmp.common.auth.models import NMP_PRINCIPAL_ENVVAR from nmp.common.jobs.constants import NEMO_JOB_SECRETS_ENVVAR from nmp.core.jobs.controllers.backends.subprocess_runtime import ( + NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH_ENVVAR, + NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT_ENVVAR, SubprocessOtelLogger, + _build_otlp_log_exporter, + _UnixSocketOTLPSession, inject_secret_env_vars, parse_secret_references, start_log_capture, @@ -122,3 +126,37 @@ def test_local_otel_logger_close_flushes_and_shuts_down(): mock_otel_logger.emit.assert_called_once() mock_provider.force_flush.assert_called_once() mock_provider.shutdown.assert_called_once() + + +def test_build_otlp_log_exporter_keeps_default_http_exporter_for_tcp(): + with patch("nmp.core.jobs.controllers.backends.subprocess_runtime.OTLPLogExporter") as exporter: + result = _build_otlp_log_exporter({}, "http://files.example/otlp/v1/logs", {"x-test": "yes"}) + + assert result is exporter.return_value + exporter.assert_called_once_with(endpoint="http://files.example/otlp/v1/logs", headers={"x-test": "yes"}) + + +def test_build_otlp_log_exporter_uses_unix_socket_session_for_uds(): + env = { + NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT_ENVVAR: "uds", + NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH_ENVVAR: "/tmp/nemo-platform.sock", + } + + with patch("nmp.core.jobs.controllers.backends.subprocess_runtime.OTLPLogExporter") as exporter: + result = _build_otlp_log_exporter(env, "http://nemo-platform.local/otlp/v1/logs", {}) + + assert result is exporter.return_value + kwargs = exporter.call_args.kwargs + assert kwargs["endpoint"] == "http://nemo-platform.local/otlp/v1/logs" + assert kwargs["headers"] is None + assert isinstance(kwargs["session"], _UnixSocketOTLPSession) + kwargs["session"].close() + + +def test_build_otlp_log_exporter_rejects_uds_without_socket_path(): + with pytest.raises(ValueError, match=NMP_JOB_LAUNCHER_OTLP_LOGS_SOCKET_PATH_ENVVAR): + _build_otlp_log_exporter( + {NMP_JOB_LAUNCHER_OTLP_LOGS_TRANSPORT_ENVVAR: "uds"}, + "http://nemo-platform.local/otlp/v1/logs", + {}, + )