From 3b1b41b6128458b76a93b47891e8300924db80f8 Mon Sep 17 00:00:00 2001 From: "Paul A. Parkanzky" Date: Thu, 6 Aug 2026 16:54:49 -0400 Subject: [PATCH 1/2] fix(auditor): connect aggregated artifacts download Signed-off-by: Paul A. Parkanzky --- .../nemo_platform_plugin/jobs/api_factory.py | 35 +++- .../src/nemo_auditor/jobs/artifacts_route.py | 57 ++++++ .../src/nemo_auditor/jobs/audit.py | 2 + .../nemo-auditor/src/nemo_auditor/service.py | 10 +- .../tests/test_artifacts_route.py | 171 ++++++++++++++++++ plugins/nemo-auditor/tests/test_service.py | 11 +- 6 files changed, 274 insertions(+), 12 deletions(-) create mode 100644 plugins/nemo-auditor/src/nemo_auditor/jobs/artifacts_route.py create mode 100644 plugins/nemo-auditor/tests/test_artifacts_route.py diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/api_factory.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/api_factory.py index e61e991107..f5cb82deae 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/api_factory.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/api_factory.py @@ -74,7 +74,9 @@ PlatformJobResponse as PlatformJob, ) from nemo_platform_plugin.schema import DatetimeFilter, Filter, Page, PaginationData, StringFilter -from pydantic import BaseModel, Field, TypeAdapter, field_validator +from collections.abc import Awaitable, Callable +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, field_validator, model_validator +from typing_extensions import Self logger = logging.getLogger(__name__) @@ -464,8 +466,16 @@ class JobRouteOption(StrEnum): class PlatformJobResultRoute(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) name: str - serializer: ResultSerializer + serializer: ResultSerializer | None = None + handler: Callable[..., Awaitable[Response]] | None = None + + @model_validator(mode="after") + def _require_handler_or_serializer(self) -> Self: + if self.handler is None and self.serializer is None: + raise ValueError("PlatformJobResultRoute requires either 'serializer' or 'handler'") + return self # Compiler types: compiler receives both input spec (user-provided) and output spec (with auto-generated fields) @@ -1233,12 +1243,21 @@ async def route( return route for job_result_route in job_result_routes: - router.add_api_route( - name=f"download_job_result_{job_result_route.name}", - path=f"/jobs/{{job}}/results/{job_result_route.name}/download", - endpoint=_stamp(_make_explicit_download_endpoint(job_result_route), perm="read", write=False), - **job_result_route.serializer.route_kwargs(), - ) + if job_result_route.handler is not None: + router.add_api_route( + name=f"download_job_result_{job_result_route.name}", + path=f"/jobs/{{job}}/results/{job_result_route.name}/download", + endpoint=_stamp(job_result_route.handler, perm="read", write=False), + methods=["GET"], + response_class=FileResponse, + ) + else: + router.add_api_route( + name=f"download_job_result_{job_result_route.name}", + path=f"/jobs/{{job}}/results/{job_result_route.name}/download", + endpoint=_stamp(_make_explicit_download_endpoint(job_result_route), perm="read", write=False), + **job_result_route.serializer.route_kwargs(), # type: ignore[union-attr] + ) # Add one final route for wildcard `{name}`, for undeclared results. # This route will simply return the result's artifact as a file. diff --git a/plugins/nemo-auditor/src/nemo_auditor/jobs/artifacts_route.py b/plugins/nemo-auditor/src/nemo_auditor/jobs/artifacts_route.py new file mode 100644 index 0000000000..627f7330ae --- /dev/null +++ b/plugins/nemo-auditor/src/nemo_auditor/jobs/artifacts_route.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""On-the-fly aggregate artifact download handler for audit jobs.""" + +from __future__ import annotations + +import shutil +import tarfile +import tempfile +from pathlib import Path + +from fastapi import BackgroundTasks, Depends, HTTPException, Response +from fastapi.responses import FileResponse +from nemo_platform import AsyncNeMoPlatform +from nemo_platform_plugin.client.adapter import client_from_platform +from nemo_platform_plugin.dependencies import get_sdk_client +from nemo_platform_plugin.jobs.client import AsyncJobsClient +from nemo_platform_plugin.jobs.result_manager import download_from_result_info + +from nemo_auditor.jobs.audit import GARAK_RESULT_NAMES + + +async def aggregate_artifacts_download( + workspace: str, + job: str, + background_tasks: BackgroundTasks, + sdk: AsyncNeMoPlatform = Depends(get_sdk_client), +) -> Response: + """Download all available Garak report results and return them as a single tar.gz.""" + jobs_client = client_from_platform(sdk, AsyncJobsClient) + all_results = (await jobs_client.list_job_results(name=job, workspace=workspace)).data() + relevant = [r for r in all_results.data if r.name in GARAK_RESULT_NAMES] + + if not relevant: + raise HTTPException(status_code=404, detail="No artifact results found for this audit job") + + tmp_dirs = [] + for result in relevant: + filename, tmp_dir_path = await download_from_result_info( + result_name=result.name, + job_name=job, + workspace=workspace, + artifact_url=result.artifact_url, + files_sdk=sdk, + ) + tmp_dirs.append((filename, tmp_dir_path)) + background_tasks.add_task(tmp_dir_path.cleanup_tmp_dir) + + agg_tmp = Path(tempfile.mkdtemp()) + tar_path = agg_tmp / "artifacts.tar.gz" + with tarfile.open(tar_path, "w:gz") as tar: + for filename, tmp_dir_path in tmp_dirs: + tar.add(tmp_dir_path.path, arcname=filename) + background_tasks.add_task(lambda: shutil.rmtree(agg_tmp, ignore_errors=True)) + + return FileResponse(path=tar_path, media_type="application/gzip", filename="artifacts.tar.gz") diff --git a/plugins/nemo-auditor/src/nemo_auditor/jobs/audit.py b/plugins/nemo-auditor/src/nemo_auditor/jobs/audit.py index fe244da61e..7c907b8c9c 100644 --- a/plugins/nemo-auditor/src/nemo_auditor/jobs/audit.py +++ b/plugins/nemo-auditor/src/nemo_auditor/jobs/audit.py @@ -57,6 +57,8 @@ ("report-hitlog-jsonl", ".hitlog.jsonl"), ) +GARAK_RESULT_NAMES: frozenset[str] = frozenset(name for name, _ in _GARAK_OUTPUT_TYPES) + # garak refuses to start unless these are set even when unused (e.g. when # the actual creds come through IGW). services/auditor sets the same four. _REQUIRED_API_KEY_VARS = ( diff --git a/plugins/nemo-auditor/src/nemo_auditor/service.py b/plugins/nemo-auditor/src/nemo_auditor/service.py index 96b6576c63..125222fed6 100644 --- a/plugins/nemo-auditor/src/nemo_auditor/service.py +++ b/plugins/nemo-auditor/src/nemo_auditor/service.py @@ -10,7 +10,9 @@ from fastapi import APIRouter from nemo_auditor.authz import scope from nemo_auditor.jobs.audit import AuditJob +from nemo_auditor.jobs.artifacts_route import aggregate_artifacts_download from nemo_platform_plugin.authz import CallerKind, path_rule +from nemo_platform_plugin.jobs.api_factory import PlatformJobResultRoute from nemo_platform_plugin.jobs.routes import add_job_routes from nemo_platform_plugin.service import NemoService, RouterSpec @@ -58,7 +60,13 @@ async def healthz() -> dict[str, object]: prefix=crud_prefix, ), RouterSpec( - add_job_routes(AuditJob, authz=scope.child("audit")), + add_job_routes( + AuditJob, + authz=scope.child("audit"), + job_result_routes=[ + PlatformJobResultRoute(name="artifacts", handler=aggregate_artifacts_download), + ], + ), tag="Auditor Jobs", description="Audit job submission and retrieval.", prefix=crud_prefix, diff --git a/plugins/nemo-auditor/tests/test_artifacts_route.py b/plugins/nemo-auditor/tests/test_artifacts_route.py new file mode 100644 index 0000000000..ce75207d73 --- /dev/null +++ b/plugins/nemo-auditor/tests/test_artifacts_route.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the on-the-fly aggregate artifacts download handler.""" + +from __future__ import annotations + +import io +import tarfile +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import HTTPException +from fastapi.responses import FileResponse +from nemo_auditor.jobs.artifacts_route import aggregate_artifacts_download +from nemo_platform_plugin.jobs.file_manager import TmpDirPath + + +def _make_result(name: str, artifact_url: str = "file:///fake") -> MagicMock: + r = MagicMock() + r.name = name + r.artifact_url = artifact_url + return r + + +def _make_results_page(*names: str) -> MagicMock: + page = MagicMock() + page.data = [_make_result(n) for n in names] + return page + + +def _make_tmp_file(tmp_path: Path, name: str, content: bytes = b"data") -> TmpDirPath: + tmp_dir = Path(tempfile.mkdtemp(dir=tmp_path)) + file_path = tmp_dir / name + file_path.write_bytes(content) + return TmpDirPath(path=file_path, tmp_dir=tmp_dir) + + +@dataclass +class _FakeSdk: + pass + + +def _make_jobs_client_mock(results_page: Any) -> MagicMock: + client = MagicMock() + list_resp = MagicMock() + list_resp.data.return_value = results_page + client.list_job_results = AsyncMock(return_value=list_resp) + return client + + +@pytest.mark.asyncio +class TestAggregateArtifactsDownload: + async def test_happy_path_returns_tar_with_all_results(self, tmp_path: Path) -> None: + results = _make_results_page("report-html", "report-jsonl") + html_tmp = _make_tmp_file(tmp_path, "report-html", b"") + jsonl_tmp = _make_tmp_file(tmp_path, "report-jsonl", b'{"probe":"x"}') + + download_side_effects = [ + ("report-html", html_tmp), + ("report-jsonl", jsonl_tmp), + ] + + background_tasks = MagicMock() + sdk = _FakeSdk() + + with ( + patch( + "nemo_auditor.jobs.artifacts_route.client_from_platform", + return_value=_make_jobs_client_mock(results), + ), + patch( + "nemo_auditor.jobs.artifacts_route.download_from_result_info", + new=AsyncMock(side_effect=download_side_effects), + ), + ): + response = await aggregate_artifacts_download( + workspace="default", + job="audit-job-123", + background_tasks=background_tasks, + sdk=sdk, # type: ignore[arg-type] + ) + + assert isinstance(response, FileResponse) + tar_path = Path(response.path) + assert tar_path.exists() + with tarfile.open(tar_path, "r:gz") as tar: + members = {m.name for m in tar.getmembers()} + assert "report-html" in members + assert "report-jsonl" in members + # cleanup scheduled for both individual tmp dirs + aggregate dir + assert background_tasks.add_task.call_count == 3 + + async def test_partial_results_skips_missing(self, tmp_path: Path) -> None: + # Only report-html present; report-jsonl and report-hitlog-jsonl absent + results = _make_results_page("report-html") + html_tmp = _make_tmp_file(tmp_path, "report-html", b"") + + background_tasks = MagicMock() + sdk = _FakeSdk() + + with ( + patch( + "nemo_auditor.jobs.artifacts_route.client_from_platform", + return_value=_make_jobs_client_mock(results), + ), + patch( + "nemo_auditor.jobs.artifacts_route.download_from_result_info", + new=AsyncMock(return_value=("report-html", html_tmp)), + ), + ): + response = await aggregate_artifacts_download( + workspace="default", + job="audit-job-456", + background_tasks=background_tasks, + sdk=sdk, # type: ignore[arg-type] + ) + + assert isinstance(response, FileResponse) + with tarfile.open(Path(response.path), "r:gz") as tar: + members = {m.name for m in tar.getmembers()} + assert members == {"report-html"} + + async def test_no_relevant_results_raises_404(self) -> None: + # Job has a result, but not one of the known Garak names + results = _make_results_page("some-other-result") + + background_tasks = MagicMock() + sdk = _FakeSdk() + + with ( + patch( + "nemo_auditor.jobs.artifacts_route.client_from_platform", + return_value=_make_jobs_client_mock(results), + ), + ): + with pytest.raises(HTTPException) as exc_info: + await aggregate_artifacts_download( + workspace="default", + job="audit-job-789", + background_tasks=background_tasks, + sdk=sdk, # type: ignore[arg-type] + ) + + assert exc_info.value.status_code == 404 + + async def test_empty_results_raises_404(self) -> None: + results = _make_results_page() + + background_tasks = MagicMock() + sdk = _FakeSdk() + + with ( + patch( + "nemo_auditor.jobs.artifacts_route.client_from_platform", + return_value=_make_jobs_client_mock(results), + ), + ): + with pytest.raises(HTTPException) as exc_info: + await aggregate_artifacts_download( + workspace="default", + job="audit-job-000", + background_tasks=background_tasks, + sdk=sdk, # type: ignore[arg-type] + ) + + assert exc_info.value.status_code == 404 diff --git a/plugins/nemo-auditor/tests/test_service.py b/plugins/nemo-auditor/tests/test_service.py index 501a06d7a4..156542d312 100644 --- a/plugins/nemo-auditor/tests/test_service.py +++ b/plugins/nemo-auditor/tests/test_service.py @@ -11,15 +11,20 @@ from nemo_platform_plugin.scheduler import submit_path_for -def _mounted_post_paths() -> set[str]: +def _mounted_paths_by_method(method: str) -> set[str]: service = AuditorPluginService() paths: set[str] = set() for spec in service.get_routers(): for route in spec.router.routes: - if isinstance(route, APIRoute) and "POST" in route.methods: + if isinstance(route, APIRoute) and method in route.methods: paths.add(f"/apis/auditor{spec.prefix}{route.path}") return paths def test_audit_job_submit_route_is_mounted() -> None: - assert submit_path_for(AuditJob, workspace="{workspace}") in _mounted_post_paths() + assert submit_path_for(AuditJob, workspace="{workspace}") in _mounted_paths_by_method("POST") + + +def test_audit_job_artifacts_download_route_is_mounted() -> None: + get_paths = _mounted_paths_by_method("GET") + assert "/apis/auditor/v2/workspaces/{workspace}/jobs/audit/{job}/results/artifacts/download" in get_paths From 650d1f5594a1fa655f8b37f3a57efd93a11db2f0 Mon Sep 17 00:00:00 2001 From: "Paul A. Parkanzky" Date: Fri, 7 Aug 2026 10:47:58 -0400 Subject: [PATCH 2/2] lint fix Signed-off-by: Paul A. Parkanzky --- docs/cli/reference.mdx | 7040 ------------------------- packages/nemo_platform/pyproject.toml | 1 + 2 files changed, 1 insertion(+), 7040 deletions(-) diff --git a/docs/cli/reference.mdx b/docs/cli/reference.mdx index d7c47002de..e69de29bb2 100644 --- a/docs/cli/reference.mdx +++ b/docs/cli/reference.mdx @@ -1,7040 +0,0 @@ ---- -title: "Full CLI Reference" -description: "" ---- -Command-line interface for NeMo Platform. - -**Getting started:** -- Browse documentation with **`nemo docs --list`** -- Run local platform services with **`nemo services run --help`** -- Read the Kubernetes deployment guide with **`nemo docs set-up/helm/install`** - -**Examples:** - -```shell -nemo workspaces list --output-format markdown -nemo workspaces get default -f json -``` - -**Usage:** - -```shell -nemo [GLOBAL OPTIONS] COMMAND [ARGS]... -``` - -**Global Options:** - -* `--base-url`: Base URL for the NeMo Platform API -* `--output-format, -f `: Output format for how results are printed. [possible values: table, json, yaml, markdown, csv, raw, code] -* `--no-truncate`: Don't truncate long values in table/markdown/csv output -* `--timestamp-format `: Timestamp format for table/markdown/csv output [possible values: relative, iso8601] -* `--verbose, -v`: Enable verbose messaging. This only impacts logs that are visible, it doesn't change any data outputs. -* `--agent-mode, -A`: Enable agent-friendly output mode with extra context for coding agents. -* `--no-telemetry`: Disable anonymous usage telemetry for this invocation. - -**Help:** - -* `--version, -V`: Show version information and exit. -* `--install-completion`: Install completion for the current shell. -* `--show-completion`: Show completion for the current shell, to copy it or customize the installation. -* `--help, -h`: Show this message and exit. - -## Setup - -### nemo setup - -Set up NeMo Platform: connect or start services, configure a provider, install skills. - -Uses an already-running platform, starts local services, or connects the -CLI to an existing remote deployment. Then selects and registers an -inference provider, picks a default model, installs coding agent skills, -and optionally deploys a demo agent. - -The active config context remembers the Platform URL. When a remote -deployment is already reachable, setup asks whether to continue with it, -start local services instead, or connect to a different remote URL. - -To override the URL for one run only: - nemo --base-url http://localhost:8080 setup - -To persist a different URL: - nemo config set --base-url http://localhost:8080 - -Requires an interactive terminal (TTY). In non-interactive contexts -(CI, piped input), pass --auto to use environment variables instead. - -Use --auto for non-interactive setup from environment variables -(NEMO_DEFAULT_INFERENCE_KEY, NVIDIA_API_KEY, OPENAI_API_KEY, -ANTHROPIC_API_KEY, GEMINI_API_KEY). -Override the default model with NEMO_DEFAULT_MODEL. - -**Examples:** - -```shell -nemo setup -nemo setup --auto -nemo setup --auto --start-services --install-skills --deploy-agent -nemo setup --auto --start-services --ready-timeout 360 -NMP_BASE_URL=https://nmp.example.com NMP_ACCESS_TOKEN=... nemo setup --auto --no-start-services -nemo setup --workspace my-workspace -nemo setup --no-install-skills --no-deploy-agent -nemo --base-url http://localhost:8080 setup -``` - -**Usage:** - -```shell -nemo setup [OPTIONS] -``` - -**Options:** - -* `--auto`: Non-interactive mode: register provider from environment variables -* `--workspace, -w`: Target workspace [default: default] -* `--start-services, --no-start-services`: Start local platform services -* `--install-skills, --no-install-skills`: Install NeMo skills for coding agents -* `--skills-agents`: Comma-separated list of agents to install skills for (e.g. 'codex,cursor'). Default: all detected. Only applied when --install-skills is set. -* `--skills-scope `: Install scope for skills: 'project' (this repo) or 'user' (home). Default: project. Only applied when --install-skills is set. [possible values: project, user] -* `--skills-from`: Comma-separated list of skill sources to install from (e.g. 'nemo-platform,nemo-evaluator-plugin'). Use 'nemo-platform' for the built-in set. Default: all sources. Only applied when --install-skills is set. -* `--deploy-agent, --no-deploy-agent`: Deploy the demo calculator agent -* `--ready-timeout `: Seconds to wait for platform readiness (default: 240) - -**Help:** - -* `--help, -h`: Show this message and exit. - -### nemo auth - -Manage authentication for NeMo Platform. - -**Usage:** - -```shell -nemo auth [OPTIONS] [COMMAND] [ARGS]... -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Commands:** - -* `login`: Authenticate with the NeMo Platform cluster. -* `logout`: Remove stored credentials for the current context. -* `refresh`: Refresh the current access token. -* `token`: Print the current access token (for use with SDK or curl). -* `status`: Show current authentication status. -* `access-keys`: Manage NeMo Platform Scoped Access Keys. - -#### nemo auth login - -Authenticate with the NeMo Platform cluster. - -Uses device flow (browser) by default, or password grant when username and password are provided (e.g. for CI). - -For quickstart, use **`--unsigned-token`** to generate an unsigned JWT. - -**Examples:** - -```shell -# Set base URL and log in -nemo auth login --base-url https://nemo.example.com -# Context-specific login -nemo auth login --context dev --base-url https://nemo.dev.example.com -# Device flow, open browser -nemo auth login -# Device flow, show code only -nemo auth login --no-browser -``` - -**Usage:** - -```shell -nemo auth login [OPTIONS] -``` - -**Options:** - -* `--context`: Context to use for this login command. -* `--base-url`: Set cluster base URL for the selected context before login -* `--no-browser`: Don't open browser (device flow only) -* `--scope`: OAuth scopes to request (space-separated; quote for multiple, e.g. --scope "platform:read secrets:write") -* `--username`: Username for password grant (CI / non-interactive) -* `--password`: Password for password grant (prefer env NMP_OIDC_PASSWORD) - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Unsigned Token Options:** - -* `--unsigned-token`: Generate and save an unsigned JWT for local/testing authentication. -* `--principal-id`: Principal ID for the unsigned token (`sub` claim). Defaults to --email. -* `--email`: Email claim for the unsigned token (required with --unsigned-token). -* `--group`: Group claim value for unsigned token (repeat for multiple). -* `--expires-in `: Unsigned token expiry in seconds from now. [default: 3600] -* `--no-exp`: Omit the exp claim from the unsigned token. -* `--audience`: Audience (`aud`) claim for unsigned token. -* `--issuer`: Issuer (`iss`) claim for unsigned token. - -#### nemo auth logout - -Remove stored credentials for the current context. - -**Usage:** - -```shell -nemo auth logout [OPTIONS] -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo auth refresh - -Refresh the current access token. - -This command uses the saved refresh token to obtain a new access token without requiring you to re-authenticate through the browser. - -**Usage:** - -```shell -nemo auth refresh [OPTIONS] -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo auth token - -Print the current access token (for use with SDK or curl). - -By default this outputs the raw token to stdout, suitable for piping or capture. - -**Examples:** - -```shell -# Print token -nemo auth token -# Inspect token claims -nemo auth token --decode -# Capture in env var -export TOKEN=$(nemo auth token) -curl -H "Authorization: Bearer $(nemo auth token)" ... -``` - -**Usage:** - -```shell -nemo auth token [OPTIONS] -``` - -**Options:** - -* `--decode`: Decode the JWT payload claims as JSON. This does not verify the token signature. - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo auth status - -Show current authentication status. - -**Usage:** - -```shell -nemo auth status [OPTIONS] -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo auth access-keys - -Manage NeMo Platform Scoped Access Keys. - -**Usage:** - -```shell -nemo auth access-keys [OPTIONS] COMMAND [ARGS]... -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Commands:** - -* `create`: Create a Scoped Access Key for the current authenticated... - -##### nemo auth access-keys create - -Create a Scoped Access Key for the current authenticated user. - -**Usage:** - -```shell -nemo auth access-keys create [OPTIONS] -``` - -**Options:** - -* `--name, -n`: Optional human-readable label for the Scoped Access Key. -* `--expires-in`: Scoped Access Key lifetime in seconds. Use 'none' to request no expiration. - -**Help:** - -* `--help, -h`: Show this message and exit. - -### nemo services - -Run platform services locally. - -**Usage:** - -```shell -nemo services [OPTIONS] [COMMAND] [ARGS]... -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Commands:** - -* `run`: Run platform services in the foreground. -* `start`: Start platform services in the background. -* `stop`: Stop running platform services. -* `restart`: Restart platform services. -* `status`: Show status of the platform services instance for this... -* `ls`: List service instances on this host. -* `rm`: Remove a stopped instance directory and its logs. -* `prune`: Remove all stopped instance directories on this host. -* `logs`: Show or locate the service log file. - -#### nemo services run - -Run platform services in the foreground. Ctrl-C to stop. - -**Usage:** - -```shell -nemo services run [OPTIONS] -``` - -**Options:** - -* `--services`: Comma-separated services to run, e.g. models,entities,jobs. Defaults to all available services. -* `--service-group`: Run a predefined service group. Cannot be combined with --services. -* `--controllers`: Comma-separated controllers to run, e.g. jobs,models. -* `--controller-group`: Run a predefined controller group. Cannot be combined with --controllers. -* `--sidecars`: Comma-separated sidecars to run, e.g. adapters,cache. -* `--config`: Path to a platform configuration YAML file. -* `--host`: Host to bind to. [default: 127.0.0.1] -* `--port `: Port to bind to. [default: 8080] -* `--instance`: Instance name. Defaults to a name derived from the working directory and port. - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo services start - -Start platform services in the background. - -Detaches the process, polls /status, then returns. - -**Examples:** - -```shell -nemo services start -nemo services start --services entities,models --port 9090 -``` - -**Usage:** - -```shell -nemo services start [OPTIONS] -``` - -**Options:** - -* `--services`: Comma-separated services to run, e.g. models,entities,jobs. -* `--service-group`: Run a predefined service group. Cannot be combined with --services. -* `--controllers`: Comma-separated controllers to run, e.g. jobs,models. -* `--controller-group`: Run a predefined controller group. Cannot be combined with --controllers. -* `--sidecars`: Comma-separated sidecars to run, e.g. adapters,cache. -* `--config`: Path to a platform configuration YAML file. -* `--host`: Host to bind to. [default: 127.0.0.1] -* `--port `: Port to bind to. [default: 8080] -* `--instance`: Instance name. Defaults to a name derived from the working directory and port. - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo services stop - -Stop running platform services. - -Sends SIGTERM to the running service process and waits for it to exit. -Falls back to SIGKILL after a timeout. Foreground instances (started -with ``run``) are protected; use ``--force`` to override. - -**Examples:** - -```shell -nemo services stop -nemo services stop --timeout 60 -``` - -**Usage:** - -```shell -nemo services stop [OPTIONS] -``` - -**Options:** - -* `--timeout `: Seconds to wait before SIGKILL. [default: 30.0] -* `--instance`: Instance name. Defaults to a name derived from the working directory and port. -* `--port `: Port (used for scope computation if --instance not given). [default: 8080] -* `--force`: Stop even if the instance is running in the foreground. - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo services restart - -Restart platform services. - -Stops any running services and relaunches them. Without flags, preserves -the service set from the previous run. Errors if no previously tracked -instance exists for the computed scope; does not start a fresh instance. - -**Examples:** - -```shell -nemo services restart -nemo services restart --services entities,models,agents -``` - -**Usage:** - -```shell -nemo services restart [OPTIONS] -``` - -**Options:** - -* `--services`: Comma-separated services to run. Overrides previous service set. -* `--service-group`: Run a predefined service group. Overrides previous setting. -* `--controllers`: Comma-separated controllers to run. Overrides previous controller set. -* `--controller-group`: Run a predefined controller group. Overrides previous setting. -* `--sidecars`: Comma-separated sidecars to run. Overrides previous setting. -* `--config`: Path to a platform configuration YAML file. -* `--host`: Host to bind to. Defaults to previous value or 127.0.0.1. -* `--port `: Port to bind to. Defaults to previous value or 8080. -* `--instance`: Instance name. Defaults to a name derived from the working directory and port. - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo services status - -Show status of the platform services instance for this scope. - -**Usage:** - -```shell -nemo services status [OPTIONS] -``` - -**Options:** - -* `--instance`: Instance name. Defaults to a name derived from the working directory and port. -* `--port `: Port (used for scope computation if --instance not given). [default: 8080] - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo services ls - -List service instances on this host. - -By default shows running instances only. Use ``--all`` to include stopped -instance directories that still have logs on disk. - -**Examples:** - -```shell -nemo services ls -nemo services ls --all -``` - -**Usage:** - -```shell -nemo services ls [OPTIONS] -``` - -**Options:** - -* `--all, -a`: Include stopped instance directories (like docker ps -a). - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo services rm - -Remove a stopped instance directory and its logs. - -The scope must match a row from ``nemo services ls --all``. Running -instances are refused; stop them first. - -Unlike ``run``/``start``, ``--instance`` here does not derive a scope from -cwd and port — it is an alternate spelling for the ``SCOPE`` argument. - -**Examples:** - -```shell -nemo services rm abc12345-8080 -nemo services rm --instance abc12345-8080 -``` - -**Usage:** - -```shell -nemo services rm [OPTIONS] [SCOPE] -``` - -**Arguments:** - -* ``: Instance scope from 'nemo services ls --all'. - -**Options:** - -* `--instance`: Scope from 'nemo services ls --all' (same value as the SCOPE positional). - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo services prune - -Remove all stopped instance directories on this host. - -Stopped instance directories may include service logs from prior runs. Logs are -deleted with the instance directory. - -**Examples:** - -```shell -nemo services prune -nemo services prune --force -``` - -**Usage:** - -```shell -nemo services prune [OPTIONS] -``` - -**Options:** - -* `--force`: Remove without confirmation. - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo services logs - -Show or locate the service log file. - -**Examples:** - -```shell -nemo services logs -nemo services logs --path -nemo services logs -n 100 -``` - -**Usage:** - -```shell -nemo services logs [OPTIONS] -``` - -**Options:** - -* `--path`: Print the log file path instead of tailing. -* `-n, --lines `: Number of lines to show from end of log. [default: 50] -* `--instance`: Instance name. Defaults to a name derived from the working directory and port. -* `--port `: Port (used for scope computation if --instance not given). [default: 8080] - -**Help:** - -* `--help, -h`: Show this message and exit. - -### nemo skills - -Install AI agent skill files for Nemo. - -Supported agents: claude, codex, cursor, opencode - -**Examples:** - -```shell -# List available skills. -nemo skills list -# Show a skill's content. -nemo skills show inference -# Install all skills for Claude Code. -nemo skills install --agent claude -# Install specific skills only. -nemo skills install --agent claude --skill inference -``` - -**Usage:** - -```shell -nemo skills [OPTIONS] [COMMAND] [ARGS]... -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Commands:** - -* `list`: List available skills. -* `show`: Print skill content to stdout. -* `install`: Install Nemo skill files for an AI coding agent. - -#### nemo skills list - -List available skills. - -The default table word-wraps long descriptions; use `--no-truncate` to let -descriptions fill the full terminal width. For structured output use -`-f json|yaml|csv|markdown`. When stdout is not a TTY (pipe/redirect), -JSON is the default so callers get parseable output. - -**Examples:** - -```shell -nemo skills list -nemo skills list --no-truncate -nemo skills list -f json -nemo skills list --source nemo-platform -nemo skills list --source nemo-platform --source nemo-agents-plugin -``` - -**Usage:** - -```shell -nemo skills list [OPTIONS] -``` - -**Options:** - -* `--source`: Filter to skills from a specific source (distribution / plugin name as shown in the `Source` column, e.g. `nemo-platform`, `nemo-agents-plugin`). Can be repeated to include multiple sources. Matching is case-insensitive. - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Output Options:** - -* `--output-format, -f `: Output format for the list of results. [possible values: table, json, yaml, markdown, csv, raw, code] -* `--no-truncate`: Don't truncate long values in table/markdown/csv output. -* `--output-columns, -c`: Columns to display: 'default', 'all', or comma-separated names. Only affects table/csv/markdown formats. - -#### nemo skills show - -Print skill content to stdout. - -Without --agent, prints the raw skill content. -With --agent, prints the agent-specific formatted version. - -**Examples:** - -```shell -nemo skills show inference -nemo skills show --agent claude inference -nemo skills show inference | pbcopy -``` - -**Usage:** - -```shell -nemo skills show [OPTIONS] NAME -``` - -**Arguments:** - -* ``: Skill name to show (use 'nemo skills list' to see available skills) - -**Options:** - -* `--agent, -a`: Agent to format for. Supported: claude, codex, cursor, opencode - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo skills install - -Install Nemo skill files for an AI coding agent. - -By default, installs all skills to project scope. -Use --skill to select specific skills, --user for user scope, or ---project-dir to explicitly select the project install directory. - -**Examples:** - -```shell -nemo skills install --agent claude -nemo skills install --agent claude --user -nemo skills install --agent claude --skill inference -nemo skills install --agent claude --project-dir /path/to/project -``` - -**Usage:** - -```shell -nemo skills install [OPTIONS] -``` - -**Options:** - -* `--agent, -a`: Agent to install for (required). Supported: claude, codex, cursor, opencode -* `--skill, -s`: Install specific skill(s) only. Can be repeated. -* `--user`: Install to user scope (default: project scope) -* `--project-dir, --project-root `: Project directory to install into (default: current working directory) - -**Help:** - -* `--help, -h`: Show this message and exit. - -## CLI functions - -### nemo chat - -Start an interactive chat session with a model. - -By default, uses model entity routing where the model name should match -what's shown in 'nemo models list'. - -Use --provider for direct provider routing, where the model argument is -passed directly to the provider's API. - -Passing PROMPT sends one message and exits unless --interactive is set. -Omitting PROMPT in a TTY starts the interactive chat UI. In non-TTY -contexts, PROMPT may also be piped on stdin. Piped stdin is read in full -before sending. If both PROMPT and piped stdin are provided, PROMPT takes -precedence. - -**Examples:** - -```shell -nemo chat nvidia/llama-3.3-nemotron-super-49b-v1.5 -nemo chat nvidia/llama-3.3-nemotron-super-49b-v1.5 "What is machine learning?" -nemo chat nvidia/llama-3.3-nemotron-super-49b-v1.5 "What is machine learning?" --interactive -echo "What is machine learning?" | nemo chat nvidia/llama-3.3-nemotron-super-49b-v1.5 -nemo chat nvidia/llama-3.3-nemotron-super-49b-v1.5 "What is machine learning?" -f json -nemo chat nvidia/llama-3.3-nemotron-super-49b-v1.5 --provider nvidia-build -``` - -**Usage:** - -```shell -nemo chat [OPTIONS] MODEL [PROMPT] -``` - -**Arguments:** - -* ``: Model entity name (from 'nemo models list') or model ID when using --provider -* ``: Prompt for one-shot mode. Takes precedence over piped stdin. - -**Options:** - -* `--provider`: Provider name for direct provider routing (bypasses model entity routing) -* `--workspace`: Workspace name - -**Chat Options:** - -* `--interactive`: Start the terminal chat UI; cannot be used with piped stdin. With PROMPT, send it first. - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Model Options:** - -* `--temperature `: Sampling temperature (0.0 to 2.0) -* `--max-tokens `: Maximum tokens to generate -* `--system-message`: System message to set context for the conversation - -**Output Options:** - -* `--output-format, --format, -f `: Output format for one-shot responses. [possible values: text, json, raw] - -### nemo docs - -Read NeMo Platform documentation. - -**Examples:** - -```shell -nemo docs get-started/setup -nemo docs set-up/helm/install -nemo docs --list -nemo docs cli/configuration -``` - -**Usage:** - -```shell -nemo docs [OPTIONS] [PATH] -``` - -**Arguments:** - -* ``: Path to a doc topic (e.g., get-started/setup or set-up/helm/install). Omit to see available topics. - -**Options:** - -* `--list, -l`: List available documentation topics. - -**Help:** - -* `--help, -h`: Show this message and exit. - -### nemo wait - -Wait for resources to reach a desired status. - -**Usage:** - -```shell -nemo wait [OPTIONS] COMMAND [ARGS]... -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Commands:** - -* `inference`: Wait for inference resources - -#### nemo wait inference - -Wait for inference resources - -**Usage:** - -```shell -nemo wait inference [OPTIONS] COMMAND [ARGS]... -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Commands:** - -* `deployment`: Wait for a deployment to reach a desired status. -* `provider`: Wait for the inference gateway to be ready to route to a... - -##### nemo wait inference deployment - -Wait for a deployment to reach a desired status. - -Polls the deployment status until it reaches the desired state or times out. -For READY status, optionally verifies the gateway can route to the provider. -For DELETED status, waits for the resource to be fully garbage collected. - -Exit codes: - 0: Desired status reached - 1: Timeout or error - -**Examples:** - -```shell -nemo wait inference deployment my-deployment --status READY -nemo wait inference deployment my-deployment --status READY --timeout 600 --no-check-gateway -nemo wait inference deployment my-deployment --status DELETED --timeout 90 -``` - -**Usage:** - -```shell -nemo wait inference deployment [OPTIONS] NAME -``` - -**Arguments:** - -* ``: Name of the deployment to wait for - -**Options:** - -* `--workspace`: Workspace name -* `--status, -s `: Desired status to wait for [possible values: READY, DELETED, PENDING, ERROR; default: READY] -* `--timeout, -t `: Maximum time to wait in seconds [default: 1200] -* `--check-gateway, --no-check-gateway`: When waiting for READY, also verify gateway can route to the provider -* `--poll-interval `: Seconds between status checks [default: 3] - -**Help:** - -* `--help, -h`: Show this message and exit. - -##### nemo wait inference provider - -Wait for the inference gateway to be ready to route to a provider. - -Polls the gateway's ready endpoint until it can route requests to the -specified provider. This is useful after creating a deployment to ensure -the gateway has refreshed its cache. - -Exit codes: - 0: Gateway is ready - 1: Timeout - -**Examples:** - -```shell -nemo wait inference provider my-deployment -nemo wait inference provider my-deployment --timeout 120 -``` - -**Usage:** - -```shell -nemo wait inference provider [OPTIONS] NAME -``` - -**Arguments:** - -* ``: Name of the provider to wait for - -**Options:** - -* `--workspace`: Workspace name -* `--timeout, -t `: Maximum time to wait in seconds [default: 60] -* `--poll-interval `: Seconds between status checks [default: 1] - -**Help:** - -* `--help, -h`: Show this message and exit. - -### nemo agent - -Commands for AI agent context and capability discovery. - -**Examples:** - -```shell -# Dump full agent context (plugins, commands, skills). -nemo agent context -# List all available commands. -nemo agent commands -``` - -**Usage:** - -```shell -nemo agent [OPTIONS] [COMMAND] [ARGS]... -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Commands:** - -* `context`: Dump everything an agent needs in one call. -* `commands`: List all available top-level CLI commands. - -#### nemo agent context - -Dump everything an agent needs in one call. - -Outputs installed plugins, CLI commands, entry-point catalog, -available skills, and quick-reference patterns. Runs without a -connected cluster (metadata-only). - -**Examples:** - -```shell -nemo agent context -``` - -**Usage:** - -```shell -nemo agent context [OPTIONS] -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo agent commands - -List all available top-level CLI commands. - -Outputs a flat list of commands with descriptions, useful for -agent capability discovery. - -**Examples:** - -```shell -nemo agent commands -``` - -**Usage:** - -```shell -nemo agent commands [OPTIONS] -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -### nemo plugins - -Commands for plugin discovery. - -**Examples:** - -```shell -# List installed plugins. -nemo plugins list -``` - -**Usage:** - -```shell -nemo plugins [OPTIONS] [COMMAND] [ARGS]... -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Commands:** - -* `list`: List installed plugins. - -#### nemo plugins list - -List installed plugins. - -Discovers installed plugins from registered NeMo plugin entry points. - -**Examples:** - -```shell -nemo plugins list -nemo plugins list -f json -``` - -**Usage:** - -```shell -nemo plugins list [OPTIONS] -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Output Options:** - -* `--output-format, -f `: Output format for the list of results. [possible values: table, json, yaml, markdown, csv, raw, code] -* `--no-truncate`: Don't truncate long values in table/markdown/csv output. -* `--output-columns, -c`: Columns to display: 'default', 'all', or comma-separated names. Only affects table/csv/markdown formats. - -## Core plugins - -### nemo files - -Manage files. - -**Usage:** - -```shell -nemo files [OPTIONS] COMMAND [ARGS]... -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Commands:** - -* `upload`: Upload local files to a fileset. -* `download`: Download files from a fileset to a local path. -* `list`: List files in a fileset. -* `delete`: Delete a file from a fileset. -* `filesets`: Manage filesets -* `otlp`: Otlp operations - -#### nemo files upload - -Upload local files to a fileset. - -Supports uploading single files or directories. For directories, contents -are uploaded recursively. - -**Examples:** - -```shell -# Upload a file to the root of a fileset -nemo files upload ./data.csv my-fileset -``` - -\# Upload a directory to a subdirectory in the fileset -nemo files upload ./data/ my-fileset --remote-path uploads/ - -\# Upload without specifying a fileset (auto-creates one) -nemo files upload ./data.csv - -**Usage:** - -```shell -nemo files upload [OPTIONS] LOCAL_PATH [FILESET] -``` - -**Arguments:** - -* ``: Local path to upload -* ``: Name of the fileset to upload to. If not provided, a new fileset is created. - -**Options:** - -* `--workspace` -* `--remote-path`: Path within the fileset. Defaults to root. [default: ] - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo files download - -Download files from a fileset to a local path. - -Supports downloading single files or directories. For directories, contents -are downloaded recursively. - -**Examples:** - -```shell -# Download entire fileset to current directory -nemo files download my-fileset -o ./ -``` - -\# Download a subdirectory from the fileset -nemo files download my-fileset --remote-path data/ -o ./downloads/ - -**Usage:** - -```shell -nemo files download [OPTIONS] FILESET -``` - -**Arguments:** - -* ``: Name of the fileset to download from - -**Options:** - -* `--workspace` -* `--remote-path`: Path within the fileset. Defaults to root. [default: ] -* `--output, -o `: Local path to download to. - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo files list - -List files in a fileset. - -Lists all files recursively from the specified path within the fileset. - -**Examples:** - -```shell -# List all files in a fileset -nemo files list my-fileset -``` - -\# List files in a subdirectory -nemo files list my-fileset --remote-path data/ - -**Usage:** - -```shell -nemo files list [OPTIONS] FILESET -``` - -**Arguments:** - -* ``: Name of the fileset to list files from - -**Options:** - -* `--workspace` -* `--remote-path`: Path within the fileset. Defaults to root. [default: ] - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Output Options:** - -* `--output-format, -f `: Output format for the list of results. [possible values: table, json, yaml, markdown, csv, raw, code] -* `--output-columns, -c`: Columns to display: 'default', 'all', or comma-separated names. Only affects table/csv/markdown formats. -* `--no-truncate`: Don't truncate long values in table/markdown/csv output. - -#### nemo files delete - -Delete a file from a fileset. - -**Examples:** - -```shell -# Delete a specific file -nemo files delete my-fileset --remote-path data/old-file.txt -``` - -**Usage:** - -```shell -nemo files delete [OPTIONS] FILESET -``` - -**Arguments:** - -* ``: Name of the fileset containing the file - -**Options:** - -* `--workspace` -* `--remote-path`: Path of the file to delete within the fileset - -**Help:** - -* `--help, -h`: Show this message and exit. - -#### nemo files filesets - -Manage filesets - -**Usage:** - -```shell -nemo files filesets [OPTIONS] COMMAND [ARGS]... -``` - -**Help:** - -* `--help, -h`: Show this message and exit. - -**Commands:** - -* `create`: Create a new fileset. -* `delete`: Delete Fileset. -* `list`: List Filesets endpoint with filtering and pagination. -* `get`: Get Fileset by Workspace and Name. -* `update`: Update Fileset Metadata. - -##### nemo files filesets create - -Create a new fileset. - -If no storage configuration is provided, the default storage backend will be -used. - -**Required fields:** name - -**Examples:** - -```shell -nemo files filesets create --input-file config.json -nemo files filesets create --input-data '{"name": "value"}' -echo '{"json": "data"}' | nemo files filesets create --input-file - -nemo files filesets create --