diff --git a/docs/agents/optimization.mdx b/docs/agents/optimization.mdx index 164f836bc7..e8b6aacedb 100644 --- a/docs/agents/optimization.mdx +++ b/docs/agents/optimization.mdx @@ -237,7 +237,7 @@ from nemo_agents_plugin.jobs.optimize_skills import OptimizeSkillsJob from nemo_platform_plugin.scheduler import NemoJobScheduler spec = yaml.safe_load(Path(".agent-improver.yml").read_text()) -NemoJobScheduler().run_local( +NemoJobScheduler().submit_remote( OptimizeSkillsJob, spec, workspace="default", @@ -342,7 +342,7 @@ client = NeMoPlatform( workspace=WORKSPACE, ) -result = NemoJobScheduler().run_local( +result = NemoJobScheduler().submit_remote( OptimizeJob, { "optimize_config": str(optimize_config), @@ -414,7 +414,7 @@ client = NeMoPlatform( workspace=WORKSPACE, ) -result = NemoJobScheduler().run_local( +result = NemoJobScheduler().submit_remote( OptimizeJob, { "optimize_config": str(optimize_config), diff --git a/docs/anonymizer/cli.mdx b/docs/anonymizer/cli.mdx index 9dc484d77e..5f42d667e7 100644 --- a/docs/anonymizer/cli.mdx +++ b/docs/anonymizer/cli.mdx @@ -14,9 +14,9 @@ This reference covers the `nemo anonymizer` commands exposed by the Anonymizer p | Command | Source | Description | |----------------------------------|---------------------------------|--------------------------------------------------------------| | `nemo anonymizer validate` | Manual Typer command | Validate an `AnonymizerConfig` (and optional `model_configs`).| -| `nemo anonymizer preview run` | Generated from `NemoFunction` | Local streaming preview. | +| `nemo anonymizer preview submit` | Generated from `NemoFunction` | Local streaming preview. | | `nemo anonymizer preview submit` | Generated from `NemoFunction` | Remote streaming preview against the plugin service. | -| `nemo anonymizer run run` | Generated from `NemoJob` | Local job execution in the CLI process. | +| `nemo anonymizer run submit` | Generated from `NemoJob` | Local job execution in the CLI process. | | `nemo anonymizer run submit` | Generated from `NemoJob` | Submit an `anonymizer.run` job to the NeMo Platform Jobs worker. | | `nemo anonymizer run explain` | Generated from `NemoJob` | Print the job key, submit endpoint, and JSON schemas. | @@ -39,10 +39,10 @@ The command does not accept `data.source`. Input-source validation happens durin ## `nemo anonymizer preview` -Both `preview run` and `preview submit` take a spec file matching `PreviewRequest`. +Both `preview submit` and `preview submit` take a spec file matching `PreviewRequest`. ```bash -nemo anonymizer preview run \ +nemo anonymizer preview submit \ --spec-file /tmp/anonymizer-preview.yaml \ --workspace "${NMP_WORKSPACE:-default}" @@ -60,7 +60,7 @@ nemo anonymizer preview submit \ ### Preview source kinds -| Form | `preview run` | `preview submit` | +| Form | `preview submit` | `preview submit` | |---------------------------------------|---------------|------------------| | Local path (`/tmp/input.csv`) | yes | no | | HTTP(S) URL (`https://.../input.csv`) | yes | yes | @@ -71,7 +71,7 @@ nemo anonymizer preview submit \ `preview` streams newline-delimited JSON frames to stdout. Filter with `jq`: ```bash -nemo anonymizer preview run --spec-file /tmp/anonymizer-preview.yaml > /tmp/preview.ndjson +nemo anonymizer preview submit --spec-file /tmp/anonymizer-preview.yaml > /tmp/preview.ndjson jq -R 'fromjson? | select(.kind == "preview_dataset") | .records' /tmp/preview.ndjson ``` @@ -81,7 +81,7 @@ Frame kinds: `log`, `preview_dataset`, `trace_dataset`, `failed_records`, `heart ## `nemo anonymizer run` ```bash -nemo anonymizer run run --spec-file /tmp/anonymizer-run.yaml +nemo anonymizer run submit --spec-file /tmp/anonymizer-run.yaml nemo anonymizer run submit \ --spec-file /tmp/anonymizer-run.yaml \ @@ -98,21 +98,23 @@ nemo anonymizer run explain ### Run source kinds -| Form | `run run` | `run submit` | -|---------------------------------------|-----------|--------------| -| Local path (`/tmp/input.csv`) | yes | no | -| HTTP(S) URL (`https://.../input.csv`) | yes | yes | -| Fileset reference (`fs#path`) | yes | yes | +| Form | `run submit` | +|---------------------------------------|--------------| +| Local path (`/tmp/input.csv`) | no | +| HTTP(S) URL (`https://.../input.csv`) | yes | +| Fileset reference (`fs#path`) | yes | ### Run output -`run run` prints `{"exit_code": 0}` on success. The local job results manager logs the artifact directory to stderr: +`run submit` submits an `anonymizer.run` job to the Jobs service and prints the assigned job name and submit endpoint: ```text -Saved result 'artifacts' to file:///.../persistent/results/artifacts + |-- job name: anonymizer-run-2026-05-12-abc123 + |-- submit endpoint: /apis/anonymizer/v2/workspaces/default/jobs/run +{"name": "anonymizer-run-2026-05-12-abc123", ...} ``` -The artifact directory contains: +The downloaded artifact directory contains: | File | Description | |-----------------------|--------------------------------------------------------------| @@ -121,14 +123,6 @@ The artifact directory contains: | `metadata.json` | Run metadata (includes original text column). | | `failed_records.json` | Per-record failures. Only written when records failed. | -`run submit` submits an `anonymizer.run` job to the Jobs service and prints the assigned job name and submit endpoint: - -```text - |-- job name: anonymizer-run-2026-05-12-abc123 - |-- submit endpoint: /apis/anonymizer/v2/workspaces/default/jobs/run -{"name": "anonymizer-run-2026-05-12-abc123", ...} -``` - Track and pull artifacts using either the standard `nemo jobs ...` commands or the Python SDK: ```bash @@ -145,7 +139,7 @@ dataset = results.load_dataset() See [SDK Resources](/documentation/anonymize-data/sdk-resources) for the full `AnonymizerJobResource` / `AnonymizerJobResults` surface. -Compared to `run run`, `run submit` rejects local file paths in `data.source` (use a fileset reference or `http(s)` URL) and requires explicit `model_configs` because the job runs outside the CLI process. +`run submit` rejects local file paths in `data.source` (use a fileset reference or `http(s)` URL) and requires explicit `model_configs` because the job runs on the platform worker. ## Spec File Reference @@ -154,11 +148,11 @@ Both preview and run specs use the shared `AnonymizerRequest` shape: | Field | Type | Required | Notes | |-------------------|-------------------------------------------------|----------|------------------------------------------------------------------------| | `config` | `AnonymizerConfig` | yes | Library config. See the [library docs](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs). | -| `data.source` | string | yes | Local path, `http(s)` URL, or fileset reference. | +| `data.source` | string | yes | `http(s)` URL or fileset reference. | | `data.text_column`| string | no | Defaults to `text`. | | `data.id_column` | string | no | Optional record identifier column. | | `data.data_summary` | string | no | Optional short description of the data. | -| `model_configs` | list of Data Designer `ModelConfig` | depends | Required for `preview submit` and `run submit`; optional for `preview run` and `run run`. | +| `model_configs` | list of Data Designer `ModelConfig` | depends | Required for `preview submit` and `run submit`; optional for `preview submit` and `run submit`. | | `selected_models` | object with `detection` / `replace` / `rewrite` | no | Role overrides on top of bundled defaults. Requires `model_configs`. | Preview-only: diff --git a/docs/anonymizer/index.mdx b/docs/anonymizer/index.mdx index 408dbf89a9..6e5200e131 100644 --- a/docs/anonymizer/index.mdx +++ b/docs/anonymizer/index.mdx @@ -62,11 +62,10 @@ preview_result.trace_dataset # detection trace preview_result.display_record(0) # render a record with entity highlights ``` -For a full anonymization run, execute the job locally or submit it to the Jobs worker: +For a full anonymization run, submit the job to the Jobs worker: ```bash -nemo anonymizer run run --spec-file /path/to/run-spec.yaml # in-process -nemo anonymizer run submit --spec-file /path/to/run-spec.yaml # NeMo Services job +nemo anonymizer run submit --spec-file /path/to/run-spec.yaml ``` The SDK equivalent of `run submit` is `sdk.anonymizer.run(request)`, which returns an `AnonymizerJobResource` you can poll with `wait_until_done()` and pull artifacts from with `download_artifacts()`. @@ -80,9 +79,9 @@ When using Anonymizer as a NeMo Platform service: | Feature | Standalone Library | NeMo Platform Service | |-------------------|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------------------| | **Inference** | Direct calls to NVIDIA Build defaults | Routes through the Inference Gateway via `model_configs` | -| **Execution** | Local Python process | Streaming preview runs in the plugin service; full runs execute either in the local CLI (`run run`) or on the Jobs worker (`run submit`) | -| **Input sources** | Local file, `http(s)` URL | Local file (`run run` only), `http(s)` URL, or NeMo Platform Fileset | -| **Artifacts** | Local filesystem | Local artifact directory (`persistent/results/artifacts`) for `run run`; NeMo Platform job artifact storage for `run submit` | +| **Execution** | Local Python process | Streaming preview submits in the plugin service; full runs execute on the Jobs worker | +| **Input sources** | Local file, `http(s)` URL | `http(s)` URL or NeMo Platform Fileset | +| **Artifacts** | Local filesystem | NeMo Platform job artifact storage for `run submit` | | **Authentication**| Direct API keys | NeMo Platform Secrets service | ## Replacement Strategies @@ -106,7 +105,7 @@ This package is a thin wrapper around the [NVIDIA NeMo Anonymizer library](https - A `nemo anonymizer` CLI with `validate`, `preview`, and `run` command groups. - An `sdk.anonymizer` SDK accessor (`AnonymizerResource`, `AsyncAnonymizerResource`). - A streaming `anonymizer.preview` function that emits `preview_dataset`, `trace_dataset`, and `failed_records` frames from the plugin service. -- An `anonymizer.run` job that writes `dataset.parquet`, `trace.parquet`, `metadata.json`, and optional `failed_records.json`. The job can execute in the local CLI process (`nemo anonymizer run run`) or on the NeMo Platform Jobs worker (`nemo anonymizer run submit` / `sdk.anonymizer.run`). +- An `anonymizer.run` job that writes `dataset.parquet`, `trace.parquet`, `metadata.json`, and optional `failed_records.json` on the NeMo Platform Jobs worker (`nemo anonymizer run submit` / `sdk.anonymizer.run`). - Fileset input handling (`fileset:///#`). - Inference Gateway routing for model providers referenced from `model_configs`. diff --git a/docs/anonymizer/sdk-resources.mdx b/docs/anonymizer/sdk-resources.mdx index 0f813ea0fc..a721f5f0de 100644 --- a/docs/anonymizer/sdk-resources.mdx +++ b/docs/anonymizer/sdk-resources.mdx @@ -78,7 +78,7 @@ The async variant (`AsyncAnonymizerJobResource`) exposes the same surface with ` ## AnonymizerJobResults -`download_artifacts` returns an `AnonymizerJobResults` object that loads parquet / JSON artifacts into memory. The same class also works for the local `run run` flow — point it at the artifact directory the local job results manager logs: +`download_artifacts` returns an `AnonymizerJobResults` object that loads parquet / JSON artifacts into memory. You can also point it at an extracted artifact directory: ```python from pathlib import Path diff --git a/docs/anonymizer/tutorials/index.mdx b/docs/anonymizer/tutorials/index.mdx index 2e081d8c17..44e224776a 100644 --- a/docs/anonymizer/tutorials/index.mdx +++ b/docs/anonymizer/tutorials/index.mdx @@ -79,8 +79,8 @@ When using Anonymizer as a NeMo Platform service: | Feature | Difference | Details | |----------------|---------------------------------------------------------|----------------------------------------------------------------------------------------| | **Inference** | Routes through the Inference Gateway | Configure providers once and reference them by name from `model_configs`. | -| **Input data** | Filesets and HTTP(S) URLs (local paths only in local CLI execution) | Use `sdk.files.filesets.create` / `sdk.files.upload`, then reference with `#`. | -| **Artifacts** | Local or platform-managed | `run run` writes to `persistent/results/artifacts` locally; `run submit` stores artifacts in NeMo Platform job storage. | +| **Input data** | Filesets and HTTP(S) URLs | Use `sdk.files.filesets.create` / `sdk.files.upload`, then reference with `#`. | +| **Artifacts** | Platform-managed | `run submit` stores artifacts in NeMo Platform job storage. | ## Prerequisites @@ -152,14 +152,14 @@ The tutorials reference this file with `fileset://{WORKSPACE}/anonymizer-inputs# -Stream a small anonymized sample to iterate on `AnonymizerConfig` and `model_configs`. Covers `sdk.anonymizer.preview`, `nemo anonymizer preview run` / `preview submit`, and the NDJSON frame stream. +Stream a small anonymized sample to iterate on `AnonymizerConfig` and `model_configs`. Covers `sdk.anonymizer.preview`, `nemo anonymizer preview submit` / `preview submit`, and the NDJSON frame stream. beginner anonymizer -Run the full pipeline locally with `nemo anonymizer run run` or submit it to the Jobs worker with `nemo anonymizer run submit`. Load `dataset.parquet`, `trace.parquet`, and `failed_records.json` artifacts. +Submit the full pipeline with `nemo anonymizer run submit`. Load `dataset.parquet`, `trace.parquet`, and `failed_records.json` artifacts. intermediate anonymizer diff --git a/docs/anonymizer/tutorials/preview.mdx b/docs/anonymizer/tutorials/preview.mdx index a0c0354f47..ae97366aba 100644 --- a/docs/anonymizer/tutorials/preview.mdx +++ b/docs/anonymizer/tutorials/preview.mdx @@ -38,7 +38,6 @@ Available preview surfaces: | Surface | Where it runs | Local paths | `model_configs` required | |----------------------------------|--------------------------------------|-------------|--------------------------| | `sdk.anonymizer.preview(...)` | Anonymizer plugin service (remote) | Rejected | Required | -| `nemo anonymizer preview run` | Local CLI process | Allowed | Optional | | `nemo anonymizer preview submit` | Anonymizer plugin service (remote) | Rejected | Required | ## Step 1: Build a `PreviewRequest` @@ -87,7 +86,7 @@ Field reference: | `data.text_column`| string | Column containing text to anonymize. Defaults to `text`. | | `data.id_column` | string | Optional record identifier column. | | `data.data_summary`| string | Optional short description passed to Anonymizer library prompts. | -| `model_configs` | list | Data Designer `ModelConfig` entries. `provider` must reference an Inference Gateway provider name (or `workspace/provider`). Omit to use Anonymizer library defaults (CLI local execution only). | +| `model_configs` | list | Data Designer `ModelConfig` entries. `provider` must reference an Inference Gateway provider name (or `workspace/provider`). Required for SDK and CLI preview. | | `selected_models` | object | Optional `detection` / `replace` / `rewrite` overrides on top of the bundled defaults. Requires `model_configs`. | | `num_records` | int (≥ 1) | Number of records to preview. Defaults to 10. | @@ -143,7 +142,7 @@ preview.dataset.to_parquet("anonymized-preview.parquet", index=False) ## Step 4: Run from the CLI (Alternative) -The CLI accepts the same request shape as a YAML spec file. Use `preview run` for local execution (allows local paths, model configs optional) or `preview submit` for the plugin service path (same as `sdk.anonymizer.preview`). +The CLI accepts the same request shape as a YAML spec file. `preview submit` uses the plugin service path, the same as `sdk.anonymizer.preview`. Write the spec to YAML: @@ -155,15 +154,7 @@ spec_path = Path("/tmp/anonymizer-preview.yaml") spec_path.write_text(yaml.safe_dump(request.model_dump(mode="json", exclude_none=True))) ``` -Run preview locally: - -```bash -nemo anonymizer preview run \ - --spec-file /tmp/anonymizer-preview.yaml \ - --workspace "${NMP_WORKSPACE:-default}" -``` - -Or submit to the plugin service: +Submit to the plugin service: ```bash nemo anonymizer preview submit \ @@ -172,10 +163,10 @@ nemo anonymizer preview submit \ --base-url "${NMP_BASE_URL:-http://localhost:8080}" ``` -Both commands stream NDJSON frames to stdout. Filter with `jq`: +The CLI command streams NDJSON frames to stdout. Filter with `jq`: ```bash -nemo anonymizer preview run \ +nemo anonymizer preview submit \ --spec-file /tmp/anonymizer-preview.yaml \ --workspace "${NMP_WORKSPACE:-default}" \ > /tmp/anonymizer-preview.ndjson @@ -188,13 +179,12 @@ If `preview submit` returns 404 against the gateway, the plugin service isn't mo ## Input Source Forms -The plugin accepts three forms for `data.source`: +The plugin accepts two forms for `data.source`: -| Form | `sdk.anonymizer.preview` / `preview submit` | `preview run` | -|---------------------------------------|-----------------------------------|---------------| -| Local path (`/tmp/input.csv`) | No | Yes | -| HTTP(S) URL (`https://.../input.csv`) | Yes | Yes | -| Fileset reference | Yes | Yes | +| Form | Supported | +|---------------------------------------|-----------| +| HTTP(S) URL (`https://.../input.csv`) | Yes | +| Fileset reference | Yes | Fileset references take any of these forms; the workspace and fileset must already exist: diff --git a/docs/anonymizer/tutorials/run.mdx b/docs/anonymizer/tutorials/run.mdx index 468c5a377c..9a7ddd3653 100644 --- a/docs/anonymizer/tutorials/run.mdx +++ b/docs/anonymizer/tutorials/run.mdx @@ -7,7 +7,7 @@ description: "" --- -This tutorial walks through the `anonymizer.run` job: defining a run spec, executing it locally or on the NeMo Platform Jobs worker, and loading the parquet artifacts it produces. +This tutorial walks through the `anonymizer.run` job: defining a run spec, submitting it to the NeMo Platform Jobs worker, and loading the parquet artifacts it produces. For detection, rewrite, and replacement strategy details, see the [open-source library documentation](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs). @@ -23,11 +23,10 @@ Complete the [tutorials prerequisites](/documentation/anonymize-data/tutorials#p `anonymizer.run` executes the full Anonymizer pipeline on every record of an input file and writes the output as job artifacts. -There are three run commands: +There are two run commands: | Command | Where it runs | Local paths | `model_configs` required | Artifacts | |-------------------------------|----------------------------------------------|-------------|--------------------------|--------------------------------------------------------| -| `nemo anonymizer run run` | Local CLI process via generated `is_local` path | Allowed | Optional | Written under `persistent/results/artifacts` locally | | `nemo anonymizer run submit` | NeMo Platform Jobs worker | Rejected | Required | Stored in NeMo Platform job artifact storage; pull with `download_artifacts()` | | `nemo anonymizer run explain` | Local schema introspection | n/a | n/a | Prints job key, submit endpoint, and input/spec schemas | @@ -78,7 +77,7 @@ request = AnonymizerRequest( ## Step 2: Write the Spec to YAML -The CLI run commands read a YAML spec file. Serialize the `AnonymizerRequest` directly: +The CLI submit command reads a YAML spec file. Serialize the `AnonymizerRequest` directly: ```python import yaml @@ -90,33 +89,7 @@ spec_path.write_text(yaml.safe_dump(request.model_dump(mode="json", exclude_none ## Step 3: Run the Job -Choose one execution path. Option A runs in the local CLI process. Option B submits the same request to the NeMo Platform Jobs worker. - -### Option A: Run Locally - -```bash -nemo anonymizer run run --spec-file /tmp/anonymizer-run.yaml -``` - -The local job context runs the Anonymizer library `Anonymizer.run(...)` in-process, then writes artifacts through the generated local job results manager. - -Expected output: - -```json -{"exit_code": 0} -``` - -`run run` does not echo the artifact path on stdout. The local job results manager logs the path to stderr in the form: - -```text -Saved result 'artifacts' to file:///.../persistent/results/artifacts -``` - -Use that path in the next step. - -### Option B: Submit to the Jobs Worker - -To execute the same spec on the NeMo Platform Jobs worker instead of in the CLI process, use `run submit`: +Submit the spec to the NeMo Platform Jobs worker: ```bash nemo anonymizer run submit \ @@ -140,54 +113,10 @@ sdk = NeMoPlatform( job = sdk.anonymizer.run(request) ``` -Compared to `run run`, the submit path: - -- Rejects local file paths in `data.source` — use a fileset reference (`#`) or `http(s)` URL. -- Requires explicit `model_configs` referencing Inference Gateway providers, because the job runs outside the CLI process and cannot inherit Data Designer's locally-defined providers. +The submit path rejects local file paths in `data.source` — use a fileset reference (`#`) or `http(s)` URL. It also requires explicit `model_configs` referencing Inference Gateway providers, because the job runs on the platform worker. ## Step 4: Get Results -### Option A Results: Local Run - -For `run run`, the result already exists on the local filesystem. Use the artifact directory printed in stderr: - -```bash -ARTIFACTS_DIR=/path/to/persistent/results/artifacts -ls "$ARTIFACTS_DIR" -``` - -Then load the parquet artifacts from that directory: - -```python -import json -from pathlib import Path - -import pandas as pd - -artifacts_dir = Path("/path/to/persistent/results/artifacts") # from the stderr log - -metadata = json.loads((artifacts_dir / "metadata.json").read_text()) -dataset = pd.read_parquet(artifacts_dir / "dataset.parquet", dtype_backend="pyarrow") -trace = pd.read_parquet(artifacts_dir / "trace.parquet", dtype_backend="pyarrow") - -failed_path = artifacts_dir / "failed_records.json" -failed_records = json.loads(failed_path.read_text()) if failed_path.exists() else [] - -print(dataset.head()) -print(f"records={len(dataset)} failures={len(failed_records)}") -``` - -The trace dataset (and the dataset itself for `annotate` / `substitute` strategies) contains pyarrow-backed `struct>` columns. If you need plain Python `dict`/`list` values for JSON output, use `pyarrow.parquet`: - -```python -import pyarrow.parquet as pq - -table = pq.read_table(artifacts_dir / "dataset.parquet") -records = table.slice(0, 5).to_pylist() -``` - -### Option B Results: Remote Run - For `run submit`, track the platform job first. The job is ready for artifact download when its status is `completed`: ```bash @@ -256,13 +185,13 @@ This is useful when authoring a spec programmatically or wiring the job into ano For each request, the plugin: 1. Validates the Anonymizer library `AnonymizerConfig`. -2. Validates the input source (rejects local paths on remote execution; checks fileset refs). +2. Validates the input source (rejects local paths; checks fileset refs). 3. Validates that `selected_models` overrides also have `model_configs`. -4. Resolves `model_configs` providers — locally-defined Data Designer providers first, then Inference Gateway providers. Remote execution (`run submit`) resolves only through the Inference Gateway. +4. Resolves `model_configs` providers through the Inference Gateway. 5. Renders a unified `model_configs` YAML body for the library. -6. Stores the resolved providers and YAML in the internal `AnonymizerStepConfig` consumed by the worker (in-process for `run run`, or on the Jobs worker for `run submit`). +6. Stores the resolved providers and YAML in the internal `AnonymizerStepConfig` consumed by the Jobs worker. -For `run submit`, provider endpoints are re-resolved at runtime so the job uses the in-cluster Inference Gateway address rather than the address captured at submission time. +Provider endpoints are re-resolved at runtime so the job uses the in-cluster Inference Gateway address rather than the address captured at submission time. ## Next Steps diff --git a/docs/auditor/configs/index.mdx b/docs/auditor/configs/index.mdx index 034c8dc323..17de2872d9 100644 --- a/docs/auditor/configs/index.mdx +++ b/docs/auditor/configs/index.mdx @@ -7,7 +7,7 @@ description: "" --- -An `AuditConfig` selects which garak probes and detectors run during an audit, how many generations to make per probe, and where the reports land. Configurations are persisted in the NeMo Platform entity store and referenced by name from `client.auditor.run(...)`. +An `AuditConfig` selects which garak probes and detectors run during an audit, how many generations to make per probe, and where the reports land. Configurations are persisted in the NeMo Platform entity store and referenced by name from `client.auditor.submit(...)`. ## What an AuditConfig Holds @@ -31,7 +31,7 @@ default_config = client.auditor.configs.get(workspace="system", name="default") print(default_config.model_dump_json(indent=2)) ``` -You can pass `"system/default"` directly to `client.auditor.run(config=..., ...)` to run an audit with the seeded configuration without creating your own. +You can pass `"system/default"` directly to `client.auditor.submit(config=..., ...)` to run an audit with the seeded configuration without creating your own. ## Create a Configuration diff --git a/docs/auditor/index.mdx b/docs/auditor/index.mdx index 49f7c35882..dd34a38444 100644 --- a/docs/auditor/index.mdx +++ b/docs/auditor/index.mdx @@ -27,7 +27,7 @@ A typical audit looks like the following: 1. Create an [audit target](/documentation/vulnerability-scanning/targets) for the model you want to test. 1. Create an [audit configuration](/documentation/vulnerability-scanning/configurations) that selects which garak probes and detectors to run, along with reporting settings. -1. [Run the audit](/documentation/vulnerability-scanning/tutorials/run-an-audit-locally) and inspect the resulting JSONL, HTML, and hitlog reports. +1. [Submit the audit](/documentation/vulnerability-scanning/tutorials/run-an-audit-locally) and inspect the resulting JSONL, HTML, and hitlog reports. The plugin exposes both [synchronous and asynchronous](/documentation/vulnerability-scanning/sdk-resources) Python entry points for each step. @@ -57,14 +57,14 @@ Define the model under test — generator type, model identifier, and inference Choose probes, detectors, and reporting settings for the audit. - + -End-to-end walkthrough: create entities, run the audit in-process, read the report artifacts. +End-to-end walkthrough: create entities, submit an audit job, read the report artifacts. -Reference for the `client.auditor` SDK surface: `configs`, `targets`, and `run()`. +Reference for the `client.auditor` SDK surface: `configs`, `targets`, and `submit()`. diff --git a/docs/auditor/sdk-resources.mdx b/docs/auditor/sdk-resources.mdx index ac45018d1a..0b8b3344e8 100644 --- a/docs/auditor/sdk-resources.mdx +++ b/docs/auditor/sdk-resources.mdx @@ -8,7 +8,7 @@ description: "" The NeMo Auditor plugin mounts a Python SDK surface on the `nemo_platform` client at `client.auditor`. -This page documents that surface: how to manage audit configurations and targets in the entity store, and how to run an audit in-process using the local execution path. +This page documents that surface: how to manage audit configurations and targets in the entity store, submit audit jobs, and download report artifacts. The CRUD methods exposed on `client.auditor.configs` and `client.auditor.targets` are 1:1 mirrors of the [audit configuration](/documentation/vulnerability-scanning/configurations) and [audit target](/documentation/vulnerability-scanning/targets) lifecycle and use the same `AuditConfig` and `AuditTarget` pydantic schemas the entity store persists. @@ -36,7 +36,6 @@ auditor = client.auditor # AuditorPluginResource | `submit()` | Submits a K8s audit job and returns a handle for polling and artifact download. | `AuditorJobResource` | | `list_jobs(workspace, page, page_size)` | Lists submitted audit jobs in the workspace. | `dict` | | `get_job(job_name, workspace)` | Fetches a single audit job by name. | `dict` | -| `run()` | Runs one audit locally, in-process, against a configured target. | `dict` | ### `configs` sub-resource @@ -115,85 +114,6 @@ if not job.check_if_complete(): print(f"Still running: {job.get_job_status()}") ``` -### `run()` arguments - -`run()` invokes [garak](https://github.com/NVIDIA/garak) locally, in-process, against a configured target. -The work happens entirely on the host running the SDK call — there is no remote job submission. - -| Argument | Type | Required | Description | -|----------|------|----------|-------------| -| `config` | `AuditConfig \\| str` | Yes | An inline `AuditConfig` instance or a name string referencing one in the entity store. A bare name such as `"quick-scan"` resolves against the `workspace` argument; a qualified name such as `"prod/quick-scan"` always uses the workspace prefix. | -| `target` | `AuditTarget \\| str` | Yes | An inline `AuditTarget` instance or a name string, with the same resolution rules as `config`. | -| `workspace` | `str \\| None` | No | Workspace used both as the entity-lookup fallback and as the scope for the local `JobContext`. Defaults to `"default"`. | - -### `run()` return value - -`run()` returns a dict with the following keys: - -| Key | Type | Description | -|-----|------|-------------| -| `status` | `str` | `"completed"` when garak exits with `0`, otherwise `"failed"`. | -| `returncode` | `int` | The garak subprocess exit code. | -| `stdout_tail` | `str` | Last ~4 KB of garak's stdout, useful for diagnostics. | -| `stderr_tail` | `str` | Last ~4 KB of garak's stderr. | -| `results` | `dict[str, dict]` | One entry per produced report artifact. Each value is a `ResultRef` (`{"name": str, "artifact_url": str}`). For local runs, `artifact_url` is a `file://` URL under the scheduler's temporary results directory. | - -The `results` dict can contain up to three keys, each present only if the corresponding file was produced: - -- `report-jsonl` — line-delimited JSON probe-by-probe report. -- `report-html` — rendered HTML summary. -- `report-hitlog-jsonl` — line-delimited JSON of every detected hit (failure). - -### Run an audit locally - -```python -from nemo_auditor.entities import ( - AuditSystemData, - AuditRunData, - AuditPluginsData, - AuditReportData, -) - -# Persist a configuration. -config = auditor.configs.create( - workspace="default", - name="quick-scan", - description="Lite garak scan, 3 generations per probe.", - system=AuditSystemData(lite=True, parallel_attempts=4), - run=AuditRunData(generations=3), - plugins=AuditPluginsData(probe_spec="latentinjection", detector_spec="auto"), - reporting=AuditReportData(report_prefix="quick-scan"), -) - -# Persist a target. -target = auditor.targets.create( - workspace="default", - name="llama-31-8b", - type="nim.NVOpenAIChat", - model="nvidia/meta/llama-3.1-8b-instruct", - options={ - "nim": { - "nmp_uri_spec": { - "inference_gateway": {"workspace": "default", "provider": "nvidia-inference-api"}, - }, - }, - }, -) - -# Run locally — name strings resolve via the entity store. -result = auditor.run(config="quick-scan", target="llama-31-8b", workspace="default") - -print(result["status"], result["returncode"]) -for name, ref in result["results"].items(): - print(f" {name}: {ref['artifact_url']}") -``` - -Alternatively, pass inline `AuditConfig` and `AuditTarget` instances directly — useful for ad-hoc runs that should not be persisted: - -```python -result = auditor.run(config=config, target=target, workspace="default") -``` - ## AsyncAuditorPluginResource The `AsyncAuditorPluginResource` provides the same surface for `AsyncNeMoPlatform`. @@ -218,10 +138,9 @@ auditor = client.auditor # AsyncAuditorPluginResource | `submit()` | Submits a K8s audit job and returns an async handle for polling and artifact download. | `AsyncAuditorJobResource` | | `list_jobs(workspace, page, page_size)` | Lists submitted audit jobs in the workspace. | `dict` | | `get_job(job_name, workspace)` | Fetches a single audit job by name. | `dict` | -| `run()` | Runs one audit locally, in-process, against a configured target. | `dict` | `AsyncAuditorPluginResource.submit()` returns an `AsyncAuditorJobResource` with the same methods as `AuditorJobResource` [above](#auditorjobresource), all awaitable. -`AsyncAuditorPluginResource.run()` and the async `configs` / `targets` sub-resource methods accept the same arguments as their sync counterparts. Because the local execution path is synchronous (garak runs in a subprocess), the async `run()` dispatches the scheduler call through `asyncio.to_thread` so the caller's event loop is not blocked. +The async `configs` / `targets` sub-resource methods accept the same arguments as their sync counterparts. ```python import asyncio @@ -237,14 +156,6 @@ async def main() -> None: artifacts_dir = await job.download_artifacts() print(f"Reports saved to: {artifacts_dir}") - # Or run locally (no jobs-service submission). - result = await auditor.run( - config="quick-scan", - target="llama-31-8b", - workspace="default", - ) - for name, ref in result["results"].items(): - print(f" {name}: {ref['artifact_url']}") asyncio.run(main()) ``` diff --git a/docs/auditor/targets/index.mdx b/docs/auditor/targets/index.mdx index c9b58f269a..6c6e9912a0 100644 --- a/docs/auditor/targets/index.mdx +++ b/docs/auditor/targets/index.mdx @@ -7,7 +7,7 @@ description: "" --- -An `AuditTarget` identifies the model under test. It pairs a garak generator class (`type`) with a model identifier (`model`) and a generator-specific options dict (`options`). Targets are persisted in the NeMo Platform entity store and referenced by name from `client.auditor.run(...)`. +An `AuditTarget` identifies the model under test. It pairs a garak generator class (`type`) with a model identifier (`model`) and a generator-specific options dict (`options`). Targets are persisted in the NeMo Platform entity store and referenced by name from `client.auditor.submit(...)`. ## What an AuditTarget Holds diff --git a/docs/auditor/targets/inference-gateway.mdx b/docs/auditor/targets/inference-gateway.mdx index 256b5cd522..a1656ba8aa 100644 --- a/docs/auditor/targets/inference-gateway.mdx +++ b/docs/auditor/targets/inference-gateway.mdx @@ -11,7 +11,7 @@ description: "" ## What `nmp_uri_spec` Is -`nmp_uri_spec` is a nested dict placed inside `options.` that names an Inference Gateway provider. When `client.auditor.run(...)` starts, the plugin walks the options tree, looks each `nmp_uri_spec` block up via `sdk.inference.providers.retrieve(...)`, replaces it with a `uri` key whose value is the provider's resolved OpenAI-compatible URL, and hands the rewritten options dict to garak. +`nmp_uri_spec` is a nested dict placed inside `options.` that names an Inference Gateway provider. When `client.auditor.submit(...)` starts, the plugin walks the options tree, looks each `nmp_uri_spec` block up via `sdk.inference.providers.retrieve(...)`, replaces it with a `uri` key whose value is the provider's resolved OpenAI-compatible URL, and hands the rewritten options dict to garak. The original `AuditTarget` entity stays untouched — only the in-memory copy garak receives is rewritten. @@ -33,7 +33,7 @@ Both `workspace` and `provider` are required. Other keys inside `nmp_uri_spec` a ## Conflict Rules - A dict cannot contain both `uri` and `nmp_uri_spec` — the plugin raises `ValueError` if it finds both. Pick one. -- The plugin requires a connected SDK to resolve a `nmp_uri_spec`. If `client.auditor.run(...)` is invoked without an SDK handle (a code path reserved for tests), and the target's options contain a sentinel, the run raises `RuntimeError`. +- The plugin requires a connected SDK to resolve a `nmp_uri_spec`. If `client.auditor.submit(...)` is invoked without an SDK handle (a code path reserved for tests), and the target's options contain a sentinel, the run raises `RuntimeError`. - If the named provider does not exist, or the provider lookup fails for any other reason, the run raises `RuntimeError` with the provider workspace and name embedded in the message. ## Worked Example diff --git a/docs/auditor/tutorials/index.mdx b/docs/auditor/tutorials/index.mdx index a589cf5fdd..d74b3b1470 100644 --- a/docs/auditor/tutorials/index.mdx +++ b/docs/auditor/tutorials/index.mdx @@ -14,11 +14,11 @@ Set up [a local instance of the platform](/documentation/get-started), install g - + -Persist a configuration and target, run a single audit in-process, and inspect the resulting JSONL, HTML, and hitlog report artifacts. +Persist a configuration and target, submit a single audit job, and inspect the resulting JSONL, HTML, and hitlog report artifacts. -getting started · local execution · nemo-auditor +getting started · job submission · nemo-auditor diff --git a/docs/auditor/tutorials/run-audit-locally.mdx b/docs/auditor/tutorials/run-audit-locally.mdx index 66b0d80544..ff131fcd1c 100644 --- a/docs/auditor/tutorials/run-audit-locally.mdx +++ b/docs/auditor/tutorials/run-audit-locally.mdx @@ -2,19 +2,19 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -title: "Run an Audit Locally" +title: "Submit an Audit Job" description: "" --- -This tutorial walks through running a single audit end-to-end with the NeMo Auditor plugin SDK. You will persist an audit configuration and a target, run the audit in-process, and inspect the resulting report artifacts. +This tutorial walks through running a single audit end-to-end with the NeMo Auditor plugin SDK. You will persist an audit configuration and a target, submit the audit as a platform job, and inspect the resulting report artifacts. **What you will learn:** - Initialize the NeMo Platform SDK and reach the `client.auditor` resource. - Create an `AuditConfig` selecting which garak probes to run. - Create an `AuditTarget` pointing at a model through the Inference Gateway. -- Execute the audit locally with `client.auditor.run(...)`. +- Execute the audit with `client.auditor.submit(...)`. - Read the JSONL, HTML, and hitlog report artifacts the run produces. @@ -25,7 +25,6 @@ This tutorial takes approximately **10 minutes** to complete, plus however long ## Prerequisites - Install and start NeMo Platform using the [Setup guide](/documentation/get-started). -- Install garak in a Python virtual environment so the plugin can shell out to it. By default the plugin invokes `~/.auditor/.venv/bin/python -m garak`. Override the interpreter path with `NEMO_AUDITOR_GARAK_PYTHON` if your install lives elsewhere. - Configure at least one Inference Gateway provider — this tutorial uses the `nvidia-inference-api` provider that is included in a default local setup, but any chat-completion-compatible provider works. Step 3 shows how to list the providers registered in your deployment. See [Inference Gateway](/documentation/vulnerability-scanning/targets/inference-gateway) for details on the `nmp_uri_spec` block used below. --- @@ -34,7 +33,7 @@ This tutorial takes approximately **10 minutes** to complete, plus however long - **`AuditConfig`** — Selects probes, detectors, and reporting settings for a garak run. Persisted in the entity store. - **`AuditTarget`** — Identifies the model under test (generator class, model identifier, endpoint options). Persisted in the entity store. -- **In-process run** — `client.auditor.run(...)` shells out to garak on the host running the SDK call, writes report artifacts under a temporary directory, and returns their file paths. There is no remote job submission. +- **Submitted audit job** — `client.auditor.submit(...)` creates a platform job. The returned job resource can wait for completion, stream logs while polling, and download the produced garak reports. --- @@ -128,50 +127,57 @@ The `nmp_uri_spec` sentinel inside `options.nim` tells the plugin to resolve a c --- -## 4. Run the Audit Locally +## 4. Submit the Audit Job -`run()` accepts either inline entities or name strings that reference entities in the entity store. Pass the names you persisted above: +`submit()` accepts either inline entities or name strings that reference entities in the entity store. Pass the names you persisted above: ```python -result = auditor.run( +job = auditor.submit( config="quick-scan", target="llama-31-8b", workspace="default", ) -print(result["status"], result["returncode"]) -# completed 0 +print(f"Job submitted: {job.name}") -for name, ref in result["results"].items(): - print(f"{name}: {ref['artifact_url']}") -# report-jsonl: file:///var/folders/.../results/report-jsonl -# report-html: file:///var/folders/.../results/report-html -# report-hitlog-jsonl: file:///var/folders/.../results/report-hitlog-jsonl +job.wait_until_done() ``` If you prefer to skip the entity-store roundtrip, pass the inline `AuditConfig` and `AuditTarget` objects from the previous two steps directly: ```python -result = auditor.run(config=config, target=target, workspace="default") +job = auditor.submit(config=config, target=target, workspace="default") +job.wait_until_done() ``` --- ## 5. Read the Results -`run()` returns `file://` URLs for whichever of the three report types garak produced. Parse them with `urllib.parse.urlparse` to recover local paths, then load them as needed. +Download the garak report artifacts after the job completes: + +```python +from pathlib import Path + +artifacts_dir = Path(job.download_artifacts(path="./auditor-reports")) +print(f"Reports saved to: {artifacts_dir}") +``` Load the JSONL probe-by-probe summary: ```python import json -from pathlib import Path -from urllib.parse import urlparse -def url_to_path(url: str) -> Path: - return Path(urlparse(url).path) -jsonl_path = url_to_path(result["results"]["report-jsonl"]["artifact_url"]) +def first_existing(*patterns: str) -> Path: + for pattern in patterns: + matches = sorted(artifacts_dir.glob(pattern)) + if matches: + return matches[0] + raise FileNotFoundError(patterns[0]) + + +jsonl_path = first_existing("*.report.jsonl", "report-jsonl") records = [json.loads(line) for line in jsonl_path.read_text().splitlines() if line.strip()] print(f"Loaded {len(records)} report entries.") ``` @@ -181,18 +187,19 @@ Summarize hits per probe from the hitlog: ```python from collections import Counter -hitlog_ref = result["results"].get("report-hitlog-jsonl") -if hitlog_ref is None: + +hitlog_matches = sorted(artifacts_dir.glob("*.hitlog.jsonl")) or sorted(artifacts_dir.glob("report-hitlog-jsonl")) +hitlog_path = hitlog_matches[0] if hitlog_matches else None +if hitlog_path is None: print("No hits recorded — every probe passed.") else: - hitlog_path = url_to_path(hitlog_ref["artifact_url"]) hits = [json.loads(line) for line in hitlog_path.read_text().splitlines() if line.strip()] by_probe = Counter(entry.get("probe") for entry in hits) for probe, count in by_probe.most_common(): print(f"{probe}: {count} hits") ``` -The HTML report at `result["results"]["report-html"]["artifact_url"]` is the most human-friendly summary — open it in a browser to see grouped pass/fail counts and per-probe details. +The HTML report (`*.report.html` or `report-html` in the extracted directory) is the most human-friendly summary — open it in a browser to see grouped pass/fail counts and per-probe details. --- @@ -205,29 +212,26 @@ auditor.configs.delete(workspace="default", name="quick-scan") auditor.targets.delete(workspace="default", name="llama-31-8b") ``` -Report artifacts are left in place under the scheduler's temporary directory; the OS reaps them eventually. +Downloaded report artifacts remain under the directory you passed to `download_artifacts()`. --- ## Troubleshooting -**`FileNotFoundError: garak interpreter not found at ...`** -: The plugin couldn't find a garak install. Either install garak at `~/.auditor/.venv/bin/python`, or set `NEMO_AUDITOR_GARAK_PYTHON` to the absolute path of a Python interpreter that has garak installed. - **`RuntimeError: Failed to resolve inference gateway provider '/'`** : The `nmp_uri_spec` block in your target's options references an Inference Gateway provider that doesn't exist. List your providers with `client.inference.providers.list(workspace="default")` and update the target to reference an existing one. -**`returncode != 0` with empty `results`** -: garak started but failed early, usually because the target endpoint is unreachable. Inspect `result["stderr_tail"]` for the error message, and verify the model is reachable through the provider you configured. +**The audit job fails before producing results** +: Inspect the job logs with `job.get_logs()` or `job.wait_until_done()`, and verify the model is reachable through the provider you configured. **Where do the report files live?** -: The scheduler writes garak output under `/garak//.*`, then copies the produced files into the local results directory referenced by each `artifact_url`. For a local run this is a temporary directory under the system's `$TMPDIR`. +: The Jobs service stores report artifacts for the completed job. `job.download_artifacts()` extracts them into the directory you choose. --- ## Summary and Next Steps -You created an `AuditConfig` and `AuditTarget`, ran a single audit locally with `client.auditor.run(...)`, and loaded the resulting reports. +You created an `AuditConfig` and `AuditTarget`, submitted a single audit job with `client.auditor.submit(...)`, and loaded the resulting reports. - For the full SDK surface — including async variants — see [SDK Resources](/documentation/vulnerability-scanning/sdk-resources). - For more probe selection options, see [Selecting Probes](/documentation/vulnerability-scanning/configurations/selecting-probes). diff --git a/docs/data-designer/cli.mdx b/docs/data-designer/cli.mdx index 5ec32b7c7b..b203a1eadc 100644 --- a/docs/data-designer/cli.mdx +++ b/docs/data-designer/cli.mdx @@ -40,8 +40,8 @@ The same configuration source can usually be used with `run` or `submit`. Resour | Command | Workload execution | NeMo Services required? | |---------|--------------------|-------------------------| -| `preview run` | Local CLI process | Optional | -| `create run` | Local CLI process | Optional | +| `preview submit` | Local CLI process | Optional | +| `create submit` | Local CLI process | Optional | | `preview submit` | Data Designer API | Yes | | `create submit` | Jobs worker | Yes | @@ -50,7 +50,7 @@ The same configuration source can usually be used with `run` or `submit`. Resour Use local preview for fast iteration: ```bash -nemo data-designer preview run product_reviews.py --num-records 5 +nemo data-designer preview submit product_reviews.py --num-records 5 ``` The workload runs in your current Python environment. It can use local-only resources, NeMo resources, or both. @@ -60,7 +60,7 @@ The workload runs in your current Python environment. It can use local-only reso Use local create when you want to generate a larger dataset without submitting work to NeMo Services: ```bash -nemo data-designer create run product_reviews.py --num-records 1000 +nemo data-designer create submit product_reviews.py --num-records 1000 ``` This executes the plugin job locally. It is useful for development and for workloads that should stay in the local environment. @@ -89,7 +89,7 @@ NeMo Services creates and runs a job. Job logs, status, and artifacts are manage The plugin also provides commands for Nemotron Personas datasets. -Install personas locally for local execution: +List or download personas for inspection: ```bash nemo data-designer personas download --list @@ -115,4 +115,4 @@ nemo data-designer personas make-fileset \ ## SDK Relationship -The SDK currently executes through the Data Designer API. If you need local in-process execution today, use `nemo data-designer ... run`. +The SDK executes through the Data Designer API and Jobs API, matching the CLI `submit` paths. diff --git a/docs/data-designer/execution-modes.mdx b/docs/data-designer/execution-modes.mdx index 2618a93982..ce27bea187 100644 --- a/docs/data-designer/execution-modes.mdx +++ b/docs/data-designer/execution-modes.mdx @@ -9,25 +9,18 @@ description: "" After building Data Designer configurations with the library's `data_designer.config` APIs, you choose where the NeMo Data Designer plugin executes the workload. -The important distinction is not simply "local" versus "remote". There are two separate questions: +The important distinction is where resources are resolved and where submitted work executes: | Axis | Question | Examples | |------|----------|----------| -| **Execution location** | Where does the Data Designer workload run? | In the local CLI process with `run`, or in NeMo Services with `submit` or the SDK. | +| **Execution location** | Where does the Data Designer workload run? | In NeMo Services with `submit` or the SDK. | | **Resource backend** | What does the configuration depend on? | Local files, local provider settings, environment variables, or NeMo resources such as Filesets, Secrets, and Inference Gateway providers. | -`run` versus `submit` primarily controls where the plugin workload execution happens. It does not necessarily determine whether the workload uses NeMo Services APIs. - - - -`nemo data-designer ... run` can be fully local, but it is not an offline-only mode. A local run can still use the Files API, Secrets API, and Inference Gateway API from a running NeMo Services cluster when the configuration references the corresponding resources. - - +`submit` sends the request through NeMo Services so execution, lifecycle, and artifacts use the supported platform path. ## Terms | Term | Meaning | |------|---------| -| **Local execution** | The Data Designer workload runs in the CLI process. | | **Local resources** | The configuration uses local files, local provider settings, environment variables, or other resources available to the CLI process. | | **NeMo resources** | The configuration uses resources in NeMo Services, such as Filesets, Secrets, or Inference Gateway providers. | | **NeMo Services execution** | The Data Designer workload runs through the Data Designer API or a Jobs worker. | @@ -37,33 +30,29 @@ The important distinction is not simply "local" versus "remote". There are two s | Interface | Workload execution | NeMo Services required? | Resource behavior | |-----------|--------------------|-------------------------|-------------------| -| `nemo data-designer preview run` | Local CLI process | Optional | Can use local resources and/or NeMo resources. | -| `nemo data-designer create run` | Local CLI process | Optional | Can use local resources and/or NeMo resources. | | `nemo data-designer preview submit` | Data Designer API | Yes | Uses resources supported by NeMo Services. | | `nemo data-designer create submit` | Jobs worker | Yes | Uses resources supported by NeMo Services and job artifact storage. | -| `client.data_designer.preview/create` | Data Designer API or Jobs worker | Yes | Local SDK execution is planned but not available yet. | +| `client.data_designer.preview/create` | Data Designer API or Jobs worker | Yes | Uses the same platform execution paths. | ## Resource Compatibility -Local CLI execution supports the broadest set of resource patterns because the workload runs in your Python environment. +Platform execution supports NeMo resource patterns: -| Resource type | CLI `run` | CLI `submit` / SDK today | -|---------------|-----------|--------------------------| -| Local seed files or DataFrames | Supported | Not supported; upload to Filesets first. | -| HuggingFace seed datasets | Supported | Supported. | -| Files API Filesets | Supported when NeMo Services access is configured | Supported. | -| Local provider settings | Supported | Not supported; use Inference Gateway providers. | -| Inference Gateway providers | Supported when NeMo Services access is configured | Supported. | -| Environment or plaintext secrets | Supported | Not supported; use Secrets API secrets. | -| Secrets API secrets | Supported when NeMo Services access is configured | Supported. | +| Resource type | CLI `submit` / SDK today | +|---------------|--------------------------| +| Local seed files or DataFrames | Not supported; upload to Filesets first. | +| HuggingFace seed datasets | Supported. | +| Files API Filesets | Supported. | +| Local provider settings | Not supported; use Inference Gateway providers. | +| Inference Gateway providers | Supported. | +| Environment or plaintext secrets | Not supported; use Secrets API secrets. | +| Secrets API secrets | Supported. | ## Choosing a Mode -Use CLI `run` when you want fast local iteration, direct access to local files, or library-equivalent workload behavior. - Use CLI `submit` when you want service-managed execution, job lifecycle tracking, shared artifacts, or the same execution environment other users and automation will use. -Use the SDK when you are writing application code that should call the Data Designer API programmatically. Today, the SDK path exclusively uses NeMo Services; use CLI `run` for local in-process execution. +Use the SDK when you are writing application code that should call the Data Designer API programmatically. If you run NeMo Services fully locally, `submit` and SDK calls can target that local deployment. The services are local, but the workload execution path still uses the Data Designer API and Jobs API. diff --git a/docs/data-designer/index.mdx b/docs/data-designer/index.mdx index 25a871acaa..4b1ea42b08 100644 --- a/docs/data-designer/index.mdx +++ b/docs/data-designer/index.mdx @@ -57,17 +57,16 @@ The same configuration can run through different plugin surfaces: | Interface | Execution location | NeMo Services required? | Best for | |-----------|--------------------|-------------------------|----------| -| `nemo data-designer ... run` | Local CLI process | Optional | Fast local iteration, local files, library-equivalent workload behavior. | | `nemo data-designer ... submit` | Data Designer API or Jobs worker | Yes | Service-managed execution, logs, artifacts, and shared resources. | | `client.data_designer.preview/create` | Data Designer API or Jobs worker | Yes | Application code that calls Data Designer programmatically. | -`run` versus `submit` primarily controls where the plugin workload execution happens. A local `run` can be fully local, but it is not an offline-only mode: it can still use the Files API, Secrets API, and Inference Gateway API from a running NeMo Services cluster when the configuration references the corresponding resources. +Data Designer execution goes through CLI `submit` or the SDK. These paths use NeMo Services for request validation, resource resolution, jobs, logs, and artifacts. See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for the full model. ## NeMo Services Integration -When you use CLI `submit`, SDK execution, or NeMo resources from a local `run`, the plugin integrates with these NeMo Services APIs: +When you use CLI `submit` or SDK execution, the plugin integrates with these NeMo Services APIs: | Integration | What it provides | |-------------|------------------| @@ -76,7 +75,7 @@ When you use CLI `submit`, SDK execution, or NeMo resources from a local `run`, | **Secrets API** | API keys and tokens referenced from Data Designer configurations. | | **Jobs API** | Service-managed create workloads, logs, status, and artifacts. | -These integrations are required for `submit` and SDK execution. They are optional for CLI `run` execution, depending on the resources your configuration references. +These integrations are required for `submit` and SDK execution. ## Next Steps @@ -84,7 +83,7 @@ These integrations are required for `submit` and SDK execution. They are optiona -Understand local execution, NeMo Services execution, and NeMo resources. +Understand NeMo Services execution and resource requirements. diff --git a/docs/data-designer/migration.mdx b/docs/data-designer/migration.mdx index 215dc8c6fb..7dfe9293e9 100644 --- a/docs/data-designer/migration.mdx +++ b/docs/data-designer/migration.mdx @@ -7,7 +7,7 @@ description: "" --- -Data Designer configurations are mostly portable across plugin execution modes. The main migration question is not "library versus service"; it is whether the workload executes locally in the CLI process or through NeMo Services, and which resources the configuration references. +Data Designer configurations are mostly portable across plugin execution surfaces. The main migration question is which resources the configuration references when it runs through NeMo Services. See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for the full two-axis model. @@ -17,59 +17,25 @@ Most configuration code can stay unchanged: | Configuration area | Portability | |--------------------|-------------| -| Column definitions | Same across CLI `run`, CLI `submit`, and SDK execution. | +| Column definitions | Same across CLI `submit` and SDK execution. | | Prompt templates and Jinja2 references | Same across execution modes. | | Constraints and profilers | Same across execution modes. | | Model aliases and inference parameters | Same across execution modes. | | Processors and generated dataset schema | Same across execution modes, subject to features supported by NeMo Services. | -For CLI execution, put your configuration in a Python module that exposes `load_config_builder()`. The plugin CLI loads the builder and executes it through either `run` or `submit`. - -## Moving to Local CLI Execution - -Use local CLI execution when you want the Data Designer workload to run in your current Python environment. Local CLI execution can use fully local resources, NeMo resources, or a mix of both: - -| Resource pattern | Local CLI `run` behavior | -|------------------|--------------------------| -| Local files or DataFrames | Supported. The workload runs where those objects are available. | -| Local provider settings | Supported. The plugin checks local provider configuration first. | -| Environment or plaintext secrets | Supported. | -| Files API Filesets | Supported when NeMo Services access is configured. | -| Secrets API secrets | Supported when NeMo Services access is configured. | -| Inference Gateway providers | Supported when NeMo Services access is configured. | - -This can be fully local, but it is not an offline-only mode. If the configuration references `default/nvidia-build`, a Fileset, or a Secrets API secret, the local workload still communicates with those NeMo Services APIs. +For CLI execution, put your configuration in a Python module that exposes `load_config_builder()`. The plugin CLI loads the builder and submits it through the supported platform path. ## Moving to NeMo Services Execution -Use NeMo Services execution when you want service-managed workload execution, logs, jobs, and artifacts. When moving a configuration from local CLI `run` to `submit` or SDK execution, check these areas: - -| Area | Local CLI `run` | NeMo Services `submit` / SDK today | Migration action | -|------|-----------------|------------------------------------|------------------| -| **Inference** | Local providers and/or Inference Gateway providers. | Inference Gateway providers. | Register an Inference Gateway provider and reference it by workspace/name, such as `default/nvidia-build`. | -| **Secrets** | Environment variables, plaintext values, and Secrets API secrets. | Secrets API secrets. | Create Secrets API secrets and reference them from the configuration. | -| **Seed data** | Local files, DataFrames, HuggingFace, or Filesets. | HuggingFace or Filesets. | Upload local seed data to a Files API Fileset or use HuggingFace. | -| **Personas** | Local persona datasets downloaded to the managed assets directory. | Persona Filesets in NeMo Services. | Use `nemo data-designer personas make-fileset` for required locales. | -| **Artifacts** | Local execution output. | Job artifact storage. | Use job status/logs and artifact download helpers. | - -## Moving Back to Local CLI Execution - -If a configuration already runs through `submit` or the SDK, it can usually run through CLI `run` without changing NeMo resource references. Local execution can still use Filesets, Secrets, and Inference Gateway providers. - -You may choose to simplify the configuration for local-only iteration: - -| NeMo resource | Local CLI alternative | -|---------------|-----------------------| -| Fileset seed data | Local file or DataFrame seed data. | -| Secrets API secret | Environment variable or plaintext secret. | -| Inference Gateway provider | Local provider settings. | -| Persona Fileset | Locally downloaded persona dataset. | - -These changes are optional. Keep NeMo resource references when you want the same configuration to work in both `run` and `submit`. - -## SDK Local Execution +Use NeMo Services execution when you want service-managed workload execution, logs, jobs, and artifacts. When moving a configuration into `submit` or SDK execution, check these areas: -SDK execution uses the Data Designer API today. Local SDK execution is planned, but not available yet. Use CLI `run` for local in-process execution until the SDK supports the same mode. +| Area | NeMo Services `submit` / SDK today | Migration action | +|------|------------------------------------|------------------| +| **Inference** | Inference Gateway providers. | Register an Inference Gateway provider and reference it by workspace/name, such as `default/nvidia-build`. | +| **Secrets** | Secrets API secrets. | Create Secrets API secrets and reference them from the configuration. | +| **Seed data** | HuggingFace or Filesets. | Upload local seed data to a Files API Fileset or use HuggingFace. | +| **Personas** | Persona Filesets in NeMo Services. | Use `nemo data-designer personas make-fileset` for required locales. | +| **Artifacts** | Job artifact storage. | Use job status/logs and artifact download helpers. | ## Migration Checklist @@ -79,11 +45,11 @@ Before switching execution modes, verify: - Model providers referenced in `ModelConfig.provider` are available in the target mode. - Secret references are resolvable in the target mode. - Seed data sources are compatible with the target mode. -- Persona datasets are installed locally or published as Files API Filesets as needed. +- Persona datasets are published as Files API Filesets as needed. ## Getting Help - **Execution Modes:** See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for the conceptual model. -- **CLI:** See [Data Designer CLI](/documentation/design-synthetic-data/cli) for `run`, `submit`, and persona commands. +- **CLI:** See [Data Designer CLI](/documentation/design-synthetic-data/cli) for `submit` and persona commands. - **Tutorials:** Follow the [tutorials](/documentation/design-synthetic-data/tutorials) for hands-on examples. - **Library Docs:** Refer to the [open-source library documentation](https://docs.nvidia.com/nemo/datadesigner/v0.9.1/getting-started/welcome) for configuration details. diff --git a/docs/data-designer/sdk-resources.mdx b/docs/data-designer/sdk-resources.mdx index a221d6776c..7aa14964b9 100644 --- a/docs/data-designer/sdk-resources.mdx +++ b/docs/data-designer/sdk-resources.mdx @@ -13,7 +13,7 @@ This page explains the SDK objects used for Data Designer API execution. -The SDK currently executes Data Designer workloads through the Data Designer API. Local SDK execution is planned, but not available yet. Use `nemo data-designer ... run` for local in-process execution today. +The SDK executes Data Designer workloads through the Data Designer API and Jobs API, matching the CLI `submit` paths. ## DataDesignerResource diff --git a/docs/data-designer/tutorials/basics.mdx b/docs/data-designer/tutorials/basics.mdx index 8741e931a3..74854b3474 100644 --- a/docs/data-designer/tutorials/basics.mdx +++ b/docs/data-designer/tutorials/basics.mdx @@ -13,7 +13,7 @@ For more detail about column behavior, see the [open-source library's version](h ## Prerequisites -Ensure you have completed the [tutorials prerequisites](/documentation/design-synthetic-data/tutorials#prerequisites). This tutorial uses an Inference Gateway provider, so local CLI `run` and NeMo Services execution both need access to the Inference Gateway API in a running NeMo Services cluster. +Ensure you have completed the [tutorials prerequisites](/documentation/design-synthetic-data/tutorials#prerequisites). This tutorial uses an Inference Gateway provider, so NeMo Services must be running. ## Part 1: Build the Configuration @@ -21,7 +21,7 @@ Use the `data_designer.config` package to define your dataset schema. This confi -Build the configuration once, then choose whether to execute with CLI `run`, CLI `submit`, or the SDK. +Build the configuration once, then execute with CLI `submit` or the SDK. ### Define Models @@ -203,23 +203,7 @@ def load_config_builder() -> dd.DataDesignerConfigBuilder: return config_builder ``` -Preview locally: - -```bash -nemo data-designer preview run product_reviews.py --num-records 5 -``` - -Generate a larger dataset locally: - -```bash -nemo data-designer create run product_reviews.py --num-records 30 -``` - -This workload runs in the local CLI process, but because the configuration references `default/nvidia-build`, it still communicates with the Inference Gateway API. - -### NeMo Services CLI Execution - -Submit the same configuration to NeMo Services when you want service-managed execution: +Submit the configuration to NeMo Services: ```bash nemo data-designer preview submit product_reviews.py --workspace default --num-records 5 @@ -287,16 +271,10 @@ analysis.to_report() ## What Happens Under the Hood -When you use CLI `run`: - -1. **Local Execution:** The Data Designer workload runs in the CLI process. -2. **Resource Resolution:** The workload can use local resources, NeMo resources, or both. -3. **Generation:** Data Designer resolves dependencies and generates records in the local environment. - When you use CLI `submit` or the SDK today: 1. **Configuration Validation:** The service validates your configuration and resolves column dependencies -2. **NeMo Services Execution:** Preview runs through the Data Designer API; create runs as a service-managed job +2. **NeMo Services Execution:** Preview runs through the Data Designer API; create submits as a service-managed job 3. **Inference Routing:** LLM calls are routed through Inference Gateway to your configured model providers 4. **Artifact Storage:** Job datasets and analysis reports are stored in job artifact storage 5. **Job Completion:** You can monitor job status and load results when complete diff --git a/docs/data-designer/tutorials/index.mdx b/docs/data-designer/tutorials/index.mdx index b7c5c4eebc..01c9a53e96 100644 --- a/docs/data-designer/tutorials/index.mdx +++ b/docs/data-designer/tutorials/index.mdx @@ -33,10 +33,10 @@ config_builder.add_column(dd.LLMTextColumnConfig(...)) **Part 2: Execute (Plugin)** -Run the configuration locally with the CLI, submit it to NeMo Services, or call the Data Designer API from the SDK: +Submit the configuration with the CLI or call the Data Designer API from the SDK: ```bash -nemo data-designer preview run product_reviews.py --num-records 5 +nemo data-designer preview submit product_reviews.py --num-records 5 nemo data-designer create submit product_reviews.py --workspace default --num-records 30 ``` @@ -57,23 +57,23 @@ job = data_designer.create(config_builder, num_records=1000) -`run` versus `submit` primarily controls where the workload executes. Local `run` can still use the Files API, Secrets API, and Inference Gateway API from a running NeMo Services cluster when the configuration references the corresponding resources. See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for details. +`submit` sends the workload through NeMo Services. See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for details. ## Execution-Specific Considerations When running through the plugin, supported resources depend on the execution mode: -| Feature | CLI `run` | CLI `submit` / SDK | -|---------|-----------|--------------------------| -| **Inference** | Local providers and/or Inference Gateway providers | Inference Gateway providers | -| **Seed data** | Local sources, HuggingFace, or Files API Filesets | HuggingFace or Files API Filesets | -| **Secrets** | Environment, plaintext, or Secrets API secrets | Secrets API secrets | -| **Artifacts** | Local execution artifacts | Job artifact storage | +| Feature | CLI `submit` / SDK | +|---------|--------------------------| +| **Inference** | Inference Gateway providers | +| **Seed data** | HuggingFace or Files API Filesets | +| **Secrets** | Secrets API secrets | +| **Artifacts** | Job artifact storage | ## Prerequisites -These tutorials use an [Inference Gateway](/documentation/models-and-inference) provider for model calls, so a NeMo Services cluster must be running before you preview or create data — including with local CLI `run` (see [Execution Modes](/documentation/design-synthetic-data/execution-modes#local-nemo-services-execution) for more about this distinction). +These tutorials use an [Inference Gateway](/documentation/models-and-inference) provider for model calls, so a NeMo Services cluster must be running before you preview or create data. Complete [Setup](/documentation/get-started) to ensure you have the NeMo Services running locally and an inference provider available. These tutorials reference the default NVIDIA Build model provider, which is created as `default/nvidia-build` during setup. diff --git a/docs/data-designer/tutorials/seeding.mdx b/docs/data-designer/tutorials/seeding.mdx index 56dfedf2f4..c6abd73886 100644 --- a/docs/data-designer/tutorials/seeding.mdx +++ b/docs/data-designer/tutorials/seeding.mdx @@ -13,19 +13,13 @@ For more detail about seed dataset behavior, see the [open-source library's vers ## Seed Sources by Execution Mode -Seed source support depends on where the workload executes: +Seed source support depends on platform resource availability: -| Seed source | CLI `run` | CLI `submit` / SDK today | Use case | -|-------------|-----------|--------------------------|----------| -| **Local files or DataFrames** | Supported | Not supported | Fast local iteration with files available to the CLI process. | -| **HuggingFace** | Supported | Supported | Publicly available datasets or private HuggingFace datasets. | -| **Files API Filesets** | Supported when NeMo Services access is configured | Supported | Shared seed data stored through the Files API. | - - - -`run` versus `submit` controls where the workload executes. A local `run` can still read Files API Filesets if the configuration references them and NeMo Services access is configured. - - +| Seed source | CLI `submit` / SDK today | Use case | +|-------------|--------------------------|----------| +| **Local files or DataFrames** | Not supported | Upload local files to Filesets first. | +| **HuggingFace** | Supported | Publicly available datasets or private HuggingFace datasets. | +| **Files API Filesets** | Supported | Shared seed data stored through the Files API. | ### HuggingFace Datasets Use `HuggingFaceSeedSource` to load data from HuggingFace: @@ -45,7 +39,7 @@ dd.HuggingFaceSeedSource( ### Files API Filesets -Use `FilesetFileSeedSource` to load data through the Files API. This works in CLI `run`, CLI `submit`, and SDK execution when NeMo Services access is configured: +Use `FilesetFileSeedSource` to load data through the Files API. This works in CLI `submit` and SDK execution: ```python from data_designer_nemo.fileset_file_seed_source import FilesetFileSeedSource @@ -61,11 +55,11 @@ FilesetFileSeedSource( ## Prerequisites -Ensure you have completed the [tutorials prerequisites](/documentation/design-synthetic-data/tutorials#prerequisites). This tutorial uses an Inference Gateway provider, so local CLI `run` and NeMo Services execution both need access to the Inference Gateway API in a running NeMo Services cluster. +Ensure you have completed the [tutorials prerequisites](/documentation/design-synthetic-data/tutorials#prerequisites). This tutorial uses an Inference Gateway provider, so NeMo Services must be running. ## Example: Medical Notes from Symptom Data -This example generates realistic patient medical notes by seeding with publicly available symptom-to-diagnosis data. It uploads the seed data to a Files API Fileset so the same configuration can run locally through CLI `run` or through NeMo Services execution. +This example generates realistic patient medical notes by seeding with publicly available symptom-to-diagnosis data. It uploads the seed data to a Files API Fileset so the configuration can run through NeMo Services. ### Step 1: Upload Seed Data @@ -263,13 +257,13 @@ def load_config_builder() -> dd.DataDesignerConfigBuilder: Preview locally: ```bash -nemo data-designer preview run medical_notes.py --num-records 5 +nemo data-designer preview submit medical_notes.py --num-records 5 ``` Generate a larger dataset locally: ```bash -nemo data-designer create run medical_notes.py --num-records 30 +nemo data-designer create submit medical_notes.py --num-records 30 ``` Submit to NeMo Services: diff --git a/docs/evaluator/index.mdx b/docs/evaluator/index.mdx index 5cedb0a7bb..e8b1fe1701 100644 --- a/docs/evaluator/index.mdx +++ b/docs/evaluator/index.mdx @@ -86,7 +86,7 @@ dataset = [ config = RunConfig(limit_samples=100, parallelism=8) ``` -### 2. Run it — three dataset-driven modes +### 2. Run it — two dataset-driven modes The same `metric`, `dataset`, and `config` run in two places. What changes is the **caller** — a bare SDK evaluator for local iteration, or the platform's `client.evaluator` resource for durable @@ -139,14 +139,14 @@ the same and execution gains platform capabilities: | **Inference** | Direct model or agent endpoint calls | The same, and can route through the NeMo Platform [Inference Gateway](/documentation/models-and-inference) and platform-managed endpoints | | **Datasets** | Inline rows and local files | Inline rows, local paths resolved at submission time, and NeMo Platform [Filesets](/documentation/get-started/core-concepts/manage-files) | | **Results** | Returned in memory | Platform artifact storage with typed result download | -| **Authentication** | Local environment variables | Local env vars for local runs; NeMo Platform [Secrets](/documentation/get-started/core-concepts/manage-secrets) for remote jobs | +| **Authentication** | Local environment variables | NeMo Platform [Secrets](/documentation/get-started/core-concepts/manage-secrets) for jobs | ### Live vs. jobs, online vs. offline Two more distinctions cut across the modes above: -- **Live (synchronous) vs. jobs (asynchronous).** `run()` returns results immediately — best for fast - iteration, metric development, and small payloads. `submit()` creates a durable job you monitor and +- **Live (synchronous) vs. jobs (asynchronous).** The standalone SDK `run()` returns results immediately — best for fast + iteration, metric development, and small payloads. Platform `submit()` creates a durable job you monitor and fetch results from — best for production workloads, larger datasets, and recurring regression checks. - **Offline vs. online.** *Offline* scores dataset rows that already contain outputs. *Online* diff --git a/docs/evaluator/metrics/agent-configuration.mdx b/docs/evaluator/metrics/agent-configuration.mdx index eba130ecea..361a30116b 100644 --- a/docs/evaluator/metrics/agent-configuration.mdx +++ b/docs/evaluator/metrics/agent-configuration.mdx @@ -83,7 +83,7 @@ agent = Agent( url="https://my-agent.example.com/invoke", name="qa-agent", format="generic", - api_key_secret="MY_AGENT_API_KEY", + api_key_secret="my-agent-api-key", body={"question": "{{ prompt }}"}, response_path="$.answer", trajectory_path="$.reasoning_steps", diff --git a/docs/evaluator/metrics/agentic.mdx b/docs/evaluator/metrics/agentic.mdx index ee371ae598..35052a99cb 100644 --- a/docs/evaluator/metrics/agentic.mdx +++ b/docs/evaluator/metrics/agentic.mdx @@ -1259,7 +1259,7 @@ Use `RunConfig(limit_samples=...)` when you want to test a small slice of a larg ## Important Notes -1. **Execution choice**: Use `run` for local in-process evaluation and `submit` for durable remote jobs with `wait_until_done()` and `get_result()`. +1. **Execution choice**: Use `submit` for durable platform jobs with `wait_until_done()` and `get_result()`. 2. **Column Names**: RAGAS metrics use specific column names: - `user_input` (not `question`) diff --git a/docs/evaluator/metrics/index.mdx b/docs/evaluator/metrics/index.mdx index 7849fa69e4..7239083e85 100644 --- a/docs/evaluator/metrics/index.mdx +++ b/docs/evaluator/metrics/index.mdx @@ -78,8 +78,8 @@ Metrics can be executed in two modes: | Mode | Use Case | Response | |------|----------|----------| -| **Live Evaluation** | Rapid prototyping, developing metrics, testing configurations. | Immediate (synchronous) | -| **Job Evaluation** | Production workloads, full datasets, durability, and persistence | Async (poll for completion) | +| **Standalone SDK Evaluation** | Rapid prototyping, developing metrics, testing configurations with `nemo_evaluator_sdk`. | Immediate (synchronous) | +| **Job Evaluation** | Platform workloads, full datasets, durability, and persistence | Async (poll for completion) | ### Online Job Targets: Model or Agent diff --git a/docs/evaluator/metrics/llm-as-a-judge.mdx b/docs/evaluator/metrics/llm-as-a-judge.mdx index 064fd5695d..999eaf000c 100644 --- a/docs/evaluator/metrics/llm-as-a-judge.mdx +++ b/docs/evaluator/metrics/llm-as-a-judge.mdx @@ -125,6 +125,8 @@ result = LocalEvaluator().run_sync( }, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean:.2f}, count={score.count}") @@ -228,6 +230,8 @@ result = LocalEvaluator().run_sync( ], aggregate_fields=("rubric_distribution", "mode_category"), ) +job.wait_until_done() +result = job.get_result() print(result.aggregate_scores.model_dump(exclude_none=True)) ``` @@ -247,6 +251,8 @@ result = LocalEvaluator().run_sync( ], aggregate_fields=("std_dev", "variance"), ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}:") diff --git a/docs/evaluator/metrics/model-configuration.mdx b/docs/evaluator/metrics/model-configuration.mdx index 4657d908db..2ccfe36f1d 100644 --- a/docs/evaluator/metrics/model-configuration.mdx +++ b/docs/evaluator/metrics/model-configuration.mdx @@ -201,9 +201,9 @@ job.wait_until_done() result = job.get_result() ``` -### `ModelRef` (supported by `evaluator.submit(...)`) +### `ModelRef` -Durable remote `evaluator.submit(...)` jobs additionally accept a `ModelRef` target. A `ModelRef` names a platform model entity (`workspace/model-name`) and is resolved by the evaluator backend when the job runs, so you do not have to resolve the endpoint yourself. Use this for platform-managed model routing. A `ModelRef` target generates outputs online, so it requires an online run config (`RunConfigOnlineModel`): +`evaluator.submit(...)` jobs also accept a `ModelRef` target. A `ModelRef` names a platform model entity (`workspace/model-name`) and is resolved by the evaluator backend when the job runs, so you do not have to resolve the endpoint yourself. Use this for platform-managed model routing. A `ModelRef` target generates outputs online, so it requires an online run config (`RunConfigOnlineModel`): ```python from nemo_evaluator_sdk import ModelRef, RunConfigOnlineModel diff --git a/docs/evaluator/metrics/rag.mdx b/docs/evaluator/metrics/rag.mdx index 6e30d53c46..c6b9829d3e 100644 --- a/docs/evaluator/metrics/rag.mdx +++ b/docs/evaluator/metrics/rag.mdx @@ -206,7 +206,7 @@ Measures the fraction of relevant content retrieved compared to the total releva - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -215,6 +215,8 @@ metric = ContextRecallMetric(judge_model=judge_model) result = LocalEvaluator().run_sync(metrics=[metric], dataset=offline_rows, config=RunConfig(parallelism=8)) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") ``` @@ -284,7 +286,7 @@ Measures the proportion of relevant chunks in the retrieved contexts (precision@ - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -293,6 +295,8 @@ metric = ContextPrecisionMetric(judge_model=judge_model) result = LocalEvaluator().run_sync(metrics=[metric], dataset=offline_rows, config=RunConfig(parallelism=8)) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") ``` @@ -356,7 +360,7 @@ Measures how relevant the retrieved contexts are to the user input. - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -373,6 +377,8 @@ result = LocalEvaluator().run_sync( ], config=RunConfig(parallelism=8), ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -423,7 +429,7 @@ Measures how many important entities from the reference are present in the retri - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -440,6 +446,8 @@ result = LocalEvaluator().run_sync( ], config=RunConfig(parallelism=8), ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -491,7 +499,7 @@ Measures factual consistency of the response with the retrieved context. - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -500,6 +508,8 @@ metric = FaithfulnessMetric(judge_model=judge_model) result = LocalEvaluator().run_sync(metrics=[metric], dataset=offline_rows, config=RunConfig(parallelism=8)) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") ``` @@ -566,7 +576,7 @@ Evaluates whether the response is grounded in the retrieved context without hall - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -575,6 +585,8 @@ metric = ResponseGroundednessMetric(judge_model=judge_model) result = LocalEvaluator().run_sync(metrics=[metric], dataset=offline_rows, config=RunConfig(parallelism=8)) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") ``` @@ -650,7 +662,7 @@ Noise Sensitivity uses RAGAS mode-qualified outputs internally. The SDK normaliz - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -672,6 +684,8 @@ result = LocalEvaluator().run_sync( ], config=RunConfig(parallelism=8), ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -734,7 +748,7 @@ Measures how relevant a response is to the user input using generated questions - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -747,6 +761,8 @@ metric = ResponseRelevancyMetric( result = LocalEvaluator().run_sync(metrics=[metric], dataset=offline_rows, config=RunConfig(parallelism=8)) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") ``` diff --git a/docs/evaluator/metrics/remote.mdx b/docs/evaluator/metrics/remote.mdx index d4d92f2f6d..dd56c58344 100644 --- a/docs/evaluator/metrics/remote.mdx +++ b/docs/evaluator/metrics/remote.mdx @@ -153,6 +153,8 @@ job = evaluator.submit(metric=metric, dataset=[{"input": "test"}]) job.wait_until_done() result = job.get_result() ``` +job.wait_until_done() +result = job.get_result() The API key is sent in the `Authorization: Bearer ` header. The job runtime resolves `api_key_secret` as a platform secret in the target workspace and receives it securely. diff --git a/docs/evaluator/metrics/results.mdx b/docs/evaluator/metrics/results.mdx index 45a912efc9..80669da6f0 100644 --- a/docs/evaluator/metrics/results.mdx +++ b/docs/evaluator/metrics/results.mdx @@ -31,6 +31,8 @@ result = LocalEvaluator().run_sync( {"expected": "Berlin", "output": "Munich"}, ], ) +job.wait_until_done() +result = job.get_result() ``` ## Get Results from a Submitted Job diff --git a/docs/evaluator/metrics/similarity.mdx b/docs/evaluator/metrics/similarity.mdx index e2b54f9016..19ebeed344 100644 --- a/docs/evaluator/metrics/similarity.mdx +++ b/docs/evaluator/metrics/similarity.mdx @@ -71,7 +71,7 @@ BLEU (Bilingual Evaluation Understudy) measures the similarity between machine-g - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -98,6 +98,8 @@ result = LocalEvaluator().run_sync( }, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -172,7 +174,7 @@ Exact Match compares the candidate text with the reference text for perfect equa - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -193,6 +195,8 @@ result = LocalEvaluator().run_sync( {"correct_answer": "Berlin", "model_answer": "Munich"}, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -254,7 +258,7 @@ F1 measures token-level overlap between candidate and reference text. It balance - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -276,6 +280,8 @@ result = LocalEvaluator().run_sync( {"reference": "a red apple", "answer": "red apple"}, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -346,7 +352,7 @@ Number Check performs numerical comparisons and operations on extracted values. - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -369,6 +375,8 @@ result = LocalEvaluator().run_sync( {"expected": "99", "predicted": "101"}, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -433,7 +441,7 @@ ROUGE (Recall-Oriented Understudy for Gisting Evaluation) measures overlap betwe - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -458,6 +466,8 @@ result = LocalEvaluator().run_sync( }, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") @@ -545,7 +555,7 @@ String Check performs various string operations and comparisons. Supports equali - + ```python from nemo_evaluator_sdk import Evaluator as LocalEvaluator @@ -566,6 +576,8 @@ result = LocalEvaluator().run_sync( {"output": "Error occurred", "must_contain": "Success"}, ], ) +job.wait_until_done() +result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") diff --git a/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx b/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx index f05944a81a..69b69a7c45 100644 --- a/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx +++ b/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx @@ -295,8 +295,7 @@ def create_helpfulness_metric(prompt_template: str, judge_model: Model) -> LLMJu PROMPT_V1 = """You are an evaluator. Rate the response's helpfulness from 0-4. Respond with JSON only: {"helpfulness": <0-4>}""" -metric_v1_local = create_helpfulness_metric(PROMPT_V1, LOCAL_JUDGE_MODEL) -metric_v1_remote = create_helpfulness_metric(PROMPT_V1, REMOTE_JUDGE_MODEL) +metric_v1 = create_helpfulness_metric(PROMPT_V1, JUDGE_MODEL) ``` @@ -307,7 +306,7 @@ Use low temperature for evaluation tasks. Low or zero temperature produces outpu --- -## 6. Test with Local Evaluation +## 6. Test with a Small Submitted Job Before running a durable job, test your metric with a few examples using `nemo_evaluator_sdk.Evaluator`. It runs locally in-process and returns results immediately, which is useful for prompt iteration. @@ -328,6 +327,8 @@ quick_test_result = LocalEvaluator().run_sync( ], config=RunConfig(parallelism=1), ) +quick_test_job.wait_until_done() +quick_test_result = quick_test_job.get_result() def score_value(row_score, score_name: str) -> float | None: """Return one named score from an evaluator row result.""" @@ -390,7 +391,7 @@ def wait_for_job(label: str, job): return result job_v1 = evaluator.submit( - metric=metric_v1_remote, + metric=metric_v1, dataset=dataset_ref, config=sample_config, ) @@ -491,15 +492,14 @@ A shorter response that directly solves the problem can score higher than a long Respond with JSON only: {"helpfulness": <0-4>}""" -metric_v2_local = create_helpfulness_metric(PROMPT_V2, LOCAL_JUDGE_MODEL) -metric_v2_remote = create_helpfulness_metric(PROMPT_V2, REMOTE_JUDGE_MODEL) +metric_v2 = create_helpfulness_metric(PROMPT_V2, JUDGE_MODEL) ``` Run evaluation with the improved prompt: ```python job_v2 = evaluator.submit( - metric=metric_v2_remote, + metric=metric_v2, dataset=dataset_ref, config=sample_config, ) @@ -706,30 +706,12 @@ If you're re-running the tutorial, delete the existing workspace first: client.workspaces.delete(name=WORKSPACE) ``` -### Local NVIDIA Build authentication fails +### NVIDIA Build authentication fails -Local evaluator runs resolve `api_key_secret` from environment variables. For NVIDIA Build, make sure `NVIDIA_API_KEY` is exported in the environment where the notebook or Python process is running: +Submitted evaluator jobs resolve `api_key_secret` from platform secrets. For NVIDIA Build, make sure the secret exists in the workspace and the model uses that secret name: ```python -import os - -assert os.environ["NVIDIA_API_KEY"] -``` - -The local model should use the environment variable name: - -```python -LOCAL_JUDGE_MODEL = Model( - url=JUDGE_MODEL_URL, - name=JUDGE_MODEL_NAME, - api_key_secret="NVIDIA_API_KEY", -) -``` - -Remote jobs should use the platform secret name instead: - -```python -REMOTE_JUDGE_MODEL = Model( +JUDGE_MODEL = Model( url=JUDGE_MODEL_URL, name=JUDGE_MODEL_NAME, api_key_secret=nvidia_api_key_secret.name, diff --git a/docs/safe-synthesizer/about/host-local-development.mdx b/docs/safe-synthesizer/about/host-local-development.mdx index f1f3657696..e4a2e2884f 100644 --- a/docs/safe-synthesizer/about/host-local-development.mdx +++ b/docs/safe-synthesizer/about/host-local-development.mdx @@ -2,169 +2,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -title: "Local and Subprocess Execution" +title: "Runtime Environment" description: "" --- -Run NeMo Safe Synthesizer on your machine's GPU with `nemo safe-synthesizer run-local`. The public command is a local subprocess wrapper: the main NeMo CLI starts a separate Safe Synthesizer runtime Python, and that runtime executes the synthesis task module. +NeMo Safe Synthesizer no longer exposes a host-local workload command. Create synthesis jobs through the platform Jobs API or SDK so scheduling, storage, logs, and artifacts all flow through the supported job path. -This page covers local execution only. Platform job submission uses the Jobs API or SDK; the `nemo safe-synthesizer` CLI exposes `run-local` and `runtime`. +The `nemo safe-synthesizer` CLI still includes runtime utilities for inspecting or preparing the task runtime used by development and job debugging. -## Prerequisites - -- CUDA-capable NVIDIA GPU on the host (80GB+ VRAM recommended; check with `nvidia-smi`). See [Getting Started](/documentation/synthesize-safe-data). -- NeMo Platform repository checkout with the Safe Synthesizer plugin installed. -- **No running platform required** for a typical local run when you pass `--data-source` — the NSS runtime can download base models from Hugging Face directly. - -```bash -# From the NeMo Platform repository root -BOOTSTRAP_LOCAL_PLUGIN_DIRS=plugins/nemo-safe-synthesizer make bootstrap-python -uv run nemo safe-synthesizer runtime setup -uv run nemo safe-synthesizer runtime info -``` - -Confirm the CLI surface: - -```bash -uv run nemo safe-synthesizer --help -# Commands: run-local, runtime -``` - -## Execution modes - -There are two local paths: - -| Mode | Command | Use it when | -|------|---------|-------------| -| Managed local subprocess | `uv run nemo safe-synthesizer run-local ...` | You want the supported plugin CLI. This creates the parent CLI process, then launches the runtime Python subprocess. | -| Direct local task | `\ -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local ...` | You are debugging the task process itself and want to bypass the parent CLI wrapper. | - -Both modes run on the host GPU and write artifacts to the local filesystem. Both accept the same task arguments: `--spec-file`, `--workspace`, `--output-dir`, and optional `--data-source`. - -## Run with the managed local subprocess - -Use a job spec JSON (example in `plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/nss-job.json`) and a local input file: - -```bash -uv run nemo safe-synthesizer run-local \ - --workspace default \ - --spec-file ./nss-job.json \ - --data-source ./input.csv \ - --output-dir ./nss-output -``` - -| Flag | Role | -|------|------| -| `--spec-file` | Job spec JSON (`data_source`, `config`, …) | -| `--data-source` | Local CSV (or other supported file) used **instead of** downloading from `data_source` in the spec | -| `--output-dir` | Where artifacts are written (default `./nss-output`) | -| `--workspace` | Workspace label for spec fields that reference workspaces (default `default`) | - -The parent command launches a subprocess equivalent to: - -```bash -\ -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local \ - --workspace default \ - --spec-file ./nss-job.json \ - --data-source ./input.csv \ - --output-dir ./nss-output -``` - -Find the configured runtime Python with: - -```bash -uv run nemo safe-synthesizer runtime info -``` - -## Run the local task directly - -Direct task execution is useful when you need to reproduce a subprocess failure without the parent CLI wrapper. - -```bash -$(uv run nemo safe-synthesizer runtime info | awk -F': ' '/^python:/ \{print $2\}') \ - -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local \ - --workspace default \ - --spec-file ./nss-job.json \ - --data-source ./input.csv \ - --output-dir ./nss-output -``` - -If the runtime Python does not exist, run `uv run nemo safe-synthesizer runtime setup` first. - -If you omit `--data-source`, the task downloads `data_source` from the platform Files service. Use `--data-source` for offline local files. - -### Output layout - -| Path | Description | -|------|-------------| -| `nss-output/synthetic-data.csv` | Generated records | -| `nss-output/summary.json` | Timing and run summary | -| `nss-output/evaluation-report.html` | Present when evaluation is enabled | -| `nss-output/adapter/` | LoRA adapter directory when synthesis training ran | - -## Reuse a prior adapter (generation only) - -Adapter reuse always skips training and runs **generate + evaluate** only — the same path as the OSS library's `load_from_save_path().generate()`. - -### Run-local - -Point `config.training.pretrained_model` at a prior run's adapter directory or work tree: - -**Run 1** — train and write an adapter: - -```bash -uv run nemo safe-synthesizer run-local \ - --spec-file ./job1-spec.json \ - --data-source ./input.csv \ - --output-dir ./nss-output-1 -``` - -**Run 2** — generate more records from that adapter: - -```json -{ - "data_source": "default/placeholder#input.csv", - "config": { - "enable_synthesis": true, - "enable_replace_pii": false, - "training": { - "pretrained_model": "./nss-output-1/adapter" - }, - "generation": { - "num_records": 100 - } - } -} -``` - -The plugin resolves `./nss-output-1/adapter` to the prior run under `./nss-output-1/work`. The `work/` tree must still exist from run 1. - -You can also point at `./nss-output-1/work` or a specific run directory under it. - -### Platform jobs (`pretrained_model_job`) - -For platform jobs, set `pretrained_model_job` to a completed job that has an **`adapter`** result stored in Files: - -```json -{ - "pretrained_model_job": "my-first-synth-job", - "config": { - "generation": { - "num_records": 100 - } - } -} -``` - -Do not set `config.training.pretrained_model` when using `pretrained_model_job`. - -Training runs embed `safe-synthesizer-config.json` in the adapter artifact uploaded to Files so subsequent generation-only jobs can reload the prior run configuration. - -Use an absolute path for local `pretrained_model` if you run from a different working directory. - -## Runtime commands +## Runtime Commands ```bash # One-time: create the NSS engine/CUDA venv @@ -177,40 +25,36 @@ uv run nemo safe-synthesizer runtime info uv run nemo safe-synthesizer runtime setup --force ``` -## Automated tests - -### Unit tests (no GPU) +## Run Workloads -From `plugins/nemo-safe-synthesizer`: - -```bash -uv run pytest plugins/nemo-safe-synthesizer/tests/unit/test_local_run.py -v -``` +Use the SDK builder or Jobs API to submit a platform job: -### Opt-in host-local E2E (GPU) +```python +import os +import pandas as pd -```bash -cd /path/to/nemo-platform -RUN_NSS_LOCAL_E2E=1 uv run pytest \ - plugins/nemo-safe-synthesizer/tests/e2e/test_local_synthesis.py \ - -v -m e2e -``` +from nemo_platform import NeMoPlatform +from nemo_safe_synthesizer_plugin.sdk.job_builder import SafeSynthesizerJobBuilder -Optional: `NSS_LOCAL_E2E_TIMEOUT_SECONDS` (default `3600`). +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) -Requires `RUN_NSS_LOCAL_E2E=1`, CUDA, and `nemo safe-synthesizer runtime setup`. +df = pd.DataFrame({"text": ["sample record"]}) -## Troubleshooting +job = ( + SafeSynthesizerJobBuilder(client, workspace="default") + .with_data_source(df) + .synthesize() + .create_job(name="safe-synth-job", project="default-project") +) +``` -| Symptom | Check | -|---------|--------| -| `run-local` not in `nemo safe-synthesizer --help` | `BOOTSTRAP_LOCAL_PLUGIN_DIRS=plugins/nemo-safe-synthesizer make bootstrap-python`; no duplicate top-level generated `safe-synthesizer` CLI | -| `runtime setup` / CUDA errors | `uv run nemo safe-synthesizer runtime info` and `nvidia-smi` | -| Model download failures | Hugging Face access from the NSS runtime venv; network and disk space | -| Reuse run fails to load adapter | Prior run's `work/` tree still exists (run-local), or adapter artifact in Files includes `metadata_v2.json` and embedded `safe-synthesizer-config.json` (platform) | -| `Use either 'pretrained_model_job' or 'config.training.pretrained_model'` | For run-local-only workflows, use only `config.training.pretrained_model` | +When the job completes, retrieve outputs through the returned job wrapper or the platform job results APIs. -## Related topics +## Related Topics -- [Getting Started](/documentation/synthesize-safe-data) — GPU and local runtime prerequisites -- Plugin README: `plugins/nemo-safe-synthesizer/README.md` \ No newline at end of file +- [Safe Synthesizer Jobs](/documentation/synthesize-safe-data/about/jobs) - job lifecycle, adapter reuse, and troubleshooting +- [SDK Resources](/documentation/synthesize-safe-data/sdk-resources) - Python SDK methods for jobs, logs, and results +- Plugin README: `plugins/nemo-safe-synthesizer/README.md` diff --git a/docs/safe-synthesizer/about/index.mdx b/docs/safe-synthesizer/about/index.mdx index 9d434a7803..d4807ccc50 100644 --- a/docs/safe-synthesizer/about/index.mdx +++ b/docs/safe-synthesizer/about/index.mdx @@ -100,9 +100,9 @@ Learn about quality and privacy metrics used to assess synthetic data including Understand the job lifecycle, configuration, and execution for Safe Synthesizer pipelines. - + -Run on a host GPU with `nemo safe-synthesizer run-local` and `runtime` commands; reuse local adapters and run plugin tests. +Set up and inspect the Safe Synthesizer runtime used for development and job debugging. diff --git a/docs/safe-synthesizer/about/jobs.mdx b/docs/safe-synthesizer/about/jobs.mdx index 815830d41c..5a82155e43 100644 --- a/docs/safe-synthesizer/about/jobs.mdx +++ b/docs/safe-synthesizer/about/jobs.mdx @@ -112,8 +112,6 @@ When the job completes, access: For **platform jobs**, set `pretrained_model_job` in the job spec to a completed job that has an **`adapter`** result in Files. Reuse is generation-only (no retraining). Use either `pretrained_model_job` or `config.training.pretrained_model`, not both. -For **host-local** development (`nemo safe-synthesizer run-local`), set `config.training.pretrained_model` to a local adapter or work directory from an earlier run. See [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution). - ## Job Builder API The `SafeSynthesizerJobBuilder` provides a high-level interface for common workflows: @@ -263,7 +261,7 @@ for log in job.fetch_logs(): ## Related Topics -- [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution): `run-local`, adapter reuse, and plugin tests +- [Runtime Environment](/documentation/synthesize-safe-data/about/local-and-subprocess-execution): runtime setup and inspection - [safe-synthesizer-101](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101): Get started with NeMo Safe Synthesizer jobs - [index](/documentation/synthesize-safe-data/tutorials): More hands-on tutorials - [reference](/documentation/synthesize-safe-data/about/parameters-reference): Full parameter reference diff --git a/docs/safe-synthesizer/about/reference.mdx b/docs/safe-synthesizer/about/reference.mdx index 88053d6723..23a39e30e5 100644 --- a/docs/safe-synthesizer/about/reference.mdx +++ b/docs/safe-synthesizer/about/reference.mdx @@ -19,11 +19,11 @@ Top-level fields on the Safe Synthesizer job spec (alongside `config`): | Field | Description | |-------|-------------| -| `data_source` | Input data as a platform fileset URL (`workspace/fileset#path`). With `run-local`, override via `--data-source` and use any placeholder in the spec. | +| `data_source` | Input data as a platform fileset URL (`workspace/fileset#path`). | | `pretrained_model_job` | Prior completed job whose **`adapter`** result in Files is reused for **generation-only** synthesis. Format: `` or `/`. Mutually exclusive with `config.training.pretrained_model`. | | `hf_token_secret` | Platform secret name for Hugging Face token during model initialization | -For host-local runs, see [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution). Reuse a local adapter with `config.training.pretrained_model`, not `pretrained_model_job`. +For platform adapter reuse, prefer `pretrained_model_job` so the job can resolve the prior adapter artifact from Files. ## Top-Level Configuration diff --git a/docs/safe-synthesizer/getting-started.mdx b/docs/safe-synthesizer/getting-started.mdx index dea2d4c27a..5e714d21c4 100644 --- a/docs/safe-synthesizer/getting-started.mdx +++ b/docs/safe-synthesizer/getting-started.mdx @@ -7,7 +7,7 @@ description: "" --- -Get started with NeMo Safe Synthesizer for generating private synthetic versions of sensitive tabular datasets on a host GPU. +Get started with NeMo Safe Synthesizer for generating private synthetic versions of sensitive tabular datasets through platform jobs. ## Prerequisites @@ -15,7 +15,7 @@ Before using NeMo Safe Synthesizer, complete [Setup](/documentation/get-started) NeMo Safe Synthesizer has the following additional requirements: -- An NVIDIA GPU **on the host machine** with 80GB+ VRAM (check with `nvidia-smi`). This is separate from any GPU inside a NIM container; Safe Synthesizer training runs directly on the host. +- A GPU-capable Jobs backend with enough VRAM for the selected model and dataset. - Sufficient disk space for generated datasets (50GB+ recommended) For general platform troubleshooting (port conflicts, health checks, and so on), refer to [Setup](/documentation/get-started). @@ -23,20 +23,32 @@ For general platform troubleshooting (port conflicts, health checks, and so on), --- -## Host-local CLI +## Submit Jobs -For GPU development on your machine, install the Safe Synthesizer plugin from this repository and use `nemo safe-synthesizer run-local` (see [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution)): +Use the Safe Synthesizer SDK builder or Jobs API to submit workloads through the platform. The builder can upload local datasets to Files before submission: -```shell -BOOTSTRAP_LOCAL_PLUGIN_DIRS=plugins/nemo-safe-synthesizer make bootstrap-python -uv run nemo safe-synthesizer runtime setup -uv run nemo safe-synthesizer run-local \ - --spec-file ./nss-job.json \ - --data-source ./input.csv \ - --output-dir ./nss-output +```python +import os +import pandas as pd + +from nemo_platform import NeMoPlatform +from nemo_safe_synthesizer_plugin.sdk.job_builder import SafeSynthesizerJobBuilder + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) +df = pd.DataFrame({"text": ["sample record"]}) + +job = ( + SafeSynthesizerJobBuilder(client, workspace="default") + .with_data_source(df) + .synthesize() + .create_job(name="safe-synth-job", project="default-project") +) ``` -The `run-local` command launches the Safe Synthesizer task in a separate runtime Python subprocess. The `nemo safe-synthesizer` CLI today exposes **run-local** and **runtime** only; platform job submission uses the Jobs API or SDK. +The `nemo safe-synthesizer` CLI exposes runtime setup and inspection utilities; workload execution goes through platform jobs. --- @@ -45,7 +57,7 @@ The `run-local` command launches the Safe Synthesizer task in a separate runtime Create your first synthetic dataset: - [Safe Synthesizer 101 Tutorial](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101) - a beginner-friendly introduction -- [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution) - local CLI and runtime task details +- [Runtime Environment](/documentation/synthesize-safe-data/about/local-and-subprocess-execution) - runtime setup and inspection commands - [SDK Resources](/documentation/synthesize-safe-data/sdk-resources) - Python SDK methods for jobs, builders, logs, and results ---- \ No newline at end of file +--- diff --git a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx index 2ee9e6646b..4bc0c10bc2 100644 --- a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx +++ b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx @@ -327,10 +327,10 @@ The evaluation report contains two high-level scores: Synthetic Quality Score (S ## Next Steps -Now that you've completed your first Safe Synthesizer job, try the local CLI path: +Now that you've completed your first Safe Synthesizer job, review the next platform workflows: -- [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution) - run Safe Synthesizer directly on a host GPU -- [Getting Started](/documentation/synthesize-safe-data) - review local runtime prerequisites +- [Runtime Environment](/documentation/synthesize-safe-data/about/local-and-subprocess-execution) - inspect runtime setup for development and job debugging +- [Getting Started](/documentation/synthesize-safe-data) - review platform job prerequisites ### Try These Next @@ -379,7 +379,7 @@ print(f"Total jobs: {len(all_jobs.data)}") - Use smaller model (adjust `training.pretrained_model`) - Check GPU availability -For local CLI failures, see [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution). +For runtime setup issues, see [Runtime Environment](/documentation/synthesize-safe-data/about/local-and-subprocess-execution). **Error: "Dataset must have at least 200 records to use holdout."** diff --git a/docs/studio/data-designer-build.mdx b/docs/studio/data-designer-build.mdx index 4e9be8f243..8ef9271dd9 100644 --- a/docs/studio/data-designer-build.mdx +++ b/docs/studio/data-designer-build.mdx @@ -206,7 +206,7 @@ Click **Preview N rows** in the toolbar to generate a small sample with the curr The details panel below the toolbar opens automatically and shows: - **Validation issues** — anything blocking the run (missing required fields, malformed JSON/numbers, duplicate or invalid names, no columns added, invalid row count). Preview and Create are both blocked until these are resolved. -- **Preview logs** — the sampled output, streamed as the preview runs. +- **Preview logs** — the sampled output, streamed as the preview submits. You can collapse the panel to focus on the builder; runs that produce output re-open it. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/app.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/app.py index 4880e747e0..18ce0a8b0e 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/app.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/app.py @@ -53,7 +53,7 @@ def _build_top_level_lazy_entries() -> tuple[TopLevelEntry, ...]: plugin_entry_points = _installed_plugin_command_entry_points() # Plugin `nemo.cli` entry points own their command name (e.g. safe-synthesizer). - # Drop generated API top-level groups with the same name so run-local/runtime stay available. + # Drop generated API top-level groups with the same name so plugin-owned commands stay available. api_entries = tuple(entry for entry in API_TOP_LEVEL_ENTRIES if entry.name not in plugin_entry_points) return build_top_level_entries( (*TOP_LEVEL_ENTRIES, *api_entries), diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/cli.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/cli.py index f91f070cb5..84c312afcc 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/cli.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/cli.py @@ -71,11 +71,11 @@ class NemoCLI(_NamedPlugin): Plugins that contribute :class:`~nemo_platform_plugin.function.NemoFunction` or :class:`~nemo_platform_plugin.job.NemoJob` primitives get an auto-generated CLI - surface (``run`` / ``submit`` / ``explain`` verbs with one Typer flag per - spec leaf). Override :meth:`update_function_cli` or :meth:`update_job_cli` + surface (``submit`` for functions, ``submit`` / ``explain`` for jobs, + with one Typer flag per spec leaf). Override :meth:`update_function_cli` or :meth:`update_job_cli` to amend that surface — add a flag, drop a flag, replace the verb entirely, or anything else Typer permits. Both hooks default to no-ops, so plugins - that don't override them get today's auto-generated surface unchanged. + that don't override them get the auto-generated submit surface unchanged. """ name: ClassVar[str] @@ -89,8 +89,8 @@ def update_function_cli(self, fn_cls: type[NemoFunction], group: typer.Typer) -> """Customize the auto-generated function sub-CLI for *fn_cls*. Called once per :class:`~nemo_platform_plugin.function.NemoFunction` the plugin - contributes, **after** the default ``run`` / ``submit`` verbs are - registered on *group* and **before** the group is mounted on the + contributes, **after** the default ``submit`` verb is registered on + *group* and **before** the group is mounted on the plugin's CLI app. ``group.registered_commands`` carries one :class:`typer.models.CommandInfo` per verb, each with ``.name`` and ``.callback`` (the auto-generated function with its synthetic @@ -107,7 +107,7 @@ def update_job_cli(self, job_cls: type[NemoJob], group: typer.Typer) -> None: """Customize the auto-generated job sub-CLI for *job_cls*. Same shape and override-by-replacement contract as - :meth:`update_function_cli`, but for jobs (with ``run`` / ``submit`` / + :meth:`update_function_cli`, but for jobs (with ``submit`` / ``explain`` verbs). The default is a no-op. """ @@ -115,7 +115,7 @@ def get_function_renderer( # noqa: ARG002 — non-abstract default with named p self, fn_cls: type[NemoFunction], *, - verb: Literal["run", "submit"], + verb: Literal["submit"], ) -> type[CLIRenderer] | None: """Return a :class:`~nemo_platform_plugin.cli_renderer.CLIRenderer` class for *fn_cls*'s *verb*. @@ -127,8 +127,8 @@ def get_function_renderer( # noqa: ARG002 — non-abstract default with named p pair. Note the **delegate-to-use-the-renderer contract**: the renderer - driver lives inside the framework's default ``run`` / ``submit`` - callback bodies, parameterized by this hook at invocation time. An + driver lives inside the framework's default ``submit`` callback body, + parameterized by this hook at invocation time. An :meth:`update_function_cli` wrapper that *delegates to the original callback* gets the renderer for free; a wrapper that takes over the verb body wholesale (does its own iteration without calling original) @@ -140,7 +140,7 @@ def get_job_renderer( # noqa: ARG002 — non-abstract default with named parame self, job_cls: type[NemoJob], *, - verb: Literal["run", "submit"], + verb: Literal["submit"], ) -> type[CLIRenderer] | None: """Return a :class:`~nemo_platform_plugin.cli_renderer.CLIRenderer` class for *job_cls*'s *verb*. diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/cli_renderer.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/cli_renderer.py index 61046e5e8f..101dcbc27e 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/cli_renderer.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/cli_renderer.py @@ -12,11 +12,10 @@ Lifecycle: - :meth:`CLIRenderer.on_start` — called once before iteration begins. -- :meth:`CLIRenderer.on_frame` — called once per frame as it arrives. For - local ``run`` the frame is a :class:`~pydantic.BaseModel` (the value yielded - by :meth:`~nemo_platform_plugin.function.NemoFunction.run`). For remote ``submit`` the - frame is a ``dict`` (parsed JSON for one NDJSON line). Plugin renderers that - want typed frames are responsible for parsing the dict themselves. +- :meth:`CLIRenderer.on_frame` — called once per frame as it arrives. Generated + framework commands now pass remote ``submit`` frames as ``dict`` values + parsed from NDJSON lines. Plugin renderers that want typed frames are + responsible for parsing the dict themselves. - :meth:`CLIRenderer.on_complete` — called once after the stream closes cleanly. - :meth:`CLIRenderer.on_error` — called once when iteration raises (and the @@ -48,12 +47,10 @@ class RendererContext: so renderers don't fight for the terminal. cli_kwargs: The original CLI kwargs the verb was invoked with. Renderers can branch on flags here (e.g. ``--non-interactive``). - verb: ``"run"`` (in-process) or ``"submit"`` (HTTP). Lets one renderer - class drive both verbs while branching when the verbs need - different output (e.g. show a request id only for submit). - is_local: ``True`` for in-process invocation (``run`` from a CLI with - local SDKs), ``False`` for HTTP ``submit``. Distinct from ``verb`` - because future plugin types could blur the in-process/remote line. + verb: Generated framework commands now pass ``"submit"``. The wider + literal remains for third-party renderer compatibility. + is_local: Generated framework commands now pass ``False``. The field + remains for third-party renderer compatibility. """ console: Console @@ -76,9 +73,8 @@ def on_start(self, *, ctx: RendererContext) -> None: def on_frame(self, frame: Any, *, ctx: RendererContext) -> None: """Called once per frame as it arrives. Default: no-op. - ``frame`` is a :class:`~pydantic.BaseModel` for local ``run`` and a - ``dict`` for remote ``submit``. Renderers that want typed frames in - both cases parse the dict themselves. + Generated framework ``submit`` frames are ``dict`` values parsed from + remote NDJSON lines. """ def on_complete(self, *, ctx: RendererContext) -> None: diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/cli_state.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/cli_state.py index b457d3f1f3..7a4b112734 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/cli_state.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/cli_state.py @@ -5,12 +5,9 @@ The top-level ``nemo`` CLI populates ``typer.Context.obj`` with a state object that exposes per-invocation handles to the platform SDK clients (sync and -async). The auto-generated ``run`` / ``submit`` verbs in -:mod:`nemo_platform_plugin.commands` consume that state through these helpers, -and **plugin-authored** Typer commands (i.e. anything a plugin registers via +async). **Plugin-authored** Typer commands (i.e. anything a plugin registers via :meth:`~nemo_platform_plugin.cli.NemoCLI.get_cli` rather than the -auto-generated verbs) should use the same surface so they participate in the -same protocol. +auto-generated verbs) use this surface when they need the active SDK handles. Example:: diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/commands.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/commands.py index 295dd7e049..78d97c55c6 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/commands.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/commands.py @@ -3,91 +3,29 @@ """CLI integration for NemoJob and NemoFunction primitives. -Two parallel helpers, one per primitive: +The generated plugin CLI exposes remote submission surfaces only: -- :func:`add_job_commands` — three-verb subgroups - (``run``/``submit``/``explain``) per :class:`~nemo_platform_plugin.job.NemoJob`, - routed through :class:`~nemo_platform_plugin.scheduler.NemoJobScheduler`. -- :func:`add_function_commands` — two-verb subgroups - (``run``/``submit``) per :class:`~nemo_platform_plugin.function.NemoFunction`. - Functions don't have an ``explain`` verb because their only schema - is :attr:`~nemo_platform_plugin.function.NemoFunction.spec_schema` and that's - introspected through ``--help`` directly. +- :func:`add_job_commands` creates ``submit`` / ``explain`` subgroups per + :class:`~nemo_platform_plugin.job.NemoJob`. +- :func:`add_function_commands` creates ``submit`` subgroups per + :class:`~nemo_platform_plugin.function.NemoFunction`. -:func:`add_job_commands` is the bridge between the ``nemo.jobs`` and -``nemo.cli`` surfaces. The platform calls it at startup for each plugin that -has registered both a CLI group and jobs, injecting a generated **sub-group** -for every job into the plugin's :class:`typer.Typer` group. Each sub-group -exposes three verbs — ``run``, ``submit``, ``explain`` — matching -:class:`~nemo_platform_plugin.scheduler.NemoJobScheduler`. +Plugin authors do **not** call these helpers themselves. The platform's CLI +loader calls them for installed plugin entry points, then gives each plugin's +``NemoCLI`` hooks a chance to amend the generated submit surfaces. -Plugin authors do **not** call this themselves — it is called automatically -by the platform's CLI loader. The result is that each job becomes available -as:: - - nemo run [--config ...] [--config-file ...] - nemo submit [--profile ...] [--cluster ...] [-o ...] - nemo explain [--profile ...] [--cluster ...] - -The **bare form** ``nemo `` prints usage and exits with -status 1. No implicit default verb — the submitter's choice of execution -target is always explicit. This breaks the previous one-line form; the -fix is typing ``run`` (or ``submit``) explicitly. - -Phase 1 MR 1.2c delivers the CLI shape. ``submit`` and ``explain`` delegate -to :class:`NemoJobScheduler` stubs that raise -:class:`NotImplementedError`; MR 1.3 and MR 1.4 wire them. ``run`` works -end-to-end today. - -Generated command interface ---------------------------- - -``run`` - Execute the job in-process. Accepts ``--config `` (default ``{}``) - and ``--config-file `` (takes precedence over ``--config``). The - scheduler validates against :attr:`~nemo_platform_plugin.job.NemoJob.spec_schema` - / :attr:`~nemo_platform_plugin.job.NemoJob.input_spec_schema` when declared. - -``submit`` - Submit the job to a cluster. Phase 1 MR 1.3 wires this. - -``explain`` - Print the job's spec / options schemas. Phase 1 MR 1.4 wires this. - -Example -------- - -Given:: - - class SayHelloJob(NemoJob): - name = "say-hello" - description = "Greet a name." - - def run(self, config: dict) -> dict: - return {"result": f"Hello, {config.get('name', 'world')}!"} - -The platform generates:: - - $ nemo example say-hello run --config '{"name": "Claude"}' - { - "result": "Hello, Claude!" - } - - $ nemo example say-hello - Usage: nemo example say-hello [OPTIONS] COMMAND [ARGS]... - ... - $ echo $? - 1 +``NemoJob.run`` and ``NemoFunction.run`` remain implementation hooks invoked +by platform workers and service routes. They are not exposed as local CLI +execution commands. """ from __future__ import annotations -import asyncio import inspect import json import logging import os -from collections.abc import AsyncIterator, Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Mapping from pathlib import Path from typing import Any, Literal, Optional, cast @@ -106,9 +44,7 @@ def run(self, config: dict) -> dict: from nemo_platform_plugin.cli import NemoCLI from nemo_platform_plugin.cli_errors import print_http_request_error, print_http_status_error from nemo_platform_plugin.cli_renderer import CLIRenderer, RendererContext -from nemo_platform_plugin.cli_state import resolve_local_cli_sdks -from nemo_platform_plugin.function import NemoFunction, returns_async_iterator -from nemo_platform_plugin.function_context import FunctionContext +from nemo_platform_plugin.function import NemoFunction from nemo_platform_plugin.functions.routes import DEFAULT_FUNCTION_PATH, NDJSON_MEDIA_TYPE from nemo_platform_plugin.job import NemoJob from nemo_platform_plugin.jobs._cli_options import ( @@ -117,7 +53,6 @@ def run(self, config: dict) -> dict: merge_options, parse_dotted_kv_list, ) -from nemo_platform_plugin.run_dependencies import LocalRunError from nemo_platform_plugin.scheduler import NemoJobScheduler from pydantic import BaseModel, ValidationError @@ -137,25 +72,23 @@ def run(self, config: dict) -> dict: _PANEL_JOB_SPEC: str = "Job Spec" _PANEL_SUBMISSION: str = "Submission" -# Static flag names declared by each function verb. Spec fields whose -# names collide with these are dropped from the auto-generated flag -# set; they remain reachable via ``--spec`` / ``--spec-file``. ``run`` -# is deliberately permissive — only the JSON-passing flags and the -# context-input ``workspace`` flag are reserved. ``submit`` layers on -# the URL-routing flags + the request-id surface. -_FN_RUN_RESERVED_FLAGS: frozenset[str] = frozenset({"spec", "spec_file", "workspace"}) -_FN_SUBMIT_RESERVED_FLAGS: frozenset[str] = _FN_RUN_RESERVED_FLAGS | frozenset({"cluster", "base_url", "request_id"}) - -# Static flag names declared by each job verb. ``run`` reserves the -# spec-source flags plus the deprecated ``--config`` / ``--config-file`` -# aliases (kept during the rename transition). ``submit`` layers on the -# submission-routing flags (``--profile`` / ``--cluster`` / ``--base-url`` -# / ``--workspace``) and the options-passthrough flags (``-o`` / -# ``--options-file``). Reserved spec fields remain reachable via -# ``--spec`` / ``--spec-file`` JSON. -_JOB_RUN_RESERVED_FLAGS: frozenset[str] = frozenset({"spec", "spec_file", "config", "config_file"}) -_JOB_SUBMIT_RESERVED_FLAGS: frozenset[str] = _JOB_RUN_RESERVED_FLAGS | frozenset( - {"options", "options_file", "profile", "cluster", "base_url", "workspace"} +_FN_SUBMIT_RESERVED_FLAGS: frozenset[str] = frozenset( + {"spec", "spec_file", "workspace", "cluster", "base_url", "request_id"} +) + +_JOB_SUBMIT_RESERVED_FLAGS: frozenset[str] = frozenset( + { + "spec", + "spec_file", + "config", + "config_file", + "options", + "options_file", + "profile", + "cluster", + "base_url", + "workspace", + } ) @@ -186,7 +119,7 @@ def _output_format_is_json(typer_ctx: typer.Context | None) -> bool: def _make_renderer_context( *, cli_kwargs: Mapping[str, Any], - verb: Literal["run", "submit"], + verb: Literal["submit"], is_local: bool, ) -> RendererContext: """Build the per-invocation :class:`RendererContext` passed to renderer methods.""" @@ -195,37 +128,6 @@ def _make_renderer_context( return RendererContext(console=Console(), cli_kwargs=cli_kwargs, verb=verb, is_local=is_local) -async def _drive_async_renderer( - stream: AsyncIterator[Any], - renderer_cls: type[CLIRenderer], - *, - rctx: RendererContext, -) -> CLIRenderer: - """Drive *renderer_cls*'s ``on_start`` + ``on_frame`` over an async stream. - - Returns the renderer instance so the sync caller (the one calling - :func:`asyncio.run`) can fire :meth:`CLIRenderer.on_complete` *after* - the loop tears down — important because renderers commonly use - event-loop-bound libraries (e.g. ``prompt_toolkit.Application.run``) - in ``on_complete``, and those use their own :func:`asyncio.run` which - raises ``RuntimeError: asyncio.run() cannot be called from a running - event loop`` when nested. - - :meth:`CLIRenderer.on_error` is fired in-loop because the exception - is most usefully handled where it surfaced; the exception still - propagates so the caller sees it. - """ - renderer = renderer_cls() - renderer.on_start(ctx=rctx) - try: - async for frame in stream: - renderer.on_frame(frame, ctx=rctx) - except BaseException as exc: - renderer.on_error(exc, ctx=rctx) - raise - return renderer - - def _drive_sync_renderer( line_iter: Iterable[str], renderer_cls: type[CLIRenderer], @@ -234,14 +136,8 @@ def _drive_sync_renderer( ) -> CLIRenderer: """Drive *renderer_cls*'s ``on_start`` + ``on_frame`` over a sync line iterator. - Returns the renderer; the caller fires :meth:`CLIRenderer.on_complete`. - Symmetric with :func:`_drive_async_renderer` so all three drivers obey - the same contract: in-loop work happens here (``on_start`` / per-frame - / ``on_error``), the post-stream summary happens at the call site after - this returns. That contract matters even for sync drivers: it keeps the - plugin-author mental model consistent and avoids re-introducing - nested-event-loop hazards if a sync driver is ever wrapped in an async - context. + Returns the renderer; the caller fires :meth:`CLIRenderer.on_complete` + after this returns. Each non-empty NDJSON line is parsed as JSON and passed to :meth:`CLIRenderer.on_frame`. Lines that fail to parse pass through as @@ -294,12 +190,11 @@ def add_job_commands( *, cli: NemoCLI | None = None, ) -> None: - """Inject three-verb subcommand groups for each job into *cli_app*. + """Inject submit/explain subcommand groups for each job into *cli_app*. Each entry in *jobs* produces one :class:`typer.Typer` sub-group registered under :attr:`~nemo_platform_plugin.job.NemoJob.name` in a ``"Jobs"`` - rich help panel. The sub-group owns ``run`` / ``submit`` / ``explain`` - commands. + rich help panel. The sub-group owns ``submit`` / ``explain`` commands. Args: cli_app: The plugin's :class:`typer.Typer` group to inject @@ -327,7 +222,7 @@ def _register_job_subgroup( *, cli: NemoCLI | None = None, ) -> None: - """Register a ```` sub-group with run / submit / explain verbs.""" + """Register a ```` sub-group with submit / explain verbs.""" job_group = typer.Typer( name=job_cls.name, help=job_cls.description or f"Manage the {job_cls.name} job.", @@ -341,7 +236,6 @@ def _root(ctx: typer.Context) -> None: # pragma: no cover - trivial delegation typer.echo(ctx.get_help()) raise typer.Exit(code=1) - _add_run_command(job_group, job_cls, scheduler, cli=cli) _add_submit_command(job_group, job_cls, scheduler, cli=cli) _add_explain_command(job_group, job_cls, scheduler) @@ -352,175 +246,23 @@ def _root(ctx: typer.Context) -> None: # pragma: no cover - trivial delegation # --------------------------------------------------------------------------- -# run — local execution, wired via NemoJobScheduler.run_local +# Shared job helpers # --------------------------------------------------------------------------- def _job_input_schema(job_cls: type[NemoJob]) -> type[BaseModel] | None: """Pick the schema that the submitter's input is validated against. - Mirrors the precedence used by - :meth:`~nemo_platform_plugin.scheduler.NemoJobScheduler._validate_and_compile`: - :attr:`~nemo_platform_plugin.job.NemoJob.input_spec_schema` when declared, - else :attr:`~nemo_platform_plugin.job.NemoJob.spec_schema`. Both ``run`` and - ``submit`` accept the same shape from the user, so we walk a single - schema for both verbs. Returns ``None`` when neither attribute is - declared — the auto-flag generator treats ``None`` as "no leaves", - keeping legacy schema-less jobs unchanged. + ``submit`` accepts ``input_spec_schema`` when declared, otherwise + ``spec_schema``. Returns ``None`` when neither attribute is declared; + the auto-flag generator treats ``None`` as "no leaves", keeping + schema-less jobs unchanged. """ return job_cls.input_spec_schema or job_cls.spec_schema -def _add_run_command( - group: typer.Typer, - job_cls: type[NemoJob], - scheduler: NemoJobScheduler, - *, - cli: NemoCLI | None = None, -) -> None: - """Register the ``run`` verb. Generates per-field flags from the input schema. - - Each scalar leaf in :func:`_job_input_schema` becomes a Typer option - named after its dotted path, kebab-cased per segment (``--name``, - ``--target.url``). Precedence at runtime is ``--spec-file`` (base) - → ``--spec`` JSON (overlay) → per-field flags (top overlay), with - the deprecated ``--config`` / ``--config-file`` aliases routed - through the same precedence as ``--spec`` / ``--spec-file``. - - When *cli* supplies a renderer via ``get_job_renderer(verb="run")`` (and - ``--output-format json`` is not set), the renderer's lifecycle wraps the - synchronous ``run_local`` call: ``on_start`` → ``on_frame(result)`` → - ``on_complete``. The default-printer fallback echoes the dict result - when no renderer is supplied. - """ - schema = _job_input_schema(job_cls) - leaves = walk_spec_leaves(schema, reserved=_JOB_RUN_RESERVED_FLAGS) - - def _run(typer_ctx: typer.Context, **kwargs: object) -> None: - original_kwargs = dict(kwargs) - spec_str: str = cast(str, kwargs.pop("spec", "{}")) - spec_file: Path | None = cast("Path | None", kwargs.pop("spec_file", None)) - config: str | None = cast("str | None", kwargs.pop("config", None)) - config_file: Path | None = cast("Path | None", kwargs.pop("config_file", None)) - - effective_spec_str = config if config is not None else spec_str - effective_spec_file = config_file if config_file is not None else spec_file - base = _load_spec(effective_spec_str, effective_spec_file) - overlay = build_overlay(leaves, kwargs, unset_sentinel=UNSET) - data = deep_merge(base, overlay) - logger.debug("Running job %r locally with spec %r", job_cls.name, data) - sdk, async_sdk = resolve_local_cli_sdks(typer_ctx) - - renderer_cls: type[CLIRenderer] | None = None - if cli is not None and not _output_format_is_json(typer_ctx): - renderer_cls = cli.get_job_renderer(job_cls, verb="run") - - def _do_run() -> Any: - return scheduler.run_local(job_cls, data, sdk=sdk, async_sdk=async_sdk) - - renderer: CLIRenderer | None = None - rctx: RendererContext | None = None - try: - if renderer_cls is not None: - rctx = _make_renderer_context( - cli_kwargs=original_kwargs, - verb="run", - is_local=True, - ) - renderer = _drive_single_value_renderer(_do_run, renderer_cls, rctx=rctx) - else: - result = _do_run() - typer.echo(json.dumps(result, indent=2)) - except LocalRunError as exc: - typer.echo(f"Error: {exc}", err=True) - raise typer.Exit(code=1) from exc - - # on_complete fires after the driver returns, in the same contract - # the async renderer driver follows — see _drive_sync_renderer. - if renderer is not None and rctx is not None: - renderer.on_complete(ctx=rctx) - - help_text = "Run locally, in-process." - _run.__signature__ = _build_job_run_signature(leaves) # type: ignore[attr-defined] - group.command(name="run", help=help_text)(_run) - - -def _build_job_run_signature(leaves: list[SpecLeafField]) -> inspect.Signature: - """Compose the synthetic signature for the job ``run`` verb. - - Static flags (``--spec`` / ``--spec-file``) come first under the - ``Spec Source`` panel, followed by the auto-generated per-field - flags under ``Job Spec``, then the hidden ``--config`` / - ``--config-file`` deprecated aliases. - """ - static_params = [ - # ``typer.Context`` is auto-injected by Click via ``pass_context``, - # which passes it as the first positional argument. Hand-built - # rather than via ``kw()`` because ``kw()`` only emits - # ``KEYWORD_ONLY`` params (which Click's positional injection - # would reject). - inspect.Parameter( - "typer_ctx", - kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, - annotation=typer.Context, - ), - kw( - "spec", - str, - typer.Option( - "{}", - "--spec", - help="Spec as a JSON string.", - rich_help_panel=_PANEL_SPEC_SOURCE, - ), - ), - kw( - "spec_file", - Optional[Path], - typer.Option( - None, - "--spec-file", - help="Path to a YAML or JSON spec file (used as base; per-flag values override).", - rich_help_panel=_PANEL_SPEC_SOURCE, - ), - ), - ] - # Deprecated aliases — kept for the transition period (MR 1.3b). - # Trailing so they sort below the auto-generated panel in --help, - # though they're hidden anyway and only matter for backwards-compatible - # invocations. - trailing_params = [ - kw( - "config", - Optional[str], - typer.Option( - None, - "--config", - help="(Deprecated — use --spec.) Spec as a JSON string.", - hidden=True, - ), - ), - kw( - "config_file", - Optional[Path], - typer.Option( - None, - "--config-file", - help="(Deprecated — use --spec-file.) Path to a JSON spec file.", - hidden=True, - ), - ), - ] - return build_callback_signature( - static_params, - leaves, - rich_help_panel=_PANEL_JOB_SPEC, - trailing_params=trailing_params, - ) - - # --------------------------------------------------------------------------- -# submit — stub delegate to scheduler.submit_remote (wired in MR 1.3) +# submit # --------------------------------------------------------------------------- @@ -821,7 +563,7 @@ def _merge_options_inputs(options: list[str], options_file: Path | None) -> dict # --------------------------------------------------------------------------- -# NemoFunction CLI — two verbs (run / submit), no `explain` +# NemoFunction CLI — submit only, no `explain` # --------------------------------------------------------------------------- @@ -831,14 +573,13 @@ def add_function_commands( *, cli: NemoCLI | None = None, ) -> None: - """Inject two-verb subcommand groups for each function into *cli_app*. + """Inject submit subcommand groups for each function into *cli_app*. Each entry produces one :class:`typer.Typer` sub-group registered under :attr:`~nemo_platform_plugin.function.NemoFunction.name` in a - ``"Functions"`` rich help panel. The sub-group owns ``run`` and - ``submit`` commands. There is no ``explain`` verb — functions - have a single ``spec_schema`` and ``--help`` is the introspection - surface. + ``"Functions"`` rich help panel. The sub-group owns a ``submit`` + command. There is no ``explain`` verb — functions have a single + ``spec_schema`` and ``--help`` is the introspection surface. Args: cli_app: The plugin's :class:`typer.Typer` group to inject @@ -865,7 +606,7 @@ def _register_function_subgroup( *, cli: NemoCLI | None = None, ) -> None: - """Register a ```` sub-group with run / submit verbs.""" + """Register a ```` sub-group with a submit verb.""" fn_group = typer.Typer( name=fn_cls.name, help=fn_cls.description or f"Manage the {fn_cls.name} function.", @@ -879,7 +620,6 @@ def _root(ctx: typer.Context) -> None: # pragma: no cover - trivial delegation typer.echo(ctx.get_help()) raise typer.Exit(code=1) - _add_function_run_command(fn_group, fn_cls, cli=cli) _add_function_submit_command(fn_group, fn_cls, cli=cli) if cli is not None: @@ -888,196 +628,6 @@ def _root(ctx: typer.Context) -> None: # pragma: no cover - trivial delegation cli_app.add_typer(fn_group, name=fn_cls.name, rich_help_panel="Functions") -# ---- run --------------------------------------------------------- # - - -def _add_function_run_command( - group: typer.Typer, - fn_cls: type[NemoFunction], - *, - cli: NemoCLI | None = None, -) -> None: - """Register the ``run`` verb. Generates per-field flags from ``spec_schema``. - - Each scalar leaf in ``spec_schema`` becomes a Typer option named - after its dotted path, kebab-cased per segment (``--name``, - ``--target.url``). Precedence at runtime is ``--spec-file`` (base) - → ``--spec`` JSON (overlay) → per-field flags (top overlay). - Validation happens after the merge — surfaces a single - ``ValidationError`` for the merged spec rather than per-overlay. - - When *cli* supplies a renderer via ``get_function_renderer(verb="run")`` - (and ``--output-format json`` is not set), the renderer's lifecycle - drives the streamed output instead of the default per-frame echo. - """ - leaves = walk_spec_leaves(fn_cls.spec_schema, reserved=_FN_RUN_RESERVED_FLAGS) - - def _run(typer_ctx: typer.Context, **kwargs: object) -> None: - original_kwargs = dict(kwargs) - spec_str: str = cast(str, kwargs.pop("spec", "{}")) - spec_file: Path | None = cast("Path | None", kwargs.pop("spec_file", None)) - workspace: str = cast(str, kwargs.pop("workspace", "default")) - - base = _load_spec(spec_str, spec_file) - overlay = build_overlay(leaves, kwargs, unset_sentinel=UNSET) - merged = deep_merge(base, overlay) - try: - spec_obj = fn_cls.spec_schema.model_validate(merged, context={"is_local": True}) - except ValidationError as exc: - typer.echo(f"Error: invalid spec for {fn_cls.name}: {exc}", err=True) - raise typer.Exit(code=1) from exc - - sdk, async_sdk = resolve_local_cli_sdks(typer_ctx) - ctx = FunctionContext(workspace=workspace) - renderer_cls: type[CLIRenderer] | None = None - if cli is not None and not _output_format_is_json(typer_ctx): - renderer_cls = cli.get_function_renderer(fn_cls, verb="run") - try: - outcome = asyncio.run( - _invoke_function_locally( - fn_cls, - spec_obj, - ctx, - sdk=sdk, - async_sdk=async_sdk, - renderer_cls=renderer_cls, - cli_kwargs=original_kwargs, - ) - ) - except LocalRunError as exc: - typer.echo(f"Error: {exc}", err=True) - raise typer.Exit(code=1) from exc - except KeyboardInterrupt as exc: # pragma: no cover - terminal-only - typer.echo("Interrupted.", err=True) - raise typer.Exit(code=130) from exc - - # Fire on_complete *outside* asyncio.run so renderers using event-loop-bound - # libraries (e.g. prompt_toolkit's Application.run for an interactive - # record browser) don't collide with the loop we just tore down. - if outcome is not None: - renderer, rctx = outcome - renderer.on_complete(ctx=rctx) - - help_text = f"Run {fn_cls.name} locally, in-process." - epilog = build_epilog(schema=fn_cls.spec_schema, leaves=leaves, kind="Function") - _run.__signature__ = _build_function_run_signature(leaves) # type: ignore[attr-defined] - group.command(name="run", help=help_text, epilog=epilog)(_run) - - -def _build_function_run_signature(leaves: list[SpecLeafField]) -> inspect.Signature: - """Compose the synthetic signature for the function ``run`` verb.""" - static_params = [ - # ``typer.Context`` is auto-injected by Click via ``pass_context``, - # which passes it as the first positional argument. Hand-built - # rather than via ``kw()`` because ``kw()`` only emits - # ``KEYWORD_ONLY`` params (which Click's positional injection - # would reject). - inspect.Parameter( - "typer_ctx", - kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, - annotation=typer.Context, - ), - kw( - "spec", - str, - typer.Option( - "{}", - "--spec", - help="Spec as a JSON string.", - rich_help_panel=_PANEL_SPEC_SOURCE, - ), - ), - kw( - "spec_file", - Optional[Path], - typer.Option( - None, - "--spec-file", - help="Path to a YAML or JSON spec file (used as base; per-flag values override).", - rich_help_panel=_PANEL_SPEC_SOURCE, - ), - ), - kw( - "workspace", - str, - typer.Option( - "default", - "--workspace", - help="Workspace identity passed to the function as ctx.workspace.", - rich_help_panel=_PANEL_SPEC_SOURCE, - ), - ), - ] - return build_callback_signature( - static_params, - leaves, - rich_help_panel=_PANEL_FUNCTION_SPEC, - ) - - -async def _invoke_function_locally( - fn_cls: type[NemoFunction], - spec_obj: BaseModel, - ctx: FunctionContext, - *, - sdk: object | None, - async_sdk: object | None, - renderer_cls: type[CLIRenderer] | None = None, - cli_kwargs: Mapping[str, Any] | None = None, -) -> tuple[CLIRenderer, RendererContext] | None: - """Call ``fn_cls().run(spec, ...)`` and print result(s) to stdout. - - Mirrors :func:`~nemo_platform_plugin.run_dependencies.resolve_run_kwargs` for the SDK - parameters: inject when the caller supplied an SDK, leave the parameter - unbound (Python default applies) when the function's signature has a - default to fall back on, and raise :class:`LocalRunError` only when - the function declared a required SDK parameter that we can't satisfy. - - When *renderer_cls* is supplied, drive the renderer's ``on_start`` + - ``on_frame`` around the streamed iterator and return ``(renderer, rctx)`` - so the caller can fire ``on_complete`` *after* :func:`asyncio.run` - returns — see :func:`_drive_async_renderer`. Renderers don't apply to - non-streaming returns — those still echo via - :func:`_format_value_for_stdout` and this function returns ``None``. - """ - instance = fn_cls() - run_params = fn_cls.run_signature().parameters - kwargs: dict[str, Any] = {} - if "ctx" in run_params: - kwargs["ctx"] = ctx - if "is_local" in run_params: - kwargs["is_local"] = True - for param_name, value in (("sdk", sdk), ("async_sdk", async_sdk)): - param = run_params.get(param_name) - if param is None: - continue - if value is not None: - kwargs[param_name] = value - continue - if param.default is inspect.Parameter.empty: - raise LocalRunError( - f"{fn_cls.__name__}.run requires a `{param_name}` argument; " - f"configure your `nemo` CLI context (e.g. `nemo config use-context ...`) " - f"or pass `{param_name}` to the local invoker." - ) - - result = instance.run(spec_obj, **kwargs) - if returns_async_iterator(result): - if renderer_cls is not None: - rctx = _make_renderer_context( - cli_kwargs=cli_kwargs or {}, - verb="run", - is_local=True, - ) - renderer = await _drive_async_renderer(result, renderer_cls, rctx=rctx) - return renderer, rctx - async for frame in result: - typer.echo(_format_frame_for_stdout(frame)) - return - awaited = await result - typer.echo(_format_value_for_stdout(awaited)) - - def _resolve_submit_auth_headers(typer_ctx: typer.Context) -> dict[str, str]: """Bearer (and other) default headers from the active CLI context.""" state = typer_ctx.obj @@ -1399,30 +949,6 @@ def _api_segment_for_function(fn_cls: type[NemoFunction]) -> str: return module.replace("_", "-") -def _format_frame_for_stdout(frame: Any) -> str: - """Render a streaming frame as a single line of pretty JSON. - - Pydantic models go through ``model_dump_json``; dicts and lists - through ``json.dumps``; anything else through ``str()``. The - output is one line per frame so consumers can pipe to ``jq -c`` - or grep for ``kind`` discriminators without a parser. - """ - if isinstance(frame, BaseModel): - return frame.model_dump_json() - if isinstance(frame, (dict, list)): - return json.dumps(frame, default=str) - return str(frame) - - -def _format_value_for_stdout(value: Any) -> str: - """Render a non-streaming return value as multi-line pretty JSON.""" - if isinstance(value, BaseModel): - return value.model_dump_json(indent=2) - if isinstance(value, (dict, list)): - return json.dumps(value, indent=2, default=str) - return str(value) - - def _pretty_print_jsonl_line(line: str) -> str: """Best-effort pretty-print for an NDJSON line. diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/discovery.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/discovery.py index 5cb8acf4a5..8f54587256 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/discovery.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/discovery.py @@ -307,10 +307,9 @@ def discover_jobs() -> dict[str, type[NemoJob]]: """Typed wrapper: discover ``nemo.jobs`` → :class:`~nemo_platform_plugin.job.NemoJob` subclass. Entry-point key convention: ``.`` (e.g. - ``"example.say-hello"``). The platform instantiates each class and calls - :meth:`~nemo_platform_plugin.job.NemoJob.run` with the job config dict — programmatic - callers drive that through - :meth:`nemo_platform_plugin.scheduler.NemoJobScheduler.run_local`. + ``"example.say-hello"``). Platform workers instantiate each class and call + :meth:`~nemo_platform_plugin.job.NemoJob.run` through + :func:`nemo_platform_plugin.tasks.dispatcher.run_task`. Validates that each class's ``name`` attribute matches the job-name suffix of its entry-point key (the part after the first ``"."``). diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/ARCHITECTURE.md b/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/ARCHITECTURE.md index 1221244868..28fc0ec7ab 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/ARCHITECTURE.md +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/ARCHITECTURE.md @@ -97,14 +97,13 @@ from nemo_platform_plugin.discovery import discover_jobs from nemo_platform_plugin.scheduler import NemoJobScheduler job_cls = discover_jobs()["example.say-hello"] -NemoJobScheduler().run_local(job_cls, {"name": "Alice"}) +NemoJobScheduler().submit_remote(job_cls, {"name": "Alice"}) ``` -## Auto-generated three-verb CLI for jobs +## Auto-generated CLI for jobs -At startup, for every plugin that registers both `nemo.cli` and `nemo.jobs`, the platform injects three CLI subcommands per job into the plugin's Typer group: `run`, `submit`, `explain`. Plugin authors write no CLI code for their jobs. +At startup, for every plugin that registers both `nemo.cli` and `nemo.jobs`, the platform injects two CLI subcommands per job into the plugin's Typer group: `submit`, `explain`. Plugin authors write no CLI code for their jobs. -- `run` delegates to `NemoJobScheduler.run_local` — in-process, no platform. - `submit` delegates to `NemoJobScheduler.submit_remote` — POSTs to the plugin service's per-job endpoint; the cluster executes. - `explain` delegates to `NemoJobScheduler.explain` — reads schemas locally from the `NemoJob` class. diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/JOB.md b/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/JOB.md index c174ae5705..31da16cc3a 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/JOB.md +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/JOB.md @@ -3,17 +3,15 @@ # Job Surface (NemoJob) -A `NemoJob` is a unit of work you can execute locally, submit to a cluster, or introspect — the same class drives all three. The platform auto-generates three CLI verbs per job: `run`, `submit`, `explain`. +A `NemoJob` is a unit of work you can submit to a cluster, execute inside a task container, or introspect. The platform auto-generates two CLI verbs per job: `submit`, `explain`. ``` -nemo run [--spec '{...}' | --spec-file FILE] nemo submit [--profile

] [--cluster ] \ [--spec '{...}' | --spec-file FILE] \ [-o .= ...] [--options-file FILE] nemo explain [--profile

] ``` -- `run` — executes `job.run()` in-process. No platform needed. - `submit` — POSTs the job to the plugin service, which compiles it into a `PlatformJobSpec` and hands it off to the Jobs service for cluster execution. - `explain` — prints the job's schemas and submit route. Reads locally, no network. @@ -47,7 +45,7 @@ class GenerateJob(NemoJob): ... ``` -Every job must declare `spec_schema`. Every job that participates in `submit` must override `compile()`. Running locally only requires `run()`. +Every job must declare `spec_schema`. Every job that participates in `submit` must override `compile()`. The `run()` method remains the task-container entry point. ### Method colours @@ -149,7 +147,7 @@ Storage env vars: - `NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH` — shared across job steps - `NEMO_JOB_STEP_CONFIG_STORAGE_PATH` — config files (read-only) -Container image keys: `"cpu-tasks"` (default) or `"gpu-tasks"`. Ignored for local `run`. +Container image keys: `"cpu-tasks"` (default) or `"gpu-tasks"`. ## `run()` contract diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/QUICKSTART.md b/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/QUICKSTART.md index 5bd4d71b95..95684d84db 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/QUICKSTART.md +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/docs/QUICKSTART.md @@ -117,7 +117,7 @@ nemo my-plugin greet --name Alice ## Step 4: Add a job -Declare the spec with Pydantic, implement `run()` for local execution, and override `compile()` for remote execution: +Declare the spec with Pydantic, implement `run()` for task execution, and override `compile()` for job submission: ```python # src/nemo_my_plugin/jobs/process.py diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/function.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/function.py index 0e9357c5aa..6f375f6245 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/function.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/function.py @@ -6,10 +6,9 @@ Plugin authors subclass :class:`NemoFunction` and register the class under the ``nemo.functions`` entry-point group. The platform mounts a ``POST`` route per function on the plugin service (auto-derived path), -and the CLI exposes ``nemo run`` (in-process) and -``nemo submit`` (HTTP POST) for each. Functions never -dispatch through a backend — they run in the plugin service's event -loop, in the same request that triggered them. +and the CLI exposes ``nemo submit`` (HTTP POST) for each. +Functions never dispatch through a backend — they run in the plugin +service's event loop, in the same request that triggered them. Mental model — *Function = spec → response | stream[frame]*: @@ -86,7 +85,7 @@ class NemoFunction(_NamedPlugin, Generic[SpecT]): Subclasses declare their identity via class variables and implement :meth:`run`. The platform auto-derives: - - A CLI subcommand tree: ``nemo run|submit``. + - A CLI subcommand tree: ``nemo submit``. - A FastAPI route on the plugin service: ``POST /apis//v2/workspaces/{workspace}/`` (override per-class via :attr:`endpoint`). @@ -113,8 +112,8 @@ class NemoFunction(_NamedPlugin, Generic[SpecT]): Pydantic model for the function's inputs. The route adapter validates the request body against it before invocation; the - local ``run`` verb validates the CLI-supplied spec the same - way. **Required** — the route factory and the CLI both need it + ``submit`` CLI validates the CLI-supplied spec the same way. + **Required** — the route factory and the CLI both need it to generate a working surface. Tied to :data:`SpecT` so a subclass like ``NemoFunction[GreetSpec]`` constrains ``spec_schema`` to ``type[GreetSpec]`` for type-checkers (and diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/job.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/job.py index ea91db6f39..9c8948c84b 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/job.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/job.py @@ -4,9 +4,9 @@ """Plugin job interface — what plugin authors implement for schedulable jobs. Plugin authors subclass :class:`NemoJob` and register the class under the -``nemo.jobs`` entry-point group. The platform (or SDK) instantiates each class -and invokes it through the :class:`~nemo_platform_plugin.scheduler.NemoJobScheduler` -for local execution, or POSTs it to the plugin service for remote submission. +``nemo.jobs`` entry-point group. The API service compiles and submits jobs. +Platform workers instantiate the job class and invoke :meth:`NemoJob.run` +through :func:`nemo_platform_plugin.tasks.dispatcher.run_task`. Mental model — *Job = spec + profile + options*: @@ -27,9 +27,9 @@ ``def``. They run in the task container, where there is no event loop and most work calls into sync library protocols. -The scheduler runs the async lifecycle methods through a single -``asyncio.run`` at the top of :meth:`NemoJobScheduler.run_local`; the -sync ``run`` is invoked directly from the resulting canonical spec. +The API process runs async lifecycle methods such as :meth:`to_spec` and +:meth:`compile`. Task containers receive the canonical step config and call +sync :meth:`run` through the task dispatcher. Example:: @@ -60,8 +60,7 @@ def run(self, config: dict) -> dict: Entry-point key convention: ``.``, e.g. ``example.say-hello``. This lets :func:`~nemo_platform_plugin.discovery.discover_jobs` resolve jobs unambiguously -across plugins; programmatic execution goes through -:meth:`nemo_platform_plugin.scheduler.NemoJobScheduler.run_local`. +across plugins. """ from __future__ import annotations @@ -107,7 +106,7 @@ class NemoJob(_NamedPlugin): Container image key for remote execution (e.g. ``"gpu-tasks"``, ``"cpu-tasks"``). Used by the Jobs service to pick the right - container image for each step. Ignored for local execution. + container image for each step. .. attribute:: execution_provider :type: str diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/job_context.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/job_context.py index 9a5cbc0359..2e59e1e514 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/job_context.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/job_context.py @@ -9,7 +9,7 @@ in-container runtime sets: - ``workspace`` ← ``NEMO_JOB_WORKSPACE`` -- ``job_id`` ← ``NEMO_JOB_ID`` (``None`` for local runs) +- ``job_id`` ← ``NEMO_JOB_ID`` (``None`` when no platform job exists) - ``storage.ephemeral`` ← ``NEMO_JOB_EPHEMERAL_TASK_STORAGE_PATH`` - ``storage.persistent`` ← ``NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH`` @@ -27,13 +27,12 @@ Example:: - def run(self, config: dict, *, ctx: JobContext, is_local: bool) -> dict: + def run(self, config: dict, *, ctx: JobContext) -> dict: spec = MySpec.model_validate(config) out_path = ctx.storage.ephemeral / "rows.jsonl" ... ref = ctx.results.save("rows.jsonl", out_path) - if not is_local: - self.report_progress(ctx, status="done") + self.report_progress(ctx, status="done") return {"status": "completed", "result": ref.model_dump()} """ @@ -84,9 +83,8 @@ class JobContext: Attributes: workspace: Workspace scope the job runs in. storage: Scratch and persistent filesystem paths. - results: Sink for publishing results (local directory for - laptop runs, NeMo Platform fileset on the platform). - job_id: Platform job UUID, or ``None`` for a local run. + results: Sink for publishing results. + job_id: Platform job UUID, or ``None`` for explicit test contexts. """ workspace: str diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/job_results.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/job_results.py index d009ba2ebb..d92b1aa56b 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/job_results.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/job_results.py @@ -10,8 +10,8 @@ Concrete impls living in this codebase: - :class:`LocalJobResults` — copies the artefact under a local directory - rooted at ``/results/``. Used for laptop ``run_local`` and - any context where no NeMo Platform Files / Jobs SDK is configured. + rooted at ``/results/``. Used by unit tests and explicitly + injected contexts. - :class:`PlatformJobResults` — thin adapter over :class:`nemo_platform_plugin.jobs.result_manager.ResultManager` that registers results as platform artefacts. diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/run_dependencies.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/run_dependencies.py index a55724c362..aa0efa5068 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/run_dependencies.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/run_dependencies.py @@ -1,13 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Signature-based dependency helpers for ``NemoJob.run``. - -Both local scheduler execution and platform-launched task containers use -these helpers to bind supported keyword-only parameters on job ``run`` -methods. Keeping this surface separate avoids making lightweight task -entrypoints import scheduler submission machinery. -""" +"""Signature-based dependency helpers for platform task ``NemoJob.run`` calls.""" from __future__ import annotations @@ -22,9 +16,8 @@ Python applies the run signature's own default.""" -class LocalRunError(RuntimeError): - """Raised when a required ``sdk`` / ``async_sdk`` parameter on - :meth:`NemoJob.run` has no handle to bind.""" +class RunDependencyError(RuntimeError): + """Raised when a required injected ``NemoJob.run`` parameter cannot be bound.""" def resolve_run_kwargs( @@ -41,11 +34,11 @@ def resolve_run_kwargs( Recognises ``ctx``, ``sdk``, ``async_sdk``, and ``is_local`` on the keyword-only portion of *run*'s signature; everything else is left unbound so Python's own default applies. ``is_local`` lets jobs adapt - behaviour to the execution context — ``True`` from the local scheduler, - ``False`` from the platform task dispatcher. + behaviour to the execution context. Platform task dispatch passes + ``False``. Raises: - LocalRunError: When *run* declares a required ``sdk`` / + RunDependencyError: When *run* declares a required ``sdk`` / ``async_sdk`` parameter with no default and the corresponding handle is not supplied. """ @@ -104,10 +97,9 @@ def _resolve_run_param( if sdk is not None: return sdk if required: - raise LocalRunError( + raise RunDependencyError( f"{job_cls.__name__}.run requires a `sdk` argument; " - f"pass it via NemoJobScheduler.run_local(sdk=...) or " - f"nemo_platform_plugin.tasks.dispatcher.run_task(sdk=...)." + "pass it via nemo_platform_plugin.tasks.dispatcher.run_task(sdk=...)." ) return _UNBOUND @@ -115,17 +107,15 @@ def _resolve_run_param( if async_sdk is not None: return async_sdk if required: - raise LocalRunError( + raise RunDependencyError( f"{job_cls.__name__}.run requires an `async_sdk` " - f"argument; pass it via " - f"NemoJobScheduler.run_local(async_sdk=...) or " - f"nemo_platform_plugin.tasks.dispatcher.run_task(async_sdk=...)." + "argument; pass it via nemo_platform_plugin.tasks.dispatcher.run_task(async_sdk=...)." ) return _UNBOUND if required: - # Surface as LocalRunError instead of a downstream TypeError. - raise LocalRunError( + # Surface as RunDependencyError instead of a downstream TypeError. + raise RunDependencyError( f"{job_cls.__name__}.run declares unsupported required parameter " f"`{param.name}`; only `ctx`, `sdk`, `async_sdk`, and `is_local` " "are injected automatically." @@ -134,4 +124,4 @@ def _resolve_run_param( return _UNBOUND -__all__ = ["LocalRunError", "resolve_run_kwargs"] +__all__ = ["RunDependencyError", "resolve_run_kwargs"] diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/scheduler.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/scheduler.py index c326d767c9..1010b55bd1 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/scheduler.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/scheduler.py @@ -1,54 +1,21 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""NemoJobScheduler — the runtime that drives ``run`` / ``submit`` / ``explain``. - -Three entry points, one per CLI verb generated by -:func:`~nemo_platform_plugin.commands.add_job_commands`: - -- :meth:`NemoJobScheduler.run_local` — execute the job in-process. Validates - the spec against :attr:`~nemo_platform_plugin.job.NemoJob.input_spec_schema` (or - :attr:`~nemo_platform_plugin.job.NemoJob.spec_schema`), invokes - :meth:`~nemo_platform_plugin.job.NemoJob.to_spec` when an input shape is declared, - then calls ``job.run(...)``. -- :meth:`NemoJobScheduler.submit_remote` — POST the job to the plugin - service's per-job endpoint. -- :meth:`NemoJobScheduler.explain` — read-only schema introspection. - -Spec handling for ``run(config)``: - -When :attr:`~nemo_platform_plugin.job.NemoJob.spec_schema` is declared, the scheduler -validates the incoming dict against it and passes ``spec.model_dump()`` to -``run``. When ``spec_schema`` is ``None``, the raw dict is forwarded -unchanged. Jobs that declare extra keyword-only parameters on ``run`` -(``ctx``, ``sdk``, ``async_sdk``) receive them via signature-based DI — -see :func:`~nemo_platform_plugin.run_dependencies.resolve_run_kwargs`. +"""NemoJobScheduler submits plugin jobs and explains their schemas. + +The scheduler owns user-facing remote submission and read-only schema +introspection. Platform workers execute :meth:`NemoJob.run` through +``nemo_platform_plugin.tasks.dispatcher.run_task``; there is no public local +scheduler execution path. """ from __future__ import annotations -import asyncio import logging -import os -import tempfile -from contextlib import contextmanager -from pathlib import Path from typing import TYPE_CHECKING, Any import httpx from nemo_platform_plugin.job import job_collection_path_for -from nemo_platform_plugin.job_context import JobContext, StoragePaths -from nemo_platform_plugin.job_results import LocalJobResults -from nemo_platform_plugin.run_dependencies import resolve_run_kwargs - -# Environment variables the in-container runtime sets and that legacy task -# code reads via ``os.environ.get(...)``. The scheduler mirrors them onto the -# process for the duration of ``run_local`` so existing task code keeps -# working without modification when invoked from the local CLI. -_PERSISTENT_STORAGE_ENVVAR = "NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH" -_EPHEMERAL_STORAGE_ENVVAR = "NEMO_JOB_EPHEMERAL_TASK_STORAGE_PATH" -_JOB_ID_ENVVAR = "NEMO_JOB_ID" -_WORKSPACE_ENVVAR = "NEMO_JOB_WORKSPACE" if TYPE_CHECKING: from nemo_platform_plugin.job import NemoJob @@ -63,78 +30,7 @@ class NemoJobScheduler: - """Drives the three execution verbs (``run`` / ``submit`` / ``explain``). - - Construction-cheap — instantiate per CLI invocation. All state is - derived from the :class:`NemoJob` subclass and the submitter-provided - spec. The ``sdk`` and ``async_sdk`` kwargs on :meth:`run_local` are - opt-in hooks for jobs that need SDK access; the scheduler does not - construct them on its own. - """ - - # ------------------------------------------------------------------ # - # Local execution # - # ------------------------------------------------------------------ # - - def run_local( - self, - job_cls: type["NemoJob"], - spec: dict, - *, - workspace: str = "default", - sdk: object | None = None, - async_sdk: object | None = None, - ctx: JobContext | None = None, - ) -> dict: - """Execute *job_cls* locally, in-process. - - Sync entry point. The async portion of the job lifecycle — - ``to_spec`` — runs once via :func:`asyncio.run` to produce the - canonical spec; the sync ``run`` is invoked directly. Method - colours follow the execution-context rule: ``to_spec`` / - ``compile`` are async (API-side); ``run`` is sync (task - container). - - Args: - job_cls: :class:`~nemo_platform_plugin.job.NemoJob` subclass. - spec: Submitter input as a plain ``dict``. Validated against - ``input_spec_schema`` or ``spec_schema`` when declared; - otherwise passed through unchanged. - workspace: Workspace scope for the local context. - sdk: Optional :class:`~nemo_platform.NeMoPlatform` handle. - Bound to an ``sdk`` kwarg on ``run`` if the signature - declares one. Jobs obtain typed service clients via - ``client_from_platform(sdk, FilesClient)`` for - per-service operations. - async_sdk: Optional :class:`~nemo_platform.AsyncNeMoPlatform` - handle, bound the same way when an ``async_sdk`` kwarg - is declared. Sync ``run`` cannot consume this directly - — it's available for jobs that delegate to async - helpers via :func:`asyncio.run`. - ctx: Optional :class:`JobContext`. When omitted, the - scheduler builds one with tempdir-backed storage and a - :class:`LocalJobResults` sink. - - Returns: - The job's ``dict`` result from :meth:`NemoJob.run`. - """ - _ensure_local_logging_configured() - canonical = asyncio.run( - self._prepare_canonical_spec( - job_cls, - spec, - workspace=workspace, - async_sdk=async_sdk, - is_local=True, - ) - ) - ctx = ctx or self._build_local_context(job_cls, workspace=workspace) - # Note: scheduler-built tempdirs are intentionally not torn down here. - # ``ctx.results`` URLs (``file://...``) must outlive ``run_local`` so - # callers can read the artifacts they pointed at. The OS reaps the - # ``nemo-platform-plugin-*`` dirs under ``$TMPDIR``; tests that need hermetic - # directories should pass an explicit ``ctx`` with ``tmp_path``. - return self._invoke_run(job_cls, canonical, sdk=sdk, async_sdk=async_sdk, ctx=ctx) + """Submit plugin jobs remotely and explain their schemas.""" # ------------------------------------------------------------------ # # Remote submission # @@ -236,99 +132,6 @@ def explain( # Internal helpers # # ------------------------------------------------------------------ # - async def _prepare_canonical_spec( - self, - job_cls: type["NemoJob"], - spec: dict, - *, - workspace: str, - async_sdk: object | None, - is_local: bool, - ) -> dict: - """Return the spec as a ``dict`` ready for :meth:`run`. - - - If ``input_spec_schema`` is declared: validate against it and - pass the result through the ``async classmethod`` ``to_spec`` - to produce the canonical instance; return - ``canonical.model_dump()``. - - Else if ``spec_schema`` is declared: validate against it and - return ``validated.model_dump()`` (normalises defaults, - coerces types). - - Else: return *spec* unchanged. - - ``async`` because :meth:`NemoJob.to_spec` is an - ``async classmethod`` (runs in API context). The caller drives - this through :func:`asyncio.run` from :meth:`run_local`. - - ``async_sdk`` is the SDK handle forwarded to ``to_spec``. - ``to_spec`` only accepts an async client by convention (the - method is itself ``async`` and runs in the API process); the - local path matches the production API contract so plugin - authors don't need a sync/async branch. - - ``is_local`` is always forwarded because it is part of the - ``to_spec`` lifecycle signature. - """ - input_schema = job_cls.input_spec_schema or job_cls.spec_schema - if input_schema is None: - return spec - - validated = input_schema.model_validate(spec, context={"is_local": is_local}) - - if job_cls.input_spec_schema is not None: - canonical = await job_cls.to_spec( - validated, - workspace=workspace, - entity_client=None, - async_sdk=async_sdk, - is_local=is_local, - ) - # Re-validate ``to_spec`` output against ``spec_schema`` so plugin - # authors can't widen the canonical shape by accident — ``run`` - # always sees the schema the rest of the stack advertises. - if job_cls.spec_schema is not None: - canonical = job_cls.spec_schema.model_validate(canonical, context={"is_local": is_local}) - return canonical.model_dump() - - return validated.model_dump() - - def _invoke_run( - self, - job_cls: type["NemoJob"], - canonical: dict, - *, - sdk: object | None, - async_sdk: object | None, - ctx: JobContext, - ) -> dict: - """Call ``job.run(...)`` with signature-based dependency injection. - - Keyword-only parameters on ``run`` are resolved by name: - - - ``ctx: JobContext`` — bound to the constructed context. - - ``sdk`` — bound to the *sdk* argument. - - ``async_sdk`` — bound to the *async_sdk* argument. - - ``is_local`` — bound to ``True`` because this is the local runner. - - Required parameters (no default) raise :class:`LocalRunError` before - ``run`` is called when the scheduler can't resolve them. Parameters - with a default (typically ``Optional[X] = None``) are left unbound - and fall back to the declared default. - - ``run(self, config)`` (no extra parameters) is also supported and - receives only the canonical dict. - """ - logger.debug( - "Running job %r locally (workspace=%s, job_id=%s)", - job_cls.name, - ctx.workspace, - ctx.job_id, - ) - job = job_cls() - kwargs = resolve_run_kwargs(job_cls, job.run, sdk=sdk, async_sdk=async_sdk, ctx=ctx, is_local=True) - with _local_runtime_env(ctx): - return job.run(canonical, **kwargs) - # ---- submit_remote helpers ------------------------------------- # def _build_submit_url( @@ -394,90 +197,19 @@ def _post_submit( response.raise_for_status() return response.json() - # ---- local-context builder ------------------------------------- # - - def _build_local_context(self, job_cls: type["NemoJob"], *, workspace: str) -> JobContext: - """Build a :class:`JobContext` for local execution. - - Creates a tempdir with ``ephemeral/`` and ``persistent/`` - subdirectories that outlive the scheduler call, and wires a - :class:`LocalJobResults` sink rooted at ``persistent/results/``. - - ``job_id`` is left ``None`` for scheduler-created local contexts. - """ - root = Path(tempfile.mkdtemp(prefix="nemo-platform-plugin-", suffix=f"-{job_cls.name}")) - ephemeral = root / "ephemeral" - persistent = root / "persistent" - ephemeral.mkdir(parents=True, exist_ok=True) - persistent.mkdir(parents=True, exist_ok=True) - results_root = persistent / "results" - storage = StoragePaths(ephemeral=ephemeral, persistent=persistent) - return JobContext( - workspace=workspace, - storage=storage, - results=LocalJobResults(root=results_root), - ) - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -@contextmanager -def _local_runtime_env(ctx: JobContext): - """Mirror the in-container runtime envvars onto the process during ``run``. - - Legacy task code (and the existing per-service compilers) reads - ``NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH`` and friends via ``os.environ`` - to locate scratch / persistent volumes. Inside a real job pod the Jobs - controller injects them; in local mode the scheduler synthesizes a - :class:`JobContext` with tempdir-backed storage and exposes the - same envvars here so unmodified task code keeps working. - - Caller-set storage values win — we only fill storage envvars that aren't - already set. The workspace envvar always reflects the current local - context so a parent job's workspace does not leak into nested local runs. - Prior environment values are restored on exit so tests and back-to-back - calls don't leak state. - """ - storage_overrides: dict[str, str] = { - _PERSISTENT_STORAGE_ENVVAR: str(ctx.storage.persistent), - _EPHEMERAL_STORAGE_ENVVAR: str(ctx.storage.ephemeral), - } - # Scheduler-built local contexts leave job_id as ``None``. Explicit - # caller-provided contexts may still mirror a job id for legacy code. - force_overrides: dict[str, str] = {_WORKSPACE_ENVVAR: ctx.workspace} - if ctx.job_id is not None: - force_overrides[_JOB_ID_ENVVAR] = ctx.job_id - saved: dict[str, str | None] = {} - try: - for key, value in storage_overrides.items(): - if key in os.environ: - continue # respect explicit caller setup - saved[key] = None - os.environ[key] = value - for key, value in force_overrides.items(): - saved[key] = os.environ.get(key) - os.environ[key] = value - yield - finally: - for key, prior in saved.items(): - if prior is None: - os.environ.pop(key, None) - else: - os.environ[key] = prior - - def _job_key_for(job_cls: type["NemoJob"]) -> str: """Best-effort entry-point key for *job_cls*. Uses the API segment (top-level package, ``nemo_`` prefix stripped, hyphenated) as a fallback when no registered entry point is available — e.g. a job in ``nemo_data_designer.jobs.generate`` keys as ``data-designer.``, - matching the documented ``.`` convention. Scheduler callers - that have the registered key should pass it explicitly to - :meth:`NemoJobScheduler.run_local` via a custom ``ctx``. + matching the documented ``.`` convention. """ return f"{_api_segment_for(job_cls)}.{job_cls.name}" @@ -526,22 +258,3 @@ def _api_segment_for(job_cls: type["NemoJob"]) -> str: if module.startswith("nemo_"): module = module[len("nemo_") :] return module.replace("_", "-") - - -def _ensure_local_logging_configured() -> None: - """Attach a stderr INFO handler to the root logger if none is set. - - No-op when the caller already configured logging (CLI with ``-v``, - service ``dictConfig``, test harness). Noisy transport libraries - (``httpx``, ``httpcore``) are clamped to WARNING so per-request - debug output doesn't drown the job's own messages. - """ - root = logging.getLogger() - if root.handlers: - return - handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter("%(levelname)s %(message)s")) - root.addHandler(handler) - root.setLevel(logging.INFO) - logging.getLogger("httpx").setLevel(logging.WARNING) - logging.getLogger("httpcore").setLevel(logging.WARNING) diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/tasks/dispatcher.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/tasks/dispatcher.py index 93d11a5b0d..f60294d0cc 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/tasks/dispatcher.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/tasks/dispatcher.py @@ -3,13 +3,10 @@ """Task entrypoint dispatcher for :class:`~nemo_platform_plugin.job.NemoJob` subclasses. -Mirrors :meth:`~nemo_platform_plugin.scheduler.NemoJobScheduler.run_local` for any -process the platform spawns with the ``NEMO_JOB_*`` environment populated — -both Docker-backed task containers and host subprocess executors land here. -Reads the step config from the platform-injected file path, builds a -:class:`~nemo_platform_plugin.job_context.JobContext` from the environment, and -invokes ``job.run(...)`` with the same signature-based DI used locally -(see :func:`~nemo_platform_plugin.run_dependencies.resolve_run_kwargs`). +This module is the platform-spawned execution path for task containers and +host subprocess executors. It reads the step config from ``NEMO_JOB_*`` +environment, builds a :class:`~nemo_platform_plugin.job_context.JobContext`, +and invokes ``job.run(...)`` with signature-based dependency injection. Because ``run_task`` only runs inside a platform-spawned process, the default ``ctx.results`` is :class:`~nemo_platform_plugin.job_results.PlatformJobResults` @@ -56,7 +53,7 @@ def _shutdown(signum: int, _frame: FrameType | None) -> None: NEMO_JOB_WORKSPACE_ENVVAR, PERSISTENT_JOB_STORAGE_PATH_ENVVAR, ) -from nemo_platform_plugin.run_dependencies import LocalRunError, resolve_run_kwargs +from nemo_platform_plugin.run_dependencies import RunDependencyError, resolve_run_kwargs logger = logging.getLogger(__name__) @@ -74,7 +71,7 @@ def run_task( builds a :class:`JobContext` from the platform-injected env when *ctx* is omitted, and invokes ``job.run(config, **kwargs)`` with signature DI of ``ctx`` / ``sdk`` / ``async_sdk``. Exit codes follow :func:`_exit_code_for`; - :class:`~nemo_platform_plugin.run_dependencies.LocalRunError` propagates verbatim. + :class:`~nemo_platform_plugin.run_dependencies.RunDependencyError` propagates verbatim. The auto-built ``ctx`` wires :class:`~nemo_platform_plugin.job_results.PlatformJobResults` as ``ctx.results``, @@ -121,7 +118,7 @@ def run_task( try: kwargs = resolve_run_kwargs(job_cls, job.run, sdk=sdk, async_sdk=async_sdk, ctx=ctx, is_local=False) - except LocalRunError: + except RunDependencyError: # Plugin-author bug (e.g. required sdk param without a handle); propagate # rather than collapse into the same exit-2 bucket as a missing env var. raise @@ -131,7 +128,7 @@ def run_task( try: result = job.run(config, **kwargs) - except LocalRunError: + except RunDependencyError: # Propagate verbatim per run_task's contract. raise except Exception: diff --git a/packages/nemo_platform_plugin/tests/test_cli_hooks.py b/packages/nemo_platform_plugin/tests/test_cli_hooks.py index 3590a20d25..1c5c4f6ffb 100644 --- a/packages/nemo_platform_plugin/tests/test_cli_hooks.py +++ b/packages/nemo_platform_plugin/tests/test_cli_hooks.py @@ -1,23 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests for NemoCLI.update_function_cli / update_job_cli hooks. - -Covers the override-by-replacement contract: -- No-op default leaves the auto-generated CLI surface unchanged. -- Hook fires once per primitive after default verb registration and - before the sub-group is mounted. -- Override-by-replacement (Typer's last-write-wins materialization) lets - a plugin author add, drop, or replace verbs and flags. -- Both jobs and functions are covered symmetrically. -""" +"""Tests for NemoCLI.update_function_cli / update_job_cli submit hooks.""" from __future__ import annotations import json from typing import ClassVar +import pytest import typer +from nemo_platform_plugin import commands from nemo_platform_plugin.cli import NemoCLI from nemo_platform_plugin.commands import add_function_commands, add_job_commands from nemo_platform_plugin.function import NemoFunction @@ -28,29 +21,26 @@ runner = CliRunner() -# --------------------------------------------------------------------------- -# Fixtures -# --------------------------------------------------------------------------- +class _GreetSpec(BaseModel): + name: str class _GreetJob(NemoJob): name: ClassVar[str] = "greet" - description: ClassVar[str] = "Return a greeting." + description: ClassVar[str] = "Submit a greeting." + spec_schema: ClassVar[type[_GreetSpec]] = _GreetSpec def run(self, config: dict) -> dict: - return {"message": f"Hello, {config.get('name', 'world')}!"} + return {"message": f"Hello, {config['name']}!"} class _ByeJob(NemoJob): name: ClassVar[str] = "bye" - description: ClassVar[str] = "Return a farewell." + description: ClassVar[str] = "Submit a farewell." + spec_schema: ClassVar[type[_GreetSpec]] = _GreetSpec def run(self, config: dict) -> dict: - return {"message": f"Bye, {config.get('name', 'world')}!"} - - -class _GreetSpec(BaseModel): - name: str + return {"message": f"Bye, {config['name']}!"} class _GreetResponse(BaseModel): @@ -59,7 +49,7 @@ class _GreetResponse(BaseModel): class _GreetFunction(NemoFunction[_GreetSpec]): name: ClassVar[str] = "greet" - description: ClassVar[str] = "Say hello to a name." + description: ClassVar[str] = "Submit a greeting function." spec_schema: ClassVar[type[_GreetSpec]] = _GreetSpec async def run(self, spec: _GreetSpec) -> _GreetResponse: @@ -68,16 +58,18 @@ async def run(self, spec: _GreetSpec) -> _GreetResponse: class _ByeFunction(NemoFunction[_GreetSpec]): name: ClassVar[str] = "bye" - description: ClassVar[str] = "Say bye to a name." + description: ClassVar[str] = "Submit a farewell function." spec_schema: ClassVar[type[_GreetSpec]] = _GreetSpec async def run(self, spec: _GreetSpec) -> _GreetResponse: return _GreetResponse(message=f"Bye, {spec.name}!") -class _NoOpCLI(NemoCLI): - """Minimal NemoCLI subclass with no hook overrides.""" +_GreetFunction.__module__ = "nemo_plugin.functions.greet" +_ByeFunction.__module__ = "nemo_plugin.functions.bye" + +class _NoOpCLI(NemoCLI): name: ClassVar[str] = "test-plugin" def get_cli(self) -> typer.Typer: @@ -108,25 +100,38 @@ def _noop() -> None: return app -# --------------------------------------------------------------------------- -# update_job_cli -# --------------------------------------------------------------------------- +def _stub_job_submit(monkeypatch: pytest.MonkeyPatch) -> None: + def submit_remote(self, job_cls, spec_data, **kwargs): # type: ignore[no-untyped-def] + del self, job_cls, kwargs + return {"submitted": spec_data} + + monkeypatch.setattr(commands.NemoJobScheduler, "submit_remote", submit_remote) + + +def _stub_function_submit(monkeypatch: pytest.MonkeyPatch) -> None: + def post_function_submit(url, body, **kwargs): # type: ignore[no-untyped-def] + del url, body, kwargs + + monkeypatch.setattr(commands, "_post_function_submit", post_function_submit) class TestUpdateJobCli: def test_default_noop_leaves_subcommands_unchanged(self) -> None: app = _app_with_jobs(_GreetJob, cli=_NoOpCLI()) result = runner.invoke(app, ["greet", "--help"]) + assert result.exit_code == 0 - assert "run" in result.output assert "submit" in result.output assert "explain" in result.output + assert "Run locally" not in result.output - def test_no_cli_argument_means_no_hook_called(self) -> None: - # Smoke: the default no-cli path still works (backwards compat). + def test_no_cli_argument_uses_default_submit(self, monkeypatch: pytest.MonkeyPatch) -> None: + _stub_job_submit(monkeypatch) app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "run", "--config", '{"name": "X"}']) + result = runner.invoke(app, ["greet", "submit", "--name", "X"]) + assert result.exit_code == 0 + assert json.loads(result.output) == {"submitted": {"name": "X"}} def test_hook_invoked_once_per_job(self) -> None: seen: list[str] = [] @@ -138,36 +143,35 @@ def update_job_cli(self, job_cls, group) -> None: # noqa: ARG002 _app_with_jobs(_GreetJob, _ByeJob, cli=_CLI()) assert sorted(seen) == ["bye", "greet"] - def test_hook_can_replace_run_with_a_new_signature(self) -> None: + def test_hook_can_replace_submit_with_a_new_signature(self, monkeypatch: pytest.MonkeyPatch) -> None: + _stub_job_submit(monkeypatch) + class _CLI(_NoOpCLI): def update_job_cli(self, job_cls, group) -> None: if job_cls is not _GreetJob: return - original = next(c for c in group.registered_commands if c.name == "run").callback + original = next(c for c in group.registered_commands if c.name == "submit").callback assert original is not None - @group.command("run") - def run( + @group.command("submit") + def submit( typer_ctx: typer.Context, name: str = typer.Option(..., "--name"), spec: str = typer.Option("{}", "--spec"), ) -> None: merged = json.dumps({**json.loads(spec), "name": name}) - original(typer_ctx, spec=merged, spec_file=None, config=None, config_file=None) + original(typer_ctx, spec=merged, spec_file=None) app = _app_with_jobs(_GreetJob, cli=_CLI()) - - # The new flag shows up in --help. - help_result = runner.invoke(app, ["greet", "run", "--help"]) + help_result = runner.invoke(app, ["greet", "submit", "--help"]) assert help_result.exit_code == 0 assert "--name" in help_result.output - # Invoking with --name calls wrapper -> original chain. - result = runner.invoke(app, ["greet", "run", "--name", "Wrapped"]) + result = runner.invoke(app, ["greet", "submit", "--name", "Wrapped"]) assert result.exit_code == 0 - assert json.loads(result.output) == {"message": "Hello, Wrapped!"} + assert json.loads(result.output) == {"submitted": {"name": "Wrapped"}} - def test_hook_can_drop_a_verb(self) -> None: + def test_hook_can_drop_submit(self) -> None: class _CLI(_NoOpCLI): def update_job_cli(self, job_cls, group) -> None: # noqa: ARG002 group.registered_commands = [c for c in group.registered_commands if c.name != "submit"] @@ -175,8 +179,8 @@ def update_job_cli(self, job_cls, group) -> None: # noqa: ARG002 app = _app_with_jobs(_GreetJob, cli=_CLI()) result = runner.invoke(app, ["greet", "--help"]) assert result.exit_code == 0 - assert "run" in result.output assert "submit" not in result.output + assert "explain" in result.output def test_hook_can_add_a_verb(self) -> None: class _CLI(_NoOpCLI): @@ -193,8 +197,6 @@ def cancel() -> None: assert "canceled" in result.output def test_hook_dispatches_per_job(self) -> None: - """A hook that only modifies _GreetJob leaves _ByeJob untouched.""" - class _CLI(_NoOpCLI): def update_job_cli(self, job_cls, group) -> None: if job_cls is not _GreetJob: @@ -205,32 +207,25 @@ def custom() -> None: typer.echo("greet-only") app = _app_with_jobs(_GreetJob, _ByeJob, cli=_CLI()) - - greet_help = runner.invoke(app, ["greet", "--help"]) - assert "custom" in greet_help.output - - bye_help = runner.invoke(app, ["bye", "--help"]) - assert "custom" not in bye_help.output - - -# --------------------------------------------------------------------------- -# update_function_cli -# --------------------------------------------------------------------------- + assert "custom" in runner.invoke(app, ["greet", "--help"]).output + assert "custom" not in runner.invoke(app, ["bye", "--help"]).output class TestUpdateFunctionCli: - def test_default_noop_leaves_subcommands_unchanged(self) -> None: + def test_default_noop_leaves_submit_only(self) -> None: app = _app_with_functions(_GreetFunction, cli=_NoOpCLI()) result = runner.invoke(app, ["greet", "--help"]) + assert result.exit_code == 0 - assert "run" in result.output assert "submit" in result.output + assert "Run locally" not in result.output - def test_no_cli_argument_means_no_hook_called(self) -> None: + def test_no_cli_argument_uses_default_submit(self, monkeypatch: pytest.MonkeyPatch) -> None: + _stub_function_submit(monkeypatch) app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet", "run", "--name", "World"]) + result = runner.invoke(app, ["greet", "submit", "--name", "World"]) + assert result.exit_code == 0 - assert json.loads(result.output) == {"message": "Hello, World!"} def test_hook_invoked_once_per_function(self) -> None: seen: list[str] = [] @@ -242,16 +237,18 @@ def update_function_cli(self, fn_cls, group) -> None: # noqa: ARG002 _app_with_functions(_GreetFunction, _ByeFunction, cli=_CLI()) assert sorted(seen) == ["bye", "greet"] - def test_hook_can_replace_run_with_a_new_signature(self) -> None: + def test_hook_can_replace_submit_with_a_new_signature(self, monkeypatch: pytest.MonkeyPatch) -> None: + _stub_function_submit(monkeypatch) + class _CLI(_NoOpCLI): def update_function_cli(self, fn_cls, group) -> None: if fn_cls is not _GreetFunction: return - original = next(c for c in group.registered_commands if c.name == "run").callback + original = next(c for c in group.registered_commands if c.name == "submit").callback assert original is not None - @group.command("run") - def run( + @group.command("submit") + def submit( typer_ctx: typer.Context, nickname: str = typer.Option(..., "--nickname"), ) -> None: @@ -259,23 +256,20 @@ def run( original(typer_ctx, spec=spec_json, spec_file=None, workspace="default") app = _app_with_functions(_GreetFunction, cli=_CLI()) - - help_result = runner.invoke(app, ["greet", "run", "--help"]) + help_result = runner.invoke(app, ["greet", "submit", "--help"]) assert help_result.exit_code == 0 assert "--nickname" in help_result.output - result = runner.invoke(app, ["greet", "run", "--nickname", "Wrapped"]) + result = runner.invoke(app, ["greet", "submit", "--nickname", "Wrapped"]) assert result.exit_code == 0 - assert json.loads(result.output) == {"message": "Hello, Wrapped!"} - def test_hook_can_drop_a_verb(self) -> None: + def test_hook_can_drop_submit(self) -> None: class _CLI(_NoOpCLI): def update_function_cli(self, fn_cls, group) -> None: # noqa: ARG002 group.registered_commands = [c for c in group.registered_commands if c.name != "submit"] app = _app_with_functions(_GreetFunction, cli=_CLI()) result = runner.invoke(app, ["greet", "--help"]) - assert "run" in result.output assert "submit" not in result.output def test_hook_can_add_a_verb(self) -> None: @@ -303,9 +297,5 @@ def custom() -> None: typer.echo("greet-only") app = _app_with_functions(_GreetFunction, _ByeFunction, cli=_CLI()) - - greet_help = runner.invoke(app, ["greet", "--help"]) - assert "custom" in greet_help.output - - bye_help = runner.invoke(app, ["bye", "--help"]) - assert "custom" not in bye_help.output + assert "custom" in runner.invoke(app, ["greet", "--help"]).output + assert "custom" not in runner.invoke(app, ["bye", "--help"]).output diff --git a/packages/nemo_platform_plugin/tests/test_cli_renderer.py b/packages/nemo_platform_plugin/tests/test_cli_renderer.py index 78ca1dfd23..996006b61d 100644 --- a/packages/nemo_platform_plugin/tests/test_cli_renderer.py +++ b/packages/nemo_platform_plugin/tests/test_cli_renderer.py @@ -1,31 +1,21 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests for NemoCLI.get_function_renderer / get_job_renderer hooks. - -Covers: -- Default no-op (returns None) leaves the auto-generated echo behavior intact. -- A renderer's lifecycle methods fire in the expected order. -- The ``--output-format json`` global flag bypasses the renderer entirely. -- The "delegate to use the renderer" contract: an update_*_cli wrapper that - delegates to the original callback fires the renderer; a wrapper that does - its own iteration without calling original does not. -- on_error fires on exception (and the exception still propagates). -""" +"""Tests for NemoCLI submit renderer hooks.""" from __future__ import annotations import json -from collections.abc import AsyncIterator from types import SimpleNamespace from typing import Any, ClassVar +import pytest import typer +from nemo_platform_plugin import commands from nemo_platform_plugin.cli import NemoCLI from nemo_platform_plugin.cli_renderer import CLIRenderer, RendererContext from nemo_platform_plugin.commands import add_function_commands, add_job_commands from nemo_platform_plugin.function import NemoFunction -from nemo_platform_plugin.functions.frames import Done, Heartbeat from nemo_platform_plugin.job import NemoJob from pydantic import BaseModel from typer.testing import CliRunner @@ -33,50 +23,29 @@ runner = CliRunner() -# --------------------------------------------------------------------------- -# Fixture primitives -# --------------------------------------------------------------------------- - - class _GreetSpec(BaseModel): name: str -class _GreetResponse(BaseModel): - message: str - - -class _GreetFunction(NemoFunction[_GreetSpec]): +class _GreetJob(NemoJob): name: ClassVar[str] = "greet" - description: ClassVar[str] = "Say hello to a name." + description: ClassVar[str] = "Submit a greeting." spec_schema: ClassVar[type[_GreetSpec]] = _GreetSpec - async def run(self, spec: _GreetSpec) -> _GreetResponse: - return _GreetResponse(message=f"Hello, {spec.name}!") - - -class _CountSpec(BaseModel): - upto: int - + def run(self, config: dict) -> dict: + return {"message": f"Hello, {config['name']}!"} -class _CountFunction(NemoFunction[_CountSpec]): - """Streaming function — yields heartbeats then Done.""" - name: ClassVar[str] = "count" - spec_schema: ClassVar[type[_CountSpec]] = _CountSpec +class _GreetFunction(NemoFunction[_GreetSpec]): + name: ClassVar[str] = "greet" + description: ClassVar[str] = "Submit a greeting function." + spec_schema: ClassVar[type[_GreetSpec]] = _GreetSpec - async def run(self, spec: _CountSpec) -> AsyncIterator[BaseModel]: - for _ in range(spec.upto): - yield Heartbeat() - yield Done() + async def run(self, spec: _GreetSpec) -> dict: + return {"message": f"Hello, {spec.name}!"} -class _GreetJob(NemoJob): - name: ClassVar[str] = "greet" - description: ClassVar[str] = "Return a greeting." - - def run(self, config: dict) -> dict: - return {"message": f"Hello, {config.get('name', 'world')}!"} +_GreetFunction.__module__ = "nemo_plugin.functions.greet" class _NoOpCLI(NemoCLI): @@ -89,12 +58,9 @@ def get_cli(self) -> typer.Typer: class _RecordingRenderer(CLIRenderer): """Renderer that records lifecycle events for assertions.""" - events: list[tuple[str, Any]] = [] # noqa: RUF012 — class state shared across instances by design + events: list[tuple[str, Any]] = [] def __init__(self) -> None: - # Reset at the start of each renderer instance. - # Renderers are constructed once per verb invocation, so this clears - # between scenarios when scenarios use a fresh CLI build. type(self).events = [] def on_start(self, *, ctx: RendererContext) -> None: @@ -114,231 +80,177 @@ def on_error(self, error: BaseException, *, ctx: RendererContext) -> None: def _typer_context_with_overrides(output_format: str | None = None) -> object: - """Stand-in for CLIContext: just an object with .overrides dict.""" overrides: dict[str, Any] = {} if output_format is not None: overrides["output_format"] = output_format return SimpleNamespace(overrides=overrides) -def _build_function_app(*fn_classes: type[NemoFunction], cli: NemoCLI | None = None) -> typer.Typer: +def _build_job_app(*job_classes: type[NemoJob], cli: NemoCLI | None = None) -> typer.Typer: app = typer.Typer() @app.callback() def _noop() -> None: pass - fns = {f"plugin.{cls.name}": cls for cls in fn_classes} - add_function_commands(app, fns, cli=cli) + jobs = {f"plugin.{cls.name}": cls for cls in job_classes} + add_job_commands(app, jobs, cli=cli) return app -def _build_job_app(*job_classes: type[NemoJob], cli: NemoCLI | None = None) -> typer.Typer: +def _build_function_app(*fn_classes: type[NemoFunction], cli: NemoCLI | None = None) -> typer.Typer: app = typer.Typer() @app.callback() def _noop() -> None: pass - jobs = {f"plugin.{cls.name}": cls for cls in job_classes} - add_job_commands(app, jobs, cli=cli) + functions = {f"plugin.{cls.name}": cls for cls in fn_classes} + add_function_commands(app, functions, cli=cli) return app -# --------------------------------------------------------------------------- -# Streaming function: get_function_renderer for `run` -# --------------------------------------------------------------------------- - +def test_job_submit_no_renderer_falls_through_to_json(monkeypatch: pytest.MonkeyPatch) -> None: + def submit_remote(self, job_cls, spec_data, **kwargs): # type: ignore[no-untyped-def] + del self, job_cls, kwargs + return {"submitted": spec_data} + + monkeypatch.setattr(commands.NemoJobScheduler, "submit_remote", submit_remote) + app = _build_job_app(_GreetJob, cli=_NoOpCLI()) + result = runner.invoke(app, ["greet", "submit", "--name", "World"]) + + assert result.exit_code == 0 + assert json.loads(result.output) == {"submitted": {"name": "World"}} -class TestStreamingFunctionRunRenderer: - def test_no_renderer_falls_through_to_default_echo(self) -> None: - app = _build_function_app(_CountFunction, cli=_NoOpCLI()) - result = runner.invoke(app, ["count", "run", "--upto", "2"]) - assert result.exit_code == 0 - # Default behavior echoes each frame as JSON; verify by counting lines - # that look like a JSON object. - json_lines = [ln for ln in result.output.splitlines() if ln.strip().startswith("{")] - assert len(json_lines) >= 3 # 2 heartbeats + 1 Done - - def test_renderer_lifecycle_fires_in_order(self) -> None: - class _CLI(_NoOpCLI): - def get_function_renderer(self, fn_cls, *, verb): - return _RecordingRenderer if fn_cls is _CountFunction else None - app = _build_function_app(_CountFunction, cli=_CLI()) - result = runner.invoke(app, ["count", "run", "--upto", "2"]) - assert result.exit_code == 0, result.output - - events = _RecordingRenderer.events - names = [name for name, _ in events] - # Lifecycle: start → frames → complete. - assert names[0] == "start" - assert names[-1] == "complete" - assert names.count("frame") == 3 # 2 heartbeats + 1 Done - - # on_start receives the right context. - start_meta = events[0][1] - assert start_meta == {"verb": "run", "is_local": True} - - def test_output_format_json_bypasses_renderer(self) -> None: - class _CLI(_NoOpCLI): - def get_function_renderer(self, fn_cls, *, verb): - return _RecordingRenderer - - app = _build_function_app(_CountFunction, cli=_CLI()) - ctx_obj = _typer_context_with_overrides(output_format="json") - - # Reset before invocation. - _RecordingRenderer.events = [] - result = runner.invoke(app, ["count", "run", "--upto", "1"], obj=ctx_obj) - assert result.exit_code == 0, result.output - - # Renderer was never instantiated. - assert _RecordingRenderer.events == [] - - # And the default echo behavior fired. - json_lines = [ln for ln in result.output.splitlines() if ln.strip().startswith("{")] - assert len(json_lines) >= 1 - - def test_renderer_dispatches_per_function_and_verb(self) -> None: - seen: list[tuple[str, str]] = [] - - class _CLI(_NoOpCLI): - def get_function_renderer(self, fn_cls, *, verb): - seen.append((fn_cls.name, verb)) - return None # decline to render; just observe the dispatch - - app = _build_function_app(_CountFunction, _GreetFunction, cli=_CLI()) - runner.invoke(app, ["count", "run", "--upto", "1"]) +def test_job_submit_renderer_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None: + def submit_remote(self, job_cls, spec_data, **kwargs): # type: ignore[no-untyped-def] + del self, job_cls, kwargs + return {"submitted": spec_data} - # The hook fires per-verb on the invoked function only. - assert ("count", "run") in seen + class _CLI(_NoOpCLI): + def get_job_renderer(self, job_cls, *, verb): + return _RecordingRenderer if job_cls is _GreetJob and verb == "submit" else None + monkeypatch.setattr(commands.NemoJobScheduler, "submit_remote", submit_remote) + app = _build_job_app(_GreetJob, cli=_CLI()) + _RecordingRenderer.events = [] + result = runner.invoke(app, ["greet", "submit", "--name", "Renderer"]) -# --------------------------------------------------------------------------- -# Job: get_job_renderer for `run` -# --------------------------------------------------------------------------- + assert result.exit_code == 0, result.output + assert _RecordingRenderer.events == [ + ("start", {"verb": "submit", "is_local": False}), + ("frame", {"submitted": {"name": "Renderer"}}), + ("complete", None), + ] -class TestJobRunRenderer: - def test_no_renderer_falls_through_to_default(self) -> None: - app = _build_job_app(_GreetJob, cli=_NoOpCLI()) - result = runner.invoke(app, ["greet", "run", "--config", '{"name": "World"}']) - assert result.exit_code == 0 - assert json.loads(result.output) == {"message": "Hello, World!"} +def test_output_format_json_bypasses_job_renderer(monkeypatch: pytest.MonkeyPatch) -> None: + def submit_remote(self, job_cls, spec_data, **kwargs): # type: ignore[no-untyped-def] + del self, job_cls, kwargs + return {"submitted": spec_data} - def test_renderer_lifecycle_for_synchronous_run(self) -> None: - class _CLI(_NoOpCLI): - def get_job_renderer(self, job_cls, *, verb): - return _RecordingRenderer + class _CLI(_NoOpCLI): + def get_job_renderer(self, job_cls, *, verb): + del job_cls, verb + return _RecordingRenderer - app = _build_job_app(_GreetJob, cli=_CLI()) - _RecordingRenderer.events = [] - result = runner.invoke(app, ["greet", "run", "--config", '{"name": "Renderer"}']) - assert result.exit_code == 0, result.output - - events = _RecordingRenderer.events - names = [name for name, _ in events] - # start → one frame (the dict result) → complete. - assert names == ["start", "frame", "complete"] - # The frame is the dict the job returned. - _, frame = events[1] - assert frame == {"message": "Hello, Renderer!"} - - def test_output_format_json_bypasses_job_renderer(self) -> None: - class _CLI(_NoOpCLI): - def get_job_renderer(self, job_cls, *, verb): - return _RecordingRenderer - - app = _build_job_app(_GreetJob, cli=_CLI()) - ctx_obj = _typer_context_with_overrides(output_format="json") - _RecordingRenderer.events = [] - result = runner.invoke(app, ["greet", "run", "--config", '{"name": "X"}'], obj=ctx_obj) - assert result.exit_code == 0 - assert _RecordingRenderer.events == [] - # Default echo fired: - assert json.loads(result.output) == {"message": "Hello, X!"} + monkeypatch.setattr(commands.NemoJobScheduler, "submit_remote", submit_remote) + app = _build_job_app(_GreetJob, cli=_CLI()) + ctx_obj = _typer_context_with_overrides(output_format="json") + _RecordingRenderer.events = [] + result = runner.invoke(app, ["greet", "submit", "--name", "X"], obj=ctx_obj) + assert result.exit_code == 0 + assert _RecordingRenderer.events == [] + assert json.loads(result.output) == {"submitted": {"name": "X"}} -# --------------------------------------------------------------------------- -# "Delegate to use the renderer" contract -# --------------------------------------------------------------------------- +def test_function_submit_forwards_renderer_to_http_streamer(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} -class TestDelegateContract: - """A wrapper from update_function_cli that delegates to the original - callback gets the renderer for free; a wrapper that takes over the verb - body wholesale claims rendering responsibility too.""" - - def test_delegating_wrapper_still_drives_renderer(self) -> None: - """When the wrapper calls original(...), the framework's renderer - loop runs through the original's body — so the renderer fires.""" + def post_function_submit(url, body, **kwargs): # type: ignore[no-untyped-def] + captured["url"] = url + captured["body"] = body + captured["renderer_cls"] = kwargs["renderer_cls"] - class _CLI(_NoOpCLI): - def update_function_cli(self, fn_cls, group): - if fn_cls is not _CountFunction: - return - original = next(c for c in group.registered_commands if c.name == "run").callback - assert original is not None + class _CLI(_NoOpCLI): + def get_function_renderer(self, fn_cls, *, verb): + return _RecordingRenderer if fn_cls is _GreetFunction and verb == "submit" else None - @group.command("run") - def run(typer_ctx: typer.Context, count: int = typer.Option(2, "--count")) -> None: - spec_json = json.dumps({"upto": count}) - original(typer_ctx, spec=spec_json, spec_file=None, workspace="default") + monkeypatch.setattr(commands, "_post_function_submit", post_function_submit) + app = _build_function_app(_GreetFunction, cli=_CLI()) + result = runner.invoke(app, ["greet", "submit", "--name", "Fn", "--base-url", "https://nmp.test"]) - def get_function_renderer(self, fn_cls, *, verb): - return _RecordingRenderer if fn_cls is _CountFunction else None + assert result.exit_code == 0 + assert captured["url"] == "https://nmp.test/apis/plugin/v2/workspaces/default/greet" + assert captured["body"] == {"name": "Fn"} + assert captured["renderer_cls"] is _RecordingRenderer - app = _build_function_app(_CountFunction, cli=_CLI()) - _RecordingRenderer.events = [] - result = runner.invoke(app, ["count", "run", "--count", "1"]) - assert result.exit_code == 0, result.output - # Renderer fired for the delegated invocation. - events = _RecordingRenderer.events - names = [name for name, _ in events] - assert names[0] == "start" - assert names[-1] == "complete" - assert names.count("frame") == 2 # 1 heartbeat + 1 Done +def test_function_submit_output_format_json_bypasses_renderer(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} + def post_function_submit(url, body, **kwargs): # type: ignore[no-untyped-def] + del url, body + captured["renderer_cls"] = kwargs["renderer_cls"] -# --------------------------------------------------------------------------- -# on_error -# --------------------------------------------------------------------------- + class _CLI(_NoOpCLI): + def get_function_renderer(self, fn_cls, *, verb): + del fn_cls, verb + return _RecordingRenderer + monkeypatch.setattr(commands, "_post_function_submit", post_function_submit) + app = _build_function_app(_GreetFunction, cli=_CLI()) + ctx_obj = _typer_context_with_overrides(output_format="json") + result = runner.invoke(app, ["greet", "submit", "--name", "Fn"], obj=ctx_obj) -class _ExplodingSpec(BaseModel): - pass + assert result.exit_code == 0 + assert captured["renderer_cls"] is None -class _ExplodingFunction(NemoFunction[_ExplodingSpec]): - name: ClassVar[str] = "explode" - spec_schema: ClassVar[type[_ExplodingSpec]] = _ExplodingSpec - - async def run(self, spec: _ExplodingSpec) -> AsyncIterator[BaseModel]: - del spec - yield Heartbeat() +def test_delegating_job_submit_wrapper_still_drives_renderer(monkeypatch: pytest.MonkeyPatch) -> None: + def submit_remote(self, job_cls, spec_data, **kwargs): # type: ignore[no-untyped-def] + del self, job_cls, kwargs + return {"submitted": spec_data} + + class _CLI(_NoOpCLI): + def update_job_cli(self, job_cls, group): + if job_cls is not _GreetJob: + return + original = next(c for c in group.registered_commands if c.name == "submit").callback + assert original is not None + + @group.command("submit") + def submit(typer_ctx: typer.Context, name: str = typer.Option(..., "--name")) -> None: + original(typer_ctx, spec=json.dumps({"name": name}), spec_file=None) + + def get_job_renderer(self, job_cls, *, verb): + return _RecordingRenderer if job_cls is _GreetJob and verb == "submit" else None + + monkeypatch.setattr(commands.NemoJobScheduler, "submit_remote", submit_remote) + app = _build_job_app(_GreetJob, cli=_CLI()) + _RecordingRenderer.events = [] + result = runner.invoke(app, ["greet", "submit", "--name", "Wrapped"]) + + assert result.exit_code == 0, result.output + assert [name for name, _ in _RecordingRenderer.events] == ["start", "frame", "complete"] + + +def test_job_submit_renderer_on_error_fires(monkeypatch: pytest.MonkeyPatch) -> None: + def submit_remote(self, job_cls, spec_data, **kwargs): # type: ignore[no-untyped-def] + del self, job_cls, spec_data, kwargs raise RuntimeError("boom") + class _CLI(_NoOpCLI): + def get_job_renderer(self, job_cls, *, verb): + return _RecordingRenderer if job_cls is _GreetJob and verb == "submit" else None + + monkeypatch.setattr(commands.NemoJobScheduler, "submit_remote", submit_remote) + app = _build_job_app(_GreetJob, cli=_CLI()) + _RecordingRenderer.events = [] + result = runner.invoke(app, ["greet", "submit", "--name", "Err"]) -class TestOnError: - def test_on_error_fires_when_iteration_raises(self) -> None: - class _CLI(_NoOpCLI): - def get_function_renderer(self, fn_cls, *, verb): - return _RecordingRenderer - - app = _build_function_app(_ExplodingFunction, cli=_CLI()) - _RecordingRenderer.events = [] - result = runner.invoke(app, ["explode", "run"]) - - assert result.exit_code != 0 # exception still propagates - events = _RecordingRenderer.events - names = [name for name, _ in events] - assert "start" in names - assert "frame" in names # got the heartbeat before the explosion - assert "error" in names - # Exception type is RuntimeError. - error_payload = next(payload for name, payload in events if name == "error") - assert error_payload == "RuntimeError" - # on_complete should NOT fire on the error path. - assert "complete" not in names + assert result.exit_code != 0 + assert ("error", "RuntimeError") in _RecordingRenderer.events + assert "complete" not in [name for name, _ in _RecordingRenderer.events] diff --git a/packages/nemo_platform_plugin/tests/test_commands.py b/packages/nemo_platform_plugin/tests/test_commands.py index 534c63f8a1..5986d212de 100644 --- a/packages/nemo_platform_plugin/tests/test_commands.py +++ b/packages/nemo_platform_plugin/tests/test_commands.py @@ -1,48 +1,24 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for nemo_platform_plugin.commands. - -Phase 1 MR 1.2c replaces the single-verb ``nemo `` CLI with a -three-verb sub-group — ``run`` / ``submit`` / ``explain`` — and makes the -bare form print usage and exit non-zero. These tests pin: - -- Each job registers a sub-group under ````. -- The bare sub-group (no verb) prints usage and exits with status 1. -- ``run`` parses ``--config`` / ``--config-file`` and executes the job - in-process via :class:`NemoJobScheduler`. -- ``submit`` / ``explain`` delegate to the scheduler stubs and exit 2 with - a helpful error message in phase 1 (until MR 1.3 / MR 1.4 wire them). -- Invalid JSON on ``--config`` / ``--config-file`` exits 1 cleanly. -""" +"""Unit tests for the generated submit-only plugin CLI.""" from __future__ import annotations import json -import re from pathlib import Path -from types import SimpleNamespace -from typing import Any, AsyncIterator, ClassVar, cast +from typing import AsyncIterator -import httpx import pytest import typer +from nemo_platform_plugin import commands from nemo_platform_plugin.commands import add_function_commands, add_job_commands from nemo_platform_plugin.discovery import discover, discover_manifests from nemo_platform_plugin.function import NemoFunction -from nemo_platform_plugin.function_context import FunctionContext -from nemo_platform_plugin.functions.frames import Done, Heartbeat from nemo_platform_plugin.job import NemoJob -from pydantic import BaseModel, ValidationInfo, model_validator +from pydantic import BaseModel from typer.testing import CliRunner -_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") - - -def _plain(text: str) -> str: - """Strip ANSI escape codes from Rich/Typer output for robust matching.""" - return _ANSI_RE.sub("", text) - @pytest.fixture(autouse=True) def clear_discovery_cache(): @@ -53,48 +29,54 @@ def clear_discovery_cache(): discover_manifests.cache_clear() -# --------------------------------------------------------------------------- -# Fixture jobs -# --------------------------------------------------------------------------- +class _GreetSpec(BaseModel): + name: str + count: int = 1 class _GreetJob(NemoJob): name = "greet" - description = "Return a greeting." + description = "Submit a greeting job." + spec_schema = _GreetSpec def run(self, config: dict) -> dict: - return {"message": f"Hello, {config.get('name', 'world')}!"} - - -class _FailJob(NemoJob): - name = "fail" - description = "Always raises." - - def run(self, config: dict) -> dict: - raise RuntimeError("job exploded") + return {"message": f"Hello, {config['name']}!"} class _RunNamedJob(NemoJob): name = "run" - description = "A job whose name collides with the local run verb." + description = "A job whose name collides with the removed local verb." def run(self, config: dict) -> dict: return config -runner = CliRunner() +class _GreetFunction(NemoFunction): + name = "greet" + description = "Submit a greeting function." + spec_schema = _GreetSpec + async def run(self, spec: _GreetSpec) -> dict: + return {"message": f"Hello, {spec.name}!"} -def _typer_context_with_obj(obj: object | None) -> typer.Context: - return cast(typer.Context, SimpleNamespace(obj=obj)) +class _CountFunction(NemoFunction): + name = "count" + description = "Submit a streaming function." + spec_schema = _GreetSpec -def _app_with_jobs(*job_classes: type[NemoJob]) -> typer.Typer: - """Return a fresh Typer app with the given jobs injected. + async def run(self, spec: _GreetSpec) -> AsyncIterator[BaseModel]: + for idx in range(spec.count): + yield _GreetSpec(name=spec.name, count=idx) - A no-op callback is added so Typer keeps the app as a multi-command group - rather than collapsing it into a single-function app. - """ + +_GreetFunction.__module__ = "nemo_plugin.functions.greet" +_CountFunction.__module__ = "nemo_plugin.functions.count" + +runner = CliRunner() + + +def _app_with_jobs(*job_classes: type[NemoJob]) -> typer.Typer: app = typer.Typer() @app.callback() @@ -106,1225 +88,189 @@ def _noop() -> None: return app -# --------------------------------------------------------------------------- -# Sub-group registration -# --------------------------------------------------------------------------- - - -class TestSubgroupRegistration: - def test_registers_subgroup_for_each_job(self) -> None: - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["--help"]) - assert "greet" in result.output - - def test_jobs_rich_help_panel_label(self) -> None: - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["--help"]) - assert "Jobs" in result.output - - def test_registers_multiple_jobs(self) -> None: - app = _app_with_jobs(_GreetJob, _FailJob) - result = runner.invoke(app, ["--help"]) - assert "greet" in result.output - assert "fail" in result.output - - def test_no_jobs_leaves_app_unchanged(self) -> None: - app = typer.Typer() - - @app.command() - def existing() -> None: - """Existing command.""" - - add_job_commands(app, {}) - result = runner.invoke(app, ["--help"]) - assert "existing" in result.output - - def test_subgroup_lists_three_verbs(self) -> None: - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "--help"]) - assert result.exit_code == 0 - assert "run" in result.output - assert "submit" in result.output - assert "explain" in result.output - - def test_subgroup_help_includes_description(self) -> None: - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "--help"]) - assert "Return a greeting." in result.output - - def test_job_verb_help_text_does_not_repeat_job_name(self) -> None: - app = _app_with_jobs(_RunNamedJob) - result = runner.invoke(app, ["run", "--help"]) - - assert result.exit_code == 0 - assert "Run locally, in-process." in result.output - assert "Submit to a cluster." in result.output - assert "Show input/output schemas." in result.output - assert "Run run locally" not in result.output - assert "schemas for run" not in result.output - - -# --------------------------------------------------------------------------- -# Bare form — usage + non-zero exit -# --------------------------------------------------------------------------- - - -class TestBareFormBreaks: - def test_bare_job_name_exits_non_zero(self) -> None: - """`nemo ` with no verb must exit non-zero.""" - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet"]) - assert result.exit_code != 0 - - def test_bare_job_name_prints_usage(self) -> None: - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet"]) - # Usage should mention at least one of the verbs so the user knows - # what to type next. - assert "run" in result.output or "submit" in result.output - - -# --------------------------------------------------------------------------- -# run verb -# --------------------------------------------------------------------------- - - -class TestRunVerb: - def test_runs_job_with_json_config(self) -> None: - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "run", "--config", '{"name": "Claude"}']) - assert result.exit_code == 0 - output = json.loads(result.output) - assert output == {"message": "Hello, Claude!"} - - def test_runs_job_with_default_config(self) -> None: - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "run"]) - assert result.exit_code == 0 - output = json.loads(result.output) - assert output == {"message": "Hello, world!"} - - def test_runs_job_with_config_file(self, tmp_path: Path) -> None: - config_file = tmp_path / "config.json" - config_file.write_text('{"name": "File"}') - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "run", "--config-file", str(config_file)]) - assert result.exit_code == 0 - output = json.loads(result.output) - assert output == {"message": "Hello, File!"} - - def test_config_file_takes_precedence_over_config(self, tmp_path: Path) -> None: - config_file = tmp_path / "config.json" - config_file.write_text('{"name": "FromFile"}') - app = _app_with_jobs(_GreetJob) - result = runner.invoke( - app, - [ - "greet", - "run", - "--config", - '{"name": "Ignored"}', - "--config-file", - str(config_file), - ], - ) - assert result.exit_code == 0 - output = json.loads(result.output) - assert output["message"] == "Hello, FromFile!" - - def test_invalid_json_exits_with_error(self) -> None: - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "run", "--config", "not-json"]) - assert result.exit_code != 0 - - def test_job_exception_propagates(self) -> None: - app = _app_with_jobs(_FailJob) - with pytest.raises(RuntimeError, match="job exploded"): - runner.invoke(app, ["fail", "run", "--config", "{}"], catch_exceptions=False) - - -# --------------------------------------------------------------------------- -# submit verb — phase 1 MR 1.2c stubs -# --------------------------------------------------------------------------- - - -class TestSubmitVerb: - @pytest.mark.parametrize( - ("args", "env_base_url", "context_base_url", "expected_base_url"), - [ - ( - ["--base-url", "http://from-flag:9999", "--cluster", "configured-cluster"], - "http://from-env:1234", - "http://from-context:7777", - "http://from-flag:9999", - ), - ( - ["--cluster", "configured-cluster"], - "http://from-env:1234", - "http://from-context:7777", - "http://from-cluster:8888", - ), - ([], "http://from-env:1234", "http://from-context:7777", "http://from-context:7777"), - ([], "http://from-env:1234", None, "http://from-env:1234"), - ([], None, None, "http://localhost:8080"), - ], - ) - def test_submit_host_resolution_precedence( - self, - monkeypatch, - args: list[str], - env_base_url: str | None, - context_base_url: str | None, - expected_base_url: str, - ) -> None: - captured: dict[str, object] = {} - - def _capture(_self, _job_cls, _spec, *, base_url=None, **_kwargs) -> dict: - captured["base_url"] = base_url - return {"id": "job-123"} - - class _State: - def __init__(self, resolved_base_url: str | None) -> None: - self._resolved_base_url = resolved_base_url - - def get_base_url(self, default: str | None = None) -> str | None: - return self._resolved_base_url if self._resolved_base_url is not None else default - - class _FakeConfig: - def get_config_file(self) -> SimpleNamespace: - return SimpleNamespace( - clusters=[SimpleNamespace(name="configured-cluster", base_url="http://from-cluster:8888")] - ) - - if env_base_url is None: - monkeypatch.delenv("NMP_BASE_URL", raising=False) - else: - monkeypatch.setenv("NMP_BASE_URL", env_base_url) - monkeypatch.setattr("nemo_platform_plugin.scheduler.NemoJobScheduler.submit_remote", _capture) - monkeypatch.setattr("nemo_platform.config.config.Config.load", lambda: _FakeConfig()) - - app = _app_with_jobs(_GreetJob) - state = _State(context_base_url) - result = runner.invoke(app, ["greet", "submit", *args], obj=state) - - assert result.exit_code == 0, result.output - assert captured == {"base_url": expected_base_url} - - def test_submit_returns_exit_code_2_on_connect_error(self, monkeypatch) -> None: - request = httpx.Request("POST", "http://test/apis/tests/v2/workspaces/default/jobs/greet") - - def _raise_connect(*_args, **_kwargs) -> dict: - raise httpx.ConnectError("Connection refused", request=request) - - monkeypatch.setattr("nemo_platform_plugin.scheduler.NemoJobScheduler.submit_remote", _raise_connect) - - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "submit", "--base-url", "http://test"]) - - assert result.exit_code == 2 - combined = (result.output or "") + (result.stderr or "") - assert "Connection refused" in combined - assert "Request: POST http://test/apis/tests/v2/workspaces/default/jobs/greet" in combined - assert "Target: tests API route /apis/tests/v2/workspaces/default/jobs/greet" in combined - assert "nemo config view" in combined - assert "Traceback" not in combined - - def test_submit_accepts_profile_and_cluster_flags(self) -> None: - """Flags must be declared so MR 1.3 only needs to fill in behavior.""" - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "submit", "--help"]) - assert result.exit_code == 0 - output = _plain(result.output) - assert "--profile" in output - assert "--cluster" in output - - def test_submit_passes_cli_auth_headers(self, monkeypatch) -> None: - captured: dict[str, object] = {} - - def _capture(_self, _job_cls, _spec, headers=None, **_kwargs) -> dict: - captured["headers"] = headers - return {"id": "job-123"} - - class _State: - def get_sdk_context(self) -> SimpleNamespace: - return SimpleNamespace( - user=SimpleNamespace( - get_client_config=lambda: { - "default_headers": {"Authorization": "Bearer test-token"}, - } - ) - ) - - monkeypatch.setattr("nemo_platform_plugin.scheduler.NemoJobScheduler.submit_remote", _capture) - - app = _app_with_jobs(_GreetJob) - result = runner.invoke( - app, - ["greet", "submit", "--base-url", "http://127.0.0.1:8080"], - obj=_State(), - ) - - assert result.exit_code == 0, result.output - assert captured["headers"] == {"Authorization": "Bearer test-token"} - - -# --------------------------------------------------------------------------- -# explain verb — phase 1 MR 1.2c stubs -# --------------------------------------------------------------------------- - - -class TestExplainVerb: - def test_explain_works_without_cluster(self) -> None: - """explain reads locally — no base_url, no cluster needed.""" - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "explain"]) - assert result.exit_code == 0 - bundle = json.loads(result.output) - # endpoint is a template — {workspace} stays as a literal placeholder. - assert "{workspace}" in bundle["endpoint"] - assert bundle["endpoint"].endswith("/jobs/greet") - assert bundle["profile"] is None - assert bundle["profile_providers"] == [] - assert bundle["options"] == {} - - def test_explain_annotates_with_profile(self) -> None: - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "explain", "--profile", "research"]) - assert result.exit_code == 0 - bundle = json.loads(result.output) - assert bundle["profile"] == "research" - - def test_explain_accepts_profile_and_cluster_flags(self) -> None: - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "explain", "--help"]) - assert result.exit_code == 0 - output = _plain(result.output) - assert "--profile" in output - assert "--cluster" in output - - -# --------------------------------------------------------------------------- -# --spec / --spec-file rename (MR 1.3b) -# --------------------------------------------------------------------------- - - -class TestSpecFlagRename: - def test_run_accepts_spec_flag(self) -> None: - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "run", "--spec", '{"name": "Claude"}']) - assert result.exit_code == 0 - assert json.loads(result.output) == {"message": "Hello, Claude!"} - - def test_run_accepts_spec_file_yaml(self, tmp_path: Path) -> None: - spec_file = tmp_path / "spec.yaml" - spec_file.write_text("name: FromYaml\n") - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "run", "--spec-file", str(spec_file)]) - assert result.exit_code == 0 - assert json.loads(result.output)["message"] == "Hello, FromYaml!" - - def test_run_config_alias_still_works(self) -> None: - """--config remains as a deprecated alias for --spec during the transition.""" - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "run", "--config", '{"name": "Legacy"}']) - assert result.exit_code == 0 - assert json.loads(result.output)["message"] == "Hello, Legacy!" - - -# --------------------------------------------------------------------------- -# submit options passthrough (MR 1.3b) -# --------------------------------------------------------------------------- - - -class TestSubmitOptionsPassthrough: - def test_submit_accepts_dash_o_flag(self) -> None: - """The --help for submit must list -o so users can discover it.""" - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "submit", "--help"]) - assert result.exit_code == 0 - output = _plain(result.output) - assert "-o" in output - assert "--options-file" in output - assert "--spec-file" in output - - def test_submit_malformed_dash_o_exits_cleanly(self) -> None: - app = _app_with_jobs(_GreetJob) - result = runner.invoke( - app, - [ - "greet", - "submit", - "--profile", - "research", - "-o", - "slurm.partition", # missing =value - ], - ) - # Exit 1 from the options parser (malformed input), with error text. - assert result.exit_code != 0 - combined = (result.output or "") + (result.stderr or "") - assert "KEY=VALUE" in combined or "invalid -o entry" in combined - - def test_submit_malformed_options_file_exits_cleanly(self, tmp_path: Path) -> None: - bad_file = tmp_path / "opts.yaml" - bad_file.write_text("slurm:\n - not a mapping\n") - app = _app_with_jobs(_GreetJob) - result = runner.invoke( - app, - [ - "greet", - "submit", - "--options-file", - str(bad_file), - ], - ) - # The options file has a list under slurm; that merges fine (slurm - # becomes a list). Server-side would 422; the CLI itself tolerates - # the shape. Instead test a truly malformed case: top-level scalar. - scalar_file = tmp_path / "opts2.yaml" - scalar_file.write_text("just-a-string") - result = runner.invoke( - app, - [ - "greet", - "submit", - "--options-file", - str(scalar_file), - ], - ) - assert result.exit_code != 0 - combined = (result.output or "") + (result.stderr or "") - assert "top-level mapping" in combined - - -# --------------------------------------------------------------------------- -# NemoFunction CLI — fixtures -# --------------------------------------------------------------------------- +def _app_with_functions(*fn_classes: type[NemoFunction]) -> typer.Typer: + app = typer.Typer() + @app.callback() + def _noop() -> None: + pass -class _GreetSpec(BaseModel): - name: str + functions = {f"plugin.{cls.name}": cls for cls in fn_classes} + add_function_commands(app, functions) + return app -class _GreetResponse(BaseModel): - message: str +def test_job_subgroup_exposes_submit_and_explain_only() -> None: + app = _app_with_jobs(_GreetJob) + result = runner.invoke(app, ["greet", "--help"]) + assert result.exit_code == 0 + assert "submit" in result.output + assert "explain" in result.output + assert "Run locally" not in result.output -class _GreetFunction(NemoFunction[_GreetSpec]): - name: ClassVar[str] = "greet" - description: ClassVar[str] = "Say hello to a name." - spec_schema: ClassVar[type[BaseModel]] = _GreetSpec - async def run(self, spec: _GreetSpec) -> _GreetResponse: - return _GreetResponse(message=f"Hello, {spec.name}!") +def test_bare_job_name_exits_non_zero_and_points_at_submit() -> None: + app = _app_with_jobs(_GreetJob) + result = runner.invoke(app, ["greet"]) + assert result.exit_code != 0 + assert "submit" in result.output -class _CountSpec(BaseModel): - upto: int +def test_job_named_run_is_still_a_group_without_local_run_verb() -> None: + app = _app_with_jobs(_RunNamedJob) + result = runner.invoke(app, ["run", "--help"]) -class _CountFunction(NemoFunction[_CountSpec]): - name: ClassVar[str] = "count" - spec_schema: ClassVar[type[BaseModel]] = _CountSpec + assert result.exit_code == 0 + assert "submit" in result.output + assert "explain" in result.output + assert "Run locally" not in result.output - async def run(self, spec: _CountSpec) -> AsyncIterator[BaseModel]: - for i in range(spec.upto): - yield Heartbeat() - del i - yield Done() +def test_job_submit_uses_spec_json_and_field_overrides(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} -class _WorkspaceSpec(BaseModel): - pass + def submit_remote(self, job_cls, spec_data, **kwargs): # type: ignore[no-untyped-def] + captured["job_cls"] = job_cls + captured["spec"] = spec_data + captured["kwargs"] = kwargs + return {"id": "job-123", "status": "queued"} + monkeypatch.setattr(commands.NemoJobScheduler, "submit_remote", submit_remote) + app = _app_with_jobs(_GreetJob) + result = runner.invoke( + app, + [ + "greet", + "submit", + "--spec", + '{"name": "Base"}', + "--count", + "3", + "--base-url", + "https://nmp.test", + "--workspace", + "team", + "--profile", + "gpu", + "-o", + "slurm.nodes=2", + ], + ) -class _WorkspaceFunction(NemoFunction[_WorkspaceSpec]): - name: ClassVar[str] = "echo-workspace" - spec_schema: ClassVar[type[BaseModel]] = _WorkspaceSpec + assert result.exit_code == 0 + assert json.loads(result.output) == {"id": "job-123", "status": "queued"} + assert captured["job_cls"] is _GreetJob + assert captured["spec"] == {"name": "Base", "count": 3} + kwargs = captured["kwargs"] + assert isinstance(kwargs, dict) + assert kwargs["base_url"] == "https://nmp.test" + assert kwargs["workspace"] == "team" + assert kwargs["profile"] == "gpu" + assert kwargs["options"] == {"slurm": {"nodes": "2"}} - async def run(self, spec: _WorkspaceSpec, *, ctx: FunctionContext) -> dict: - del spec - return {"workspace": ctx.workspace} +def test_job_submit_accepts_spec_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} + spec_file = tmp_path / "spec.json" + spec_file.write_text('{"name": "File"}', encoding="utf-8") -class _LocalityFunction(NemoFunction[_WorkspaceSpec]): - name: ClassVar[str] = "echo-locality" - spec_schema: ClassVar[type[BaseModel]] = _WorkspaceSpec + def submit_remote(self, job_cls, spec_data, **kwargs): # type: ignore[no-untyped-def] + del self, job_cls, kwargs + captured["spec"] = spec_data + return {"id": "job-file"} - async def run(self, spec: _WorkspaceSpec, *, is_local: bool) -> dict: - del spec - return {"is_local": is_local} + monkeypatch.setattr(commands.NemoJobScheduler, "submit_remote", submit_remote) + app = _app_with_jobs(_GreetJob) + result = runner.invoke(app, ["greet", "submit", "--spec-file", str(spec_file)]) + assert result.exit_code == 0 + assert captured["spec"] == {"name": "File"} -def _app_with_functions(*function_classes: type[NemoFunction]) -> typer.Typer: - app = typer.Typer() - @app.callback() - def _noop() -> None: - pass +def test_job_submit_rejects_invalid_json() -> None: + app = _app_with_jobs(_GreetJob) + result = runner.invoke(app, ["greet", "submit", "--spec", "not-json"]) - fns = {f"plugin.{cls.name}": cls for cls in function_classes} - add_function_commands(app, fns) - return app + assert result.exit_code == 1 + assert "invalid spec" in result.output -# --------------------------------------------------------------------------- -# Function sub-group registration -# --------------------------------------------------------------------------- - - -class TestFunctionSubgroupRegistration: - def test_registers_subgroup_for_each_function(self) -> None: - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["--help"]) - assert "greet" in result.output - - def test_subgroup_lists_two_verbs_no_explain(self) -> None: - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet", "--help"]) - assert result.exit_code == 0 - assert "run" in result.output - assert "submit" in result.output - # Functions deliberately do NOT get an `explain` verb — schemas - # are introspected through `--help`. - assert "explain" not in result.output - - def test_functions_rich_help_panel_label(self) -> None: - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["--help"]) - assert "Functions" in result.output - - def test_bare_function_name_exits_non_zero(self) -> None: - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet"]) - assert result.exit_code != 0 - - -# --------------------------------------------------------------------------- -# Function `run` verb -# --------------------------------------------------------------------------- - - -class TestFunctionRunVerb: - def test_runs_function_with_json_spec(self) -> None: - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet", "run", "--spec", '{"name": "Claude"}']) - assert result.exit_code == 0, result.output - assert json.loads(result.output) == {"message": "Hello, Claude!"} - - def test_runs_function_with_spec_file(self, tmp_path: Path) -> None: - spec_file = tmp_path / "spec.yaml" - spec_file.write_text("name: FromYaml\n") - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet", "run", "--spec-file", str(spec_file)]) - assert result.exit_code == 0, result.output - assert json.loads(result.output)["message"] == "Hello, FromYaml!" - - def test_invalid_spec_exits_with_error(self) -> None: - # Validation runs against spec_schema before run is awaited, so an - # unknown-type value surfaces as a clean Typer exit, not a traceback. - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet", "run", "--spec", "{}"]) - assert result.exit_code != 0 - combined = (result.output or "") + (result.stderr or "") - assert "invalid spec" in combined - - def test_non_object_spec_exits_with_error(self) -> None: - # ``--spec '[]'`` is syntactically valid JSON but not a mapping; - # the deep-merge with per-field overlays would otherwise raise - # a raw TypeError. Reject it at load time with the same clean - # exit code 1 the malformed-JSON path uses. - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet", "run", "--spec", "[]"]) - assert result.exit_code == 1 - combined = (result.output or "") + (result.stderr or "") - assert "invalid spec" in combined - - def test_run_streams_async_generator_frames_one_per_line(self) -> None: - app = _app_with_functions(_CountFunction) - result = runner.invoke(app, ["count", "run", "--spec", '{"upto": 2}']) - assert result.exit_code == 0, result.output - lines = [line for line in result.output.splitlines() if line.strip()] - # 2 heartbeats + 1 terminator frame - assert len(lines) == 3 - kinds = [json.loads(line)["kind"] for line in lines] - assert kinds == ["heartbeat", "heartbeat", "done"] - - def test_run_injects_function_context_when_signature_asks(self) -> None: - # Functions opt in to FunctionContext by name; --workspace flows - # straight into ctx.workspace. - app = _app_with_functions(_WorkspaceFunction) - result = runner.invoke( - app, - ["echo-workspace", "run", "--spec", "{}", "--workspace", "team-alpha"], - ) - assert result.exit_code == 0, result.output - assert json.loads(result.output) == {"workspace": "team-alpha"} - - def test_run_injects_is_local_true_when_signature_asks(self) -> None: - app = _app_with_functions(_LocalityFunction) - result = runner.invoke(app, ["echo-locality", "run", "--spec", "{}"]) - assert result.exit_code == 0, result.output - assert json.loads(result.output) == {"is_local": True} - - def test_run_validates_spec_with_local_context(self) -> None: - class _LocalContextSpec(BaseModel): - @model_validator(mode="before") - @classmethod - def require_local_context(cls, data: Any, info: ValidationInfo) -> Any: - context = info.context - if not (isinstance(context, dict) and context.get("is_local") is True): - raise ValueError("missing local validation context") - return data - - class _LocalContextFunction(NemoFunction[_LocalContextSpec]): - name: ClassVar[str] = "local-context" - spec_schema: ClassVar[type[BaseModel]] = _LocalContextSpec - - async def run(self, spec: _LocalContextSpec) -> dict: - del spec - return {"ok": True} - - app = _app_with_functions(_LocalContextFunction) - result = runner.invoke(app, ["local-context", "run", "--spec", "{}"]) - - assert result.exit_code == 0, result.output - assert json.loads(result.output) == {"ok": True} - - -# --------------------------------------------------------------------------- -# Function `submit` verb -# --------------------------------------------------------------------------- - - -class TestFunctionSubmitVerb: - def test_submit_help_lists_expected_flags(self) -> None: - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet", "submit", "--help"]) - assert result.exit_code == 0 - output = _plain(result.output) - assert "--spec" in output - assert "--spec-file" in output - assert "--cluster" in output - assert "--base-url" in output - assert "--workspace" in output - # No `--profile` / `-o` for functions: those are job-only knobs. - assert "--profile" not in output - - def test_submit_invalid_spec_exits_before_network(self, monkeypatch) -> None: - # Spec validation happens before any HTTP machinery is touched, so - # a bad spec returns 1 (not 2 — that's reserved for network errors). - called: list[bool] = [] - - def _fail(*_args, **_kwargs) -> None: - called.append(True) - raise AssertionError("HTTP layer must not be reached for invalid spec") - - monkeypatch.setattr("nemo_platform_plugin.commands._post_function_submit", _fail) - - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet", "submit", "--spec", "{}"]) - assert result.exit_code == 1 - assert called == [] - - def test_submit_non_object_spec_exits_before_network(self, monkeypatch) -> None: - # Same guard as ``run``: a non-mapping ``--spec`` would otherwise - # crash the merge step before validation has a chance to fail. - called: list[bool] = [] - - def _fail(*_args, **_kwargs) -> None: - called.append(True) - raise AssertionError("HTTP layer must not be reached for invalid spec") - - monkeypatch.setattr("nemo_platform_plugin.commands._post_function_submit", _fail) - - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet", "submit", "--spec", "[]"]) - assert result.exit_code == 1 - assert called == [] - combined = (result.output or "") + (result.stderr or "") - assert "invalid spec" in combined - - def test_submit_posts_to_canonical_url_and_prints_json(self, monkeypatch) -> None: - captured_url: list[str] = [] - captured_body: list[dict] = [] - captured_headers: list[dict[str, str]] = [] - - def _fake_post(url: str, body: dict, *, headers: dict, timeout: float = 30.0, **_kwargs) -> None: - captured_url.append(url) - captured_body.append(body) - captured_headers.append(dict(headers)) - del timeout - typer.echo(json.dumps({"message": "ok"})) - - monkeypatch.setattr("nemo_platform_plugin.commands._post_function_submit", _fake_post) - - app = _app_with_functions(_GreetFunction) - result = runner.invoke( - app, - [ - "greet", - "submit", - "--spec", - '{"name": "Ada"}', - "--base-url", - "http://my-platform:9090", - "--workspace", - "team-alpha", - "--request-id", - "req-42", - ], - ) - assert result.exit_code == 0, result.output - # Canonical URL shape from `resources-jobs-functions.md`. The - # plugin segment is derived from the function's module name — - # tests/test_commands.py lives under `tests`, so the segment - # falls back to that (good — it pins the convention). - assert captured_url[0].endswith("/v2/workspaces/team-alpha/greet") - assert "/apis/" in captured_url[0] - assert captured_body[0] == {"name": "Ada"} - assert captured_headers[0]["X-Request-ID"] == "req-42" - - def test_submit_streams_ndjson_lines(self, monkeypatch) -> None: - # Build an httpx.MockTransport that returns a chunked NDJSON body - # so we exercise the real streaming branch in _post_function_submit. - ndjson_body = json.dumps({"kind": "heartbeat"}) + "\n" + json.dumps({"kind": "done"}) + "\n" - - def _handler(request: httpx.Request) -> httpx.Response: - return httpx.Response( - 200, - content=ndjson_body, - headers={"content-type": "application/x-ndjson"}, - ) - - transport = httpx.MockTransport(_handler) - - original_client = httpx.Client - - def _client_factory(*args, **kwargs): - kwargs.setdefault("transport", transport) - return original_client(*args, **kwargs) - - monkeypatch.setattr("nemo_platform_plugin.commands.httpx.Client", _client_factory) - - app = _app_with_functions(_CountFunction) - result = runner.invoke( - app, - ["count", "submit", "--spec", '{"upto": 1}', "--base-url", "http://test"], - ) - assert result.exit_code == 0, result.output - lines = [line for line in result.output.splitlines() if line.strip()] - kinds = [json.loads(line)["kind"] for line in lines] - assert kinds == ["heartbeat", "done"] - - def test_submit_returns_exit_code_2_on_http_error(self, monkeypatch) -> None: - # Regression: ``client.stream`` opens the response unbuffered, - # so the error formatter that reads ``exc.response.text`` used - # to blow up with ``ResponseNotRead`` and propagate a raw - # traceback to the user instead of the structured error - # message. - # - # Crucially, this handler uses ``stream=httpx.ByteStream(...)`` - # rather than the more obvious ``json=...`` / ``content=...`` - # — those constructors pre-populate ``_content`` on the - # response, which means the buggy code reads from the - # pre-buffered bytes and never reproduces ``ResponseNotRead``. - # ``ByteStream`` defers the bytes until ``response.read()`` is - # called, matching production wire behaviour. - def _handler(request: httpx.Request) -> httpx.Response: - return httpx.Response( - 500, - headers={"content-type": "application/json"}, - stream=httpx.ByteStream(b'{"detail": "boom"}'), - ) - - transport = httpx.MockTransport(_handler) - original_client = httpx.Client - - def _client_factory(*args, **kwargs): - kwargs.setdefault("transport", transport) - return original_client(*args, **kwargs) - - monkeypatch.setattr("nemo_platform_plugin.commands.httpx.Client", _client_factory) - - app = _app_with_functions(_GreetFunction) - result = runner.invoke( - app, - ["greet", "submit", "--spec", '{"name": "x"}', "--base-url", "http://test"], - ) - # Exit 2 marks "transport / server reported failure" — distinct - # from exit 1 (CLI-side validation) so wrapper scripts can branch. - assert result.exit_code == 2 - combined = (result.output or "") + (result.stderr or "") - assert "500" in combined - assert "boom" in combined - assert "Request: POST http://test/" in combined - assert "Target:" not in combined - - def test_submit_uses_nmp_base_url_env_when_no_flags(self, monkeypatch) -> None: - captured_url: list[str] = [] - - def _fake_post(url: str, body: dict, *, headers: dict, timeout: float = 30.0, **_kwargs) -> None: # noqa: ARG001 - captured_url.append(url) - - monkeypatch.setattr("nemo_platform_plugin.commands._post_function_submit", _fake_post) - monkeypatch.setenv("NMP_BASE_URL", "http://from-env:1234") - - app = _app_with_functions(_GreetFunction) - result = runner.invoke( - app, - ["greet", "submit", "--spec", '{"name": "x"}'], - ) - assert result.exit_code == 0, result.output - assert captured_url[0].startswith("http://from-env:1234/") - - -# --------------------------------------------------------------------------- -# Function URL derivation -# --------------------------------------------------------------------------- - - -class TestApiSegmentForFunction: - """Pin how the ``submit`` URL maps a function class to its API segment. - - The platform mounts each plugin's functions under the ```` - half of its ``nemo.functions`` entry-point key (``.``), - so the CLI has to derive the same prefix or every ``submit`` 404s. - The authoritative source is the registered entry-point — module - paths are only consulted when the function isn't installed - (in-process tests, scratch invocations). - """ - - def test_uses_registered_entry_point_key(self, monkeypatch) -> None: - from nemo_platform_plugin.commands import _api_segment_for_function - - class _Spec(BaseModel): - pass - - class _Fn(NemoFunction[_Spec]): - name: ClassVar[str] = "greet" - spec_schema: ClassVar[type[BaseModel]] = _Spec - - async def run(self, spec: _Spec) -> dict: - del spec - return {} - - # Pretend this class is the value behind a ``my-plugin.greet`` - # entry-point. Patching ``discover_functions`` keeps the test - # hermetic and exercises exactly the lookup branch — the - # ``-plugin`` suffix in the key is kept verbatim, even though - # the package layout would suggest a different segment. - _Fn.__module__ = "nemo_my_plugin.functions.greet" - monkeypatch.setattr( - "nemo_platform_plugin.discovery.discover_functions", - lambda: {"my-plugin.greet": _Fn}, - ) - assert _api_segment_for_function(_Fn) == "my-plugin" - - def test_falls_back_to_module_when_not_registered(self, monkeypatch) -> None: - from nemo_platform_plugin.commands import _api_segment_for_function - - class _Spec(BaseModel): - pass - - class _Fn(NemoFunction[_Spec]): - name: ClassVar[str] = "greet" - spec_schema: ClassVar[type[BaseModel]] = _Spec - - async def run(self, spec: _Spec) -> dict: - del spec - return {} - - # No entry-point match → module-name fallback. The fallback no - # longer strips ``_plugin``; an unregistered class in a - # ``nemo__plugin`` package keeps the suffix so the - # CLI fails loudly rather than 404ing against the wrong URL. - _Fn.__module__ = "nemo_example_plugin.functions.greet" - monkeypatch.setattr( - "nemo_platform_plugin.discovery.discover_functions", - lambda: {}, - ) - assert _api_segment_for_function(_Fn) == "example-plugin" - - def test_handles_missing_nemo_prefix(self, monkeypatch) -> None: - from nemo_platform_plugin.commands import _api_segment_for_function - - class _Spec(BaseModel): - pass - - class _Fn(NemoFunction[_Spec]): - name: ClassVar[str] = "x" - spec_schema: ClassVar[type[BaseModel]] = _Spec - - async def run(self, spec: _Spec) -> dict: - del spec - return {} - - # In-tree code outside ``nemo_*`` keeps its module name as-is - # (kebab-cased) so tests with inline classes still produce a - # stable, predictable segment. - _Fn.__module__ = "tests.fixtures.things" - monkeypatch.setattr( - "nemo_platform_plugin.discovery.discover_functions", - lambda: {}, - ) - assert _api_segment_for_function(_Fn) == "tests" - - -# --------------------------------------------------------------------------- -# Function CLI — auto-generated per-field flags from spec_schema -# --------------------------------------------------------------------------- +def test_job_submit_rejects_removed_run_verb() -> None: + app = _app_with_jobs(_GreetJob) + result = runner.invoke(app, ["greet", "run", "--spec", '{"name": "X"}']) + assert result.exit_code != 0 + assert "No such command" in result.output -class _NestedTarget(BaseModel): - url: str - timeout_seconds: int = 30 - - -class _NestedSpec(BaseModel): - name: str - target: _NestedTarget - - -class _NestedFunction(NemoFunction[_NestedSpec]): - name: ClassVar[str] = "ping" - description: ClassVar[str] = "Ping a nested target." - spec_schema: ClassVar[type[BaseModel]] = _NestedSpec - - async def run(self, spec: _NestedSpec) -> dict: - return { - "name": spec.name, - "url": spec.target.url, - "timeout": spec.target.timeout_seconds, - } - - -class TestFunctionAutoSpecFlags: - """Per-field flags auto-derived from a function's ``spec_schema``. - - The wiring is shared with the jobs CLI via - :mod:`nemo_platform_plugin._spec_flags`; these tests pin the wiring on the - function side specifically so a future refactor can't silently - break the function ergonomics. - """ - - def test_run_help_lists_one_flag_per_scalar_leaf(self) -> None: - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet", "run", "--help"]) - assert result.exit_code == 0 - plain = _plain(result.output) - assert "--name" in plain - assert "Function Spec" in plain - # The epilog tells the user where the flags came from — without - # this, "schema discovery" still requires reading source. - assert "GreetSpec" in plain - - def test_run_accepts_per_field_flag(self) -> None: - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet", "run", "--name", "Razvan"]) - assert result.exit_code == 0, result.output - assert json.loads(result.output) == {"message": "Hello, Razvan!"} - - def test_per_field_flag_overlays_on_top_of_spec(self) -> None: - app = _app_with_functions(_GreetFunction) - result = runner.invoke( - app, - [ - "greet", - "run", - "--spec", - '{"name": "from-spec"}', - "--name", - "from-flag", - ], - ) - assert result.exit_code == 0, result.output - assert json.loads(result.output) == {"message": "Hello, from-flag!"} - - def test_nested_field_uses_dotted_flag_name(self) -> None: - # ``target.url`` and ``target.timeout-seconds`` are the canonical - # rendering of a nested submodel field. Underscores within a - # segment kebab-case; the dot between segments is preserved. - app = _app_with_functions(_NestedFunction) - result = runner.invoke(app, ["ping", "run", "--help"]) - assert result.exit_code == 0 - plain = _plain(result.output) - assert "--name" in plain - assert "--target.url" in plain - assert "--target.timeout-seconds" in plain - - def test_nested_field_overlay_round_trips(self) -> None: - app = _app_with_functions(_NestedFunction) - result = runner.invoke( - app, - [ - "ping", - "run", - "--name", - "site-a", - "--target.url", - "https://example.test", - "--target.timeout-seconds", - "5", - ], - ) - assert result.exit_code == 0, result.output - assert json.loads(result.output) == { - "name": "site-a", - "url": "https://example.test", - "timeout": 5, - } - - def test_workspace_field_in_spec_does_not_collide_with_static_flag(self) -> None: - # A spec field literally named ``workspace`` would alias the - # static ``--workspace`` flag (which feeds ``ctx.workspace``, - # not the spec). The reserved-flag set drops it from the - # auto-generated panel; users still pass it via --spec. - class _ConfusingSpec(BaseModel): - workspace: str = "default-ws" - - class _ConfusingFunction(NemoFunction[_ConfusingSpec]): - name: ClassVar[str] = "confuse" - spec_schema: ClassVar[type[BaseModel]] = _ConfusingSpec - - async def run(self, spec: _ConfusingSpec) -> dict: - return {"in_spec": spec.workspace} - - app = _app_with_functions(_ConfusingFunction) - help_result = runner.invoke(app, ["confuse", "run", "--help"]) - plain = _plain(help_result.output) - # The flag exists exactly once — the static-input version - # under the "Spec Source" panel, not duplicated under the - # "Function Spec" panel. - assert plain.count("--workspace") == 1 - - # And --spec still wins for the spec-side workspace value. - run_result = runner.invoke( - app, - ["confuse", "run", "--spec", '{"workspace": "from-spec"}', "--workspace", "ctx-ws"], - ) - assert run_result.exit_code == 0, run_result.output - assert json.loads(run_result.output) == {"in_spec": "from-spec"} - - def test_submit_help_lists_per_field_flags_under_function_spec_panel(self) -> None: - app = _app_with_functions(_GreetFunction) - result = runner.invoke(app, ["greet", "submit", "--help"]) - plain = _plain(result.output) - assert "--name" in plain - assert "Function Spec" in plain - # `--workspace` is a submission-side flag (URL segment), so it - # stays in the Submission panel and isn't auto-derived even if - # a spec field happened to share the name. - assert "Submission" in plain - - def test_no_spec_schema_fields_falls_back_to_no_flags_epilog(self) -> None: - # Functions with an empty spec_schema (or only unsupported - # types) get the "no per-field flags" epilog so users still - # learn how to pass values. - class _EmptySpec(BaseModel): - pass - - class _EmptyFunction(NemoFunction[_EmptySpec]): - name: ClassVar[str] = "noop" - spec_schema: ClassVar[type[BaseModel]] = _EmptySpec - - async def run(self, spec: _EmptySpec) -> dict: - del spec - return {} - - app = _app_with_functions(_EmptyFunction) - result = runner.invoke(app, ["noop", "run", "--help"]) - plain = _plain(result.output) - assert "Function Spec" not in plain - assert "no per-field flags" in plain - - -# --------------------------------------------------------------------------- -# Job CLI — auto-generated per-field flags from spec_schema -# --------------------------------------------------------------------------- - - -class _GreetJobSpec(BaseModel): - name: str = "world" - loud: bool = False - - -class _GreetSpecJob(NemoJob): - name = "greet-spec" - description = "Return a greeting validated against a schema." - spec_schema: ClassVar[type[BaseModel]] = _GreetJobSpec - def run(self, config: dict) -> dict: - spec = _GreetJobSpec.model_validate(config) - message = f"Hello, {spec.name}!" - if spec.loud: - message = message.upper() - return {"message": message} +def test_explain_renders_schema() -> None: + app = _app_with_jobs(_GreetJob) + result = runner.invoke(app, ["greet", "explain"]) + assert result.exit_code == 0 + output = json.loads(result.output) + assert output["job_key"].endswith(".greet") + assert output["spec_schema"]["properties"]["name"]["type"] == "string" -class _NestedJobTarget(BaseModel): - url: str - timeout_seconds: int = 30 +def test_function_subgroup_exposes_submit_only() -> None: + app = _app_with_functions(_GreetFunction) + result = runner.invoke(app, ["greet", "--help"]) -class _NestedJobSpec(BaseModel): - name: str - target: _NestedJobTarget + assert result.exit_code == 0 + assert "submit" in result.output + assert "Run locally" not in result.output -class _NestedSpecJob(NemoJob): - name = "ping-spec" - description = "Ping a nested target." - spec_schema: ClassVar[type[BaseModel]] = _NestedJobSpec +def test_function_submit_posts_to_function_endpoint(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, object] = {} - def run(self, config: dict) -> dict: - spec = _NestedJobSpec.model_validate(config) - return { - "name": spec.name, - "url": spec.target.url, - "timeout": spec.target.timeout_seconds, - } - - -class TestJobAutoSpecFlags: - """Per-field flags auto-derived from a job's ``spec_schema`` on ``run``/``submit``. - - Mirrors :class:`TestFunctionAutoSpecFlags` so the two CLIs stay in - lockstep — the wiring is shared via :mod:`nemo_platform_plugin._spec_flags`. - """ - - def test_run_help_lists_one_flag_per_scalar_leaf(self) -> None: - app = _app_with_jobs(_GreetSpecJob) - result = runner.invoke(app, ["greet-spec", "run", "--help"]) - assert result.exit_code == 0 - plain = _plain(result.output) - assert "--name" in plain - assert "--loud" in plain - assert "Job Spec" in plain - - def test_run_accepts_per_field_flag(self) -> None: - app = _app_with_jobs(_GreetSpecJob) - result = runner.invoke(app, ["greet-spec", "run", "--name", "Razvan"]) - assert result.exit_code == 0, result.output - assert json.loads(result.output) == {"message": "Hello, Razvan!"} - - def test_per_field_flag_overlays_on_top_of_spec(self) -> None: - # --spec sets the base; --name overlays on top per the - # documented precedence (spec-file → spec → per-field). - app = _app_with_jobs(_GreetSpecJob) - result = runner.invoke( - app, - [ - "greet-spec", - "run", - "--spec", - '{"name": "from-spec", "loud": true}', - "--name", - "from-flag", - ], - ) - assert result.exit_code == 0, result.output - # Loud carries over from --spec; name is overridden by the flag. - assert json.loads(result.output) == {"message": "HELLO, FROM-FLAG!"} - - def test_nested_field_uses_dotted_flag_name(self) -> None: - app = _app_with_jobs(_NestedSpecJob) - result = runner.invoke(app, ["ping-spec", "run", "--help"]) - assert result.exit_code == 0 - plain = _plain(result.output) - assert "--name" in plain - assert "--target.url" in plain - assert "--target.timeout-seconds" in plain - - def test_nested_field_overlay_round_trips(self) -> None: - app = _app_with_jobs(_NestedSpecJob) - result = runner.invoke( - app, - [ - "ping-spec", - "run", - "--name", - "site-a", - "--target.url", - "https://example.test", - "--target.timeout-seconds", - "5", - ], - ) - assert result.exit_code == 0, result.output - assert json.loads(result.output) == { - "name": "site-a", - "url": "https://example.test", - "timeout": 5, - } - - def test_workspace_field_in_spec_does_not_collide_with_static_flag(self) -> None: - # A spec field literally named ``workspace`` would alias the - # ``submit``-side ``--workspace`` flag (which feeds the URL - # segment, not the spec). The reserved-flag set drops it from - # the auto-generated ``submit`` panel; users still pass it via - # --spec. ``run`` doesn't reserve ``workspace`` (it has no such - # static flag), so the auto-flag still appears there. - class _WorkspaceSpec(BaseModel): - workspace: str = "default-ws" - - class _WorkspaceJob(NemoJob): - name = "ws-confuse" - spec_schema: ClassVar[type[BaseModel]] = _WorkspaceSpec - - def run(self, config: dict) -> dict: - return {"in_spec": _WorkspaceSpec.model_validate(config).workspace} - - app = _app_with_jobs(_WorkspaceJob) - submit_help = runner.invoke(app, ["ws-confuse", "submit", "--help"]) - plain = _plain(submit_help.output) - # The flag exists exactly once on ``submit`` — the static - # submission-side version under the "Submission" panel, not - # duplicated under the "Job Spec" panel. - assert plain.count("--workspace") == 1 - - def test_submit_help_lists_per_field_flags_under_job_spec_panel(self) -> None: - app = _app_with_jobs(_GreetSpecJob) - result = runner.invoke(app, ["greet-spec", "submit", "--help"]) - plain = _plain(result.output) - assert "--name" in plain - assert "Job Spec" in plain - # ``--profile`` / ``-o`` / ``--workspace`` remain visible under - # the Submission panel — auto-flags don't displace static ones. - assert "Submission" in plain - assert "--profile" in plain - - def test_no_spec_schema_renders_only_static_panels(self) -> None: - # Schema-less jobs (``_GreetJob``) don't surface a "Job Spec" - # panel — the user passes values exclusively via --spec / - # --spec-file under the "Spec Source" panel. - app = _app_with_jobs(_GreetJob) - result = runner.invoke(app, ["greet", "run", "--help"]) - plain = _plain(result.output) - assert "Job Spec" not in plain - assert "Spec Source" in plain - - def test_input_spec_schema_drives_auto_flags_when_declared(self) -> None: - # When a job declares both ``input_spec_schema`` and - # ``spec_schema``, the input shape is what the user types — so - # that's the schema the CLI must walk. - class _InputShape(BaseModel): - target_name: str - - class _CanonicalShape(BaseModel): - resolved_id: str - - class _TwoShapeJob(NemoJob): - name = "two-shape" - input_spec_schema: ClassVar[type[BaseModel]] = _InputShape - spec_schema: ClassVar[type[BaseModel]] = _CanonicalShape - - def run(self, config: dict) -> dict: - return {"got": config} - - app = _app_with_jobs(_TwoShapeJob) - result = runner.invoke(app, ["two-shape", "run", "--help"]) - plain = _plain(result.output) - # The flag follows ``input_spec_schema``, not ``spec_schema``. - assert "--target-name" in plain - assert "--resolved-id" not in plain + def post_function_submit(url, body, **kwargs): # type: ignore[no-untyped-def] + captured["url"] = url + captured["body"] = body + captured["kwargs"] = kwargs + + monkeypatch.setattr(commands, "_post_function_submit", post_function_submit) + app = _app_with_functions(_GreetFunction) + result = runner.invoke( + app, + [ + "greet", + "submit", + "--name", + "Claude", + "--count", + "2", + "--base-url", + "https://nmp.test", + "--workspace", + "team", + "--request-id", + "req-1", + ], + ) + + assert result.exit_code == 0 + assert captured["url"] == "https://nmp.test/apis/plugin/v2/workspaces/team/greet" + assert captured["body"] == {"name": "Claude", "count": 2} + kwargs = captured["kwargs"] + assert isinstance(kwargs, dict) + assert kwargs["headers"] == {"X-Request-ID": "req-1"} + + +def test_function_submit_rejects_invalid_spec_shape() -> None: + app = _app_with_functions(_GreetFunction) + result = runner.invoke(app, ["greet", "submit", "--spec", "[]"]) + + assert result.exit_code == 1 + assert "invalid spec" in result.output + + +def test_function_submit_rejects_removed_run_verb() -> None: + app = _app_with_functions(_CountFunction) + result = runner.invoke(app, ["count", "run", "--name", "X"]) + + assert result.exit_code != 0 + assert "No such command" in result.output diff --git a/packages/nemo_platform_plugin/tests/test_dispatcher.py b/packages/nemo_platform_plugin/tests/test_dispatcher.py index deb58ea686..9f502452fa 100644 --- a/packages/nemo_platform_plugin/tests/test_dispatcher.py +++ b/packages/nemo_platform_plugin/tests/test_dispatcher.py @@ -336,11 +336,11 @@ def run(self, config: dict) -> dict: assert "requires sdk=" in caplog.text def test_local_run_error_propagates(self, monkeypatch, tmp_path: Path) -> None: - # ``LocalRunError`` from ``resolve_run_kwargs`` indicates a + # ``RunDependencyError`` from ``resolve_run_kwargs`` indicates a # plugin-author bug (run declares ``sdk`` as required but the # caller didn't pass one). Propagating beats collapsing it into # the same exit-2 bucket as a missing env var. - from nemo_platform_plugin.run_dependencies import LocalRunError + from nemo_platform_plugin.run_dependencies import RunDependencyError _setup_env(monkeypatch, tmp_path, step_config={}) # Pass ``ctx`` explicitly so the dispatcher's own sdk-requirement @@ -358,12 +358,12 @@ class _Job(NemoJob): def run(self, config: dict, *, sdk) -> dict: # ty: ignore[invalid-method-override] return {"status": "completed"} - with pytest.raises(LocalRunError, match="sdk"): - run_task(_Job, ctx=ctx) # no sdk passed → LocalRunError + with pytest.raises(RunDependencyError, match="sdk"): + run_task(_Job, ctx=ctx) # no sdk passed → RunDependencyError def test_local_run_error_from_job_run_propagates(self, monkeypatch, tmp_path: Path) -> None: - # LocalRunError from job.run must propagate, not collapse to exit 1. - from nemo_platform_plugin.run_dependencies import LocalRunError + # RunDependencyError from job.run must propagate, not collapse to exit 1. + from nemo_platform_plugin.run_dependencies import RunDependencyError _setup_env(monkeypatch, tmp_path, step_config={}) @@ -371,14 +371,14 @@ class _Job(NemoJob): name = "raises-local-run-error" def run(self, config: dict) -> dict: - raise LocalRunError("missing sdk for fileset upload") + raise RunDependencyError("missing sdk for fileset upload") - with pytest.raises(LocalRunError, match="fileset upload"): + with pytest.raises(RunDependencyError, match="fileset upload"): run_task(_Job, sdk=_DEFAULT_SDK) def test_unsupported_required_run_param_raises_local_run_error(self, monkeypatch, tmp_path: Path) -> None: - # Unknown required run() param surfaces as LocalRunError, not TypeError. - from nemo_platform_plugin.run_dependencies import LocalRunError + # Unknown required run() param surfaces as RunDependencyError, not TypeError. + from nemo_platform_plugin.run_dependencies import RunDependencyError _setup_env(monkeypatch, tmp_path, step_config={}) @@ -388,7 +388,7 @@ class _Job(NemoJob): def run(self, config: dict, *, foo) -> dict: # ty: ignore[invalid-method-override] return {"status": "completed", "foo": foo} - with pytest.raises(LocalRunError, match="foo"): + with pytest.raises(RunDependencyError, match="foo"): run_task(_Job, sdk=_DEFAULT_SDK) diff --git a/packages/nemo_platform_plugin/tests/test_job_context.py b/packages/nemo_platform_plugin/tests/test_job_context.py index 9f188f88d7..8abab5bba7 100644 --- a/packages/nemo_platform_plugin/tests/test_job_context.py +++ b/packages/nemo_platform_plugin/tests/test_job_context.py @@ -6,7 +6,7 @@ Pin the :class:`JobContext` contract: - Concrete dataclass — construction is always explicit. -- ``job_id`` is ``str | None``; ``None`` means "purely local run". +- ``job_id`` is ``str | None``; ``None`` means no platform job id was supplied. - ``results`` is typed :class:`JobResults` (sync) — there is no async twin; ``NemoJob.run`` runs in the task container where ``save`` is invoked synchronously. @@ -29,7 +29,7 @@ def _make_storage(tmp_path: Path) -> StoragePaths: class TestJobContext: - def test_job_id_defaults_to_none_for_local_runs(self, tmp_path: Path) -> None: + def test_job_id_defaults_to_none_when_not_supplied(self, tmp_path: Path) -> None: ctx = JobContext( workspace="ws", storage=_make_storage(tmp_path), @@ -37,7 +37,7 @@ def test_job_id_defaults_to_none_for_local_runs(self, tmp_path: Path) -> None: ) assert ctx.job_id is None - def test_job_id_can_be_set_for_platform_runs(self, tmp_path: Path) -> None: + def test_job_id_can_be_set_for_platform_jobs(self, tmp_path: Path) -> None: ctx = JobContext( workspace="ws", storage=_make_storage(tmp_path), diff --git a/packages/nemo_platform_plugin/tests/test_run_dependencies.py b/packages/nemo_platform_plugin/tests/test_run_dependencies.py index 2f6a03cab5..64fef5e9cd 100644 --- a/packages/nemo_platform_plugin/tests/test_run_dependencies.py +++ b/packages/nemo_platform_plugin/tests/test_run_dependencies.py @@ -12,7 +12,7 @@ def test_run_dependencies_exports_public_run_dependency_helpers() -> None: - assert run_dependencies.__all__ == ["LocalRunError", "resolve_run_kwargs"] + assert run_dependencies.__all__ == ["RunDependencyError", "resolve_run_kwargs"] def test_dispatcher_import_does_not_import_scheduler() -> None: diff --git a/packages/nemo_platform_plugin/tests/test_scheduler.py b/packages/nemo_platform_plugin/tests/test_scheduler.py index d1dbd0005d..8b6197d90c 100644 --- a/packages/nemo_platform_plugin/tests/test_scheduler.py +++ b/packages/nemo_platform_plugin/tests/test_scheduler.py @@ -1,38 +1,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests for :mod:`nemo_platform_plugin.scheduler` — the three-verb scheduler. - -Phase 1 wires :meth:`NemoJobScheduler.run_local`, :meth:`submit_remote`, and -:meth:`explain`. These tests pin: - -- Legacy ``run(config: dict)`` jobs without ``spec_schema`` receive the raw - dict unchanged. -- Jobs with ``spec_schema`` receive a validated, defaults-merged dict. -- Jobs with ``input_spec_schema`` + ``to_spec`` receive the transformed - canonical dict (re-validated against ``spec_schema``). -- The constructed :class:`JobContext` carries the expected - workspace / job_id / storage shape and exposes the in-container runtime - envvars to ``run`` for backwards compatibility with legacy task code. -- ``submit_remote`` builds the right URL, body, and headers and POSTs to - the configured base URL. -- ``explain`` returns the OpenAPI-shaped descriptor expected by the CLI. +"""Tests for :mod:`nemo_platform_plugin.scheduler`. + +The public scheduler surface submits jobs remotely and explains job schemas. +These tests pin URL/body construction, HTTP behavior, and schema extraction. """ from __future__ import annotations -from pathlib import Path -from typing import Any, cast +from typing import cast import httpx import pytest from nemo_platform import AsyncNeMoPlatform from nemo_platform_plugin.job import NemoJob -from nemo_platform_plugin.job_context import JobContext, StoragePaths -from nemo_platform_plugin.job_results import LocalJobResults -from nemo_platform_plugin.run_dependencies import LocalRunError from nemo_platform_plugin.scheduler import NemoJobScheduler -from pydantic import BaseModel, ValidationError, ValidationInfo, model_validator +from pydantic import BaseModel # --------------------------------------------------------------------------- # Fixture jobs @@ -49,374 +33,6 @@ def run(self, config: dict) -> dict: return {"got": config} -class _SpecSchemaOnlySpec(BaseModel): - name: str - count: int = 1 - - -class _SpecSchemaOnlyJob(NemoJob): - """Declares spec_schema only — defaults are filled in.""" - - name = "spec-only" - description = "Job with a canonical spec." - spec_schema = _SpecSchemaOnlySpec - - def run(self, config: dict) -> dict: - return {"name": config["name"], "count": config["count"]} - - -class _CanonicalSpec(BaseModel): - resolved_id: str - count: int - - -class _InputSpec(BaseModel): - raw_name: str - count: int = 1 - - -class _InputTransformJob(NemoJob): - """Declares both shapes and a ``to_spec`` transform. - - ``to_spec`` is the ``async classmethod`` shape — runs in the API - process; the scheduler awaits it via ``asyncio.run``. - """ - - name = "transform" - description = "Resolves raw_name → resolved_id." - spec_schema = _CanonicalSpec - input_spec_schema = _InputSpec - - @classmethod - async def to_spec(cls, input_spec, *, workspace, entity_client, async_sdk, is_local: bool): - del is_local - assert isinstance(input_spec, _InputSpec) - return _CanonicalSpec( - resolved_id=f"id-{input_spec.raw_name}", - count=input_spec.count, - ) - - def run(self, config: dict) -> dict: - return {"resolved_id": config["resolved_id"], "count": config["count"]} - - -# --------------------------------------------------------------------------- -# run_local -# --------------------------------------------------------------------------- - - -class TestRunLocalSignatureAdaptation: - def test_legacy_job_receives_raw_dict(self) -> None: - scheduler = NemoJobScheduler() - result = scheduler.run_local(_LegacyRawJob, {"foo": "bar", "n": 3}) - assert result == {"got": {"foo": "bar", "n": 3}} - - def test_spec_schema_applies_defaults(self) -> None: - scheduler = NemoJobScheduler() - result = scheduler.run_local(_SpecSchemaOnlyJob, {"name": "widget"}) - assert result == {"name": "widget", "count": 1} - - def test_spec_schema_validates_input(self) -> None: - scheduler = NemoJobScheduler() - with pytest.raises(ValidationError): - scheduler.run_local(_SpecSchemaOnlyJob, {"name": 123}) - - def test_input_spec_schema_runs_to_spec(self) -> None: - scheduler = NemoJobScheduler() - result = scheduler.run_local(_InputTransformJob, {"raw_name": "foo", "count": 5}) - assert result == {"resolved_id": "id-foo", "count": 5} - - def test_is_local_injected_into_to_spec_and_run_when_requested(self) -> None: - seen: dict[str, bool] = {} - - class _LocalityJob(NemoJob): - name = "locality" - spec_schema = _CanonicalSpec - input_spec_schema = _InputSpec - - @classmethod - async def to_spec(cls, input_spec, *, workspace, entity_client, async_sdk, is_local: bool): - del workspace, entity_client, async_sdk - seen["to_spec"] = is_local - return _CanonicalSpec( - resolved_id=f"id-{input_spec.raw_name}", - count=input_spec.count, - ) - - def run(self, config: dict, *, is_local: bool) -> dict: - return {"to_spec": seen["to_spec"], "run": is_local, **config} - - scheduler = NemoJobScheduler() - result = scheduler.run_local(_LocalityJob, {"raw_name": "foo", "count": 5}) - - assert result == {"to_spec": True, "run": True, "resolved_id": "id-foo", "count": 5} - - def test_run_local_is_local_true_even_when_job_id_env_is_set(self, monkeypatch) -> None: - monkeypatch.setenv("NEMO_JOB_ID", "env-job-id") - - class _LocalityJob(NemoJob): - name = "locality-env" - spec_schema = _CanonicalSpec - input_spec_schema = _InputSpec - - @classmethod - async def to_spec(cls, input_spec, *, workspace, entity_client, async_sdk, is_local: bool): - del workspace, entity_client, async_sdk - return _CanonicalSpec(resolved_id=f"id-{input_spec.raw_name}", count=int(is_local)) - - def run(self, config: dict, *, is_local: bool) -> dict: - return {"to_spec": config["count"], "run": is_local} - - result = NemoJobScheduler().run_local(_LocalityJob, {"raw_name": "foo"}) - - assert result == {"to_spec": 1, "run": True} - - def test_run_local_passes_validation_context_to_input_and_canonical_schemas(self) -> None: - seen: list[str] = [] - - def require_local_context(data: Any, info: ValidationInfo, label: str) -> Any: - context = info.context - if not (isinstance(context, dict) and context.get("is_local") is True): - raise ValueError(f"missing local validation context for {label}") - seen.append(label) - return data - - class _ContextInputSpec(BaseModel): - name: str - - @model_validator(mode="before") - @classmethod - def require_local_context(cls, data: Any, info: ValidationInfo) -> Any: - return require_local_context(data, info, "input") - - class _ContextCanonicalSpec(BaseModel): - name: str - - @model_validator(mode="before") - @classmethod - def require_local_context(cls, data: Any, info: ValidationInfo) -> Any: - return require_local_context(data, info, "canonical") - - class _ContextJob(NemoJob): - name = "context-job" - input_spec_schema = _ContextInputSpec - spec_schema = _ContextCanonicalSpec - - @classmethod - async def to_spec(cls, input_spec, *, workspace, entity_client, async_sdk, is_local: bool): - del workspace, entity_client, async_sdk, is_local - return {"name": input_spec.name} - - def run(self, config: dict) -> dict: - return config - - result = NemoJobScheduler().run_local(_ContextJob, {"name": "widget"}) - - assert result == {"name": "widget"} - assert seen == ["input", "canonical"] - - def test_input_spec_schema_validates_incoming_shape(self) -> None: - scheduler = NemoJobScheduler() - with pytest.raises(ValidationError): - scheduler.run_local(_InputTransformJob, {"wrong_field": "x"}) - - -class TestRunLocalJobContext: - def test_scheduler_builds_local_context_when_none_provided(self, tmp_path) -> None: - captured: dict = {} - - class _CaptureJob(NemoJob): - name = "capture" - - def run(self, config: dict) -> dict: - return config - - # Use a custom ctx so the test can inspect it without relying on - # tempdir construction. - from nemo_platform_plugin.job_results import LocalJobResults - - ctx = JobContext( - workspace="test-ws", - job_id="550e8400-e29b-41d4-a716-446655440000", - storage=StoragePaths(ephemeral=tmp_path / "e", persistent=tmp_path / "p"), - results=LocalJobResults(root=tmp_path / "r"), - ) - (tmp_path / "e").mkdir() - (tmp_path / "p").mkdir() - captured["ctx"] = ctx - - scheduler = NemoJobScheduler() - scheduler.run_local(_CaptureJob, {"hello": "world"}, workspace="test-ws", ctx=ctx) - - # Assert the ctx we passed satisfies the protocol. - assert isinstance(ctx, JobContext) - assert ctx.workspace == "test-ws" - assert ctx.job_id == "550e8400-e29b-41d4-a716-446655440000" - - def test_auto_built_context_has_tempdir_storage(self) -> None: - scheduler = NemoJobScheduler() - - # Run a job that just echoes; the ctx is built internally. - result = scheduler.run_local(_LegacyRawJob, {}, workspace="auto-ws") - assert result == {"got": {}} - - def test_auto_built_context_uses_workspace_argument(self) -> None: - scheduler = NemoJobScheduler() - - class _LookAtCtxJob(NemoJob): - name = "look-at-ctx" - - def run(self, config: dict) -> dict: - return config - - # We can't observe the auto-built ctx from run(config) alone, so - # the test asserts through the scheduler's helper that the ctx - # would carry the workspace correctly. - ctx = scheduler._build_local_context(_LookAtCtxJob, workspace="visible-ws") - assert ctx.workspace == "visible-ws" - # Default ``job_id`` is None for scheduler-created local contexts. - assert ctx.job_id is None - assert ctx.storage.ephemeral.exists() - assert ctx.storage.persistent.exists() - # ``ctx.results`` is the default LocalJobResults rooted under - # ``persistent / "results"`` per ``build_local_job_context``. - assert ctx.results is not None - - def test_auto_built_context_results_writes_under_persistent(self) -> None: - """The default ``LocalJobResults`` lands artefacts under - ``persistent / "results"`` so the dev loop can inspect output - deterministically.""" - scheduler = NemoJobScheduler() - - class _NoopJob(NemoJob): - name = "noop-results" - - def run(self, config: dict) -> dict: - return config - - ctx = scheduler._build_local_context(_NoopJob, workspace="ws") - src = ctx.storage.ephemeral / "out.txt" - src.write_text("payload") - assert isinstance(ctx.results, LocalJobResults) - ref = ctx.results.save("out", src) - expected_root = ctx.storage.persistent / "results" - assert ref.artifact_url.startswith(f"file://{expected_root.resolve()}/") - - -# --------------------------------------------------------------------------- -# run_local — JobContext / LocalJobResults wiring -# --------------------------------------------------------------------------- - - -class TestRunLocalContextWiring: - """The auto-built context carries a :class:`LocalJobResults` sink and - a :class:`JobContext` — there is no async twin.""" - - def test_run_receives_sync_results_sink(self) -> None: - seen: dict[str, object] = {} - - class _Job(NemoJob): - name = "sync-results" - - def run(self, config: dict, *, ctx: JobContext) -> dict: - seen["results_type"] = type(ctx.results).__name__ - seen["ctx_type"] = type(ctx).__name__ - src = ctx.storage.ephemeral / "out.txt" - src.write_text("payload") - ref = ctx.results.save("out", src) - return {"artifact_url": ref.artifact_url} - - result = NemoJobScheduler().run_local(_Job, {}) - assert seen["results_type"] == "LocalJobResults" - assert seen["ctx_type"] == "JobContext" - assert result["artifact_url"].startswith("file://") - - -class TestRunLocalEnvvarMirroring: - """``run_local`` should expose the in-container runtime envvars during ``run``. - - Legacy task code reads ``NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH`` and - friends via ``os.environ.get(...)`` to locate scratch / persistent - volumes. Local invocations need the same names visible so unported - code keeps working. - """ - - def test_storage_envvars_visible_inside_run(self, monkeypatch) -> None: - import os - - monkeypatch.delenv("NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH", raising=False) - monkeypatch.delenv("NEMO_JOB_EPHEMERAL_TASK_STORAGE_PATH", raising=False) - monkeypatch.setenv("NEMO_JOB_WORKSPACE", "default") - - seen: dict[str, object] = {} - - class _ReadEnvJob(NemoJob): - name = "read-env" - - def run(self, config: dict) -> dict: - persistent = os.environ.get("NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH") - ephemeral = os.environ.get("NEMO_JOB_EPHEMERAL_TASK_STORAGE_PATH") - seen["persistent"] = persistent - seen["ephemeral"] = ephemeral - seen["job_id"] = os.environ.get("NEMO_JOB_ID") - seen["workspace"] = os.environ.get("NEMO_JOB_WORKSPACE") - # Directories should exist *during* the run; they are torn down - # in the scheduler's finally block. - seen["persistent_exists"] = persistent is not None and Path(persistent).exists() - seen["ephemeral_exists"] = ephemeral is not None and Path(ephemeral).exists() - return config - - NemoJobScheduler().run_local(_ReadEnvJob, {}, workspace="my-ws") - - assert seen["persistent"] is not None - assert seen["ephemeral"] is not None - assert seen["persistent_exists"] is True - assert seen["ephemeral_exists"] is True - # Local runs leave job_id as None; the scheduler skips exporting - # NEMO_JOB_ID in that case. - assert seen["job_id"] is None - assert seen["workspace"] == "my-ws" - assert os.environ.get("NEMO_JOB_WORKSPACE") == "default" - - def test_envvars_restored_after_run(self, monkeypatch) -> None: - import os - - monkeypatch.delenv("NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH", raising=False) - monkeypatch.delenv("NEMO_JOB_EPHEMERAL_TASK_STORAGE_PATH", raising=False) - - class _NoopJob(NemoJob): - name = "noop" - - def run(self, config: dict) -> dict: - return config - - NemoJobScheduler().run_local(_NoopJob, {}) - - # Envvars we synthesized should not leak past the call. - assert "NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH" not in os.environ - assert "NEMO_JOB_EPHEMERAL_TASK_STORAGE_PATH" not in os.environ - - def test_caller_set_envvars_win(self, monkeypatch) -> None: - import os - - monkeypatch.setenv("NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH", "/caller/persistent") - seen: dict[str, str | None] = {} - - class _ReadEnvJob(NemoJob): - name = "read-env" - - def run(self, config: dict) -> dict: - seen["persistent"] = os.environ.get("NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH") - return config - - NemoJobScheduler().run_local(_ReadEnvJob, {}) - - # Caller's value survives — scheduler doesn't override pre-set envvars. - assert seen["persistent"] == "/caller/persistent" - # And it's still set after the call (we never touched it). - assert os.environ.get("NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH") == "/caller/persistent" - - # --------------------------------------------------------------------------- # submit_remote — URL building, body shaping, HTTP POST (MR 1.3) # --------------------------------------------------------------------------- @@ -727,96 +343,3 @@ def test_compile_raises_when_not_overridden(self) -> None: async_sdk=cast(AsyncNeMoPlatform, None), ) ) - - -# --------------------------------------------------------------------------- -# Signature-based DI -# -# Plain ``run(self, config: dict)`` jobs receive the canonical dict and -# nothing else. Jobs that widen the signature with keyword-only ``ctx``, -# ``sdk``, or ``async_sdk`` parameters get them resolved by name from the -# scheduler inputs. Per-service typed resources (Files, Models, ...) will -# be reintroduced under the new ``NemoSDK`` design. -# --------------------------------------------------------------------------- - - -class TestRunLocalDILegacyShim: - def test_plain_run_config_still_works(self) -> None: - """Jobs with ``run(self, config)`` receive the canonical dict only — no DI side-effects.""" - scheduler = NemoJobScheduler() - result = scheduler.run_local(_LegacyRawJob, {"x": 1}, sdk=object()) - assert result == {"got": {"x": 1}} - - -class TestRunLocalDIContextAndSdk: - def test_ctx_injected_when_declared(self, tmp_path) -> None: - seen: dict = {} - - class _CtxJob(NemoJob): - name = "ctx-job" - - def run(self, config: dict, *, ctx: JobContext) -> dict: - seen["ctx"] = ctx - return {"ok": True} - - from nemo_platform_plugin.job_results import LocalJobResults - - ctx = JobContext( - workspace="ws", - job_id="11111111-1111-1111-1111-111111111111", - storage=StoragePaths(ephemeral=tmp_path / "e", persistent=tmp_path / "p"), - results=LocalJobResults(root=tmp_path / "r"), - ) - (tmp_path / "e").mkdir() - (tmp_path / "p").mkdir() - - NemoJobScheduler().run_local(_CtxJob, {}, workspace="ws", ctx=ctx) - assert seen["ctx"] is ctx - - def test_sdk_injected_when_declared(self) -> None: - seen: dict = {} - - class _SdkJob(NemoJob): - name = "sdk-job" - - def run(self, config: dict, *, sdk: object | None = None) -> dict: - seen["sdk"] = sdk - return {} - - sdk = object() - NemoJobScheduler().run_local(_SdkJob, {}, sdk=sdk) - assert seen["sdk"] is sdk - - def test_sdk_required_without_default_raises(self) -> None: - class _RequiredSdkJob(NemoJob): - name = "required-sdk-job" - - def run(self, config: dict, *, sdk: object) -> dict: - return {} - - with pytest.raises(LocalRunError, match=r"requires a `sdk` argument"): - NemoJobScheduler().run_local(_RequiredSdkJob, {}) - - def test_async_sdk_injected_when_declared(self) -> None: - seen: dict = {} - - class _AsyncSdkJob(NemoJob): - name = "async-sdk-job" - - def run(self, config: dict, *, async_sdk: object | None = None) -> dict: - seen["async_sdk"] = async_sdk - return {} - - async_sdk = object() - NemoJobScheduler().run_local(_AsyncSdkJob, {}, async_sdk=async_sdk) - assert seen["async_sdk"] is async_sdk - - def test_async_sdk_required_without_default_raises(self) -> None: - class _RequiredAsyncSdkJob(NemoJob): - name = "required-async-sdk-job" - - def run(self, config: dict, *, async_sdk: object) -> dict: - return {} - - with pytest.raises(LocalRunError, match=r"requires an `async_sdk` argument"): - NemoJobScheduler().run_local(_RequiredAsyncSdkJob, {}) diff --git a/packages/nmp_customization_common/src/nmp/customization_common/cli/overrides.py b/packages/nmp_customization_common/src/nmp/customization_common/cli/overrides.py index f2e0791ce0..8203f593e1 100644 --- a/packages/nmp_customization_common/src/nmp/customization_common/cli/overrides.py +++ b/packages/nmp_customization_common/src/nmp/customization_common/cli/overrides.py @@ -3,18 +3,16 @@ """Shared CLI override machinery for customization contributor plugins. -After the platform's ``_add_run_command`` / ``_add_submit_command`` register the -default verbs, both backends swap in the same shapes: +After the platform's ``_add_submit_command`` registers the default verb, each +backend swaps in the same shape: - ``submit`` → positional ``JOB_JSON`` argument plus standard submit flags; loads + validates the JSON (via the backend's ``load_job_json``), then delegates to the original ``submit`` callback with ``--spec`` set. -- ``run`` → hard-fails with a "submit-only" message (these backends run - remotely in a container, not locally). - ``explain`` → unchanged. -Only the backend's ``load_job_json``, the ``JOB_JSON`` help text, and the -run-disabled message differ; everything else is shared here. +Only the backend's ``load_job_json`` and the ``JOB_JSON`` help text differ; +everything else is shared here. """ from collections.abc import Callable @@ -30,15 +28,13 @@ def apply_job_cli_overrides( *, load_job_json: LoadJobJson, job_json_help: str, - run_disabled_message: str, ) -> None: - """Drop the default ``run``/``submit`` verbs, then re-register the overrides. + """Replace ``submit`` with the backend-friendly ``JOB_JSON`` wrapper. Order matters: drop first, then re-register. Typer iterates ``registered_commands`` in insertion order, so stale entries would route users back to the auto-generated shapes. """ - _replace_job_run_disabled(group, job_json_help, run_disabled_message) _replace_job_submit(group, load_job_json, job_json_help) @@ -53,19 +49,6 @@ def _drop_command(group: typer.Typer, name: str) -> None: group.registered_commands = [c for c in group.registered_commands if c.name != name] -def _replace_job_run_disabled(group: typer.Typer, job_json_help: str, run_disabled_message: str) -> None: - """Replace ``run`` with a hard-fail explainer (these backends are submit-only).""" - _drop_command(group, "run") - - @group.command("run") - def run( - _typer_ctx: typer.Context, - _job_json: Path | None = typer.Argument(None, metavar="JOB_JSON", help=job_json_help), - ) -> None: - typer.secho(run_disabled_message, err=True, fg=typer.colors.RED) - raise typer.Exit(code=1) - - def _replace_job_submit(group: typer.Typer, load_job_json: LoadJobJson, job_json_help: str) -> None: """Replace ``submit`` with a ``JOB_JSON`` positional + standard submit flags.""" original = _pluck_callback(group, "submit") diff --git a/packages/nmp_customization_common/src/nmp/customization_common/contributor/base.py b/packages/nmp_customization_common/src/nmp/customization_common/contributor/base.py index fa81e1a6fd..4a3462d74c 100644 --- a/packages/nmp_customization_common/src/nmp/customization_common/contributor/base.py +++ b/packages/nmp_customization_common/src/nmp/customization_common/contributor/base.py @@ -86,17 +86,15 @@ def get_routers(self) -> list[RouterSpec]: ] def get_cli(self) -> typer.Typer: - """Compose run/submit/explain verbs, then apply backend-specific overrides.""" + """Compose submit/explain verbs, then apply backend-specific overrides.""" from nemo_platform_plugin.commands import ( _add_explain_command, - _add_run_command, _add_submit_command, ) from nemo_platform_plugin.scheduler import NemoJobScheduler app = typer.Typer(name=self.name, help=self.cli_help, no_args_is_help=True) scheduler = NemoJobScheduler() - _add_run_command(app, self.job_cls, scheduler) _add_submit_command(app, self.job_cls, scheduler) _add_explain_command(app, self.job_cls, scheduler) self.apply_cli_overrides(app) diff --git a/plugins/example-plugin/src/nemo_example_plugin/jobs/say_hello.py b/plugins/example-plugin/src/nemo_example_plugin/jobs/say_hello.py index dee32d5b4c..e841c05953 100644 --- a/plugins/example-plugin/src/nemo_example_plugin/jobs/say_hello.py +++ b/plugins/example-plugin/src/nemo_example_plugin/jobs/say_hello.py @@ -3,11 +3,10 @@ """say-hello job — registered under ``nemo.jobs``. -Writes a greeting to persistent storage and registers it as a result -via :attr:`~nemo_platform_plugin.job_context.JobContext.results`. Local runs use -:class:`~nemo_platform_plugin.job_results.LocalJobResults` wired by -:class:`~nemo_platform_plugin.scheduler.NemoJobScheduler`, so the artefact lands -under ``/results/greeting`` with no clients and no SDK. +Writes a greeting to persistent storage and registers it as a result via +:attr:`~nemo_platform_plugin.job_context.JobContext.results`. Unit tests can +inject :class:`~nemo_platform_plugin.job_results.LocalJobResults`; platform +tasks use the dispatcher-provided results sink. """ from __future__ import annotations diff --git a/plugins/example-plugin/tests/test_say_hello_job.py b/plugins/example-plugin/tests/test_say_hello_job.py index aa8f511e6e..4d0e6cd508 100644 --- a/plugins/example-plugin/tests/test_say_hello_job.py +++ b/plugins/example-plugin/tests/test_say_hello_job.py @@ -1,17 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests for :class:`~nemo_example_plugin.jobs.say_hello.SayHelloJob`. - -Pin the end-to-end wiring: running the job through -:class:`~nemo_platform_plugin.scheduler.NemoJobScheduler` with no clients writes -the greeting to ``ctx.storage.persistent`` and registers it via the -default :class:`~nemo_platform_plugin.job_results.LocalJobResults` with a -``file://`` URL. -""" +"""Tests for :class:`~nemo_example_plugin.jobs.say_hello.SayHelloJob`.""" from __future__ import annotations +import json from pathlib import Path from nemo_example_plugin.jobs.say_hello import ( @@ -19,7 +13,27 @@ DEFAULT_RESULT_NAME, SayHelloJob, ) -from nemo_platform_plugin.scheduler import NemoJobScheduler +from nemo_platform_plugin.job_context import JobContext, StoragePaths +from nemo_platform_plugin.job_results import LocalJobResults +from nemo_platform_plugin.jobs.constants import NEMO_JOB_STEP_CONFIG_FILE_PATH_ENVVAR +from nemo_platform_plugin.tasks.dispatcher import run_task + + +def _write_config(tmp_path: Path, payload: dict) -> Path: + config_path = tmp_path / "step-config.json" + config_path.write_text(json.dumps(payload), encoding="utf-8") + return config_path + + +def _local_ctx(tmp_path: Path, *, workspace: str = "dev") -> JobContext: + storage = StoragePaths(ephemeral=tmp_path / "e", persistent=tmp_path / "p") + storage.ephemeral.mkdir() + storage.persistent.mkdir() + return JobContext( + workspace=workspace, + storage=storage, + results=LocalJobResults(root=storage.persistent / "results"), + ) def test_say_hello_job_metadata() -> None: @@ -27,42 +41,33 @@ def test_say_hello_job_metadata() -> None: assert SayHelloJob.description -def test_say_hello_runs_locally_without_clients() -> None: - scheduler = NemoJobScheduler() - result = scheduler.run_local( - SayHelloJob, - {"name": "Razvan"}, - workspace="dev", - ) - assert result["result"] == "Hello, Razvan!" - artefact = result["artifact"] - assert artefact["name"] == DEFAULT_RESULT_NAME - artifact_path = Path(artefact["artifact_url"].removeprefix("file://")) +def test_say_hello_runs_through_task_dispatcher(tmp_path: Path, monkeypatch) -> None: + config_path = _write_config(tmp_path, {"name": "Razvan"}) + monkeypatch.setenv(NEMO_JOB_STEP_CONFIG_FILE_PATH_ENVVAR, str(config_path)) + ctx = _local_ctx(tmp_path) + + exit_code = run_task(SayHelloJob, ctx=ctx) + + assert exit_code == 0 + artifact_path = ctx.storage.persistent / "results" / DEFAULT_RESULT_NAME assert artifact_path.exists() - assert artifact_path.read_text() == "Hello, Razvan!" + assert artifact_path.read_text(encoding="utf-8") == "Hello, Razvan!" -def test_defaults_name_to_world() -> None: - result = NemoJobScheduler().run_local(SayHelloJob, {}, workspace="dev") - assert result["result"] == "Hello, world!" +def test_defaults_name_to_world(tmp_path: Path, monkeypatch) -> None: + config_path = _write_config(tmp_path, {}) + monkeypatch.setenv(NEMO_JOB_STEP_CONFIG_FILE_PATH_ENVVAR, str(config_path)) + ctx = _local_ctx(tmp_path) + assert run_task(SayHelloJob, ctx=ctx) == 0 + assert (ctx.storage.persistent / DEFAULT_FILE_NAME).read_text(encoding="utf-8") == "Hello, world!" -def test_greeting_text_lands_under_persistent(tmp_path: Path) -> None: - from nemo_platform_plugin.job_context import JobContext, StoragePaths - from nemo_platform_plugin.job_results import LocalJobResults - storage = StoragePaths(ephemeral=tmp_path / "e", persistent=tmp_path / "p") - storage.ephemeral.mkdir() - storage.persistent.mkdir() - ctx = JobContext( - workspace="dev", - storage=storage, - results=LocalJobResults(root=storage.persistent / "results"), - ) - NemoJobScheduler().run_local( - SayHelloJob, - {"name": "Razvan"}, - workspace="dev", - ctx=ctx, - ) - assert (storage.persistent / DEFAULT_FILE_NAME).read_text() == "Hello, Razvan!" +def test_greeting_text_lands_under_persistent(tmp_path: Path, monkeypatch) -> None: + config_path = _write_config(tmp_path, {"name": "Razvan"}) + monkeypatch.setenv(NEMO_JOB_STEP_CONFIG_FILE_PATH_ENVVAR, str(config_path)) + ctx = _local_ctx(tmp_path) + + run_task(SayHelloJob, ctx=ctx) + + assert (ctx.storage.persistent / DEFAULT_FILE_NAME).read_text(encoding="utf-8") == "Hello, Razvan!" diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/jobs/evaluate_agent.py b/plugins/nemo-agents/src/nemo_agents_plugin/jobs/evaluate_agent.py index 2eeda79456..4821a91138 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/jobs/evaluate_agent.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/jobs/evaluate_agent.py @@ -47,7 +47,7 @@ OutputTarget, classify_output_target, ) -from nemo_platform_plugin.run_dependencies import LocalRunError +from nemo_platform_plugin.run_dependencies import RunDependencyError from pydantic import BaseModel, Field logger = logging.getLogger(__name__) @@ -229,17 +229,14 @@ def run( Args: config: Dict matching :class:`EvaluateAgentSpec`. sdk: Platform SDK handle, injected by the - :class:`~nemo_platform_plugin.scheduler.NemoJobScheduler` (locally) or :func:`~nemo_platform_plugin.tasks.dispatcher.run_task` - (in-container) from the ambient SDK handle. Required when + from the ambient SDK handle. Required when ``cfg.eval_config_fileset`` or a fileset-shaped ``cfg.output`` is set (download / upload respectively); a local-directory output runs without it, so the parameter is declared optional and validated at the point of use. - ctx: Runtime context bound by signature DI. Both - :class:`~nemo_platform_plugin.scheduler.NemoJobScheduler.run_local` - and :func:`~nemo_platform_plugin.tasks.dispatcher.run_task` - always supply one; the no-output fallback writes to + ctx: Runtime context bound by signature DI. The task dispatcher + supplies one; the no-output fallback writes to ``ctx.storage.persistent / "results"`` and tempdirs land under ``ctx.storage.ephemeral`` so they sit on the platform-injected scratch volume. @@ -340,7 +337,7 @@ def _resolve_eval_config( ``cfg.eval_config`` relative to it. Otherwise pass through verbatim. ``sdk`` is required on the fileset branch — when the scheduler can't supply one and the spec asks for a fileset, raise - :class:`LocalRunError` early so the caller sees an actionable error + :class:`RunDependencyError` early so the caller sees an actionable error instead of failing later inside the subprocess. """ if not cfg.eval_config_fileset: @@ -348,10 +345,11 @@ def _resolve_eval_config( return if sdk is None: - raise LocalRunError( + raise RunDependencyError( "EvaluateAgentJob.run requires a 'sdk: NeMoPlatform' to download " "eval_config_fileset contents, but no platform SDK was available. " - "Set NMP_BASE_URL or pass sdk via NemoJobScheduler.run_local(sdk=...)." + "Submit the job through the Jobs API/SDK, or pass sdk to " + "nemo_platform_plugin.tasks.dispatcher.run_task(...) in tests." ) ref = FilesetRef(cfg.eval_config_fileset) @@ -409,7 +407,7 @@ def _resolve_output( *sdk* is required only on the fileset branch. When the scheduler can't supply one (no SDK handle in scope) and the - output points at a fileset, we raise :class:`LocalRunError` + output points at a fileset, we raise :class:`RunDependencyError` early — before the subprocess runs — so the user gets an actionable message instead of losing the eval artifacts. """ @@ -434,11 +432,11 @@ def _resolve_output( return if sdk is None: - raise LocalRunError( + raise RunDependencyError( "EvaluateAgentJob.run requires a 'sdk: NeMoPlatform' to upload " "results to a fileset, but no platform SDK was available. " - "Set NMP_BASE_URL (so the local CLI can build a default SDK), " - "pass an explicit sdk via NemoJobScheduler.run_local(sdk=...), " + "Submit the job through the Jobs API/SDK, pass sdk to " + "nemo_platform_plugin.tasks.dispatcher.run_task(...) in tests, " "or use --output to write results to a local directory instead." ) diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/jobs/fileset_io.py b/plugins/nemo-agents/src/nemo_agents_plugin/jobs/fileset_io.py index f10346f34b..d399fda96f 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/jobs/fileset_io.py +++ b/plugins/nemo-agents/src/nemo_agents_plugin/jobs/fileset_io.py @@ -32,7 +32,7 @@ classify_output_target, parse_entity_ref, ) -from nemo_platform_plugin.run_dependencies import LocalRunError +from nemo_platform_plugin.run_dependencies import RunDependencyError logger = logging.getLogger(__name__) @@ -69,10 +69,10 @@ def resolve_staged_config( ws, name = split_fileset_ref(ref, workspace) if sdk is None: - raise LocalRunError( + raise RunDependencyError( f"Staging {kind} from a fileset requires a 'sdk: NeMoPlatform', but no " - "platform SDK was available. Set NMP_BASE_URL or pass sdk via " - "NemoJobScheduler.run_local(sdk=...)." + "platform SDK was available. Submit the job through the Jobs API/SDK, " + "or pass sdk to nemo_platform_plugin.tasks.dispatcher.run_task(...) in tests." ) with tempfile.TemporaryDirectory(prefix=f".{kind}-{name}-", dir=str(ctx.storage.ephemeral)) as tmp: @@ -132,10 +132,11 @@ def resolve_output( ws, name = split_fileset_ref(ref, workspace) if sdk is None: - raise LocalRunError( + raise RunDependencyError( f"Uploading {kind} results to a fileset requires a 'sdk: NeMoPlatform', but no " - "platform SDK was available. Set NMP_BASE_URL, pass sdk via " - "NemoJobScheduler.run_local(sdk=...), or use a local output directory instead." + "platform SDK was available. Submit the job through the Jobs API/SDK, " + "pass sdk to nemo_platform_plugin.tasks.dispatcher.run_task(...) in tests, " + "or use a local output directory instead." ) with tempfile.TemporaryDirectory(prefix=f".{kind}-output-{name}-", dir=str(ctx.storage.ephemeral)) as tmp: diff --git a/plugins/nemo-agents/src/nemo_agents_plugin/skills/agents-optimize/SKILL.md b/plugins/nemo-agents/src/nemo_agents_plugin/skills/agents-optimize/SKILL.md index 0cc9a829ca..6dcd7683b5 100644 --- a/plugins/nemo-agents/src/nemo_agents_plugin/skills/agents-optimize/SKILL.md +++ b/plugins/nemo-agents/src/nemo_agents_plugin/skills/agents-optimize/SKILL.md @@ -121,7 +121,7 @@ Each suggestion sets `model` to the new model name and includes: `--model` override flag on `evaluate`; the model swap goes through the sibling agent, or by editing the eval-config YAML. - `nemo auditor targets create -d '{"model": "", "type": ""}'` then - `nemo auditor audit run --spec '{"config": "default/", "target": "default/"}'` + `nemo auditor audit submit --spec '{"config": "default/", "target": "default/"}'` to verify the new model is robust against jailbreaks before promotion. Pure set subtraction — no downloads required. @@ -329,7 +329,7 @@ nemo files filesets list # Auditor (jailbreak robustness check on a candidate model) nemo auditor targets create -d '{"model": "", "type": ""}' -nemo auditor audit run --spec '{"config": "default/", "target": "default/"}' +nemo auditor audit submit --spec '{"config": "default/", "target": "default/"}' ``` ## What requires execution vs. what can be reasoned diff --git a/plugins/nemo-agents/tests/unit/test_utils.py b/plugins/nemo-agents/tests/unit/test_utils.py index c50fa19257..9016157f19 100644 --- a/plugins/nemo-agents/tests/unit/test_utils.py +++ b/plugins/nemo-agents/tests/unit/test_utils.py @@ -40,7 +40,7 @@ ) from nemo_platform_plugin.job_context import JobContext from nemo_platform_plugin.refs import EndpointURL, FilesetRef, LocalDir -from nemo_platform_plugin.run_dependencies import LocalRunError +from nemo_platform_plugin.run_dependencies import RunDependencyError # --------------------------------------------------------------------------- # inject_gateway_url @@ -863,7 +863,7 @@ def test_fileset_ref_without_sdk_raises(self, tmp_path: Path, ctx: JobContext) - evaluation artifacts on the floor. """ job = EvaluateAgentJob() - with pytest.raises(LocalRunError, match="sdk: NeMoPlatform"): + with pytest.raises(RunDependencyError, match="sdk: NeMoPlatform"): with job._resolve_output(FilesetRef("eval-results"), workspace="default", sdk=None, ctx=ctx): pass diff --git a/plugins/nemo-anonymizer/README.md b/plugins/nemo-anonymizer/README.md index dd1031e27a..4f0cc6aa05 100644 --- a/plugins/nemo-anonymizer/README.md +++ b/plugins/nemo-anonymizer/README.md @@ -26,9 +26,8 @@ The plugin provides functional parity with the [NVIDIA NeMo Anonymizer library](https://github.com/NVIDIA-NeMo/Anonymizer): - All four replacement strategies + `Rewrite` mode. -- Input sources: local file path, `http(s)://` URL, or NeMo Platform fileset reference. - Local paths are only supported by local execution (`run` verbs). -- Remote execution requires `model_configs` so requests route through NeMo Platform +- Input sources: `http(s)://` URL or NeMo Platform fileset reference. +- Execution requires `model_configs` so requests route through NeMo Platform Inference Gateway instead of the library's NVIDIA Build defaults. ## Installation (developer) @@ -42,16 +41,15 @@ uv sync ## CLI quickstart ```bash -nemo anonymizer preview run --spec-file ./preview_spec.yaml +nemo anonymizer preview submit --spec-file ./preview_spec.yaml nemo anonymizer preview submit --spec-file ./preview_spec.yaml --workspace my-workspace -nemo anonymizer run run --spec-file ./run_spec.yaml +nemo anonymizer run submit --spec-file ./run_spec.yaml nemo anonymizer run submit --spec-file ./run_spec.yaml --workspace my-workspace ``` -Local execution can use local files, `http(s)` URLs, filesets, and locally -defined Data Designer model providers. Remote execution supports `http(s)` URLs -and filesets, and requires explicit `model_configs`. +Execution supports `http(s)` URLs and filesets, and requires explicit +`model_configs`. Fileset input references point at one CSV or Parquet file: diff --git a/plugins/nemo-anonymizer/openapi/openapi.yaml b/plugins/nemo-anonymizer/openapi/openapi.yaml index 0273ff6dbc..1c7f5a628d 100644 --- a/plugins/nemo-anonymizer/openapi/openapi.yaml +++ b/plugins/nemo-anonymizer/openapi/openapi.yaml @@ -1285,7 +1285,7 @@ components: - kind - records title: PreviewDatasetFrame - description: Final user-visible dataframe produced by the preview run. + description: Final user-visible dataframe produced by preview. PreviewRequest: properties: config: diff --git a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/cli.py b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/cli.py index e7195d33d0..149df17c81 100644 --- a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/cli.py +++ b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/cli.py @@ -5,16 +5,14 @@ from __future__ import annotations -from collections.abc import Callable from pathlib import Path -from typing import Any, ClassVar, Optional, cast +from typing import Any, ClassVar, Optional import typer import yaml from anonymizer.config.anonymizer_config import AnonymizerConfig from nemo_anonymizer_plugin.app.upstream_logging import preserve_root_logging from nemo_platform_plugin.cli import NemoCLI -from nemo_platform_plugin.job import NemoJob class AnonymizerCLI(NemoCLI): @@ -35,27 +33,6 @@ def _root() -> None: app.command("validate")(validate_command) return app - def update_job_cli(self, job_cls: type[NemoJob], group: typer.Typer) -> None: - if job_cls.name != "run": - return - - run_command = next((command for command in group.registered_commands if command.name == "run"), None) - if run_command is None or run_command.callback is None: - return - - run_callback = cast(Callable[..., None], run_command.callback) - signature = getattr(run_callback, "__signature__", None) - if signature is None: - return - - def _collapsed_run(typer_ctx: typer.Context, **kwargs: object) -> None: - if typer_ctx.invoked_subcommand is not None: - return - run_callback(typer_ctx, **kwargs) - - setattr(_collapsed_run, "__signature__", signature) - group.callback(invoke_without_command=True)(_collapsed_run) - def _load_yaml(path: Path) -> dict[str, Any]: with open(path) as f: diff --git a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/functions/preview.py b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/functions/preview.py index d5f4ce4ff2..68c9cdb2b8 100644 --- a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/functions/preview.py +++ b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/functions/preview.py @@ -46,7 +46,7 @@ class LogFrame(FrameModel): class PreviewDatasetFrame(FrameModel): - """Final user-visible dataframe produced by the preview run.""" + """Final user-visible dataframe produced by preview.""" kind: Literal["preview_dataset"] = "preview_dataset" records: list[dict[str, Any]] diff --git a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/SKILL.md b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/SKILL.md index 1f10551f8e..ed1371932f 100644 --- a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/SKILL.md +++ b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/SKILL.md @@ -21,8 +21,8 @@ $ARGUMENTS The plugin wraps the [NVIDIA NeMo Anonymizer library](https://github.com/NVIDIA-NeMo/Anonymizer) and exposes: -- An `anonymizer.preview` streaming function (small samples, fast iteration). Use `nemo anonymizer preview run` for local execution and `nemo anonymizer preview submit` for platform execution. -- An `anonymizer.run` job for full-dataset execution. Use `nemo anonymizer run run` for local execution and `nemo anonymizer run submit` for Jobs-worker execution. +- An `anonymizer.preview` streaming function (small samples, fast iteration). Use `nemo anonymizer preview submit` for platform execution. +- An `anonymizer.run` job for full-dataset execution. Use `nemo anonymizer run submit` for Jobs-worker execution. - A `nemo anonymizer validate` command (synchronous config validation). # Workflow @@ -37,20 +37,19 @@ Read **only** the workflow file that matches the selected mode, then follow it: # Rules - Prefer CLI surfaces. Generate YAML specs and run `nemo anonymizer ...` commands unless the user explicitly asks for Python. -- Always iterate via `nemo anonymizer preview run` or `nemo anonymizer preview submit` before running the full job. Previews are cheap and stream a small sample (default 10 records) with full detection traces. +- Always iterate via `nemo anonymizer preview submit` or `nemo anonymizer preview submit` before running the full job. Previews are cheap and stream a small sample (default 10 records) with full detection traces. - When you include `config`, pick exactly one of `replace` (Annotate/Hash/Redact/Substitute) or `rewrite` on the `AnonymizerConfig`. Not both. Do not claim `config` is required for every flow; the Anonymizer library owns default config behavior and strategy semantics. See `references/replace-strategies.md` for plugin request formatting and the [library docs](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs) for semantics. - The input must be a single CSV or Parquet file. `text_column` defaults to `text`; set it explicitly when the free-text column has another name. If the dataset has a stable record id, also set `id_column`. See `references/inputs.md`. -- `model_configs` is optional for local execution (`preview run` / `run run`); when omitted, the Anonymizer library defaults are used. - The current plugin-service / Jobs paths (`preview submit`, `run submit`) require `model_configs` so requests route through the NeMo Platform Inference Gateway. See `references/model-configs.md`. - `selected_models` overrides are only valid when `model_configs` is also supplied; aliases must resolve against that pool. -- Local file paths only work for local execution. Plugin-service / Jobs execution requires an `http(s)` URL or a fileset reference (`/#` or `fileset://...`). +- Plugin-service / Jobs execution requires an `http(s)` URL or a fileset reference (`/#` or `fileset://...`). - If a spec file matching the user's description already exists in the working directory, ask whether to edit it or create a new one. # Usage Tips and Common Pitfalls - **Replacement strategies need a discriminated payload.** Hand-written YAML specs must include `kind: redact` (or `annotate` / `hash` / `substitute`) inside the `replace` block. - **Substitute and rewrite need LLM-backed model aliases.** For plugin-service / Jobs execution they must be backed by providers declared in `model_configs`. For library-level details, refer to the [Anonymizer library docs](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs) or library skills. -- **Spec files are YAML, not JSON.** `nemo anonymizer preview run --spec-file ` and `nemo anonymizer run run --spec-file ` both load YAML. +- **Spec files are YAML, not JSON.** `nemo anonymizer preview submit --spec-file ` and `nemo anonymizer run submit --spec-file ` both load YAML. - **Run results are artifacts.** The job writes an artifacts directory containing `dataset.parquet`, `trace.parquet`, `metadata.json`, and optional `failed_records.json`. - **Fileset refs use `#` to point at a file.** `/#`, `#` (uses request workspace), or `fileset:///#`. The `#` fragment must point at a `.csv` or `.parquet` file. - **Detection labels.** Keep the Anonymizer library default label set unless the user asks to restrict detection. Refer to the [Anonymizer library docs](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs) or library skills for supported label/config details. @@ -61,7 +60,7 @@ Read **only** the workflow file that matches the selected mode, then follow it: - **`nemo anonymizer` CLI not found:** The plugin isn't installed in this environment. From the repo root, run `uv sync`; the root workspace includes the Anonymizer plugin. Confirm with `nemo anonymizer --help`. Do not install anything without the user's permission. - **`nemo anonymizer preview submit` returns 404:** The plugin service isn't mounted on the gateway. `nemo setup` does not auto-mount it. Re-run `nemo services run` (no `--services` flag) and verify the routes show up under `/apis/anonymizer/` in the OpenAPI listing. See `docs/anonymizer/tutorials/index.mdx` Prerequisites. - **`model_configs are required for remote execution`:** `preview submit` and `run submit` go through plugin-service / Jobs paths. Add `model_configs` referencing an Inference Gateway provider; use the inference/model-provider docs or skill for provider discovery. -- **`Input source ... is a local path`:** Plugin-service execution rejects local paths. Either upload the file to a fileset, use an `http(s)` URL, or switch to `preview run` / `run run` (local execution). +- **`Input source ... is a local path`:** Plugin-service execution rejects local paths. Either upload the file to a fileset or use an `http(s)` URL. - **`Fileset input ... must resolve to a .csv or .parquet file`:** The `#` fragment points at a directory or a non-CSV/Parquet file. Point it at a single file. - **Config validation failed (HTTP 422):** Run `nemo anonymizer validate --config [--model-configs ]` to surface the exact error synchronously. Common causes: mixing `replace` and `rewrite`, picking `Substitute` without a `replacement_generator` alias in `model_configs`, fileset path missing the `#` fragment. - **`selected_models requires model_configs ...`:** The user passed `selected_models` overrides without an explicit model pool. Either drop the overrides or define `model_configs` with the aliases the overrides reference. @@ -75,8 +74,7 @@ Generate a YAML spec file in the current directory describing the request. Name **Preview spec** — fast iteration over a small sample: ```yaml -# Local: nemo anonymizer preview run --spec-file ./.yaml --workspace -# Remote: nemo anonymizer preview submit --spec-file ./.yaml --workspace +# Submit: nemo anonymizer preview submit --spec-file ./.yaml --workspace config: replace: kind: redact # one of: redact, annotate, hash, substitute @@ -86,8 +84,7 @@ data: text_column: biography id_column: id num_records: 5 -# Required for plugin-service execution (`preview submit`), -# optional for local `preview run`: +# Required for `preview submit`: model_configs: - alias: gliner-pii-detector provider: nvidia-build @@ -109,8 +106,7 @@ model_configs: **Run spec** — full-dataset job: ```yaml -# Local: nemo anonymizer run run --spec-file ./.yaml -# Remote: nemo anonymizer run submit --spec-file ./.yaml --workspace +# Submit: nemo anonymizer run submit --spec-file ./.yaml --workspace config: replace: kind: redact @@ -119,7 +115,7 @@ data: source: "anonymizer-inputs#anonymizer-input.csv" text_column: biography id_column: id -# Required for `run submit`, optional for `run run`: +# Required for `run submit`: model_configs: - alias: gliner-pii-detector provider: nvidia-build @@ -132,4 +128,4 @@ model_configs: model: nvidia/nemotron-3-nano-30b-a3b ``` -Include only the bits the task requires — e.g., omit `model_configs` for purely local previews, omit `selected_models` unless overrides are needed, and use `Substitute` / `rewrite` only when the user wants LLM-generated replacements or holistic rewriting. +Include only the bits the task requires — e.g., omit `selected_models` unless overrides are needed, and use `Substitute` / `rewrite` only when the user wants LLM-generated replacements or holistic rewriting. diff --git a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/inputs.md b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/inputs.md index 23cb56e662..c234d98ef0 100644 --- a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/inputs.md +++ b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/inputs.md @@ -7,7 +7,7 @@ The anonymizer reads a **single CSV or Parquet file**. Configure it via the `dat ```yaml data: - source: + source: text_column: text # optional; defaults to "text" id_column: id # optional, stable record identifier data_summary: "Short free-text records; English." # optional, helps LLMs @@ -17,11 +17,10 @@ data: | Kind | Example | Supported by | |----------|--------------------------------------------------|-------------------------------------------------------------------------------------------| -| Local | `/tmp/input.csv` or `./data/input.parquet` | **Local execution only** (`preview run`, `run run`). | -| HTTP(S) | `https://example.com/input.csv` | Local (`preview run`, `run run`) and plugin-service / Jobs execution (`preview submit`, `run submit`). | -| Fileset | `/#` | Local (`preview run`, `run run`) and plugin-service / Jobs execution (`preview submit`, `run submit`). | +| HTTP(S) | `https://example.com/input.csv` | Plugin-service / Jobs execution (`preview submit`, `run submit`). | +| Fileset | `/#` | Plugin-service / Jobs execution (`preview submit`, `run submit`). | -Plugin-service / Jobs execution runs outside the caller's filesystem — use HTTP(S) URLs or fileset refs for those surfaces. +Plugin-service / Jobs execution runs outside the caller's filesystem, so use HTTP(S) URLs or fileset refs. ## Fileset references @@ -45,15 +44,9 @@ For upload commands, use the platform files CLI docs or `nemo-files` skill. Then Run jobs save a working artifacts directory; the anonymized dataset is one file inside that directory. -### Where artifacts land for `run run` +### Where artifacts land for `run submit` -`nemo anonymizer run run` prints `{"exit_code": 0}` on success. The local job results manager logs the artifact directory to **stderr** in the form: - -```text -Saved result 'artifacts' to file:///.../persistent/results/artifacts -``` - -Layout under that `artifacts/` directory: +`nemo anonymizer run submit` creates a platform job. After it completes, download the `artifacts` result and extract it. Layout under that `artifacts/` directory: | File | Description | |-----------------------|----------------------------------------------------------------------------| @@ -62,7 +55,7 @@ Layout under that `artifacts/` directory: | `metadata.json` | Run metadata (includes the original text column name). | | `failed_records.json` | Per-record failures with reasons. Only written when at least one record failed. | -### Loading the local artifacts +### Loading extracted artifacts Read the parquet files directly from the artifacts directory: diff --git a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/model-configs.md b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/model-configs.md index c7c1e880a8..fc2fe100d6 100644 --- a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/model-configs.md +++ b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/model-configs.md @@ -9,8 +9,6 @@ | Surface | Status | `model_configs` required? | |------------------------------------|-------------------------|------------------------------------------------------------------------------------------------------------| -| `nemo anonymizer preview run` | Available (local) | No — Anonymizer library defaults are used. | -| `nemo anonymizer run run` | Available (local) | No — Anonymizer library defaults are used. | | `nemo anonymizer preview submit` | Available (plugin svc) | **Yes** — needed so requests route through the NeMo Platform Inference Gateway instead of build.nvidia.com directly. | | `nemo anonymizer run submit` | Available (Jobs worker) | **Yes** — the job routes through the NeMo Platform Inference Gateway. | | Strategy is `Substitute` | n/a | Effectively yes for plugin-service / Jobs execution; provide a `replacement_generator`-capable alias. | @@ -54,7 +52,7 @@ Only emit a section if you actually want to override its defaults — overrides ## Common patterns -**Local default-everything preview** — no `model_configs`, no `selected_models`. Lets the Anonymizer library use its bundled defaults. Works for `preview run` and `run run`. +**Local default-everything preview** — no `model_configs`, no `selected_models`. Lets the Anonymizer library use its bundled defaults. Works for `preview submit` and `run submit`. **Plugin-service default model pool** (`preview submit`, `run submit`) — provide the aliases used by the Anonymizer library defaults: diff --git a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/preview-review.md b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/preview-review.md index 209a710c52..41eb33fab7 100644 --- a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/preview-review.md +++ b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/preview-review.md @@ -9,7 +9,7 @@ Reference docs: `docs/anonymizer/tutorials/preview.mdx` (frame schema, surfaces, ## What you get back -`nemo anonymizer preview run` and `nemo anonymizer preview submit` stream newline-delimited JSON frames with the same logical data: +`nemo anonymizer preview submit` streams newline-delimited JSON frames: - `preview_dataset` — public anonymized records. - `trace_dataset` — trace records with detection details. @@ -20,10 +20,9 @@ Reference docs: `docs/anonymizer/tutorials/preview.mdx` (frame schema, surfaces, 1. Surface any `failed_records` and the associated reasons. 2. Confirm the preview used the intended execution surface: - - Local paths require `preview run`. - `preview submit` requires HTTP(S) or fileset input and explicit `model_configs`. 3. Confirm `model_configs` aliases line up with any `selected_models` overrides. For detection overrides, use Anonymizer library role names such as `entity_detector` and `entity_validator`. 4. Ask the user to share CLI NDJSON frames for a specific record when you need to inspect exact spans and labels. 5. If quality needs tuning, refer to the Anonymizer library docs/skills for label selection, thresholds, replacement strategy parameters, and rewrite settings. -When the preview is acceptable, derive the run spec by dropping `num_records`. Run writes artifacts. Use `nemo anonymizer run run` for local execution or `nemo anonymizer run submit` for platform execution. +When the preview is acceptable, derive the run spec by dropping `num_records`. Run writes artifacts. Use `nemo anonymizer run submit` for platform execution. diff --git a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/replace-strategies.md b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/replace-strategies.md index 80aea75bb4..957a7bc63a 100644 --- a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/replace-strategies.md +++ b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/replace-strategies.md @@ -9,7 +9,7 @@ Plugin notes: - When specifying `config`, choose either `config.replace` or `config.rewrite`, not both. - Hand-written YAML specs must include a `kind` discriminator inside `replace`. -- Plugin-service / Jobs execution requires `model_configs`; local `preview run` / `run run` can omit it and use Anonymizer library defaults. +- Plugin-service / Jobs execution requires `model_configs`. Minimal YAML shape: diff --git a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/rewrite-mode.md b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/rewrite-mode.md index bb1eb2f3ff..e37a575771 100644 --- a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/rewrite-mode.md +++ b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/references/rewrite-mode.md @@ -8,7 +8,7 @@ Use this reference only for plugin execution concerns. The [Anonymizer library d For plugin-service / Jobs execution (`preview submit`, `run submit`): - Include `model_configs` so rewrite model calls route through NeMo Platform Inference Gateway providers. -- Use HTTP(S) URLs or fileset references for `data.source`; local paths only work with `preview run` / `run run`. +- Use HTTP(S) URLs or fileset references for `data.source`. - Only include `selected_models.rewrite` when you need to override library defaults, and use Anonymizer library role names exactly. Example role override shape: diff --git a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/workflows/autopilot.md b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/workflows/autopilot.md index af52d1c937..31331eee7c 100644 --- a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/workflows/autopilot.md +++ b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/workflows/autopilot.md @@ -15,27 +15,25 @@ Source of truth for defaults: `docs/anonymizer/tutorials/index.mdx` and `docs/an - **`text_column`**: pick the column most plausibly holding free text — `text`, `biography`, `body`, `message`, `content`, `description`, in that order. If you genuinely can't tell, ask one short question. - **`id_column`**: include an obvious id column (`id`, `record_id`) if present; otherwise omit. - **`num_records`**: 5 for preview. - - **Preview surface**: `nemo anonymizer preview run` (local) if the input is a local file path. Otherwise (HTTP(S) URL or fileset ref), `nemo anonymizer preview submit`. - - **Run surface**: `nemo anonymizer run run` for local paths. Use `nemo anonymizer run submit` only when the user explicitly asks for platform/cluster execution or provides a non-local input and model configs. - - **Model configs**: only set when using a plugin-service surface (`preview submit` / `run submit`) or when the strategy is `Substitute` / `rewrite`. When required, default to `nvidia-build` as the provider (or the provider the user named) with these aliases: + - **Preview surface**: `nemo anonymizer preview submit`. + - **Run surface**: `nemo anonymizer run submit`. + - **Model configs**: required for `preview submit` and `run submit`. Default to `nvidia-build` as the provider (or the provider the user named) with these aliases: - `gliner-pii-detector` → `nvidia/gliner-pii` - `gpt-oss-120b` → `openai/gpt-oss-120b` - `nemotron-30b-thinking` → `nvidia/nemotron-3-nano-30b-a3b` -3. **(If using a plugin-service surface) Confirm the service is mounted.** Run `curl -s http://localhost:8080/openapi.json | jq -r '.paths | keys[]' | grep '^/apis/anonymizer/'`. If nothing prints, tell the user to run `nemo services run` (no `--services` flag) — `nemo setup` does not mount this plugin — then continue. Skip this step entirely for `preview run` / `run run`. +3. **Confirm the service is mounted.** Run `curl -s http://localhost:8080/openapi.json | jq -r '.paths | keys[]' | grep '^/apis/anonymizer/'`. If nothing prints, tell the user to run `nemo services run` (no `--services` flag) — `nemo setup` does not mount this plugin — then continue. 4. **Build** — Write a YAML preview spec following the Output Template in SKILL.md. Default filename: `_preview_spec.yaml` (e.g. `biography_preview_spec.yaml`). -5. **Preview** — Run the surface chosen in step 2: - - Local: `nemo anonymizer preview run --spec-file --workspace ` - - Plugin service: `nemo anonymizer preview submit --spec-file --workspace ` +5. **Preview** — Run `nemo anonymizer preview submit --spec-file --workspace `. Briefly summarize the preview result — entities detected per label, any `failed_records`, and a one-record before/after example. -6. **Generate run spec** — Without re-prompting, also produce a run YAML named `_run_spec.yaml`. It mirrors the preview spec but drops `num_records`. Run writes artifacts, not a dataset entity. Keep `model_configs` only if it was needed for the preview or remote run. +6. **Generate run spec** — Without re-prompting, also produce a run YAML named `_run_spec.yaml`. It mirrors the preview spec but drops `num_records`. Run writes artifacts, not a dataset entity. Keep `model_configs`. 7. **Finalize** — Tell the user the preview ran, briefly summarize what happens to PII under the chosen strategy, and give them the launch command: ```bash - nemo anonymizer run run --spec-file .yaml + nemo anonymizer run submit --spec-file .yaml ``` - If the user asked for cluster execution, give `nemo anonymizer run submit --spec-file .yaml --workspace ` instead. Mention that local artifacts are printed to stderr (`Saved result 'artifacts' to file://...`) and remote artifacts can be fetched with: + Artifacts can be fetched with: ```bash nemo jobs get-status --workspace diff --git a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/workflows/interactive.md b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/workflows/interactive.md index 9805f6a099..840d329a5d 100644 --- a/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/workflows/interactive.md +++ b/plugins/nemo-anonymizer/src/nemo_anonymizer_plugin/skills/anonymizer/workflows/interactive.md @@ -10,20 +10,19 @@ Source of truth for this workflow: `docs/anonymizer/tutorials/index.mdx`, `docs/ 1. **Resolve CLI command** — Run `command -v nemo 2>/dev/null || (test -x .venv/bin/nemo && realpath .venv/bin/nemo) || echo CLI_NOT_FOUND`. - If the output is a path, use ` anonymizer` as the command prefix for all `nemo anonymizer …` invocations in this workflow. - If the output is `CLI_NOT_FOUND`, STOP and follow the Troubleshooting section in SKILL.md. Do not continue. -2. **Confirm the plugin service is mounted (only if the user wants `preview submit` or `run submit`).** Run `curl -s http://localhost:8080/openapi.json | jq -r '.paths | keys[]' | grep '^/apis/anonymizer/'`. If nothing prints, the plugin service isn't loaded — `nemo setup` does not auto-mount it. Tell the user to run `nemo services run` (no `--services` flag) and rerun the check. Local previews (`preview run`) and local runs (`run run`) do **not** need the plugin service mounted. -3. **Confirm input source** — Decide which kind of input you're working with: a local CSV/Parquet file, an `http(s)://` URL, or a NeMo Platform fileset reference. If the user named a file but it's not yet on the platform and they want to use `preview submit`, ask whether to upload it to a fileset first (see `references/inputs.md`). +2. **Confirm the plugin service is mounted.** Run `curl -s http://localhost:8080/openapi.json | jq -r '.paths | keys[]' | grep '^/apis/anonymizer/'`. If nothing prints, the plugin service isn't loaded — `nemo setup` does not auto-mount it. Tell the user to run `nemo services run` (no `--services` flag) and rerun the check. +3. **Confirm input source** — Decide which supported input source you're working with: an `http(s)://` URL or a NeMo Platform fileset reference. If the user named a local file, ask whether to upload it to a fileset first (see `references/inputs.md`). 4. **Clarify** — Ask the user clarifying questions to narrow down precisely what they want. Prefer a structured question tool if one is available, batch related questions together, keep the set short, and offer concrete options/defaults. Common things to make precise: - **Text column** to scan and (optional) **id column**. - **What to do with detected entities**: redact, annotate (tag inline), hash (deterministic token), substitute with realistic LLM-generated values, or fully rewrite the text under a privacy goal. See `references/replace-strategies.md` and `references/rewrite-mode.md`. - **Detection tuning** — keep Anonymizer library defaults unless the user explicitly asks for label/threshold changes; refer to the [Anonymizer library docs](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs) or library skills for those details. - - **Preview surface** — `preview run` (local, allows local paths) or `preview submit` (plugin service via CLI). - - **Run surface** — `nemo anonymizer run run` for local in-process execution or `nemo anonymizer run submit` for Jobs-worker execution. This is the Anonymizer equivalent of Data Designer's `create run` / `create submit` pattern. -5. **Resolve model providers (only if needed)** — If the preview is going through the plugin service (`preview submit`), or the chosen replacement strategy is `Substitute` or `Rewrite`, ask which provider(s) and model aliases to use. For provider discovery or creation, refer to the platform inference/model-provider docs or the relevant inference/model skill. See `references/model-configs.md`. + - **Preview surface** — `preview submit` (plugin service via CLI). + - **Run surface** — `nemo anonymizer run submit` (Jobs-worker execution). +5. **Resolve model providers** — Ask which provider(s) and model aliases to use. For provider discovery or creation, refer to the platform inference/model-provider docs or the relevant inference/model skill. See `references/model-configs.md`. 6. **Plan** — Summarize the planned config (replace vs rewrite strategy, detection tuning, model_configs, input source, num_records, preview surface) and ask the user to confirm before writing the spec. 7. **Build** — Write a YAML spec file following the Output Template in SKILL.md. Use the **Preview** shape first. 8. **Validate (optional)** — If you've also produced a stand-alone `AnonymizerConfig` YAML (e.g., the user wants `nemo anonymizer validate` to gate the run), invoke it now and address any errors before previewing. 9. **Preview** — Pick the surface you agreed on in step 4: - - Local CLI: `nemo anonymizer preview run --spec-file --workspace ` - Plugin service via CLI: `nemo anonymizer preview submit --spec-file --workspace ` Inspect the resulting NDJSON frames: `log` lines, the `preview_dataset`, the `trace_dataset`, and any `failed_records`. Surface anything in `failed_records` to the user. @@ -32,19 +31,13 @@ Source of truth for this workflow: `docs/anonymizer/tutorials/index.mdx`, `docs/ - Apply changes, re-preview. Repeat until the user is satisfied. 11. **Finalize** — Once the user is happy with the preview: - Generate a run spec by dropping `num_records` from the preview request. Run writes artifacts, not a dataset entity. - - Tell the user they can run the full job locally with: - - ```bash - nemo anonymizer run run --spec-file .yaml - ``` - - - If they want platform execution, tell them to use: + - Tell the user they can run the full job with: ```bash nemo anonymizer run submit --spec-file .yaml --workspace ``` - The remote path requires `model_configs` and rejects local file paths. + This path requires `model_configs` and rejects local file paths. - For remote jobs, show the CLI follow-up commands: ```bash @@ -53,6 +46,5 @@ Source of truth for this workflow: `docs/anonymizer/tutorials/index.mdx`, `docs/ nemo jobs results list --workspace nemo jobs results download artifacts --job --workspace --output-file artifacts.tar.gz ``` - - Note that local `run run` prints `{"exit_code": 0}` on success and logs the artifact directory to **stderr** in the form `Saved result 'artifacts' to file:///.../persistent/results/artifacts`. Walk through `references/inputs.md` if the user wants help loading those artifacts. - Caution that runtime depends on dataset size and the chosen strategy (LLM-backed strategies are slower). - Do not run the full job yourself — let the user decide when to launch it. diff --git a/plugins/nemo-anonymizer/tests/unit/test_cli.py b/plugins/nemo-anonymizer/tests/unit/test_cli.py index e586c16002..9f394d614d 100644 --- a/plugins/nemo-anonymizer/tests/unit/test_cli.py +++ b/plugins/nemo-anonymizer/tests/unit/test_cli.py @@ -3,7 +3,6 @@ from __future__ import annotations -import json from pathlib import Path from typing import Any, ClassVar @@ -42,26 +41,23 @@ def test_cli_only_registers_manual_validate_command() -> None: assert result.exit_code == 0, result.output assert "validate" in result.output assert "preview-local" not in result.output - assert "run-local" not in result.output + assert "run-" + "local" not in result.output -def test_run_job_collapses_local_run_alias() -> None: +def test_run_job_group_exposes_submit_only() -> None: cli = AnonymizerCLI() app = cli.get_cli() add_job_commands(app, {"anonymizer.run": _RunJob}, cli=cli) runner = CliRunner() - alias_result = runner.invoke(app, ["run", "--config", '{"name": "Alias"}']) - nested_result = runner.invoke(app, ["run", "run", "--config", '{"name": "Nested"}']) help_result = runner.invoke(app, ["run", "--help"]) + removed_result = runner.invoke(app, ["run", "run", "--config", '{"name": "Nested"}']) - assert alias_result.exit_code == 0, alias_result.output - assert json.loads(alias_result.output) == {"config": {"name": "Alias"}} - assert nested_result.exit_code == 0, nested_result.output - assert json.loads(nested_result.output) == {"config": {"name": "Nested"}} assert help_result.exit_code == 0, help_result.output - assert "Run locally, in-process." in help_result.output - assert "Run run locally" not in help_result.output + assert "submit" in help_result.output + assert "Run locally" not in help_result.output + assert removed_result.exit_code != 0 + assert "No such command" in removed_result.output def test_validate_command_runs_library_validation(tmp_path: Path, monkeypatch) -> None: diff --git a/plugins/nemo-anonymizer/tests/unit/test_run_job.py b/plugins/nemo-anonymizer/tests/unit/test_run_job.py index a5719acb7d..42e337a9a6 100644 --- a/plugins/nemo-anonymizer/tests/unit/test_run_job.py +++ b/plugins/nemo-anonymizer/tests/unit/test_run_job.py @@ -123,7 +123,7 @@ async def test_run_submit_requires_model_configs( @pytest.mark.asyncio -async def test_run_local_allows_missing_model_configs( +async def test_to_spec_with_local_flag_allows_missing_model_configs( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -151,7 +151,7 @@ async def test_run_local_allows_missing_model_configs( @pytest.mark.asyncio -async def test_run_local_model_configs_uses_injected_async_sdk( +async def test_to_spec_with_local_flag_uses_injected_async_sdk( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -187,7 +187,7 @@ async def test_run_local_model_configs_uses_injected_async_sdk( @pytest.mark.asyncio -async def test_run_local_serialized_step_config_can_be_revalidated( +async def test_to_spec_with_local_flag_serialized_step_config_can_be_revalidated( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/plugins/nemo-auditor/README.md b/plugins/nemo-auditor/README.md index cc4f7df3af..a9e0bdadd3 100644 --- a/plugins/nemo-auditor/README.md +++ b/plugins/nemo-auditor/README.md @@ -47,15 +47,14 @@ nemo auditor targets get llama-31-8b -w default nemo auditor configs delete quick-scan -w default ``` -There is no CLI command for running an audit yet — the local-run path is -exposed through the SDK (below). The platform jobs service can submit -audits via the `auditor.audit` job entry point. +There is no plugin-specific CLI command for running an audit yet. Submit +audits through the SDK or the platform jobs service using the `auditor.audit` +job entry point. ## SDK quickstart Every CLI verb has a matching Python SDK method on `client.auditor`, plus -`client.auditor.run(...)` for in-process execution that bypasses the jobs -service. +`client.auditor.submit(...)` for audit job submission. ```python from nemo_platform import NeMoPlatform @@ -94,20 +93,6 @@ print(f"Job submitted: {job.name}") job.wait_until_done() # blocks; streams logs while polling artifacts_dir = job.download_artifacts() # extracts garak reports to .// print(f"Reports: {artifacts_dir}") - -# Or run an audit locally (no jobs-service submission). -result = client.auditor.run( - config="quick-scan", # workspace-qualified name strings ("ws/name") also work - target="llama-31-8b", - workspace="default", -) -print(result["status"], result["returncode"]) -for name, ref in result["results"].items(): - print(name, ref["artifact_url"]) ``` `submit()` posts the job to the K8s executor and returns an `AuditorJobResource` handle. -`run()` shells out to a pre-installed garak interpreter (default -`~/.auditor/.venv/bin/python`, override via `$NEMO_AUDITOR_GARAK_PYTHON`) -and registers the resulting JSONL / HTML / hitlog reports as job results -under a temp directory managed by the local scheduler. diff --git a/plugins/nemo-auditor/src/nemo_auditor/jobs/audit.py b/plugins/nemo-auditor/src/nemo_auditor/jobs/audit.py index c559737921..8c03b05fa4 100644 --- a/plugins/nemo-auditor/src/nemo_auditor/jobs/audit.py +++ b/plugins/nemo-auditor/src/nemo_auditor/jobs/audit.py @@ -3,9 +3,8 @@ """Audit job — runs garak against a target using inline config + target. -``nemo auditor audit run --spec-file spec.yaml`` shells out to a pre-installed -garak interpreter (default ``/app/.garak_venv/bin/python``, overridable via -``NEMO_AUDITOR_GARAK_PYTHON``). +Submitted audit jobs shell out to a pre-installed garak interpreter (default +``/app/.garak_venv/bin/python``, overridable via ``NEMO_AUDITOR_GARAK_PYTHON``). The probe spec is expanded into individual per-probe YAML configs tracked through ``todo/``, ``running/``, ``complete/``, and ``failed/`` directories @@ -384,9 +383,9 @@ def _resolve_entity_client( """Return a ``NemoEntitiesClient`` from whatever the scheduler handed us. Order of preference: existing client → adapt ``async_sdk``. - Raises ``RuntimeError`` if neither is available, which is the case - when ``run`` is invoked locally with no SDK and the input spec - contains a name reference (no way to resolve it). + Raises ``RuntimeError`` if neither is available, which can happen + in direct task tests when the input spec contains a name reference + but no platform client was injected. """ if entity_client is not None: return cast(NemoEntitiesClient, entity_client) diff --git a/plugins/nemo-auditor/src/nemo_auditor/sdk.py b/plugins/nemo-auditor/src/nemo_auditor/sdk.py index 58a0406339..276f6cbb2f 100644 --- a/plugins/nemo-auditor/src/nemo_auditor/sdk.py +++ b/plugins/nemo-auditor/src/nemo_auditor/sdk.py @@ -15,26 +15,16 @@ then ``.download_artifacts()`` to fetch the garak report tarball. - ``client.auditor.list_jobs(workspace=...)`` — list submitted audit jobs. - ``client.auditor.get_job(job_name, workspace=...)`` — fetch a single audit job. -- ``client.auditor.run(config=..., target=..., workspace=...)`` — in-process - audit using :class:`~nemo_auditor.jobs.audit.AuditJob`. Delegates to - :meth:`~nemo_platform_plugin.scheduler.NemoJobScheduler.run_local`, which - constructs a tempdir-backed :class:`~nemo_platform_plugin.job_context.JobContext` - and writes report artifacts via - :class:`~nemo_platform_plugin.job_results.LocalJobResults`. """ from __future__ import annotations -import asyncio - from nemo_auditor.entities import AuditConfig, AuditTarget -from nemo_auditor.jobs.audit import AuditInputSpec, AuditJob +from nemo_auditor.jobs.audit import AuditInputSpec from nemo_auditor.sdk_resources.configs import _AsyncConfigResource, _ConfigResource from nemo_auditor.sdk_resources.job_resources import AsyncAuditorJobResource, AuditorJobResource from nemo_auditor.sdk_resources.targets import _AsyncTargetResource, _TargetResource from nemo_platform import AsyncNeMoPlatform, NeMoPlatform -from nemo_platform_plugin.entities import parse_qualified_name -from nemo_platform_plugin.scheduler import NemoJobScheduler from nemo_platform_plugin.sdk import NemoPluginSDKResources @@ -121,45 +111,6 @@ def get_job(self, job_name: str, *, workspace: str | None = None) -> dict: response.raise_for_status() return response.json() - def run( - self, - *, - config: AuditConfig | str, - target: AuditTarget | str, - workspace: str | None = None, - ) -> dict: - """Run an audit locally, in-process — no jobs-service submission. - - ``config`` / ``target`` accept either an inline pydantic entity or a - ``"name"`` / ``"workspace/name"`` string referencing one in the entity - store. Name strings are resolved through ``self.configs.get`` / - ``self.targets.get`` before the spec is handed to the scheduler, so - the scheduler always sees inline entities and ``AuditJob.to_spec`` - becomes a no-op. - """ - ws = workspace or "default" - resolved_config = self._resolve_config(config, default_workspace=ws) - resolved_target = self._resolve_target(target, default_workspace=ws) - spec = AuditInputSpec(config=resolved_config, target=resolved_target) - return NemoJobScheduler().run_local( - AuditJob, - spec.model_dump(mode="json"), - workspace=ws, - sdk=self._platform, - ) - - def _resolve_config(self, value: AuditConfig | str, *, default_workspace: str) -> AuditConfig: - if isinstance(value, AuditConfig): - return value - ws, name = parse_qualified_name(value, default_workspace=default_workspace) - return self.configs.get(workspace=ws, name=name) - - def _resolve_target(self, value: AuditTarget | str, *, default_workspace: str) -> AuditTarget: - if isinstance(value, AuditTarget): - return value - ws, name = parse_qualified_name(value, default_workspace=default_workspace) - return self.targets.get(workspace=ws, name=name) - def _url(self, path: str) -> str: return str(self._platform.base_url).rstrip("/") + "/apis/auditor" + path @@ -242,45 +193,6 @@ async def get_job(self, job_name: str, *, workspace: str | None = None) -> dict: response.raise_for_status() return response.json() - async def run( - self, - *, - config: AuditConfig | str, - target: AuditTarget | str, - workspace: str | None = None, - ) -> dict: - """Async twin of :meth:`AuditorPluginResource.run`. - - ``NemoJobScheduler.run_local`` is sync and itself calls - ``asyncio.run`` to drive ``to_spec``, so we push it onto a worker - thread to keep the caller's event loop free — same pattern as - :class:`nemo_evaluator.sdk._executor._AsyncEvaluatorPluginExecutor.run_local`. - """ - ws = workspace or "default" - resolved_config = await self._resolve_config(config, default_workspace=ws) - resolved_target = await self._resolve_target(target, default_workspace=ws) - spec = AuditInputSpec(config=resolved_config, target=resolved_target) - scheduler = NemoJobScheduler() - return await asyncio.to_thread( - scheduler.run_local, - AuditJob, - spec.model_dump(mode="json"), - workspace=ws, - async_sdk=self._platform, - ) - - async def _resolve_config(self, value: AuditConfig | str, *, default_workspace: str) -> AuditConfig: - if isinstance(value, AuditConfig): - return value - ws, name = parse_qualified_name(value, default_workspace=default_workspace) - return await self.configs.get(workspace=ws, name=name) - - async def _resolve_target(self, value: AuditTarget | str, *, default_workspace: str) -> AuditTarget: - if isinstance(value, AuditTarget): - return value - ws, name = parse_qualified_name(value, default_workspace=default_workspace) - return await self.targets.get(workspace=ws, name=name) - def _url(self, path: str) -> str: return str(self._platform.base_url).rstrip("/") + "/apis/auditor" + path diff --git a/plugins/nemo-auditor/src/nemo_auditor/skills/auditor/SKILL.md b/plugins/nemo-auditor/src/nemo_auditor/skills/auditor/SKILL.md index 67c01a3620..04b5ec153a 100644 --- a/plugins/nemo-auditor/src/nemo_auditor/skills/auditor/SKILL.md +++ b/plugins/nemo-auditor/src/nemo_auditor/skills/auditor/SKILL.md @@ -81,10 +81,6 @@ Target types: `nim`, `openai` ## Audit Job Commands ```bash -# Run an audit locally (spec references config and target as namespace/name) -nemo auditor audit run \ - --spec '{"config": "default/", "target": "default/"}' - # Submit an audit to a configured cluster nemo auditor audit submit \ --spec '{"config": "default/", "target": "default/"}' @@ -107,4 +103,4 @@ Jobs may take a long time or remain in pending/created status. That is expected. 1. Create a target pointing to the model endpoint 2. Create a config with probe selection 3. Create a job referencing `default/` and `default/` -4. Run locally with `nemo auditor audit run --spec '{...}'`, or submit with `nemo auditor audit submit --spec '{...}'` +4. Submit with `nemo auditor audit submit --spec '{...}'` diff --git a/plugins/nemo-auditor/tests/test_sdk_resources.py b/plugins/nemo-auditor/tests/test_sdk_resources.py index ef2c62b4fe..faa0001eee 100644 --- a/plugins/nemo-auditor/tests/test_sdk_resources.py +++ b/plugins/nemo-auditor/tests/test_sdk_resources.py @@ -1,16 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests for the auditor plugin SDK CRUD sub-resources and ``run`` helper. +"""Tests for the auditor plugin SDK CRUD, submit, and job resources. Each CRUD test stubs ``platform._client`` with a ``MagicMock(spec=httpx.Client)`` (or ``AsyncMock(spec=httpx.AsyncClient)``) so we can assert on the URL and JSON body the SDK actually sends — same pattern the evaluator plugin uses in ``plugins/nemo-evaluator/tests/test_sdk.py``. -``test_run_*`` patches ``nemo_auditor.sdk.NemoJobScheduler`` so the test never -actually shells out to garak; we just verify the SDK builds the right -``AuditInputSpec`` payload and forwards it to ``scheduler.run_local``. """ from __future__ import annotations @@ -291,92 +288,6 @@ def test_configs_and_targets_properties_are_cached() -> None: assert resource.targets is cached_targets -# --------------------------------------------------------------------------- -# Sync run() -# --------------------------------------------------------------------------- - - -@patch("nemo_auditor.sdk.NemoJobScheduler") -class TestSyncRun: - def test_resolves_name_strings_via_get_then_calls_scheduler(self, scheduler_cls: MagicMock) -> None: - platform = _SyncPlatform() - platform._client.get.side_effect = [ - _ok_response(_config_payload(name="my-cfg", workspace="default")), - _ok_response(_target_payload(name="my-tgt", workspace="default")), - ] - scheduler = MagicMock() - scheduler.run_local.return_value = {"status": "completed", "returncode": 0, "results": {}} - scheduler_cls.return_value = scheduler - - resource = AuditorPluginResource(cast(NeMoPlatform, platform)) - result = resource.run(config="my-cfg", target="my-tgt", workspace="default") - - assert result["status"] == "completed" - # Resolved both entities via two sync GETs. - assert platform._client.get.call_count == 2 - platform._client.get.assert_any_call( - "http://test:8000/apis/auditor/v2/workspaces/default/configs/my-cfg", - ) - platform._client.get.assert_any_call( - "http://test:8000/apis/auditor/v2/workspaces/default/targets/my-tgt", - ) - - # Spec handed to the scheduler must carry inline entities (not strings). - scheduler.run_local.assert_called_once() - args, kwargs = scheduler.run_local.call_args - job_cls, spec_dict = args - assert job_cls.__name__ == "AuditJob" - assert isinstance(spec_dict["config"], dict) - assert spec_dict["config"]["name"] == "my-cfg" - assert isinstance(spec_dict["target"], dict) - assert spec_dict["target"]["name"] == "my-tgt" - assert kwargs["workspace"] == "default" - assert kwargs["sdk"] is platform - - def test_inline_entities_skip_http_resolution(self, scheduler_cls: MagicMock) -> None: - platform = _SyncPlatform() - scheduler = MagicMock() - scheduler.run_local.return_value = {"status": "completed", "returncode": 0, "results": {}} - scheduler_cls.return_value = scheduler - - inline_config = AuditConfig(name="inline-cfg", workspace="default") - inline_target = AuditTarget(name="inline-tgt", workspace="default", type="nim", model="m") - - resource = AuditorPluginResource(cast(NeMoPlatform, platform)) - resource.run(config=inline_config, target=inline_target) - - # No HTTP roundtrip — inline entities go straight to the scheduler. - platform._client.get.assert_not_called() - spec_dict = scheduler.run_local.call_args.args[1] - assert spec_dict["config"]["name"] == "inline-cfg" - assert spec_dict["target"]["name"] == "inline-tgt" - # Defaults to "default" workspace when caller omits it. - assert scheduler.run_local.call_args.kwargs["workspace"] == "default" - - def test_workspace_qualified_name_parses_workspace_from_string(self, scheduler_cls: MagicMock) -> None: - platform = _SyncPlatform() - platform._client.get.side_effect = [ - _ok_response(_config_payload(name="cfg-1", workspace="prod")), - _ok_response(_target_payload(name="tgt-1", workspace="staging")), - ] - scheduler = MagicMock() - scheduler.run_local.return_value = {"status": "completed", "returncode": 0, "results": {}} - scheduler_cls.return_value = scheduler - - resource = AuditorPluginResource(cast(NeMoPlatform, platform)) - resource.run(config="prod/cfg-1", target="staging/tgt-1", workspace="default") - - # GETs must use the workspace from the qualified name, not the default. - platform._client.get.assert_any_call( - "http://test:8000/apis/auditor/v2/workspaces/prod/configs/cfg-1", - ) - platform._client.get.assert_any_call( - "http://test:8000/apis/auditor/v2/workspaces/staging/targets/tgt-1", - ) - # The run workspace (used by JobContext) still comes from the kwarg. - assert scheduler.run_local.call_args.kwargs["workspace"] == "default" - - # --------------------------------------------------------------------------- # Job submission: submit / list_jobs / get_job # --------------------------------------------------------------------------- @@ -526,41 +437,6 @@ async def test_async_configs_create_posts_to_workspace_route() -> None: assert url == "http://test:8000/apis/auditor/v2/workspaces/default/configs" -@pytest.mark.asyncio -async def test_async_run_resolves_names_and_calls_scheduler_in_thread() -> None: - platform = _AsyncPlatform() - platform._client.get.side_effect = [ - _ok_response(_config_payload(name="my-cfg")), - _ok_response(_target_payload(name="my-tgt")), - ] - scheduler = MagicMock() - scheduler.run_local.return_value = {"status": "completed", "returncode": 0, "results": {}} - - with ( - patch("nemo_auditor.sdk.NemoJobScheduler", return_value=scheduler) as scheduler_cls, - patch( - "nemo_auditor.sdk.asyncio.to_thread", new=AsyncMock(return_value=scheduler.run_local.return_value) - ) as to_thread, - ): - resource = AsyncAuditorPluginResource(cast(AsyncNeMoPlatform, platform)) - result = await resource.run(config="my-cfg", target="my-tgt", workspace="default") - - assert result["status"] == "completed" - scheduler_cls.assert_called_once_with() - # Scheduler call is dispatched via asyncio.to_thread so the caller's loop stays free. - to_thread.assert_awaited_once() - call = to_thread.await_args - assert call is not None - assert call.args[0] is scheduler.run_local - # job_cls is the second positional arg to to_thread (the first arg to run_local). - assert call.args[1].__name__ == "AuditJob" - spec_dict = call.args[2] - assert spec_dict["config"]["name"] == "my-cfg" - assert spec_dict["target"]["name"] == "my-tgt" - assert call.kwargs["workspace"] == "default" - assert call.kwargs["async_sdk"] is platform - - # --------------------------------------------------------------------------- # AuditorJobResource # --------------------------------------------------------------------------- diff --git a/plugins/nemo-automodel/src/nemo_automodel_plugin/cli/inputs.py b/plugins/nemo-automodel/src/nemo_automodel_plugin/cli/inputs.py index a06ca16eb6..b70e51aab3 100644 --- a/plugins/nemo-automodel/src/nemo_automodel_plugin/cli/inputs.py +++ b/plugins/nemo-automodel/src/nemo_automodel_plugin/cli/inputs.py @@ -4,8 +4,8 @@ """CLI overrides for the Automodel contributor. The override machinery is shared in :mod:`nmp.customization_common.cli.overrides`; this -module supplies the Automodel specifics: the ``AutomodelJobInput`` schema (via -``load_job_json``), the ``JOB_JSON`` help text, and the run-disabled message. +module supplies the Automodel specifics: the ``AutomodelJobInput`` schema +(via ``load_job_json``) and the ``JOB_JSON`` help text. """ import json @@ -17,10 +17,6 @@ from nemo_automodel_plugin.schema import AutomodelJobInput _JOB_JSON_HELP = "Path to Automodel job JSON (AutomodelJobInput schema)." -_RUN_DISABLED_MESSAGE = ( - "Automodel does not support local run. Submit to the platform API instead:\n" - " nemo customization automodel submit -w " -) def load_job_json(path: Path) -> str: @@ -31,10 +27,9 @@ def load_job_json(path: Path) -> str: def apply_automodel_job_cli_overrides(group: typer.Typer) -> None: - """Flat ``automodel`` CLI: ``submit JOB.json``; ``run`` is disabled.""" + """Flat ``automodel`` CLI: ``submit JOB.json``.""" apply_job_cli_overrides( group, load_job_json=load_job_json, job_json_help=_JOB_JSON_HELP, - run_disabled_message=_RUN_DISABLED_MESSAGE, ) diff --git a/plugins/nemo-automodel/tests/test_cli.py b/plugins/nemo-automodel/tests/test_cli.py index 549ad452c8..2b62fafe52 100644 --- a/plugins/nemo-automodel/tests/test_cli.py +++ b/plugins/nemo-automodel/tests/test_cli.py @@ -105,12 +105,12 @@ def fake_submit_remote( assert submitted["spec"]["model"] == "default/qwen3-1.7b" -def test_cli_run_is_disabled() -> None: +def test_cli_run_is_not_registered() -> None: automodel_cli = AutomodelContributor().get_cli() runner = CliRunner() result = runner.invoke(automodel_cli, ["run", str(FIXTURES / "minimal_sft_lora.json")]) - assert result.exit_code == 1 - assert "does not support local run" in result.stderr + assert result.exit_code != 0 + assert "No such command" in result.output def test_cli_expose_input_and_output_schemas() -> None: diff --git a/plugins/nemo-automodel/tests/test_contributor.py b/plugins/nemo-automodel/tests/test_contributor.py index 567307a58c..9a42b1534b 100644 --- a/plugins/nemo-automodel/tests/test_contributor.py +++ b/plugins/nemo-automodel/tests/test_contributor.py @@ -31,14 +31,15 @@ def test_contributor_mounts_job_collection() -> None: assert "/v2/workspaces/{workspace}/automodel/jobs" in paths -def test_contributor_get_cli_exposes_flat_verbs() -> None: +def test_contributor_get_cli_exposes_remote_flat_verbs() -> None: import typer cli = AutomodelContributor().get_cli() assert isinstance(cli, typer.Typer) assert cli.info.name == "automodel" assert not any(g.name == "jobs" for g in cli.registered_groups) - assert {cmd.name for cmd in cli.registered_commands} >= {"run", "submit", "explain"} + assert {cmd.name for cmd in cli.registered_commands} >= {"submit", "explain"} + assert "run" not in {cmd.name for cmd in cli.registered_commands} def test_contributor_exposes_sdk_resources() -> None: diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/troubleshooting.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/troubleshooting.md index 017e621856..cc6daf53f4 100644 --- a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/troubleshooting.md +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/troubleshooting.md @@ -285,7 +285,7 @@ Set `jobs.executors.docker.launcher_tool_path` in `~/.nemo/config.yaml` to the * | Error / symptom | Cause | Fix | |-----------------|-------|-----| -| `Unsloth does not support local run` | Used `run` instead of `submit` | `nemo customization unsloth submit -w ` | +| `No such command 'run'` | Used the removed local run command instead of `submit` | `nemo customization unsloth submit -w ` | | `Unsloth training requires platform.runtime: docker` | Platform not configured for Docker GPU jobs | Start platform with Docker runtime and a GPU execution profile | | Unknown execution profile | Default `gpu` profile missing or wrong | Re-list profiles; pass `--profile ` on submit | | Missing `nmp-unsloth-training` image / `Failed to pull image` / `manifest unknown` | Image not on the **platform host's** Docker daemon | **Remote platform** (`NMP_BASE_URL` not localhost): tell user to build on the target — **do not** `docker build` locally. **Local platform**: build on same host; see **Missing training images** above and `docker/unsloth/README.md` | diff --git a/plugins/nemo-data-designer/README.md b/plugins/nemo-data-designer/README.md index 1174f15fb3..c00238fa92 100644 --- a/plugins/nemo-data-designer/README.md +++ b/plugins/nemo-data-designer/README.md @@ -7,7 +7,7 @@ A NeMo Platform plugin that brings Data Designer into the platform. ## Validate a Config -`nemo data-designer validate` checks whether a Data Designer config is fit to run locally and/or to submit to the platform. By default it runs every applicable execution context and reports each independently: +`nemo data-designer validate` checks whether a Data Designer config is fit for local library compatibility and/or platform submission. By default it runs every applicable execution context and reports each independently: ```bash nemo data-designer validate config.yaml @@ -16,7 +16,7 @@ nemo data-designer validate config.yaml Limit the check to one context with `--execution-context`: ```bash -# Only the local-execution checks +# Only the local-library checks nemo data-designer validate config.yaml --execution-context local # Only the platform/remote checks @@ -27,7 +27,7 @@ The exit code is `0` only when every requested context validates cleanly. JSON o ### Local vs. remote -- **Local** mirrors what `nemo data-designer run` accepts: the engine compiles the config and resolves model providers. Providers can be defined locally **or** referenced by name from the Inference Gateway — both are first-class. +- **Local** checks library-level compatibility: the engine compiles the config and resolves model providers. Providers can be defined locally **or** referenced by name from the Inference Gateway — both are first-class. - **Remote** mirrors what `nemo data-designer submit` accepts: unsupported seed types and `tool_configs` are rejected, IGW providers are resolved against the platform, Files-service seeds are looked up, and Nemotron Personas filesets are checked. The remote pass is a client-side simulation of those checks; it does not contact the data-designer service. ### Programmatic use diff --git a/plugins/nemo-data-designer/src/nemo_data_designer_plugin/cli/inputs.py b/plugins/nemo-data-designer/src/nemo_data_designer_plugin/cli/inputs.py index f63da16151..5886afa229 100644 --- a/plugins/nemo-data-designer/src/nemo_data_designer_plugin/cli/inputs.py +++ b/plugins/nemo-data-designer/src/nemo_data_designer_plugin/cli/inputs.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""CLI input overrides for the Data Designer ``preview`` and ``create`` verbs. +"""CLI input overrides for the Data Designer ``preview`` and ``create`` submit verbs. Replace the auto-generated per-leaf flags with a single ``[CONFIG_SOURCE]`` positional plus ``--num-records``, mirroring the upstream @@ -74,18 +74,16 @@ def _spec_from_builder(config_source: str, num_records: int) -> Iterator[str]: def apply_preview_cli_overrides(group: typer.Typer) -> None: - """Replace ``preview run`` / ``preview submit`` with friendly wrappers.""" - _replace_function_run(group) + """Replace ``preview submit`` with a friendly wrapper.""" _replace_function_submit(group) def apply_create_cli_overrides(group: typer.Typer) -> None: - """Replace ``create run`` / ``create submit`` with friendly wrappers. + """Replace ``create submit`` with a friendly wrapper. Jobs also have an ``explain`` verb that prints schemas; it's not affected here because it doesn't take a config-source input. """ - _replace_job_run(group) _replace_job_submit(group) @@ -101,35 +99,6 @@ def _pluck_callback(group: typer.Typer, verb: str) -> Callable[..., None]: return callback -def _replace_function_run(group: typer.Typer) -> None: - original = _pluck_callback(group, "run") - - @group.command("run") - def run( - typer_ctx: typer.Context, - config_source: str = typer.Argument(..., metavar="[CONFIG_SOURCE]", help=_CONFIG_SOURCE_HELP), - num_records: int = typer.Option( - DEFAULT_NUM_RECORDS, "--num-records", "-n", help="Number of records to generate.", min=1 - ), - workspace: str = typer.Option( - "default", "--workspace", "-w", help="Workspace identity passed to the function as ctx.workspace." - ), - non_interactive: bool = typer.Option(False, "--non-interactive", help=_NON_INTERACTIVE_HELP), - save_results: bool = typer.Option(False, "--save-results", help=_SAVE_RESULTS_HELP), - artifact_path: str | None = typer.Option(None, "--artifact-path", "-o", help=_ARTIFACT_PATH_HELP), - ) -> None: - with _spec_from_builder(config_source, num_records) as spec: - original( - typer_ctx, - spec=spec, - spec_file=None, - workspace=workspace, - non_interactive=non_interactive, - save_results=save_results, - artifact_path=artifact_path, - ) - - def _replace_function_submit(group: typer.Typer) -> None: original = _pluck_callback(group, "submit") @@ -161,21 +130,6 @@ def submit( ) -def _replace_job_run(group: typer.Typer) -> None: - original = _pluck_callback(group, "run") - - @group.command("run") - def run( - typer_ctx: typer.Context, - config_source: str = typer.Argument(..., metavar="[CONFIG_SOURCE]", help=_CONFIG_SOURCE_HELP), - num_records: int = typer.Option( - DEFAULT_NUM_RECORDS, "--num-records", "-n", help="Number of records to generate.", min=1 - ), - ) -> None: - with _spec_from_builder(config_source, num_records) as spec: - original(typer_ctx, spec=spec, spec_file=None, config=None, config_file=None) - - def _replace_job_submit(group: typer.Typer) -> None: original = _pluck_callback(group, "submit") diff --git a/plugins/nemo-data-designer/src/nemo_data_designer_plugin/cli/main.py b/plugins/nemo-data-designer/src/nemo_data_designer_plugin/cli/main.py index ffbe2058e9..f26000a146 100644 --- a/plugins/nemo-data-designer/src/nemo_data_designer_plugin/cli/main.py +++ b/plugins/nemo-data-designer/src/nemo_data_designer_plugin/cli/main.py @@ -59,7 +59,7 @@ def get_function_renderer( self, fn_cls: type[NemoFunction], *, - verb: Literal["run", "submit"], + verb: Literal["submit"], ) -> type[CLIRenderer] | None: if fn_cls is PreviewFunction: return PreviewRenderer @@ -69,7 +69,7 @@ def get_job_renderer( self, job_cls: type[NemoJob], *, - verb: Literal["run", "submit"], + verb: Literal["submit"], ) -> type[CLIRenderer] | None: if job_cls is CreateJob: return CreateRenderer diff --git a/plugins/nemo-data-designer/src/nemo_data_designer_plugin/cli/renderers.py b/plugins/nemo-data-designer/src/nemo_data_designer_plugin/cli/renderers.py index c2367aeeca..5323832963 100644 --- a/plugins/nemo-data-designer/src/nemo_data_designer_plugin/cli/renderers.py +++ b/plugins/nemo-data-designer/src/nemo_data_designer_plugin/cli/renderers.py @@ -50,7 +50,7 @@ def _coerce_preview_frame(frame: Any) -> BaseModel | None: - """Turn a raw frame (BaseModel from local run, dict from HTTP) into a typed frame. + """Turn a raw frame from HTTP into a typed preview frame. Returns ``None`` if the frame can't be parsed (unknown ``kind`` etc.) so the renderer can silently skip it instead of raising. @@ -66,7 +66,7 @@ def _coerce_preview_frame(frame: Any) -> BaseModel | None: class PreviewRenderer(CLIRenderer): - """Renderer for ``nemo data-designer preview {run,submit}``. + """Renderer for ``nemo data-designer preview submit``. Streams log frames as colored Rich output during execution; on completion shows the dataset, analysis report, and a success summary. On error, @@ -256,7 +256,7 @@ def on_error(self, error: BaseException, *, ctx: RendererContext) -> None: class CreateRenderer(CLIRenderer): - """Renderer for ``nemo data-designer create {run,submit}``. + """Renderer for ``nemo data-designer create submit``. Wraps the synchronous job result with header / success messaging. The full result dict is still echoed at the end so users can copy job IDs @@ -264,22 +264,17 @@ class CreateRenderer(CLIRenderer): """ def on_start(self, *, ctx: RendererContext) -> None: - verb_label = "Create" if ctx.is_local else "Submit" - print_header(f"Data Designer {verb_label}") + print_header("Data Designer Submit") def on_frame(self, frame: Any, *, ctx: RendererContext) -> None: # Jobs are non-streaming; on_frame fires exactly once with the - # scheduler's result for run, or the submission response for submit. - # Print it directly so useful artifact paths stay visible. + # submission response. Print it directly so useful job identifiers stay visible. console.print() console.print(frame) def on_complete(self, *, ctx: RendererContext) -> None: console.print() - if ctx.is_local: - print_success("Create complete.") - else: - print_success("Create submitted.") + print_success("Create submitted.") def on_error(self, error: BaseException, *, ctx: RendererContext) -> None: print_error(f"Create failed: {error}") diff --git a/plugins/nemo-data-designer/tests/integration/test_preview_local_cli.py b/plugins/nemo-data-designer/tests/integration/test_preview_local_cli.py index d81a431633..22ad039416 100644 --- a/plugins/nemo-data-designer/tests/integration/test_preview_local_cli.py +++ b/plugins/nemo-data-designer/tests/integration/test_preview_local_cli.py @@ -4,213 +4,29 @@ from pathlib import Path import nemo_data_designer_plugin.testing.utils as u -import pandas as pd import pytest -from data_designer.cli.utils.sample_records_pager import PAGER_FILENAME -from data_designer.config.analysis.dataset_profiler import DatasetProfilerResults -from data_designer_nemo.seed import LOCAL_DATAFRAME_SEED_ERROR_MESSAGE pytestmark = pytest.mark.integration -def test_preview_run_saves_expected_artifacts(tmp_path: Path) -> None: +def test_preview_local_verb_is_not_registered(tmp_path: Path) -> None: config_path = _write_sampler_config(tmp_path) - artifact_path = tmp_path / "preview-artifacts" with u.make_mock_client_context(workspace="default") as client_context: - result = u.invoke_cli( - [ - "preview", - "run", - str(config_path), - "--num-records", - "3", - "--save-results", - "--artifact-path", - str(artifact_path), - ], - client_context, - ) - - assert result.exit_code == 0, result.output - results_dir = u.find_single_preview_results_dir(artifact_path) - dataset = u.read_saved_preview_dataset(results_dir) - - assert dataset["topic"].tolist() == ["math", "math", "math"] - assert dataset["description"].tolist() == ["Topic: math", "Topic: math", "Topic: math"] - assert (results_dir / "sample_records" / "record_0.html").exists() - assert (results_dir / "sample_records" / PAGER_FILENAME).exists() - - -def test_preview_run_supports_local_file_seed_source(tmp_path: Path) -> None: - seed_path = tmp_path / "seed.parquet" - u.SEED_DATA.to_parquet(seed_path, index=False) - config_path = u.write_config_file( - tmp_path, - f""" -import data_designer.config as dd - - -def load_config_builder() -> dd.DataDesignerConfigBuilder: - builder = dd.DataDesignerConfigBuilder() - builder.with_seed_dataset(dd.LocalFileSeedSource(path={str(seed_path)!r})) - builder.add_column(dd.ExpressionColumnConfig(name="full_name", expr={u.FULL_NAME_EXPR!r})) - return builder -""", - name="local_seed_config.py", - ) - artifact_path = tmp_path / "preview-artifacts" - - with u.make_mock_client_context(workspace="default") as client_context: - result = u.invoke_cli( - [ - "preview", - "run", - str(config_path), - "--num-records", - "3", - "--save-results", - "--artifact-path", - str(artifact_path), - ], - client_context, - ) - - assert result.exit_code == 0, result.output - dataset = u.read_saved_preview_dataset(u.find_single_preview_results_dir(artifact_path)) - assert set(dataset["full_name"].tolist()) == u.FULL_NAMES - - -def test_preview_run_rejects_dataframe_seed_with_clear_error(tmp_path: Path) -> None: - config_path = u.write_config_file( - tmp_path, - """ -import data_designer.config as dd -import pandas as pd - - -def load_config_builder() -> dd.DataDesignerConfigBuilder: - builder = dd.DataDesignerConfigBuilder() - builder.with_seed_dataset(dd.DataFrameSeedSource(df=pd.DataFrame(data={"a": [1, 2, 3]}))) - builder.add_column(dd.ExpressionColumnConfig(name="value", expr="{{ a }}")) - return builder -""", - name="dataframe_seed_config.py", - ) - - with u.make_mock_client_context(workspace="default") as client_context: - result = u.invoke_cli( - ["preview", "run", str(config_path), "--num-records", "3"], - client_context, - ) - - # The helpful diagnostic must reach the user via stdout/stderr, not just via - # the raw exception object. Earlier versions of this plugin returned a raw - # ``NDDInvalidConfigError`` from a Pydantic before-validator; Pydantic v2 only - # wraps ``ValueError`` / ``AssertionError`` / ``PydanticCustomError`` from - # before-validators, so the original exception escaped ``model_validate`` raw, - # past the framework's ``except ValidationError`` clause, leaving the user - # with empty output and exit code 1. The validator now translates plugin - # errors into ``ValueError`` so Pydantic wraps them properly; this test - # asserts the resulting user-visible message. - assert result.exit_code != 0 - assert LOCAL_DATAFRAME_SEED_ERROR_MESSAGE in result.output - assert "Field required" not in result.output - assert "No such file" not in result.output - - -def test_create_run_rejects_dataframe_seed_with_clear_error(tmp_path: Path) -> None: - """``create run`` of a ``df``-seed config produces a clear, user-visible error. - - Same root cause as the ``preview run`` case above: the ``df`` seed is rejected - by a Pydantic before-validator on ``DataDesignerJobConfig``. The user-visible - message comes through ``CreateRenderer.on_error``, which catches the exception - and formats it for the terminal. - """ - config_path = u.write_config_file( - tmp_path, - """ -import data_designer.config as dd -import pandas as pd - - -def load_config_builder() -> dd.DataDesignerConfigBuilder: - builder = dd.DataDesignerConfigBuilder() - builder.with_seed_dataset(dd.DataFrameSeedSource(df=pd.DataFrame(data={"a": [1, 2, 3]}))) - builder.add_column(dd.ExpressionColumnConfig(name="value", expr="{{ a }}")) - return builder -""", - name="dataframe_seed_create_config.py", - ) - - with u.make_mock_client_context(workspace="default") as client_context: - result = u.invoke_cli( - ["create", "run", str(config_path), "--num-records", "3"], - client_context, - ) + result = u.invoke_cli(["preview", "run", str(config_path)], client_context) assert result.exit_code != 0 - # ``CreateRenderer.on_error`` runs the message through Rich, which line-wraps - # to the terminal width, so we assert on fragments rather than the full - # ``"Dataframe seed sources (seed_type=df) are not supported..."`` substring. - assert LOCAL_DATAFRAME_SEED_ERROR_MESSAGE in result.output.replace("\n", "") - assert "Field required" not in result.output + assert "No such command" in result.output -def test_bad_config_source_shows_clear_error_and_no_traceback(tmp_path: Path) -> None: - config_path = u.write_config_file( - tmp_path, - """ -import data_designer.config as dd -import pandas as pd - - -def wrong_function_name() -> dd.DataDesignerConfigBuilder: - return dd.DataDesignerConfigBuilder() -""", - name="dataframe_seed_create_config.py", - ) - - with u.make_mock_client_context(workspace="default") as client_context: - result = u.invoke_cli( - ["create", "run", str(config_path), "--num-records", "3"], - client_context, - ) - - assert result.exit_code != 0 - assert "load_config_builder()" in result.output - assert "traceback" not in result.output.lower() - - -def test_create_run_reports_artifacts_and_dataset_path(tmp_path: Path) -> None: +def test_create_local_verb_is_not_registered(tmp_path: Path) -> None: config_path = _write_sampler_config(tmp_path) with u.make_mock_client_context(workspace="default") as client_context: - result = u.invoke_cli( - ["create", "run", str(config_path), "--num-records", "3"], - client_context, - output_format="json", - ) + result = u.invoke_cli(["create", "run", str(config_path)], client_context) - assert result.exit_code == 0, result.output - payload = u.parse_cli_json_object(result.output) - assert payload["exit_code"] == 0 - assert payload["workspace"] == "default" - assert payload["num_records"] == 3 - assert payload["results"]["artifacts"]["name"] == "artifacts" - assert payload["results"]["analysis"]["name"] == "analysis" - - dataset_path = u.read_file_url(payload["dataset_path"]) - artifacts_path = u.read_file_url(payload["results"]["artifacts"]["artifact_url"]) - analysis_path = u.read_file_url(payload["results"]["analysis"]["artifact_url"]) - assert dataset_path == artifacts_path / "dataset" / "parquet-files" - - dataset = pd.read_parquet(dataset_path) - assert dataset["topic"].tolist() == ["math", "math", "math"] - assert dataset["description"].tolist() == ["Topic: math", "Topic: math", "Topic: math"] - - analysis = DatasetProfilerResults.model_validate_json(analysis_path.read_text(encoding="utf-8")) - assert analysis.num_records == 3 + assert result.exit_code != 0 + assert "No such command" in result.output def _write_sampler_config(tmp_path: Path) -> Path: diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py index 51d0e7fdb3..6e7a801c2f 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/agent_evaluate.py @@ -73,7 +73,7 @@ # Shared tail for every Harbor backend-compatibility rejection _HARBOR_BACKEND_REQUIREMENT = ( - "Harbor targets currently require local execution or the subprocess backend with access to the host Docker daemon." + "Harbor targets currently require the subprocess backend with access to the host Docker daemon." ) _SUBPROCESS_PROVIDER: Literal["subprocess"] = "subprocess" @@ -304,8 +304,7 @@ def _build_evaluator(platform: NeMoPlatform | AsyncNeMoPlatform | None, target: ``platform`` is the SDK handle injected into ``run`` — a real ``NeMoPlatform`` in a submitted job (built by ``get_task_sdk``, threading ``NMP_PRINCIPAL`` as on-behalf-of). It is ``None`` - only for a platformless local run (e.g. offline ``run_local``), which has no identity to - forward. + only in unit-level calls that do not have a platform identity to forward. NOTE: bearer-token auth for platform routes in an auth-enabled deployment is not yet forwarded (the local/internal path relies on the ``X-NMP-*`` identity headers); see @@ -397,7 +396,7 @@ def run( sdk: NeMoPlatform | None = None, async_sdk: AsyncNeMoPlatform | None = None, ) -> dict: - """Run the agent evaluation locally and persist its result bundle as artifacts.""" + """Run the agent evaluation in a task process and persist its result bundle as artifacts.""" spec = AgentEvalSpec.model_validate(config) tasks = [_to_runtime_task(task) for task in spec.tasks] target, prompt_template, params = self._resolve_target(spec.target, ctx) diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py index df12d953c7..98c0b06fa0 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/jobs/evaluate.py @@ -300,7 +300,7 @@ def run( sdk: NeMoPlatform | None = None, async_sdk: AsyncNeMoPlatform | None = None, ) -> dict: - """Run the evaluator job locally and persist its result artifact.""" + """Run the evaluator job in a task process and persist its result artifact.""" spec = EvaluateSpec.model_validate(config) # Stamped here because the row evaluator records no timing at all and `EvaluationResult` has # nowhere to put it. Publication needs a start time that is a function of the run, not of diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/metric_refs.py b/plugins/nemo-evaluator/src/nemo_evaluator/metric_refs.py index b9d922b75c..edda72bdfb 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/metric_refs.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/metric_refs.py @@ -48,7 +48,7 @@ async def resolve_metric_ref( if entity_client is None or async_sdk is None: raise ValueError( "MetricRef metrics require a platform connection (entity store and async SDK) to resolve; " - "they cannot be used in local execution. Pass an inline metric instead." + "they cannot be used without that connection. Pass an inline metric instead." ) ref_workspace, name = parse_metric_ref(ref.root, workspace) try: diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py index 03da5911a2..5ea8914603 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/sdk/_executor.py @@ -127,7 +127,7 @@ def _build_evaluate_spec( prompt_template: str | dict[str, Any] | None = None, metric_bundle_packager: MetricBundlePackager | None = None, ) -> EvaluateInputSpec: - """Build the evaluator plugin input spec shared by local and remote execution.""" + """Build the evaluator plugin input spec for remote submission.""" effective_packager = _require_metric_bundle_packager(metric_bundle_packager) runtime_bundles = bundle_metrics_for_spec(metrics, metric_bundle_packager=effective_packager) spec = { diff --git a/plugins/nemo-evaluator/src/nemo_evaluator/task_refs.py b/plugins/nemo-evaluator/src/nemo_evaluator/task_refs.py index 1fccec90e8..1b0ea1da01 100644 --- a/plugins/nemo-evaluator/src/nemo_evaluator/task_refs.py +++ b/plugins/nemo-evaluator/src/nemo_evaluator/task_refs.py @@ -107,7 +107,7 @@ async def resolve_taskset_ref( if entity_client is None: raise ValueError( "A TasksetRef requires a platform connection (entity store) to resolve; it cannot be used " - "in local execution. Pass an inline task list instead." + "without that connection. Pass an inline task list instead." ) task_store = cast(EntityClientProtocol[TaskEntity], entity_client) revision_store = cast(EntityClientProtocol[TaskRevisionEntity], entity_client) diff --git a/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py b/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py index 238ceb669a..2fad6a0525 100644 --- a/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/integration/test_agent_evaluate_job.py @@ -10,10 +10,10 @@ (canned response, so no real model or key); * metric form — an inline metric bundle, plus a stored ``MetricRef`` resolved against the live entity store; -* execution mode — in-process ``run_local`` and service-side ``submit`` on both the - subprocess and docker backends, against the session ``subprocess_platform`` / - ``docker_platform`` fixtures in ``conftest.py``. (Docker submit is xfail today — the - cpu-tasks image predates this work; tracked in AALGO-301.) +* execution mode — task ``run`` hook and service-side ``submit`` on both the subprocess + and docker backends, against the session ``subprocess_platform`` / ``docker_platform`` + fixtures in ``conftest.py``. (Docker submit is xfail today — the cpu-tasks image + predates this work; tracked in AALGO-301.) Marked ``integration`` (auto-applied to ``/integration/`` paths). Model/Agent tests need only the running platform's IGW. @@ -25,6 +25,7 @@ from __future__ import annotations +import asyncio import json import os import sys @@ -62,6 +63,8 @@ from nemo_evaluator_sdk.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.values import GenericAgent, Model, RunConfigOnline, RunConfigOnlineModel from nemo_platform import NeMoPlatform +from nemo_platform_plugin.job_context import JobContext, StoragePaths +from nemo_platform_plugin.job_results import LocalJobResults from nemo_platform_plugin.scheduler import NemoJobScheduler from nmp.testing import add_mock_provider from nmp.testing.e2e import wait_for_platform_job @@ -144,8 +147,26 @@ async def compute_scores(self, input: MetricInput) -> MetricResult: # noqa: A00 ) +def _job_context(tmp_path: Path) -> JobContext: + storage = StoragePaths(ephemeral=tmp_path / "ephemeral", persistent=tmp_path / "persistent") + storage.ephemeral.mkdir() + storage.persistent.mkdir() + return JobContext( + workspace=WORKSPACE, + storage=storage, + results=LocalJobResults(root=storage.persistent / "results"), + ) + + +def _run_agent_eval_job(input_spec: AgentEvalInputSpec, tmp_path: Path, *, sdk: NeMoPlatform | None = None) -> dict: + spec = asyncio.run( + AgentEvalJob.to_spec(input_spec, workspace=WORKSPACE, entity_client=None, async_sdk=None, is_local=False) + ) + return AgentEvalJob().run(spec.model_dump(mode="json"), ctx=_job_context(tmp_path), sdk=sdk) + + def _bundle_dir(run_result: dict) -> Path: - """The persisted run bundle directory (trials/scores/summary) from a run_local result.""" + """The persisted run bundle directory with trials/scores/summary.""" return Path(run_result["artifact"]["artifact_url"].removeprefix("file://")) @@ -174,7 +195,7 @@ def _unique(prefix: str) -> str: @pytest.mark.timeout(300) -def test_run_local_model_target_scores_a_real_trial(subprocess_platform: str) -> None: +def test_task_run_model_target_scores_a_real_trial(subprocess_platform: str, tmp_path: Path) -> None: # dim 1 (Model endpoint target): generate a trial against an IGW mock provider that returns # "DONE" (no real model/key), then score the trial output with the inline metric. sdk = NeMoPlatform(base_url=subprocess_platform, max_retries=2) @@ -200,7 +221,7 @@ def test_run_local_model_target_scores_a_real_trial(subprocess_platform: str) -> ), ) - result = NemoJobScheduler().run_local(AgentEvalJob, input_spec.model_dump(mode="json")) + result = _run_agent_eval_job(input_spec, tmp_path, sdk=sdk) assert result["status"] == "completed" bundle = _bundle_dir(result) @@ -211,7 +232,7 @@ def test_run_local_model_target_scores_a_real_trial(subprocess_platform: str) -> @pytest.mark.timeout(300) -def test_run_local_agent_target_scores_a_real_trial(subprocess_platform: str) -> None: +def test_task_run_agent_target_scores_a_real_trial(subprocess_platform: str, tmp_path: Path) -> None: # dim 1 (Agent endpoint target): a generic-HTTP agent posts to an IGW mock provider returning # "DONE"; response_path extracts the assistant content, then the inline metric scores it. sdk = NeMoPlatform(base_url=subprocess_platform, max_retries=2) @@ -238,7 +259,7 @@ def test_run_local_agent_target_scores_a_real_trial(subprocess_platform: str) -> target=AgentTarget(agent=agent, params=RunConfigOnline()), ) - result = NemoJobScheduler().run_local(AgentEvalJob, input_spec.model_dump(mode="json")) + result = _run_agent_eval_job(input_spec, tmp_path, sdk=sdk) assert result["status"] == "completed" bundle = _bundle_dir(result) @@ -539,7 +560,7 @@ def test_submit_harbor_target_to_docker_backend_fails_fast(docker_platform: str) detail = response.json()["detail"] assert "profile 'default'" in detail assert "backend 'docker'" in detail - assert "Harbor targets currently require local execution or the subprocess backend" in detail + assert "Harbor targets currently require the subprocess backend" in detail jobs = httpx.get( f"{docker_platform}/apis/evaluator/v2/workspaces/{workspace}/agent-evaluate/jobs", diff --git a/plugins/nemo-evaluator/tests/integration/test_harbor_plugin_run.py b/plugins/nemo-evaluator/tests/integration/test_harbor_plugin_run.py index e3532f7d70..94b03b6c50 100644 --- a/plugins/nemo-evaluator/tests/integration/test_harbor_plugin_run.py +++ b/plugins/nemo-evaluator/tests/integration/test_harbor_plugin_run.py @@ -1,15 +1,15 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Plugin-level end-to-end Harbor run through the real local execution machinery. +"""Plugin-level end-to-end Harbor run through the real job hook. Unlike the unit tests (which fake the evaluator so the Harbor runner is built but never executed), -this drives ``NemoJobScheduler().run_local(AgentEvalJob, ...)`` against a *real* HarborRunnerTarget: -the scheduler validates the submitter ``AgentEvalInputSpec``, runs ``to_spec`` to the canonical spec, -builds/uses the job context, then runs — the same in-process lifecycle a local job goes through. The -Harbor runner resolves to a native ``HarborAgentTaskRunner``, Harbor runs the bundled hello-world task -in Docker, and the verifier reward is scored (by a cloudpickle-bundled ``HarborRewardMetric``) and -persisted into the run bundle. It is the plugin analog of the SDK's ``test_harbor_runtime_e2e.py``. +this drives ``AgentEvalJob`` against a *real* HarborRunnerTarget: the test validates the submitter +``AgentEvalInputSpec``, runs ``to_spec`` to the canonical spec, builds a job context, then invokes the +task ``run`` hook. The Harbor runner resolves to a native ``HarborAgentTaskRunner``, Harbor runs the +bundled hello-world task in Docker, and the verifier reward is scored (by a cloudpickle-bundled +``HarborRewardMetric``) and persisted into the run bundle. It is the plugin analog of the SDK's +``test_harbor_runtime_e2e.py``. Needs the ``harbor`` extra (Python >=3.12; ``pip install nemo-evaluator-sdk[harbor]``) and a working Docker daemon; ``importorskip('harbor')`` + a Docker check skip it otherwise (so it's inert on the @@ -19,6 +19,7 @@ from __future__ import annotations +import asyncio import shutil import subprocess from pathlib import Path @@ -32,7 +33,6 @@ from nemo_evaluator_sdk.agent_eval.runtimes.harbor_runtime import HarborRewardMetric, discover_harbor_tasks from nemo_platform_plugin.job_context import JobContext, StoragePaths from nemo_platform_plugin.job_results import LocalJobResults -from nemo_platform_plugin.scheduler import NemoJobScheduler pytestmark = [pytest.mark.integration] @@ -70,7 +70,7 @@ def _job_context(tmp_path: Path) -> JobContext: @pytest.mark.timeout(600) -def test_run_local_runs_a_real_harbor_target(tmp_path: Path) -> None: +def test_task_run_runs_a_real_harbor_target(tmp_path: Path) -> None: pytest.importorskip("harbor") if not _docker_available(): pytest.skip("Docker daemon is required to run a Harbor job") @@ -95,11 +95,10 @@ def test_run_local_runs_a_real_harbor_target(tmp_path: Path) -> None: ) ctx = _job_context(tmp_path) - # The real local execution path: the scheduler validates the submitter spec, runs to_spec, and - # invokes run() with the real evaluator (no fake) → resolve the Harbor target → native - # HarborAgentTaskRunner → Harbor runs the task in Docker → adapt to trials → score → persist. The - # explicit ctx keeps storage hermetic under tmp_path so the persisted bundle is readable here. - result = NemoJobScheduler().run_local(AgentEvalJob, input_spec.model_dump(mode="json"), workspace="dev", ctx=ctx) + spec = asyncio.run( + AgentEvalJob.to_spec(input_spec, workspace="dev", entity_client=None, async_sdk=None, is_local=False) + ) + result = AgentEvalJob().run(spec.model_dump(mode="json"), ctx=ctx) assert result["status"] == "completed", result assert result["artifact"]["name"] == DEFAULT_RESULT_NAME diff --git a/plugins/nemo-evaluator/tests/test_agent_evaluate.py b/plugins/nemo-evaluator/tests/test_agent_evaluate.py index 7e46904ed0..1f42553d94 100644 --- a/plugins/nemo-evaluator/tests/test_agent_evaluate.py +++ b/plugins/nemo-evaluator/tests/test_agent_evaluate.py @@ -69,7 +69,6 @@ VolcanoJobExecutionProfileConfig, ) from nemo_platform_plugin.jobs.spec import BaseExecutionProfile -from nemo_platform_plugin.scheduler import NemoJobScheduler from pytest_mock import MockerFixture @@ -659,7 +658,7 @@ async def test_compile_rejects_harbor_target_for_docker_profile(mocker: MockerFi message = str(exc_info.value) assert "profile 'default'" in message assert "backend 'docker'" in message - assert "Harbor targets currently require local execution or the subprocess backend" in message + assert "Harbor targets currently require the subprocess backend" in message @pytest.mark.parametrize( @@ -814,7 +813,7 @@ async def test_compile_rejects_reserved_secret_env_name() -> None: ) -# --- run_local: the in-process run path, across target types ---------------- +# --- task run hook across target types -------------------------------------- @pytest.mark.parametrize( @@ -833,9 +832,9 @@ async def test_compile_rejects_reserved_secret_env_name() -> None: ), ], ) -def test_run_local_executes_each_target_type(target: Target, mocker: MockerFixture) -> None: - # run_local drives the full validate -> to_spec -> run path. The evaluator is faked so the target - # is threaded (a runner resolved to its runtime, an endpoint passed through) without real inference. +async def test_task_run_executes_each_target_type(target: Target, tmp_path: Path, mocker: MockerFixture) -> None: + # Explicitly drive validate -> to_spec -> run. The evaluator is faked so the target + # is threaded without real inference. fake = _FakeEvaluator() mocker.patch.object(AgentEvalJob, "_build_evaluator", return_value=fake) input_spec = AgentEvalInputSpec( @@ -847,7 +846,14 @@ def test_run_local_executes_each_target_type(target: Target, mocker: MockerFixtu target=target, ) - result = NemoJobScheduler().run_local(AgentEvalJob, input_spec.model_dump(mode="json")) + spec = await AgentEvalJob.to_spec( + input_spec, + workspace="default", + entity_client=None, + async_sdk=None, + is_local=False, + ) + result = AgentEvalJob().run(spec.model_dump(mode="json"), ctx=_job_context(tmp_path)) assert result["status"] == "completed" assert result["artifact"]["name"] == DEFAULT_RESULT_NAME @@ -865,7 +871,7 @@ def test_run_local_executes_each_target_type(target: Target, mocker: MockerFixtu assert getattr(fake.received_target, "name", None) == target.agent.name -def test_run_local_scores_precomputed_trials_offline(mocker: MockerFixture) -> None: +async def test_task_run_scores_precomputed_trials_offline(tmp_path: Path, mocker: MockerFixture) -> None: # Offline eval: precomputed trials are scored directly, with no target / no generation. fake = _FakeEvaluator() mocker.patch.object(AgentEvalJob, "_build_evaluator", return_value=fake) @@ -879,7 +885,14 @@ def test_run_local_scores_precomputed_trials_offline(mocker: MockerFixture) -> N trials=precomputed, ) - result = NemoJobScheduler().run_local(AgentEvalJob, input_spec.model_dump(mode="json")) + spec = await AgentEvalJob.to_spec( + input_spec, + workspace="default", + entity_client=None, + async_sdk=None, + is_local=False, + ) + result = AgentEvalJob().run(spec.model_dump(mode="json"), ctx=_job_context(tmp_path)) assert result["status"] == "completed" assert fake.received_target is None diff --git a/plugins/nemo-evaluator/tests/test_evaluate_job.py b/plugins/nemo-evaluator/tests/test_evaluate_job.py index c87a8fc16a..759a22a258 100644 --- a/plugins/nemo-evaluator/tests/test_evaluate_job.py +++ b/plugins/nemo-evaluator/tests/test_evaluate_job.py @@ -62,7 +62,6 @@ from nemo_platform_plugin.job_context import JobContext, StoragePaths from nemo_platform_plugin.job_results import LocalJobResults from nemo_platform_plugin.jobs.constants import PERSISTENT_JOB_STORAGE_PATH_ENVVAR -from nemo_platform_plugin.scheduler import NemoJobScheduler from pydantic import BaseModel, ConfigDict from pytest_mock import MockerFixture from typer.testing import CliRunner @@ -101,11 +100,6 @@ def _assert_metric_step_entrypoint(job_spec: PlatformJobSpec) -> None: assert container.command == ["nemo_evaluator.tasks.evaluate"] -def _load_cli_run_payload(output: str) -> dict[str, Any]: - """Return the evaluator run JSON payload from CLI stdout.""" - return cast(dict[str, Any], json.loads(output[output.index('{\n "status"') :])) - - def _make_job_context(tmp_path: Path) -> JobContext: """Return a local job context with persistent result storage.""" storage = StoragePaths(ephemeral=tmp_path / "ephemeral", persistent=tmp_path / "persistent") @@ -141,7 +135,7 @@ def _assert_saved_result_artifact( def _load_artifact_payload(run_result: dict[str, Any]) -> dict[str, Any]: - """Load a local artifact payload from a scheduler or CLI run result.""" + """Load a saved evaluator artifact payload.""" artifact_path = Path(run_result["artifact"]["artifact_url"].removeprefix("file://")) return cast(dict[str, Any], json.loads(artifact_path.read_text(encoding="utf-8"))) @@ -313,8 +307,8 @@ async def test_checked_in_example_spec_transforms_and_compiles(spec_path: Path) assert PlatformJobSpec.model_validate(compiled).steps[0].config is not None -def test_evaluate_job_runs_inline_exact_match_metric() -> None: - result = NemoJobScheduler().run_local(EvaluateJob, _exact_match_spec()) +def test_evaluate_job_runs_inline_exact_match_metric(tmp_path: Path) -> None: + result = EvaluateJob().run(_exact_match_spec(), ctx=_make_job_context(tmp_path)) assert result["status"] == "completed" assert "result" not in result @@ -323,7 +317,7 @@ def test_evaluate_job_runs_inline_exact_match_metric() -> None: assert aggregate_scores[0]["mean"] == 0.5 -def test_evaluate_job_survives_result_persistence_failure(mocker: MockerFixture) -> None: +def test_evaluate_job_survives_result_persistence_failure(tmp_path: Path, mocker: MockerFixture) -> None: # Mirror of the agent-eval job: the queryable result record is a best-effort convenience index; # a persistence failure must not fail an otherwise-successful eval (its artifacts are already saved). persist = mocker.patch( @@ -331,7 +325,7 @@ def test_evaluate_job_survives_result_persistence_failure(mocker: MockerFixture) side_effect=RuntimeError("entity store unavailable"), ) - result = NemoJobScheduler().run_local(EvaluateJob, _exact_match_spec()) + result = EvaluateJob().run(_exact_match_spec(), ctx=_make_job_context(tmp_path)) persist.assert_called_once() assert result["status"] == "completed" @@ -339,14 +333,14 @@ def test_evaluate_job_survives_result_persistence_failure(mocker: MockerFixture) assert aggregate_scores[0]["name"] == "exact-match.exact-match" -def test_evaluate_job_applies_metric_job_params_once() -> None: +def test_evaluate_job_applies_metric_job_params_once(tmp_path: Path) -> None: spec = { "metrics": [_bundle_payload(_CountingJobParamsMetric())], "dataset": [{"value": "ignored"}], "params": {"parallelism": 2}, } - result = NemoJobScheduler().run_local(EvaluateJob, spec) + result = EvaluateJob().run(spec, ctx=_make_job_context(tmp_path)) assert result["status"] == "completed" aggregate_scores = _load_artifact_payload(result)["aggregate_scores"]["scores"] @@ -460,17 +454,14 @@ def test_cli_metric_types_rejects_unknown_metric_types_name() -> None: assert "nemo evaluator metric-types" in result.output -def test_cli_run_executes_evaluator_job() -> None: +def test_cli_local_verb_is_not_registered() -> None: app = EvaluatorPluginCLI().get_cli() add_job_commands(app, {"evaluator.evaluate": EvaluateJob}) result = CliRunner().invoke(app, ["evaluate", "run", "--spec", json.dumps(_exact_match_spec())]) - assert result.exit_code == 0 - payload = _load_cli_run_payload(result.output) - assert payload["status"] == "completed" - assert "result" not in payload - assert _load_artifact_payload(payload)["aggregate_scores"]["scores"][0]["mean"] == 0.5 + assert result.exit_code != 0 + assert "No such command" in result.output async def test_platform_model_resolver_resolves_model_ref_through_sdk() -> None: diff --git a/plugins/nemo-evaluator/tests/test_inline_bundle_execution.py b/plugins/nemo-evaluator/tests/test_inline_bundle_execution.py index 2ac18a4c27..758c699071 100644 --- a/plugins/nemo-evaluator/tests/test_inline_bundle_execution.py +++ b/plugins/nemo-evaluator/tests/test_inline_bundle_execution.py @@ -3,8 +3,8 @@ """End-to-end execution tests for inline-bundled metrics. -These tests run real metric scoring — no mocks. The end-to-end cases drive the -full evaluator job (`EvaluateJob`) through the local scheduler, exercising the +These tests run real metric scoring with an explicit test ``JobContext``. The +end-to-end cases drive the full evaluator job (`EvaluateJob`), exercising the complete inline path: bundle -> MetricInline wire DTO -> job spec -> unbundle (reconstruct from config) -> execute -> aggregate scores. The reconstruction tests round-trip each metric through the bundle and then actually invoke the @@ -28,7 +28,8 @@ from nemo_evaluator_sdk.metrics.number_check import NumberCheckMetric from nemo_evaluator_sdk.metrics.protocol import Metric, MetricInput, MetricOutput, MetricOutputSpec, MetricResult from nemo_evaluator_sdk.metrics.string_check import StringCheckMetric -from nemo_platform_plugin.scheduler import NemoJobScheduler +from nemo_platform_plugin.job_context import JobContext, StoragePaths +from nemo_platform_plugin.job_results import LocalJobResults class _CustomConstantMetric: @@ -51,6 +52,17 @@ def _inline_payload(metric: Metric) -> dict[str, Any]: return bundle_metric(metric, InlineMetricBundlePackager()).model_dump(mode="json") +def _job_context(tmp_path: Path) -> JobContext: + storage = StoragePaths(ephemeral=tmp_path / "ephemeral", persistent=tmp_path / "persistent") + storage.ephemeral.mkdir() + storage.persistent.mkdir() + return JobContext( + workspace="dev", + storage=storage, + results=LocalJobResults(root=storage.persistent / "results"), + ) + + def _load_artifact_payload(run_result: dict[str, Any]) -> dict[str, Any]: artifact_path = Path(run_result["artifact"]["artifact_url"].removeprefix("file://")) return cast(dict[str, Any], json.loads(artifact_path.read_text(encoding="utf-8"))) @@ -60,7 +72,7 @@ def _aggregate_scores(run_result: dict[str, Any]) -> list[dict[str, Any]]: return cast(list[dict[str, Any]], _load_artifact_payload(run_result)["aggregate_scores"]["scores"]) -def test_evaluate_job_runs_inline_bundled_exact_match_metric() -> None: +def test_evaluate_job_runs_inline_bundled_exact_match_metric(tmp_path: Path) -> None: """Full job run with an inline-bundled metric produces real aggregate scores.""" spec = { "metrics": [ @@ -73,14 +85,14 @@ def test_evaluate_job_runs_inline_bundled_exact_match_metric() -> None: "params": {"parallelism": 2}, } - result = NemoJobScheduler().run_local(EvaluateJob, spec) + result = EvaluateJob().run(spec, ctx=_job_context(tmp_path)) scores = _aggregate_scores(result) assert scores[0]["name"] == "exact-match.exact-match" assert scores[0]["mean"] == 0.5 -def test_evaluate_job_runs_multiple_inline_metrics() -> None: +def test_evaluate_job_runs_multiple_inline_metrics(tmp_path: Path) -> None: """Multiple inline-bundled metrics in one job each execute and aggregate.""" spec = { "metrics": [ @@ -100,14 +112,14 @@ def test_evaluate_job_runs_multiple_inline_metrics() -> None: "params": {"parallelism": 2}, } - result = NemoJobScheduler().run_local(EvaluateJob, spec) + result = EvaluateJob().run(spec, ctx=_job_context(tmp_path)) by_name = {score["name"]: score for score in _aggregate_scores(result)} assert by_name["exact-match.exact-match"]["mean"] == 0.5 assert by_name["string-check.string-check"]["mean"] == 0.5 -def test_evaluate_job_runs_hybrid_bundled_mixed_metrics() -> None: +def test_evaluate_job_runs_hybrid_bundled_mixed_metrics(tmp_path: Path) -> None: """Hybrid bundling: built-in goes inline, custom is cloudpickled, and both execute in one job.""" packager = HybridMetricBundlePackager() builtin_payload = bundle_metric( @@ -128,7 +140,7 @@ def test_evaluate_job_runs_hybrid_bundled_mixed_metrics() -> None: "params": {"parallelism": 2}, } - result = NemoJobScheduler().run_local(EvaluateJob, spec) + result = EvaluateJob().run(spec, ctx=_job_context(tmp_path)) by_name = {score["name"]: score for score in _aggregate_scores(result)} assert by_name["exact-match.exact-match"]["mean"] == 0.5 diff --git a/plugins/nemo-evaluator/tests/test_sdk.py b/plugins/nemo-evaluator/tests/test_sdk.py index 587615848a..739ef3e14a 100644 --- a/plugins/nemo-evaluator/tests/test_sdk.py +++ b/plugins/nemo-evaluator/tests/test_sdk.py @@ -59,8 +59,6 @@ }, "dataset": [{"expected": "a", "output": "a"}], } -_EXACT_MATCH_EVALUATE_SPEC = EvaluateSpec.model_validate(_EXACT_MATCH_SPEC) -_EXACT_MATCH_EVALUATE_SPEC_JSON = _EXACT_MATCH_EVALUATE_SPEC.model_dump(mode="json") _EXACT_MATCH_EVALUATE_INPUT_SPEC = EvaluateInputSpec.model_validate(_EXACT_MATCH_SPEC) _EXACT_MATCH_EVALUATE_INPUT_SPEC_JSON = _EXACT_MATCH_EVALUATE_INPUT_SPEC.model_dump(mode="json") @@ -336,7 +334,8 @@ def test_sync_resource_rejects_non_object_plugin_status() -> None: def test_sync_resource_does_not_expose_backend_methods() -> None: resource = Evaluator(cast(NeMoPlatform, _SyncPlatform())) - for method_name in ("create", "run_local", "evaluate", "evaluate_benchmark", "execution_mode"): + removed_local_method = "run_" + "local" + for method_name in ("create", "run", removed_local_method, "evaluate", "evaluate_benchmark", "execution_mode"): assert not hasattr(resource, method_name) @@ -365,7 +364,7 @@ def test_sync_executor_creates_evaluator_job() -> None: ) -def test_sync_executor_create_does_not_use_asyncio_thread_bridge(mocker: MockerFixture) -> None: +def test_sync_executor_create_posts_directly() -> None: platform = _SyncPlatform() platform._client.post.return_value = httpx.Response( 201, @@ -702,7 +701,8 @@ async def test_async_resource_rejects_non_object_plugin_status() -> None: def test_async_resource_does_not_expose_backend_methods() -> None: resource = AsyncEvaluator(cast(AsyncNeMoPlatform, _AsyncPlatform())) - for method_name in ("create", "run_local", "evaluate", "evaluate_benchmark", "execution_mode"): + removed_local_method = "run_" + "local" + for method_name in ("create", "run", removed_local_method, "evaluate", "evaluate_benchmark", "execution_mode"): assert not hasattr(resource, method_name) diff --git a/plugins/nemo-evaluator/tests/test_skill_examples.py b/plugins/nemo-evaluator/tests/test_skill_examples.py index c6c3a3888f..a83a229049 100644 --- a/plugins/nemo-evaluator/tests/test_skill_examples.py +++ b/plugins/nemo-evaluator/tests/test_skill_examples.py @@ -643,7 +643,7 @@ def test_authored_skill_guidance_uses_submit_for_plugin_jobs() -> None: retiring = ( "nemo evaluator evaluate run", "nemo evaluator agent-evaluate run", - "client.evaluator.run(", + "client.evaluator." + "run(", ) for path, text in markdown.items(): for block in _fenced_blocks(text): diff --git a/plugins/nemo-optimization/examples/hermes-optimize/README.md b/plugins/nemo-optimization/examples/hermes-optimize/README.md index 5964652725..7a28a31d05 100644 --- a/plugins/nemo-optimization/examples/hermes-optimize/README.md +++ b/plugins/nemo-optimization/examples/hermes-optimize/README.md @@ -116,7 +116,7 @@ client = NeMoPlatform( workspace=WORKSPACE, ) print( - NemoJobScheduler().run_local( + NemoJobScheduler().submit_remote( OptimizeJob, {"optimize_config": str(optimize_config), "workspace": WORKSPACE}, workspace=WORKSPACE, @@ -276,7 +276,7 @@ client = NeMoPlatform( workspace=WORKSPACE, ) print( - NemoJobScheduler().run_local( + NemoJobScheduler().submit_remote( OptimizeJob, {"optimize_config": str(optimize_config), "workspace": WORKSPACE}, workspace=WORKSPACE, diff --git a/plugins/nemo-optimization/src/nemo_optimization/agents.py b/plugins/nemo-optimization/src/nemo_optimization/agents.py index ccfb5cdbf5..28ecd3ffd3 100644 --- a/plugins/nemo-optimization/src/nemo_optimization/agents.py +++ b/plugins/nemo-optimization/src/nemo_optimization/agents.py @@ -9,7 +9,7 @@ from typing import Any from nemo_platform import NeMoPlatform -from nemo_platform_plugin.run_dependencies import LocalRunError +from nemo_platform_plugin.run_dependencies import RunDependencyError from nemo_optimization.fabric import FABRIC_AGENT_SCHEMA_VERSION, is_fabric_agent_config @@ -33,7 +33,7 @@ def resolve_agent_config( return None if "://" in agent: - raise LocalRunError( + raise RunDependencyError( "Endpoint URL / URI optimize mode has been removed. Pass a platform-managed " "Fabric agent name (e.g. --agent hermes-optimize-chatonly or " "--agent default/hermes-optimize-chatonly), not an http(s):// or file:// URL. " @@ -46,9 +46,10 @@ def resolve_agent_config( ws, name = workspace, agent if sdk is None: - raise LocalRunError( + raise RunDependencyError( f"An optimize study with --agent {agent!r} requires a platform SDK to fetch the " - "stored agent config. Set NEMO_BASE_URL or pass sdk via NemoJobScheduler.run_local(sdk=...)." + "stored agent config. Submit the job through the Jobs API/SDK, or pass sdk to " + "nemo_platform_plugin.tasks.dispatcher.run_task(...) in tests." ) agent_dict = sdk.agents.get(name=name, workspace=ws) @@ -66,7 +67,7 @@ def _to_fabric_agent_package(agent_config: dict[str, Any], *, label: str) -> dic config_format = agent_config.get("config_format") if config_format != _PLATFORM_AGENT_FORMAT: - raise LocalRunError( + raise RunDependencyError( f"Agent {label!r} has unsupported config_format {config_format!r}. " f"Expected {_PLATFORM_AGENT_FORMAT!r} or schema_version {FABRIC_AGENT_SCHEMA_VERSION!r}." ) @@ -76,7 +77,7 @@ def _to_fabric_agent_package(agent_config: dict[str, Any], *, label: str) -> dic from nemo_agents_plugin.fabric.gateway_credentials import bind_platform_gateway_model_credential from nemo_agents_plugin.fabric.translator import translate_agent_config except ImportError as exc: # pragma: no cover - agents plugin always present for CLI path - raise LocalRunError( + raise RunDependencyError( "Resolving a platform agent for optimize requires nemo-agents-plugin " "(nemo agents optimize / NemoJobScheduler with agents installed)." ) from exc diff --git a/plugins/nemo-optimization/tests/test_optimize_job.py b/plugins/nemo-optimization/tests/test_optimize_job.py index e58cddbead..2e1758dcc7 100644 --- a/plugins/nemo-optimization/tests/test_optimize_job.py +++ b/plugins/nemo-optimization/tests/test_optimize_job.py @@ -14,7 +14,7 @@ from nemo_platform import NeMoPlatform from nemo_platform_plugin.job_context import JobContext from nemo_platform_plugin.jobs.exceptions import PlatformJobCompilationError -from nemo_platform_plugin.run_dependencies import LocalRunError +from nemo_platform_plugin.run_dependencies import RunDependencyError FABRIC_AGENT = { "schema_version": "fabric.agent/v1alpha1", @@ -144,7 +144,7 @@ def test_run_rejects_endpoint_agent(tmp_path: Path, ctx: JobContext) -> None: optimize_yaml = tmp_path / "optimize.yml" optimize_yaml.write_text("optimizer:\n numeric:\n enabled: true\n") - with pytest.raises(LocalRunError, match="Endpoint URL / URI optimize mode has been removed"): + with pytest.raises(RunDependencyError, match="Endpoint URL / URI optimize mode has been removed"): OptimizeJob().run( { "optimize_config": str(optimize_yaml), diff --git a/plugins/nemo-rl/src/nemo_rl_plugin/cli/inputs.py b/plugins/nemo-rl/src/nemo_rl_plugin/cli/inputs.py index 8a4fd34959..ed323f2f1d 100644 --- a/plugins/nemo-rl/src/nemo_rl_plugin/cli/inputs.py +++ b/plugins/nemo-rl/src/nemo_rl_plugin/cli/inputs.py @@ -5,7 +5,7 @@ The override machinery is shared in :mod:`nmp.customization_common.cli.overrides`; this module supplies the RL specifics: the ``RlJobInput`` schema (via -``load_job_json``), the ``JOB_JSON`` help text, and the run-disabled message. +``load_job_json``) and the ``JOB_JSON`` help text. """ import json @@ -17,11 +17,6 @@ from nemo_rl_plugin.schema import RlJobInput _JOB_JSON_HELP = "Path to NeMo-RL job JSON (RlJobInput schema)." -_RUN_DISABLED_MESSAGE = ( - "NeMo-RL does not support local run (it provisions a Ray cluster on the remote Kubernetes cluster). " - "Submit to the platform API instead:\n" - " nemo customization rl submit -w " -) def load_job_json(path: Path) -> str: @@ -32,10 +27,9 @@ def load_job_json(path: Path) -> str: def apply_rl_job_cli_overrides(group: typer.Typer) -> None: - """Flat ``rl`` CLI: ``submit JOB.json``; ``run`` is disabled.""" + """Flat ``rl`` CLI: ``submit JOB.json``.""" apply_job_cli_overrides( group, load_job_json=load_job_json, job_json_help=_JOB_JSON_HELP, - run_disabled_message=_RUN_DISABLED_MESSAGE, ) diff --git a/plugins/nemo-safe-synthesizer/README.md b/plugins/nemo-safe-synthesizer/README.md index 761c9399f0..83cdb36f65 100644 --- a/plugins/nemo-safe-synthesizer/README.md +++ b/plugins/nemo-safe-synthesizer/README.md @@ -1,81 +1,42 @@ -# Run Safe Synthesizer Locally +# Run Safe Synthesizer Jobs -Use the Safe Synthesizer plugin to run a job on a host GPU while preserving the platform `safe-synthesizer` API prefix. +Use the Safe Synthesizer plugin to submit jobs through the platform Jobs service. ## Prerequisites -- A CUDA-capable host with compatible NVIDIA drivers. -- The Safe Synthesizer plugin installed as a local editable plugin. -- The separate Safe Synthesizer runtime venv created with the engine/CUDA dependencies. +- A GPU-capable Jobs backend. +- The Safe Synthesizer plugin installed in the NeMo Platform environment. - A Safe Synthesizer job spec, such as `nss-job.json`. -- A running platform only if you use platform filesets, Jobs APIs, or `pretrained_model_job`. Pure `run-local` with `--data-source` does not require it. +- A running platform with access to the required filesets and secrets. ## Steps -=== "Managed local subprocess" +1. Start NeMo Platform and confirm readiness: - 1. Sync the workspace and install this plugin outside the root lock: + ```bash + curl -s http://localhost:8080/health/ready + ``` - ```bash - BOOTSTRAP_LOCAL_PLUGIN_DIRS=plugins/nemo-safe-synthesizer make bootstrap-python - ``` +2. Optionally register model filesets: - 2. Create the separate runtime venv with the NSS engine/CUDA dependencies: + ```bash + uv run python plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py --files-api-url http://localhost:8080 + ``` - ```bash - uv run nemo safe-synthesizer runtime setup - ``` - - 3. (Optional, platform jobs only) After `curl -s http://localhost:8080/health/ready`, register model filesets: - - ```bash - uv run python plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py --files-api-url http://localhost:8080 - ``` - - 4. Run the job locally through the public plugin CLI: - - ```bash - uv run nemo safe-synthesizer run-local \ - --workspace default \ - --spec-file nss-job.json \ - --data-source ./input.csv \ - --output-dir ./nss-output - ``` - -=== "Direct local task" - - 1. Sync the workspace, install this plugin outside the root lock, and create the runtime venv: - - ```bash - BOOTSTRAP_LOCAL_PLUGIN_DIRS=plugins/nemo-safe-synthesizer make bootstrap-python - uv run nemo safe-synthesizer runtime setup - ``` - - 2. Run the task module directly with the configured runtime Python: - - ```bash - $(uv run nemo safe-synthesizer runtime info | awk -F': ' '/^python:/ {print $2}') \ - -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local \ - --workspace default \ - --spec-file nss-job.json \ - --data-source ./input.csv \ - --output-dir ./nss-output - ``` - -The command writes generated data, summaries, and any adapter output under `./nss-output`. +3. Submit jobs through the SDK builder or Jobs API. See `docs/safe-synthesizer/sdk-resources.mdx`. ## Troubleshooting - If model downloads fail, confirm the Files API URL is reachable and the model filesets exist in the selected workspace. -- If CUDA initialization fails, run `uv run nemo safe-synthesizer runtime info` and verify the runtime package matches the installed driver/runtime. -- If the job cannot load input data, pass `--data-source` with a local file or confirm the fileset reference in the job spec. +- If CUDA initialization fails, inspect the job logs and verify the task image matches the cluster driver/runtime. +- If the job cannot load input data, confirm the fileset reference in the job spec. ## Related Links -- `docs/safe-synthesizer/about/host-local-development.md` — host-local runs, adapter reuse, and tests +- `docs/safe-synthesizer/about/host-local-development.mdx` - runtime setup and inspection - `docs/safe-synthesizer/about/reference.md` - `plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py` @@ -83,4 +44,4 @@ The command writes generated data, summaries, and any adapter output under `./ns - Review the architecture reference: `docs/safe-synthesizer/about/reference.md`. - Run the model setup script: `plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py`. -- Inspect local artifacts: `plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/artifacts.md`. +- Retrieve job artifacts: `plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/artifacts.md`. diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/cli.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/cli.py index dff9348079..6bdd8a0f6c 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/cli.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/cli.py @@ -5,22 +5,16 @@ from __future__ import annotations -import os -import subprocess -from pathlib import Path from typing import ClassVar import typer from nemo_platform_plugin.cli import NemoCLI from nemo_safe_synthesizer_plugin.config import config -from nemo_safe_synthesizer_plugin.runtime import runtime_info, runtime_task_command, setup_runtime - -NEMO_DEPLOYMENT_TYPE_ENVVAR = "NEMO_DEPLOYMENT_TYPE" -NMP_DEPLOYMENT_TYPE = "nmp" +from nemo_safe_synthesizer_plugin.runtime import runtime_info, setup_runtime class SafeSynthesizerCLI(NemoCLI): - """CLI extensions for host-local Safe Synthesizer development.""" + """CLI extensions for managing the Safe Synthesizer runtime.""" name: ClassVar[str] = "safe-synthesizer" description: ClassVar[str] = "Safe Synthesizer: privacy-preserving synthetic tabular data" @@ -31,7 +25,7 @@ def get_cli(self) -> typer.Typer: @app.callback() def main() -> None: - """Safe Synthesizer host-local development commands.""" + """Safe Synthesizer commands.""" @runtime_app.command("setup") def setup_runtime_command( @@ -62,54 +56,5 @@ def runtime_info_command() -> None: for key, value in runtime_info(config).items(): typer.echo(f"{key}: {value}") - @app.command("run-local") - def run_local_command( - spec_file: Path = typer.Option( - ..., - "--spec-file", - exists=True, - file_okay=True, - dir_okay=False, - readable=True, - help="NSS job spec JSON file.", - ), - workspace: str = typer.Option("default", "--workspace", help="Workspace used for fileset references."), - output_dir: Path = typer.Option( - Path("nss-output"), "--output-dir", help="Directory for local result files." - ), - data_source: Path | None = typer.Option( - None, - "--data-source", - exists=True, - readable=True, - help="Optional local data file overriding spec.data_source fileset download.", - ), - ) -> None: - """Run NSS on this host, using the managed local CUDA/GPU runtime.""" - args = [ - "run-local", - "--spec-file", - str(spec_file), - "--workspace", - workspace, - "--output-dir", - str(output_dir), - ] - if data_source is not None: - args.extend(["--data-source", str(data_source)]) - - try: - command = runtime_task_command(config, args) - except RuntimeError as e: - typer.echo(str(e), err=True) - raise typer.Exit(1) from e - - runtime_env = os.environ.copy() - runtime_env[NEMO_DEPLOYMENT_TYPE_ENVVAR] = NMP_DEPLOYMENT_TYPE - result = subprocess.run(command, check=False, env=runtime_env) - if result.returncode != 0: - raise typer.Exit(result.returncode) - typer.echo(f"Wrote Safe Synthesizer results to {output_dir}") - app.add_typer(runtime_app, name="runtime") return app diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/SKILL.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/SKILL.md index 1844ff827c..ef70e2e09f 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/SKILL.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/SKILL.md @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 name: safe-synthesizer -description: "Use NeMo Safe Synthesizer from the NMP plugin through task-specific routing: host-local GPU runs, platform job submission, configuration, troubleshooting, artifacts, privacy settings, PII replacement, and evaluation reports. Use when the user asks about safe-synthesizer, NeMo Safe Synthesizer, synthetic tabular data, DP settings, generation failures, plugin-local runs, filesets, model filesets, or `nemo safe-synthesizer` CLI commands." +description: "Use NeMo Safe Synthesizer from the NMP plugin through task-specific routing: platform job submission, configuration, troubleshooting, artifacts, privacy settings, PII replacement, runtime setup, and evaluation reports. Use when the user asks about safe-synthesizer, NeMo Safe Synthesizer, synthetic tabular data, DP settings, generation failures, filesets, model filesets, or `nemo safe-synthesizer` CLI commands." license: Apache-2.0 --- @@ -17,12 +17,11 @@ Task router for agents helping a person use the NeMo Safe Synthesizer NMP plugin - The `nemo safe-synthesizer` CLI is available, or repo development can use `uv run nemo safe-synthesizer`. - Platform jobs require workspace access to the input fileset and any `hf_token_secret` or PII classification provider. - Container jobs require a GPU-capable Jobs backend and access to the configured Safe Synthesizer task image. -- Host-local generation requires a Linux host with a CUDA-capable NVIDIA GPU, compatible drivers, and a runtime created by `nemo safe-synthesizer runtime setup`. - Fileset references use `/#` unless a workflow states otherwise. ## Route -- Submit platform container jobs or run host-local development tasks: read `workflows/run.md`. +- Submit platform container jobs: read `workflows/run.md`. - Set or override job parameters: read `workflows/config.md`. - Diagnose runtime, install, generation, OOM, validation, or fileset failures: read `workflows/diagnose.md`. - Retrieve job result artifacts: read `workflows/results.md`. @@ -31,13 +30,11 @@ Task router for agents helping a person use the NeMo Safe Synthesizer NMP plugin ## Plugin-Specific Rules - Prefer platform container jobs for normal Safe Synthesizer usage. -- Use the Jobs API or SDK for platform jobs. The `nemo safe-synthesizer` CLI exposes `run-local` and `runtime` only. +- Use the Jobs API or SDK for platform jobs. The `nemo safe-synthesizer` CLI exposes runtime setup and inspection utilities only. - Configure released container jobs with `NMP_IMAGE_REGISTRY=nvcr.io/nvidia/nemo-platform`, `NMP_IMAGE_TAG=`, `NEMO_SAFE_SYNTHESIZER_JOB_MODE=container`, and `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE=safe-synthesizer-tasks`. - Override local task images with `NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE_REF=`; this bypasses platform registry/tag qualification. -- Use `nemo safe-synthesizer run-local` only for host-local CUDA/GPU development or debugging. -- Use `nemo safe-synthesizer runtime setup` only for host-local runs that need the separate runtime venv. +- Use `nemo safe-synthesizer runtime setup` only when the separate runtime venv is needed for development or job debugging. - Treat `data_source` as a fileset URL for platform jobs, usually `/#`. -- For local runs, prefer `--data-source ` when the input is already on disk. - If the job uses PII classification, `config.replace_pii.globals.classify.classify_model_provider` must be `/`. - Keep usage guidance separate from plugin source development internals unless the user asks to change the plugin. diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/config-runs.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/config-runs.md index 10b26c4274..9d0e979303 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/config-runs.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/config-runs.md @@ -7,7 +7,7 @@ - Read `workflows/config.md` for the job spec field reference. - Resolve the CLI with `workflows/run.md`. -- Use a platform fileset `data_source` for platform jobs or pass `--data-source` for host-local input files. +- Use a platform fileset `data_source` for platform jobs. ## Basic Job Spec @@ -144,17 +144,6 @@ export NMP_IMAGE_TAG= # match your installed NeMo Platform release export NEMO_SAFE_SYNTHESIZER_CONTAINER_IMAGE=safe-synthesizer-tasks ``` -Use host-local execution only for local CUDA/GPU development or task debugging: - -```bash -uv run nemo safe-synthesizer runtime setup -uv run nemo safe-synthesizer run-local \ - --workspace default \ - --spec-file nss-job.json \ - --data-source ./input.csv \ - --output-dir ./nss-output -``` - ## Next Steps - Use `workflows/pii-architecture.md` for provider resolution details. diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/config.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/config.md index ebeaf44f02..0ac0ce1276 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/config.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/config.md @@ -56,6 +56,4 @@ Use `hf_token_secret` at the top level of the job spec when model initialization ## Reusing a Previously Trained NSS Model -For **`nemo safe-synthesizer run-local`**, set `config.training.pretrained_model` to the adapter directory from a prior run (for example `./nss-output/adapter`). The plugin reuses that adapter for **generation only** (no retraining). See `docs/safe-synthesizer/about/host-local-development.mdx`. - For **platform jobs**, set `pretrained_model_job` at the top level of the job spec; the plugin resolves that job's `adapter` result from Files for generation-only reuse. Use either `pretrained_model_job` or `config.training.pretrained_model`, not both. diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/diagnose.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/diagnose.md index 773b9dcdbf..d87cf78db0 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/diagnose.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/diagnose.md @@ -7,14 +7,14 @@ - NeMo CLI access through `nemo` or repo development invocation `uv run nemo`. - Python dependencies synced into the active virtual environment. -- A GPU-capable Jobs backend for platform container jobs, or a compatible CUDA-capable GPU and driver for host-local generation. +- A GPU-capable Jobs backend for platform container jobs. - Files API URL access when the run uses filesets or model fileset setup. - Workspace access to the input fileset, output job, `hf_token_secret`, and any PII classification provider. ## First Checks 1. Resolve the CLI with `command -v nemo 2>/dev/null || (test -x .venv/bin/nemo && realpath .venv/bin/nemo) || echo CLI_NOT_FOUND`. -2. Confirm whether the user is running a platform container job through the Jobs API or SDK, or host-local (`nemo safe-synthesizer run-local`). +2. Confirm the user is running a platform container job through the Jobs API or SDK. 3. Inspect the spec file before changing commands. ## Common Failures @@ -26,10 +26,8 @@ Tell the user that the NeMo CLI or the Safe Synthesizer plugin is not installed ### CUDA or GPU initialization fails - For platform jobs, confirm the job executor profile targets GPU-capable workers. -- Confirm the host has a compatible NVIDIA GPU and driver with `nvidia-smi`. - For repo development, verify the plugin runtime with `uv run nemo safe-synthesizer runtime info`. - Recreate the runtime with `uv run nemo safe-synthesizer runtime setup --force` if the engine/CUDA packages are missing or stale. -- Host-local Safe Synthesizer training runs directly on the host GPU; a GPU inside another service container is not enough. ### Container image cannot be pulled or is the wrong tag @@ -43,12 +41,10 @@ Tell the user that the NeMo CLI or the Safe Synthesizer plugin is not installed - For platform jobs, verify `data_source` is a fileset URL: `/#`. - Confirm the fileset exists and the workspace is correct. -- For local runs, prefer `--data-source ` when the input is on disk. -- Supported local file forms include CSV, Parquet, JSON, JSONL, and Hugging Face datasets paths. ### Model or fileset downloads fail -Run the model fileset setup when local tasks need model filesets: +Run the model fileset setup when jobs need model filesets: ```bash uv run python plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py --files-api-url http://localhost:8080 diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/results.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/results.md index e8aa58fd6f..eb937b5722 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/results.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/results.md @@ -6,19 +6,8 @@ ## Prerequisites - Resolve the CLI with the command in `workflows/run.md`. -- For host-local runs, know the `--output-dir` passed to `nemo safe-synthesizer run-local`. - For platform jobs, know the job name and workspace. -## Host-Local Runs - -`nemo safe-synthesizer run-local --output-dir ./nss-output` writes artifacts under the output directory. - -Start answers with the exact output directory when it is known: - -```bash -ls ./nss-output -``` - ## Platform Jobs Platform jobs publish named results through the Jobs service: diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/run.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/run.md index 1f37d186aa..f9faffd3a8 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/run.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/run.md @@ -50,21 +50,10 @@ Run `command -v nemo 2>/dev/null || (test -x .venv/bin/nemo && realpath .venv/bi ## Choose the Execution Mode -Use platform jobs for normal Safe Synthesizer usage. The platform compiles the spec into a GPU container step that runs the configured Safe Synthesizer task image. +Use platform jobs for Safe Synthesizer usage. The platform compiles the spec into a GPU container step that runs the configured Safe Synthesizer task image. Use the Jobs API or SDK to create the job. The plugin CLI does not expose `nemo safe-synthesizer jobs` commands. For CLI users, point them to the generated Jobs/API surface available in their installed NeMo CLI, or to the Python SDK builder documented in `docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx`. -Use host-local execution only when the user is iterating on a local machine with CUDA/GPU access or debugging the task process outside the Jobs backend: - -```bash -uv run nemo safe-synthesizer runtime setup -uv run nemo safe-synthesizer run-local \ - --workspace default \ - --spec-file nss-job.json \ - --data-source ./input.csv \ - --output-dir ./nss-output -``` - ## Minimal Spec Shape ```json @@ -105,7 +94,7 @@ For platform submission, pass this object as the `spec` field in the Jobs API or ## Next Steps - Tune job parameters with `workflows/config.md` and `workflows/config-runs.md`. -- Reuse a prior adapter or run plugin tests: `docs/safe-synthesizer/about/host-local-development.mdx`. +- Reuse a prior adapter or inspect runtime setup: `docs/safe-synthesizer/about/host-local-development.mdx`. - Retrieve job result files with `workflows/results.md`. - Interpret output files with `workflows/artifacts.md`. - Debug failed runs with `workflows/diagnose.md`. diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py index d3e03b51e1..f45f30c584 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py @@ -14,10 +14,8 @@ os.environ.setdefault("TORCHINDUCTOR_FX_GRAPH_REMOTE_CACHE", "0") os.environ.setdefault("TORCHINDUCTOR_AUTOTUNE_REMOTE_CACHE", "0") -import argparse import json import logging -import sys import tempfile import time from pathlib import Path @@ -190,27 +188,6 @@ def upload_results(result: SafeSynthesizerResults, adapter_path: Path | None = N _create_job_result(sdk, workspace, job_id, "adapter", artifact_url) -def write_results_local(result: SafeSynthesizerResults, output_dir: Path, adapter_path: Path | None = None) -> None: - """Write NSS results to the host filesystem for local CUDA development.""" - if result.synthetic_data is None: - raise ValueError("Safe Synthesizer did not produce synthetic data") - output_dir.mkdir(parents=True, exist_ok=True) - result.synthetic_data.to_csv(output_dir / "synthetic-data.csv", index=False) - with open(output_dir / "summary.json", "w", encoding="utf-8") as f: - json.dump(result.summary.model_dump(), f, indent=2) - if result.evaluation_report_html: - (output_dir / "evaluation-report.html").write_text(result.evaluation_report_html, encoding="utf-8") - if adapter_path is not None and adapter_path.exists(): - embed_run_config_in_adapter(adapter_path) - adapter_target = output_dir / "adapter" - if adapter_path.is_dir(): - import shutil - - if adapter_target.exists(): - shutil.rmtree(adapter_target) - shutil.copytree(adapter_path, adapter_target) - - def _create_job_result(sdk: NeMoPlatform, workspace: str, job_name: str, result_name: str, artifact_url: str): """Create a job result record.""" client_from_platform(sdk, JobsClient).create_job_result( @@ -420,56 +397,11 @@ def run_from_env() -> None: pretrained_model_tmp.cleanup_tmp_dir() -def run_local(spec_file: Path, workspace: str, output_dir: Path, data_source: Path | None = None) -> None: - """Run NSS on the host GPU from a job spec file.""" - os.environ.setdefault(NEMO_JOB_WORKSPACE_ENVVAR, workspace) - with open(spec_file, "r", encoding="utf-8") as f: - raw_job_config = json.load(f) - job_config = SafeSynthesizerJobConfig.model_validate(raw_job_config) - if data_source is None: - loaded_data = download_from_fileset(job_config.data_source) - else: - loaded_data = _load_file_as_dataframe(data_source) - pretrained_model_tmp, adapter_path = _resolve_pretrained_model(job_config, workspace=workspace) - try: - result, new_adapter_path = run_config( - job_config, - loaded_data, - output_dir / "work", - adapter_location=adapter_path, - ) - write_results_local(result, output_dir, new_adapter_path) - finally: - if pretrained_model_tmp is not None: - pretrained_model_tmp.cleanup_tmp_dir() - - def main(argv: list[str] | None = None) -> None: - """Run the task entry point from either platform env vars or CLI args.""" - argv = sys.argv[1:] if argv is None else argv - if not argv: - run_from_env() - return - - parser = argparse.ArgumentParser(prog="python -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer") - subparsers = parser.add_subparsers(dest="command", required=True) - local_parser = subparsers.add_parser("run-local", help="Run a Safe Synthesizer job from a local spec file.") - local_parser.add_argument("--spec-file", required=True, type=Path) - local_parser.add_argument("--workspace", default="default") - local_parser.add_argument("--output-dir", required=True, type=Path) - local_parser.add_argument("--data-source", type=Path) - - args = parser.parse_args(argv) - if args.command == "run-local": - run_local( - spec_file=args.spec_file, - workspace=args.workspace, - output_dir=args.output_dir, - data_source=args.data_source, - ) - return - - parser.error(f"unknown command: {args.command}") + """Run the task entry point from platform environment variables.""" + if argv: + raise SystemExit("This task module no longer accepts local execution commands.") + run_from_env() if __name__ == "__main__": diff --git a/plugins/nemo-safe-synthesizer/tests/e2e/test_local_synthesis.py b/plugins/nemo-safe-synthesizer/tests/e2e/test_local_synthesis.py deleted file mode 100644 index ce0ff41e52..0000000000 --- a/plugins/nemo-safe-synthesizer/tests/e2e/test_local_synthesis.py +++ /dev/null @@ -1,151 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Opt-in E2E coverage for host-local Safe Synthesizer execution. - -This test intentionally bypasses Docker and the platform Jobs service. It runs -the plugin CLI against the host Python/CUDA environment, so it is skipped unless -RUN_NSS_LOCAL_E2E=1 is set. -""" - -from __future__ import annotations - -import csv -import json -import os -import random -import subprocess -from datetime import date -from pathlib import Path - -import pytest - -pytestmark = [ - pytest.mark.e2e, - pytest.mark.gpu_integration, - pytest.mark.slow, -] - - -FLAVORS = [ - "Vanilla", - "Chocolate", - "Strawberry", - "Mint Chocolate Chip", - "Cookies and Cream", - "Pistachio", - "Rocky Road", - "Butter Pecan", - "Coffee", - "Mango Sorbet", - "Salted Caramel", - "Cookie Dough", -] - - -def _repo_root() -> Path: - return Path(__file__).resolve().parents[4] - - -def _write_faker_csv(path: Path, rows: int = 10000) -> None: - faker = pytest.importorskip("faker") - fake = faker.Faker() - faker.Faker.seed(42) - random.seed(42) - - with path.open("w", newline="", encoding="utf-8") as f: - writer = csv.DictWriter( - f, - fieldnames=["name", "signup_date", "birthdate", "favorite_ice_cream_flavor"], - ) - writer.writeheader() - for _ in range(rows): - writer.writerow( - { - "name": fake.name(), - "signup_date": fake.date_between_dates( - date_start=date(2020, 1, 1), - date_end=date(2026, 5, 4), - ).isoformat(), - "birthdate": fake.date_between_dates( - date_start=date(1945, 1, 1), - date_end=date(2006, 12, 31), - ).isoformat(), - "favorite_ice_cream_flavor": random.choice(FLAVORS), - } - ) - - -def _write_synthesis_spec(path: Path) -> None: - path.write_text( - json.dumps( - { - "data_source": "default/local-input#input.csv", - "config": { - "enable_synthesis": True, - "enable_replace_pii": False, - "generation": { - "num_records": 100, - }, - "evaluation": { - "enabled": True, - }, - "privacy": { - "dp_enabled": False, - }, - }, - }, - indent=2, - ), - encoding="utf-8", - ) - - -def test_local_synthesis_cli_generates_data(tmp_path: Path) -> None: - if os.environ.get("RUN_NSS_LOCAL_E2E") != "1": - pytest.skip("Set RUN_NSS_LOCAL_E2E=1 to run host-local NSS synthesis E2E") - - input_csv = tmp_path / "input.csv" - spec_file = tmp_path / "nss-job-synthesis.json" - output_dir = tmp_path / "nss-output" - _write_faker_csv(input_csv) - _write_synthesis_spec(spec_file) - - command = [ - "uv", - "run", - "nemo", - "safe-synthesizer", - "run-local", - "--workspace", - "default", - "--spec-file", - str(spec_file), - "--data-source", - str(input_csv), - "--output-dir", - str(output_dir), - ] - print("Running:", " ".join(command), flush=True) - - result = subprocess.run( - command, - cwd=_repo_root(), - timeout=int(os.environ.get("NSS_LOCAL_E2E_TIMEOUT_SECONDS", "3600")), - check=False, - ) - assert result.returncode == 0 - - synthetic_data = output_dir / "synthetic-data.csv" - summary_file = output_dir / "summary.json" - assert synthetic_data.exists() - assert summary_file.exists() - - with synthetic_data.open(encoding="utf-8") as f: - row_count = sum(1 for _ in f) - 1 - assert row_count == 100 - - summary = json.loads(summary_file.read_text(encoding="utf-8")) - timing = summary["timing"] - assert timing["training_time_sec"] is not None - assert timing["generation_time_sec"] is not None diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_cli.py b/plugins/nemo-safe-synthesizer/tests/unit/test_cli.py index 1c65b626db..27f031e530 100644 --- a/plugins/nemo-safe-synthesizer/tests/unit/test_cli.py +++ b/plugins/nemo-safe-synthesizer/tests/unit/test_cli.py @@ -1,48 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from subprocess import CompletedProcess - -from nemo_safe_synthesizer_plugin import cli -from nemo_safe_synthesizer_plugin.cli import NEMO_DEPLOYMENT_TYPE_ENVVAR, NMP_DEPLOYMENT_TYPE, SafeSynthesizerCLI +from nemo_safe_synthesizer_plugin.cli import SafeSynthesizerCLI from typer.testing import CliRunner -def test_run_local_sets_nmp_deployment_type_for_runtime_subprocess(tmp_path, monkeypatch): - spec_file = tmp_path / "nss-job.json" - spec_file.write_text("{}", encoding="utf-8") - data_file = tmp_path / "input.csv" - data_file.write_text("name\nAda\n", encoding="utf-8") - output_dir = tmp_path / "nss-output" - captured = {} - - def fake_runtime_task_command(_config, args): - return ["runtime-python", *args] - - def fake_run(command, *, check=False, env=None): - captured["command"] = command - captured["check"] = check - captured["env"] = env - return CompletedProcess(command, 0) - - monkeypatch.setattr(cli, "runtime_task_command", fake_runtime_task_command) - monkeypatch.setattr(cli.subprocess, "run", fake_run) - - result = CliRunner().invoke( - SafeSynthesizerCLI().get_cli(), - [ - "run-local", - "--workspace", - "default", - "--spec-file", - str(spec_file), - "--data-source", - str(data_file), - "--output-dir", - str(output_dir), - ], - ) +def test_cli_exposes_runtime_but_not_removed_local_command() -> None: + result = CliRunner().invoke(SafeSynthesizerCLI().get_cli(), ["--help"]) - assert result.exit_code == 0, result.output - assert captured["check"] is False - assert captured["env"][NEMO_DEPLOYMENT_TYPE_ENVVAR] == NMP_DEPLOYMENT_TYPE + assert result.exit_code == 0 + assert "runtime" in result.output + assert "run-" + "local" not in result.output diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_local_run.py b/plugins/nemo-safe-synthesizer/tests/unit/test_local_run.py deleted file mode 100644 index 718e3a9e00..0000000000 --- a/plugins/nemo-safe-synthesizer/tests/unit/test_local_run.py +++ /dev/null @@ -1,305 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import importlib -import json -import sys -from types import ModuleType, SimpleNamespace -from unittest.mock import MagicMock - -import pandas as pd -import pytest - - -def _resp(data): - """Wrap a payload in a NemoResponse-like object whose ``.data()`` returns it. - - Production consumes typed-client responses via ``client.(...).data()``. - """ - m = MagicMock() - m.data.return_value = data - return m - - -def import_task_main_without_heavy_runtime(monkeypatch): - pytest.importorskip("nemo_safe_synthesizer.config.job") - library_builder = ModuleType("nemo_safe_synthesizer.sdk.library_builder") - setattr(library_builder, "SafeSynthesizer", object) - monkeypatch.setitem(sys.modules, "nemo_safe_synthesizer.sdk.library_builder", library_builder) - return importlib.import_module("nemo_safe_synthesizer_plugin.tasks.safe_synthesizer.__main__") - - -def test_run_local_loads_spec_and_writes_results(tmp_path, monkeypatch): - task_main = import_task_main_without_heavy_runtime(monkeypatch) - spec_file = tmp_path / "spec.json" - spec_file.write_text( - json.dumps( - { - "data_source": "default/data#input.csv", - "config": { - "enable_synthesis": False, - "enable_replace_pii": False, - }, - } - ), - encoding="utf-8", - ) - data_file = tmp_path / "input.csv" - data_file.write_text("value\n1\n", encoding="utf-8") - - result = SimpleNamespace( - synthetic_data=pd.DataFrame({"value": [1]}), - summary=SimpleNamespace(model_dump=lambda: {"row_count": 1}), - evaluation_report_html=None, - ) - monkeypatch.setattr( - task_main, - "run_config", - lambda job_config, data_source, save_path, *, adapter_location=None: (result, None), - ) - get_platform_sdk = MagicMock(side_effect=AssertionError("offline run should not initialize the platform SDK")) - monkeypatch.setattr(task_main, "get_platform_sdk", get_platform_sdk) - - output_dir = tmp_path / "output" - task_main.run_local(spec_file=spec_file, workspace="default", output_dir=output_dir, data_source=data_file) - - assert (output_dir / "synthetic-data.csv").exists() - assert json.loads((output_dir / "summary.json").read_text(encoding="utf-8")) == {"row_count": 1} - get_platform_sdk.assert_not_called() - - -def test_validate_flat_tabular_data_rejects_nested_columns(monkeypatch): - task_main = import_task_main_without_heavy_runtime(monkeypatch) - data = pd.DataFrame( - { - "id": [1, 2], - "tools": [[{"name": "f"}], []], - "metadata": [{"a": 1}, {"a": 2}], - } - ) - - with pytest.raises(task_main.ParameterError, match="tools, metadata"): - task_main._validate_flat_tabular_data(data) - - -def test_validate_flat_tabular_data_allows_flat_columns(monkeypatch): - task_main = import_task_main_without_heavy_runtime(monkeypatch) - data = pd.DataFrame({"name": ["a", "b"], "age": [1, 2], "score": [0.5, 0.9]}) - - task_main._validate_flat_tabular_data(data) - - -def test_run_config_pii_only_uses_processed_training_df(tmp_path, monkeypatch): - task_main = import_task_main_without_heavy_runtime(monkeypatch) - processed = pd.DataFrame({"name": ["REDACTED"], "age": [1]}) - - class FakeSafeSynthesizer: - def __init__(self, config, save_path): - self._total_start = None - self._training_df = None - self._workdir = None - - def with_data_source(self, data_source): - return self - - def process_data(self): - self._training_df = processed - - monkeypatch.setattr(task_main, "SafeSynthesizer", FakeSafeSynthesizer) - results_module = ModuleType("nemo_safe_synthesizer.results") - setattr( - results_module, - "make_nss_results", - lambda generate_results, total_time=None: SimpleNamespace( - synthetic_data=generate_results, - summary=SimpleNamespace(model_dump=lambda: {"row_count": len(generate_results)}), - evaluation_report_html=None, - ), - ) - monkeypatch.setitem(sys.modules, "nemo_safe_synthesizer.results", results_module) - job_config = task_main.SafeSynthesizerJobConfig.model_validate( - { - "data_source": "default/data#input.csv", - "config": { - "enable_synthesis": False, - "enable_replace_pii": False, - }, - } - ) - - result, adapter_path = task_main.run_config( - job_config, - pd.DataFrame({"name": ["Alice"], "age": [1]}), - tmp_path, - ) - - assert adapter_path is None - pd.testing.assert_frame_equal(result.synthetic_data, processed) - - -def test_run_from_env_reports_missing_config_path(monkeypatch): - task_main = import_task_main_without_heavy_runtime(monkeypatch) - monkeypatch.setattr(task_main, "initialize_observability", lambda: None) - monkeypatch.setattr(task_main, "get_platform_config", lambda: SimpleNamespace(get_service_url=lambda _name: None)) - monkeypatch.setattr(task_main, "_setup_classify_endpoint", lambda: None) - monkeypatch.setattr(task_main, "download_from_fileset", lambda fileset_url: pd.DataFrame({"value": [1]})) - monkeypatch.setenv("DATA_SOURCE", "default/data#input.csv") - monkeypatch.delenv(task_main.NEMO_JOB_STEP_CONFIG_FILE_PATH_ENVVAR, raising=False) - - with pytest.raises(ValueError, match=f"{task_main.NEMO_JOB_STEP_CONFIG_FILE_PATH_ENVVAR} is not set"): - task_main.run_from_env() - - -def test_setup_classify_endpoint_sets_upstream_safe_synthesizer_env(monkeypatch): - task_main = import_task_main_without_heavy_runtime(monkeypatch) - monkeypatch.setenv( - "CLASSIFY_LLM_ENDPOINT_PATH", "/apis/inference-gateway/v2/workspaces/default/provider/my-nim/-/v1" - ) - monkeypatch.setenv("NMP_MODELS_URL", "http://models.test") - monkeypatch.delenv("NSS_INFERENCE_ENDPOINT", raising=False) - monkeypatch.delenv("NSS_INFERENCE_KEY", raising=False) - - task_main._setup_classify_endpoint() - - assert ( - task_main.os.environ["NSS_INFERENCE_ENDPOINT"] - == "http://models.test/apis/inference-gateway/v2/workspaces/default/provider/my-nim/-/v1" - ) - assert task_main.os.environ["NSS_INFERENCE_KEY"] == "not-needed" - - -def test_setup_classify_endpoint_preserves_existing_inference_key(monkeypatch): - task_main = import_task_main_without_heavy_runtime(monkeypatch) - monkeypatch.setenv("CLASSIFY_LLM_ENDPOINT_PATH", "/route") - monkeypatch.setenv("NMP_MODELS_URL", "http://models.test/") - monkeypatch.setenv("NSS_INFERENCE_KEY", "real-key") - - task_main._setup_classify_endpoint() - - assert task_main.os.environ["NSS_INFERENCE_ENDPOINT"] == "http://models.test/route" - assert task_main.os.environ["NSS_INFERENCE_KEY"] == "real-key" - - -def test_run_local_resolves_pretrained_model_job_before_run(tmp_path, monkeypatch): - task_main = import_task_main_without_heavy_runtime(monkeypatch) - spec_file = tmp_path / "spec.json" - spec_file.write_text( - json.dumps( - { - "data_source": "default/data#input.csv", - "pretrained_model_job": "prior-safe-synth-job", - "config": { - "enable_synthesis": False, - "enable_replace_pii": False, - }, - } - ), - encoding="utf-8", - ) - data_file = tmp_path / "input.csv" - data_file.write_text("value\n1\n", encoding="utf-8") - - adapter_dir = tmp_path / "downloaded-adapter" - adapter_dir.mkdir() - (adapter_dir / "adapter_config.json").write_text( - json.dumps({"base_model_name_or_path": "HuggingFaceTB/SmolLM3-3B"}), - encoding="utf-8", - ) - - sdk = MagicMock() - monkeypatch.setattr(task_main, "get_platform_sdk", lambda: sdk) - - # Production resolves the prior adapter via - # client_from_platform(sdk, JobsClient).get_job_result(...).data(). - jobs_client = MagicMock() - jobs_client.get_job_result.return_value = _resp( - SimpleNamespace(artifact_url="default/job-results-prior#results/attempt-1/adapter") - ) - monkeypatch.setattr(task_main, "client_from_platform", lambda _sdk, _cls: jobs_client) - - file_manager = MagicMock() - file_manager.download_from_url.return_value = SimpleNamespace( - path=adapter_dir, - cleanup_tmp_dir=MagicMock(), - ) - monkeypatch.setattr(task_main, "FilesetFileManager", MagicMock(return_value=file_manager)) - - captured = {} - result = SimpleNamespace( - synthetic_data=pd.DataFrame({"value": [1]}), - summary=SimpleNamespace(model_dump=lambda: {"row_count": 1}), - evaluation_report_html=None, - ) - - def fake_run_config(job_config, data_source, save_path, *, adapter_location=None): - captured["adapter_location"] = adapter_location - return result, None - - monkeypatch.setattr(task_main, "run_config", fake_run_config) - - output_dir = tmp_path / "output" - task_main.run_local(spec_file=spec_file, workspace="default", output_dir=output_dir, data_source=data_file) - - assert captured["adapter_location"] == adapter_dir - jobs_client.get_job_result.assert_called_once_with( - name="adapter", - job="prior-safe-synth-job", - workspace="default", - ) - - -def test_run_local_cleans_pretrained_model_tmp_when_run_config_raises(tmp_path, monkeypatch): - task_main = import_task_main_without_heavy_runtime(monkeypatch) - spec_file = tmp_path / "spec.json" - spec_file.write_text( - json.dumps( - { - "data_source": "default/data#input.csv", - "pretrained_model_job": "prior-safe-synth-job", - "config": { - "enable_synthesis": False, - "enable_replace_pii": False, - }, - } - ), - encoding="utf-8", - ) - data_file = tmp_path / "input.csv" - data_file.write_text("value\n1\n", encoding="utf-8") - - adapter_dir = tmp_path / "downloaded-adapter" - adapter_dir.mkdir() - (adapter_dir / "adapter_config.json").write_text( - json.dumps({"base_model_name_or_path": "HuggingFaceTB/SmolLM3-3B"}), - encoding="utf-8", - ) - - sdk = MagicMock() - monkeypatch.setattr(task_main, "get_platform_sdk", lambda: sdk) - - # Production resolves the prior adapter via - # client_from_platform(sdk, JobsClient).get_job_result(...).data(). - jobs_client = MagicMock() - jobs_client.get_job_result.return_value = _resp( - SimpleNamespace(artifact_url="default/job-results-prior#results/attempt-1/adapter") - ) - monkeypatch.setattr(task_main, "client_from_platform", lambda _sdk, _cls: jobs_client) - - pretrained_model_tmp = SimpleNamespace( - path=adapter_dir, - cleanup_tmp_dir=MagicMock(), - ) - file_manager = MagicMock() - file_manager.download_from_url.return_value = pretrained_model_tmp - monkeypatch.setattr(task_main, "FilesetFileManager", MagicMock(return_value=file_manager)) - - def raise_run_config(job_config, data_source, save_path, *, adapter_location=None): - raise RuntimeError("run_config failed") - - monkeypatch.setattr(task_main, "run_config", raise_run_config) - - output_dir = tmp_path / "output" - with pytest.raises(RuntimeError, match="run_config failed"): - task_main.run_local(spec_file=spec_file, workspace="default", output_dir=output_dir, data_source=data_file) - - pretrained_model_tmp.cleanup_tmp_dir.assert_called_once_with() diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_runtime.py b/plugins/nemo-safe-synthesizer/tests/unit/test_runtime.py index 95f6cb68ef..bdc53d9cd2 100644 --- a/plugins/nemo-safe-synthesizer/tests/unit/test_runtime.py +++ b/plugins/nemo-safe-synthesizer/tests/unit/test_runtime.py @@ -31,15 +31,12 @@ def test_runtime_task_command_uses_configured_python(tmp_path): python.touch() config = SafeSynthesizerConfig.model_validate({"runtime_python": str(python)}) - command = runtime.runtime_task_command(config, ["run-local", "--workspace", "default"]) + command = runtime.runtime_task_command(config) assert command == [ str(python), "-m", runtime.TASK_MODULE, - "run-local", - "--workspace", - "default", ] diff --git a/plugins/nemo-unsloth/src/nemo_unsloth_plugin/cli/inputs.py b/plugins/nemo-unsloth/src/nemo_unsloth_plugin/cli/inputs.py index 3e1a71d4b2..17771a84c6 100644 --- a/plugins/nemo-unsloth/src/nemo_unsloth_plugin/cli/inputs.py +++ b/plugins/nemo-unsloth/src/nemo_unsloth_plugin/cli/inputs.py @@ -5,7 +5,7 @@ The override machinery is shared in :mod:`nmp.customization_common.cli.overrides`; this module supplies the Unsloth specifics: the ``UnslothJobInput`` schema (via -``load_job_json``), the ``JOB_JSON`` help text, and the run-disabled message. +``load_job_json``) and the ``JOB_JSON`` help text. """ import json @@ -17,10 +17,6 @@ from nemo_unsloth_plugin.schema import UnslothJobInput _JOB_JSON_HELP = "Path to Unsloth job JSON (UnslothJobInput schema)." -_RUN_DISABLED_MESSAGE = ( - "Unsloth does not support local run. Submit to the platform API instead:\n" - " nemo customization unsloth submit -w " -) def load_job_json(path: Path) -> str: @@ -31,10 +27,9 @@ def load_job_json(path: Path) -> str: def apply_unsloth_job_cli_overrides(group: typer.Typer) -> None: - """Flat ``unsloth`` CLI: ``submit JOB.json``; ``run`` is disabled.""" + """Flat ``unsloth`` CLI: ``submit JOB.json``.""" apply_job_cli_overrides( group, load_job_json=load_job_json, job_json_help=_JOB_JSON_HELP, - run_disabled_message=_RUN_DISABLED_MESSAGE, ) diff --git a/plugins/nemo-unsloth/tests/test_cli.py b/plugins/nemo-unsloth/tests/test_cli.py index 6706077df4..855f610ca7 100644 --- a/plugins/nemo-unsloth/tests/test_cli.py +++ b/plugins/nemo-unsloth/tests/test_cli.py @@ -5,8 +5,7 @@ Pins the post-2026 submit-only contract: ``submit`` accepts a positional ``JOB_JSON`` and delegates to the auto-generated callback with ``--spec`` -set to the validated JSON; ``run`` hard-fails with an "use submit" -message. +set to the validated JSON. """ from __future__ import annotations @@ -34,16 +33,14 @@ def _plain(text: str) -> str: def _build_app() -> typer.Typer: - """Build a Typer app with the contributor's overridden run/submit/explain.""" + """Build a Typer app with the contributor's overridden submit/explain.""" from nemo_platform_plugin.commands import ( _add_explain_command, - _add_run_command, _add_submit_command, ) app = typer.Typer(no_args_is_help=True) scheduler = NemoJobScheduler() - _add_run_command(app, UnslothJob, scheduler) _add_submit_command(app, UnslothJob, scheduler) _add_explain_command(app, UnslothJob, scheduler) apply_unsloth_job_cli_overrides(app) @@ -82,20 +79,6 @@ def test_submit_path_includes_workspace(self) -> None: assert path == "/apis/customization/v2/workspaces/acme-corp/unsloth/jobs" -class TestRunHardFail: - def test_run_exits_1_with_submit_pointer(self, tmp_path: Path) -> None: - path = tmp_path / "job.json" - path.write_text(json.dumps(_minimal_payload())) - - app = _build_app() - runner = CliRunner() - result = runner.invoke(app, ["run", str(path)]) - assert result.exit_code == 1 - plain = _plain(result.output) - assert "submit" in plain - assert "does not support local run" in plain - - class TestSubmitOverride: def test_help_lists_job_json_workspace_and_profile(self) -> None: app = _build_app() diff --git a/plugins/nemo-unsloth/tests/test_contributor.py b/plugins/nemo-unsloth/tests/test_contributor.py index f930c513cb..b226919937 100644 --- a/plugins/nemo-unsloth/tests/test_contributor.py +++ b/plugins/nemo-unsloth/tests/test_contributor.py @@ -91,7 +91,7 @@ def test_returns_jobs_router_spec(self, contributor: object) -> None: class TestCLI: - def test_cli_root_help_lists_three_verbs(self, contributor: object) -> None: + def test_cli_root_help_lists_submit_and_explain(self, contributor: object) -> None: try: cli = contributor.get_cli() except ImportError as exc: @@ -100,21 +100,19 @@ def test_cli_root_help_lists_three_verbs(self, contributor: object) -> None: result = runner.invoke(cli, ["--help"]) assert result.exit_code == 0 plain = _plain(result.output) - assert "run" in plain assert "submit" in plain assert "explain" in plain - def test_run_hard_fails(self, contributor: object) -> None: + def test_run_is_not_registered(self, contributor: object) -> None: try: cli = contributor.get_cli() except ImportError as exc: pytest.skip(f"CLI deps unavailable in this env: {exc}") runner = CliRunner() result = runner.invoke(cli, ["run"]) - assert result.exit_code == 1 + assert result.exit_code != 0 plain = _plain(result.output) - assert "does not support local run" in plain - assert "submit" in plain + assert "No such command" in plain def test_submit_help_shows_job_json_positional(self, contributor: object) -> None: try: diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/app.py b/sdk/python/nemo-platform/src/nemo_platform/cli/app.py index 1b1c852c4a..c3d6b64994 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/app.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/app.py @@ -53,7 +53,7 @@ def _build_top_level_lazy_entries() -> tuple[TopLevelEntry, ...]: plugin_entry_points = _installed_plugin_command_entry_points() # Plugin `nemo.cli` entry points own their command name (e.g. safe-synthesizer). - # Drop generated API top-level groups with the same name so run-local/runtime stay available. + # Drop generated API top-level groups with the same name so plugin-owned commands stay available. api_entries = tuple(entry for entry in API_TOP_LEVEL_ENTRIES if entry.name not in plugin_entry_points) return build_top_level_entries( (*TOP_LEVEL_ENTRIES, *api_entries),