From 2baf8e488b22bd1bde3ed1121c79f07525cac804 Mon Sep 17 00:00:00 2001 From: mkornfield Date: Thu, 4 Jun 2026 22:06:22 +0000 Subject: [PATCH 1/5] fix: move nss jobs into plugin, update docs Signed-off-by: mkornfield --- docs/_snippets/nvidia-build-model-provider.md | 6 +- docs/safe-synthesizer/.gitignore | 1 + .../about/host-local-development.md | 56 +++- docs/safe-synthesizer/about/jobs.md | 2 +- docs/safe-synthesizer/about/reference.md | 2 +- docs/safe-synthesizer/getting-started.md | 18 +- docs/safe-synthesizer/llms.txt | 2 +- .../tutorials/differential-privacy.md | 4 +- .../tutorials/safe-synthesizer-101.md | 78 ++++-- mkdocs.yml | 25 +- packages/nemo_platform/pyproject.toml | 1 + plugins/nemo-safe-synthesizer/README.md | 25 +- plugins/nemo-safe-synthesizer/pyproject.toml | 3 + .../api/v2/jobs/endpoints.py | 15 +- .../sdk/config.py | 31 +++ .../sdk/http_utils.py | 57 +++++ .../nemo_safe_synthesizer_plugin/sdk/job.py | 180 +++++++++++++ .../sdk/job_builder.py | 241 ++++++++++++++++++ .../sdk/resources.py | 224 ++++++++++++++++ .../safe-synthesizer/workflows/diagnose.md | 8 +- .../safe-synthesizer/workflows/results.md | 11 +- .../skills/safe-synthesizer/workflows/run.md | 11 +- .../tests/unit/test_jobs.py | 21 ++ .../tests/unit/test_sdk.py | 131 ++++++++++ pyproject.toml | 1 + third_party/licenses.jsonl | 1 + third_party/osv-licenses.json | 12 +- third_party/requirements-main.txt | 43 +++- uv.lock | 6 + 29 files changed, 1121 insertions(+), 95 deletions(-) create mode 100644 docs/safe-synthesizer/.gitignore create mode 100644 plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/config.py create mode 100644 plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/http_utils.py create mode 100644 plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job.py create mode 100644 plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job_builder.py create mode 100644 plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/resources.py create mode 100644 plugins/nemo-safe-synthesizer/tests/unit/test_sdk.py diff --git a/docs/_snippets/nvidia-build-model-provider.md b/docs/_snippets/nvidia-build-model-provider.md index ed54b6e50db..cac6bf7f97b 100644 --- a/docs/_snippets/nvidia-build-model-provider.md +++ b/docs/_snippets/nvidia-build-model-provider.md @@ -1,8 +1,8 @@ !!! note - The platform pre-configures a `system/nvidia-build` model provider during startup. + The platform pre-configures a `default/nvidia-build` model provider during startup. This provider routes inference requests to models hosted on `build.nvidia.com` using the API base URL `https://integrate.api.nvidia.com` - and the NGC API key with `Public API Endpoints` permissions provided during deployment (automatically saved as the built-in `system/ngc-api-key` secret). + and the NGC API key with `Public API Endpoints` permissions provided during deployment. - You can verify this provider exists by running `nemo inference providers list --workspace system`. + You can verify this provider exists by running `nemo inference providers list --workspace default`. The tutorials in these docs use this provider for inference, but you can alternatively create your own and use it instead. diff --git a/docs/safe-synthesizer/.gitignore b/docs/safe-synthesizer/.gitignore new file mode 100644 index 00000000000..84cf576933c --- /dev/null +++ b/docs/safe-synthesizer/.gitignore @@ -0,0 +1 @@ +tutorials/evaluation_report.html diff --git a/docs/safe-synthesizer/about/host-local-development.md b/docs/safe-synthesizer/about/host-local-development.md index 701f16e8695..f5b2214b648 100644 --- a/docs/safe-synthesizer/about/host-local-development.md +++ b/docs/safe-synthesizer/about/host-local-development.md @@ -1,9 +1,11 @@ -# Host-Local Development and Testing +# Local and Subprocess Execution -Run {{nss_short_name}} on your machine's GPU with `nemo safe-synthesizer run-local`. This page covers the plugin CLI only (`run-local` and `runtime`). It does not cover platform job submission or `nemo safe-synthesizer jobs …` commands (not exposed in the CLI today). +Run {{nss_short_name}} on your machine's GPU with `nemo safe-synthesizer run-local`. The public command is a local subprocess wrapper: the main NeMo CLI starts a separate Safe Synthesizer runtime Python, and that runtime executes the synthesis task module. + +This page covers local execution only. Platform job submission uses the Jobs API or SDK; the `nemo safe-synthesizer` CLI exposes `run-local` and `runtime`. ## Prerequisites @@ -25,7 +27,18 @@ uv run nemo safe-synthesizer --help # Commands: run-local, runtime ``` -## Run a job locally +## Execution modes + +There are two local paths: + +| Mode | Command | Use it when | +|------|---------|-------------| +| Managed local subprocess | `uv run nemo safe-synthesizer run-local ...` | You want the supported plugin CLI. This creates the parent CLI process, then launches the runtime Python subprocess. | +| Direct local task | ` -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local ...` | You are debugging the task process itself and want to bypass the parent CLI wrapper. | + +Both modes run on the host GPU and write artifacts to the local filesystem. Both accept the same task arguments: `--spec-file`, `--workspace`, `--output-dir`, and optional `--data-source`. + +## Run with the managed local subprocess Use a job spec JSON (example in `plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/nss-job.json`) and a local input file: @@ -44,6 +57,39 @@ uv run nemo safe-synthesizer run-local \ | `--output-dir` | Where artifacts are written (default `./nss-output`) | | `--workspace` | Workspace label for spec fields that reference workspaces (default `default`) | +The parent command launches a subprocess equivalent to: + +```bash + -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local \ + --workspace default \ + --spec-file ./nss-job.json \ + --data-source ./input.csv \ + --output-dir ./nss-output +``` + +Find the configured runtime Python with: + +```bash +uv run nemo safe-synthesizer runtime info +``` + +## Run the local task directly + +Direct task execution is useful when you need to reproduce a subprocess failure without the parent CLI wrapper. + +```bash +$(uv run nemo safe-synthesizer runtime info | awk -F': ' '/^python:/ {print $2}') \ + -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local \ + --workspace default \ + --spec-file ./nss-job.json \ + --data-source ./input.csv \ + --output-dir ./nss-output +``` + +If the runtime Python does not exist, run `uv run nemo safe-synthesizer runtime setup` first. + +If you omit `--data-source`, the task downloads `data_source` from the platform Files service. Use `--data-source` for offline local files. + ### Output layout | Path | Description | @@ -161,7 +207,5 @@ Requires `RUN_NSS_LOCAL_E2E=1`, CUDA, and `nemo safe-synthesizer runtime setup`. ## Related topics -- [Parameters Reference](reference.md) — spec and `config` fields -- [Getting Started](../getting-started.md) — GPU and platform context -- [Jobs](jobs.md) — platform job lifecycle (separate from this run-local guide) +- [Getting Started](../getting-started.md) — GPU and local runtime prerequisites - Plugin README: `plugins/nemo-safe-synthesizer/README.md` diff --git a/docs/safe-synthesizer/about/jobs.md b/docs/safe-synthesizer/about/jobs.md index 187d9e0a766..9b5ded16c3f 100644 --- a/docs/safe-synthesizer/about/jobs.md +++ b/docs/safe-synthesizer/about/jobs.md @@ -120,7 +120,7 @@ import os import pandas as pd from nemo_platform import NeMoPlatform -from nemo_platform.beta.safe_synthesizer.job_builder import SafeSynthesizerJobBuilder +from nemo_safe_synthesizer_plugin.sdk.job_builder import SafeSynthesizerJobBuilder # Placeholders df: pd.DataFrame = pd.DataFrame() diff --git a/docs/safe-synthesizer/about/reference.md b/docs/safe-synthesizer/about/reference.md index 543897d7f8b..9cda0abeee6 100644 --- a/docs/safe-synthesizer/about/reference.md +++ b/docs/safe-synthesizer/about/reference.md @@ -95,7 +95,7 @@ import os import pandas as pd from nemo_platform import NeMoPlatform -from nemo_platform.beta.safe_synthesizer.job_builder import SafeSynthesizerJobBuilder +from nemo_safe_synthesizer_plugin.sdk.job_builder import SafeSynthesizerJobBuilder # Placeholders df: pd.DataFrame = pd.DataFrame() diff --git a/docs/safe-synthesizer/getting-started.md b/docs/safe-synthesizer/getting-started.md index 26eb896567d..bf5acb1dd2c 100644 --- a/docs/safe-synthesizer/getting-started.md +++ b/docs/safe-synthesizer/getting-started.md @@ -1,18 +1,18 @@ # Getting Started with {{nss_short_name}} -Get started with {{nss_short_name}} for generating private synthetic versions of sensitive tabular datasets. +Get started with {{nss_short_name}} for generating private synthetic versions of sensitive tabular datasets on a host GPU. ## Prerequisites -Before using {{nss_short_name}}, complete the [{{platform_name}} Quickstart](../get-started/quickstart.md) to install the CLI/SDK and deploy the platform. +Before using {{nss_short_name}}, complete [Setup](../get-started/setup.md) to install the CLI/SDK. {{nss_short_name}} has the following additional requirements: -- An NVIDIA GPU **on the host machine** with 80GB+ VRAM (check with `nvidia-smi`). This is separate from any GPU inside a NIM container — Safe Synthesizer training runs directly on the host. +- An NVIDIA GPU **on the host machine** with 80GB+ VRAM (check with `nvidia-smi`). This is separate from any GPU inside a NIM container; Safe Synthesizer training runs directly on the host. - Sufficient disk space for generated datasets (50GB+ recommended) -For general platform troubleshooting (port conflicts, health checks, and so on), refer to the [main quickstart guide](../get-started/quickstart.md). +For general platform troubleshooting (port conflicts, health checks, and so on), refer to [Setup](../get-started/setup.md). --8<-- "_snippets/nvidia-build-model-provider.md" @@ -20,7 +20,7 @@ For general platform troubleshooting (port conflicts, health checks, and so on), ## Host-local CLI -For GPU development on your machine, install the Safe Synthesizer plugin from this repository and use `nemo safe-synthesizer run-local` (see [Host-Local Development and Testing](about/host-local-development.md)): +For GPU development on your machine, install the Safe Synthesizer plugin from this repository and use `nemo safe-synthesizer run-local` (see [Local and Subprocess Execution](about/host-local-development.md)): ```shell BOOTSTRAP_LOCAL_PLUGIN_DIRS=plugins/nemo-safe-synthesizer make bootstrap-python @@ -31,15 +31,15 @@ uv run nemo safe-synthesizer run-local \ --output-dir ./nss-output ``` -Platform job submission (Jobs API, Studio, tutorials) is documented separately in [Jobs](about/jobs.md) and the [tutorials](tutorials/index.md). The `nemo safe-synthesizer` CLI today exposes **run-local** and **runtime** only. +The `run-local` command launches the Safe Synthesizer task in a separate runtime Python subprocess. The `nemo safe-synthesizer` CLI today exposes **run-local** and **runtime** only; platform job submission uses the Jobs API or SDK. --- ## Next Steps -Run one of the [tutorials](tutorials/index.md) to create your first synthetic dataset: +Create your first synthetic dataset: -- [Safe Synthesizer 101 Tutorial](tutorials/safe-synthesizer-101.md) - A beginner-friendly introduction -- [Differential Privacy Tutorial](tutorials/differential-privacy.md) - Generate differentially-private synthetic data +- [Safe Synthesizer 101 Tutorial](tutorials/safe-synthesizer-101.md) - a beginner-friendly introduction +- [Local and Subprocess Execution](about/host-local-development.md) - local CLI and runtime task details --- diff --git a/docs/safe-synthesizer/llms.txt b/docs/safe-synthesizer/llms.txt index 1e7912e4413..ee5a8ec941f 100644 --- a/docs/safe-synthesizer/llms.txt +++ b/docs/safe-synthesizer/llms.txt @@ -26,7 +26,7 @@ Jobs are created using `SafeSynthesizerJobBuilder` from the Python SDK: ```python -from nemo_platform.beta.safe_synthesizer.job_builder import SafeSynthesizerJobBuilder +from nemo_safe_synthesizer_plugin.sdk.job_builder import SafeSynthesizerJobBuilder builder = ( SafeSynthesizerJobBuilder(client) diff --git a/docs/safe-synthesizer/tutorials/differential-privacy.md b/docs/safe-synthesizer/tutorials/differential-privacy.md index 43c7870a41a..61c73521ac9 100644 --- a/docs/safe-synthesizer/tutorials/differential-privacy.md +++ b/docs/safe-synthesizer/tutorials/differential-privacy.md @@ -71,7 +71,7 @@ fi import os import pandas as pd from nemo_platform import NeMoPlatform -from nemo_platform.beta.safe_synthesizer.job_builder import SafeSynthesizerJobBuilder +from nemo_safe_synthesizer_plugin.sdk.job_builder import SafeSynthesizerJobBuilder # Configure client client = NeMoPlatform( @@ -262,7 +262,7 @@ for i, exp in enumerate(experiments): Configure differential privacy with custom parameters: ```python -from nemo_platform.beta.safe_synthesizer.config import DifferentialPrivacyHyperparams +from nemo_safe_synthesizer_plugin.sdk.config import DifferentialPrivacyHyperparams # Create custom privacy configuration privacy_config = DifferentialPrivacyHyperparams( diff --git a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md index 7521fb348ce..ded0d252117 100644 --- a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md +++ b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md @@ -73,8 +73,11 @@ try: print("✅ Successfully connected to Safe Synthesizer service") print(f"Found {len(jobs.data)} existing jobs") except Exception as e: - print(f"❌ Cannot connect to service: {e}") - print("Please verify base_url and service status") + raise RuntimeError( + "Cannot connect to the Safe Synthesizer service. Restart the platform after installing " + "the safe-synthesizer plugin, then verify that /apis/safe-synthesizer/v2/workspaces/default/jobs " + "appears in the platform OpenAPI schema." + ) from e ``` --- @@ -85,7 +88,7 @@ For this tutorial, we'll use a women's clothing reviews dataset from Kaggle that ```python import pandas as pd -import kagglehub # type: ignore[import-not-found] +import kagglehub # Download the dataset path = kagglehub.dataset_download("nicapotato/womens-ecommerce-clothing-reviews") @@ -116,7 +119,9 @@ Before running jobs, set up column classification for accurate PII detection. ```python # Use the pre-configured NVIDIA Build model provider # This provider is set up automatically during platform deployment -provider_name = "system/nvidia-build" +provider_name = os.environ.get("NSS_CLASSIFY_MODEL_PROVIDER", "default/nvidia-build") +provider_workspace, provider_id = provider_name.split("/", 1) +client.inference.providers.retrieve(provider_id, workspace=provider_workspace) print(f"✅ Using model provider: {provider_name}") ``` @@ -141,6 +146,8 @@ if hf_token: # Store your HuggingFace token as a platform secret client.secrets.create(workspace="default", name=hf_secret_name, value=hf_token) print(f"✓ Created secret: {hf_secret_name}") +else: + hf_secret_name = None ``` ## Step 7: Create and Run a Safe Synthesizer Job @@ -149,7 +156,7 @@ Use the `SafeSynthesizerJobBuilder` to configure and create a job: ```python import pandas as pd -from nemo_platform.beta.safe_synthesizer.job_builder import SafeSynthesizerJobBuilder +from nemo_safe_synthesizer_plugin.sdk.job_builder import SafeSynthesizerJobBuilder # Create a project for our jobs (creates if it doesn't exist) project_name = "test-project" @@ -164,7 +171,6 @@ builder = ( SafeSynthesizerJobBuilder(client) .with_data_source(df) .with_classify_model_provider(provider_name) # Enable column classification - .with_replace_pii() # Enable PII replacement .synthesize() # Enable synthesis ) @@ -226,10 +232,10 @@ except RuntimeError as e: If the job fails with **"No GPUs available on this system"**, ensure your quickstart is configured with GPU access: + + ```bash -nemo quickstart configure -# Select "host-gpu" when prompted -nemo quickstart up +nemo setup --start-services ``` Verify GPU access with `nvidia-smi` on the host. @@ -240,6 +246,8 @@ Verify GPU access with `nvidia-smi` on the host. Once the job is complete, retrieve the generated synthetic data: + + ```python synthetic_df = job.fetch_data() @@ -301,25 +309,57 @@ job.display_report_in_notebook() --- -## Understanding the Results +## Step 11: Generate More Records from the Same Adapter -### Interpreting Scores +The completed job stores its LoRA adapter as an `adapter` result in Files. You can submit a second job with `pretrained_model_job` to reuse that adapter and skip training. This is useful when you want more synthetic records from the same trained model. -The evaluation report contains two high-level scores: Synthetic Quality Score (SQS) and Data Privacy Score (DPS). Both are measured out of 10, and higher is better. To learn more about how to interpret the scores, refer to the [evaluation guide](../about/evaluation.md). +```python +reuse_job_name = f"synthesis-reuse-{pd.Timestamp.now().strftime('%Y%m%d-%H%M%S')}" +reuse_builder = ( + SafeSynthesizerJobBuilder(client) + .with_data_source(df) + .with_pretrained_model_job(job.job_name) + .with_generate(num_records=100) +) + +reuse_job = reuse_builder.create_job(name=reuse_job_name, project=project_name) +print(f"✅ Adapter reuse job created: {reuse_job.job_name}") +``` + +Wait for the generation-only job and fetch the additional records: + +```python +print("⏳ Waiting for adapter reuse job to complete...") +try: + reuse_job.wait_for_completion() + print("✅ Adapter reuse job completed!") +except RuntimeError as e: + print(f"❌ Adapter reuse job failed: {e}") + raise + +additional_synthetic_df = reuse_job.fetch_data() +print(f"✅ Generated {len(additional_synthetic_df)} additional synthetic records") +print(additional_synthetic_df.head()) +``` + +`pretrained_model_job` accepts either a job in the current workspace (`job.job_name`) or a fully-qualified `/` reference. Do not set `config.training.pretrained_model` when using `pretrained_model_job`; that local path field is only for `run-local` workflows. --- -## Next Steps +## Understanding the Results + +### Interpreting Scores -Now that you've completed your first Safe Synthesizer job, explore more advanced features: +The evaluation report contains two high-level scores: Synthetic Quality Score (SQS) and Data Privacy Score (DPS). Both are measured out of 10, and higher is better. -### Advanced Tutorials +--- -- [Differential Privacy Tutorial](differential-privacy.md) - Apply mathematical privacy guarantees +## Next Steps -### Documentation +Now that you've completed your first Safe Synthesizer job, try the local CLI path: -- [index](../about/index.md) - Understand core concepts +- [Local and Subprocess Execution](../about/host-local-development.md) - run Safe Synthesizer directly on a host GPU +- [Getting Started](../getting-started.md) - review local runtime prerequisites ### Try These Next @@ -368,7 +408,7 @@ print(f"Total jobs: {len(all_jobs.data)}") - Use smaller model (adjust `training.pretrained_model`) - Check GPU availability -For more help, see [jobs](../about/jobs.md). +For local CLI failures, see [Local and Subprocess Execution](../about/host-local-development.md). **Error: "Dataset must have at least 200 records to use holdout."** diff --git a/mkdocs.yml b/mkdocs.yml index e509663059e..60cd7737027 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -206,8 +206,6 @@ extra: - Platform - Fine-tune Models - Customizer - - Synthesize Safe Data - - Safe Synthesizer - Example Applications paths: - auth/** @@ -221,7 +219,14 @@ extra: - get-started/quickstart.md - helm/** - run-inference/tutorials/deploy-models.md - - safe-synthesizer/** + - safe-synthesizer/about/data-synthesis.md + - safe-synthesizer/about/evaluation.md + - safe-synthesizer/about/index.md + - safe-synthesizer/about/jobs.md + - safe-synthesizer/about/pii-replacement.md + - safe-synthesizer/about/reference.md + - safe-synthesizer/tutorials/differential-privacy.md + - safe-synthesizer/tutorials/index.md - set-up/** - troubleshooting/cluster-setup.md - troubleshooting/customizer.md @@ -320,19 +325,9 @@ nav: - SDK Resources: data-designer/sdk-resources.md - Migrating from Standalone Library: data-designer/migration.md - Synthesize Safe Data: - - About: - - Overview: safe-synthesizer/about/index.md - - Data Synthesis: safe-synthesizer/about/data-synthesis.md - - Evaluation: safe-synthesizer/about/evaluation.md - - Jobs: safe-synthesizer/about/jobs.md - - Host-Local Development: safe-synthesizer/about/host-local-development.md - - PII Replacement: safe-synthesizer/about/pii-replacement.md - - Parameters Reference: safe-synthesizer/about/reference.md - Getting Started: safe-synthesizer/getting-started.md - - Tutorials: - - Overview: safe-synthesizer/tutorials/index.md - - Safe Synthesizer 101: safe-synthesizer/tutorials/safe-synthesizer-101.md - - Differential Privacy: safe-synthesizer/tutorials/differential-privacy.md + - Local and Subprocess Execution: safe-synthesizer/about/host-local-development.md + - Safe Synthesizer 101: safe-synthesizer/tutorials/safe-synthesizer-101.md - Anonymize Data: - About: anonymizer/index.md - Quickstart: anonymizer/quickstart.md diff --git a/packages/nemo_platform/pyproject.toml b/packages/nemo_platform/pyproject.toml index 664d85da368..4ce894028c4 100644 --- a/packages/nemo_platform/pyproject.toml +++ b/packages/nemo_platform/pyproject.toml @@ -557,6 +557,7 @@ anonymizer = "nemo_anonymizer_plugin.sdk.resources:anonymizer_sdk_resources" auditor = "nemo_auditor.sdk:auditor_sdk_resources" data_designer = "nemo_data_designer_plugin.sdk.resources:data_designer_sdk_resources" evaluator = "nemo_evaluator.sdk.resources:evaluator_sdk_resources" +safe_synthesizer = "nemo_safe_synthesizer_plugin.sdk.resources:safe_synthesizer_sdk_resources" # Generated from [tool.bundle-package]; do not edit this table by hand. [project.entry-points."nemo.seed"] diff --git a/plugins/nemo-safe-synthesizer/README.md b/plugins/nemo-safe-synthesizer/README.md index 1399058b672..0a38678fca9 100644 --- a/plugins/nemo-safe-synthesizer/README.md +++ b/plugins/nemo-safe-synthesizer/README.md @@ -12,7 +12,7 @@ Use the Safe Synthesizer plugin to run a job on a host GPU while preserving the ## Steps -=== "CLI" +=== "Managed local subprocess" 1. Sync the workspace and install this plugin outside the root lock: @@ -32,27 +32,34 @@ Use the Safe Synthesizer plugin to run a job on a host GPU while preserving the uv run python plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py --files-api-url http://localhost:8080 ``` - 4. Run the job locally: + 4. Run the job locally through the public plugin CLI: ```bash - uv run nemo safe-synthesizer run-local --workspace default --spec-file nss-job.json --output-dir ./nss-output + uv run nemo safe-synthesizer run-local \ + --workspace default \ + --spec-file nss-job.json \ + --data-source ./input.csv \ + --output-dir ./nss-output ``` -=== "Managed Runtime" +=== "Direct local task" 1. Sync the workspace, install this plugin outside the root lock, and create the runtime venv: ```bash BOOTSTRAP_LOCAL_PLUGIN_DIRS=plugins/nemo-safe-synthesizer make bootstrap-python uv run nemo safe-synthesizer runtime setup - # Optional after platform is up — see step 3 in the CLI section - uv run python plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py --files-api-url http://localhost:8080 ``` - 2. Run the same local job through the managed runtime: + 2. Run the task module directly with the configured runtime Python: ```bash - uv run nemo safe-synthesizer run-local --workspace default --spec-file nss-job.json --output-dir ./nss-output + $(uv run nemo safe-synthesizer runtime info | awk -F': ' '/^python:/ {print $2}') \ + -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local \ + --workspace default \ + --spec-file nss-job.json \ + --data-source ./input.csv \ + --output-dir ./nss-output ``` The command writes generated data, summaries, and any adapter output under `./nss-output`. @@ -61,7 +68,7 @@ The command writes generated data, summaries, and any adapter output under `./ns - If model downloads fail, confirm the Files API URL is reachable and the model filesets exist in the selected workspace. - If CUDA initialization fails, run `uv run nemo safe-synthesizer runtime info` and verify the runtime package matches the installed driver/runtime. -- If the job cannot load input data, pass a local data source or confirm the fileset reference in the job spec. +- If the job cannot load input data, pass `--data-source` with a local file or confirm the fileset reference in the job spec. ## Related Links diff --git a/plugins/nemo-safe-synthesizer/pyproject.toml b/plugins/nemo-safe-synthesizer/pyproject.toml index 54576643cc5..8fec62dc687 100644 --- a/plugins/nemo-safe-synthesizer/pyproject.toml +++ b/plugins/nemo-safe-synthesizer/pyproject.toml @@ -28,6 +28,9 @@ safe-synthesizer = "nemo_safe_synthesizer_plugin.service:SafeSynthesizerService" [project.entry-points."nemo.cli"] safe-synthesizer = "nemo_safe_synthesizer_plugin.cli:SafeSynthesizerCLI" +[project.entry-points."nemo.sdk"] +safe_synthesizer = "nemo_safe_synthesizer_plugin.sdk.resources:safe_synthesizer_sdk_resources" + [project.entry-points."nemo.skills"] safe-synthesizer = "nemo_safe_synthesizer_plugin.skills:get_skills_path" diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py index ad0f5a1a425..abfe76568dd 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py @@ -130,7 +130,7 @@ def _validate_pretrained_model_source(cls, data: Any) -> Any: return data config_data = data.get("config") training_data = config_data.get("training") if isinstance(config_data, dict) else None - if isinstance(training_data, dict) and "pretrained_model" in training_data: + if isinstance(training_data, dict) and training_data.get("pretrained_model") is not None: raise ValueError("Use either 'pretrained_model_job' or 'config.training.pretrained_model', not both.") return data @@ -154,6 +154,15 @@ def parse_pretrained_model_job_ref(job_ref: str, workspace_fallback: str) -> tup return workspace, job_name +def _runtime_job_config(job_config: SafeSynthesizerJobConfig) -> dict[str, Any]: + config = job_config.model_dump() + if job_config.pretrained_model_job: + training = config.get("config", {}).get("training") + if isinstance(training, dict): + training.pop("pretrained_model", None) + return config + + def _create_job_step(job_config: SafeSynthesizerJobConfig, environment: list[EnvironmentVariable]) -> PlatformJobStep: if config.job_mode == "subprocess-local": try: @@ -168,7 +177,7 @@ def _create_job_step(job_config: SafeSynthesizerJobConfig, environment: list[Env profile=config.job_executor_profile, command=command, ), - config=job_config.model_dump(), + config=_runtime_job_config(job_config), environment=environment, ) @@ -196,7 +205,7 @@ def _create_job_step(job_config: SafeSynthesizerJobConfig, environment: list[Env ), resources=resources, ), - config=job_config.model_dump(), + config=_runtime_job_config(job_config), environment=environment, ) diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/config.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/config.py new file mode 100644 index 00000000000..93d69cc99e2 --- /dev/null +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/config.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Safe Synthesizer config re-exports for plugin SDK users.""" + +try: + from nemo_safe_synthesizer.config import ( + DataParameters, + DifferentialPrivacyHyperparams, + EvaluationParameters, + GenerateParameters, + PiiReplacerConfig, + SafeSynthesizerJobConfig, + SafeSynthesizerParameters, + TimeSeriesParameters, + TrainingHyperparams, + ) + + __all__ = [ + "DataParameters", + "DifferentialPrivacyHyperparams", + "EvaluationParameters", + "GenerateParameters", + "PiiReplacerConfig", + "SafeSynthesizerJobConfig", + "SafeSynthesizerParameters", + "TimeSeriesParameters", + "TrainingHyperparams", + ] +except ImportError: + __all__ = [] diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/http_utils.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/http_utils.py new file mode 100644 index 00000000000..2ee3b212d4f --- /dev/null +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/http_utils.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""HTTP helpers for the Safe Synthesizer plugin SDK.""" + +from __future__ import annotations + +from urllib.parse import quote, urljoin + +from nemo_platform import AsyncNeMoPlatform, NeMoPlatform + +PlatformClient = NeMoPlatform | AsyncNeMoPlatform + +_API_PREFIX = "/apis/safe-synthesizer" + + +def base_url(source: str) -> str: + """Return the normalized base URL for a raw URL string.""" + return source.rstrip("/") + + +def resolve_workspace(platform: PlatformClient, workspace: str | None, *, strict: bool = False) -> str: + """Return the explicit, platform, or default workspace for Safe Synthesizer routes.""" + resolved = workspace or platform.workspace + if resolved is None: + if strict: + raise ValueError("workspace must be provided when the client has no default workspace") + return "default" + return resolved + + +def url(platform: PlatformClient, path: str, workspace: str | None = None) -> str: + """Build a full Safe Synthesizer plugin API URL for the provided route path.""" + resolved_path = path.format(workspace=resolve_workspace(platform, workspace)) + return _join_url(str(platform.base_url), f"{_API_PREFIX}/{resolved_path}") + + +def platform_default_headers(platform: PlatformClient) -> dict[str, str]: + """Return string-valued default platform headers for direct plugin HTTP calls.""" + return {str(key): value for key, value in platform.default_headers.items() if isinstance(value, str)} + + +def job_route_base_url(*, raw_base_url: str, workspace: str, job_name: str) -> str: + """Build the stable Safe Synthesizer plugin URL prefix for one submitted job.""" + encoded_workspace = quote(workspace, safe="") + encoded_job_name = quote(job_name, safe="") + return _join_url(raw_base_url, f"{_API_PREFIX}/v2/workspaces/{encoded_workspace}/jobs/{encoded_job_name}") + + +def job_route_resource_url(*, job_base_url: str, resource_path: str) -> str: + """Build a full URL below a stable Safe Synthesizer job route.""" + return _join_url(job_base_url, resource_path) + + +def _join_url(root: str, relative_path: str) -> str: + """Join a root URL and a relative path using URL parsing rules.""" + return urljoin(f"{base_url(root)}/", relative_path.lstrip("/")) diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job.py new file mode 100644 index 00000000000..bb8246a768b --- /dev/null +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""High-level Safe Synthesizer job SDK helpers.""" + +from __future__ import annotations + +import json +import logging +import time +from base64 import b64encode +from io import BytesIO +from pathlib import Path +from typing import Iterator + +import pandas as pd +from nemo_platform import NeMoPlatform +from nemo_platform._types import Omit, omit +from nemo_platform.types import PlatformJobLog, PlatformJobStatusResponse +from nemo_safe_synthesizer.config.external_results import SafeSynthesizerSummary +from typing_extensions import Self + +logger = logging.getLogger(__name__) + + +class ReportHtml: + """Container for a Safe Synthesizer HTML report.""" + + def __init__(self, html: str): + self.raw_html = html + self.as_data_uri = f"data:text/html;base64,{b64encode(self.raw_html.encode()).decode()}" + + def save(self, path: str | Path) -> None: + """Save the evaluation report to a file.""" + Path(path).write_text(self.raw_html, encoding="utf-8") + + def display_report_in_notebook(self, width: str = "100%", height: int = 1000) -> None: + """Display the evaluation report in a Jupyter notebook.""" + try: + from IPython.display import IFrame, display + except ImportError: + logger.warning("IPython is required to display reports in notebooks. Report will not be displayed.") + return + + display(IFrame(self.as_data_uri, width=width, height=height)) + + @classmethod + def read(cls, path: str | Path) -> Self: + """Read an evaluation report from a file.""" + return cls(Path(path).read_text(encoding="utf-8")) + + +class SafeSynthesizerJob: + """Convenience wrapper for a Safe Synthesizer platform job.""" + + def __init__(self, job_name: str, client: NeMoPlatform, workspace: str = "default"): + self.job_name = job_name + self._client = client + self._workspace = workspace + + def fetch_status(self) -> str: + """Fetch the current job status.""" + return self.fetch_status_info().status + + def fetch_status_info(self) -> PlatformJobStatusResponse: + """Fetch the current job status response.""" + return self._client.jobs.get_status(self.job_name, workspace=self._workspace) + + def wait_for_completion( + self, poll_interval: int = 10, verbose: bool = True, log_timeout: float | None = None + ) -> None: + """Block until the job reaches a terminal state.""" + last_page_cursor: str | None = None + seen_log_keys: set[str] = set() + previous_status_info = None + current_status_info = self.fetch_status_info() + while current_status_info.status not in ["completed", "error", "cancelled"]: + if verbose: + logging_level = None + try: + httpx_logger = logging.getLogger("httpx") + logging_level = httpx_logger.level + httpx_logger.setLevel("ERROR") + new_logs, last_page_cursor = self._fetch_logs_incremental( + page_cursor=last_page_cursor, timeout=log_timeout + ) + for new_log in new_logs: + log_key = f"{new_log.timestamp}:{hash(new_log.message)}" + if log_key not in seen_log_keys: + print(new_log.message.strip()) + seen_log_keys.add(log_key) + except Exception: + logger.exception("Error fetching logs") + finally: + if logging_level is not None: + logging.getLogger("httpx").setLevel(logging_level) + current_status_info = self.fetch_status_info() + if current_status_info != previous_status_info: + print( + f"Job status changed to status: '{current_status_info.status}',", + f"status_details: {current_status_info.status_details},", + f"error_details: {current_status_info.error_details}", + ) + previous_status_info = current_status_info + time.sleep(poll_interval) + if current_status_info.status in ["error", "cancelled"]: + raise RuntimeError( + f"Job '{self.job_name}' ended with status '{current_status_info.status}'. " + f"Details: {current_status_info.status_details}. " + f"Error: {current_status_info.error_details}. " + "Check job logs with job.print_logs() for more details." + ) + + def fetch_summary(self) -> SafeSynthesizerSummary: + """Fetch the machine-readable job summary.""" + response = self._client.jobs.results.download("summary", job=self.job_name, workspace=self._workspace) + return SafeSynthesizerSummary.model_validate(json.loads(response.read().decode("utf-8"))) + + def fetch_report(self) -> ReportHtml: + """Fetch the evaluation report as HTML.""" + response = self._client.jobs.results.download("evaluation-report", job=self.job_name, workspace=self._workspace) + return ReportHtml(html=response.read().decode("utf-8")) + + def display_report_in_notebook(self, width: str = "100%", height: int = 1000) -> None: + """Display the evaluation report in a Jupyter notebook.""" + self.fetch_report().display_report_in_notebook(width=width, height=height) + + def save_report(self, path: str | Path) -> None: + """Save the evaluation report to a file.""" + self.fetch_report().save(path) + + def fetch_data(self) -> pd.DataFrame: + """Fetch generated synthetic data as a pandas DataFrame.""" + response = self._client.jobs.results.download("synthetic-data", job=self.job_name, workspace=self._workspace) + return pd.read_csv(BytesIO(response.read())) + + def _fetch_logs_incremental( + self, page_cursor: str | None = None, timeout: float | None = None + ) -> tuple[list[PlatformJobLog], str | None]: + """Fetch logs incrementally starting from a page cursor.""" + timeout = 300.0 if timeout is None else timeout + all_logs: list[PlatformJobLog] = [] + current_cursor: str | Omit = omit if page_cursor is None else page_cursor + last_cursor_with_data: str | None = page_cursor + + while True: + response = self._client.with_options(timeout=timeout).jobs.get_logs( + self.job_name, + page_cursor=current_cursor, + workspace=self._workspace, + ) + + if response.data: + all_logs.extend(response.data) + if isinstance(current_cursor, str): + last_cursor_with_data = current_cursor + + if response.next_page is None: + return all_logs, last_cursor_with_data + current_cursor = response.next_page + + def fetch_logs(self, timeout: float | None = None) -> Iterator[PlatformJobLog]: + """Fetch job logs as an iterator over log objects.""" + timeout = 300.0 if timeout is None else timeout + page_cursor: str | Omit = omit + while True: + response = self._client.with_options(timeout=timeout).jobs.get_logs( + self.job_name, + page_cursor=page_cursor, + workspace=self._workspace, + ) + yield from response.data + if response.next_page is None: + break + page_cursor = response.next_page + + def print_logs(self, timeout: float | None = None) -> None: + """Print job logs to stdout.""" + for log in self.fetch_logs(timeout=timeout): + print(log.message.strip()) diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job_builder.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job_builder.py new file mode 100644 index 00000000000..e292b51b5c5 --- /dev/null +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job_builder.py @@ -0,0 +1,241 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Builder for Safe Synthesizer jobs submitted through the plugin SDK.""" + +from __future__ import annotations + +import logging +import random +import string +import tempfile +from pathlib import Path +from typing import Any + +import pandas as pd +from nemo_platform import NeMoPlatform +from nemo_safe_synthesizer_plugin.sdk.job import SafeSynthesizerJob +from typing_extensions import Self + +logger = logging.getLogger(__name__) + +_ConfigInput = Any + + +def _to_dict(config: _ConfigInput) -> dict[str, Any]: + """Coerce a Pydantic model, dict, or None to a plain dict.""" + if config is None: + return {} + if hasattr(config, "model_dump"): + return config.model_dump() + if isinstance(config, dict): + return config.copy() + raise ValueError(f"Config must be a dict, a Pydantic model, or None; got {type(config)!r}") + + +def _merge_config(config: _ConfigInput, kwargs: dict[str, Any]) -> dict[str, Any]: + base = _to_dict(config) + base.update(kwargs) + return base + + +class SafeSynthesizerJobBuilder: + """Fluent builder for Safe Synthesizer plugin jobs.""" + + def __init__(self, client: NeMoPlatform, workspace: str = "default"): + self._client = client + self._workspace = workspace + + self._hf_token_secret: str | None = None + self._classify_model_provider: str | None = None + self._pretrained_model_job: str | None = None + self._data_source: pd.DataFrame | str | Path | None = None + self._data_source_path: str | None = None + + self._enable_synthesis = False + self._enable_replace_pii = False + + self._data_config: dict[str, Any] = {} + self._training_config: dict[str, Any] = {} + self._generation_config: dict[str, Any] = {} + self._evaluation_config: dict[str, Any] = {} + self._privacy_config: dict[str, Any] | None = None + self._time_series_config: dict[str, Any] = {} + self._replace_pii_config: dict[str, Any] = {} + + def with_data_source(self, df_source: pd.DataFrame | str | Path) -> Self: + """Set the data source as a DataFrame or local data file path.""" + self._data_source = df_source + self._data_source_path = None + return self + + def synthesize(self) -> Self: + """Enable data synthesis for the job run.""" + self._enable_synthesis = True + return self + + def with_data(self, config: _ConfigInput = None, **kwargs: Any) -> Self: + """Configure data parameters.""" + self._data_config = _merge_config(config, kwargs) + return self + + def with_train(self, config: _ConfigInput = None, **kwargs: Any) -> Self: + """Configure training hyperparameters and enable synthesis.""" + self._training_config = _merge_config(config, kwargs) + self._enable_synthesis = True + return self + + def with_generate(self, config: _ConfigInput = None, **kwargs: Any) -> Self: + """Configure generation parameters and enable synthesis.""" + self._generation_config = _merge_config(config, kwargs) + self._enable_synthesis = True + return self + + def with_evaluate(self, config: _ConfigInput = None, **kwargs: Any) -> Self: + """Configure evaluation parameters.""" + self._evaluation_config = _merge_config(config, kwargs) + return self + + def with_differential_privacy(self, config: _ConfigInput = None, **kwargs: Any) -> Self: + """Configure differential privacy parameters.""" + self._privacy_config = _merge_config(config, kwargs) + return self + + def with_time_series(self, config: _ConfigInput = None, **kwargs: Any) -> Self: + """Configure time-series parameters.""" + self._time_series_config = _merge_config(config, kwargs) + return self + + def with_replace_pii(self, config: _ConfigInput = None, **kwargs: Any) -> Self: + """Configure and enable PII replacement.""" + self._replace_pii_config = _merge_config(config, kwargs) + self._enable_replace_pii = True + return self + + def with_classify_model_provider(self, provider_name: str) -> Self: + """Configure column classification with an Inference Gateway model provider.""" + if "/" in provider_name: + self._classify_model_provider = provider_name + else: + self._classify_model_provider = f"{self._workspace}/{provider_name}" + logger.info("Configured classify model provider: %s", self._classify_model_provider) + return self + + def with_hf_token_secret(self, secret_name: str) -> Self: + """Configure Hugging Face authentication through a platform secret.""" + self._hf_token_secret = secret_name + return self + + def with_pretrained_model_job(self, job_name: str) -> Self: + """Reuse a previous Safe Synthesizer job's adapter for generation-only synthesis. + + Args: + job_name: Completed job name in the current workspace, or a fully-qualified + ``/`` reference. + """ + self._pretrained_model_job = job_name + self._enable_synthesis = True + return self + + def resolve_job_config(self) -> Self: + """Upload input data and validate the final job configuration without submitting.""" + self._resolve_datasource() + self._build_job_spec() + return self + + def create_job(self, **kwargs: Any) -> SafeSynthesizerJob: + """Upload input data and submit the Safe Synthesizer job.""" + self._resolve_datasource() + response = self._client.safe_synthesizer.jobs.create( + workspace=self._workspace, + spec=self._build_job_spec(), + **kwargs, + ) + return SafeSynthesizerJob(response.name, self._client, workspace=self._workspace) + + def _resolve_datasource(self, **kwargs: Any) -> None: + if self._data_source_path is not None: + return + if isinstance(self._data_source, pd.DataFrame): + df = self._data_source + elif isinstance(self._data_source, str | Path): + df = pd.read_csv(self._data_source, **kwargs) + else: + raise ValueError("Data source must be a pandas DataFrame or local data file path") + + tmp_path: Path | None = None + try: + with tempfile.NamedTemporaryFile(mode="w+", suffix=".csv", delete=False) as tmp: + tmp_path = Path(tmp.name) + df.to_csv(tmp_path, index=False) + file_name = f"dataset{self._generate_random_string()}.csv" + self._data_source_path = self._upload_to_fileset( + dataset_path=tmp_path, + filename=file_name, + fileset_name="safe-synthesizer-inputs", + ) + finally: + if tmp_path is not None: + tmp_path.unlink(missing_ok=True) + + def _build_job_spec(self) -> dict[str, Any]: + if not self._enable_replace_pii and not self._enable_synthesis: + raise ValueError("Data synthesis and/or replace PII must be enabled") + if not self._data_source_path: + raise ValueError("No data source path found after uploading dataset") + + pii_config: dict[str, Any] | None = None + if self._enable_replace_pii: + pii_config = self._replace_pii_config.copy() + if self._classify_model_provider: + globals_cfg = pii_config.setdefault("globals", {}) + classify_cfg = globals_cfg.setdefault("classify", {}) + classify_cfg["classify_model_provider"] = self._classify_model_provider + + nss_config: dict[str, Any] = { + "enable_synthesis": self._enable_synthesis, + "enable_replace_pii": self._enable_replace_pii, + "data": self._data_config, + "training": self._training_config, + "generation": self._generation_config, + "evaluation": self._evaluation_config, + "time_series": self._time_series_config, + } + if self._privacy_config is not None: + nss_config["privacy"] = self._privacy_config + if pii_config is not None: + nss_config["replace_pii"] = pii_config + + spec: dict[str, Any] = { + "data_source": self._data_source_path, + "config": nss_config, + } + if self._hf_token_secret: + spec["hf_token_secret"] = self._hf_token_secret + if self._pretrained_model_job: + spec["pretrained_model_job"] = self._pretrained_model_job + return spec + + def _upload_to_fileset(self, dataset_path: str | Path, filename: str, fileset_name: str) -> str: + dataset_path = self._validate_dataset_path(dataset_path) + self._client.files.upload( + local_path=str(dataset_path), + remote_path=filename, + fileset=fileset_name, + workspace=self._workspace, + fileset_auto_create=True, + ) + return f"{self._workspace}/{fileset_name}#{filename}" + + def _generate_random_string(self, length: int = 6) -> str: + characters = string.ascii_uppercase + string.digits + return "".join(random.choice(characters) for _ in range(length)) + + @staticmethod + def _validate_dataset_path(dataset_path: str | Path) -> Path: + path = Path(dataset_path) + if not path.is_file(): + raise ValueError("To upload a dataset, you must provide a valid file path.") + if path.suffix not in {".parquet", ".csv", ".json", ".jsonl"}: + raise ValueError("Dataset files must be in parquet, csv, json, or jsonl format.") + return path diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/resources.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/resources.py new file mode 100644 index 00000000000..bcb4039689f --- /dev/null +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/resources.py @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Plugin SDK resources for Safe Synthesizer.""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import quote + +import httpx +from nemo_platform import AsyncNeMoPlatform, NeMoPlatform +from nemo_platform_plugin.sdk import NemoPluginSDKResources +from nemo_safe_synthesizer_plugin.sdk import http_utils + + +class SafeSynthesizerJobsResource: + """Sync SDK namespace mounted as ``client.safe_synthesizer.jobs``.""" + + def __init__(self, platform: NeMoPlatform) -> None: + self._platform = platform + self._http_client = platform._client + + def create( + self, + *, + spec: dict[str, Any], + workspace: str | None = None, + name: str | None = None, + project: str | None = None, + description: str | None = None, + ownership: dict[str, object] | None = None, + custom_fields: dict[str, object] | None = None, + timeout: float | None = None, + ) -> Any: + """Create a Safe Synthesizer platform job through the plugin route.""" + payload: dict[str, Any] = {"spec": spec} + if name is not None: + payload["name"] = name + if project is not None: + payload["project"] = project + if description is not None: + payload["description"] = description + if ownership is not None: + payload["ownership"] = ownership + if custom_fields is not None: + payload["custom_fields"] = custom_fields + + response = self._http_client.post( + http_utils.url(self._platform, "/v2/workspaces/{workspace}/jobs", workspace), + json=payload, + headers=http_utils.platform_default_headers(self._platform), + timeout=timeout, + ) + _raise_for_status(response) + return _object_from_mapping(response.json()) + + def list(self, *, workspace: str | None = None, **params: Any) -> Any: + """List Safe Synthesizer jobs.""" + response = self._http_client.get( + http_utils.url(self._platform, "/v2/workspaces/{workspace}/jobs", workspace), + params={key: value for key, value in params.items() if value is not None}, + headers=http_utils.platform_default_headers(self._platform), + ) + _raise_for_status(response) + return _object_from_mapping(response.json()) + + def retrieve(self, name: str, *, workspace: str | None = None) -> Any: + """Retrieve one Safe Synthesizer job by name.""" + response = self._http_client.get( + http_utils.url( + self._platform, + f"/v2/workspaces/{{workspace}}/jobs/{quote(name, safe='')}", + workspace, + ), + headers=http_utils.platform_default_headers(self._platform), + ) + _raise_for_status(response) + return _object_from_mapping(response.json()) + + def get_status(self, name: str, *, workspace: str | None = None) -> Any: + """Retrieve Safe Synthesizer job status.""" + return self._platform.jobs.get_status(name, workspace=workspace) + + def get_logs(self, name: str, *, workspace: str | None = None, **kwargs: Any) -> Any: + """Retrieve paginated Safe Synthesizer job logs from the Jobs service.""" + return self._platform.jobs.get_logs(name, workspace=workspace, **kwargs) + + +class SafeSynthesizerResource: + """Sync SDK namespace mounted as ``client.safe_synthesizer``.""" + + def __init__(self, platform: NeMoPlatform) -> None: + self._platform = platform + self.jobs = SafeSynthesizerJobsResource(platform) + + +class AsyncSafeSynthesizerJobsResource: + """Async SDK namespace mounted as ``client.safe_synthesizer.jobs``.""" + + def __init__(self, platform: AsyncNeMoPlatform) -> None: + self._platform = platform + self._http_client = platform._client + + async def create( + self, + *, + spec: dict[str, Any], + workspace: str | None = None, + name: str | None = None, + project: str | None = None, + description: str | None = None, + ownership: dict[str, object] | None = None, + custom_fields: dict[str, object] | None = None, + timeout: float | None = None, + ) -> Any: + """Create a Safe Synthesizer platform job through the plugin route.""" + payload: dict[str, Any] = {"spec": spec} + if name is not None: + payload["name"] = name + if project is not None: + payload["project"] = project + if description is not None: + payload["description"] = description + if ownership is not None: + payload["ownership"] = ownership + if custom_fields is not None: + payload["custom_fields"] = custom_fields + + response = await self._http_client.post( + http_utils.url(self._platform, "/v2/workspaces/{workspace}/jobs", workspace), + json=payload, + headers=http_utils.platform_default_headers(self._platform), + timeout=timeout, + ) + _raise_for_status(response) + return _object_from_mapping(response.json()) + + async def list(self, *, workspace: str | None = None, **params: Any) -> Any: + """List Safe Synthesizer jobs.""" + response = await self._http_client.get( + http_utils.url(self._platform, "/v2/workspaces/{workspace}/jobs", workspace), + params={key: value for key, value in params.items() if value is not None}, + headers=http_utils.platform_default_headers(self._platform), + ) + _raise_for_status(response) + return _object_from_mapping(response.json()) + + async def retrieve(self, name: str, *, workspace: str | None = None) -> Any: + """Retrieve one Safe Synthesizer job by name.""" + response = await self._http_client.get( + http_utils.url( + self._platform, + f"/v2/workspaces/{{workspace}}/jobs/{quote(name, safe='')}", + workspace, + ), + headers=http_utils.platform_default_headers(self._platform), + ) + _raise_for_status(response) + return _object_from_mapping(response.json()) + + async def get_status(self, name: str, *, workspace: str | None = None) -> Any: + """Retrieve Safe Synthesizer job status.""" + return await self._platform.jobs.get_status(name, workspace=workspace) + + def get_logs(self, name: str, *, workspace: str | None = None, **kwargs: Any) -> Any: + """Retrieve paginated Safe Synthesizer job logs from the Jobs service.""" + return self._platform.jobs.get_logs(name, workspace=workspace, **kwargs) + + +class AsyncSafeSynthesizerResource: + """Async SDK namespace mounted as ``client.safe_synthesizer``.""" + + def __init__(self, platform: AsyncNeMoPlatform) -> None: + self._platform = platform + self.jobs = AsyncSafeSynthesizerJobsResource(platform) + + +def _object_from_mapping(value: Any) -> Any: + """Convert JSON objects into attribute-accessible objects recursively.""" + if isinstance(value, dict): + return _SDKObject({str(key): _object_from_mapping(child) for key, child in value.items()}) + if isinstance(value, list): + return [_object_from_mapping(child) for child in value] + return value + + +def _raise_for_status(response: httpx.Response) -> None: + """Raise HTTP errors with FastAPI detail text included.""" + try: + response.raise_for_status() + except httpx.HTTPStatusError as e: + detail = _response_detail(response) + if detail: + message = f"{e}. Response detail: {detail}" + raise httpx.HTTPStatusError(message, request=e.request, response=e.response) from e + raise + + +def _response_detail(response: httpx.Response) -> str | None: + try: + body = response.json() + except ValueError: + text = response.text.strip() + return text or None + if isinstance(body, dict) and "detail" in body: + return str(body["detail"]) + return str(body) if body else None + + +class _SDKObject(dict[str, Any]): + """Small dict wrapper with attribute access for plugin route responses.""" + + def __getattr__(self, name: str) -> Any: + try: + return self[name] + except KeyError as e: + raise AttributeError(name) from e + + +safe_synthesizer_sdk_resources = NemoPluginSDKResources( + sync_resource=SafeSynthesizerResource, + async_resource=AsyncSafeSynthesizerResource, +) diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/diagnose.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/diagnose.md index 81e90a860e3..81dc6153a6e 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/diagnose.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/diagnose.md @@ -11,7 +11,7 @@ ## First Checks 1. Resolve the CLI with `command -v nemo 2>/dev/null || (test -x .venv/bin/nemo && realpath .venv/bin/nemo) || echo CLI_NOT_FOUND`. -2. Confirm whether the user is running host-local (`nemo safe-synthesizer run-local`) or platform jobs (`nemo safe-synthesizer jobs create`). +2. Confirm whether the user is running host-local (`nemo safe-synthesizer run-local`) or a platform job through the Jobs API or SDK. 3. Inspect the spec file before changing commands. ## Common Failures @@ -52,8 +52,8 @@ Then confirm the Files API URL is reachable and the target workspace contains th ### Job remains pending or results are missing -- Check the platform job status with `nemo safe-synthesizer jobs get --workspace `. -- If the command supports waiting, retry creation with `--wait --timeout ` for a synchronous status loop. +- Check the platform job status with the Jobs API or SDK. +- If the submission path supports waiting, retry creation with its documented wait or polling option. - Inspect job result names from the artifacts workflow. ## Source Files for Development Debugging @@ -68,5 +68,5 @@ Only inspect these when the user asks to change or debug plugin code: - Re-run with the command shape in `workflows/run.md`. - Recreate model filesets with `plugins/nemo-safe-synthesizer/scripts/setup_model_filesets.py`. -- Check job status with `nemo safe-synthesizer jobs get --workspace `. +- Check platform job status with the Jobs API or SDK. - Retrieve result names with `workflows/results.md`, then inspect `summary` or `summary.json` first. diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/results.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/results.md index 1095c9b8537..9123c385367 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/results.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/results.md @@ -25,17 +25,10 @@ Platform jobs publish named results through the Jobs service: - `evaluation-report` - `adapter` -Use the generated Safe Synthesizer jobs result commands when available: - -```bash -nemo safe-synthesizer jobs results list --workspace default -nemo safe-synthesizer jobs results get --job --workspace default -``` - -If those generated commands differ in the installed CLI, run `nemo safe-synthesizer jobs results --help` and follow the current help text. +Use the Jobs API or SDK to list and fetch result records for platform jobs. The plugin CLI does not expose `nemo safe-synthesizer jobs ...` result commands. ## Next Steps - Interpret artifact names and missing output cases with `workflows/artifacts.md`. -- Check job status with `nemo safe-synthesizer jobs get --workspace `. +- Check platform job status with the Jobs API or SDK. - Diagnose failures with `workflows/diagnose.md`. diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/run.md b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/run.md index 6e826dcce98..215214d5ead 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/run.md +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/skills/safe-synthesizer/workflows/run.md @@ -44,14 +44,9 @@ uv run nemo safe-synthesizer run-local \ --output-dir ./nss-output ``` -Use platform job submission when the user wants the NMP Jobs service to run Safe Synthesizer: +Use the Jobs API or SDK when the user wants the NMP Jobs service to run Safe Synthesizer. The plugin CLI does not expose `nemo safe-synthesizer jobs` commands. -```bash -nemo safe-synthesizer jobs create my-safe-synthesizer-job \ - --workspace default \ - --input-file platform-job.json \ - --wait -``` +For CLI users, point them to the generated Jobs/API surface available in their installed NeMo CLI, or to the Python SDK builder documented in `docs/safe-synthesizer/tutorials/safe-synthesizer-101.md`. ## Minimal Spec Shape @@ -74,7 +69,7 @@ nemo safe-synthesizer jobs create my-safe-synthesizer-job \ } ``` -For platform submission, pass this object as the `spec` field when using `--input-data`; with `--input-file`, the generated API command accepts the same create payload shape documented by CLI help. +For platform submission, pass this object as the `spec` field in the Jobs API or SDK create payload. `platform-job.json` wraps the job spec: diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py b/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py index a88f6a999f9..cc44990ce16 100644 --- a/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py +++ b/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py @@ -119,6 +119,27 @@ async def test_job_config_compiler_validates_pretrained_model_job(mock_sdk): ) +@pytest.mark.asyncio +async def test_plugin_job_config_allows_pretrained_model_job_runtime_config(mock_sdk): + mock_sdk.jobs.results.retrieve = AsyncMock( + return_value=MagicMock(artifact_url="default/job-results-prior#results/attempt-1/adapter") + ) + spec = PluginJobConfig.model_validate( + { + "data_source": DEFAULT_DATA_SOURCE, + "pretrained_model_job": "prior-safe-synth-job", + "config": {}, + } + ) + + compiled = await _compile(spec, mock_sdk) + step = next(iter(compiled["steps"])) + reparsed = PluginJobConfig.model_validate(step["config"]) + + assert "pretrained_model" not in step["config"]["config"]["training"] + assert reparsed.pretrained_model_job == "prior-safe-synth-job" + + @pytest.mark.asyncio async def test_job_config_compiler_pretrained_model_job_not_found(mock_sdk): mock_sdk.jobs.results.retrieve = AsyncMock( diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_sdk.py b/plugins/nemo-safe-synthesizer/tests/unit/test_sdk.py new file mode 100644 index 00000000000..c463e7836b6 --- /dev/null +++ b/plugins/nemo-safe-synthesizer/tests/unit/test_sdk.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import MagicMock + +import httpx +import pandas as pd +from nemo_platform import NeMoPlatform +from nemo_platform_plugin.discovery import discover, discover_entry_points +from nemo_safe_synthesizer_plugin.sdk.job_builder import SafeSynthesizerJobBuilder +from nemo_safe_synthesizer_plugin.sdk.resources import SafeSynthesizerResource + + +def _mock_platform(requests: list[httpx.Request]) -> NeMoPlatform: + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 201, + json={"name": "safe-synth-job", "status": "created", "spec": {"data_source": "default/data#input.csv"}}, + ) + + http_client = httpx.Client(transport=httpx.MockTransport(handler)) + return NeMoPlatform(base_url="http://nmp.test", http_client=http_client, workspace="default") + + +def test_safe_synthesizer_resource_creates_job_through_plugin_route() -> None: + requests: list[httpx.Request] = [] + platform = _mock_platform(requests) + resource = SafeSynthesizerResource(platform) + + response = resource.jobs.create( + workspace="default", + name="safe-synth-job", + spec={"data_source": "default/data#input.csv", "config": {}}, + ) + + assert response.name == "safe-synth-job" + assert requests[0].method == "POST" + assert str(requests[0].url) == "http://nmp.test/apis/safe-synthesizer/v2/workspaces/default/jobs" + assert json.loads(requests[0].read()) == { + "spec": {"data_source": "default/data#input.csv", "config": {}}, + "name": "safe-synth-job", + } + + +def test_safe_synthesizer_resource_mounts_on_platform_client() -> None: + discover.cache_clear() + discover_entry_points.cache_clear() + requests: list[httpx.Request] = [] + platform = _mock_platform(requests) + + response = platform.safe_synthesizer.jobs.create( + workspace="default", + name="safe-synth-job", + spec={"data_source": "default/data#input.csv", "config": {}}, + ) + + assert response.name == "safe-synth-job" + assert str(requests[0].url) == "http://nmp.test/apis/safe-synthesizer/v2/workspaces/default/jobs" + + +def test_safe_synthesizer_resource_includes_response_detail_in_errors() -> None: + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(422, json={"detail": "Failed to compile safe-synthesizer job spec"}) + + http_client = httpx.Client(transport=httpx.MockTransport(handler)) + platform = NeMoPlatform(base_url="http://nmp.test", http_client=http_client, workspace="default") + resource = SafeSynthesizerResource(platform) + + try: + resource.jobs.create(workspace="default", spec={"data_source": "default/data#input.csv", "config": {}}) + except httpx.HTTPStatusError as e: + assert "Response detail: Failed to compile safe-synthesizer job spec" in str(e) + else: + raise AssertionError("Expected HTTPStatusError") + + +def test_job_builder_uploads_dataframe_and_submits_spec() -> None: + client = MagicMock() + client.files.upload = MagicMock() + client.safe_synthesizer.jobs.create.return_value = SimpleNamespace(name="safe-synth-job") + + builder = ( + SafeSynthesizerJobBuilder(client, workspace="default") + .with_data_source(pd.DataFrame({"value": [1]})) + .with_classify_model_provider("nvidia-build") + .with_replace_pii() + .synthesize() + .with_generate(num_records=10) + .with_hf_token_secret("hf-token") + ) + + job = builder.create_job(name="safe-synth-job") + + assert job.job_name == "safe-synth-job" + client.files.upload.assert_called_once() + create_kwargs = client.safe_synthesizer.jobs.create.call_args.kwargs + assert create_kwargs["workspace"] == "default" + assert create_kwargs["name"] == "safe-synth-job" + assert create_kwargs["spec"]["data_source"].startswith("default/safe-synthesizer-inputs#dataset") + assert create_kwargs["spec"]["hf_token_secret"] == "hf-token" + config = create_kwargs["spec"]["config"] + assert config["enable_synthesis"] is True + assert config["enable_replace_pii"] is True + assert config["generation"] == {"num_records": 10} + assert config["replace_pii"]["globals"]["classify"]["classify_model_provider"] == "default/nvidia-build" + + +def test_job_builder_submits_pretrained_model_job_for_adapter_reuse() -> None: + client = MagicMock() + client.files.upload = MagicMock() + client.safe_synthesizer.jobs.create.return_value = SimpleNamespace(name="adapter-reuse-job") + + builder = ( + SafeSynthesizerJobBuilder(client, workspace="default") + .with_data_source(pd.DataFrame({"value": [1]})) + .with_pretrained_model_job("first-synth-job") + .with_generate(num_records=25) + ) + + job = builder.create_job(name="adapter-reuse-job") + + assert job.job_name == "adapter-reuse-job" + create_kwargs = client.safe_synthesizer.jobs.create.call_args.kwargs + assert create_kwargs["spec"]["pretrained_model_job"] == "first-synth-job" + assert create_kwargs["spec"]["config"]["generation"] == {"num_records": 25} + assert "pretrained_model" not in create_kwargs["spec"]["config"]["training"] diff --git a/pyproject.toml b/pyproject.toml index 5ff85157b0e..1783d2b2dd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -168,6 +168,7 @@ enabled-plugins = [ "nemo-evaluator-plugin", "nemo-guardrails-plugin", "nemo-auditor-plugin", + "nemo-safe-synthesizer-plugin", "nemo-switchyard", "nemo-agents-plugin", ] diff --git a/third_party/licenses.jsonl b/third_party/licenses.jsonl index 974e60584c5..04e4a3d3d88 100644 --- a/third_party/licenses.jsonl +++ b/third_party/licenses.jsonl @@ -153,6 +153,7 @@ {"name": "multiprocess", "license": "BSD-3-CLAUSE", "compatible": true} {"name": "mypy-extensions", "license": "MIT", "compatible": true} {"name": "nemo-anonymizer", "license": "APACHE-2.0", "compatible": true} +{"name": "nemo-safe-synthesizer", "license": "APACHE-2.0", "compatible": true} {"name": "nemoguardrails", "license": "APACHE-2.0", "compatible": true} {"name": "nest-asyncio", "license": "BSD-3-CLAUSE", "compatible": true} {"name": "nest-asyncio2", "license": "BSD-3-CLAUSE", "compatible": true} diff --git a/third_party/osv-licenses.json b/third_party/osv-licenses.json index 878575ed8f7..db54176cf97 100644 --- a/third_party/osv-licenses.json +++ b/third_party/osv-licenses.json @@ -2507,6 +2507,16 @@ "Apache-2.0" ] }, + { + "package": { + "name": "nemo-safe-synthesizer", + "version": "0.1.1", + "ecosystem": "PyPI" + }, + "licenses": [ + "Apache-2.0" + ] + }, { "package": { "name": "nemoguardrails", @@ -5727,7 +5737,7 @@ }, { "name": "Apache-2.0", - "count": 79 + "count": 80 }, { "name": "non-standard", diff --git a/third_party/requirements-main.txt b/third_party/requirements-main.txt index 015f897c8c9..869950db5d2 100644 --- a/third_party/requirements-main.txt +++ b/third_party/requirements-main.txt @@ -27,6 +27,7 @@ # nemo-auditor-plugin # nemo-data-designer-plugin # nemo-guardrails-plugin + # nemo-safe-synthesizer-plugin # nemoplatform -e ./packages/nemo_platform_ext ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') # via nemoplatform @@ -41,6 +42,7 @@ # nemo-platform # nemo-platform-ext # nemo-platform-sdk + # nemo-safe-synthesizer-plugin # nemo-switchyard # nmp-common # nmp-inference-gateway @@ -53,6 +55,7 @@ # nemo-data-designer-plugin # nemo-evaluator-plugin # nemo-platform + # nemo-safe-synthesizer-plugin # nemoplatform # nmp-auth # nmp-core-mcp @@ -85,6 +88,7 @@ -e ./plugins/nemo-data-designer ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -e ./plugins/nemo-evaluator ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -e ./plugins/nemo-guardrails ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') +-e ./plugins/nemo-safe-synthesizer ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -e ./plugins/nemo-switchyard ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') -e ./plugins/nemo-switchyard/vendor/switchyard ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') # via nemo-switchyard @@ -536,6 +540,7 @@ colorama==0.4.6 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 # via + # nemo-safe-synthesizer # nvidia-nat-core # sacrebleu # sqlfluff @@ -716,7 +721,9 @@ expandvars==1.1.2 ; (platform_machine == 'arm64' and sys_platform == 'darwin') o faker==20.1.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:562a3a09c3ed3a1a7b20e13d79f904dfdfc5e740f72813ecf95e4cf71e5a2f52 \ --hash=sha256:aeb3e26742863d1e387f9d156f1c36e14af63bf5e6f36fb39b8c27f6a903be38 - # via data-designer-engine + # via + # data-designer-engine + # nemo-safe-synthesizer fastapi==0.129.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:61315cebd2e65df5f97ec298c888f9de30430dd0612d59d6480beafbc10655af \ --hash=sha256:b4946880e48f462692b31c083be0432275cbfb6e2274566b1be91479cc1a84ec @@ -724,6 +731,7 @@ fastapi==0.129.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or # nemo-anonymizer-plugin # nemo-data-designer-plugin # nemo-platform-plugin + # nemo-safe-synthesizer-plugin # nemoguardrails # nemoplatform # nmp-auth @@ -901,6 +909,7 @@ fsspec==2025.3.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or # filesets # huggingface-hub # nemo-platform-sdk + # nemo-safe-synthesizer-plugin # nmp-guardrails gitdb==4.0.12 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571 \ @@ -968,7 +977,9 @@ grpcio==1.80.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or ( gunicorn==25.3.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:cacea387dab08cd6776501621c295a904fe8e3b7aae9a1a3cbb26f4e7ed54660 \ --hash=sha256:f74e1b2f9f76f6cd1ca01198968bd2dd65830edc24b6e8e4d78de8320e2fe889 - # via nmp-guardrails + # via + # nemo-safe-synthesizer-plugin + # nmp-guardrails h11==0.16.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 @@ -1040,6 +1051,8 @@ httpx==0.28.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (p # nemo-auditor-plugin # nemo-data-designer-plugin # nemo-platform-sdk + # nemo-safe-synthesizer + # nemo-safe-synthesizer-plugin # nemoguardrails # nmp-auth # nmp-guardrails @@ -1065,6 +1078,7 @@ huggingface-hub==1.15.0 ; (platform_machine == 'arm64' and sys_platform == 'darw # datasets # fastembed # langchain-huggingface + # nemo-safe-synthesizer # nmp-common # nmp-evaluator # tokenizers @@ -1252,6 +1266,7 @@ jsonschema==4.23.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') # litellm # mcp # nemo-evaluator-sdk + # nemo-safe-synthesizer jsonschema-path==0.3.4 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:8365356039f16cc65fddffafda5f58766e34bebab7d6d105616ab52bc4297001 \ --hash=sha256:f502191fdc2b22050f9a81c9237be9d27145b9001c55842bece5e94e382e52f8 @@ -1298,6 +1313,7 @@ langchain-community==0.3.27 ; (platform_machine == 'arm64' and sys_platform == ' --hash=sha256:e1037c3b9da0c6d10bf06e838b034eb741e016515c79ef8f3f16e53ead33d882 # via # nemoguardrails + # nmp-evaluator # nvidia-nat-langchain # ragas langchain-core==1.3.3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ @@ -1655,6 +1671,9 @@ mypy-extensions==1.0.0 ; (platform_machine == 'arm64' and sys_platform == 'darwi nemo-anonymizer==0.2.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:369ee9f717e3c346328bcef9767da3b596d5b927b5f9cc162ece95766b1d8aad # via nemo-anonymizer-plugin +nemo-safe-synthesizer==0.1.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ + --hash=sha256:e92564b8522ffc2360fb6daaca36e88a722206e165a4fe157e7ccf65b1cac260 + # via nemo-safe-synthesizer-plugin nemoguardrails==0.21.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:b338453b371751f5b09637415702e2ee25f0885317b691cbb0d2f2f164eeea5d # via @@ -2078,6 +2097,7 @@ pandas==2.3.3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (p # nemo-anonymizer-plugin # nemo-data-designer-plugin # nemo-evaluator-sdk + # nemo-safe-synthesizer # nemoguardrails # nmp-files # nmp-jobs @@ -2390,6 +2410,8 @@ pydantic==2.12.5 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or # nemo-platform-ext # nemo-platform-plugin # nemo-platform-sdk + # nemo-safe-synthesizer + # nemo-safe-synthesizer-plugin # nemoguardrails # nemoplatform # nmp-auth @@ -2467,6 +2489,8 @@ pydantic-settings==2.8.1 ; (platform_machine == 'arm64' and sys_platform == 'dar # langchain-community # mcp # nemo-platform-plugin + # nemo-safe-synthesizer + # nemo-safe-synthesizer-plugin # nmp-auth # nmp-common # nmp-customizer @@ -2564,6 +2588,7 @@ python-multipart==0.0.32 ; (platform_machine == 'arm64' and sys_platform == 'dar # data-designer-engine # fastapi # mcp + # nemo-safe-synthesizer-plugin # nmp-guardrails # nvidia-nat-core pytz==2026.1.post1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ @@ -2610,6 +2635,7 @@ pyyaml==6.0.3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (p # nemo-platform-ext # nemo-platform-plugin # nemo-platform-sdk + # nemo-safe-synthesizer # nemoguardrails # nemoplatform # nmp-auth @@ -2710,6 +2736,7 @@ requests==2.33.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or # langsmith # nemo-platform-ext # nemo-platform-sdk + # nemo-safe-synthesizer-plugin # nemoplatform # ngcsdk # nmp-evaluator @@ -2741,6 +2768,7 @@ rich==14.3.3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (pl # nemo-agents-plugin # nemo-platform-ext # nemo-platform-sdk + # nemo-safe-synthesizer # nemoguardrails # ngcsdk # nmp-platform @@ -2934,7 +2962,9 @@ sentry-sdk==2.57.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') setuptools==82.0.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9 \ --hash=sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb - # via pymilvus + # via + # nemo-safe-synthesizer + # pymilvus shellingham==1.5.4 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de @@ -3045,7 +3075,9 @@ streaming-form-data==2.0.0 ; (platform_machine == 'arm64' and sys_platform == 'd structlog==25.5.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:098522a3bebed9153d4570c6d0288abf80a031dfdb2048d59a49e9dc2190fc98 \ --hash=sha256:a8453e9b9e636ec59bd9e79bbd4a72f025981b3ba0f5837aebf48f02f37a7f9f - # via nmp-common + # via + # nemo-safe-synthesizer + # nmp-common sympy==1.14.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ --hash=sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517 \ --hash=sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5 @@ -3130,6 +3162,7 @@ tqdm==4.67.3 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (pl # datasets # fastembed # huggingface-hub + # nemo-safe-synthesizer # nltk # openai # optuna @@ -3149,6 +3182,7 @@ typer==0.24.1 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (p # nemo-platform-ext # nemo-platform-plugin # nemo-platform-sdk + # nemo-safe-synthesizer-plugin # nemoguardrails # ragas types-aioboto3==15.5.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') \ @@ -3274,6 +3308,7 @@ uvicorn==0.42.0 ; (platform_machine == 'arm64' and sys_platform == 'darwin') or # fastapi-cloud-cli # fastmcp # mcp + # nemo-safe-synthesizer-plugin # nemoguardrails # nemoplatform # nmp-auth diff --git a/uv.lock b/uv.lock index 5b90aa17d2d..990999da7ca 100644 --- a/uv.lock +++ b/uv.lock @@ -5946,6 +5946,7 @@ core-services = [ { name = "nemo-guardrails-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-platform", extra = ["services"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-platform-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-safe-synthesizer-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-switchyard", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nmp-auth", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nmp-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -6038,6 +6039,7 @@ enabled-plugins = [ { name = "nemo-data-designer-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-evaluator-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-guardrails-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-safe-synthesizer-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-switchyard", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] functional-services = [ @@ -6049,6 +6051,7 @@ functional-services = [ { name = "nemo-guardrails-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-platform", extra = ["services"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-platform-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nemo-safe-synthesizer-plugin", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nemo-switchyard", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nmp-auth", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "nmp-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, @@ -6140,6 +6143,7 @@ core-services = [ { name = "nemo-platform", editable = "packages/nemo_platform" }, { name = "nemo-platform", extras = ["services"], editable = "packages/nemo_platform" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, + { name = "nemo-safe-synthesizer-plugin", editable = "plugins/nemo-safe-synthesizer" }, { name = "nemo-switchyard", editable = "plugins/nemo-switchyard" }, { name = "nmp-auth", editable = "services/core/auth" }, { name = "nmp-common", editable = "packages/nmp_common" }, @@ -6234,6 +6238,7 @@ enabled-plugins = [ { name = "nemo-data-designer-plugin", editable = "plugins/nemo-data-designer" }, { name = "nemo-evaluator-plugin", editable = "plugins/nemo-evaluator" }, { name = "nemo-guardrails-plugin", editable = "plugins/nemo-guardrails" }, + { name = "nemo-safe-synthesizer-plugin", editable = "plugins/nemo-safe-synthesizer" }, { name = "nemo-switchyard", editable = "plugins/nemo-switchyard" }, ] functional-services = [ @@ -6246,6 +6251,7 @@ functional-services = [ { name = "nemo-platform", editable = "packages/nemo_platform" }, { name = "nemo-platform", extras = ["services"], editable = "packages/nemo_platform" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, + { name = "nemo-safe-synthesizer-plugin", editable = "plugins/nemo-safe-synthesizer" }, { name = "nemo-switchyard", editable = "plugins/nemo-switchyard" }, { name = "nmp-auth", editable = "services/core/auth" }, { name = "nmp-common", editable = "packages/nmp_common" }, From 4ebd512b972cb3b13a2e9f5bf68a704ad84e49d1 Mon Sep 17 00:00:00 2001 From: mkornfield Date: Thu, 4 Jun 2026 22:35:13 +0000 Subject: [PATCH 2/5] chore: address feedback Signed-off-by: mkornfield --- .../about/host-local-development.md | 1 + .../tutorials/safe-synthesizer-101.md | 12 ++++++++---- mkdocs.yml | 1 + .../src/nemo_platform_plugin/scheduler.py | 17 +++++++++++------ .../tests/test_scheduler.py | 2 ++ .../src/nemo_safe_synthesizer_plugin/sdk/job.py | 11 ++++++----- .../sdk/job_builder.py | 14 ++++++++++++-- 7 files changed, 41 insertions(+), 17 deletions(-) diff --git a/docs/safe-synthesizer/about/host-local-development.md b/docs/safe-synthesizer/about/host-local-development.md index f5b2214b648..7868815e7d1 100644 --- a/docs/safe-synthesizer/about/host-local-development.md +++ b/docs/safe-synthesizer/about/host-local-development.md @@ -1,6 +1,7 @@ + # Local and Subprocess Execution Run {{nss_short_name}} on your machine's GPU with `nemo safe-synthesizer run-local`. The public command is a local subprocess wrapper: the main NeMo CLI starts a separate Safe Synthesizer runtime Python, and that runtime executes the synthesis task module. diff --git a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md index ded0d252117..a8f53725ef0 100644 --- a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md +++ b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md @@ -74,8 +74,8 @@ try: print(f"Found {len(jobs.data)} existing jobs") except Exception as e: raise RuntimeError( - "Cannot connect to the Safe Synthesizer service. Restart the platform after installing " - "the safe-synthesizer plugin, then verify that /apis/safe-synthesizer/v2/workspaces/default/jobs " + "Cannot connect to the {{nss_short_name}} service. Restart the platform after installing " + "the {{nss_plugin_slug}} plugin, then verify that /apis/{{nss_plugin_slug}}/v2/workspaces/default/jobs " "appears in the platform OpenAPI schema." ) from e ``` @@ -120,8 +120,12 @@ Before running jobs, set up column classification for accurate PII detection. # Use the pre-configured NVIDIA Build model provider # This provider is set up automatically during platform deployment provider_name = os.environ.get("NSS_CLASSIFY_MODEL_PROVIDER", "default/nvidia-build") -provider_workspace, provider_id = provider_name.split("/", 1) -client.inference.providers.retrieve(provider_id, workspace=provider_workspace) +if "/" in provider_name: + provider_workspace, provider_id = provider_name.split("/", 1) + client.inference.providers.retrieve(provider_id, workspace=provider_workspace) +else: + provider_id = provider_name + client.inference.providers.retrieve(provider_id) print(f"✅ Using model provider: {provider_name}") ``` diff --git a/mkdocs.yml b/mkdocs.yml index 60cd7737027..f141f0e05b4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -261,6 +261,7 @@ extra: nsm_short_name: "NeMo Studio" nss_long_name: "NVIDIA NeMo Safe Synthesizer" nss_short_name: "NeMo Safe Synthesizer" + nss_plugin_slug: "safe-synthesizer" nop_long_name: "NVIDIA NeMo Operator" nop_short_name: "NeMo Operator" studio_long_name: "NVIDIA NeMo Studio" diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/scheduler.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/scheduler.py index 3c5828f4f10..7fe9c620064 100644 --- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/scheduler.py +++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/scheduler.py @@ -435,26 +435,31 @@ def _local_runtime_env(ctx: JobContext): :class:`JobContext` with tempdir-backed storage and exposes the same envvars here so unmodified task code keeps working. - Caller-set values win — we only fill envvars that aren't already set, - and we restore the prior environment on exit so tests and back-to-back + Caller-set storage values win — we only fill storage envvars that aren't + already set. The workspace envvar always reflects the current local + context so a parent job's workspace does not leak into nested local runs. + Prior environment values are restored on exit so tests and back-to-back calls don't leak state. """ - overrides: dict[str, str] = { + storage_overrides: dict[str, str] = { _PERSISTENT_STORAGE_ENVVAR: str(ctx.storage.persistent), _EPHEMERAL_STORAGE_ENVVAR: str(ctx.storage.ephemeral), - _WORKSPACE_ENVVAR: ctx.workspace, } # Scheduler-built local contexts leave job_id as ``None``. Explicit # caller-provided contexts may still mirror a job id for legacy code. + force_overrides: dict[str, str] = {_WORKSPACE_ENVVAR: ctx.workspace} if ctx.job_id is not None: - overrides[_JOB_ID_ENVVAR] = ctx.job_id + force_overrides[_JOB_ID_ENVVAR] = ctx.job_id saved: dict[str, str | None] = {} try: - for key, value in overrides.items(): + for key, value in storage_overrides.items(): if key in os.environ: continue # respect explicit caller setup saved[key] = None os.environ[key] = value + for key, value in force_overrides.items(): + saved[key] = os.environ.get(key) + os.environ[key] = value yield finally: for key, prior in saved.items(): diff --git a/packages/nemo_platform_plugin/tests/test_scheduler.py b/packages/nemo_platform_plugin/tests/test_scheduler.py index 87ed1b487ac..d1dbd0005d9 100644 --- a/packages/nemo_platform_plugin/tests/test_scheduler.py +++ b/packages/nemo_platform_plugin/tests/test_scheduler.py @@ -346,6 +346,7 @@ def test_storage_envvars_visible_inside_run(self, monkeypatch) -> None: monkeypatch.delenv("NEMO_JOB_PERSISTENT_JOB_STORAGE_PATH", raising=False) monkeypatch.delenv("NEMO_JOB_EPHEMERAL_TASK_STORAGE_PATH", raising=False) + monkeypatch.setenv("NEMO_JOB_WORKSPACE", "default") seen: dict[str, object] = {} @@ -375,6 +376,7 @@ def run(self, config: dict) -> dict: # NEMO_JOB_ID in that case. assert seen["job_id"] is None assert seen["workspace"] == "my-ws" + assert os.environ.get("NEMO_JOB_WORKSPACE") == "default" def test_envvars_restored_after_run(self, monkeypatch) -> None: import os diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job.py index bb8246a768b..9b34836a2d3 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job.py @@ -96,11 +96,12 @@ def wait_for_completion( logging.getLogger("httpx").setLevel(logging_level) current_status_info = self.fetch_status_info() if current_status_info != previous_status_info: - print( - f"Job status changed to status: '{current_status_info.status}',", - f"status_details: {current_status_info.status_details},", - f"error_details: {current_status_info.error_details}", - ) + if verbose: + print( + f"Job status changed to status: '{current_status_info.status}',", + f"status_details: {current_status_info.status_details},", + f"error_details: {current_status_info.error_details}", + ) previous_status_info = current_status_info time.sleep(poll_interval) if current_status_info.status in ["error", "cancelled"]: diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job_builder.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job_builder.py index e292b51b5c5..37d58f868aa 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job_builder.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job_builder.py @@ -10,7 +10,7 @@ import string import tempfile from pathlib import Path -from typing import Any +from typing import Any, cast import pandas as pd from nemo_platform import NeMoPlatform @@ -156,10 +156,20 @@ def create_job(self, **kwargs: Any) -> SafeSynthesizerJob: def _resolve_datasource(self, **kwargs: Any) -> None: if self._data_source_path is not None: return + df: pd.DataFrame if isinstance(self._data_source, pd.DataFrame): df = self._data_source elif isinstance(self._data_source, str | Path): - df = pd.read_csv(self._data_source, **kwargs) + data_source_path = Path(self._data_source) + match data_source_path.suffix.lower(): + case ".parquet": + df = cast(pd.DataFrame, pd.read_parquet(data_source_path, **kwargs)) + case ".jsonl": + df = cast(pd.DataFrame, pd.read_json(data_source_path, lines=True, **kwargs)) + case ".json": + df = cast(pd.DataFrame, pd.read_json(data_source_path, **kwargs)) + case _: + df = cast(pd.DataFrame, pd.read_csv(data_source_path, **kwargs)) else: raise ValueError("Data source must be a pandas DataFrame or local data file path") From 1bda39a0eea6f2ab1340079297149f8ad34f138f Mon Sep 17 00:00:00 2001 From: mkornfield Date: Fri, 5 Jun 2026 19:06:57 +0000 Subject: [PATCH 3/5] address cr Signed-off-by: mkornfield --- docs/_snippets/nvidia-build-model-provider.md | 2 +- docs/safe-synthesizer/about/index.md | 2 +- docs/safe-synthesizer/about/jobs.md | 4 +- docs/safe-synthesizer/about/reference.md | 2 +- docs/safe-synthesizer/getting-started.md | 1 + docs/safe-synthesizer/sdk-resources.md | 97 ++++++++++++++ .../tutorials/safe-synthesizer-101.md | 1 + mkdocs.yml | 23 ++-- packages/nemo_platform/BUNDLING.md | 2 +- .../api/v2/jobs/endpoints.py | 120 +---------------- .../job_config.py | 123 ++++++++++++++++++ .../sdk/config.py | 4 +- .../nemo_safe_synthesizer_plugin/sdk/job.py | 5 +- .../sdk/job_builder.py | 6 +- .../sdk/resources.py | 4 +- .../tasks/safe_synthesizer/__main__.py | 2 +- .../tests/unit/test_jobs.py | 41 ++++++ .../tests/unit/test_sdk.py | 102 ++++++++++++++- 18 files changed, 400 insertions(+), 141 deletions(-) create mode 100644 docs/safe-synthesizer/sdk-resources.md create mode 100644 plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/job_config.py diff --git a/docs/_snippets/nvidia-build-model-provider.md b/docs/_snippets/nvidia-build-model-provider.md index cac6bf7f97b..e81f7d92e59 100644 --- a/docs/_snippets/nvidia-build-model-provider.md +++ b/docs/_snippets/nvidia-build-model-provider.md @@ -1,5 +1,5 @@ !!! note - The platform pre-configures a `default/nvidia-build` model provider during startup. + `nemo setup` pre-configures a `default/nvidia-build` model provider during local startup. This provider routes inference requests to models hosted on `build.nvidia.com` using the API base URL `https://integrate.api.nvidia.com` and the NGC API key with `Public API Endpoints` permissions provided during deployment. diff --git a/docs/safe-synthesizer/about/index.md b/docs/safe-synthesizer/about/index.md index bc3bda32f6b..4edd6d815ab 100644 --- a/docs/safe-synthesizer/about/index.md +++ b/docs/safe-synthesizer/about/index.md @@ -106,7 +106,7 @@ Get hands-on experience with Safe Synthesizer through step-by-step tutorials. Understand the job lifecycle, configuration, and execution for Safe Synthesizer pipelines. -- **[Host-Local Development](host-local-development.md)** +- **[Local and Subprocess Execution](host-local-development.md)** --- diff --git a/docs/safe-synthesizer/about/jobs.md b/docs/safe-synthesizer/about/jobs.md index 9b5ded16c3f..0945abe5c83 100644 --- a/docs/safe-synthesizer/about/jobs.md +++ b/docs/safe-synthesizer/about/jobs.md @@ -109,7 +109,7 @@ When the job completes, access: For **platform jobs**, set `pretrained_model_job` in the job spec to a completed job that has an **`adapter`** result in Files. Reuse is generation-only (no retraining). Use either `pretrained_model_job` or `config.training.pretrained_model`, not both. -For **host-local** development (`nemo safe-synthesizer run-local`), set `config.training.pretrained_model` to a local adapter or work directory from an earlier run. See [Host-Local Development and Testing](host-local-development.md). +For **host-local** development (`nemo safe-synthesizer run-local`), set `config.training.pretrained_model` to a local adapter or work directory from an earlier run. See [Local and Subprocess Execution](host-local-development.md). ## Job Builder API @@ -309,7 +309,7 @@ kubectl get events -n --sort-by='.lastTimestamp' ## Related Topics -- [Host-Local Development and Testing](host-local-development.md): `run-local`, adapter reuse, and plugin tests +- [Local and Subprocess Execution](host-local-development.md): `run-local`, adapter reuse, and plugin tests - [safe-synthesizer-101](../tutorials/safe-synthesizer-101.md): Get started with {{nss_short_name}} jobs - [index](../tutorials/index.md): More hands-on tutorials - [reference](reference.md): Full parameter reference diff --git a/docs/safe-synthesizer/about/reference.md b/docs/safe-synthesizer/about/reference.md index 9cda0abeee6..c049eba082c 100644 --- a/docs/safe-synthesizer/about/reference.md +++ b/docs/safe-synthesizer/about/reference.md @@ -17,7 +17,7 @@ Top-level fields on the Safe Synthesizer job spec (alongside `config`): | `pretrained_model_job` | Prior completed job whose **`adapter`** result in Files is reused for **generation-only** synthesis. Format: `` or `/`. Mutually exclusive with `config.training.pretrained_model`. | | `hf_token_secret` | Platform secret name for Hugging Face token during model initialization | -For host-local runs, see [Host-Local Development and Testing](host-local-development.md). Reuse a local adapter with `config.training.pretrained_model`, not `pretrained_model_job`. +For host-local runs, see [Local and Subprocess Execution](host-local-development.md). Reuse a local adapter with `config.training.pretrained_model`, not `pretrained_model_job`. ## Top-Level Configuration diff --git a/docs/safe-synthesizer/getting-started.md b/docs/safe-synthesizer/getting-started.md index bf5acb1dd2c..0b3f6073518 100644 --- a/docs/safe-synthesizer/getting-started.md +++ b/docs/safe-synthesizer/getting-started.md @@ -41,5 +41,6 @@ Create your first synthetic dataset: - [Safe Synthesizer 101 Tutorial](tutorials/safe-synthesizer-101.md) - a beginner-friendly introduction - [Local and Subprocess Execution](about/host-local-development.md) - local CLI and runtime task details +- [SDK Resources](sdk-resources.md) - Python SDK methods for jobs, builders, logs, and results --- diff --git a/docs/safe-synthesizer/sdk-resources.md b/docs/safe-synthesizer/sdk-resources.md new file mode 100644 index 00000000000..736587227b1 --- /dev/null +++ b/docs/safe-synthesizer/sdk-resources.md @@ -0,0 +1,97 @@ + +# Safe Synthesizer {{platform_name}} SDK Resources + +The `nemo_safe_synthesizer_plugin.sdk` module provides {{platform_name}}-specific helpers for creating and monitoring {{nss_short_name}} jobs. Use these objects when you want to submit jobs through the platform Jobs service and retrieve platform-managed results. + +## SafeSynthesizerResource + +`SafeSynthesizerResource` is the entry point mounted on a `NeMoPlatform` client: + +```python +import os + +from nemo_platform import NeMoPlatform + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) +safe_synthesizer = client.safe_synthesizer +``` + +An async variant with the same namespace is available as `AsyncNeMoPlatform.safe_synthesizer`. + +## SafeSynthesizerJobsResource + +`client.safe_synthesizer.jobs` calls the plugin API for {{nss_short_name}} jobs. + +| Method | Description | +|--------|-------------| +| `create(*, spec, name=None, workspace=None, project=None, timeout=None, **params)` | Creates a {{nss_short_name}} platform job from a job spec. | +| `list(*, workspace=None, **params)` | Lists {{nss_short_name}} jobs in a workspace. | +| `retrieve(name, *, workspace=None)` | Retrieves one {{nss_short_name}} job by name. | +| `get_status(name, *, workspace=None)` | Returns the job status from the platform Jobs service. | +| `get_logs(name, *, workspace=None, **kwargs)` | Returns paginated job logs from the platform Jobs service. | + +The async resource exposes the same methods as `async def` methods. + +## SafeSynthesizerJobBuilder + +`SafeSynthesizerJobBuilder` assembles a job spec, uploads local datasets to Files, submits the job, and returns a `SafeSynthesizerJob` wrapper. + +```python +import pandas as pd + +from nemo_safe_synthesizer_plugin.sdk.job_builder import SafeSynthesizerJobBuilder + +df = pd.DataFrame({"text": ["sample record"]}) + +job = ( + SafeSynthesizerJobBuilder(client, workspace="default") + .with_data_source(df) + .with_classify_model_provider("default/nvidia-build") + .with_replace_pii() + .synthesize() + .create_job(name="safe-synth-job", project="default-project") +) +``` + +| Method | Description | +|--------|-------------| +| `with_data_source(data_source)` | Sets a pandas DataFrame or local `.csv`, `.parquet`, `.json`, or `.jsonl` file as input. Local data is uploaded to Files before submission. | +| `with_data(config=None, **kwargs)` | Sets data preparation parameters. | +| `with_train(config=None, **kwargs)` | Sets fine-tuning parameters. | +| `with_generate(config=None, **kwargs)` | Sets generation parameters and enables synthesis. | +| `synthesize()` | Enables synthesis. | +| `with_evaluate(config=None, **kwargs)` | Sets evaluation parameters. | +| `with_differential_privacy(config=None, **kwargs)` | Sets DP-SGD parameters. | +| `with_time_series(config=None, **kwargs)` | Sets time-series parameters. | +| `with_replace_pii(config=None, **kwargs)` | Enables PII replacement. | +| `with_classify_model_provider(provider_name)` | Sets the Inference Gateway provider used for PII column classification. Pair with `with_replace_pii()`. | +| `with_hf_token_secret(secret_name)` | Passes a platform secret name as `HF_TOKEN` to the runtime job. | +| `with_pretrained_model_job(job_name)` | Reuses a prior job's `adapter` result for generation-only synthesis. | +| `resolve_job_config()` | Uploads data and validates the generated job spec without submitting. | +| `create_job(**kwargs)` | Submits the job and returns `SafeSynthesizerJob`. | + +## SafeSynthesizerJob + +`SafeSynthesizerJob` is a convenience wrapper returned by the builder. + +| Method | Description | +|--------|-------------| +| `fetch_status()` | Returns the current platform job status string. | +| `fetch_status_info()` | Returns the full platform job status response. | +| `wait_for_completion(poll_interval=10, verbose=True, log_timeout=None)` | Polls status and logs until the job reaches a terminal state. Raises `RuntimeError` for `error` or `cancelled`. | +| `fetch_logs(timeout=None)` | Iterates over platform job log entries. | +| `print_logs(timeout=None)` | Prints platform job logs to stdout. | +| `fetch_data()` | Downloads the `synthetic-data` result and returns it as a pandas DataFrame. | +| `fetch_summary()` | Downloads the `summary` result as a `SafeSynthesizerSummary`. | +| `fetch_report()` | Downloads the `evaluation-report` result as HTML. | +| `save_report(path)` | Saves the HTML evaluation report to a local file. | +| `display_report_in_notebook(width="100%", height=1000)` | Displays the evaluation report in a notebook. | + +## Related Topics + +- [Safe Synthesizer 101](tutorials/safe-synthesizer-101.md) - submit and monitor a first job +- [Safe Synthesizer Jobs](about/jobs.md) - understand job lifecycle and troubleshooting +- [Parameters Reference](about/reference.md) - review job spec and configuration fields diff --git a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md index a8f53725ef0..5912bcccc33 100644 --- a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md +++ b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md @@ -175,6 +175,7 @@ builder = ( SafeSynthesizerJobBuilder(client) .with_data_source(df) .with_classify_model_provider(provider_name) # Enable column classification + .with_replace_pii() # Enable PII detection and replacement .synthesize() # Enable synthesis ) diff --git a/mkdocs.yml b/mkdocs.yml index f141f0e05b4..f90b52fe95e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -219,14 +219,6 @@ extra: - get-started/quickstart.md - helm/** - run-inference/tutorials/deploy-models.md - - safe-synthesizer/about/data-synthesis.md - - safe-synthesizer/about/evaluation.md - - safe-synthesizer/about/index.md - - safe-synthesizer/about/jobs.md - - safe-synthesizer/about/pii-replacement.md - - safe-synthesizer/about/reference.md - - safe-synthesizer/tutorials/differential-privacy.md - - safe-synthesizer/tutorials/index.md - set-up/** - troubleshooting/cluster-setup.md - troubleshooting/customizer.md @@ -327,8 +319,19 @@ nav: - Migrating from Standalone Library: data-designer/migration.md - Synthesize Safe Data: - Getting Started: safe-synthesizer/getting-started.md - - Local and Subprocess Execution: safe-synthesizer/about/host-local-development.md - - Safe Synthesizer 101: safe-synthesizer/tutorials/safe-synthesizer-101.md + - About: + - Overview: safe-synthesizer/about/index.md + - Data Synthesis: safe-synthesizer/about/data-synthesis.md + - PII Replacement: safe-synthesizer/about/pii-replacement.md + - Evaluation: safe-synthesizer/about/evaluation.md + - Jobs: safe-synthesizer/about/jobs.md + - Local and Subprocess Execution: safe-synthesizer/about/host-local-development.md + - Parameters Reference: safe-synthesizer/about/reference.md + - Tutorials: + - Overview: safe-synthesizer/tutorials/index.md + - Safe Synthesizer 101: safe-synthesizer/tutorials/safe-synthesizer-101.md + - Differential Privacy: safe-synthesizer/tutorials/differential-privacy.md + - SDK Resources: safe-synthesizer/sdk-resources.md - Anonymize Data: - About: anonymizer/index.md - Quickstart: anonymizer/quickstart.md diff --git a/packages/nemo_platform/BUNDLING.md b/packages/nemo_platform/BUNDLING.md index f6e0e0ec0e5..bf52b269b7c 100644 --- a/packages/nemo_platform/BUNDLING.md +++ b/packages/nemo_platform/BUNDLING.md @@ -130,4 +130,4 @@ The wheel gets thinner, the dependency metadata stays correct, and `pip install ## Other vendoring (`make vendor`) -The `make vendor` command also handles SDK client extensions (`nemo_platform_ext`, `data_designer_sdk`, `models`, `filesets`, `safe_synthesizer_sdk`, `nemo_evaluator_sdk`). These are **not** bundled via `[tool.bundle-package]` — they use the older `[tool.vendor-package]` mechanism which copies source files into the SDK tree with import rewriting. This is separate from the bundling described above and is only relevant to SDK client-side extensions. +The `make vendor` command also handles SDK client extensions (`nemo_platform_ext`, `data_designer_sdk`, `models`, `filesets`, `nemo_evaluator_sdk`). These are **not** bundled via `[tool.bundle-package]` — they use the older `[tool.vendor-package]` mechanism which copies source files into the SDK tree with import rewriting. This is separate from the bundling described above and is only relevant to SDK client-side extensions. diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py index abfe76568dd..183fdab01cd 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py @@ -31,129 +31,19 @@ job_route_factory, ) from nemo_safe_synthesizer.config.external_results import SafeSynthesizerSummary -from nemo_safe_synthesizer.config.job import SafeSynthesizerJobConfig as SafeSynthesizerJobConfigInternal -from nemo_safe_synthesizer.config.job import SafeSynthesizerParameters as SafeSynthesizerParametersInternal -from nemo_safe_synthesizer.config.replace_pii import PiiReplacerConfig from nemo_safe_synthesizer_plugin.config import config +from nemo_safe_synthesizer_plugin.job_config import ( + SafeSynthesizerJobConfig, + SafeSynthesizerParameters, + parse_pretrained_model_job_ref, +) from nemo_safe_synthesizer_plugin.runtime import runtime_task_command from nmp.common.jobs.exceptions import PlatformJobCompilationError from nmp.common.jobs.image import get_qualified_image -from pydantic import Field, model_validator -from pydantic.json_schema import SkipJsonSchema logger = logging.getLogger(__name__) -class SafeSynthesizerParameters(SafeSynthesizerParametersInternal): - """NMP-facing Safe Synthesizer parameters with SDK convenience flags.""" - - enable_synthesis: bool = Field( - default=True, - exclude=True, - description="Whether to run LLM training and generation phases. " - "When false the task only performs PII replacement and returns the processed data.", - ) - enable_replace_pii: bool = Field( - default=True, - exclude=True, - description="Whether to run the default PII replacement pipeline before synthesis.", - ) - - -class SafeSynthesizerJobConfig(SafeSynthesizerJobConfigInternal): - """NMP-facing Safe Synthesizer job config with SDK convenience flags.""" - - __doc__ = SafeSynthesizerJobConfigInternal.__doc__ - - config: SafeSynthesizerParameters = Field( - description="The Safe Synthesizer parameters configuration.", - ) - pretrained_model_job: str | None = Field( - default=None, - description="Optional previous NSS job whose stored adapter artifact is reused for generation-only " - "synthesis. Accepts either '' in the current workspace or '/'. " - "The plugin resolves the prior job's 'adapter' result from Files.", - ) - - enable_synthesis: SkipJsonSchema[bool] = Field( - default=True, - description="Whether to run LLM training and generation phases. " - "When False the task only performs PII replacement and returns the processed data.", - ) - - @model_validator(mode="before") - @classmethod - def _apply_enable_flags(cls, data: Any) -> Any: - if not isinstance(data, dict): - return data - cfg = data.get("config") - if not isinstance(cfg, dict): - return data - enable_synthesis = cfg.pop("enable_synthesis", True) - enable_replace_pii = cfg.pop("enable_replace_pii", True) - data.setdefault("enable_synthesis", enable_synthesis) - if not enable_replace_pii: - cfg["replace_pii"] = None - return data - - @model_validator(mode="before") - @classmethod - def _apply_pii_defaults(cls, data: Any) -> Any: - if not isinstance(data, dict): - return data - config_data = data.get("config") - if not isinstance(config_data, dict): - return data - replace_pii = config_data.get("replace_pii") - if not isinstance(replace_pii, dict) or "steps" in replace_pii: - return data - - def deep_update(base: dict, override: dict) -> dict: - for k, v in override.items(): - if isinstance(v, dict) and isinstance(base.get(k), dict): - deep_update(base[k], v) - else: - base[k] = v - return base - - default = PiiReplacerConfig.get_default_config().model_dump() - deep_update(default, replace_pii) - config_data["replace_pii"] = default - return data - - @model_validator(mode="before") - @classmethod - def _validate_pretrained_model_source(cls, data: Any) -> Any: - if not isinstance(data, dict): - return data - if not data.get("pretrained_model_job"): - return data - config_data = data.get("config") - training_data = config_data.get("training") if isinstance(config_data, dict) else None - if isinstance(training_data, dict) and training_data.get("pretrained_model") is not None: - raise ValueError("Use either 'pretrained_model_job' or 'config.training.pretrained_model', not both.") - return data - - -def parse_pretrained_model_job_ref(job_ref: str, workspace_fallback: str) -> tuple[str, str]: - """Parse a previous NSS job reference. - - Accepts either "" in the current workspace or "/". - """ - parts = job_ref.split("/", 1) - if len(parts) == 1: - workspace = workspace_fallback - job_name = parts[0] - else: - workspace, job_name = parts - - if not workspace or not job_name: - raise PlatformJobCompilationError( - f"Invalid pretrained_model_job format: {job_ref!r}. Expected '' or '/'." - ) - return workspace, job_name - - def _runtime_job_config(job_config: SafeSynthesizerJobConfig) -> dict[str, Any]: config = job_config.model_dump() if job_config.pretrained_model_job: diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/job_config.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/job_config.py new file mode 100644 index 00000000000..25402c418f9 --- /dev/null +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/job_config.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Safe Synthesizer job config shared by API handlers and task containers.""" + +from typing import Any + +from nemo_safe_synthesizer.config.job import SafeSynthesizerJobConfig as SafeSynthesizerJobConfigInternal +from nemo_safe_synthesizer.config.job import SafeSynthesizerParameters as SafeSynthesizerParametersInternal +from nemo_safe_synthesizer.config.replace_pii import PiiReplacerConfig +from nmp.common.jobs.exceptions import PlatformJobCompilationError +from pydantic import Field, model_validator +from pydantic.json_schema import SkipJsonSchema + + +class SafeSynthesizerParameters(SafeSynthesizerParametersInternal): + """NMP-facing Safe Synthesizer parameters with SDK convenience flags.""" + + enable_synthesis: bool = Field( + default=True, + exclude=True, + description="Whether to run LLM training and generation phases. " + "When false the task only performs PII replacement and returns the processed data.", + ) + enable_replace_pii: bool = Field( + default=True, + exclude=True, + description="Whether to run the default PII replacement pipeline before synthesis.", + ) + + +class SafeSynthesizerJobConfig(SafeSynthesizerJobConfigInternal): + """NMP-facing Safe Synthesizer job config with SDK convenience flags.""" + + __doc__ = SafeSynthesizerJobConfigInternal.__doc__ + + config: SafeSynthesizerParameters = Field( + description="The Safe Synthesizer parameters configuration.", + ) + pretrained_model_job: str | None = Field( + default=None, + description="Optional previous NSS job whose stored adapter artifact is reused for generation-only " + "synthesis. Accepts either '' in the current workspace or '/'. " + "The plugin resolves the prior job's 'adapter' result from Files.", + ) + + enable_synthesis: SkipJsonSchema[bool] = Field( + default=True, + description="Whether to run LLM training and generation phases. " + "When False the task only performs PII replacement and returns the processed data.", + ) + + @model_validator(mode="before") + @classmethod + def _apply_enable_flags(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + cfg = data.get("config") + if not isinstance(cfg, dict): + return data + enable_synthesis = cfg.pop("enable_synthesis", True) + enable_replace_pii = cfg.pop("enable_replace_pii", True) + data.setdefault("enable_synthesis", enable_synthesis) + if not enable_replace_pii: + cfg["replace_pii"] = None + return data + + @model_validator(mode="before") + @classmethod + def _apply_pii_defaults(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + config_data = data.get("config") + if not isinstance(config_data, dict): + return data + replace_pii = config_data.get("replace_pii") + if not isinstance(replace_pii, dict) or "steps" in replace_pii: + return data + + def deep_update(base: dict, override: dict) -> dict: + for k, v in override.items(): + if isinstance(v, dict) and isinstance(base.get(k), dict): + deep_update(base[k], v) + else: + base[k] = v + return base + + default = PiiReplacerConfig.get_default_config().model_dump() + deep_update(default, replace_pii) + config_data["replace_pii"] = default + return data + + @model_validator(mode="before") + @classmethod + def _validate_pretrained_model_source(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + if not data.get("pretrained_model_job"): + return data + config_data = data.get("config") + training_data = config_data.get("training") if isinstance(config_data, dict) else None + if isinstance(training_data, dict) and training_data.get("pretrained_model") is not None: + raise ValueError("Use either 'pretrained_model_job' or 'config.training.pretrained_model', not both.") + return data + + +def parse_pretrained_model_job_ref(job_ref: str, workspace_fallback: str) -> tuple[str, str]: + """Parse a previous NSS job reference. + + Accepts either "" in the current workspace or "/". + """ + parts = job_ref.split("/", 1) + if len(parts) == 1: + workspace = workspace_fallback + job_name = parts[0] + else: + workspace, job_name = parts + + if not workspace or not job_name: + raise PlatformJobCompilationError( + f"Invalid pretrained_model_job format: {job_ref!r}. Expected '' or '/'." + ) + return workspace, job_name diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/config.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/config.py index 93d69cc99e2..f4cc164eeeb 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/config.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/config.py @@ -27,5 +27,5 @@ "TimeSeriesParameters", "TrainingHyperparams", ] -except ImportError: - __all__ = [] +except ImportError as e: + raise ImportError("Install nemo-safe-synthesizer to use SDK config types.") from e diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job.py index 9b34836a2d3..ffd36c4cb4b 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job.py @@ -13,6 +13,7 @@ from pathlib import Path from typing import Iterator +import httpx import pandas as pd from nemo_platform import NeMoPlatform from nemo_platform._types import Omit, omit @@ -89,8 +90,8 @@ def wait_for_completion( if log_key not in seen_log_keys: print(new_log.message.strip()) seen_log_keys.add(log_key) - except Exception: - logger.exception("Error fetching logs") + except httpx.HTTPError as e: + logger.warning("Error fetching logs while waiting for job completion: %s", e) finally: if logging_level is not None: logging.getLogger("httpx").setLevel(logging_level) diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job_builder.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job_builder.py index 37d58f868aa..84b37fe59a2 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job_builder.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/job_builder.py @@ -113,7 +113,11 @@ def with_replace_pii(self, config: _ConfigInput = None, **kwargs: Any) -> Self: return self def with_classify_model_provider(self, provider_name: str) -> Self: - """Configure column classification with an Inference Gateway model provider.""" + """Configure the model provider used by PII replacement column classification. + + The provider is included in the job spec only when PII replacement is enabled with + ``with_replace_pii()``. + """ if "/" in provider_name: self._classify_model_provider = provider_name else: diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/resources.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/resources.py index bcb4039689f..910e86b8b73 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/resources.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/sdk/resources.py @@ -163,9 +163,9 @@ async def get_status(self, name: str, *, workspace: str | None = None) -> Any: """Retrieve Safe Synthesizer job status.""" return await self._platform.jobs.get_status(name, workspace=workspace) - def get_logs(self, name: str, *, workspace: str | None = None, **kwargs: Any) -> Any: + async def get_logs(self, name: str, *, workspace: str | None = None, **kwargs: Any) -> Any: """Retrieve paginated Safe Synthesizer job logs from the Jobs service.""" - return self._platform.jobs.get_logs(name, workspace=workspace, **kwargs) + return await self._platform.jobs.get_logs(name, workspace=workspace, **kwargs) class AsyncSafeSynthesizerResource: diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py index 24b6f4d8d84..1cad99c4c6c 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py @@ -30,7 +30,7 @@ from nemo_safe_synthesizer.config.internal_results import SafeSynthesizerResults from nemo_safe_synthesizer.observability import initialize_observability from nemo_safe_synthesizer.sdk.library_builder import SafeSynthesizer -from nemo_safe_synthesizer_plugin.api.v2.jobs.endpoints import ( +from nemo_safe_synthesizer_plugin.job_config import ( SafeSynthesizerJobConfig, parse_pretrained_model_job_ref, ) diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py b/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py index cc44990ce16..e9e53f33620 100644 --- a/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py +++ b/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py @@ -140,6 +140,47 @@ async def test_plugin_job_config_allows_pretrained_model_job_runtime_config(mock assert reparsed.pretrained_model_job == "prior-safe-synth-job" +def test_runtime_job_config_allows_pretrained_model_job_with_missing_training(): + job_config = MagicMock() + job_config.pretrained_model_job = "prior-safe-synth-job" + job_config.model_dump.return_value = { + "data_source": DEFAULT_DATA_SOURCE, + "pretrained_model_job": "prior-safe-synth-job", + "config": {"generation": {"num_records": 25}}, + } + + runtime_config = endpoints._runtime_job_config(job_config) + + assert runtime_config["config"] == {"generation": {"num_records": 25}} + + +def test_runtime_job_config_allows_pretrained_model_job_with_non_dict_training(): + job_config = MagicMock() + job_config.pretrained_model_job = "prior-safe-synth-job" + job_config.model_dump.return_value = { + "data_source": DEFAULT_DATA_SOURCE, + "pretrained_model_job": "prior-safe-synth-job", + "config": {"training": "local-adapter"}, + } + + runtime_config = endpoints._runtime_job_config(job_config) + + assert runtime_config["config"]["training"] == "local-adapter" + + +def test_runtime_job_config_preserves_pretrained_model_without_pretrained_model_job(): + job_config = MagicMock() + job_config.pretrained_model_job = None + job_config.model_dump.return_value = { + "data_source": DEFAULT_DATA_SOURCE, + "config": {"training": {"pretrained_model": "HuggingFaceTB/SmolLM3-3B"}}, + } + + runtime_config = endpoints._runtime_job_config(job_config) + + assert runtime_config["config"]["training"]["pretrained_model"] == "HuggingFaceTB/SmolLM3-3B" + + @pytest.mark.asyncio async def test_job_config_compiler_pretrained_model_job_not_found(mock_sdk): mock_sdk.jobs.results.retrieve = AsyncMock( diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_sdk.py b/plugins/nemo-safe-synthesizer/tests/unit/test_sdk.py index c463e7836b6..81e073072b9 100644 --- a/plugins/nemo-safe-synthesizer/tests/unit/test_sdk.py +++ b/plugins/nemo-safe-synthesizer/tests/unit/test_sdk.py @@ -4,15 +4,19 @@ from __future__ import annotations import json +from datetime import datetime, timezone +from io import BytesIO from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import httpx import pandas as pd +import pytest from nemo_platform import NeMoPlatform from nemo_platform_plugin.discovery import discover, discover_entry_points +from nemo_safe_synthesizer_plugin.sdk.job import SafeSynthesizerJob from nemo_safe_synthesizer_plugin.sdk.job_builder import SafeSynthesizerJobBuilder -from nemo_safe_synthesizer_plugin.sdk.resources import SafeSynthesizerResource +from nemo_safe_synthesizer_plugin.sdk.resources import AsyncSafeSynthesizerJobsResource, SafeSynthesizerResource def _mock_platform(requests: list[httpx.Request]) -> NeMoPlatform: @@ -79,6 +83,18 @@ def handler(_request: httpx.Request) -> httpx.Response: raise AssertionError("Expected HTTPStatusError") +@pytest.mark.asyncio +async def test_async_safe_synthesizer_resource_get_logs_awaits_platform_jobs() -> None: + platform = MagicMock() + platform.jobs.get_logs = AsyncMock(return_value=SimpleNamespace(data=[])) + resource = AsyncSafeSynthesizerJobsResource(platform) + + response = await resource.get_logs("safe-synth-job", workspace="default", limit=10) + + assert response.data == [] + platform.jobs.get_logs.assert_awaited_once_with("safe-synth-job", workspace="default", limit=10) + + def test_job_builder_uploads_dataframe_and_submits_spec() -> None: client = MagicMock() client.files.upload = MagicMock() @@ -129,3 +145,85 @@ def test_job_builder_submits_pretrained_model_job_for_adapter_reuse() -> None: assert create_kwargs["spec"]["pretrained_model_job"] == "first-synth-job" assert create_kwargs["spec"]["config"]["generation"] == {"num_records": 25} assert "pretrained_model" not in create_kwargs["spec"]["config"]["training"] + + +@pytest.mark.parametrize("status", ["error", "cancelled"]) +def test_safe_synthesizer_job_wait_for_completion_raises_on_terminal_failure(status: str) -> None: + client = MagicMock() + client.jobs.get_status.return_value = SimpleNamespace( + status=status, + status_details={"reason": "failed"}, + error_details={"message": "boom"}, + ) + job = SafeSynthesizerJob("safe-synth-job", client, workspace="default") + + with pytest.raises(RuntimeError, match=f"ended with status '{status}'"): + job.wait_for_completion(poll_interval=0, verbose=False) + + client.jobs.get_status.assert_called_once_with("safe-synth-job", workspace="default") + + +def test_safe_synthesizer_job_fetch_data_reads_synthetic_csv() -> None: + client = MagicMock() + client.jobs.results.download.return_value = BytesIO(b"name,value\nalice,1\nbob,2\n") + job = SafeSynthesizerJob("safe-synth-job", client, workspace="default") + + result = job.fetch_data() + + client.jobs.results.download.assert_called_once_with("synthetic-data", job="safe-synth-job", workspace="default") + assert list(result.columns) == ["name", "value"] + assert result["value"].tolist() == [1, 2] + + +def test_safe_synthesizer_job_fetch_summary_parses_json() -> None: + client = MagicMock() + client.jobs.results.download.return_value = BytesIO( + json.dumps( + { + "synthetic_data_quality_score": 8.5, + "data_privacy_score": 9.0, + "num_valid_records": 10, + "num_prompts": 10, + "timing": {"total_time_sec": 12.5}, + } + ).encode() + ) + job = SafeSynthesizerJob("safe-synth-job", client, workspace="default") + + summary = job.fetch_summary() + + client.jobs.results.download.assert_called_once_with("summary", job="safe-synth-job", workspace="default") + assert summary.synthetic_data_quality_score == 8.5 + assert summary.data_privacy_score == 9.0 + assert summary.timing.total_time_sec == 12.5 + + +def test_safe_synthesizer_job_fetch_logs_follows_pagination() -> None: + client = MagicMock() + log_client = MagicMock() + client.with_options.return_value = log_client + first_log = SimpleNamespace( + job="safe-synth-job", + job_step="safe-synthesizer", + job_task="task-1", + message="first", + timestamp=datetime(2026, 1, 1, tzinfo=timezone.utc), + ) + second_log = SimpleNamespace( + job="safe-synth-job", + job_step="safe-synthesizer", + job_task="task-1", + message="second", + timestamp=datetime(2026, 1, 1, 0, 0, 1, tzinfo=timezone.utc), + ) + log_client.jobs.get_logs.side_effect = [ + SimpleNamespace(data=[first_log], next_page="cursor-2"), + SimpleNamespace(data=[second_log], next_page=None), + ] + job = SafeSynthesizerJob("safe-synth-job", client, workspace="default") + + logs = list(job.fetch_logs(timeout=5.0)) + + assert [log.message for log in logs] == ["first", "second"] + assert client.with_options.call_count == 2 + assert log_client.jobs.get_logs.call_args_list[1].kwargs["page_cursor"] == "cursor-2" From 24faf94147a48c984ee366a88397798ae4b8ea2f Mon Sep 17 00:00:00 2001 From: mkornfield Date: Fri, 5 Jun 2026 21:04:36 +0000 Subject: [PATCH 4/5] chore: lint fixes Signed-off-by: mkornfield --- packages/nemo_platform_ext/tests/cli/test_docs.py | 1 - .../src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/nemo_platform_ext/tests/cli/test_docs.py b/packages/nemo_platform_ext/tests/cli/test_docs.py index 5c6a9a5faa5..d0ddc40e485 100644 --- a/packages/nemo_platform_ext/tests/cli/test_docs.py +++ b/packages/nemo_platform_ext/tests/cli/test_docs.py @@ -145,7 +145,6 @@ def test_docs_list_filters_unrendered_topics(self): assert "customizer/about" not in topics assert "evaluator/metrics/job-management" not in topics assert "helm/index" not in topics - assert "safe-synthesizer/about/index" not in topics assert "CONTRIBUTING" not in topics assert "README" not in topics assert "template/EULA" not in topics diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py index 183fdab01cd..a32632c5cde 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/api/v2/jobs/endpoints.py @@ -34,7 +34,6 @@ from nemo_safe_synthesizer_plugin.config import config from nemo_safe_synthesizer_plugin.job_config import ( SafeSynthesizerJobConfig, - SafeSynthesizerParameters, parse_pretrained_model_job_ref, ) from nemo_safe_synthesizer_plugin.runtime import runtime_task_command From 19fa74d5a8145ac0d461f8aecbf413d77fe1450d Mon Sep 17 00:00:00 2001 From: mkornfield Date: Fri, 5 Jun 2026 21:46:03 +0000 Subject: [PATCH 5/5] fix: inference key setting Signed-off-by: mkornfield --- .../tasks/safe_synthesizer/__main__.py | 5 ++-- .../tests/unit/test_jobs.py | 5 ++-- .../tests/unit/test_local_run.py | 30 +++++++++++++++++++ .../nemo_platform_ext/cli/test_docs.py | 1 - 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py index 1cad99c4c6c..861d8cf9aba 100644 --- a/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py +++ b/plugins/nemo-safe-synthesizer/src/nemo_safe_synthesizer_plugin/tasks/safe_synthesizer/__main__.py @@ -264,7 +264,7 @@ def _resolve_pretrained_model( def _setup_classify_endpoint(): - """Set up the NIM_ENDPOINT_URL for column classification from platform env vars.""" + """Set up upstream Safe Synthesizer PII classification env vars from platform env vars.""" endpoint_path = os.environ.get("CLASSIFY_LLM_ENDPOINT_PATH") if endpoint_path: models_url = os.environ.get("NMP_MODELS_URL") @@ -275,7 +275,8 @@ def _setup_classify_endpoint(): ) return full_url = models_url.rstrip("/") + endpoint_path - os.environ["NIM_ENDPOINT_URL"] = full_url + os.environ["NSS_INFERENCE_ENDPOINT"] = full_url + os.environ.setdefault("NSS_INFERENCE_KEY", "not-needed") logger.info("Configured column classification endpoint: %s", full_url) diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py b/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py index e9e53f33620..f893a6ec037 100644 --- a/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py +++ b/plugins/nemo-safe-synthesizer/tests/unit/test_jobs.py @@ -7,8 +7,9 @@ from nemo_platform import NotFoundError, PermissionDeniedError from nemo_safe_synthesizer.config.replace_pii import ClassifyConfig, Globals, PiiReplacerConfig, StepDefinition from nemo_safe_synthesizer_plugin.api.v2.jobs import endpoints -from nemo_safe_synthesizer_plugin.api.v2.jobs.endpoints import SafeSynthesizerJobConfig as PluginJobConfig -from nemo_safe_synthesizer_plugin.api.v2.jobs.endpoints import SafeSynthesizerParameters, job_config_compiler +from nemo_safe_synthesizer_plugin.api.v2.jobs.endpoints import job_config_compiler +from nemo_safe_synthesizer_plugin.job_config import SafeSynthesizerJobConfig as PluginJobConfig +from nemo_safe_synthesizer_plugin.job_config import SafeSynthesizerParameters from nemo_safe_synthesizer_plugin.runtime import TASK_MODULE from nmp.common.jobs.exceptions import PlatformJobCompilationError diff --git a/plugins/nemo-safe-synthesizer/tests/unit/test_local_run.py b/plugins/nemo-safe-synthesizer/tests/unit/test_local_run.py index a70fcc3c243..d5ff3b9db15 100644 --- a/plugins/nemo-safe-synthesizer/tests/unit/test_local_run.py +++ b/plugins/nemo-safe-synthesizer/tests/unit/test_local_run.py @@ -71,6 +71,36 @@ def test_run_from_env_reports_missing_config_path(monkeypatch): task_main.run_from_env() +def test_setup_classify_endpoint_sets_upstream_safe_synthesizer_env(monkeypatch): + task_main = import_task_main_without_heavy_runtime(monkeypatch) + monkeypatch.setenv( + "CLASSIFY_LLM_ENDPOINT_PATH", "/apis/inference-gateway/v2/workspaces/default/provider/my-nim/-/v1" + ) + monkeypatch.setenv("NMP_MODELS_URL", "http://models.test") + monkeypatch.delenv("NSS_INFERENCE_ENDPOINT", raising=False) + monkeypatch.delenv("NSS_INFERENCE_KEY", raising=False) + + task_main._setup_classify_endpoint() + + assert ( + task_main.os.environ["NSS_INFERENCE_ENDPOINT"] + == "http://models.test/apis/inference-gateway/v2/workspaces/default/provider/my-nim/-/v1" + ) + assert task_main.os.environ["NSS_INFERENCE_KEY"] == "not-needed" + + +def test_setup_classify_endpoint_preserves_existing_inference_key(monkeypatch): + task_main = import_task_main_without_heavy_runtime(monkeypatch) + monkeypatch.setenv("CLASSIFY_LLM_ENDPOINT_PATH", "/route") + monkeypatch.setenv("NMP_MODELS_URL", "http://models.test/") + monkeypatch.setenv("NSS_INFERENCE_KEY", "real-key") + + task_main._setup_classify_endpoint() + + assert task_main.os.environ["NSS_INFERENCE_ENDPOINT"] == "http://models.test/route" + assert task_main.os.environ["NSS_INFERENCE_KEY"] == "real-key" + + def test_run_local_resolves_pretrained_model_job_before_run(tmp_path, monkeypatch): task_main = import_task_main_without_heavy_runtime(monkeypatch) spec_file = tmp_path / "spec.json" diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_docs.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_docs.py index 07522c802ac..b71dbb3bc9d 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_docs.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_docs.py @@ -145,7 +145,6 @@ def test_docs_list_filters_unrendered_topics(self): assert "customizer/about" not in topics assert "evaluator/metrics/job-management" not in topics assert "helm/index" not in topics - assert "safe-synthesizer/about/index" not in topics assert "CONTRIBUTING" not in topics assert "README" not in topics assert "template/EULA" not in topics