From a35629eaa90fcb5c06c8bfb1d47efbb573a1157c Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Fri, 22 May 2026 10:36:51 -0400 Subject: [PATCH 01/26] docs(fern): scaffold fern infrastructure under docs/fern/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First step of the mkdocs → fern migration. Adds the fern config, components, styles, and scripts at docs/fern/ — no pages or content yet, those land in subsequent commits. Layout: - docs/fern/fern.config.json, package.json, docs.yml — fern build config - docs/fern/components/ — custom React components (Button, Icon, NotebookViewer, BadgeLinks, CustomFooter, MetricsTable, TrajectoryViewer, Authors) - docs/fern/styles/ — component CSS (notebook-viewer, button) - docs/fern/assets/ — shared SVGs/images (NVIDIA marks, agent eval framework diagram, etc.) - docs/fern/main.css — NVIDIA theme overrides - docs/fern/scripts/ — ipynb-to-fern-json converter, validate-mdx parse gate, substitutions.json - docs/fern/versions/latest.yml — nav scaffold (paths will be rewritten to ../../.mdx in a later commit once pages exist at docs/
/.mdx) Signed-off-by: Lawrence Lane --- docs/fern/README.md | 108 + docs/fern/assets/NVIDIA_dark.svg | 37 + docs/fern/assets/NVIDIA_light.svg | 36 + docs/fern/assets/NVIDIA_symbol.svg | 24 + docs/fern/assets/favicon.ico | 3 + .../assets/images/agent_eval_framework.png | 3 + .../images/attribute-inference-protection.png | 3 + docs/fern/assets/images/car-audio-theft.jpg | 3 + .../images/column-correlation-stability.png | 3 + .../images/column-distribution-stability.png | 3 + docs/fern/assets/images/customizations.png | 3 + docs/fern/assets/images/dashboard.png | 3 + docs/fern/assets/images/datasets.png | 3 + .../images/deep-structure-stability.png | 3 + docs/fern/assets/images/dps-subscores.png | 3 + docs/fern/assets/images/dps-usability.png | 3 + .../assets/images/evaluation_result_dir.png | 3 + docs/fern/assets/images/evaluations.png | 3 + .../assets/images/evaluator_interactions.png | 3 + docs/fern/assets/images/favicon.ico | 3 + docs/fern/assets/images/gpu_memory.png | 3 + docs/fern/assets/images/gpu_utilization.png | 3 + docs/fern/assets/images/jobs.png | 3 + .../assets/images/main_nv_logo_square.png | 3 + .../membership-inference-protection.png | 3 + .../assets/images/nemo-eval-diagram-2510.png | 3 + .../images/nemo-platform-architecture.svg | 3 + docs/fern/assets/images/nvidia-logo-white.png | 3 + docs/fern/assets/images/overall-sqs-dps.png | 3 + .../images/packed_vs_not_packed_tables.png | 3 + .../images/packed_vs_not_packed_val_loss.png | 3 + docs/fern/assets/images/pca.png | 3 + docs/fern/assets/images/pii-replay.png | 3 + docs/fern/assets/images/pod_logs.png | 3 + .../assets/images/prompt-tuned-models.png | 3 + docs/fern/assets/images/run-evaluation.png | 3 + docs/fern/assets/images/runtime.png | 3 + docs/fern/assets/images/safe-synthesizer.png | 3 + docs/fern/assets/images/secrets.png | 3 + docs/fern/assets/images/seq_packing_stats.png | 3 + docs/fern/assets/images/sqs-subscores.png | 3 + docs/fern/assets/images/sqs-usability.png | 3 + docs/fern/assets/images/street-scene.jpg | 3 + .../images/text-semantic-similarity.png | 3 + .../images/text-structure-similarity.png | 3 + .../assets/images/wandb_charts_example.png | 3 + docs/fern/assets/images/workflow_logs.png | 3 + docs/fern/assets/images/workspaces.png | 3 + docs/fern/assets/nvidia-logo-white.png | 3 + docs/fern/components/Authors.tsx | 56 + docs/fern/components/BadgeLinks.tsx | 37 + docs/fern/components/Button.tsx | 47 + docs/fern/components/CustomFooter.tsx | 96 + docs/fern/components/Icon.tsx | 88 + docs/fern/components/MetricsTable.tsx | 106 + docs/fern/components/NotebookViewer.tsx | 421 ++++ docs/fern/components/TrajectoryViewer.tsx | 144 ++ docs/fern/docs.yml | 59 + docs/fern/fern.config.json | 4 + docs/fern/main.css | 872 ++++++++ docs/fern/package-lock.json | 1815 +++++++++++++++++ docs/fern/package.json | 14 + docs/fern/scripts/README.md | 38 + docs/fern/scripts/ipynb-to-fern-json.py | 340 +++ docs/fern/scripts/requirements.txt | 3 + docs/fern/scripts/substitutions.json | 31 + docs/fern/scripts/validate-mdx.mjs | 71 + docs/fern/styles/button.css | 44 + docs/fern/styles/notebook-viewer.css | 187 ++ 69 files changed, 4813 insertions(+) create mode 100644 docs/fern/README.md create mode 100644 docs/fern/assets/NVIDIA_dark.svg create mode 100644 docs/fern/assets/NVIDIA_light.svg create mode 100644 docs/fern/assets/NVIDIA_symbol.svg create mode 100644 docs/fern/assets/favicon.ico create mode 100644 docs/fern/assets/images/agent_eval_framework.png create mode 100644 docs/fern/assets/images/attribute-inference-protection.png create mode 100644 docs/fern/assets/images/car-audio-theft.jpg create mode 100644 docs/fern/assets/images/column-correlation-stability.png create mode 100644 docs/fern/assets/images/column-distribution-stability.png create mode 100644 docs/fern/assets/images/customizations.png create mode 100644 docs/fern/assets/images/dashboard.png create mode 100644 docs/fern/assets/images/datasets.png create mode 100644 docs/fern/assets/images/deep-structure-stability.png create mode 100644 docs/fern/assets/images/dps-subscores.png create mode 100644 docs/fern/assets/images/dps-usability.png create mode 100644 docs/fern/assets/images/evaluation_result_dir.png create mode 100644 docs/fern/assets/images/evaluations.png create mode 100644 docs/fern/assets/images/evaluator_interactions.png create mode 100644 docs/fern/assets/images/favicon.ico create mode 100644 docs/fern/assets/images/gpu_memory.png create mode 100644 docs/fern/assets/images/gpu_utilization.png create mode 100644 docs/fern/assets/images/jobs.png create mode 100644 docs/fern/assets/images/main_nv_logo_square.png create mode 100644 docs/fern/assets/images/membership-inference-protection.png create mode 100644 docs/fern/assets/images/nemo-eval-diagram-2510.png create mode 100644 docs/fern/assets/images/nemo-platform-architecture.svg create mode 100644 docs/fern/assets/images/nvidia-logo-white.png create mode 100644 docs/fern/assets/images/overall-sqs-dps.png create mode 100644 docs/fern/assets/images/packed_vs_not_packed_tables.png create mode 100644 docs/fern/assets/images/packed_vs_not_packed_val_loss.png create mode 100644 docs/fern/assets/images/pca.png create mode 100644 docs/fern/assets/images/pii-replay.png create mode 100644 docs/fern/assets/images/pod_logs.png create mode 100644 docs/fern/assets/images/prompt-tuned-models.png create mode 100644 docs/fern/assets/images/run-evaluation.png create mode 100644 docs/fern/assets/images/runtime.png create mode 100644 docs/fern/assets/images/safe-synthesizer.png create mode 100644 docs/fern/assets/images/secrets.png create mode 100644 docs/fern/assets/images/seq_packing_stats.png create mode 100644 docs/fern/assets/images/sqs-subscores.png create mode 100644 docs/fern/assets/images/sqs-usability.png create mode 100644 docs/fern/assets/images/street-scene.jpg create mode 100644 docs/fern/assets/images/text-semantic-similarity.png create mode 100644 docs/fern/assets/images/text-structure-similarity.png create mode 100644 docs/fern/assets/images/wandb_charts_example.png create mode 100644 docs/fern/assets/images/workflow_logs.png create mode 100644 docs/fern/assets/images/workspaces.png create mode 100644 docs/fern/assets/nvidia-logo-white.png create mode 100644 docs/fern/components/Authors.tsx create mode 100644 docs/fern/components/BadgeLinks.tsx create mode 100644 docs/fern/components/Button.tsx create mode 100644 docs/fern/components/CustomFooter.tsx create mode 100644 docs/fern/components/Icon.tsx create mode 100644 docs/fern/components/MetricsTable.tsx create mode 100644 docs/fern/components/NotebookViewer.tsx create mode 100644 docs/fern/components/TrajectoryViewer.tsx create mode 100644 docs/fern/docs.yml create mode 100644 docs/fern/fern.config.json create mode 100644 docs/fern/main.css create mode 100644 docs/fern/package-lock.json create mode 100644 docs/fern/package.json create mode 100644 docs/fern/scripts/README.md create mode 100644 docs/fern/scripts/ipynb-to-fern-json.py create mode 100644 docs/fern/scripts/requirements.txt create mode 100644 docs/fern/scripts/substitutions.json create mode 100644 docs/fern/scripts/validate-mdx.mjs create mode 100644 docs/fern/styles/button.css create mode 100644 docs/fern/styles/notebook-viewer.css diff --git a/docs/fern/README.md b/docs/fern/README.md new file mode 100644 index 00000000000..1128d89c23e --- /dev/null +++ b/docs/fern/README.md @@ -0,0 +1,108 @@ +# NeMo Platform Fern Docs + +This directory holds the Fern MDX source for the NeMo Platform documentation site. + +All new published docs edits should land under `fern/`. Keep the legacy MkDocs tree under `../docs/` as migration source material and historical reference. + +## Quick Links + +| What | Where | +| --- | --- | +| Fern dashboard | https://dashboard.buildwithfern.com (NVIDIA org) | +| Agent skill | [`../.claude/skills/nemo-platform-docs/SKILL.md`](../.claude/skills/nemo-platform-docs/SKILL.md) | +| CI workflows | [`../.github/workflows/fern-docs-*.yml`](../.github/workflows/) | +| Publish workflow | [`../.github/workflows/publish-fern-docs.yml`](../.github/workflows/publish-fern-docs.yml) | + +## Quickstart + +First time on this machine: + +```bash +npx -y fern-api@latest login +``` + +Then run from `fern/`: + +```bash +npm run check # Validate docs.yml, navigation, and MDX +npm run dev # Start local preview at http://localhost:3000 +``` + +`package.json` intentionally shells out to `npx -y fern-api@latest`, so there is no install step for local docs work. + +## Layout + +```text +fern/ +├── fern.config.json # Fern organization + CLI version +├── package.json # npm run check|dev|generate|preview +├── docs.yml # Site config, theme, assets, redirects, versions +├── main.css # NVIDIA theme overrides +├── assets/ # Logos, shared SVGs, and page images +├── components/ # Custom TSX components +└── versions/ + ├── latest.yml # Navigation tree + └── latest/pages/ # MDX content and page-local assets +``` + +The site uses a single `Latest` version. `versions/latest.yml` defines the sidebar and maps each page file to its canonical route. + +## Authoring + +Add pages under `versions/latest/pages/` and wire them into `versions/latest.yml`. + +Use front matter for the rendered page title: + +```yaml +--- +title: "Page Title" +description: "" +--- +``` + +Do not add a duplicate first `# Page Title` heading when it matches the front matter `title`; Fern renders that title automatically. + +Use Fern-native MDX components such as ``, ``, ``, ``, ``, and ``. Do not reintroduce MkDocs Material syntax like `!!! note`, `=== "Tab"`, `--8<--`, or `
`. + +## Internal Links + +Use Fern's nav-derived canonical URLs: + +```mdx +[Workspaces](/documentation/get-started/core-concepts/workspaces) +``` + +Avoid source-path links such as `/get-started/concepts/workspaces`, `/latest/get-started/concepts/workspaces`, and relative `.md` links. If a public URL changes, add a redirect in `docs.yml`. + +## API Reference + +The API reference page lives at `versions/latest/pages/api/index.mdx` and renders `versions/latest/pages/api/openapi.yaml` with: + +```mdx + +``` + +Regenerate or replace `openapi.yaml` when the Platform API changes, then run `npm run check`. + +## CI and Publishing + +| Workflow | Trigger | Purpose | +| --- | --- | --- | +| `fern-docs-ci.yml` | trusted `pull-request/` mirror branch | Run `npm run check` with `DOCS_FERN_TOKEN` available | +| `fern-docs-preview-build.yml` | `pull_request` touching `fern/**` | Upload PR `fern/` sources with no secrets | +| `fern-docs-preview-comment.yml` | successful preview build workflow run | Generate a Fern preview with `DOCS_FERN_TOKEN` and post/update the PR comment | +| `publish-fern-docs.yml` | `main`, `docs/v*` tag, or manual dispatch | Publish the Fern docs site | + +Required secret: `DOCS_FERN_TOKEN`, generated with `fern token` from an account that can publish to the NVIDIA Fern organization. + +PRs that touch `fern/**` get a shared preview URL posted as a comment after the two-part preview workflow finishes. + +## Troubleshooting + +| Symptom | Fix | +| --- | --- | +| `HTTP 403` or organization access error | Sign in at https://dashboard.buildwithfern.com, then run `npx -y fern-api@latest login` again | +| `fern check` YAML error | Use 2-space indentation; make sure `path:` values are relative to `versions/latest.yml` | +| Page 404 in preview | Check that `versions/latest.yml` lists the page and links by canonical route | +| Broken internal link | Rewrite it to the nav-derived `/documentation/...` URL | +| JSX or MDX parse error | Escape raw `{}`, `<`, or `>` in prose, and use Fern components instead of raw MkDocs syntax | diff --git a/docs/fern/assets/NVIDIA_dark.svg b/docs/fern/assets/NVIDIA_dark.svg new file mode 100644 index 00000000000..fe67b898bfd --- /dev/null +++ b/docs/fern/assets/NVIDIA_dark.svg @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/fern/assets/NVIDIA_light.svg b/docs/fern/assets/NVIDIA_light.svg new file mode 100644 index 00000000000..568ee177ba0 --- /dev/null +++ b/docs/fern/assets/NVIDIA_light.svg @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/fern/assets/NVIDIA_symbol.svg b/docs/fern/assets/NVIDIA_symbol.svg new file mode 100644 index 00000000000..fd57037f326 --- /dev/null +++ b/docs/fern/assets/NVIDIA_symbol.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + diff --git a/docs/fern/assets/favicon.ico b/docs/fern/assets/favicon.ico new file mode 100644 index 00000000000..cea91cdbfb4 --- /dev/null +++ b/docs/fern/assets/favicon.ico @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bedcb14e0470ff8abe26405a630e9604ac4fa581b385a7cfb00420c22f541b6e +size 25214 diff --git a/docs/fern/assets/images/agent_eval_framework.png b/docs/fern/assets/images/agent_eval_framework.png new file mode 100644 index 00000000000..044bfebc87e --- /dev/null +++ b/docs/fern/assets/images/agent_eval_framework.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4683db588a0d8a00e462a5f8c452184c941d7d5c6a42361149c50fa1432c8984 +size 100009 diff --git a/docs/fern/assets/images/attribute-inference-protection.png b/docs/fern/assets/images/attribute-inference-protection.png new file mode 100644 index 00000000000..51835ae4b36 --- /dev/null +++ b/docs/fern/assets/images/attribute-inference-protection.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a4cdefe61df6630387b2c5fa92bfdc275a14d97130f007e819355d90ce34bca +size 79411 diff --git a/docs/fern/assets/images/car-audio-theft.jpg b/docs/fern/assets/images/car-audio-theft.jpg new file mode 100644 index 00000000000..3995f5be167 --- /dev/null +++ b/docs/fern/assets/images/car-audio-theft.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ed605f0ed8e5e2c3f35cdf21894646f82f6df407aae1c527110a913df3253b7 +size 29606 diff --git a/docs/fern/assets/images/column-correlation-stability.png b/docs/fern/assets/images/column-correlation-stability.png new file mode 100644 index 00000000000..9187b2897bc --- /dev/null +++ b/docs/fern/assets/images/column-correlation-stability.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bf8796bbcbe7c03d1acde5494d3110c35d5299c62f08056d0755853ebd96178 +size 270914 diff --git a/docs/fern/assets/images/column-distribution-stability.png b/docs/fern/assets/images/column-distribution-stability.png new file mode 100644 index 00000000000..1fd663e2195 --- /dev/null +++ b/docs/fern/assets/images/column-distribution-stability.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e340503e29c05d47e1e7156a2efe2d6c403f4b1e12619938653d477a9ba50ed7 +size 77566 diff --git a/docs/fern/assets/images/customizations.png b/docs/fern/assets/images/customizations.png new file mode 100644 index 00000000000..bb5dc5dc81d --- /dev/null +++ b/docs/fern/assets/images/customizations.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25dc8dc4d79105232e46d4ee4af42a1e0aa1a6a34fce63a1dec67a29adc91ec5 +size 66483 diff --git a/docs/fern/assets/images/dashboard.png b/docs/fern/assets/images/dashboard.png new file mode 100644 index 00000000000..1ede1b98db7 --- /dev/null +++ b/docs/fern/assets/images/dashboard.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afafac13cda48a9a0c91e3d0164c7dcc7a16ef12cf80edbdcb8047915c72ea5f +size 131900 diff --git a/docs/fern/assets/images/datasets.png b/docs/fern/assets/images/datasets.png new file mode 100644 index 00000000000..e62cfae2e58 --- /dev/null +++ b/docs/fern/assets/images/datasets.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb226eb3adb2d1f6896b7c7f6a1e52b527deac3ab16ee0412cd3616b1d26e1e3 +size 60171 diff --git a/docs/fern/assets/images/deep-structure-stability.png b/docs/fern/assets/images/deep-structure-stability.png new file mode 100644 index 00000000000..e62a10bf922 --- /dev/null +++ b/docs/fern/assets/images/deep-structure-stability.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96249fa81a0a9a40f552dac2ce6a8fe6b7f8d863ffb443ceb2b9d0ba53875158 +size 121483 diff --git a/docs/fern/assets/images/dps-subscores.png b/docs/fern/assets/images/dps-subscores.png new file mode 100644 index 00000000000..ca899410c4a --- /dev/null +++ b/docs/fern/assets/images/dps-subscores.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27c407368d08b5aa1efe9939e7f2c86eccea140e7469757dfcfd6cd979787f67 +size 148059 diff --git a/docs/fern/assets/images/dps-usability.png b/docs/fern/assets/images/dps-usability.png new file mode 100644 index 00000000000..e89f6bc6b0c --- /dev/null +++ b/docs/fern/assets/images/dps-usability.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07a1d60023e4f8801e71b1f2c5381f9808b5592a4fc44230fa7aced976cc9c69 +size 51168 diff --git a/docs/fern/assets/images/evaluation_result_dir.png b/docs/fern/assets/images/evaluation_result_dir.png new file mode 100644 index 00000000000..a616d08f3c8 --- /dev/null +++ b/docs/fern/assets/images/evaluation_result_dir.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa6a64998f2f2a4a5af65683c646d9226680e4e9187e41900536c10e4999f275 +size 74039 diff --git a/docs/fern/assets/images/evaluations.png b/docs/fern/assets/images/evaluations.png new file mode 100644 index 00000000000..755c2b9d719 --- /dev/null +++ b/docs/fern/assets/images/evaluations.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbd197dd7dd5304094e379e1c4af64935cc9a76b586c24da64121b77cea655dc +size 57931 diff --git a/docs/fern/assets/images/evaluator_interactions.png b/docs/fern/assets/images/evaluator_interactions.png new file mode 100644 index 00000000000..79c10a69810 --- /dev/null +++ b/docs/fern/assets/images/evaluator_interactions.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea680b65c386858426ed0b05c83d500d4cf224acba7b3115c387450a616f67e5 +size 94633 diff --git a/docs/fern/assets/images/favicon.ico b/docs/fern/assets/images/favicon.ico new file mode 100644 index 00000000000..cea91cdbfb4 --- /dev/null +++ b/docs/fern/assets/images/favicon.ico @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bedcb14e0470ff8abe26405a630e9604ac4fa581b385a7cfb00420c22f541b6e +size 25214 diff --git a/docs/fern/assets/images/gpu_memory.png b/docs/fern/assets/images/gpu_memory.png new file mode 100644 index 00000000000..0c5c31d9c11 --- /dev/null +++ b/docs/fern/assets/images/gpu_memory.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dbfd9fb92f561fe1439cde258cec7ec1e776862ca94bf6d7a77ece92d981c5d +size 103607 diff --git a/docs/fern/assets/images/gpu_utilization.png b/docs/fern/assets/images/gpu_utilization.png new file mode 100644 index 00000000000..fab38d60671 --- /dev/null +++ b/docs/fern/assets/images/gpu_utilization.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58036162ad51e43d8dbee45a70459ec37a10ab06614a4088417de9db6a9c8f64 +size 186900 diff --git a/docs/fern/assets/images/jobs.png b/docs/fern/assets/images/jobs.png new file mode 100644 index 00000000000..b5a6efff3f3 --- /dev/null +++ b/docs/fern/assets/images/jobs.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67e04e0f6f8ee07d3b22126610a071db0b966d4fd09fc3c740fee294b88ec4f4 +size 51380 diff --git a/docs/fern/assets/images/main_nv_logo_square.png b/docs/fern/assets/images/main_nv_logo_square.png new file mode 100644 index 00000000000..1dfb208cb08 --- /dev/null +++ b/docs/fern/assets/images/main_nv_logo_square.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99cd3d358ff5cc186ad406caebc90b00547fbe6dd0049890df28df4285f49f15 +size 11074 diff --git a/docs/fern/assets/images/membership-inference-protection.png b/docs/fern/assets/images/membership-inference-protection.png new file mode 100644 index 00000000000..03c108e0bd6 --- /dev/null +++ b/docs/fern/assets/images/membership-inference-protection.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e202fff24fec9baab76a980dcb3d98793a0573271ccda0c83172258e8c19e7ef +size 388868 diff --git a/docs/fern/assets/images/nemo-eval-diagram-2510.png b/docs/fern/assets/images/nemo-eval-diagram-2510.png new file mode 100644 index 00000000000..45bb6cb09be --- /dev/null +++ b/docs/fern/assets/images/nemo-eval-diagram-2510.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70d36b0e73afc7a63de97beb3795d3b7d39373386bf12b83a56ebb254fd4230c +size 286432 diff --git a/docs/fern/assets/images/nemo-platform-architecture.svg b/docs/fern/assets/images/nemo-platform-architecture.svg new file mode 100644 index 00000000000..3f54d60505d --- /dev/null +++ b/docs/fern/assets/images/nemo-platform-architecture.svg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c6ba483df553bf9dd0bf750359c27de7d34478054023894635084ae0e17a9d8 +size 132304 diff --git a/docs/fern/assets/images/nvidia-logo-white.png b/docs/fern/assets/images/nvidia-logo-white.png new file mode 100644 index 00000000000..f6c73aed43f --- /dev/null +++ b/docs/fern/assets/images/nvidia-logo-white.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00b499146c0d590c254ff8ab555298cfa88605938bf883a880a58067eccdb571 +size 47302 diff --git a/docs/fern/assets/images/overall-sqs-dps.png b/docs/fern/assets/images/overall-sqs-dps.png new file mode 100644 index 00000000000..82639ab173a --- /dev/null +++ b/docs/fern/assets/images/overall-sqs-dps.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:240532d46d8ed57992bb87dd160c7f471bcb0f365411c23cec61ab930a02799b +size 49850 diff --git a/docs/fern/assets/images/packed_vs_not_packed_tables.png b/docs/fern/assets/images/packed_vs_not_packed_tables.png new file mode 100644 index 00000000000..4d85735cdf2 --- /dev/null +++ b/docs/fern/assets/images/packed_vs_not_packed_tables.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98ec568543654899b3de6f20e90306f369aa083f5d41e5e92b04bf1dedcb863d +size 51068 diff --git a/docs/fern/assets/images/packed_vs_not_packed_val_loss.png b/docs/fern/assets/images/packed_vs_not_packed_val_loss.png new file mode 100644 index 00000000000..64716a20f8b --- /dev/null +++ b/docs/fern/assets/images/packed_vs_not_packed_val_loss.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46aa81bd023f5ac8a05b3afb14e42c178efeae20ebfce990699324767bc51b72 +size 59932 diff --git a/docs/fern/assets/images/pca.png b/docs/fern/assets/images/pca.png new file mode 100644 index 00000000000..02848475983 --- /dev/null +++ b/docs/fern/assets/images/pca.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf0da92c1242110b4c89d01eb16c3fa6d0fdcfda2dea1482093ee0c191653d20 +size 244797 diff --git a/docs/fern/assets/images/pii-replay.png b/docs/fern/assets/images/pii-replay.png new file mode 100644 index 00000000000..ed585a3a64e --- /dev/null +++ b/docs/fern/assets/images/pii-replay.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1730ad81b8527927269948253b8a1a594c65ff37ae3af5e5b31586f759fc492 +size 502381 diff --git a/docs/fern/assets/images/pod_logs.png b/docs/fern/assets/images/pod_logs.png new file mode 100644 index 00000000000..d9bd08b145a --- /dev/null +++ b/docs/fern/assets/images/pod_logs.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99c4fd813f8417e0f10f8470944648f3daeadd679e46509e7f361a976363785e +size 85826 diff --git a/docs/fern/assets/images/prompt-tuned-models.png b/docs/fern/assets/images/prompt-tuned-models.png new file mode 100644 index 00000000000..5d320cf844c --- /dev/null +++ b/docs/fern/assets/images/prompt-tuned-models.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7478e25256510953156ab50827caaa751ab80732ae33b010c904f780fc33ad80 +size 93980 diff --git a/docs/fern/assets/images/run-evaluation.png b/docs/fern/assets/images/run-evaluation.png new file mode 100644 index 00000000000..b094b83a8c4 --- /dev/null +++ b/docs/fern/assets/images/run-evaluation.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02b2248cba2109fcbfb0301fbc8b09c6308ebad94f649111bf02181d93ba334f +size 75450 diff --git a/docs/fern/assets/images/runtime.png b/docs/fern/assets/images/runtime.png new file mode 100644 index 00000000000..08e93044b95 --- /dev/null +++ b/docs/fern/assets/images/runtime.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55e3ed1c7abc734af7f1ae800586c70f0fc8a95bba94153c95a1110c348bd351 +size 40022 diff --git a/docs/fern/assets/images/safe-synthesizer.png b/docs/fern/assets/images/safe-synthesizer.png new file mode 100644 index 00000000000..6c5eef9aec1 --- /dev/null +++ b/docs/fern/assets/images/safe-synthesizer.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f90ab8fa51a8d3281697f6437d7470c413ad0979f7ed67ee29b6be89bbd315a +size 92404 diff --git a/docs/fern/assets/images/secrets.png b/docs/fern/assets/images/secrets.png new file mode 100644 index 00000000000..0c9d1549c0f --- /dev/null +++ b/docs/fern/assets/images/secrets.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dc00607d2227df31a53f8a249fdf931e8f40f41564afda8db7e70c45f104c5f +size 56868 diff --git a/docs/fern/assets/images/seq_packing_stats.png b/docs/fern/assets/images/seq_packing_stats.png new file mode 100644 index 00000000000..f58d188ce1d --- /dev/null +++ b/docs/fern/assets/images/seq_packing_stats.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:928c2eb1aeaab3c28e81716f600ef250eeb5b781548e7d3ad879c1cc9b58ccd1 +size 34452 diff --git a/docs/fern/assets/images/sqs-subscores.png b/docs/fern/assets/images/sqs-subscores.png new file mode 100644 index 00000000000..5dce6af893c --- /dev/null +++ b/docs/fern/assets/images/sqs-subscores.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac180e353c03edeee07a829935bc3ce93cd39a9cd4c16da86ed1babfba282f85 +size 210286 diff --git a/docs/fern/assets/images/sqs-usability.png b/docs/fern/assets/images/sqs-usability.png new file mode 100644 index 00000000000..e597068b237 --- /dev/null +++ b/docs/fern/assets/images/sqs-usability.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ef0663067d0e5e1da919cf6108d32e26e80f815257310ba401723befde533c2 +size 56886 diff --git a/docs/fern/assets/images/street-scene.jpg b/docs/fern/assets/images/street-scene.jpg new file mode 100644 index 00000000000..fdfe8100824 --- /dev/null +++ b/docs/fern/assets/images/street-scene.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce84c961f2ade3aa431ad77a10b8d6fc24ffba6b79a6c4d63c2f0cb4371993c9 +size 29771 diff --git a/docs/fern/assets/images/text-semantic-similarity.png b/docs/fern/assets/images/text-semantic-similarity.png new file mode 100644 index 00000000000..d4b474b7a74 --- /dev/null +++ b/docs/fern/assets/images/text-semantic-similarity.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eb77b97d7aa0c2fcc5170dbbce8c65b56e229d964af21fad3abb1b2337bd276 +size 161834 diff --git a/docs/fern/assets/images/text-structure-similarity.png b/docs/fern/assets/images/text-structure-similarity.png new file mode 100644 index 00000000000..daddded6d81 --- /dev/null +++ b/docs/fern/assets/images/text-structure-similarity.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03c4d27552697ee19f56aa0f1b6f1bb93dedcd14356732021809941a7e648f35 +size 106838 diff --git a/docs/fern/assets/images/wandb_charts_example.png b/docs/fern/assets/images/wandb_charts_example.png new file mode 100644 index 00000000000..8337792d117 --- /dev/null +++ b/docs/fern/assets/images/wandb_charts_example.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3afb36a6b944926559e07713756a5b27bb8e350a4f4bd2dd23ac810aced0dfcc +size 161897 diff --git a/docs/fern/assets/images/workflow_logs.png b/docs/fern/assets/images/workflow_logs.png new file mode 100644 index 00000000000..8c1dacd1260 --- /dev/null +++ b/docs/fern/assets/images/workflow_logs.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4a563bc0eb19d27350bb55b8dda269fa054a8d058c790222a82f18616e01359 +size 67084 diff --git a/docs/fern/assets/images/workspaces.png b/docs/fern/assets/images/workspaces.png new file mode 100644 index 00000000000..ac3e56c3eaa --- /dev/null +++ b/docs/fern/assets/images/workspaces.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a60d05fefbbea4ea30545f6de9aba7a8fe772c2a74e21815a88243b499fe28df +size 110153 diff --git a/docs/fern/assets/nvidia-logo-white.png b/docs/fern/assets/nvidia-logo-white.png new file mode 100644 index 00000000000..f6c73aed43f --- /dev/null +++ b/docs/fern/assets/nvidia-logo-white.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00b499146c0d590c254ff8ab555298cfa88605938bf883a880a58067eccdb571 +size 47302 diff --git a/docs/fern/components/Authors.tsx b/docs/fern/components/Authors.tsx new file mode 100644 index 00000000000..0c71197953c --- /dev/null +++ b/docs/fern/components/Authors.tsx @@ -0,0 +1,56 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Authors - Renders author byline with avatars for dev notes / blog posts. + * + * Uses authors data from components/devnotes/authors-data.ts (synced with .authors.yml). + * NOTE: Fern's custom component pipeline uses the automatic JSX runtime. + * + * Usage in MDX (authors from frontmatter): + * --- + * authors: + * - jdoe + * - asmith + * --- + * + * import { Authors } from "@/components/Authors"; + * + */ + +import { authors } from "./devnotes/authors-data"; + +export interface AuthorsProps { + /** Author IDs from .authors.yml. From frontmatter: ids={authors} */ + ids?: string[]; +} + +export const Authors = ({ ids }: AuthorsProps) => { + const validAuthors = (ids ?? []) + .map((id) => authors[id]) + .filter(Boolean); + + if (validAuthors.length === 0) return null; + + return ( +
+ {validAuthors.map((author, i) => ( +
+ +
+ {author.name} + {author.description} +
+
+ ))} +
+ ); +}; diff --git a/docs/fern/components/BadgeLinks.tsx b/docs/fern/components/BadgeLinks.tsx new file mode 100644 index 00000000000..c4a5949c339 --- /dev/null +++ b/docs/fern/components/BadgeLinks.tsx @@ -0,0 +1,37 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Badge links for GitHub, License, PyPI, etc. + * Uses a custom wrapper to avoid Fern's external-link icon stacking under badges. + * + * `badges` is required — there is intentionally no default. A previous + * version shipped placeholder URLs that could land in production for + * sites that rendered the component without props. See README-BadgeLinks.md. + */ +export type BadgeItem = { + href: string; + src: string; + alt: string; +}; + +export interface BadgeLinksProps { + badges: BadgeItem[]; +} + +export function BadgeLinks({ badges }: BadgeLinksProps) { + return ( +
+ {badges.map((b) => ( + + {b.alt} + + ))} +
+ ); +} diff --git a/docs/fern/components/Button.tsx b/docs/fern/components/Button.tsx new file mode 100644 index 00000000000..5945e6df422 --- /dev/null +++ b/docs/fern/components/Button.tsx @@ -0,0 +1,47 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Button — a styled link rendered as a CTA button. + * + * Replaces MkDocs Material's `{ .md-button }` / `{ .md-button--primary }` + * attribute list. The converter emits this wrapper when it sees that + * pattern in the docs source. + * + * NOTE: Fern's custom component pipeline uses the automatic JSX runtime; + * we do not import React. + * + * Usage in MDX: + * import { Button } from "@/components/Button"; + * + * + * + * + * In MDX you can also pass a markdown link via children if you prefer: + * + */ + +import type { ReactNode } from "react"; + +export interface ButtonProps { + href: string; + variant?: "primary" | "secondary"; + children: ReactNode; +} + +export function Button({ href, variant = "secondary", children }: ButtonProps) { + const external = /^https?:\/\//.test(href); + return ( + + {children} + + ); +} + +export default Button; diff --git a/docs/fern/components/CustomFooter.tsx b/docs/fern/components/CustomFooter.tsx new file mode 100644 index 00000000000..db2d9da6b0b --- /dev/null +++ b/docs/fern/components/CustomFooter.tsx @@ -0,0 +1,96 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + */ + +/** + * Custom footer for NVIDIA docs (Fern native header/footer). + * Markup and class names match the original custom-app footer 1:1 so that + * fern/main.css (footer + Built with Fern styles) applies correctly: + * dark mode logo, responsive layout, and Built with Fern tooltip. + */ +export default function CustomFooter() { + const currentYear = new Date().getFullYear(); + const logoUrl = + "https://fern-image-hosting.s3.us-east-1.amazonaws.com/nvidia/NVIDIA_Logo_0.svg"; + + return ( + + ); +} diff --git a/docs/fern/components/Icon.tsx b/docs/fern/components/Icon.tsx new file mode 100644 index 00000000000..154fcaf8cba --- /dev/null +++ b/docs/fern/components/Icon.tsx @@ -0,0 +1,88 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Icon — render a small inline icon, mapping MkDocs Material icon names to + * inline SVGs. + * + * The MkDocs source uses `:icon: code-square` (admonition option) and + * `:material-foo:` (inline). The converter strips the admonition option + * (Accordion / Note don't take an icon prop yet), so this component is + * here for hand-authored MDX pages that want to render an icon next to + * text. Add new names by extending the ICONS map. + * + * Usage: + * import { Icon } from "@/components/Icon"; + * + * + * + */ + +import type { ReactNode } from "react"; + +// Minimal Lucide-style stroke icons. Lucide is MIT and we trace just the +// names actually used in our docs to avoid a runtime icon dependency. +const ICONS: Record = { + "code-square": ( + <> + + + + + ), + terminal: ( + <> + + + + ), + gear: ( + <> + + + + ), + info: ( + <> + + + + + ), + shield: ( + + ), +}; + +export interface IconProps { + name: string; + size?: number; +} + +export function Icon({ name, size = 16 }: IconProps) { + const glyph = ICONS[name]; + if (!glyph) { + return null; + } + return ( + + {glyph} + + ); +} + +export default Icon; diff --git a/docs/fern/components/MetricsTable.tsx b/docs/fern/components/MetricsTable.tsx new file mode 100644 index 00000000000..f95dc18646e --- /dev/null +++ b/docs/fern/components/MetricsTable.tsx @@ -0,0 +1,106 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * MetricsTable - Styled comparison table for benchmark results. + * + * Optional: highlights best values per column (bold). + * NOTE: Fern's custom component pipeline uses the automatic JSX runtime. + * Do NOT import React -- the `react` module is not resolvable in Fern's build. + * + * Usage in MDX: + * import { MetricsTable } from "@/components/MetricsTable"; + * + * + */ + +export interface MetricsTableProps { + headers: string[]; + rows: (string | number)[][]; + /** Column indices where lower is better (for highlighting) */ + lowerIsBetter?: number[]; + /** Column indices where higher is better (default for non-lowerIsBetter) */ + higherIsBetter?: number[]; +} + +function findBestIndices( + rows: (string | number)[][], + colIndex: number, + lowerIsBetter: boolean +): Set { + const values = rows.map((r) => { + const v = r[colIndex]; + if (typeof v === "number") return v; + const parsed = parseFloat(String(v)); + return isNaN(parsed) ? (lowerIsBetter ? Infinity : -Infinity) : parsed; + }); + const best = lowerIsBetter ? Math.min(...values) : Math.max(...values); + const bestIndices = new Set(); + values.forEach((v, i) => { + if (v === best) bestIndices.add(i); + }); + return bestIndices; +} + +export const MetricsTable = ({ + headers, + rows, + lowerIsBetter = [], + higherIsBetter = [], +}: MetricsTableProps) => { + const lowerSet = new Set(lowerIsBetter); + const bestByCol: Record> = {}; + + for (let c = 0; c < headers.length; c++) { + if (lowerSet.has(c)) { + bestByCol[c] = findBestIndices(rows, c, true); + } else if (higherIsBetter.includes(c)) { + bestByCol[c] = findBestIndices(rows, c, false); + } else { + const numLike = rows.every((r) => { + const v = r[c]; + return typeof v === "number" || !isNaN(parseFloat(String(v))); + }); + if (numLike) { + bestByCol[c] = findBestIndices(rows, c, false); + } + } + } + + return ( +
+ + + + {headers.map((h, i) => ( + + ))} + + + + {rows.map((row, rowIdx) => ( + + {row.map((cell, colIdx) => { + const isBest = bestByCol[colIdx]?.has(rowIdx); + return ( + + ); + })} + + ))} + +
{h}
+ {cell} +
+
+ ); +}; diff --git a/docs/fern/components/NotebookViewer.tsx b/docs/fern/components/NotebookViewer.tsx new file mode 100644 index 00000000000..87c09059436 --- /dev/null +++ b/docs/fern/components/NotebookViewer.tsx @@ -0,0 +1,421 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { ReactNode } from "react"; + +/** + * NotebookViewer - Renders Jupyter notebook content in Fern docs. + * + * Uses Fern's code block structure (fern-code, fern-code-block, etc.) so input + * and output cells match the default Fern code block styling. + * + * Accepts notebook cells (markdown + code) and optionally a Colab URL. + * Designed to work with notebooks converted via `fern/scripts/ipynb-to-fern-json.py`. + * + * NOTE: Fern's custom component pipeline uses the automatic JSX runtime. + * Only type-only imports from "react" are used (erased at compile time). + * + * SECURITY / TRUST MODEL: + * -------- + * Notebook output cells of `format: "html"` (typically pandas DataFrame `_repr_html_`, + * matplotlib HTML, or similar) are rendered with `dangerouslySetInnerHTML` and + * are NOT sanitized — see the renderer near the bottom of this file. + * + * The trust boundary is the converter pipeline: + * + * docs/notebook_source/*.py (jupytext source — code-reviewed at PR time) + * └─> jupytext --execute (runs in CI/maintainer shell with NVIDIA_API_KEY) + * └─> *.ipynb (real outputs captured) + * └─> fern/scripts/ipynb-to-fern-json.py + * └─> fern/components/notebooks/*.{json,ts} (committed) + * + * Both ends of the pipeline (the .py source and the generated *.ts) are + * code-reviewed before merge. Fern's bundle is then static. + * + * If a future tutorial ever embeds output that incorporates LLM-generated HTML + * or arbitrary user-controlled content, switch the `format === "html"` branch + * to a sanitizer (e.g. DOMPurify) before merging. + * + * Usage in MDX: + * import { NotebookViewer } from "@/components/NotebookViewer"; + * import notebook from "@/components/notebooks/1-the-basics"; + * + * + */ + +export interface CellOutput { + type: "text" | "image"; + data: string; + format?: "plain" | "html"; + /** MIME for image outputs. Defaults to image/png; the converter sets + * image/jpeg when it re-encodes large outputs to keep the .ts payload small. */ + mime?: string; +} + +export interface NotebookCell { + type: "markdown" | "code"; + source: string; + /** Pre-rendered syntax-highlighted HTML (from Pygments). When present, used instead of escaped source. */ + source_html?: string; + language?: string; + outputs?: CellOutput[]; +} + +export interface NotebookData { + cells: NotebookCell[]; +} + +export interface NotebookViewerProps { + /** Notebook data with cells array. If import fails, this may be undefined. */ + notebook?: NotebookData | null; + /** Optional Colab URL for "Run in Colab" badge */ + colabUrl?: string; + /** Show code cell outputs (default: true) */ + showOutputs?: boolean; +} + +function NotebookViewerError({ message, detail }: { message: string; detail?: string }) { + return ( +
+ NotebookViewer error: {message} + {detail && ( + 
+          {detail}
+
+ )} +
+ ); +} + +function escapeHtml(text: string): string { + if (typeof text !== "string") return ""; + return text + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + + +function handleCopy(content: string, button: HTMLButtonElement) { + navigator.clipboard.writeText(content).catch(() => {}); + const originalHtml = button.innerHTML; + const originalLabel = button.getAttribute("aria-label") ?? "Copy code"; + button.innerHTML = "Copied!"; + button.setAttribute("aria-label", "Copied to clipboard"); + setTimeout(() => { + button.innerHTML = originalHtml; + button.setAttribute("aria-label", originalLabel); + }, 1500); +} + +const FLAG_ICON = ( + + + + +); + +const SCROLL_AREA_STYLE = `[data-radix-scroll-area-viewport]{scrollbar-width:thin;scrollbar-color:var(--grayscale-a7) transparent;-webkit-overflow-scrolling:touch;}[data-radix-scroll-area-viewport]::-webkit-scrollbar{height:8px;width:8px;}[data-radix-scroll-area-viewport]::-webkit-scrollbar-track{background:transparent;}[data-radix-scroll-area-viewport]::-webkit-scrollbar-thumb{background:var(--grayscale-a7);border-radius:9999px;}`; + +const BUTTON_BASE_CLASS = + "focus-visible:ring-(color:--accent) rounded-2 inline-flex items-center justify-center gap-2 whitespace-nowrap text-sm font-medium transition-colors hover:transition-none focus-visible:outline-none focus-visible:ring-1 disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0 text-(color:--grayscale-a11) hover:bg-(color:--accent-a3) hover:text-(color:--accent-11) pointer-coarse:size-9 size-7"; + +/** Fern code block structure – matches Fern docs (header with language + buttons, pre with scroll area). */ +function FernCodeBlock({ + title, + children, + className = "", + asPre = true, + copyContent, + showLineNumbers = false, + codeHtml, +}: { + title: string; + children: ReactNode; + className?: string; + /** Use div instead of pre for content (needed when children include block elements like img/div). */ + asPre?: boolean; + /** Raw text to copy when copy button is clicked. When provided, shows a copy button. */ + copyContent?: string; + /** Show line numbers in a table layout (matches Fern's code block structure). */ + showLineNumbers?: boolean; + /** Pre-rendered HTML for each line when showLineNumbers is true. Lines are split by newline. */ + codeHtml?: string; +}) { + const headerLabel = title === "Output" ? "Output" : title.charAt(0).toUpperCase() + title.slice(1); + const wrapperClasses = + "fern-code fern-code-block bg-card-background border-card-border rounded-3 shadow-card-grayscale relative mb-6 mt-4 flex w-full min-w-0 max-w-full flex-col border first:mt-0"; + const preStyle = { + backgroundColor: "rgb(255, 255, 255)", + ["--shiki-dark-bg" as string]: "#212121", + color: "rgb(36, 41, 46)", + ["--shiki-dark" as string]: "#EEFFFF", + }; + + const scrollAreaContent = () => { + if (codeHtml == null) return null; + const lines = codeHtml.split("\n"); + return ( +
+ +``` ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) -For deployment guide, see [Admin Setup](../set-up/index.md) in the {{platform_name}} documentation. +For deployment guide, see [Admin Setup](../set-up/index.md) in the NeMo Platform documentation. ## Values -The following is the complete `values.yaml` file for the {{platform_name}} Helm Chart. +The following is the complete `values.yaml` file for the NeMo Platform Helm Chart. All configuration options are documented inline with comments. - ---8<-- "_values.yaml" diff --git a/docs/index.mdx b/docs/index.mdx index eae8c123337..2944505d69d 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -1,8 +1,10 @@ -# {{platform_name}} - +--- +title: "NeMo Platform" +description: "" +--- Make the agents you ship faster, more accurate, and safer. -{{platform_name}} brings NVIDIA NeMo libraries together under one CLI, Python SDK, and web UI. Hardening, evaluation, and tuning for the agents you put in production. +NeMo Platform brings NVIDIA NeMo libraries together under one CLI, Python SDK, and web UI. Hardening, evaluation, and tuning for the agents you put in production. ## What You Can Do diff --git a/docs/notebooks/ndd_evaluator.mdx b/docs/notebooks/ndd_evaluator.mdx index 7b7623981b0..4146e4bc2ba 100644 --- a/docs/notebooks/ndd_evaluator.mdx +++ b/docs/notebooks/ndd_evaluator.mdx @@ -1,5 +1,7 @@ -# Emergency Triage LLM Evaluation - +--- +title: "Emergency Triage LLM Evaluation" +description: "" +--- **Problem:** Accurately labeling Emergency Severity Index (ESI) levels from nurse triage notes is critical for clinical research and model development, but real-world data is often sensitive and access is restricted. Additionally, obtaining high-quality human annotations is costly and slow, making it difficult to build large, diverse datasets for robust evaluation. **Opportunity:** Synthetic data generation offers a scalable, privacy-preserving alternative. By simulating realistic triage notes and ESI labels, we can create rich datasets without exposing patient information or relying on scarce human annotators. This enables rapid iteration, benchmarking, and model improvement in a domain where data scarcity is a major bottleneck. @@ -37,7 +39,7 @@ Tip: Run the cells in order. You can re-run preview/generation to explore differ ## **Step 1**: 🎨 NeMo Data Designer -First, create a client for interacting with {{platform_name}}. +First, create a client for interacting with NeMo Platform. ```python import os @@ -52,7 +54,7 @@ client = NeMoPlatform(base_url=BASE_URL, workspace=WORKSPACE) ``` Create a Secret and Model Provider to use models hosted on `build.nvidia.com`. -Note: these resources may already exist if you are running {{platform_name}} Quickstart. +Note: these resources may already exist if you are running NeMo Platform Quickstart. ```python SECRET_NAME = "nvidia-build-api-key" @@ -224,7 +226,7 @@ config_builder.add_column( ## 🦜 LLM-generated columns ```python -{% raw %} + # LLM-generated triage note config_builder.add_column( dd.LLMTextColumnConfig( @@ -247,13 +249,13 @@ config_builder.add_column( model_alias=MODEL_ALIAS_GENERATOR, ) ) -{% endraw %} + ``` ## ⚖️ LLM-as-a-Judge Evaluation Step ```python -{% raw %} + # Rubric: clinical coherence clinical_coherence_rubric = dd.Score( name="Clinical Coherence", @@ -302,7 +304,7 @@ config_builder.add_column( scores=[clinical_coherence_rubric, esi_level_complexity_rubric], ) ) -{% endraw %} + ``` ### 🧪 Generate & Preview @@ -490,7 +492,7 @@ for level, df in df_complexities.items(): This section defines the evaluation configuration used to assess model performance on triage note classification using a custom evaluator. ```python -{% raw %} + EVALUATOR_CONFIG = { "prompt_template": { "messages": [ @@ -515,7 +517,7 @@ EVALUATOR_CONFIG = { } } -{% endraw %} + ``` ## 🔍 Model evaluation loop and configuration @@ -529,7 +531,7 @@ The models evaluated are: For *each* complexity level, the accuracy score for each model is printed, allowing for side-by-side evaluation of how each model performs at every complexity. ```python -{% raw %} + import time import pandas as pd from typing import Any @@ -637,7 +639,7 @@ for complexity in COMPLEXITIES: results_dict[spec['name']][complexity.capitalize()] = 100 * accuracy # Store as percentage print(f" --> DONE: {spec['name']}, {complexity.capitalize()} (Accuracy: {100*accuracy:.2f}%)\n") -{% endraw %} + ``` ```python diff --git a/docs/pysdk/client/index.mdx b/docs/pysdk/client/index.mdx index 6035c8b0e66..71dea5605dc 100644 --- a/docs/pysdk/client/index.mdx +++ b/docs/pysdk/client/index.mdx @@ -1,6 +1,8 @@ -# Client APIs - -The following reference provides detailed documentation for the synchronous and asynchronous clients of the {{platform_name}} Python SDK. +--- +title: "Client APIs" +description: "" +--- +The following reference provides detailed documentation for the synchronous and asynchronous clients of the NeMo Platform Python SDK. ## Synchronous Client @@ -39,7 +41,7 @@ NeMoPlatform( | Parameter | Description | | --- | --- | | `workspace` | Workspace name used by workspace-scoped routes. You can set it on the client or pass it to individual methods that accept a workspace argument. | -| `base_url` | Base URL for the {{platform_name}} API. If omitted, the client reads it from the active CLI context. | +| `base_url` | Base URL for the NeMo Platform API. If omitted, the client reads it from the active CLI context. | | `inference_base_url` | Optional override for inference gateway requests. Defaults to `base_url`. | | `config_path` | Path to the CLI config file. Defaults to `~/.config/nmp/config.yaml`. | | `context_name` | Named CLI context to read from the config file. | @@ -145,7 +147,7 @@ for SDK-level errors. ## Client Attributes -The {{platform_name}} clients provide access to API resources through the following attributes. +The NeMo Platform clients provide access to API resources through the following attributes. For endpoint details, see the [REST API Reference](../../api/index.md). ### Organization diff --git a/docs/pysdk/index.mdx b/docs/pysdk/index.mdx index 70efa41aa5d..3780a398266 100644 --- a/docs/pysdk/index.mdx +++ b/docs/pysdk/index.mdx @@ -1,24 +1,32 @@ +--- +title: "NeMo Platform Python SDK Reference" +description: "" +--- +import Snippet03Usage from "@/snippets/sdk/python/overrides/nemo-platform/README/03_usage.mdx"; + -# {{platform_name}} Python SDK Reference -The [{{platform_name}} Python SDK](https://pypi.org/project/nemo-platform/) is a library for building and deploying AI models, abstracting the underlying infrastructure and providing a high-level interface for the {{platform_name}} APIs. +The [NeMo Platform Python SDK](https://pypi.org/project/nemo-platform/) is a library for building and deploying AI models, abstracting the underlying infrastructure and providing a high-level interface for the NeMo Platform APIs. ## Installation -Install the {{platform_name}} Python SDK using `pip`: +Install the NeMo Platform Python SDK using `pip`: ```bash pip install nemo-platform ``` -!!! note "This project will download and install additional third-party open source software projects. Review the license terms of these open source projects before use." + + +This project will download and install additional third-party open source software projects. Review the license terms of these open source projects before use. + + If you previously installed the `nemo-microservices` package, uninstall it first to avoid conflicts: ```bash pip uninstall nemo-microservices ``` ---8<-- "sdk/python/overrides/nemo-platform/README/03_usage.md" - + ## Next Steps - Read more about connecting with the [client APIs](./client/index.md). diff --git a/docs/requirements.mdx b/docs/requirements.mdx index f12d2c1c83f..8ab4222509a 100644 --- a/docs/requirements.mdx +++ b/docs/requirements.mdx @@ -1,8 +1,10 @@ -# Hardware and Software Requirements for {{platform_name}} +--- +title: "Hardware and Software Requirements for NeMo Platform" +description: "" +--- +This page lists the requirements for the OSS local-install path for NeMo Platform 0.1.0. For the full compatibility table, see the [Support Matrix](support-matrix.md). -This page lists the requirements for the OSS local-install path for {{platform_name}} {{ release }}. For the full compatibility table, see the [Support Matrix](support-matrix.md). - -The OSS {{ release }} documentation is scoped to local setup with the Python package and `nemo setup`. Docker Compose, Helm, Kubernetes, and OpenShift deployment guides are not part of this release scope. +The OSS 0.1.0 documentation is scoped to local setup with the Python package and `nemo setup`. Docker Compose, Helm, Kubernetes, and OpenShift deployment guides are not part of this release scope. ## Local Setup Requirements diff --git a/docs/run-inference/about.mdx b/docs/run-inference/about.mdx index 45332831c02..7f69a23c9c6 100644 --- a/docs/run-inference/about.mdx +++ b/docs/run-inference/about.mdx @@ -1,9 +1,13 @@ - - +--- +title: "About Models and Inference" +description: "" +--- +import SnippetNvidiaBuildModelProvider from "@/snippets/_snippets/nvidia-build-model-provider.mdx"; +import SnippetCliSdkSetup from "@/snippets/_snippets/tutorials/cli-sdk-setup.mdx"; + -# About Models and Inference -The {{platform_name}} provides APIs for registering external model providers and routing inference requests through a unified gateway. +The NeMo Platform provides APIs for registering external model providers and routing inference requests through a unified gateway. ```mermaid flowchart LR @@ -42,8 +46,7 @@ flowchart LR GW --> Client ``` ---8<-- "_snippets/nvidia-build-model-provider.md" - + ### Add External Providers @@ -57,138 +60,156 @@ The example below demonstrates how to recreate it in your own workspace. For disambiguation purposes, this example names the manually-created version `my-nvidia-build`. -=== "CLI" - - ```bash - # Store API key - echo "$NVIDIA_API_KEY" | nemo secrets create "nvidia-api-key" --from-file - + - # Create provider - nemo inference providers create "my-nvidia-build" \ - --host-url "https://integrate.api.nvidia.com" \ - --api-key-secret-name "nvidia-api-key" + - nemo wait inference provider my-nvidia-build +```bash +# Store API key +echo "$NVIDIA_API_KEY" | nemo secrets create "nvidia-api-key" --from-file - - # Test using interactive chat - nemo chat nvidia/llama-3.3-nemotron-super-49b-v1 'Hello!' \ - --provider my-nvidia-build - ``` +# Create provider +nemo inference providers create "my-nvidia-build" \ + --host-url "https://integrate.api.nvidia.com" \ + --api-key-secret-name "nvidia-api-key" -=== "Python SDK" +nemo wait inference provider my-nvidia-build - ```python - # Store API key - client.secrets.create(name="nvidia-api-key", data=os.environ["NVIDIA_API_KEY"]) +# Test using interactive chat +nemo chat nvidia/llama-3.3-nemotron-super-49b-v1 'Hello!' \ + --provider my-nvidia-build +``` - # Create provider - provider = client.inference.providers.create( - name="my-nvidia-build", - host_url="https://integrate.api.nvidia.com", - api_key_secret_name="nvidia-api-key", - ) + + - client.models.wait_for_provider("my-nvidia-build") +```python +# Store API key +client.secrets.create(name="nvidia-api-key", data=os.environ["NVIDIA_API_KEY"]) + +# Create provider +provider = client.inference.providers.create( + name="my-nvidia-build", + host_url="https://integrate.api.nvidia.com", + api_key_secret_name="nvidia-api-key", +) + +client.models.wait_for_provider("my-nvidia-build") + +# Use provider routing +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="my-nvidia-build", + body={ + "model": "meta/llama-3.1-8b-instruct", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` - # Use provider routing - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="my-nvidia-build", - body={ - "model": "meta/llama-3.1-8b-instruct", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` + + #### OpenAI -=== "CLI" - - ```bash - # Store API key - echo "$OPENAI_API_KEY" | nemo secrets create "openai-api-key" --from-file - + - # Create provider with enabled models - nemo inference providers create "openai" \ - --host-url "https://api.openai.com/v1" \ - --api-key-secret-name "openai-api-key" \ - --enabled-models "gpt-4" \ - --enabled-models "gpt-3.5-turbo" + - nemo wait inference provider openai +```bash +# Store API key +echo "$OPENAI_API_KEY" | nemo secrets create "openai-api-key" --from-file - - # Test using interactive chat - nemo chat gpt-4 'Hello!' \ - --provider openai - ``` +# Create provider with enabled models +nemo inference providers create "openai" \ + --host-url "https://api.openai.com/v1" \ + --api-key-secret-name "openai-api-key" \ + --enabled-models "gpt-4" \ + --enabled-models "gpt-3.5-turbo" -=== "Python SDK" +nemo wait inference provider openai - ```python - client.secrets.create(name="openai-api-key", data=os.environ["OPENAI_API_KEY"]) +# Test using interactive chat +nemo chat gpt-4 'Hello!' \ + --provider openai +``` - provider = client.inference.providers.create( - name="openai", - host_url="https://api.openai.com/v1", - api_key_secret_name="openai-api-key", - enabled_models=["gpt-4", "gpt-3.5-turbo"], - ) + + - client.models.wait_for_provider("openai") +```python +client.secrets.create(name="openai-api-key", data=os.environ["OPENAI_API_KEY"]) + +provider = client.inference.providers.create( + name="openai", + host_url="https://api.openai.com/v1", + api_key_secret_name="openai-api-key", + enabled_models=["gpt-4", "gpt-3.5-turbo"], +) + +client.models.wait_for_provider("openai") + +# Use provider routing +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="openai", + body={ + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` - # Use provider routing - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="openai", - body={ - "model": "gpt-4", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` + + #### Anthropic Anthropic's `/v1/messages` API expects the API key in an `X-Api-Key:` header (not `Authorization: Bearer`) and requires an `anthropic-version` header on every request. Use `--auth-header-format` (Jinja2 template, must contain exactly one `{{ auth_secret }}` variable) to override the default `Authorization: Bearer {{ auth_secret }}` and pass the API-version pin via `--default-extra-headers`. Without these, Anthropic rejects every request with 401. -=== "CLI" + - ```bash - # Store API key - echo "$ANTHROPIC_API_KEY" | nemo secrets create "anthropic-api-key" --from-file - + - # Create provider — override the default Bearer header and pin the API version - nemo inference providers create "anthropic" \ - --host-url "https://api.anthropic.com" \ - --api-key-secret-name "anthropic-api-key" \ - --auth-header-format "X-Api-Key: {{ auth_secret }}" \ - --default-extra-headers '{"anthropic-version": "2023-06-01"}' +```bash +# Store API key +echo "$ANTHROPIC_API_KEY" | nemo secrets create "anthropic-api-key" --from-file - - nemo wait inference provider anthropic - ``` +# Create provider — override the default Bearer header and pin the API version +nemo inference providers create "anthropic" \ + --host-url "https://api.anthropic.com" \ + --api-key-secret-name "anthropic-api-key" \ + --auth-header-format "X-Api-Key: {{ auth_secret }}" \ + --default-extra-headers '{"anthropic-version": "2023-06-01"}' -=== "Python SDK" +nemo wait inference provider anthropic +``` - ```python - # Store API key - client.secrets.create(name="anthropic-api-key", data=os.environ["ANTHROPIC_API_KEY"]) + + - # Create provider — override the default Bearer header and pin the API version - provider = client.inference.providers.create( - name="anthropic", - host_url="https://api.anthropic.com", - api_key_secret_name="anthropic-api-key", - auth_header_format="X-Api-Key: {{ auth_secret }}", - default_extra_headers={"anthropic-version": "2023-06-01"}, - ) +```python +# Store API key +client.secrets.create(name="anthropic-api-key", data=os.environ["ANTHROPIC_API_KEY"]) + +# Create provider — override the default Bearer header and pin the API version +provider = client.inference.providers.create( + name="anthropic", + host_url="https://api.anthropic.com", + api_key_secret_name="anthropic-api-key", + auth_header_format="X-Api-Key: {{ auth_secret }}", + default_extra_headers={"anthropic-version": "2023-06-01"}, +) + +client.models.wait_for_provider("anthropic") +``` - client.models.wait_for_provider("anthropic") - ``` + + `{{ auth_secret }}` is substituted with the resolved secret value at request time. --- @@ -228,8 +249,7 @@ without hand-assembling the path. Add `--provider ` or Set up the CLI or Python SDK first: ---8<-- "_snippets/tutorials/cli-sdk-setup.md" - + The SDK provides convenience methods for OpenAI compatibility: ```python diff --git a/docs/run-inference/tutorials/deploy-models.mdx b/docs/run-inference/tutorials/deploy-models.mdx index ba48667081b..1259e2a6b9a 100644 --- a/docs/run-inference/tutorials/deploy-models.mdx +++ b/docs/run-inference/tutorials/deploy-models.mdx @@ -1,15 +1,20 @@ - - +--- +title: "Deploy Models" +description: "" +--- +import SnippetCliSdkSetup from "@/snippets/_snippets/tutorials/cli-sdk-setup.mdx"; -# Deploy Models Deploy models from NGC or HuggingFace. Register external providers like OpenAI or NVIDIA Build. -!!! note "Resource names for deployments, deployment configs, and providers must contain only letters (a-z, A-Z), digits (0-9), underscores, hyphens, and dots. For example: `llama-3.1-8b`, `my-custom-model`, `qwen-fs-config`." + ---8<-- "_snippets/tutorials/cli-sdk-setup.md" +Resource names for deployments, deployment configs, and providers must contain only letters (a-z, A-Z), digits (0-9), underscores, hyphens, and dots. For example: `llama-3.1-8b`, `my-custom-model`, `qwen-fs-config`. + + + --- ## Add External Providers @@ -23,138 +28,156 @@ The example below demonstrates how to recreate it in your own workspace. For disambiguation purposes, this example names the manually-created version `my-nvidia-build`. -=== "CLI" - - ```bash - # Store API key - echo "$NVIDIA_API_KEY" | nemo secrets create "nvidia-api-key" --from-file - + - # Create provider - nemo inference providers create "my-nvidia-build" \ - --host-url "https://integrate.api.nvidia.com" \ - --api-key-secret-name "nvidia-api-key" + - nemo wait inference provider my-nvidia-build +```bash +# Store API key +echo "$NVIDIA_API_KEY" | nemo secrets create "nvidia-api-key" --from-file - - # Test using interactive chat - nemo chat nvidia/llama-3.3-nemotron-super-49b-v1 'Hello!' \ - --provider my-nvidia-build - ``` +# Create provider +nemo inference providers create "my-nvidia-build" \ +--host-url "https://integrate.api.nvidia.com" \ +--api-key-secret-name "nvidia-api-key" -=== "Python SDK" +nemo wait inference provider my-nvidia-build - ```python - # Store API key - client.secrets.create(name="nvidia-api-key", value=os.environ["NVIDIA_API_KEY"]) - - # Create provider - provider = client.inference.providers.create( - name="my-nvidia-build", - host_url="https://integrate.api.nvidia.com", - api_key_secret_name="nvidia-api-key", - ) +# Test using interactive chat +nemo chat nvidia/llama-3.3-nemotron-super-49b-v1 'Hello!' \ +--provider my-nvidia-build +``` - client.models.wait_for_provider("my-nvidia-build") + + + +```python +# Store API key +client.secrets.create(name="nvidia-api-key", value=os.environ["NVIDIA_API_KEY"]) + +# Create provider +provider = client.inference.providers.create( + name="my-nvidia-build", + host_url="https://integrate.api.nvidia.com", + api_key_secret_name="nvidia-api-key", +) + +client.models.wait_for_provider("my-nvidia-build") + +# Use provider routing +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="my-nvidia-build", + body={ + "model": "meta/llama-3.1-8b-instruct", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` - # Use provider routing - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="my-nvidia-build", - body={ - "model": "meta/llama-3.1-8b-instruct", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` + + ### OpenAI -=== "CLI" + - ```bash - # Store API key - echo "$OPENAI_API_KEY" | nemo secrets create "openai-api-key" --from-file - + - # Create provider with enabled models - nemo inference providers create "openai" \ - --host-url "https://api.openai.com/v1" \ - --api-key-secret-name "openai-api-key" \ - --enabled-models "gpt-4" \ - --enabled-models "gpt-3.5-turbo" +```bash +# Store API key +echo "$OPENAI_API_KEY" | nemo secrets create "openai-api-key" --from-file - - nemo wait inference provider openai +# Create provider with enabled models +nemo inference providers create "openai" \ +--host-url "https://api.openai.com/v1" \ +--api-key-secret-name "openai-api-key" \ +--enabled-models "gpt-4" \ +--enabled-models "gpt-3.5-turbo" - # Test using interactive chat - nemo chat gpt-4 'Hello!' \ - --provider openai - ``` +nemo wait inference provider openai -=== "Python SDK" - - ```python - client.secrets.create(name="openai-api-key", value=os.environ["OPENAI_API_KEY"]) +# Test using interactive chat +nemo chat gpt-4 'Hello!' \ +--provider openai +``` - provider = client.inference.providers.create( - name="openai", - host_url="https://api.openai.com/v1", - api_key_secret_name="openai-api-key", - enabled_models=["gpt-4", "gpt-3.5-turbo"], - ) + + + +```python +client.secrets.create(name="openai-api-key", value=os.environ["OPENAI_API_KEY"]) + +provider = client.inference.providers.create( + name="openai", + host_url="https://api.openai.com/v1", + api_key_secret_name="openai-api-key", + enabled_models=["gpt-4", "gpt-3.5-turbo"], +) + +client.models.wait_for_provider("openai") + +# Use provider routing +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="openai", + body={ + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` - client.models.wait_for_provider("openai") - - # Use provider routing - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="openai", - body={ - "model": "gpt-4", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` + + ### Anthropic Anthropic's `/v1/messages` API expects the API key in an `X-Api-Key:` header (not `Authorization: Bearer`) and requires an `anthropic-version` header on every request. Use `--auth-header-format` (Jinja2 template, must contain exactly one `{{ auth_secret }}` variable) to override the default `Authorization: Bearer {{ auth_secret }}` and pass the API-version pin via `--default-extra-headers`. Without these, Anthropic rejects every request with 401. -=== "CLI" + - ```bash - # Store API key - echo "$ANTHROPIC_API_KEY" | nemo secrets create "anthropic-api-key" --from-file - + - # Create provider — override the default Bearer header and pin the API version - nemo inference providers create "anthropic" \ - --host-url "https://api.anthropic.com" \ - --api-key-secret-name "anthropic-api-key" \ - --auth-header-format "X-Api-Key: {{ auth_secret }}" \ - --default-extra-headers '{"anthropic-version": "2023-06-01"}' +```bash +# Store API key +echo "$ANTHROPIC_API_KEY" | nemo secrets create "anthropic-api-key" --from-file - - nemo wait inference provider anthropic - ``` +# Create provider — override the default Bearer header and pin the API version +nemo inference providers create "anthropic" \ +--host-url "https://api.anthropic.com" \ +--api-key-secret-name "anthropic-api-key" \ +--auth-header-format "X-Api-Key: {{ auth_secret }}" \ +--default-extra-headers '{"anthropic-version": "2023-06-01"}' -=== "Python SDK" +nemo wait inference provider anthropic +``` - ```python - # Store API key - client.secrets.create(name="anthropic-api-key", data=os.environ["ANTHROPIC_API_KEY"]) + + - # Create provider — override the default Bearer header and pin the API version - provider = client.inference.providers.create( - name="anthropic", - host_url="https://api.anthropic.com", - api_key_secret_name="anthropic-api-key", - auth_header_format="X-Api-Key: {{ auth_secret }}", - default_extra_headers={"anthropic-version": "2023-06-01"}, - ) +```python +# Store API key +client.secrets.create(name="anthropic-api-key", data=os.environ["ANTHROPIC_API_KEY"]) - client.models.wait_for_provider("anthropic") - ``` +# Create provider — override the default Bearer header and pin the API version +provider = client.inference.providers.create( + name="anthropic", + host_url="https://api.anthropic.com", + api_key_secret_name="anthropic-api-key", + auth_header_format="X-Api-Key: {{ auth_secret }}", + default_extra_headers={"anthropic-version": "2023-06-01"}, +) +client.models.wait_for_provider("anthropic") +``` + + + + `{{ auth_secret }}` is substituted with the resolved secret value at request time. --- @@ -166,253 +189,282 @@ Deploy pre-built NIM containers from NGC. ### Deploy Llama 3.2 1B -=== "CLI" + - ```bash - nemo inference deployment-configs create \ - --name "llama-3-2-1b-config" \ - --nim-deployment '{ - "gpu": 1, - "image_name": "nvcr.io/nim/meta/llama-3.2-1b-instruct", - "image_tag": "1.8.6", - "model_name": "meta/llama-3.2-1b-instruct" - }' + - nemo inference deployments create \ - --name "llama-3-2-1b-deployment" \ - --config "llama-3-2-1b-config" +```bash +nemo inference deployment-configs create \ +--name "llama-3-2-1b-config" \ +--nim-deployment '{ +"gpu": 1, +"image_name": "nvcr.io/nim/meta/llama-3.2-1b-instruct", +"image_tag": "1.8.6", +"model_name": "meta/llama-3.2-1b-instruct" +}' - nemo wait inference deployment llama-3-2-1b-deployment +nemo inference deployments create \ +--name "llama-3-2-1b-deployment" \ +--config "llama-3-2-1b-config" - nemo chat meta/llama-3.2-1b-instruct 'Hello!' \ - --provider llama-3-2-1b-deployment \ - --max-tokens 100 - ``` +nemo wait inference deployment llama-3-2-1b-deployment -=== "Python SDK" +nemo chat meta/llama-3.2-1b-instruct 'Hello!' \ +--provider llama-3-2-1b-deployment \ +--max-tokens 100 +``` - ```python - config = client.inference.deployment_configs.create( - name="llama-3-2-1b-config", - nim_deployment={ - "gpu": 1, - "image_name": "nvcr.io/nim/meta/llama-3.2-1b-instruct", - "image_tag": "1.8.6", - "model_name": "meta/llama-3.2-1b-instruct", - }, - ) - - deployment = client.inference.deployments.create( - name="llama-3-2-1b-deployment", config="llama-3-2-1b-config" - ) - - client.models.wait_for_status( - deployment_name="llama-3-2-1b-deployment", desired_status="READY" - ) - - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="llama-3-2-1b-deployment", - body={ - "model": "meta/llama-3.2-1b-instruct", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` + + + +```python +config = client.inference.deployment_configs.create( + name="llama-3-2-1b-config", + nim_deployment={ + "gpu": 1, + "image_name": "nvcr.io/nim/meta/llama-3.2-1b-instruct", + "image_tag": "1.8.6", + "model_name": "meta/llama-3.2-1b-instruct", + }, +) + +deployment = client.inference.deployments.create( + name="llama-3-2-1b-deployment", config="llama-3-2-1b-config" +) + +client.models.wait_for_status( + deployment_name="llama-3-2-1b-deployment", desired_status="READY" +) + +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="llama-3-2-1b-deployment", + body={ + "model": "meta/llama-3.2-1b-instruct", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` + + + ### Deploy NeMo Guard Jailbreak Detection Deploy classification NIMs like NeMoGuard for content safety. Uses the `/v1/classify` endpoint instead of chat completions. -=== "CLI" + - ```bash - nemo inference deployment-configs create \ - --name "nemoguard-jailbreak-config" \ - --nim-deployment '{ - "gpu": 1, - "image_name": "nvcr.io/nim/nvidia/nemoguard-jailbreak-detect", - "image_tag": "1.10.1" - }' + - nemo inference deployments create \ - --name "nemoguard-jailbreak-deployment" \ - --config "nemoguard-jailbreak-config" +```bash +nemo inference deployment-configs create \ +--name "nemoguard-jailbreak-config" \ +--nim-deployment '{ +"gpu": 1, +"image_name": "nvcr.io/nim/nvidia/nemoguard-jailbreak-detect", +"image_tag": "1.10.1" +}' - nemo wait inference deployment nemoguard-jailbreak-deployment +nemo inference deployments create \ +--name "nemoguard-jailbreak-deployment" \ +--config "nemoguard-jailbreak-config" - nemo inference gateway provider post v1/classify \ - --name "nemoguard-jailbreak-deployment" \ - --body '{"input": "Tell me about vacation spots in Hawaii."}' - ``` +nemo wait inference deployment nemoguard-jailbreak-deployment -=== "Python SDK" - - ```python - config = client.inference.deployment_configs.create( - name="nemoguard-jailbreak-config", - nim_deployment={ - "gpu": 1, - "image_name": "nvcr.io/nim/nvidia/nemoguard-jailbreak-detect", - "image_tag": "1.10.1", - }, - ) - - deployment = client.inference.deployments.create( - name="nemoguard-jailbreak-deployment", config="nemoguard-jailbreak-config" - ) +nemo inference gateway provider post v1/classify \ +--name "nemoguard-jailbreak-deployment" \ +--body '{"input": "Tell me about vacation spots in Hawaii."}' +``` - client.models.wait_for_status( - deployment_name="nemoguard-jailbreak-deployment", desired_status="READY" - ) + + + +```python +config = client.inference.deployment_configs.create( + name="nemoguard-jailbreak-config", + nim_deployment={ + "gpu": 1, + "image_name": "nvcr.io/nim/nvidia/nemoguard-jailbreak-detect", + "image_tag": "1.10.1", + }, +) + +deployment = client.inference.deployments.create( + name="nemoguard-jailbreak-deployment", config="nemoguard-jailbreak-config" +) + +client.models.wait_for_status( + deployment_name="nemoguard-jailbreak-deployment", desired_status="READY" +) + +response = client.inference.gateway.provider.post( + "v1/classify", + name="nemoguard-jailbreak-deployment", + body={"input": "Tell me about vacation spots in Hawaii."}, +) +``` - response = client.inference.gateway.provider.post( - "v1/classify", - name="nemoguard-jailbreak-deployment", - body={"input": "Tell me about vacation spots in Hawaii."}, - ) - ``` + + --- ## Deploy from HuggingFace -!!! warning "HuggingFace deployments use the Multi-LLM NIM (`nvcr.io/nim/nvidia/llm-nim:1.13.1`) by default, which only supports specific model architectures. Check the [supported architectures list](https://docs.nvidia.com/nim/large-language-models/1.13.0/supported-architectures.html) before deploying. If your model architecture is not listed, you will need a model-specific NIM image — see [Deploy from NGC](#deploy-from-ngc) for that approach." + + +HuggingFace deployments use the Multi-LLM NIM (`nvcr.io/nim/nvidia/llm-nim:1.13.1`) by default, which only supports specific model architectures. Check the [supported architectures list](https://docs.nvidia.com/nim/large-language-models/1.13.0/supported-architectures.html) before deploying. If your model architecture is not listed, you will need a model-specific NIM image — see [Deploy from NGC](#deploy-from-ngc) for that approach. + + You can register a HuggingFace model through the Files service. This creates a fileset that acts as a proxy. The Files service handles authentication and caches the weights on first download, so subsequent deployments start faster. -=== "CLI" - - ```bash - # (Optional) Create a HuggingFace token secret for private models. - # Public models like Qwen do not require a token. - echo "$HF_TOKEN" | nemo secrets create "hf-token-secret" --from-file - - - # Create a fileset pointing to the HuggingFace model. - # "token_secret" is optional — only needed for private/gated models. - nemo files filesets create "qwen-2-5-1-5b" \ - --storage '{ - "type": "huggingface", - "repo_id": "Qwen/Qwen2.5-1.5B-Instruct", - "repo_type": "model", - "token_secret": "hf-token-secret" - }' - - # Register a model entity referencing the fileset - nemo models create "qwen-2-5-1-5b" \ - --fileset "default/qwen-2-5-1-5b" - - # Create deployment config pointing to the model entity - nemo inference deployment-configs create "qwen-fs-config" \ - --nim-deployment '{ - "model_namespace": "default", - "model_name": "qwen-2-5-1-5b", - "gpu": 1 - }' - - nemo inference deployments create "qwen-fs-deployment" \ - --config "qwen-fs-config" - - nemo wait inference deployment qwen-fs-deployment - - nemo chat default/qwen-2-5-1-5b 'Hello!' \ - --provider qwen-fs-deployment \ - --max-tokens 100 - ``` - -=== "Python SDK" - - ```python - # (Optional) Create a HuggingFace token secret for private models. - # Public models like Qwen do not require a token. - client.secrets.create(name="hf-token-secret", value=os.environ["HF_TOKEN"]) - - # Create a fileset pointing to the HuggingFace model. - # "token_secret" is optional — only needed for private/gated models. - client.files.filesets.create( - name="qwen-2-5-1-5b", - storage={ - "type": "huggingface", - "repo_id": "Qwen/Qwen2.5-1.5B-Instruct", - "repo_type": "model", - "token_secret": "hf-token-secret", - }, - ) - - # Register a model entity referencing the fileset - client.models.create(name="qwen-2-5-1-5b", fileset="default/qwen-2-5-1-5b") - - # Create deployment config pointing to the model entity - config = client.inference.deployment_configs.create( - name="qwen-fs-config", - nim_deployment={ - "model_namespace": "default", - "model_name": "qwen-2-5-1-5b", - "gpu": 1, - }, - ) - - # Deploy — no hf_token_secret_name needed - deployment = client.inference.deployments.create( - name="qwen-fs-deployment", config="qwen-fs-config" - ) - - client.models.wait_for_status( - deployment_name="qwen-fs-deployment", desired_status="READY" - ) - - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="qwen-fs-deployment", - body={ - "model": "default/qwen-2-5-1-5b", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` + + + + +```bash +# (Optional) Create a HuggingFace token secret for private models. +# Public models like Qwen do not require a token. +echo "$HF_TOKEN" | nemo secrets create "hf-token-secret" --from-file - + +# Create a fileset pointing to the HuggingFace model. +# "token_secret" is optional — only needed for private/gated models. +nemo files filesets create "qwen-2-5-1-5b" \ +--storage '{ +"type": "huggingface", +"repo_id": "Qwen/Qwen2.5-1.5B-Instruct", +"repo_type": "model", +"token_secret": "hf-token-secret" +}' + +# Register a model entity referencing the fileset +nemo models create "qwen-2-5-1-5b" \ +--fileset "default/qwen-2-5-1-5b" + +# Create deployment config pointing to the model entity +nemo inference deployment-configs create "qwen-fs-config" \ +--nim-deployment '{ +"model_namespace": "default", +"model_name": "qwen-2-5-1-5b", +"gpu": 1 +}' + +nemo inference deployments create "qwen-fs-deployment" \ +--config "qwen-fs-config" + +nemo wait inference deployment qwen-fs-deployment + +nemo chat default/qwen-2-5-1-5b 'Hello!' \ +--provider qwen-fs-deployment \ +--max-tokens 100 +``` + + + + +```python +# (Optional) Create a HuggingFace token secret for private models. +# Public models like Qwen do not require a token. +client.secrets.create(name="hf-token-secret", value=os.environ["HF_TOKEN"]) + +# Create a fileset pointing to the HuggingFace model. +# "token_secret" is optional — only needed for private/gated models. +client.files.filesets.create( + name="qwen-2-5-1-5b", + storage={ + "type": "huggingface", + "repo_id": "Qwen/Qwen2.5-1.5B-Instruct", + "repo_type": "model", + "token_secret": "hf-token-secret", + }, +) + +# Register a model entity referencing the fileset +client.models.create(name="qwen-2-5-1-5b", fileset="default/qwen-2-5-1-5b") + +# Create deployment config pointing to the model entity +config = client.inference.deployment_configs.create( + name="qwen-fs-config", + nim_deployment={ + "model_namespace": "default", + "model_name": "qwen-2-5-1-5b", + "gpu": 1, + }, +) + +# Deploy — no hf_token_secret_name needed +deployment = client.inference.deployments.create( + name="qwen-fs-deployment", config="qwen-fs-config" +) + +client.models.wait_for_status( + deployment_name="qwen-fs-deployment", desired_status="READY" +) + +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="qwen-fs-deployment", + body={ + "model": "default/qwen-2-5-1-5b", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` -!!! tip + - The `fileset` format is `/`. This tells the deployment system to pull weights from the Files service, which proxies the download from HuggingFace using the fileset `token_secret`. For public models like Qwen, the `token_secret` on the fileset is optional. + + +The `fileset` format is `/`. This tells the deployment system to pull weights from the Files service, which proxies the download from HuggingFace using the fileset `token_secret`. For public models like Qwen, the `token_secret` on the fileset is optional. + + --- ## Deployment Cleanup -=== "CLI" + + + - ```bash - # Note: Deleting the deployment will free up its GPU(s) when complete - nemo inference deployments delete - nemo wait inference deployment --status DELETED - nemo inference deployment-configs delete +```bash +# Note: Deleting the deployment will free up its GPU(s) when complete +nemo inference deployments delete +nemo wait inference deployment --status DELETED +nemo inference deployment-configs delete - # For external providers - nemo inference providers delete - nemo secrets delete - ``` +# For external providers +nemo inference providers delete +nemo secrets delete +``` -=== "Python SDK" + + - ```python - # Note: Deleting the deployment will free up its GPU(s) when complete - client.inference.deployments.delete(name="") - client.models.wait_for_status( - deployment_name="", desired_status="DELETED" - ) - client.inference.deployment_configs.delete(name="") +```python +# Note: Deleting the deployment will free up its GPU(s) when complete +client.inference.deployments.delete(name="") +client.models.wait_for_status( + deployment_name="", desired_status="DELETED" +) +client.inference.deployment_configs.delete(name="") - # For external providers - client.inference.providers.delete(name="") - client.secrets.delete(name="") - ``` +# For external providers +client.inference.providers.delete(name="") +client.secrets.delete(name="") +``` + + + --- ## Multi-GPU Deployments @@ -436,132 +488,146 @@ The multi-LLM NIM (`nvcr.io/nim/nvidia/llm-nim:1.13.1`) requires explicit parall This example deploys [Qwen2.5-14B-Instruct](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct) across 2 GPUs using tensor parallelism. -=== "CLI" - - ```bash - # Create a fileset pointing to the HuggingFace model - # Qwen models are public — token_secret is optional - nemo files filesets create \ - --name "qwen-2-5-14b" \ - --storage '{ - "type": "huggingface", - "repo_id": "Qwen/Qwen2.5-14B-Instruct", - "repo_type": "model" - }' - - # Register a model entity referencing the fileset - nemo models create \ - --name "qwen-2-5-14b" \ - --fileset "default/qwen-2-5-14b" - - # Create deployment config with 2 GPUs (TP=2 by default) - nemo inference deployment-configs create \ - --name "qwen-14b-config" \ - --nim-deployment '{ - "model_name": "default/qwen-2-5-14b", - "gpu": 2 - }' - - # Deploy - nemo inference deployments create \ - --name "qwen-14b-deployment" \ - --config "qwen-14b-config" - - nemo wait inference deployment qwen-14b-deployment - - nemo chat default/qwen-2-5-14b 'Hello!' \ - --max-tokens 100 - ``` - -=== "Python SDK" - - ```python - # Create a fileset pointing to the HuggingFace model - # Qwen models are public — token_secret is optional - client.files.filesets.create( - name="qwen-2-5-14b", - storage={ - "type": "huggingface", - "repo_id": "Qwen/Qwen2.5-14B-Instruct", - "repo_type": "model", - }, - ) - - # Register a model entity referencing the fileset - client.models.create(name="qwen-2-5-14b", fileset="default/qwen-2-5-14b") - - # Create deployment config with 2 GPUs (TP=2 by default) - config = client.inference.deployment_configs.create( - name="qwen-14b-config", - nim_deployment={"model_name": "default/qwen-2-5-14b", "gpu": 2}, - ) - - # Deploy - deployment = client.inference.deployments.create( - name="qwen-14b-deployment", config="qwen-14b-config" - ) - - client.models.wait_for_status( - deployment_name="qwen-14b-deployment", desired_status="READY" - ) - - response = client.inference.gateway.model.post( - "v1/chat/completions", - name="qwen-2-5-14b", - body={ - "model": "default/qwen-2-5-14b", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - # NIM sets NIM_TENSOR_PARALLEL_SIZE=2 automatically - ``` + + + +```bash +# Create a fileset pointing to the HuggingFace model +# Qwen models are public — token_secret is optional +nemo files filesets create \ +--name "qwen-2-5-14b" \ +--storage '{ +"type": "huggingface", +"repo_id": "Qwen/Qwen2.5-14B-Instruct", +"repo_type": "model" +}' + +# Register a model entity referencing the fileset +nemo models create \ +--name "qwen-2-5-14b" \ +--fileset "default/qwen-2-5-14b" + +# Create deployment config with 2 GPUs (TP=2 by default) +nemo inference deployment-configs create \ +--name "qwen-14b-config" \ +--nim-deployment '{ +"model_name": "default/qwen-2-5-14b", +"gpu": 2 +}' + +# Deploy +nemo inference deployments create \ +--name "qwen-14b-deployment" \ +--config "qwen-14b-config" + +nemo wait inference deployment qwen-14b-deployment + +nemo chat default/qwen-2-5-14b 'Hello!' \ +--max-tokens 100 +``` + + + + +```python +# Create a fileset pointing to the HuggingFace model +# Qwen models are public — token_secret is optional +client.files.filesets.create( + name="qwen-2-5-14b", + storage={ + "type": "huggingface", + "repo_id": "Qwen/Qwen2.5-14B-Instruct", + "repo_type": "model", + }, +) + +# Register a model entity referencing the fileset +client.models.create(name="qwen-2-5-14b", fileset="default/qwen-2-5-14b") + +# Create deployment config with 2 GPUs (TP=2 by default) +config = client.inference.deployment_configs.create( + name="qwen-14b-config", + nim_deployment={"model_name": "default/qwen-2-5-14b", "gpu": 2}, +) + +# Deploy +deployment = client.inference.deployments.create( + name="qwen-14b-deployment", config="qwen-14b-config" +) + +client.models.wait_for_status( + deployment_name="qwen-14b-deployment", desired_status="READY" +) + +response = client.inference.gateway.model.post( + "v1/chat/completions", + name="qwen-2-5-14b", + body={ + "model": "default/qwen-2-5-14b", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +# NIM sets NIM_TENSOR_PARALLEL_SIZE=2 automatically +``` + + + + ### Custom Parallelism Configuration For larger models requiring more GPUs, you can configure specific TP/PP splits using `additional_envs`. The formula is: `gpu` = `NIM_TENSOR_PARALLEL_SIZE` × `NIM_PIPELINE_PARALLEL_SIZE`. -=== "CLI" + - ```bash - nemo inference deployment-configs create \ - --name "multi-gpu-custom-config" \ - --nim-deployment '{ - "model_name": "default/qwen-2-5-14b", - "gpu": 4, - "additional_envs": { - "NIM_TENSOR_PARALLEL_SIZE": "2", - "NIM_PIPELINE_PARALLEL_SIZE": "2" - } - }' - ``` - -=== "Python SDK" - - ```python - config = client.inference.deployment_configs.create( - name="multi-gpu-custom-config", - nim_deployment={ - "model_name": "default/qwen-2-5-14b", - "gpu": 4, - "additional_envs": { - "NIM_TENSOR_PARALLEL_SIZE": "2", - "NIM_PIPELINE_PARALLEL_SIZE": "2", - }, + + +```bash +nemo inference deployment-configs create \ +--name "multi-gpu-custom-config" \ +--nim-deployment '{ +"model_name": "default/qwen-2-5-14b", +"gpu": 4, +"additional_envs": { +"NIM_TENSOR_PARALLEL_SIZE": "2", +"NIM_PIPELINE_PARALLEL_SIZE": "2" +} +}' +``` + + + + +```python +config = client.inference.deployment_configs.create( + name="multi-gpu-custom-config", + nim_deployment={ + "model_name": "default/qwen-2-5-14b", + "gpu": 4, + "additional_envs": { + "NIM_TENSOR_PARALLEL_SIZE": "2", + "NIM_PIPELINE_PARALLEL_SIZE": "2", }, - ) - ``` + }, +) +``` + + + + + -!!! tip - **Choosing Parallelism Strategy** +**Choosing Parallelism Strategy** - - **TP=8, PP=1** (default): Lowest latency, best for real-time applications - - **TP=4, PP=2**: Balanced latency and throughput - - **TP=2, PP=4**: Highest throughput, best for batch processing +- **TP=8, PP=1** (default): Lowest latency, best for real-time applications +- **TP=4, PP=2**: Balanced latency and throughput +- **TP=2, PP=4**: Highest throughput, best for batch processing - For custom models, match deployment parallelism to training parallelism for optimal performance. +For custom models, match deployment parallelism to training parallelism for optimal performance. + --- @@ -571,9 +637,11 @@ Configure custom chat templates and tool calling for NIM deployments. These sett For more information on chat templates, see the [Hugging Face chat templating guide](https://huggingface.co/docs/transformers/en/chat_templating) and the [NeMo chat templates documentation](https://docs.nvidia.com/nemo/rl/latest/design-docs/chat-datasets.html#chat-templates). -!!! warning - **Security consideration:** Chat templates are Jinja2 programs that execute on every inference call. While NIM uses a sandboxed Jinja2 environment (mitigating arbitrary code execution), a malicious or misconfigured template can still alter model behavior — for example, by injecting hidden instructions, rewriting messages, or degrading output quality. Grant chat template permissions only to trusted users, and review templates before deploying to production. See [Inference-Time Backdoors via Chat Templates (IEEE S&P 2026)](https://arxiv.org/abs/2602.04653) for further background. + +**Security consideration:** Chat templates are Jinja2 programs that execute on every inference call. While NIM uses a sandboxed Jinja2 environment (mitigating arbitrary code execution), a malicious or misconfigured template can still alter model behavior — for example, by injecting hidden instructions, rewriting messages, or degrading output quality. Grant chat template permissions only to trusted users, and review templates before deploying to production. See [Inference-Time Backdoors via Chat Templates (IEEE S&P 2026)](https://arxiv.org/abs/2602.04653) for further background. + + ### Configuration Sources There are two ways to configure chat templates and tool calling, with deployment-level settings taking highest priority: @@ -603,10 +671,12 @@ flowchart LR Set `chat_template` and tool calling configuration with fileset `metadata.model.tool_calling`. The platform automatically propagates these into the model entity spec when the model-spec background task runs. -=== "CLI" + + + + +```bash - ```bash -{% raw %} # Create fileset with chat template and tool calling config via metadata nemo files filesets create \ --name "llama-3-2-1b-tool" \ @@ -643,77 +713,83 @@ Set `chat_template` and tool calling configuration with fileset `metadata.model. --config "llama-tool-config" nemo wait inference deployment llama-tool-deployment -{% endraw %} + ``` -=== "Python SDK" - - ```python - {% raw %} - # Create fileset with chat template and tool calling config via metadata - tool_metadata: dict[str, object] = { - "model": { - "tool_calling": { - "chat_template": ( - "{%- for message in messages %}" - "{%- set content = '<{{ start_header_id }}>' + message['role'] + '<{{ end_header_id }}>\n\n'" - " + message['content'] | trim + '<{{ eot_id }}>' %}" - "{%- if loop.index0 == 0 %}{%- set content = '<{{ begin_of_text }}>' + content %}{%- endif %}" - "{{ content }}{%- endfor %}" - "{%- if add_generation_prompt %}{{ '<{{ start_header_id }}>assistant<{{ end_header_id }}>\n\n' }}{%- endif %}" - ), - "tool_call_parser": "llama3_json", - "auto_tool_choice": True, - } + + + +```python + +# Create fileset with chat template and tool calling config via metadata +tool_metadata: dict[str, object] = { + "model": { + "tool_calling": { + "chat_template": ( + "{%- for message in messages %}" + "{%- set content = '<{{ start_header_id }}>' + message['role'] + '<{{ end_header_id }}>\n\n'" + " + message['content'] | trim + '<{{ eot_id }}>' %}" + "{%- if loop.index0 == 0 %}{%- set content = '<{{ begin_of_text }}>' + content %}{%- endif %}" + "{{ content }}{%- endfor %}" + "{%- if add_generation_prompt %}{{ '<{{ start_header_id }}>assistant<{{ end_header_id }}>\n\n' }}{%- endif %}" + ), + "tool_call_parser": "llama3_json", + "auto_tool_choice": True, } } - - client.files.filesets.create( - name="llama-3-2-1b-tool", - storage={ - "type": "huggingface", - "repo_id": "meta-llama/Llama-3.2-1B-Instruct", - "repo_type": "model" - }, - metadata=tool_metadata, - ) - - # Register model entity referencing the fileset - client.models.create( - name="llama-3-2-1b-tool", - fileset="default/llama-3-2-1b-tool", - ) - - # Deploy — chat_template and tool_call_config are inherited from the fileset - config = client.inference.deployment_configs.create( - name="llama-tool-config", - nim_deployment={ - "model_name": "default/llama-3-2-1b-tool", - "gpu": 1 - } - ) - - deployment = client.inference.deployments.create( - name="llama-tool-deployment", - config="llama-tool-config" - ) - - client.models.wait_for_status( - deployment_name="llama-tool-deployment", - desired_status="READY" - ) - {% endraw %} +} + +client.files.filesets.create( + name="llama-3-2-1b-tool", + storage={ + "type": "huggingface", + "repo_id": "meta-llama/Llama-3.2-1B-Instruct", + "repo_type": "model" + }, + metadata=tool_metadata, +) + +# Register model entity referencing the fileset +client.models.create( + name="llama-3-2-1b-tool", + fileset="default/llama-3-2-1b-tool", +) + +# Deploy — chat_template and tool_call_config are inherited from the fileset +config = client.inference.deployment_configs.create( + name="llama-tool-config", + nim_deployment={ + "model_name": "default/llama-3-2-1b-tool", + "gpu": 1 + } +) + +deployment = client.inference.deployments.create( + name="llama-tool-deployment", + config="llama-tool-config" +) + +client.models.wait_for_status( + deployment_name="llama-tool-deployment", + desired_status="READY" +) + ``` + + + ### Option 2: Set via Deployment Config (Override) Set `chat_template` and `tool_call_config` directly on the deployment config. These override any values from the fileset. -=== "CLI" + + + + +```bash - ```bash -{% raw %} nemo inference deployment-configs create \ --name "llama-tool-override-config" \ --nim-deployment '{ @@ -725,154 +801,170 @@ Set `chat_template` and `tool_call_config` directly on the deployment config. Th "auto_tool_choice": false } }' -{% endraw %} + ``` -=== "Python SDK" - - ```python - {% raw %} - config = client.inference.deployment_configs.create( - name="llama-tool-override-config", - nim_deployment={ - "model_name": "default/llama-3-2-1b-tool", - "gpu": 1, - "chat_template": "{%- for message in messages %}{{ message['content'] }}{%- endfor %}", - "tool_call_config": { - "tool_call_parser": "hermes", - "auto_tool_choice": False, - } + + + +```python + +config = client.inference.deployment_configs.create( + name="llama-tool-override-config", + nim_deployment={ + "model_name": "default/llama-3-2-1b-tool", + "gpu": 1, + "chat_template": "{%- for message in messages %}{{ message['content'] }}{%- endfor %}", + "tool_call_config": { + "tool_call_parser": "hermes", + "auto_tool_choice": False, } - ) - {% endraw %} + } +) + ``` + + + ### Change Tool Calling Config for an Existing Model Updating a fileset's `metadata.model.tool_calling` does **not** propagate changes to an existing model entity. The model entity's `spec` is populated from the fileset only at creation time. To change the tool calling configuration, create a new fileset with the updated config and a new model entity that references it. -=== "CLI" + + + + +```bash +# Create a new fileset with the updated tool calling config +nemo files filesets create \ +--name "llama-3-2-1b-tool-v2" \ +--storage '{ +"type": "huggingface", +"repo_id": "meta-llama/Llama-3.2-1B-Instruct", +"repo_type": "model" +}' \ +--metadata '{ +"model": { +"tool_calling": { +"tool_call_parser": "mistral", +"auto_tool_choice": true +} +} +}' + +# Create a new model entity referencing the new fileset +nemo models create \ +--name "llama-3-2-1b-tool-v2" \ +--fileset "default/llama-3-2-1b-tool-v2" + +# Update deployment config to use the new model, or create a new one +nemo inference deployment-configs create \ +--name "llama-tool-config-v2" \ +--nim-deployment '{ +"model_name": "default/llama-3-2-1b-tool-v2", +"gpu": 1 +}' + +nemo inference deployments create \ +--name "llama-tool-deployment-v2" \ +--config "llama-tool-config-v2" + +nemo wait inference deployment llama-tool-deployment-v2 +``` + + + - ```bash - # Create a new fileset with the updated tool calling config - nemo files filesets create \ - --name "llama-3-2-1b-tool-v2" \ - --storage '{ - "type": "huggingface", - "repo_id": "meta-llama/Llama-3.2-1B-Instruct", - "repo_type": "model" - }' \ - --metadata '{ +```python +# Create a new fileset with the updated tool calling config via metadata +tool_metadata_v2: dict[str, object] = { "model": { - "tool_calling": { - "tool_call_parser": "mistral", - "auto_tool_choice": true - } - } - }' - - # Create a new model entity referencing the new fileset - nemo models create \ - --name "llama-3-2-1b-tool-v2" \ - --fileset "default/llama-3-2-1b-tool-v2" - - # Update deployment config to use the new model, or create a new one - nemo inference deployment-configs create \ - --name "llama-tool-config-v2" \ - --nim-deployment '{ - "model_name": "default/llama-3-2-1b-tool-v2", - "gpu": 1 - }' - - nemo inference deployments create \ - --name "llama-tool-deployment-v2" \ - --config "llama-tool-config-v2" - - nemo wait inference deployment llama-tool-deployment-v2 - ``` - -=== "Python SDK" - - ```python - # Create a new fileset with the updated tool calling config via metadata - tool_metadata_v2: dict[str, object] = { - "model": { - "tool_calling": { - "tool_call_parser": "mistral", - "auto_tool_choice": True, - } + "tool_calling": { + "tool_call_parser": "mistral", + "auto_tool_choice": True, } } +} + +client.files.filesets.create( + name="llama-3-2-1b-tool-v2", + storage={ + "type": "huggingface", + "repo_id": "meta-llama/Llama-3.2-1B-Instruct", + "repo_type": "model", + }, + metadata=tool_metadata_v2, +) + +# Create a new model entity referencing the new fileset +client.models.create( + name="llama-3-2-1b-tool-v2", + fileset="default/llama-3-2-1b-tool-v2", +) + +# Create a new deployment config and deployment +config = client.inference.deployment_configs.create( + name="llama-tool-config-v2", + nim_deployment={"model_name": "default/llama-3-2-1b-tool-v2", "gpu": 1}, +) + +deployment = client.inference.deployments.create( + name="llama-tool-deployment-v2", config="llama-tool-config-v2" +) + +client.models.wait_for_status( + deployment_name="llama-tool-deployment-v2", desired_status="READY" +) +``` - client.files.filesets.create( - name="llama-3-2-1b-tool-v2", - storage={ - "type": "huggingface", - "repo_id": "meta-llama/Llama-3.2-1B-Instruct", - "repo_type": "model", - }, - metadata=tool_metadata_v2, - ) - - # Create a new model entity referencing the new fileset - client.models.create( - name="llama-3-2-1b-tool-v2", - fileset="default/llama-3-2-1b-tool-v2", - ) - - # Create a new deployment config and deployment - config = client.inference.deployment_configs.create( - name="llama-tool-config-v2", - nim_deployment={"model_name": "default/llama-3-2-1b-tool-v2", "gpu": 1}, - ) - - deployment = client.inference.deployments.create( - name="llama-tool-deployment-v2", config="llama-tool-config-v2" - ) - - client.models.wait_for_status( - deployment_name="llama-tool-deployment-v2", desired_status="READY" - ) - ``` + + ### Custom Tool Call Plugin For custom tool calling parsers, store the plugin Python file in a separate fileset and reference it via `tool_call_plugin`. -!!! info - Because plugins execute arbitrary Python code inside the NIM container, `tool_call_plugin` is **disabled by default** at the platform level. - To enable it, set `models.tool_call_plugin.enabled: true` in the platform configuration and ensure the user has the `models.tool-call-plugin.set` permission (granted to Admin and PlatformAdmin roles by default). - - -=== "Python SDK" - - ```python - # 1. Create a fileset for the plugin file - client.files.filesets.create(name="my-tool-plugin") - client.files.upload( - fileset="my-tool-plugin", - local_path="my_parser.py", - remote_path="my_parser.py", - ) - - # 2. Reference the plugin fileset in the model's fileset metadata - plugin_tool_metadata: dict[str, object] = { - "model": { - "tool_calling": { - "tool_call_parser": "custom_parser", - "tool_call_plugin": "default/my-tool-plugin", - "auto_tool_choice": True, - } + + +Because plugins execute arbitrary Python code inside the NIM container, `tool_call_plugin` is **disabled by default** at the platform level. +To enable it, set `models.tool_call_plugin.enabled: true` in the platform configuration and ensure the user has the `models.tool-call-plugin.set` permission (granted to Admin and PlatformAdmin roles by default). + + + + + + +```python +# 1. Create a fileset for the plugin file +client.files.filesets.create(name="my-tool-plugin") +client.files.upload( + fileset="my-tool-plugin", + local_path="my_parser.py", + remote_path="my_parser.py", +) + +# 2. Reference the plugin fileset in the model's fileset metadata +plugin_tool_metadata: dict[str, object] = { + "model": { + "tool_calling": { + "tool_call_parser": "custom_parser", + "tool_call_plugin": "default/my-tool-plugin", + "auto_tool_choice": True, } } +} - client.files.filesets.update( - "llama-3-2-1b-tool", - metadata=plugin_tool_metadata, - ) - ``` +client.files.filesets.update( + "llama-3-2-1b-tool", + metadata=plugin_tool_metadata, +) +``` + + + The platform downloads the plugin fileset at deployment time and passes the `.py` file path to NIM via the `NIM_TOOL_PARSER_PLUGIN` environment variable. ### Tool Call Config Reference @@ -901,33 +993,43 @@ The platform translates tool calling configuration into NIM environment variable Some models (for example, `nvidia/nemotron-3-nano-30b-a3b`) enable reasoning/thinking by default. (From the model card: "[nemotron-3] responds to user queries and tasks by first generating a reasoning trace and then concluding with a final response") You can disable it on a per-request basis by passing `chat_template_kwargs` directly in the request body: -=== "CLI" - - ```bash - nemo inference gateway provider post v1/chat/completions \ - --name "my-deployment" \ - --body '{ - "model": "default/my-model", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - "chat_template_kwargs": {"thinking": false} - }' - ``` - -=== "Python SDK" - - ```python - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="my-deployment", - body={ - "model": "default/my-model", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - "chat_template_kwargs": {"thinking": False}, - }, - ) - ``` + + + + +```bash +nemo inference gateway provider post v1/chat/completions \ +--name "my-deployment" \ +--body '{ +"model": "default/my-model", +"messages": [{"role": "user", "content": "Hello!"}], +"max_tokens": 100, +"chat_template_kwargs": {"thinking": false} +}' +``` + + + + +```python +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="my-deployment", + body={ + "model": "default/my-model", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + "chat_template_kwargs": {"thinking": False}, + }, +) +``` + + + + + + +`chat_template_kwargs` is passed directly in the request body, not nested under `extra_body`. For more details on request-level reasoning overrides, see the [vLLM documentation](https://docs.vllm.ai/en/latest/features/reasoning_outputs/#request-level-override). -!!! note "`chat_template_kwargs` is passed directly in the request body, not nested under `extra_body`. For more details on request-level reasoning overrides, see the [vLLM documentation](https://docs.vllm.ai/en/latest/features/reasoning_outputs/#request-level-override)." + This parameter only applies to models which use vLLM under the hood. For non-vLLM providers such as OpenAI or NVIDIA Build, the parameter that controls reasoning differs. Consult the provider's documentation for the specific parameter. diff --git a/docs/run-inference/tutorials/index.mdx b/docs/run-inference/tutorials/index.mdx index bbd61d8d7de..a7d8d3bcb1d 100644 --- a/docs/run-inference/tutorials/index.mdx +++ b/docs/run-inference/tutorials/index.mdx @@ -1,7 +1,10 @@ +--- +title: "Tutorials" +description: "" +--- -# Tutorials -Learn how to run inference through the {{platform_name}}. +Learn how to run inference through the NeMo Platform. ## Prerequisites diff --git a/docs/run-inference/tutorials/run-inference.mdx b/docs/run-inference/tutorials/run-inference.mdx index 2ce4cabc5b9..ba5c489a143 100644 --- a/docs/run-inference/tutorials/run-inference.mdx +++ b/docs/run-inference/tutorials/run-inference.mdx @@ -1,15 +1,19 @@ - - +--- +title: "Run Inference" +description: "" +--- +import SnippetCliSdkSetup from "@/snippets/_snippets/tutorials/cli-sdk-setup.mdx"; + -# Run Inference Route inference requests through the gateway using model entity routing, provider routing, or OpenAI-compatible routing. -!!! note - This tutorial assumes you have an external provider registered. The platform pre-configures a `system/nvidia-build` provider on startup, which most of the examples below use. To register your own provider, see [About Models and Inference](../about.md#model-providers). + ---8<-- "_snippets/tutorials/cli-sdk-setup.md" +This tutorial assumes you have an external provider registered. The platform pre-configures a `system/nvidia-build` provider on startup, which most of the examples below use. To register your own provider, see [About Models and Inference](../about.md#model-providers). + + --- ## Listing Models @@ -24,29 +28,35 @@ There are two distinct ways to list models, depending on what you need: Use `nemo models list` to manage model configurations. Use `nemo inference models list` to discover what's ready to serve requests right now. -=== "CLI" + - ```bash - # All registered model entities (may include models without active deployments) - nemo models list + - # Models available for inference via the gateway (OpenAI-compatible IDs) - nemo inference models list - ``` +```bash +# All registered model entities (may include models without active deployments) +nemo models list -=== "Python SDK" +# Models available for inference via the gateway (OpenAI-compatible IDs) +nemo inference models list +``` - ```python - # All registered model entities - for model in client.models.list(): - print(model.name) + + - # Models available for inference via the gateway - models = client.inference.models.list() - for model in models.data: - print(model.id) # format: workspace/model_entity_name - ``` +```python +# All registered model entities +for model in client.models.list(): + print(model.name) + +# Models available for inference via the gateway +models = client.inference.models.list() +for model in models.data: + print(model.id) # format: workspace/model_entity_name +``` + + + --- ## Route by Model Entity @@ -56,45 +66,57 @@ Route requests using the model entity name. The gateway selects an available pro Find available model entities: -=== "CLI" + - ```bash - nemo models list - ``` + -=== "Python SDK" +```bash +nemo models list +``` + + + + +```python +for model in client.models.list(): + print(model.name) +``` - ```python - for model in client.models.list(): - print(model.name) - ``` + + Run inference by passing the Model Entity. -=== "CLI" + + + + +```bash +# Model entities are auto-discovered from deployments +# Use the model entity name from 'nemo models list' +nemo chat meta-llama-3-2-1b-instruct 'Hello!' --max-tokens 100 +``` - ```bash - # Model entities are auto-discovered from deployments - # Use the model entity name from 'nemo models list' - nemo chat meta-llama-3-2-1b-instruct 'Hello!' --max-tokens 100 - ``` + + -=== "Python SDK" +```python +# Model entities are auto-discovered from deployments +response = client.inference.gateway.model.post( + "v1/chat/completions", + name="meta-llama-3-2-1b-instruct", # Model entity name + body={ + "model": "meta/llama-3.2-1b-instruct", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` - ```python - # Model entities are auto-discovered from deployments - response = client.inference.gateway.model.post( - "v1/chat/completions", - name="meta-llama-3-2-1b-instruct", # Model entity name - body={ - "model": "meta/llama-3.2-1b-instruct", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` + + --- ## Route by Provider @@ -104,63 +126,79 @@ Route to a specific provider instance. Use for A/B testing or targeting specific Find available providers: -=== "CLI" + - ```bash - nemo inference providers list + - # List models available on a provider if its API is OpenAI compliant - nemo inference gateway provider get v1/models --name llama-3-2-1b-deployment - ``` +```bash +nemo inference providers list + +# List models available on a provider if its API is OpenAI compliant +nemo inference gateway provider get v1/models --name llama-3-2-1b-deployment +``` + + + -=== "Python SDK" +```python +for provider in client.inference.providers.list(): + print(f"{provider.name}: {provider.status}") - ```python - for provider in client.inference.providers.list(): - print(f"{provider.name}: {provider.status}") +# List models available on a provider if its API is OpenAI compliant +models = client.inference.gateway.provider.get( + "v1/models", + name="llama-3-2-1b-deployment", +) +print(models) +``` - # List models available on a provider if its API is OpenAI compliant - models = client.inference.gateway.provider.get( - "v1/models", - name="llama-3-2-1b-deployment", - ) - print(models) - ``` + + Pass inference request using provider routing. -=== "CLI" + + + - ```bash - # Provider name matches deployment name for auto-created providers - nemo chat meta/llama-3.2-1b-instruct 'Hello!' \ - --provider llama-3-2-1b-deployment \ - --max-tokens 100 - ``` +```bash +# Provider name matches deployment name for auto-created providers +nemo chat meta/llama-3.2-1b-instruct 'Hello!' \ +--provider llama-3-2-1b-deployment \ +--max-tokens 100 +``` -=== "Python SDK" + + - ```python - # Provider name matches deployment name for auto-created providers - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="llama-3-2-1b-deployment", # Provider name - body={ - "model": "meta/llama-3.2-1b-instruct", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` +```python +# Provider name matches deployment name for auto-created providers +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="llama-3-2-1b-deployment", # Provider name + body={ + "model": "meta/llama-3.2-1b-instruct", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` + + + --- ## Route using OpenAI SDK Use the OpenAI-compatible endpoint for drop-in SDK replacement. The model field uses the format `{workspace}/{model_entity}`. -!!! note "Model entity naming" + + +Model entity naming + + The `{model_entity}` segment is validated against a strict regex (lowercase letters, digits, and hyphens; 2–63 characters; **no slashes**). Vendor-style IDs such as `meta/llama-3.3-70b-instruct` are rejected with @@ -174,23 +212,29 @@ Use the OpenAI-compatible endpoint for drop-in SDK replacement. The model field List models currently reachable via the Inference Gateway. Returns results in OpenAI-compatible format; each `id` is `workspace/model_entity_name`. -=== "CLI" + + + + +```bash +nemo inference models list +``` - ```bash - nemo inference models list - ``` + + -=== "Python SDK" +```python +models = client.inference.models.list() +for model in models.data: + print(model.id) # format: workspace/model_entity_name +``` - ```python - models = client.inference.models.list() - for model in models.data: - print(model.id) # format: workspace/model_entity_name - ``` + + ### Using SDK -Make requests to the OpenAI-compatible Inference Gateway route with the {{platform_name}} SDK. +Make requests to the OpenAI-compatible Inference Gateway route with the NeMo Platform SDK. ```python response = client.inference.gateway.openai.post( @@ -223,50 +267,55 @@ response = oai_client.chat.completions.create( If you construct third-party clients manually, pass the headers returned by `client.models.get_client_default_headers()` so auth and identity headers are preserved when authorization is enabled. -=== "OpenAI (manual client)" + - ```python - from openai import OpenAI + - base_url = client.models.get_openai_route_base_url() - headers = client.models.get_client_default_headers() +```python +from openai import OpenAI - oai_client = OpenAI( - base_url=base_url, - api_key="not-needed", - default_headers=headers, - ) +base_url = client.models.get_openai_route_base_url() +headers = client.models.get_client_default_headers() - response = oai_client.chat.completions.create( - model="default/meta-llama-3-2-1b-instruct", - messages=[{"role": "user", "content": "Hello!"}], - ) - ``` +oai_client = OpenAI( + base_url=base_url, + api_key="not-needed", + default_headers=headers, +) -=== "LiteLLM" +response = oai_client.chat.completions.create( + model="default/meta-llama-3-2-1b-instruct", + messages=[{"role": "user", "content": "Hello!"}], +) +``` + + - ```bash - # make sure litellm is installed - pip install litellm - ``` +```bash +# make sure litellm is installed +pip install litellm +``` + + +```python +from litellm import completion - - ```python - from litellm import completion +base_url = client.models.get_openai_route_base_url() +headers = client.models.get_client_default_headers() - base_url = client.models.get_openai_route_base_url() - headers = client.models.get_client_default_headers() +response = completion( + model="openai/meta-llama-3-2-1b-instruct", + messages=[{"role": "user", "content": "Hello!"}], + api_base=base_url, + api_key="not-needed", + extra_headers=headers, +) +``` - response = completion( - model="openai/meta-llama-3-2-1b-instruct", - messages=[{"role": "user", "content": "Hello!"}], - api_base=base_url, - api_key="not-needed", - extra_headers=headers, - ) - ``` + + ### Using curl The OpenAI-compatible chat-completions endpoint is reachable directly over HTTP. diff --git a/docs/safe-synthesizer/about/data-synthesis.mdx b/docs/safe-synthesizer/about/data-synthesis.mdx index 395e350cab4..403d0d3e5d1 100644 --- a/docs/safe-synthesizer/about/data-synthesis.mdx +++ b/docs/safe-synthesizer/about/data-synthesis.mdx @@ -1,11 +1,14 @@ +--- +title: "Data Synthesis" +description: "" +--- -# Data Synthesis -The synthesizer component is the main component of the {{nss_short_name}} product. It uses LLM-based fine-tuning to generate realistic synthetic data that maintains the utility of your original dataset while providing privacy protection. +The synthesizer component is the main component of the NeMo Safe Synthesizer product. It uses LLM-based fine-tuning to generate realistic synthetic data that maintains the utility of your original dataset while providing privacy protection. ## How It Works -{{nss_short_name}} employs a novel approach to synthetic data generation: +NeMo Safe Synthesizer employs a novel approach to synthetic data generation: 1. **Tabular Fine-Tuning**: Fine-tunes a language model on your tabular data to learn patterns, correlations, and statistical properties 2. **Generation**: Uses the fine-tuned model to generate new synthetic records that maintain data utility @@ -17,7 +20,7 @@ Creating synthetic versions of private data allows you to unlock insights withou ### LLM-Based Fine-Tuning -{{nss_short_name}} adapts language models to understand and generate tabular data: +NeMo Safe Synthesizer adapts language models to understand and generate tabular data: - Converts tabular data into text sequences suitable for LLM training - Fine-tunes on your dataset to capture patterns and correlations @@ -26,7 +29,7 @@ Creating synthetic versions of private data allows you to unlock insights withou ### Supported Data Types -{{nss_short_name}} supports diverse tabular data: +NeMo Safe Synthesizer supports diverse tabular data: - **Numeric**: Continuous and discrete numerical values - **Categorical**: Text labels and categories @@ -37,7 +40,7 @@ Creating synthetic versions of private data allows you to unlock insights withou A high level of privacy protection is achieved simply through the process of generating synthetic data, and is often a sufficient balance between privacy and utility. For use cases that require maximum privacy, you can fine-tune with differential privacy. -Differential privacy (DP) is the gold standard for privacy protection, providing mathematical guarantees that individual records cannot be identified. When you enable DP, {{nss_short_name}} introduces calibrated noise during model training to ensure that the synthetic data generation process is provably private. +Differential privacy (DP) is the gold standard for privacy protection, providing mathematical guarantees that individual records cannot be identified. When you enable DP, NeMo Safe Synthesizer introduces calibrated noise during model training to ensure that the synthetic data generation process is provably private. #### Mathematical Guarantee @@ -57,7 +60,7 @@ Where: #### DP-SGD Implementation -{{nss_short_name}} uses Differentially Private Stochastic Gradient Descent (DP-SGD) to add privacy guarantees during model training: +NeMo Safe Synthesizer uses Differentially Private Stochastic Gradient Descent (DP-SGD) to add privacy guarantees during model training: 1. **Per-sample gradient computation** - Calculate gradients for each training example individually 2. **Gradient clipping** - Clip L2 norm to `per_sample_max_grad_norm` to bound sensitivity diff --git a/docs/safe-synthesizer/about/evaluation.mdx b/docs/safe-synthesizer/about/evaluation.mdx index 8ba95c4f900..159f395c8ff 100644 --- a/docs/safe-synthesizer/about/evaluation.mdx +++ b/docs/safe-synthesizer/about/evaluation.mdx @@ -1,8 +1,11 @@ - +--- +title: "Evaluation" +description: "" +--- + -# Evaluation -Evaluation is a critical component of {{nss_short_name}} that helps you understand both the utility and privacy of your synthetic data. The evaluation step is enabled by default and provides comprehensive reports comparing your training and synthetic datasets across multiple dimensions. +Evaluation is a critical component of NeMo Safe Synthesizer that helps you understand both the utility and privacy of your synthetic data. The evaluation step is enabled by default and provides comprehensive reports comparing your training and synthetic datasets across multiple dimensions. ## How It Works @@ -89,7 +92,7 @@ For differential privacy configuration and privacy-utility tradeoffs, refer to [ ## Evaluation Reports -Every {{nss_short_name}} job automatically generates an HTML evaluation report containing: +Every NeMo Safe Synthesizer job automatically generates an HTML evaluation report containing: - Overall SQS and DPS scores - Detailed subscores for each metric diff --git a/docs/safe-synthesizer/about/index.mdx b/docs/safe-synthesizer/about/index.mdx index df468f3331d..028d10d9ed5 100644 --- a/docs/safe-synthesizer/about/index.mdx +++ b/docs/safe-synthesizer/about/index.mdx @@ -1,22 +1,26 @@ +--- +title: "About Generating Safe Synthetic Data" +description: "" +--- +import { Button } from "@/components/Button"; -# About Generating Safe Synthetic Data -{{nss_long_name}} enables you to create private versions of sensitive tabular datasets. The resulting data is entirely synthetic, with no one-to-one mapping to your original records. {{nss_short_name}} is purpose-built for privacy compliance and data protection while preserving data utility for downstream AI tasks. +NVIDIA NeMo Safe Synthesizer enables you to create private versions of sensitive tabular datasets. The resulting data is entirely synthetic, with no one-to-one mapping to your original records. NeMo Safe Synthesizer is purpose-built for privacy compliance and data protection while preserving data utility for downstream AI tasks. -[**Quickstart**](../getting-started.md){ .md-button .md-button--primary } -[**Tutorials**](../tutorials/index.md){ .md-button } + + --- -{{nss_short_name}} allows you to generate synthetic data that maintains the statistical properties of your original dataset without exposing sensitive information about individual records. +NeMo Safe Synthesizer allows you to generate synthetic data that maintains the statistical properties of your original dataset without exposing sensitive information about individual records. -{{nss_short_name}} is best when you have the data you need, but because it is private or sensitive in nature you cannot use it as-is. {{nss_short_name}} interpolates from existing data to generate a private, synthetic version, where new records have no one-to-one mapping to original records. If you do not have any data or want to extrapolate based on a very small set of examples, refer to [index](../../data-designer/index.md). NeMo Data Designer supports synthetic data creation from scratch or small seed for AI training and development use cases. +NeMo Safe Synthesizer is best when you have the data you need, but because it is private or sensitive in nature you cannot use it as-is. NeMo Safe Synthesizer interpolates from existing data to generate a private, synthetic version, where new records have no one-to-one mapping to original records. If you do not have any data or want to extrapolate based on a very small set of examples, refer to [index](../../data-designer/index.md). NeMo Data Designer supports synthetic data creation from scratch or small seed for AI training and development use cases. -## {{nss_short_name}} Job +## NeMo Safe Synthesizer Job -A complete {{nss_short_name}} job consists of the following steps: +A complete NeMo Safe Synthesizer job consists of the following steps: 1. [Upload Data](../../get-started/concepts/manage-files.md): Add your tabular data to the Files API 2. Prepare Data: @@ -36,27 +40,26 @@ Find all Safe Synthesizer configuration parameters in [Parameters Reference](ref ## Installation Options -Try out this early access API using Docker Compose or deploying the {{platform_name}} Helm chart. +Try out this early access API using Docker Compose or deploying the NeMo Platform Helm chart. -
+ -- **[Quickstart](../getting-started.md)** + - --- +Get started with the NeMo Safe Synthesizer microservice locally using the NeMo Platform CLI. Easiest for local testing. - Get started with the {{nss_short_name}} microservice locally using the {{platform_name}} CLI. Easiest for local testing. +standalone - standalone + + -- **[Helm Chart](../../set-up/helm/index.md)** +Deploy the NeMo Platform Helm Chart, which includes NeMo Safe Synthesizer. - --- +helm-chart - Deploy the {{platform_name}} Helm Chart, which includes {{nss_short_name}}. + - helm-chart - -
+ --- @@ -64,52 +67,48 @@ Try out this early access API using Docker Compose or deploying the {{platform_n Get hands-on experience with Safe Synthesizer through step-by-step tutorials. -
+ -- **[Tutorials](../tutorials/index.md)** + - --- +Learn how to use NeMo Safe Synthesizer with hands-on tutorials covering basics to advanced topics. - Learn how to use {{nss_short_name}} with hands-on tutorials covering basics to advanced topics. +beginner intermediate - beginner intermediate + -
+ --- ## Core Concepts -
- -- **[Data Synthesis](data-synthesis.md)** - - --- - - Learn about LLM-based synthesis, differential privacy, and tabular fine-tuning for generating synthetic data. - -- **[PII Replacement](pii-replacement.md)** + - --- + - Understand how PII detection and replacement works to protect sensitive information before synthesis. +Learn about LLM-based synthesis, differential privacy, and tabular fine-tuning for generating synthetic data. -- **[Evaluation](evaluation.md)** + + - --- +Understand how PII detection and replacement works to protect sensitive information before synthesis. - Learn about quality and privacy metrics used to assess synthetic data including SQS and DPS scores. + + -- **[Jobs](jobs.md)** +Learn about quality and privacy metrics used to assess synthetic data including SQS and DPS scores. - --- + + - Understand the job lifecycle, configuration, and execution for Safe Synthesizer pipelines. +Understand the job lifecycle, configuration, and execution for Safe Synthesizer pipelines. -- **[Parameters Reference](reference.md)** + + - --- +Reference all configuration parameters available when creating Safe Synthesizer jobs. - Reference all configuration parameters available when creating Safe Synthesizer jobs. + -
\ No newline at end of file + diff --git a/docs/safe-synthesizer/about/jobs.mdx b/docs/safe-synthesizer/about/jobs.mdx index 3ddac9c1f30..c51f9bd230c 100644 --- a/docs/safe-synthesizer/about/jobs.mdx +++ b/docs/safe-synthesizer/about/jobs.mdx @@ -1,15 +1,18 @@ - - +--- +title: "Safe Synthesizer Jobs" +description: "" +--- + + -# Safe Synthesizer Jobs -{{nss_short_name}} jobs orchestrate the complete pipeline from data preparation through synthesis to evaluation. Understanding the job lifecycle and configuration options is essential for effective use of the platform. +NeMo Safe Synthesizer jobs orchestrate the complete pipeline from data preparation through synthesis to evaluation. Understanding the job lifecycle and configuration options is essential for effective use of the platform. + - ## Job Lifecycle -A {{nss_short_name}} job progresses through several states: +A NeMo Safe Synthesizer job progresses through several states: ### Job States @@ -186,7 +189,7 @@ for log in job.fetch_logs(): ### Docker Compose Deployments -When running {{nss_short_name}} using Docker Compose, view container logs directly: +When running NeMo Safe Synthesizer using Docker Compose, view container logs directly: ```bash # View safe-synthesizer service logs @@ -302,6 +305,6 @@ kubectl get events -n --sort-by='.lastTimestamp' ## Related Topics -- [safe-synthesizer-101](../tutorials/safe-synthesizer-101.md): Get started with {{nss_short_name}} jobs +- [safe-synthesizer-101](../tutorials/safe-synthesizer-101.md): Get started with NeMo Safe Synthesizer jobs - [index](../tutorials/index.md): More hands-on tutorials - [reference](reference.md): Full parameter reference diff --git a/docs/safe-synthesizer/about/pii-replacement.mdx b/docs/safe-synthesizer/about/pii-replacement.mdx index 21601831a57..00a0d66c629 100644 --- a/docs/safe-synthesizer/about/pii-replacement.mdx +++ b/docs/safe-synthesizer/about/pii-replacement.mdx @@ -1,7 +1,10 @@ - - +--- +title: "PII Replacement" +description: "" +--- + + -# PII Replacement PII (Personally Identifiable Information) replacement is a critical privacy protection step that detects and replaces sensitive information in your datasets before synthesis. This ensures that the model has no chance of learning the most sensitive information like names, addresses, and other identifiers. @@ -16,7 +19,7 @@ The PII replacement pipeline operates in multiple stages: ## Detection Methods -{{nss_short_name}} supports multiple PII detection approaches: +NeMo Safe Synthesizer supports multiple PII detection approaches: ### Nemotron PII Detection diff --git a/docs/safe-synthesizer/about/reference.mdx b/docs/safe-synthesizer/about/reference.mdx index 7f00aaa0e1f..b367389eb19 100644 --- a/docs/safe-synthesizer/about/reference.mdx +++ b/docs/safe-synthesizer/about/reference.mdx @@ -1,10 +1,13 @@ - - +--- +title: "Parameters Reference" +description: "" +--- + + -# Parameters Reference This page summarizes the main configuration groups available when creating -{{nss_short_name}} jobs. For generated REST API schema details, see the +NeMo Safe Synthesizer jobs. For generated REST API schema details, see the [Safe Synthesizer API Reference](../../api/index.md#tag-safe-synthesizer). ## Top-Level Configuration diff --git a/docs/safe-synthesizer/getting-started.mdx b/docs/safe-synthesizer/getting-started.mdx index 3fcfb93012a..15737adb559 100644 --- a/docs/safe-synthesizer/getting-started.mdx +++ b/docs/safe-synthesizer/getting-started.mdx @@ -1,26 +1,30 @@ +--- +title: "Getting Started with NeMo Safe Synthesizer" +description: "" +--- +import SnippetNvidiaBuildModelProvider from "@/snippets/_snippets/nvidia-build-model-provider.mdx"; + -# Getting Started with {{nss_short_name}} -Get started with {{nss_short_name}} for generating private synthetic versions of sensitive tabular datasets. +Get started with NeMo Safe Synthesizer for generating private synthetic versions of sensitive tabular datasets. ## Prerequisites -Before using {{nss_short_name}}, complete the [{{platform_name}} Quickstart](../get-started/quickstart.md) to install the CLI/SDK and deploy the platform. +Before using NeMo Safe Synthesizer, complete the [NeMo Platform Quickstart](../get-started/quickstart.md) to install the CLI/SDK and deploy the platform. -{{nss_short_name}} has the following additional requirements: +NeMo Safe Synthesizer has the following additional requirements: - An NVIDIA GPU **on the host machine** with 80GB+ VRAM (check with `nvidia-smi`). This is separate from any GPU inside a NIM container — Safe Synthesizer training runs directly on the host. - Sufficient disk space for generated datasets (50GB+ recommended) For general platform troubleshooting (port conflicts, health checks, and so on), refer to the [main quickstart guide](../get-started/quickstart.md). ---8<-- "_snippets/nvidia-build-model-provider.md" - + --- ## Using the CLI -Interact with {{nss_short_name}} using the `nemo` CLI: +Interact with NeMo Safe Synthesizer using the `nemo` CLI: ```shell # List jobs diff --git a/docs/safe-synthesizer/tutorials/differential-privacy.mdx b/docs/safe-synthesizer/tutorials/differential-privacy.mdx index 9b6b8a926fb..a37909ec78a 100644 --- a/docs/safe-synthesizer/tutorials/differential-privacy.mdx +++ b/docs/safe-synthesizer/tutorials/differential-privacy.mdx @@ -1,7 +1,10 @@ - - +--- +title: "Differential Privacy Tutorial" +description: "" +--- + + -# Differential Privacy Tutorial Learn how to apply differential privacy to achieve the maximum level of privacy with mathematical guarantees. This tutorial explores the privacy-utility tradeoff and demonstrates how to configure differential privacy parameters for optimal results. @@ -46,7 +49,7 @@ Differential privacy (DP) provides mathematical guarantees that synthetic data d ### Record-Level vs Group-Level Privacy -By default, {{nss_short_name}} uses **record-level** differential privacy, which protects individual records. For datasets where multiple records belong to the same entity (e.g., a patient with multiple visits), you can use **group-level** privacy by setting `group_training_examples_by` to the column that identifies each entity. See [Group-Level Privacy](#group-level-privacy) in the Advanced Configuration section for a code example. +By default, NeMo Safe Synthesizer uses **record-level** differential privacy, which protects individual records. For datasets where multiple records belong to the same entity (e.g., a patient with multiple visits), you can use **group-level** privacy by setting `group_training_examples_by` to the column that identifies each entity. See [Group-Level Privacy](#group-level-privacy) in the Advanced Configuration section for a code example. **When to use group-level privacy:** - Multiple records per person/entity in your dataset @@ -57,7 +60,7 @@ By default, {{nss_short_name}} uses **record-level** differential privacy, which ## Setup -Install the {{platform_name}} SDK with Safe Synthesizer support: +Install the NeMo Platform SDK with Safe Synthesizer support: ```shell if command -v uv &> /dev/null; then @@ -381,7 +384,7 @@ check_data_requirements(len(df)) **Guidelines:** - **10,000+ records**: Ideal for differential privacy - **5,000-10,000 records**: May work, use higher epsilon -- **< 5,000 records**: Consider quality trade-offs carefully +- **\< 5,000 records**: Consider quality trade-offs carefully ### Choosing Epsilon diff --git a/docs/safe-synthesizer/tutorials/index.mdx b/docs/safe-synthesizer/tutorials/index.mdx index 4e9239d5dd3..332a1fc21ea 100644 --- a/docs/safe-synthesizer/tutorials/index.mdx +++ b/docs/safe-synthesizer/tutorials/index.mdx @@ -1,13 +1,16 @@ +--- +title: "Tutorials" +description: "" +--- -# Tutorials -Learn how to run {{nss_short_name}} jobs through hands-on tutorials to generate private synthetic versions of sensitive tabular datasets. Each tutorial provides step-by-step guidance with executable code examples. +Learn how to run NeMo Safe Synthesizer jobs through hands-on tutorials to generate private synthetic versions of sensitive tabular datasets. Each tutorial provides step-by-step guidance with executable code examples. ## Prerequisites Before starting any tutorial, ensure you have: -- [{{nss_short_name}} deployed](../getting-started.md) using Docker Compose or Helm +- [NeMo Safe Synthesizer deployed](../getting-started.md) using Docker Compose or Helm - Python environment with `nemo-platform` SDK installed: ```bash pip install nemo-platform[safe-synthesizer] @@ -18,47 +21,47 @@ Before starting any tutorial, ensure you have: ## Getting Started -
+ -- **[Safe Synthesizer 101](safe-synthesizer-101.md)** + - --- +Learn the basics with your first Safe Synthesizer job, leveraging smart defaults. This tutorial covers uploading data, running a synthesis job with PII replacement, and reviewing evaluation reports. - Learn the basics with your first Safe Synthesizer job, leveraging smart defaults. This tutorial covers uploading data, running a synthesis job with PII replacement, and reviewing evaluation reports. +**Topics covered:** +- Installing the SDK +- Connecting to Safe Synthesizer +- Using `SafeSynthesizerJobBuilder` +- Monitoring job progress +- Retrieving synthetic data and evaluation reports - **Topics covered:** - - Installing the SDK - - Connecting to Safe Synthesizer - - Using `SafeSynthesizerJobBuilder` - - Monitoring job progress - - Retrieving synthetic data and evaluation reports +beginner 20 minutes - beginner 20 minutes + -
+ --- ## Advanced Topics -
+ -- **[Differential Privacy Tutorial](differential-privacy.md)** + - --- +Apply differential privacy to achieve the maximum level of privacy with mathematical guarantees. This tutorial explores the privacy-utility tradeoff and how to configure differential privacy parameters. - Apply differential privacy to achieve the maximum level of privacy with mathematical guarantees. This tutorial explores the privacy-utility tradeoff and how to configure differential privacy parameters. +**Topics covered:** +- Understanding differential privacy concepts (epsilon, delta) +- Configuring privacy hyperparameters +- Privacy budget analysis +- Evaluating privacy-utility tradeoffs +- Interpreting privacy metrics - **Topics covered:** - - Understanding differential privacy concepts (epsilon, delta) - - Configuring privacy hyperparameters - - Privacy budget analysis - - Evaluating privacy-utility tradeoffs - - Interpreting privacy metrics +intermediate 1.5 hours - intermediate 1.5 hours + -
+ --- diff --git a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx index 032cfaa4e3e..71cc35d58b3 100644 --- a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx +++ b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx @@ -1,15 +1,18 @@ - - +--- +title: "Safe Synthesizer 101" +description: "" +--- +import SnippetNvidiaBuildModelProvider from "@/snippets/_snippets/nvidia-build-model-provider.mdx"; + -# Safe Synthesizer 101 -Learn the fundamentals of {{nss_short_name}} by creating your first Safe Synthesizer job using provided defaults. In this tutorial, you'll upload sample customer data, replace personally identifiable information, fine-tune a model, generate synthetic records, and review the evaluation report. +Learn the fundamentals of NeMo Safe Synthesizer by creating your first Safe Synthesizer job using provided defaults. In this tutorial, you'll upload sample customer data, replace personally identifiable information, fine-tune a model, generate synthetic records, and review the evaluation report. ## Prerequisites Before you begin, make sure that you have: -- Access to a deployment of {{nss_short_name}} (see [getting-started](../getting-started.md)) +- Access to a deployment of NeMo Safe Synthesizer (see [getting-started](../getting-started.md)) - **An NVIDIA GPU with 80 GB+ VRAM** — Safe Synthesizer requires GPU access for model training, even when using remote inference for other services. Verify with `nvidia-smi`. - Python environment with `nemo-platform` SDK installed - Basic understanding of Python and pandas @@ -29,7 +32,7 @@ By the end of this tutorial, you'll understand how to: ## Step 1: Install the SDK -Install the {{platform_name}} SDK with Safe Synthesizer support. Run the following command in a **terminal (shell)**: +Install the NeMo Platform SDK with Safe Synthesizer support. Run the following command in a **terminal (shell)**: ```shell if command -v uv &> /dev/null; then @@ -108,11 +111,12 @@ print(df.head()) Before running jobs, set up column classification for accurate PII detection. -!!! tip - Column classification uses an LLM to automatically detect column types and improve PII detection accuracy. Without this setup, you may see classification errors and reduced detection quality. + ---8<-- "_snippets/nvidia-build-model-provider.md" +Column classification uses an LLM to automatically detect column types and improve PII detection accuracy. Without this setup, you may see classification errors and reduced detection quality. + + ```python # Use the pre-configured NVIDIA Build model provider # This provider is set up automatically during platform deployment @@ -120,9 +124,11 @@ provider_name = "system/nvidia-build" print(f"✅ Using model provider: {provider_name}") ``` -!!! note - If you prefer not to send column data to `build.nvidia.com`, you can [deploy your own LLM](../../run-inference/tutorials/deploy-models.md) and create a custom model provider. Pass the fully-qualified provider name (`workspace/provider-name`) to `.with_classify_model_provider()` instead. + +If you prefer not to send column data to `build.nvidia.com`, you can [deploy your own LLM](../../run-inference/tutorials/deploy-models.md) and create a custom model provider. Pass the fully-qualified provider name (`workspace/provider-name`) to `.with_classify_model_provider()` instead. + + --- ## Step 6: HuggingFace Token Usage (Optional) @@ -387,6 +393,9 @@ builder = ( ) ``` -!!! warning - Disabling holdout means you won't get quality metrics like privacy scores and synthetic data quality - scores. For production use, ensure your dataset has at least 200 records. + + +Disabling holdout means you won't get quality metrics like privacy scores and synthetic data quality +scores. For production use, ensure your dataset has at least 200 records. + + diff --git a/docs/set-up/config-reference.mdx b/docs/set-up/config-reference.mdx index 884f0298c5a..45c463e789a 100644 --- a/docs/set-up/config-reference.mdx +++ b/docs/set-up/config-reference.mdx @@ -1,6 +1,9 @@ +--- +title: "NeMo Platform configuration reference" +description: "" +--- (platform-config-reference)= -# NeMo Platform configuration reference This document describes the structure and defaults for the global config file for the NeMo Platform. All sections are shown in YAML format with inline comments for description, default, and possible values. @@ -14,15 +17,15 @@ Platform-wide configuration settings. platform: # Runtime used by the platform. Used to auto-detect default backends to use. | default: 'docker' | values: 'kubernetes' | 'docker' | 'none' runtime: docker - # Base URL for the {{platform_name}} api. Used as the default URL for all services. | default: 'http://localhost:8080' + # Base URL for the NeMo Platform api. Used as the default URL for all services. | default: 'http://localhost:8080' base_url: http://localhost:8080 # Optional loopback address override for job containers to reach platform services. If not specified, automatically determined based on platform: macOS uses 'host.docker.internal', Docker containers use container hostname, Linux host network uses no override. Can be set via config file or NMP_LOOPBACK_ADDRESS env var. loopback_address: # Global image pull secrets for the platform image_pull_secrets: [] - # Docker registry for {{platform_name}} images (e.g., 'nvcr.io/nvidia/nemo-microservices'). | default: 'my-registry' + # Docker registry for NeMo Platform images (e.g., 'nvcr.io/nvidia/nemo-microservices'). | default: 'my-registry' image_registry: my-registry - # Default tag for {{platform_name}} images. | default: 'local' + # Default tag for NeMo Platform images. | default: 'local' image_tag: local # Shared Docker configuration for services using Docker backends. docker: @@ -44,13 +47,13 @@ Common configuration shared by all services. service: # Format for logs generated by the service | default: 'plain' | values: 'json' | 'plain' log_format: plain - # Logging level for the {{platform_name}}. | default: 'INFO' | values: 'DEBUG' | 'INFO' | 'WARN' | 'ERROR' + # Logging level for the NeMo Platform. | default: 'INFO' | values: 'DEBUG' | 'INFO' | 'WARN' | 'ERROR' log_level: INFO - # Scheme for the {{platform_name}} service. | default: 'http' + # Scheme for the NeMo Platform service. | default: 'http' scheme: http - # Host for the {{platform_name}} service. | default: '0.0.0.0' + # Host for the NeMo Platform service. | default: '0.0.0.0' host: 0.0.0.0 - # Port for the {{platform_name}} service. | default: 8080 + # Port for the NeMo Platform service. | default: 8080 port: 8080 ``` @@ -80,7 +83,7 @@ auth: issuer: '' # Additional valid issuers for token validation. Useful for Azure AD where access tokens use v1.0 issuer format (https://sts.windows.net/{tenant}/) while endpoints use v2.0. additional_issuers: [] - # OAuth client ID for this {{platform_name}} deployment. Used for device flow and token audience validation. | default: '' + # OAuth client ID for this NeMo Platform deployment. Used for device flow and token audience validation. | default: '' client_id: '' # Override authorization endpoint (defaults to discovery). authorization_endpoint: @@ -442,7 +445,7 @@ models: docker_timeout: 600 # Networking mode for NIM containers: 'local' (port forwarding to localhost for local dev), 'dond' (container names on shared network for quickstart), 'dind' (port forwarding to docker service for DinD setups). | default: 'local' | values: 'local' | 'dond' | 'dind' models_docker_networking_mode: local - # Docker network name for 'dond' mode. NIMs will join this network to communicate with the {{platform_name}} container. Required when MODELS_DOCKER_NETWORKING_MODE='dond'. Quickstart sets this automatically via MODELS_DOCKER_NETWORK env var. | default: '' + # Docker network name for 'dond' mode. NIMs will join this network to communicate with the NeMo Platform container. Required when MODELS_DOCKER_NETWORKING_MODE='dond'. Quickstart sets this automatically via MODELS_DOCKER_NETWORK env var. | default: '' models_docker_network: '' # Container name for 'dond' mode. Used to replace localhost in URLs passed to NIMs so they can reach services via the Docker network. Quickstart sets this automatically via MODELS_DOCKER_CONTAINER_NAME env var. | default: '' models_docker_container_name: '' @@ -794,7 +797,7 @@ evaluator: results_dir: /jobs/results # Directory path in the job container for dataset files to be downloaded to and loaded from. | default: '/jobs/datasets' dataset_dir: /jobs/datasets - # Configuration for EvalFactory integration with {{platform_name}}. + # Configuration for EvalFactory integration with NeMo Platform. evalfactory: # default: 'nvcr.io/nvidia/eval-factory/agentic_eval:26.01' agentic_eval: nvcr.io/nvidia/eval-factory/agentic_eval:26.01 diff --git a/docs/set-up/helm/backup-and-restore.mdx b/docs/set-up/helm/backup-and-restore.mdx index da35f6be224..84960042927 100644 --- a/docs/set-up/helm/backup-and-restore.mdx +++ b/docs/set-up/helm/backup-and-restore.mdx @@ -1,6 +1,8 @@ -# Backup and Restore - -{{platform_name}} use several storage systems that require backup consideration: +--- +title: "Backup and Restore" +description: "" +--- +NeMo Platform use several storage systems that require backup consideration: 1. **PostgreSQL database**: stores entity metadata and relationships 2. **Object storage**: stores model files, datasets, and other artifacts diff --git a/docs/set-up/helm/database-setup.mdx b/docs/set-up/helm/database-setup.mdx index 92e231783f9..c28de40e7b5 100644 --- a/docs/set-up/helm/database-setup.mdx +++ b/docs/set-up/helm/database-setup.mdx @@ -1,10 +1,13 @@ +--- +title: "Database Setup" +description: "" +--- -# Database Setup -The {{platform_name}} uses a SQL-based database to store entities such as workspaces, jobs, and other records. +The NeMo Platform uses a SQL-based database to store entities such as workspaces, jobs, and other records. -By default, the {{platform_name}} Helm chart deploys an embedded PostgreSQL instance (a StatefulSet using the official Postgres image) with simplified authentication to enable quick installation. This should not be used for production use, and an external PostgreSQL instance is recommended. +By default, the NeMo Platform Helm chart deploys an embedded PostgreSQL instance (a StatefulSet using the official Postgres image) with simplified authentication to enable quick installation. This should not be used for production use, and an external PostgreSQL instance is recommended. ## External PostgreSQL Database diff --git a/docs/set-up/helm/file-storage.mdx b/docs/set-up/helm/file-storage.mdx index 4849fb0d4f4..a9e60517315 100644 --- a/docs/set-up/helm/file-storage.mdx +++ b/docs/set-up/helm/file-storage.mdx @@ -1,7 +1,10 @@ +--- +title: "File Storage" +description: "" +--- -# File Storage -The {{platform_name}} Files service stores uploaded files and datasets. By default, it uses filesystem storage backed by a Kubernetes ReadWriteMany PersistentVolumeClaim (PVC). Alternatively, you can configure S3-compatible object storage. +The NeMo Platform Files service stores uploaded files and datasets. By default, it uses filesystem storage backed by a Kubernetes ReadWriteMany PersistentVolumeClaim (PVC). Alternatively, you can configure S3-compatible object storage. ## Prerequisites @@ -127,9 +130,11 @@ platformConfig: use_sdk_auth: true ``` -!!! note - Some older S3-compatible systems may require `signature_version: s3` instead of the default `s3v4`. Only change this if you encounter signature-related errors. + +Some older S3-compatible systems may require `signature_version: s3` instead of the default `s3v4`. Only change this if you encounter signature-related errors. + + ### Configuration Reference | Field | Type | Default | Description | diff --git a/docs/set-up/helm/index.mdx b/docs/set-up/helm/index.mdx index 919d31b7241..859ca343569 100644 --- a/docs/set-up/helm/index.mdx +++ b/docs/set-up/helm/index.mdx @@ -1,80 +1,75 @@ +--- +title: "Install NeMo Platform with Helm" +description: "" +--- -# Install {{platform_name}} with Helm -The {{platform_name}} is bundled in an all-in-one Helm chart that supports end-to-end microservice deployment. Use the following guides to build and manage a complete data flywheel with the {{platform_name}} on your Kubernetes either on-prem or in the cloud. +The NeMo Platform is bundled in an all-in-one Helm chart that supports end-to-end microservice deployment. Use the following guides to build and manage a complete data flywheel with the NeMo Platform on your Kubernetes either on-prem or in the cloud. -
+ -- **[Prerequisites](prerequisites.md)** + - --- +Review the prerequisites for installing the \{\{helm_chart_short_name}}. - Review the prerequisites for installing the {{helm_chart_short_name}}. +cluster-admin - cluster-admin + + -- **[Install }](install.md)** +Install the NeMo Platform using the Helm chart. - --- +cluster-admin on-prem cloud - Install the {{platform_name}} using the Helm chart. + + - cluster-admin on-prem cloud +Set up an external database for the NeMo Platform. -- **[Database Setup](database-setup.md)** +cluster-admin on-prem cloud - --- + + - Set up an external database for the {{platform_name}}. +Set up Ingress for the NeMo Platform. - cluster-admin on-prem cloud +cluster-admin on-prem cloud -- **[Ingress](ingress.md)** + + - --- +Set up persistent volumes for the NeMo Platform. - Set up Ingress for the {{platform_name}}. +cluster-admin on-prem cloud - cluster-admin on-prem cloud + + -- **[Persistent Volumes](persistent-volumes.md)** +Configure storage options for the Files service. - --- +cluster-admin on-prem cloud - Set up persistent volumes for the {{platform_name}}. + + - cluster-admin on-prem cloud +Configure high-performance east-west networking (EFA, InfiniBand, TCP-XO, SR-IOV) for multi-node training. -- **[File Storage](file-storage.md)** +cluster-admin cloud - --- + + - Configure storage options for the Files service. +Install with OpenShift-compatible configuration. - cluster-admin on-prem cloud +cluster-admin openshift -- **[Multinode Networking](multinode-networking.md)** + + - --- +Set up backup and restore configurations for the NeMo Platform. - Configure high-performance east-west networking (EFA, InfiniBand, TCP-XO, SR-IOV) for multi-node training. +cluster-admin on-prem cloud - cluster-admin cloud + -- **[OpenShift](openshift.md)** - - --- - - Install with OpenShift-compatible configuration. - - cluster-admin openshift - -- **[Backup and Restore](backup-and-restore.md)** - - --- - - Set up backup and restore configurations for the {{platform_name}}. - - cluster-admin on-prem cloud - -
\ No newline at end of file + diff --git a/docs/set-up/helm/ingress.mdx b/docs/set-up/helm/ingress.mdx index 3e3589a41c7..b95c1702696 100644 --- a/docs/set-up/helm/ingress.mdx +++ b/docs/set-up/helm/ingress.mdx @@ -1,7 +1,10 @@ +--- +title: "Ingress" +description: "" +--- -# Ingress -The {{platform_name}} Helm chart can expose the API service externally using **Kubernetes Ingress**, the **Gateway API HTTPRoute**, or on OpenShift an **OpenShift Route**. Choose one based on your cluster. +The NeMo Platform Helm chart can expose the API service externally using **Kubernetes Ingress**, the **Gateway API HTTPRoute**, or on OpenShift an **OpenShift Route**. Choose one based on your cluster. **Prerequisites:** Complete [Prerequisites](./prerequisites.md) and [Install](./install.md) (or [OpenShift](./openshift.md)) so the platform is installed. Ensure your cluster has an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) controller (e.g. Traefik, OpenShift IngressController) or a [Gateway API](https://gateway-api.sigs.k8s.io/) Gateway configured. @@ -46,7 +49,7 @@ To use standard Kubernetes [Ingress](https://kubernetes.io/docs/concepts/service Install or upgrade with your values file so the Ingress resource is created. After installation, use the URL shown in the Helm notes (`helm status nemo-platform`) or the host you configured. -**Advanced:** For multiple hostnames or different paths per host, leave `defaultHost` unset and use `ingress.hosts` (each entry has `name` and `paths`). See the [{{helm_chart_short_name}} reference](../../helm/index.md) for the full structure. +**Advanced:** For multiple hostnames or different paths per host, leave `defaultHost` unset and use `ingress.hosts` (each entry has `name` and `paths`). See the [\{\{helm_chart_short_name}} reference](../../helm/index.md) for the full structure. ## Gateway API HTTPRoute @@ -66,7 +69,7 @@ On clusters that use the [Gateway API](https://gateway-api.sigs.k8s.io/), you ca 2. Install or upgrade with this values file. The HTTPRoute will be created and bound to the specified Gateway(s). -For full option details (e.g. `filters`, `labels`, `annotations`), see the [{{helm_chart_short_name}} reference](../../helm/index.md). +For full option details (e.g. `filters`, `labels`, `annotations`), see the [\{\{helm_chart_short_name}} reference](../../helm/index.md). ## OpenShift Route @@ -86,7 +89,7 @@ On Red Hat OpenShift you can expose the API using an [OpenShift Route](https://d 3. Install or upgrade with your values file. After installation, use the URL from the Helm notes (`helm status nemo-platform`) or the host you configured. -For all options (`targetPort`, `annotations`, `labels`), see the [{{helm_chart_short_name}} reference](../../helm/index.md). +For all options (`targetPort`, `annotations`, `labels`), see the [\{\{helm_chart_short_name}} reference](../../helm/index.md). ## Related diff --git a/docs/set-up/helm/install.mdx b/docs/set-up/helm/install.mdx index f82e878acf0..07585d1a8a3 100644 --- a/docs/set-up/helm/install.mdx +++ b/docs/set-up/helm/install.mdx @@ -1,11 +1,18 @@ +--- +title: "Install {{helm_chart_short_name}}" +description: "" +--- -# Install {{helm_chart_short_name}} -!!! tip "Note: This setup is the full enterprise platform, meant for advanced use. If you're just getting started, check out [setting up a local instance of the platform](../../get-started/setup.md) instead — it's faster and easier to explore the basics." + -To deploy the {{platform_name}}, follow these steps after completing the [Prerequisites](./prerequisites.md). +Note: This setup is the full enterprise platform, meant for advanced use. If you're just getting started, check out [setting up a local instance of the platform](../../get-started/setup.md) instead — it's faster and easier to explore the basics. -1. Add the {{helm_chart_short_name}} to your local Helm repositories. + + +To deploy the NeMo Platform, follow these steps after completing the [Prerequisites](./prerequisites.md). + +1. Add the \{\{helm_chart_short_name}} to your local Helm repositories. ```sh helm repo add nmp https://helm.ngc.nvidia.com/nvidia/nemo-microservices \ @@ -16,7 +23,7 @@ To deploy the {{platform_name}}, follow these steps after completing the [Prereq helm repo update ``` -2. Review the default values in the [{{helm_chart_short_name}} reference](../../helm/index.md). To override the default values, create a custom values file. Review the following while creating your custom values file. +2. Review the default values in the [\{\{helm_chart_short_name}} reference](../../helm/index.md). To override the default values, create a custom values file. Review the following while creating your custom values file. - To configure an external database, see [Database Setup](./database-setup.md). - To configure persistent volumes for jobs and files storage, see [Persistent Volumes](./persistent-volumes.md). @@ -28,12 +35,12 @@ To deploy the {{platform_name}}, follow these steps after completing the [Prereq 3. Install the Volcano scheduler before installing the chart. This is required for customization jobs that leverage multiple nodes. ```sh -{% raw %} + kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v{{volcano_version}}/installer/volcano-development.yaml -{% endraw %} + ``` - After applying, wait for the Volcano admission webhook to finish initializing before proceeding. The webhook registers immediately with `failurePolicy: Fail`, but TLS certificate generation runs asynchronously. If you proceed before the webhook is ready, all pod creation — including the {{platform_name}} Helm install — will fail with a certificate error. + After applying, wait for the Volcano admission webhook to finish initializing before proceeding. The webhook registers immediately with `failurePolicy: Fail`, but TLS certificate generation runs asynchronously. If you proceed before the webhook is ready, all pod creation — including the NeMo Platform Helm install — will fail with a certificate error. ```sh kubectl wait --for=condition=complete job/volcano-admission-init -n volcano-system --timeout=120s @@ -72,7 +79,11 @@ To uninstall the deployment, use the following command: helm uninstall nemo-platform ``` -!!! note "`helm uninstall` intentionally does **not** remove all resources:" + + +`helm uninstall` intentionally does **not** remove all resources: + + - **PVCs** are preserved to prevent accidental data loss. Delete them manually if no longer needed. - **CRDs** are not removed by Helm design ([upstream issue](https://github.com/helm/helm/issues/4840)) to avoid destroying custom resources across the cluster. - **Completed jobs, secrets, and the namespace** may also remain. @@ -86,7 +97,11 @@ kubectl delete namespace kubectl delete crd ``` -!!! warning "Deleting CRDs removes all custom resources of those types cluster-wide. Only do this if no other workloads depend on them." + + +Deleting CRDs removes all custom resources of those types cluster-wide. Only do this if no other workloads depend on them. + + ## Troubleshooting diff --git a/docs/set-up/helm/multinode-networking.mdx b/docs/set-up/helm/multinode-networking.mdx index 097354de0a3..ceb93decc68 100644 --- a/docs/set-up/helm/multinode-networking.mdx +++ b/docs/set-up/helm/multinode-networking.mdx @@ -1,5 +1,8 @@ +--- +title: "Multinode Networking" +description: "" +--- -# Multinode Networking Multi-node GPU training requires high-bandwidth, low-latency east-west networking between nodes. Without it, NCCL falls back to standard Ethernet and training times increase significantly. The Helm chart installs [Kyverno](https://kyverno.io/) policies that automatically inject the correct networking resources for four cloud providers. @@ -10,9 +13,11 @@ Multi-node GPU training requires high-bandwidth, low-latency east-west networkin - Nodes with high-performance networking hardware (EFA, InfiniBand, etc.) - The cloud provider's device plugin or network operator deployed so that networking resources are visible to Kubernetes -!!! note - The {{platform_name}} does not provision or manage the underlying cloud networking infrastructure. Each cloud provider section below lists the cluster-level prerequisites your administrator must configure. Refer to your cloud provider's documentation for setup instructions. + +The NeMo Platform does not provision or manage the underlying cloud networking infrastructure. Each cloud provider section below lists the cluster-level prerequisites your administrator must configure. Refer to your cloud provider's documentation for setup instructions. + + ## How It Works When a job requests more than one node, the jobs controller annotates the pod with `nmp.nvidia.com/enable-multi-node-networking: "true"`. Kyverno watches for this annotation and mutates the pod spec to inject the cloud-specific device requests, environment variables, and volume mounts that NCCL needs for high-performance inter-node communication. @@ -21,9 +26,11 @@ When a job requests more than one node, the jobs controller annotates the pod wi Enable **exactly one** cloud provider in the `multinodeNetworking` section of your `values.yaml`. Enabling more than one causes a Helm install error. -!!! info - Device-count parameters (`efaDevicesPerGPU`, `rdmaDevicesPerGPU`) must match the hardware ratio of your instance type. A mismatch causes jobs to either fail scheduling or silently run without high-speed networking. + + +Device-count parameters (`efaDevicesPerGPU`, `rdmaDevicesPerGPU`) must match the hardware ratio of your instance type. A mismatch causes jobs to either fail scheduling or silently run without high-speed networking. + ### AWS (EFA) Injects [Elastic Fabric Adapter](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html) device requests (`vpc.amazonaws.com/efa`), mounts the EFA OFI library, and configures shared memory. diff --git a/docs/set-up/helm/openshift.mdx b/docs/set-up/helm/openshift.mdx index 727ef25034c..f0d6cd9d882 100644 --- a/docs/set-up/helm/openshift.mdx +++ b/docs/set-up/helm/openshift.mdx @@ -1,11 +1,14 @@ +--- +title: "OpenShift" +description: "" +--- -# OpenShift -The {{platform_name}} chart works on Red Hat OpenShift when security context overrides are applied. OpenShift’s **restricted** or **restricted-v2** Security Context Constraint (SCC) requires pods to run as non-root with explicit `runAsUser` and `runAsNonRoot`. +The NeMo Platform chart works on Red Hat OpenShift when security context overrides are applied. OpenShift’s **restricted** or **restricted-v2** Security Context Constraint (SCC) requires pods to run as non-root with explicit `runAsUser` and `runAsNonRoot`. ## Values -You can override the default values for the {{platform_name}} chart to make it compatible with OpenShift. +You can override the default values for the NeMo Platform chart to make it compatible with OpenShift. 1. **Create the OpenShift values file.** Save the following as `openshift-values.yaml`: diff --git a/docs/set-up/helm/persistent-volumes.mdx b/docs/set-up/helm/persistent-volumes.mdx index 3baad498449..7263b5eb7a6 100644 --- a/docs/set-up/helm/persistent-volumes.mdx +++ b/docs/set-up/helm/persistent-volumes.mdx @@ -1,7 +1,10 @@ +--- +title: "Persistent Volumes" +description: "" +--- -# Persistent Volumes -The {{platform_name}} uses persistent volume claims (PVCs) for jobs and files storage that can be mounted on multiple pods and nodes in read-write mode. This access mode is called ReadWriteMany (RWX) in Kubernetes. Using a ReadWriteMany-capable StorageClass is required for both jobs and files storage. +The NeMo Platform uses persistent volume claims (PVCs) for jobs and files storage that can be mounted on multiple pods and nodes in read-write mode. This access mode is called ReadWriteMany (RWX) in Kubernetes. Using a ReadWriteMany-capable StorageClass is required for both jobs and files storage. NVIDIA NIM microservices also scale, upgrade, and deploy more smoothly with an RWX-backed storage class. @@ -9,11 +12,13 @@ The platform does not manage storage classes. You must install an appropriate st ## Jobs and Files storage -The {{platform_name}} chart creates a single shared PVC for jobs and files storage, configured under `core.storage` in `values.yaml`. +The NeMo Platform chart creates a single shared PVC for jobs and files storage, configured under `core.storage` in `values.yaml`. -!!! note - As an alternative to PVC-based file storage, you can configure the Files service to use S3 object storage. See [File Storage](./file-storage.md) for S3 configuration options. When using S3 for files, the shared PVC is still required for jobs storage. + +As an alternative to PVC-based file storage, you can configure the Files service to use S3 object storage. See [File Storage](./file-storage.md) for S3 configuration options. When using S3 for files, the shared PVC is still required for jobs storage. + + ### Option 1: Create a new PVC (default) 1. Confirm the `StorageClass` exists in your cluster: @@ -49,7 +54,7 @@ When set, the chart does not create a new PVC; pods mount the named volume. ## NIM storage class -For NIM deployments launched via the {{platform_name}}, you can set the default StorageClass used by NIM PVCs via platform config. In `values.yaml`, under `platformConfig`: +For NIM deployments launched via the NeMo Platform, you can set the default StorageClass used by NIM PVCs via platform config. In `values.yaml`, under `platformConfig`: ```yaml platformConfig: @@ -91,7 +96,7 @@ For high-performance workloads: ## Azure persistent volumes -On AKS, Azure Disk (`managed-csi`) only supports ReadWriteOnce (RWO). Use **Azure Files** for the shared RWX volumes required by {{platform_name}}. PostgreSQL must use `managed-csi` because Azure Files does not support the POSIX permissions it requires. +On AKS, Azure Disk (`managed-csi`) only supports ReadWriteOnce (RWO). Use **Azure Files** for the shared RWX volumes required by NeMo Platform. PostgreSQL must use `managed-csi` because Azure Files does not support the POSIX permissions it requires. 1. Enable the [Azure Files CSI driver](https://learn.microsoft.com/en-us/azure/aks/azure-files-csi) if not already installed in your cluster. 2. Set `core.storage.storageClass` to `azurefile` (or `azurefile-csi` on AKS 1.29+) and `postgresql.persistence.storageClass` to `managed-csi`. diff --git a/docs/set-up/helm/prerequisites.mdx b/docs/set-up/helm/prerequisites.mdx index 360f47efa1a..4221fd4c50c 100644 --- a/docs/set-up/helm/prerequisites.mdx +++ b/docs/set-up/helm/prerequisites.mdx @@ -1,13 +1,16 @@ +--- +title: "Prerequisites" +description: "" +--- -# Prerequisites -Before installing the {{helm_chart_short_name}}, review requirements and create necessary secrets using your NGC API key. +Before installing the \{\{helm_chart_short_name}}, review requirements and create necessary secrets using your NGC API key. --- ## Review Requirements -To check the hardware and software specifications required for installing the {{helm_chart_short_name}}, review [](../../requirements.md). +To check the hardware and software specifications required for installing the \{\{helm_chart_short_name}}, review [](../../requirements.md). For storage and persistent volume configuration (including ReadWriteMany and StorageClass configuration), see [](persistent-volumes.md). @@ -16,14 +19,15 @@ For storage and persistent volume configuration (including ReadWriteMany and Sto ## Create NGC API Key and Secrets -You need to have an NGC account, create an NGC API key, and create secrets using the key to access and download the {{helm_chart_short_name}} and Docker images. +You need to have an NGC account, create an NGC API key, and create secrets using the key to access and download the \{\{helm_chart_short_name}} and Docker images. To learn more about using Helm charts in the NGC Catalog console in general, refer to the [Helm Charts](https://docs.nvidia.com/ngc/gpu-cloud/ngc-catalog-user-guide/index.html#helm-charts) section in the _NGC Catalog User Guide_. 1. Create an NGC API key following the instructions at [Generating NGC API Keys](https://docs.nvidia.com/ngc/latest/ngc-private-registry-user-guide.html#generating-ngc-api-keys). - !!! tip + + While creating the key, make sure that you include the following permissions at a minimum: - NGC Catalog @@ -62,4 +66,4 @@ To learn more about using Helm charts in the NGC Catalog console in general, ref --namespace ``` -For more information about various secrets you might need to create and set up through the {{helm_chart_short_name}} depending on your use case, refer to [](../../get-started/concepts/manage-secrets.md). +For more information about various secrets you might need to create and set up through the \{\{helm_chart_short_name}} depending on your use case, refer to [](../../get-started/concepts/manage-secrets.md). diff --git a/docs/set-up/index.mdx b/docs/set-up/index.mdx index 8d408d0984c..957a4e19ff5 100644 --- a/docs/set-up/index.mdx +++ b/docs/set-up/index.mdx @@ -1,41 +1,44 @@ +--- +title: "About Platform Setup" +description: "" +--- -# About Platform Setup -This section describes how to set up the {{platform_name}} on your Kubernetes cluster using the {{helm_chart_short_name}}. -With this chart, you can deploy the {{platform_name}} as a full deployment or a subset of the APIs as you need. +This section describes how to set up the NeMo Platform on your Kubernetes cluster using the \{\{helm_chart_short_name}}. +With this chart, you can deploy the NeMo Platform as a full deployment or a subset of the APIs as you need. This Platform Setup chapter is for the following personas. -- **Cloud administrators**: Manage Kubernetes clusters and compute/storage resources. Deploy {{platform_name}} to the Kubernetes clusters on premises or cloud. +- **Cloud administrators**: Manage Kubernetes clusters and compute/storage resources. Deploy NeMo Platform to the Kubernetes clusters on premises or cloud. --- -## {{helm_chart_short_name}} +## \{\{helm_chart_short_name}} -The [{{helm_chart_short_name}}](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo-microservices/helm-charts/nemo-platform) is an all-in-one Helm chart that bundles the complete {{platform_name}} ecosystem and all required dependencies for full platform deployment. +The [\{\{helm_chart_short_name}}](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo-microservices/helm-charts/nemo-platform) is an all-in-one Helm chart that bundles the complete NeMo Platform ecosystem and all required dependencies for full platform deployment. You can also customize the configuration of your installation by updating the `values.yaml` file. You can also use the pre-configured tags to install only specific microservices that you need. -For the chart assets and additional details, refer to the [{{platform_name}} Collection](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo-microservices/collections/nemo-microservices) page in the NVIDIA NGC Catalog. +For the chart assets and additional details, refer to the [NeMo Platform Collection](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo-microservices/collections/nemo-microservices) page in the NVIDIA NGC Catalog. --- -## Deploy the {{platform_name}} with Helm +## Deploy the NeMo Platform with Helm -The following sections provide detailed instructions on how to deploy the {{platform_name}} using the {{helm_chart_short_name}}. +The following sections provide detailed instructions on how to deploy the NeMo Platform using the \{\{helm_chart_short_name}}. -
+ -- **[Install }](helm/index.md)** + - --- +Install the NeMo Platform using the chart on your Kubernetes cluster. - Install the {{platform_name}} using the chart on your Kubernetes cluster. +cluster-admin - cluster-admin + -
+ --- @@ -43,18 +46,17 @@ The following sections provide detailed instructions on how to deploy the {{plat Review and manage other cluster settings. -
- -- **[OpenTelemetry](opentelemetry.md)** + - --- + - Review and configure how {{platform_name}} use Open Telemetry for observability. +Review and configure how NeMo Platform use Open Telemetry for observability. -- **[Milvus](milvus.md)** + + - --- +Review and configure how NeMo Platform uses Milvus for vector database storage. - Review and configure how {{platform_name}} uses Milvus for vector database storage. + -
\ No newline at end of file + diff --git a/docs/set-up/manage-jobs.mdx b/docs/set-up/manage-jobs.mdx index 45d59c00992..128d910915c 100644 --- a/docs/set-up/manage-jobs.mdx +++ b/docs/set-up/manage-jobs.mdx @@ -1,7 +1,10 @@ +--- +title: "Manage Jobs" +description: "" +--- -# Manage Jobs -This section describes how to configure jobs in {{platform_name}}. The Jobs service is responsible for scheduling batch jobs, collecting telemetry and managing job results. +This section describes how to configure jobs in NeMo Platform. The Jobs service is responsible for scheduling batch jobs, collecting telemetry and managing job results. ## Execution Profiles @@ -13,7 +16,7 @@ An execution profile is defined by the following attributes: - A compute provider (e.g. `cpu` or `gpu`) - An [execution backend](#execution-backends) (e.g. `docker`, `kubernetes_job`, `volcano_job`) -By default, the {{platform_name}} defines a default CPU and default GPU provider to launch CPU and GPU bound jobs. +By default, the NeMo Platform defines a default CPU and default GPU provider to launch CPU and GPU bound jobs. You can configure multiple execution profiles to suit the shape of your compute environment. For example, if you have a compute environment with heterogeneous infrastructure (e.g., two types of GPU hardware such as A100 and H200), you can define a list of execution profiles as follows: @@ -49,7 +52,7 @@ For full configuration details, see the [platform configuration reference](confi ### Default Execution Profiles -The {{platform_name}} defines a default execution profile for each execution backend depending on the platform's control plane. The default execution profile is used when no specific execution profile is specified for a job. +The NeMo Platform defines a default execution profile for each execution backend depending on the platform's control plane. The default execution profile is used when no specific execution profile is specified for a job. - Default CPU Execution Profile (`cpu`): The default execution profile for CPU based jobs. - Default GPU Execution Profile (`gpu`): The default execution profile for GPU based jobs. @@ -87,9 +90,9 @@ jobs: ## Execution Backends -Execution backends are the containerized job execution systems that {{platform_name}} jobs are scheduled to run on. Each execution backend is responsible for launching and managing the job containers, and the Jobs service communicates with the execution backend to schedule and manage jobs. +Execution backends are the containerized job execution systems that NeMo Platform jobs are scheduled to run on. Each execution backend is responsible for launching and managing the job containers, and the Jobs service communicates with the execution backend to schedule and manage jobs. -{{platform_name}} currently supports the following execution backends: +NeMo Platform currently supports the following execution backends: - Docker (`docker`) - Kubernetes Jobs (`kubernetes_job`) @@ -97,11 +100,15 @@ Execution backends are the containerized job execution systems that {{platform_n ### Docker -The {{platform_name}} supports Docker as an execution backend for CPU and GPU based jobs. When the shared GPU pool is configured (see below), Docker will use only the configured GPU devices and will not over-schedule jobs if there are not currently enough GPUs available. +The NeMo Platform supports Docker as an execution backend for CPU and GPU based jobs. When the shared GPU pool is configured (see below), Docker will use only the configured GPU devices and will not over-schedule jobs if there are not currently enough GPUs available. -**Note:** The {{platform_name}} supports Docker-based job execution by default when running the {{platform_name}} in quickstart mode, which requires no configuration. +**Note:** The NeMo Platform supports Docker-based job execution by default when running the NeMo Platform in quickstart mode, which requires no configuration. -!!! tip "If you are running GPU jobs with Docker, see [GPU Configuration](../get-started/setup.md) for information on configuring the shared GPU pool. This configuration is shared between the jobs and models services to prevent GPU resource conflicts." + + +If you are running GPU jobs with Docker, see [GPU Configuration](../get-started/setup.md) for information on configuring the shared GPU pool. This configuration is shared between the jobs and models services to prevent GPU resource conflicts. + + ```yaml jobs: @@ -125,7 +132,7 @@ jobs: ### Kubernetes Jobs -The {{platform_name}} supports Kubernetes Jobs as an execution backend for CPU and GPU based jobs. +The NeMo Platform supports Kubernetes Jobs as an execution backend for CPU and GPU based jobs. ```yaml jobs: @@ -204,7 +211,7 @@ See the [KAI Scheduler documentation](https://github.com/kai-scheduler/KAI-Sched ### Volcano Jobs -The {{platform_name}} supports Volcano Jobs as an execution backend for launching distributed GPU jobs. +The NeMo Platform supports Volcano Jobs as an execution backend for launching distributed GPU jobs. Volcano Jobs are configured using the same configuration as Kubernetes Jobs, but with the following additional configuration: diff --git a/docs/set-up/milvus.mdx b/docs/set-up/milvus.mdx index 616a7295305..3893c5100d0 100644 --- a/docs/set-up/milvus.mdx +++ b/docs/set-up/milvus.mdx @@ -1,15 +1,18 @@ +--- +title: "Milvus" +description: "" +--- -# Milvus -{{nem_short_name}} uses Milvus for vector database storage for [Retrieval evaluations](../evaluator/metrics/retriever.md) and [RAG evaluations](../evaluator/metrics/rag.md). +NeMo Evaluator uses Milvus for vector database storage for [Retrieval evaluations](../evaluator/metrics/retriever.md) and [RAG evaluations](../evaluator/metrics/rag.md). ## Configuration -To configure {{nem_short_name}} to use Milvus, set the `milvus_url` in the [platform configuration](config-reference.md): +To configure NeMo Evaluator to use Milvus, set the `milvus_url` in the [platform configuration](config-reference.md): ```yaml evaluator: milvus_url: "milvus-standalone.default.svc.cluster.local:19530" ``` -See the [platform configuration reference](config-reference.md) for the complete {{nem_short_name}} configuration reference. +See the [platform configuration reference](config-reference.md) for the complete NeMo Evaluator configuration reference. diff --git a/docs/set-up/opentelemetry.mdx b/docs/set-up/opentelemetry.mdx index 23a50bebc47..bc59232d785 100644 --- a/docs/set-up/opentelemetry.mdx +++ b/docs/set-up/opentelemetry.mdx @@ -1,15 +1,18 @@ +--- +title: "OpenTelemetry Setup" +description: "" +--- -# OpenTelemetry Setup -Set up OpenTelemetry configurations to gain visibility into the operations and performance of the {{platform_name}}. +Set up OpenTelemetry configurations to gain visibility into the operations and performance of the NeMo Platform. ## Configuration -The {{platform_name}} uses OpenTelemetry to collect telemetry data from the platform and services. It leverages common OpenTelemetry SDK [configuration options](https://opentelemetry.io/docs/languages/sdk-configuration/) to configure the platform deployment. +The NeMo Platform uses OpenTelemetry to collect telemetry data from the platform and services. It leverages common OpenTelemetry SDK [configuration options](https://opentelemetry.io/docs/languages/sdk-configuration/) to configure the platform deployment. ## Helm Configuration -The {{helm_chart_short_name}} `values.yaml` exposes OpenTelemetry SDK options to configure the platform deployment. For example, to enable OpenTelemetry for the platform: +The \{\{helm_chart_short_name}} `values.yaml` exposes OpenTelemetry SDK options to configure the platform deployment. For example, to enable OpenTelemetry for the platform: ```yaml telemetry: diff --git a/docs/set-up/security.mdx b/docs/set-up/security.mdx index fc862f36c85..a76ef912bbf 100644 --- a/docs/set-up/security.mdx +++ b/docs/set-up/security.mdx @@ -1,25 +1,30 @@ +--- +title: "Security for NeMo Platform" +description: "" +--- -# Security for {{platform_name}} -This page provides security guidelines and best practices for deploying and managing {{platform_name}} in production environments. +This page provides security guidelines and best practices for deploying and managing NeMo Platform in production environments. ## Security Considerations -- The {{platform_name}} does not impose rate limits. You must implement a rate-limiting strategy to restrict access to your application. -- The {{platform_name}} does not have an internal notion of a user. To restrict authorization to specific endpoints or users, implement an external mechanism such as an Envoy proxy. -- The {{nds_short_name}} microservice does not provide object-class-specific access controls. All items reside within a single access control boundary. -- The {{platform_name}}, by design, can access all content in the {{nds_short_name}} microservice, including LoRA adapters, training data, evaluation data, and evaluation results. +- The NeMo Platform does not impose rate limits. You must implement a rate-limiting strategy to restrict access to your application. +- The NeMo Platform does not have an internal notion of a user. To restrict authorization to specific endpoints or users, implement an external mechanism such as an Envoy proxy. +- The NeMo Data Store microservice does not provide object-class-specific access controls. All items reside within a single access control boundary. +- The NeMo Platform, by design, can access all content in the NeMo Data Store microservice, including LoRA adapters, training data, evaluation data, and evaluation results. This access is required for model evaluation. Carefully weigh the risk of data exposure from serving customized models directly to production against the overhead of a separate deployment. -- The {{platform_name}} is not intended to be internet-facing. Deploy them as the logic (middle) tier in a three-tier architecture. +- The NeMo Platform is not intended to be internet-facing. Deploy them as the logic (middle) tier in a three-tier architecture. - You are responsible for securing access to any application using the microservices. This includes: - Implementing an authentication layer between users and your application - Applying required authorization controls - Securing communication between services in your application -!!! note - Refer to the [NVIDIA Product Security](https://www.nvidia.com/en-us/security/psirt-policies/) page for information about subscribing to bulletins and updates, managing vulnerabilities, and reporting vulnerabilities. + +Refer to the [NVIDIA Product Security](https://www.nvidia.com/en-us/security/psirt-policies/) page for information about subscribing to bulletins and updates, managing vulnerabilities, and reporting vulnerabilities. + + ## Default Network Ports The following table lists the default network ports for each microservice or default database. You can override these port numbers during deployment. @@ -27,31 +32,31 @@ The following table lists the default network ports for each microservice or def | Network Port | Microservice | | --- | --- | | 443/TCP | NeMo Admission Service API | -| 3000/TCP | {{nds_short_name}} API | -| 7331/TCP | {{nem_short_name}} API | -| 7331/TCP | {{ngm_short_name}} API | -| 8000/TCP | {{nim_short_name}} API | +| 3000/TCP | NeMo Data Store API | +| 7331/TCP | NeMo Evaluator API | +| 7331/TCP | NeMo Guardrails API | +| 8000/TCP | NIM API | | 8000/TCP | NeMo Retriever Text Embedding API | | 8000/TCP | NeMo Retriever Text Reranking API | -| 8000/TCP | {{ncm_short_name}} API | -| 8000/TCP | {{nes_short_name}} API | -| 8443/TCP | {{nop_short_name}} metrics | +| 8000/TCP | NeMo Customizer API | +| 8000/TCP | NeMo Entity Store API | +| 8443/TCP | NeMo Operator metrics | | 8080/TCP | Volcano Scheduler metrics | -| 9009/TCP | {{ncm_short_name}} callback | +| 9009/TCP | NeMo Customizer callback | -By default, the {{helm_chart_short_name}} configures databases with the following network ports. +By default, the \{\{helm_chart_short_name}} configures databases with the following network ports. Alternatively, you can configure each microservice to use an external database during installation. | Network Port | Database | | --- | --- | -| 5432/TCP | {{ncm_short_name}} Database | -| 5432/TCP | {{nds_short_name}} Database | -| 5432/TCP | {{nem_short_name}} Database | -| 5432/TCP | {{nes_short_name}} Database | +| 5432/TCP | NeMo Customizer Database | +| 5432/TCP | NeMo Data Store Database | +| 5432/TCP | NeMo Evaluator Database | +| 5432/TCP | NeMo Entity Store Database | | 9091/TCP | Milvus metrics | | 19530/TCP | Milvus API | -By default, the {{helm_chart_short_name}} installs an open telemetry collector, which uses the following network ports: +By default, the \{\{helm_chart_short_name}} installs an open telemetry collector, which uses the following network ports: - 4317/TCP - 4318/TCP diff --git a/docs/studio/agents.mdx b/docs/studio/agents.mdx index c87a32db003..f66c75545b4 100644 --- a/docs/studio/agents.mdx +++ b/docs/studio/agents.mdx @@ -1,6 +1,8 @@ -# {{studio_short_name}} Agents - -Use **Agents** in the {{studio_short_name}} workspace sidebar to review and operate agent workflows managed by {{platform_name}}. +--- +title: "NeMo Studio Agents" +description: "" +--- +Use **Agents** in the NeMo Studio workspace sidebar to review and operate agent workflows managed by NeMo Platform. ## Agent List @@ -21,7 +23,7 @@ The agent side panel has two primary areas: - **Agent Details** shows the stored agent metadata and config format. - **Deployments** lists active and historical deployments for the agent, including endpoint and status. -Agents are created and updated through the `nemo agents` CLI or Agents API. {{studio_short_name}} reflects the current workspace state after the platform services refresh. +Agents are created and updated through the `nemo agents` CLI or Agents API. NeMo Studio reflects the current workspace state after the platform services refresh. ## Related Topics diff --git a/docs/studio/index.mdx b/docs/studio/index.mdx index 0d14af405ee..ff498af3e8c 100644 --- a/docs/studio/index.mdx +++ b/docs/studio/index.mdx @@ -1,16 +1,21 @@ +--- +title: "About NVIDIA NeMo Studio" +description: "" +--- -# About {{studio_long_name}} -!!! note - Studio is still in early development. Many features are missing or should be expected to change. + + +Studio is still in early development. Many features are missing or should be expected to change. -{{studio_short_name}} is the web app for AI development with NVIDIA {{platform_name}}. It provides a workspace-oriented UI for managing local platform resources, reviewing agents, running agent optimization workflows, monitoring agent telemetry, and working with datasets, jobs, and secrets. + +NeMo Studio is the web app for AI development with NVIDIA NeMo Platform. It provides a workspace-oriented UI for managing local platform resources, reviewing agents, running agent optimization workflows, monitoring agent telemetry, and working with datasets, jobs, and secrets. --- ## Getting Started -{{studio_short_name}} is included with the platform. Follow the [Setup guide](../get-started/setup.md) to start {{platform_name}}, then access Studio at `/studio` on your running server. +NeMo Studio is included with the platform. Follow the [Setup guide](../get-started/setup.md) to start NeMo Platform, then access Studio at `/studio` on your running server. ## Features diff --git a/docs/studio/monitor.mdx b/docs/studio/monitor.mdx index 22d014de125..d43aa8d5ffc 100644 --- a/docs/studio/monitor.mdx +++ b/docs/studio/monitor.mdx @@ -1,10 +1,12 @@ -# {{studio_short_name}} Monitor - -Use {{studio_short_name}} **Agents > Monitor** to inspect recent agent activity for the selected workspace. +--- +title: "NeMo Studio Monitor" +description: "" +--- +Use NeMo Studio **Agents > Monitor** to inspect recent agent activity for the selected workspace. ## Telemetry Source -{{studio_short_name}} reads agent telemetry from the `nemo-agent-telemetry` fileset. Each telemetry file contributes run summaries, token usage, and request metadata that {{studio_short_name}} aggregates for the Monitor page. +NeMo Studio reads agent telemetry from the `nemo-agent-telemetry` fileset. Each telemetry file contributes run summaries, token usage, and request metadata that NeMo Studio aggregates for the Monitor page. ## Summary Cards diff --git a/docs/studio/suggestions.mdx b/docs/studio/suggestions.mdx index a373174b89c..80299f39211 100644 --- a/docs/studio/suggestions.mdx +++ b/docs/studio/suggestions.mdx @@ -1,10 +1,12 @@ -# {{studio_short_name}} Suggestions +--- +title: "NeMo Studio Suggestions" +description: "" +--- +Use NeMo Studio **Agents > Suggestions** to review optimizer suggestions for agents in the selected workspace. -Use {{studio_short_name}} **Agents > Suggestions** to review optimizer suggestions for agents in the selected workspace. +## What NeMo Studio Shows -## What {{studio_short_name}} Shows - -{{studio_short_name}} loads the latest optimizer snapshot and suggestions from the platform files service. Suggestions can be scoped to a workspace or to a specific agent. +NeMo Studio loads the latest optimizer snapshot and suggestions from the platform files service. Suggestions can be scoped to a workspace or to a specific agent. The Suggestions page summarizes: @@ -26,14 +28,14 @@ Use the table filters to narrow suggestions by: ## Run an Optimization Pass -If there are no suggestions, or if the latest snapshot is stale, {{studio_short_name}} starts a new optimizer pass for the workspace. The optimizer analyzes deployed agents and writes updated suggestions back to the platform files service. +If there are no suggestions, or if the latest snapshot is stale, NeMo Studio starts a new optimizer pass for the workspace. The optimizer analyzes deployed agents and writes updated suggestions back to the platform files service. ## Apply Suggestions -{{studio_short_name}} can apply supported suggestions from the Suggestions page. Model optimization suggestions may ask you to choose an evaluation config before applying the change. +NeMo Studio can apply supported suggestions from the Suggestions page. Model optimization suggestions may ask you to choose an evaluation config before applying the change. ## Next Steps -- [{{studio_short_name}} Agents](agents.md): review, deploy, chat with, and delete agents from {{studio_short_name}}. -- [{{studio_short_name}} Monitor](monitor.md): inspect recent agent telemetry, token usage, and inference logs. +- [NeMo Studio Agents](agents.md): review, deploy, chat with, and delete agents from NeMo Studio. +- [NeMo Studio Monitor](monitor.md): inspect recent agent telemetry, token usage, and inference logs. - [Optimize Agents](../agents/optimization.md): run CLI-driven optimization and review the underlying checks. diff --git a/docs/support-matrix.mdx b/docs/support-matrix.mdx index 35a740d55c8..bd53c92dda8 100644 --- a/docs/support-matrix.mdx +++ b/docs/support-matrix.mdx @@ -1,8 +1,10 @@ -# Support Matrix +--- +title: "Support Matrix" +description: "" +--- +This matrix defines the supported OSS local-install target for NeMo Platform 0.1.0. It applies to the Python package, CLI, SDK, and local services started by `nemo setup`. -This matrix defines the supported OSS local-install target for {{platform_name}} {{ release }}. It applies to the Python package, CLI, SDK, and local services started by `nemo setup`. - -Docker Compose, Helm, Kubernetes, and OpenShift deployment paths are not part of the OSS {{ release }} documentation scope. +Docker Compose, Helm, Kubernetes, and OpenShift deployment paths are not part of the OSS 0.1.0 documentation scope. ## Host Platforms @@ -13,7 +15,7 @@ Docker Compose, Helm, Kubernetes, and OpenShift deployment paths are not part of | Linux | Ubuntu 22.04 LTS, Ubuntu 24.04 LTS, RHEL 9, Rocky Linux 9, Debian 12 | Supported for CLI, SDK, local services, and NVIDIA GPU workloads. | | macOS | [macOS Tahoe 26](https://support.apple.com/en-us/122868) and macOS Sequoia 15 | Supported for CLI, SDK, local services, and cloud/provider workflows. Local NVIDIA GPU workloads are not supported on macOS. | | Architecture | x86_64 Linux; Apple Silicon and Intel macOS | NVIDIA GPU workloads require x86_64 Linux. | -| Windows and WSL | Not in the OSS {{ release }} support matrix | Use a supported Linux or macOS host for the local install path. | +| Windows and WSL | Not in the OSS 0.1.0 support matrix | Use a supported Linux or macOS host for the local install path. | ## Local Runtime @@ -53,7 +55,7 @@ Docker Compose, Helm, Kubernetes, and OpenShift deployment paths are not part of ## Out of Scope -The following are not part of the OSS {{ release }} local support matrix: +The following are not part of the OSS 0.1.0 local support matrix: - Native Windows local install. - Docker Compose, Helm, Kubernetes, and OpenShift deployment guides. diff --git a/docs/troubleshooting/cluster-setup.mdx b/docs/troubleshooting/cluster-setup.mdx index d8fd4a012f8..eabccc83d8b 100644 --- a/docs/troubleshooting/cluster-setup.mdx +++ b/docs/troubleshooting/cluster-setup.mdx @@ -1,14 +1,10 @@ -# Troubleshooting {{platform_name}} Deployment on Kubernetes +--- +title: "Troubleshooting NeMo Platform Deployment on Kubernetes" +description: "" +--- +Use this documentation to troubleshoot issues that can arise while you deploy and run the NeMo Platform on Kubernetes. -Use this documentation to troubleshoot issues that can arise while you deploy and run the {{platform_name}} on Kubernetes. - ## Network Issues @@ -72,35 +68,4 @@ Sorry, home directories outside of /home needs configuration. To configure snap to use the correct home directory, follow the instructions at [Home directories outside of /home](https://snapcraft.io/docs/home-outside-home) in the Snap documentation. - diff --git a/docs/troubleshooting/customizer.mdx b/docs/troubleshooting/customizer.mdx index ee4152986e2..b21993c7649 100644 --- a/docs/troubleshooting/customizer.mdx +++ b/docs/troubleshooting/customizer.mdx @@ -1,5 +1,7 @@ -# Troubleshooting {{ncm_short_name}} - +--- +title: "Troubleshooting NeMo Customizer" +description: "" +--- **Job fails during model download:** - Verify the HuggingFace token secret is configured correctly - Accept the model's license on the [HuggingFace model page](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) diff --git a/docs/troubleshooting/data-designer.mdx b/docs/troubleshooting/data-designer.mdx index e3714ee8432..542d23e5a49 100644 --- a/docs/troubleshooting/data-designer.mdx +++ b/docs/troubleshooting/data-designer.mdx @@ -1,7 +1,10 @@ +--- +title: "Troubleshoot Data Designer" +description: "" +--- -# Troubleshoot Data Designer -Learn how to troubleshoot common issues with {{ndd_short_name}} in a local {{platform_name}} setup. +Learn how to troubleshoot common issues with NeMo Data Designer in a local NeMo Platform setup. ## Prerequisites @@ -47,7 +50,7 @@ Look for startup errors, provider authentication failures, port conflicts, or Da ## Next steps - Tutorial: [Data Designer Tutorials](../data-designer/tutorials/index.md) covers common generation workflows. -- How-to: [Set Up {{platform_name}}](../get-started/setup.md) covers local setup and service startup. +- How-to: [Set Up NeMo Platform](../get-started/setup.md) covers local setup and service startup. - Explanation: [Data Designer Overview](../data-designer/index.md) explains configuration, execution, and service behavior. - Reference: [API Reference](../api/index.md#tag-data-designer) lists Data Designer endpoints. - Troubleshooting: [Troubleshooting Overview](index.md) links to other service troubleshooting pages. diff --git a/docs/troubleshooting/evaluator.mdx b/docs/troubleshooting/evaluator.mdx index 911734c0ce2..38b20bcac41 100644 --- a/docs/troubleshooting/evaluator.mdx +++ b/docs/troubleshooting/evaluator.mdx @@ -1,11 +1,16 @@ +--- +title: "Troubleshooting NeMo Evaluator" +description: "" +--- -# Troubleshooting {{nem_short_name}} -Use this documentation to troubleshoot issues that can arise when you work with [{{nem_long_name}}](../evaluator/index.md). +Use this documentation to troubleshoot issues that can arise when you work with [NVIDIA NeMo Evaluator](../evaluator/index.md). + + -!!! tip - You can [get metric logs](../evaluator/metrics/job-management.md) or [get benchmark logs](../evaluator/benchmarks/job-management.md) for `COMPLETED` or `FAILED` jobs and use them to help troubleshoot. +You can [get metric logs](../evaluator/metrics/job-management.md) or [get benchmark logs](../evaluator/benchmarks/job-management.md) for `COMPLETED` or `FAILED` jobs and use them to help troubleshoot. + --- @@ -35,12 +40,12 @@ Not all models make good judges. If the judge produces inconsistent output and d Incoming request body={'messages': [{'content': 'The output string did not satisfy the constraints given in the prompt. Fix the output string and return it.\nPlease return the output in a JSON format that complies with the following schema as specified in JSON Schema:\n{"properties": {"text": {"title": "Text", "type": "string"}}, "required": ["text"], "title": "StringIO", "type": "object"} ``` -## Dataset {dataset} is not in the expected format; it needs to have the files_url property set +## Dataset \{dataset} is not in the expected format; it needs to have the files_url property set This means that either the `files_url` is not provided as part of the dataset specification in the config, or that the `files_url` is not provided in the expected format. The dataset must be a JSON object with the `files_url` property set, -pointing to the path of the file in the {{nds_short_name}} in the format: `hf://datasets///`. +pointing to the path of the file in the NeMo Data Store in the format: `hf://datasets///`. ## Error connecting to inference server @@ -49,18 +54,18 @@ This means that for a custom evaluation, the target LLM endpoint is unable to co ## Inference SSL Error -An evaluation job that uses an HTTPS model endpoint can fail if the endpoint certificate or DNS name is not trusted by the local environment. Verify that the model URL is reachable from the host running {{platform_name}} and that the endpoint presents a valid certificate for its hostname. +An evaluation job that uses an HTTPS model endpoint can fail if the endpoint certificate or DNS name is not trusted by the local environment. Verify that the model URL is reachable from the host running NeMo Platform and that the endpoint presents a valid certificate for its hostname. ``` Error: HTTPSConnectionPool(host="", port=443): Max retries exceeded with url: /v1/chat/completions (Caused by SSLError(SSLError)) ``` -## Error occurred while checking the existence of file {file_ref} on {{nds_short_name}} +## Error occurred while checking the existence of file \{file_ref} on NeMo Data Store -This could mean that the dataset is not specified correctly, or that the {{nds_short_name}} itself is unresponsive. +This could mean that the dataset is not specified correctly, or that the NeMo Data Store itself is unresponsive. -- Verify that the files URL is correct and that the dataset and file exists in the {{nds_short_name}}. -- Verify that the {{nds_short_name}} is responsive and reachable. +- Verify that the files URL is correct and that the dataset and file exists in the NeMo Data Store. +- Verify that the NeMo Data Store is responsive and reachable. If the error contains the string `Dataset {file_ref} is not present on datastore`, it means that the datastore is responsive, but the file reference does not exist. @@ -91,7 +96,11 @@ If you changed the platform URL, use the value configured in `NMP_BASE_URL` or i To troubleshoot an evaluation job that has failed, download the evaluation result archive and inspect the job logs. -!!! warning "These are advanced troubleshooting steps that should only be done after all other troubleshooting fails." + + +These are advanced troubleshooting steps that should only be done after all other troubleshooting fails. + + ### Evaluation Job Logs @@ -114,36 +123,43 @@ Log files can be found in the results folder with the file extension `*.log`. ## Skip validation checks -When you launch an evaluation job, {{nem_short_name}} performs availability checks (for example, checking if the dataset and files exist in {{nds_short_name}}). +When you launch an evaluation job, NeMo Evaluator performs availability checks (for example, checking if the dataset and files exist in NeMo Data Store). To speed up job launch, or due to strict constraints of validation checks, you can pass the query parameter `skip_validation_checks` during job launch. Use the following code to create an evaluation job that skips validation checks. -=== "curl" - ```bash - curl -X 'POST' \ - 'https://${EVALUATOR_BASE_URL}/v1/evaluation/jobs?skip_validation_checks=True' \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ + + + + +```bash +curl -X 'POST' \ +'https://${EVALUATOR_BASE_URL}/v1/evaluation/jobs?skip_validation_checks=True' \ +-H 'accept: application/json' \ +-H 'Content-Type: application/json' \ +-d '{ +"namespace": "my-organization", +"target": "", +"config": "" +}' +``` + + + + +```python +data = { "namespace": "my-organization", "target": "", - "config": "" - }' - ``` + "config": "", +} -=== "Python" - ```python - data = { - "namespace": "my-organization", - "target": "", - "config": "", - } +endpoint = f"{EVALUATOR_BASE_URL}/v1/evaluation/jobs?skip_validation_checks=True" - endpoint = f"{EVALUATOR_BASE_URL}/v1/evaluation/jobs?skip_validation_checks=True" +response = requests.post(endpoint, json=data).json() +``` - response = requests.post(endpoint, json=data).json() - ``` + - + diff --git a/docs/troubleshooting/guardrails.mdx b/docs/troubleshooting/guardrails.mdx index 3a75b24ee11..fdd32de1d06 100644 --- a/docs/troubleshooting/guardrails.mdx +++ b/docs/troubleshooting/guardrails.mdx @@ -1,10 +1,12 @@ -# Troubleshooting {{ngm_short_name}} - -Use this documentation to troubleshoot issues that can arise when you work with [{{ngm_long_name}}]. +--- +title: "Troubleshooting NeMo Guardrails" +description: "" +--- +Use this documentation to troubleshoot issues that can arise when you work with [NVIDIA NeMo Guardrails]. ## API Catalog Endpoint Issues -Several sample configurations in the documentation use NIMs with model endpoints hosted at . +Several sample configurations in the documentation use NIMs with model endpoints hosted at [https://integrate.api.nvidia.com/v1](https://integrate.api.nvidia.com/v1). The purpose of using the endpoints is to avoid deploying NIMs locally to reduce the initial effort to get started. Perform the following steps to troubleshoot configurations that use model endpoints from the API catalog: @@ -17,14 +19,91 @@ Perform the following steps to troubleshoot configurations that use model endpoi 1. Access a model, such as the Llama 3.1 8B NemoGuard Content Safety, from the model endpoint: - --8<-- "troubleshooting/_snippets/input/guardrails-cmds.sh" + ```bash + + # start-content-safety + + curl https://integrate.api.nvidia.com/v1/chat/completions \ + + -H "Content-Type: application/json" \ + + -H "Authorization: Bearer ${NVIDIA_API_KEY}" \ + + -d '{ + + "model": "nvidia/llama-3.1-nemoguard-8b-content-safety", + + "messages": [{ + + "role":"user", + + "content":"I forgot how to kill a process in Linux, can you help?" + + }], + + "stream": false + + }' + + # end-content-safety + + ``` + + + + ```json + + { + + "id": "chat-2aaefebc0963475d862f1fd202f172b2", + + "object": "chat.completion", - ??? "Example Output" + "created": 1743540105, - --8<-- "troubleshooting/_snippets/output/guardrails-content-safety.json" + "model": "nvidia/llama-3.1-nemoguard-8b-content-safety", + "choices": [ + + { + + "index": 0, + + "message": { + + "role": "assistant", + + "content": "{\"User Safety\": \"safe\"} " + + }, + + "logprobs": null, + + "finish_reason": "stop", + + "stop_reason": null + + } + + ], + + "usage": { + + "prompt_tokens": 408, + + "total_tokens": 416, + + "completion_tokens": 8 + + }, + + "prompt_logprobs": null + + } + + ``` If your request is not successful, such as a 401 or 403 HTTP status code, - go to to generate a new API key. + go to [https://build.nvidia.com/settings/api-keys](https://build.nvidia.com/settings/api-keys) to generate a new API key. 1. Access other models. In the preceding sample `curl` command, replace the `model` value with one of the following: diff --git a/docs/troubleshooting/index.mdx b/docs/troubleshooting/index.mdx index 4a1435fb9c8..d603389b441 100644 --- a/docs/troubleshooting/index.mdx +++ b/docs/troubleshooting/index.mdx @@ -1,30 +1,29 @@ -# Troubleshooting Guide +--- +title: "Troubleshooting Guide" +description: "" +--- +This section provides troubleshooting information for the NeMo Platform. + -This section provides troubleshooting information for the {{platform_name}}. -
+ -- **[Evaluator Troubleshooting](evaluator.md)** +Learn how to troubleshoot common issues with NeMo Evaluator. - --- + + - Learn how to troubleshoot common issues with {{nem_short_name}}. +Learn how to troubleshoot common issues with NeMo Guardrails. -- **[Guardrails Troubleshooting](guardrails.md)** + + - --- +Learn how to troubleshoot common issues with NeMo Data Designer. - Learn how to troubleshoot common issues with {{ngm_short_name}}. + + -- **[Data Designer Troubleshooting](data-designer.md)** +Learn how to troubleshoot common issues with Studio. - --- + - Learn how to troubleshoot common issues with {{ndd_short_name}}. - -- **[Studio Troubleshooting](studio.md)** - - --- - - Learn how to troubleshoot common issues with Studio. - -
+ diff --git a/docs/troubleshooting/studio.mdx b/docs/troubleshooting/studio.mdx index 7043b4c579b..1ef869a9a15 100644 --- a/docs/troubleshooting/studio.mdx +++ b/docs/troubleshooting/studio.mdx @@ -1,7 +1,10 @@ +--- +title: "Troubleshooting Studio" +description: "" +--- -# Troubleshooting Studio -Learn how to troubleshoot common issues with {{studio_short_name}}. +Learn how to troubleshoot common issues with NeMo Studio. ## Datasets or Jobs Not Appearing in Studio diff --git a/docs/work/guardrails/README.mdx b/docs/work/guardrails/README.mdx index 47ddb7803b4..bbe2e70db63 100644 --- a/docs/work/guardrails/README.mdx +++ b/docs/work/guardrails/README.mdx @@ -1,3 +1,7 @@ +--- +title: "" +description: "" +--- docker run --rm -it \ -v $(pwd)/work/guardrails:/config-store \ -e CONFIG_STORE_PATH=/config-store \ From 790bd0023af50d1ab21d44bfa14df1d2985b5261 Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Fri, 22 May 2026 10:46:45 -0400 Subject: [PATCH 04/26] docs(fern-migrate): relocate snippets to docs/fern/snippets/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move MkDocs-style snippet partials (the files referenced by --8<-- includes) under docs/fern/snippets/ so the @/snippets/* imports the C3 converter emitted resolve via Fern's `@/` alias. Pure partials (renamed, history preserved): - docs/_snippets/* → docs/fern/snippets/_snippets/* - docs/customizer/tutorials/_snippets/customizer-prereqs.mdx → docs/fern/snippets/customizer/tutorials/_snippets/customizer-prereqs.mdx - docs/data-designer/_snippets/{job-results,preview-results}.mdx → docs/fern/snippets/data-designer/_snippets/* Page-plus-snippet (kept at their canonical doc location AND copied under snippets for include resolution): - docs/evaluator/benchmarks/{discover-industry-benchmarks,hf-secret}.mdx → also at docs/fern/snippets/evaluator/benchmarks/ (without front matter, since snippets render inline). External include (out-of-tree pre-existing SDK README mirrored as a partial): - sdk/python/overrides/nemo-platform/README/03_usage.md → docs/fern/snippets/sdk/python/overrides/nemo-platform/README/03_usage.mdx The guardrails snippet *images* (car-audio-theft.jpg, street-scene.jpg) stay at docs/guardrails/_snippets/input/ — they're referenced by multimodal-data tutorial via a relative `../_snippets/input/` path that already resolves. Signed-off-by: Lawrence Lane --- .../snippets}/_snippets/cli-summary.mdx | 0 .../snippets}/_snippets/naming-rules.mdx | 0 .../_snippets/nvidia-build-model-provider.mdx | 0 .../_snippets/tutorials/cli-sdk-setup.mdx | 0 .../snippets}/_snippets/tutorials/prereqs.mdx | 0 .../_snippets/customizer-prereqs.mdx | 0 .../data-designer/_snippets/job-results.mdx | 0 .../_snippets/preview-results.mdx | 0 .../discover-industry-benchmarks.mdx | 51 +++ .../evaluator/benchmarks/hf-secret.mdx | 11 + .../nemo-platform/README/03_usage.mdx | 408 ++++++++++++++++++ 11 files changed, 470 insertions(+) rename docs/{ => fern/snippets}/_snippets/cli-summary.mdx (100%) rename docs/{ => fern/snippets}/_snippets/naming-rules.mdx (100%) rename docs/{ => fern/snippets}/_snippets/nvidia-build-model-provider.mdx (100%) rename docs/{ => fern/snippets}/_snippets/tutorials/cli-sdk-setup.mdx (100%) rename docs/{ => fern/snippets}/_snippets/tutorials/prereqs.mdx (100%) rename docs/{ => fern/snippets}/customizer/tutorials/_snippets/customizer-prereqs.mdx (100%) rename docs/{ => fern/snippets}/data-designer/_snippets/job-results.mdx (100%) rename docs/{ => fern/snippets}/data-designer/_snippets/preview-results.mdx (100%) create mode 100644 docs/fern/snippets/evaluator/benchmarks/discover-industry-benchmarks.mdx create mode 100644 docs/fern/snippets/evaluator/benchmarks/hf-secret.mdx create mode 100644 docs/fern/snippets/sdk/python/overrides/nemo-platform/README/03_usage.mdx diff --git a/docs/_snippets/cli-summary.mdx b/docs/fern/snippets/_snippets/cli-summary.mdx similarity index 100% rename from docs/_snippets/cli-summary.mdx rename to docs/fern/snippets/_snippets/cli-summary.mdx diff --git a/docs/_snippets/naming-rules.mdx b/docs/fern/snippets/_snippets/naming-rules.mdx similarity index 100% rename from docs/_snippets/naming-rules.mdx rename to docs/fern/snippets/_snippets/naming-rules.mdx diff --git a/docs/_snippets/nvidia-build-model-provider.mdx b/docs/fern/snippets/_snippets/nvidia-build-model-provider.mdx similarity index 100% rename from docs/_snippets/nvidia-build-model-provider.mdx rename to docs/fern/snippets/_snippets/nvidia-build-model-provider.mdx diff --git a/docs/_snippets/tutorials/cli-sdk-setup.mdx b/docs/fern/snippets/_snippets/tutorials/cli-sdk-setup.mdx similarity index 100% rename from docs/_snippets/tutorials/cli-sdk-setup.mdx rename to docs/fern/snippets/_snippets/tutorials/cli-sdk-setup.mdx diff --git a/docs/_snippets/tutorials/prereqs.mdx b/docs/fern/snippets/_snippets/tutorials/prereqs.mdx similarity index 100% rename from docs/_snippets/tutorials/prereqs.mdx rename to docs/fern/snippets/_snippets/tutorials/prereqs.mdx diff --git a/docs/customizer/tutorials/_snippets/customizer-prereqs.mdx b/docs/fern/snippets/customizer/tutorials/_snippets/customizer-prereqs.mdx similarity index 100% rename from docs/customizer/tutorials/_snippets/customizer-prereqs.mdx rename to docs/fern/snippets/customizer/tutorials/_snippets/customizer-prereqs.mdx diff --git a/docs/data-designer/_snippets/job-results.mdx b/docs/fern/snippets/data-designer/_snippets/job-results.mdx similarity index 100% rename from docs/data-designer/_snippets/job-results.mdx rename to docs/fern/snippets/data-designer/_snippets/job-results.mdx diff --git a/docs/data-designer/_snippets/preview-results.mdx b/docs/fern/snippets/data-designer/_snippets/preview-results.mdx similarity index 100% rename from docs/data-designer/_snippets/preview-results.mdx rename to docs/fern/snippets/data-designer/_snippets/preview-results.mdx diff --git a/docs/fern/snippets/evaluator/benchmarks/discover-industry-benchmarks.mdx b/docs/fern/snippets/evaluator/benchmarks/discover-industry-benchmarks.mdx new file mode 100644 index 00000000000..9aae14e9c21 --- /dev/null +++ b/docs/fern/snippets/evaluator/benchmarks/discover-industry-benchmarks.mdx @@ -0,0 +1,51 @@ +### Discover Industry Benchmarks + +Discover industry benchmarks available to use for your evaluation job within the `system` workspace. List all industry benchmarks or filter by label category. + + + +The `system` workspace is a reserved workspace for NeMo Platform that contains ready-to-use benchmarks representing industry benchmarks with published datasets and metrics. + + + + +**Initialization:** This example uses `NeMoPlatform()` with no arguments so the SDK reads your active CLI context (set by `nemo auth login`). The `workspace="system"` is passed per-call to access the reserved system workspace. For the standard local initialization pattern, see [CLI and SDK initialization](../../get-started/setup.md#setup-init). + + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform() + +benchmarks = client.evaluation.benchmarks.list(workspace="system") + +print(f"{benchmarks.pagination.total_results} benchmarks") +for benchmark in benchmarks: + print(f"{benchmark.name}: {benchmark.description}") + +benchmarks = client.evaluation.benchmarks.list( + workspace="system", page=2, page_size=100 +) +for benchmark in benchmarks: + print(f"{benchmark.name}: {benchmark.description}") + +# Filter by evaluation category label +filtered_benchmarks = client.evaluation.benchmarks.list( + workspace="system", + extra_query={"filter[data.labels.eval_category]": "advanced_reasoning"}, +) +print(filtered_benchmarks) +``` + +| Category Label | Description | +|----------------|-------------| +| `agentic` | Evaluate the performance of agent-based or multi-step reasoning models, especially in scenarios requiring planning, tool use, and iterative reasoning. | +| `advanced_reasoning` | Evaluate reasoning capabilities of large language models through complex tasks. | +| `code` | Evaluate code generation capabilities using functional correctness benchmarks that test synthesis of working programs. | +| `content_safety` | Evaluate model safety risks including vulnerability to generate harmful, biased, or misleading content. | +| `instruction_following` | Evaluate the ability to follow explicit formatting and structural instructions | +| `language_understanding` | Evaluate knowledge and reasoning across diverse subjects in different languages. | +| `math` | Evaluate mathematical reasoning abilities. | +| `question_answering` | Evaluate the ability to generate answers to questions. | +| `rag` | Evaluate the quality of RAG pipelines by measuring both retrieval and answer generation performance. | +| `retrieval` | Evaluate the quality of document retriever pipelines. | diff --git a/docs/fern/snippets/evaluator/benchmarks/hf-secret.mdx b/docs/fern/snippets/evaluator/benchmarks/hf-secret.mdx new file mode 100644 index 00000000000..303fd97c9a9 --- /dev/null +++ b/docs/fern/snippets/evaluator/benchmarks/hf-secret.mdx @@ -0,0 +1,11 @@ +Most benchmarks require a Hugging Face token (`hf_token`) to access gated datasets. Create this secret before running evaluations: + +```python +import os + +client.secrets.create( + workspace=workspace, + name="hf_token", + value=os.getenv("HF_TOKEN", ""), +) +``` diff --git a/docs/fern/snippets/sdk/python/overrides/nemo-platform/README/03_usage.mdx b/docs/fern/snippets/sdk/python/overrides/nemo-platform/README/03_usage.mdx new file mode 100644 index 00000000000..4108f78b2ab --- /dev/null +++ b/docs/fern/snippets/sdk/python/overrides/nemo-platform/README/03_usage.mdx @@ -0,0 +1,408 @@ +## Usage + +This section describes how to use the NeMo Platform Python SDK. + +### Import the Main Client Class + +Import the main client class from the `nemo_platform` package and create a client instance as follows: + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform( + base_url="http://nemo.test", +) + +# Sample API call +page = client.workspaces.list() +print(page.data) +``` + +After creating the client instance, you can use the client to interact with the NeMo Platform APIs. + +## Async Usage + +If you want to use the asynchronous client, simply import `AsyncNeMoPlatform` instead of `NeMoPlatform` and use `await` with each API call: + +```python +import asyncio +from nemo_platform import AsyncNeMoPlatform + +client = AsyncNeMoPlatform( + base_url="http://nemo.test", +) + +# Sample API call +async def main() -> None: + page = await client.workspaces.list() + print(page.data) + + +asyncio.run(main()) +``` + +Functionality between the synchronous and asynchronous clients is otherwise identical. + +### With aiohttp + +By default, the async client uses `httpx` for HTTP requests. However, for improved concurrency performance you may also use `aiohttp` as the HTTP backend. + +You can enable this by installing `aiohttp`: + +```sh +pip install 'nemo-platform[aiohttp]' +``` + +Then you can enable it by instantiating the client with `http_client=DefaultAioHttpClient()`: + +```python +import asyncio +from nemo_platform import DefaultAioHttpClient +from nemo_platform import AsyncNeMoPlatform + + +async def main() -> None: + async with AsyncNeMoPlatform( + base_url="http://nemo.test", + http_client=DefaultAioHttpClient(), + ) as client: + page = await client.workspaces.list() + print(page.data) + + +asyncio.run(main()) +``` + +## Using Types + +Nested request parameters are [TypedDicts](https://docs.python.org/3/library/typing.html#typing.TypedDict). Responses are [Pydantic models](https://docs.pydantic.dev) which also provide helper methods for things like: + +- Serializing back into JSON, `model.to_json()` +- Converting to a dictionary, `model.to_dict()` + +Typed requests and responses provide autocomplete and documentation within your editor. If you would like to see type errors in VS Code to help catch bugs, set `python.analysis.typeCheckingMode` to `basic`. + +## Pagination + +List methods in the NeMo Platform API are paginated. + +This library provides auto-paginating iterators with each list response, so you do not have to request successive pages manually: + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform( + base_url="http://nemo.test", +) + +all_jobs = [] +# Automatically fetches more pages as needed. +for job in client.jobs.list(workspace="my-workspace"): + # Do something with job here + all_jobs.append(job) +print(all_jobs) +``` + +Or, asynchronously: + +```python +import asyncio +from nemo_platform import AsyncNeMoPlatform + +client = AsyncNeMoPlatform( + base_url="http://nemo.test", +) + +async def main() -> None: + all_jobs = [] + # Iterate through items across all pages, issuing requests as needed. + async for job in client.jobs.list(workspace="my-workspace"): + all_jobs.append(job) + print(all_jobs) + + +asyncio.run(main()) +``` + +Alternatively, you can use the `.has_next_page()`, `.next_page_info()`, or `.get_next_page()` methods for more granular control working with pages: + +```python +first_page = await client.jobs.list(workspace="my-workspace") +if first_page.has_next_page(): + print(f"will fetch next page using these details: {first_page.next_page_info()}") + next_page = await first_page.get_next_page() + print(f"number of items we just fetched: {len(next_page.data)}") + +# Remove `await` for non-async usage. +``` + +Or just work directly with the returned data: + +```python +first_page = await client.jobs.list(workspace="my-workspace") +for job in first_page.data: + print(job.id) + +# Remove `await` for non-async usage. +``` + +## Nested Parameters + +Nested parameters are dictionaries, typed using `TypedDict`, for example: + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform( + base_url="http://nemo.test", +) + +audit_config = client.audit.configs.create( + workspace="my-workspace", + name="name", + plugins={ + "buffs": {}, + "buffs_include_original_prompt": False, + "detector_spec": "auto", + "detectors": {}, + "extended_detectors": False, + "generators": {}, + "harnesses": {}, + "probe_spec": "all", + "probes": {"encoding": {"payloads": "bar"}}, + }, + reporting={}, + run={}, + system={}, +) +print(audit_config.plugins) +``` + +## Handling Errors + +The library raises errors when it cannot connect to the API or when the API returns a non-success status code. + +When the library cannot connect to the API (for example, due to network connection problems or a timeout), it raises a subclass of `nemo_platform.APIConnectionError`. + +When the API returns a non-success status code (that is, 4xx or 5xx +response), it raises a subclass of `nemo_platform.APIStatusError`, containing `status_code` and `response` properties. + +All errors inherit from `nemo_platform.APIError`. + +```python +import nemo_platform +from nemo_platform import NeMoPlatform + +client = NeMoPlatform() + +try: + client.workspaces.list() +except nemo_platform.APIConnectionError as e: + print("The server could not be reached") + print(e.__cause__) # an underlying Exception, likely raised within httpx. +except nemo_platform.RateLimitError as e: + print("A 429 status code was received; we should back off a bit.") +except nemo_platform.APIStatusError as e: + print("Another non-200-range status code was received") + print(e.status_code) + print(e.response) +``` + +Error codes are as follows: + +| Status Code | Error Type | +| ----------- | -------------------------- | +| 400 | `BadRequestError` | +| 401 | `AuthenticationError` | +| 403 | `PermissionDeniedError` | +| 404 | `NotFoundError` | +| 422 | `UnprocessableEntityError` | +| 429 | `RateLimitError` | +| >=500 | `InternalServerError` | +| N/A | `APIConnectionError` | + +## Retries + +Certain errors are automatically retried 2 times by default, with a short exponential backoff. +Connection errors (for example, due to a network connectivity problem), 408 Request Timeout, 409 Conflict, +429 Rate Limit, and >=500 Internal errors are all retried by default. + +You can use the `max_retries` option to configure or disable retry settings: + +```python +from nemo_platform import NeMoPlatform + +# Configure the default for all requests: +client = NeMoPlatform( + base_url="http://nemo.test", + # default is 2 + max_retries=0, +) + +# Or, configure per-request: +client.with_options(max_retries=5).workspaces.list() +``` + +## Timeouts + +By default, requests time out after 1 minute. You can configure this with a `timeout` option, +which accepts a float or an [`httpx.Timeout`](https://www.python-httpx.org/advanced/timeouts/#fine-tuning-the-configuration) object: + +```python +from nemo_platform import NeMoPlatform + +# Configure the default for all requests: +client = NeMoPlatform( + base_url="http://nemo.test", + # 20 seconds (default is 1 minute) + timeout=20.0, +) + +# More granular control: +client = NeMoPlatform( + timeout=httpx.Timeout(60.0, read=5.0, write=10.0, connect=2.0), +) + +# Override per-request: +client.with_options(timeout=5.0).workspaces.list() +``` + +On timeout, an `APITimeoutError` is thrown. + +Note that requests that time out are [retried twice by default](#retries). + +## Advanced Usage + +### Logging + +We use the standard library [`logging`](https://docs.python.org/3/library/logging.html) module. + +You can enable logging by setting the environment variable `NMP_LOG` to `info`. + +```shell +$ export NMP_LOG=info +``` + +Or to `debug` for more verbose logging. + +#### How to Tell Whether `None` Means `null` or Missing + +In an API response, a field may be explicitly `null`, or missing entirely; in either case, its value is `None` in this library. You can differentiate the two cases with `.model_fields_set`: + +```py +if response.my_field is None: + if 'my_field' not in response.model_fields_set: + print('Got json like {}, without a "my_field" key present at all.') + else: + print('Got json like {"my_field": null}.') +``` + +### Accessing Raw Response Data (e.g. Headers) + +You can access the "raw" response object by prefixing `.with_raw_response.` to any HTTP method call, for example: + +```py +from nemo_platform import NeMoPlatform + +client = NeMoPlatform(base_url="http://nemo.test") +response = client.workspaces.with_raw_response.list() +print(response.headers.get('X-My-Header')) + +workspace = response.parse() # get the object that `workspaces.list()` would have returned +print(workspace.id) +``` + +These methods return an `APIResponse` object. + +The async client returns an `AsyncAPIResponse` with the same structure, the only difference being `await`able methods for reading the response content. + +#### `.with_streaming_response` + +The above interface eagerly reads the full response body when you make the request, which may not always be what you want. + +To stream the response body, use `.with_streaming_response` instead, which requires a context manager and only reads the response body once you call `.read()`, `.text()`, `.json()`, `.iter_bytes()`, `.iter_text()`, `.iter_lines()` or `.parse()`. In the async client, these are async methods. + +```python +with client.workspaces.with_streaming_response.list() as response: + print(response.headers.get("X-My-Header")) + + for line in response.iter_lines(): + print(line) +``` + +The context manager is required so that the response will reliably be closed. + +### Making Custom/Undocumented Requests + +This library is typed for convenient access to the documented API. + +If you need to access undocumented endpoints, params, or response properties, you can still use the library. + +#### Undocumented Endpoints + +To make requests to undocumented endpoints, you can make requests using `client.get`, `client.post`, and other +http verbs. The client will respect options (such as retries) when making this request. + +```py +import httpx + +response = client.post( + "/foo", + cast_to=httpx.Response, + body={"my_param": True}, +) + +print(response.headers.get("x-foo")) +``` + +#### Undocumented Request Params + +If you want to explicitly send an extra param, you can do so with the `extra_query`, `extra_body`, and `extra_headers` request +options. + +#### Undocumented Response Properties + +To access undocumented response properties, you can access the extra fields like `response.unknown_prop`. You +can also get all the extra fields on the Pydantic model as a dict with +[`response.model_extra`](https://docs.pydantic.dev/latest/api/base_model/#pydantic.BaseModel.model_extra). + +### Configuring the HTTP Client + +You can directly override the [httpx client](https://www.python-httpx.org/api/#client) to customize it for your use case, including: + +- Support for [proxies](https://www.python-httpx.org/advanced/proxies/) +- Custom [transports](https://www.python-httpx.org/advanced/transports/) +- Additional [advanced](https://www.python-httpx.org/advanced/clients/) functionality + +```python +import httpx +from nemo_platform import NeMoPlatform, DefaultHttpxClient + +client = NeMoPlatform( + base_url="http://nemo.test", + http_client=DefaultHttpxClient( + proxy="http://my.test.proxy.example.com", + transport=httpx.HTTPTransport(local_address="0.0.0.0"), + ), +) +``` + +You can also customize the client on a per-request basis by using `with_options()`: + +```python +client.with_options(http_client=DefaultHttpxClient(...)) +``` + +### Managing HTTP Resources + +By default the library closes underlying HTTP connections whenever the client is [garbage collected](https://docs.python.org/3/reference/datamodel.html#object.__del__). You can manually close the client using the `.close()` method if desired, or use a context manager that closes when exiting. + +```py +from nemo_platform import NeMoPlatform + +with NeMoPlatform() as client: + # make requests here + ... + +# HTTP client is now closed +``` From d0209eb515aeeaea7f9cdf00856430433dd7c6dc Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Fri, 22 May 2026 10:47:37 -0400 Subject: [PATCH 05/26] docs(fern-migrate): notebook viewer pipeline + 7 MDX wrappers Adds Jupyter notebook rendering, reusing the pattern from NVIDIA-NeMo/DataDesigner (script ipynb-to-fern-json.py landed in C1 under docs/fern/scripts/). - docs/fern/components/notebooks/*.{json,ts}: 14 generated payloads for the 6 customizer tutorials and 1 example-applications notebook. The .ts file is a thin default-export wrapper around the .json (Fern's bundler resolves .ts imports cleanly). - docs/customizer/tutorials/{sft,lora,dpo,distillation,embedding, optimize-throughput}-customization-job.mdx and docs/example-applications/tool-calling.mdx: thin MDX wrapper pages that import the notebook payload and render via with a colab badge. Regenerate any notebook payload with: python3 docs/fern/scripts/ipynb-to-fern-json.py \ docs/customizer/tutorials/sft-customization-job.ipynb \ -o docs/fern/components/notebooks/sft-customization-job.json The .ipynb sources stay at their original paths so the converter has a stable input and Colab badges link back to the canonical notebook. Signed-off-by: Lawrence Lane --- .../distillation-customization-job.mdx | 12 + .../tutorials/dpo-customization-job.mdx | 12 + .../tutorials/embedding-customization-job.mdx | 12 + .../tutorials/lora-customization-job.mdx | 12 + .../tutorials/optimize-throughput.mdx | 12 + .../tutorials/sft-customization-job.mdx | 12 + docs/example-applications/tool-calling.mdx | 12 + .../distillation-customization-job.json | 203 ++++++++++++ .../distillation-customization-job.ts | 202 ++++++++++++ .../notebooks/dpo-customization-job.json | 203 ++++++++++++ .../notebooks/dpo-customization-job.ts | 202 ++++++++++++ .../embedding-customization-job.json | 186 +++++++++++ .../notebooks/embedding-customization-job.ts | 185 +++++++++++ .../notebooks/lora-customization-job.json | 139 +++++++++ .../notebooks/lora-customization-job.ts | 138 +++++++++ .../notebooks/optimize-throughput.json | 156 ++++++++++ .../notebooks/optimize-throughput.ts | 155 ++++++++++ .../notebooks/sft-customization-job.json | 195 ++++++++++++ .../notebooks/sft-customization-job.ts | 194 ++++++++++++ .../components/notebooks/tool-calling.json | 289 ++++++++++++++++++ .../fern/components/notebooks/tool-calling.ts | 288 +++++++++++++++++ 21 files changed, 2819 insertions(+) create mode 100644 docs/customizer/tutorials/distillation-customization-job.mdx create mode 100644 docs/customizer/tutorials/dpo-customization-job.mdx create mode 100644 docs/customizer/tutorials/embedding-customization-job.mdx create mode 100644 docs/customizer/tutorials/lora-customization-job.mdx create mode 100644 docs/customizer/tutorials/optimize-throughput.mdx create mode 100644 docs/customizer/tutorials/sft-customization-job.mdx create mode 100644 docs/example-applications/tool-calling.mdx create mode 100644 docs/fern/components/notebooks/distillation-customization-job.json create mode 100644 docs/fern/components/notebooks/distillation-customization-job.ts create mode 100644 docs/fern/components/notebooks/dpo-customization-job.json create mode 100644 docs/fern/components/notebooks/dpo-customization-job.ts create mode 100644 docs/fern/components/notebooks/embedding-customization-job.json create mode 100644 docs/fern/components/notebooks/embedding-customization-job.ts create mode 100644 docs/fern/components/notebooks/lora-customization-job.json create mode 100644 docs/fern/components/notebooks/lora-customization-job.ts create mode 100644 docs/fern/components/notebooks/optimize-throughput.json create mode 100644 docs/fern/components/notebooks/optimize-throughput.ts create mode 100644 docs/fern/components/notebooks/sft-customization-job.json create mode 100644 docs/fern/components/notebooks/sft-customization-job.ts create mode 100644 docs/fern/components/notebooks/tool-calling.json create mode 100644 docs/fern/components/notebooks/tool-calling.ts diff --git a/docs/customizer/tutorials/distillation-customization-job.mdx b/docs/customizer/tutorials/distillation-customization-job.mdx new file mode 100644 index 00000000000..6a4e355733a --- /dev/null +++ b/docs/customizer/tutorials/distillation-customization-job.mdx @@ -0,0 +1,12 @@ +--- +title: "Knowledge Distillation Customization" +description: "" +--- + +import { NotebookViewer } from "@/components/NotebookViewer"; +import notebook from "@/components/notebooks/distillation-customization-job"; + + diff --git a/docs/customizer/tutorials/dpo-customization-job.mdx b/docs/customizer/tutorials/dpo-customization-job.mdx new file mode 100644 index 00000000000..1b35286e600 --- /dev/null +++ b/docs/customizer/tutorials/dpo-customization-job.mdx @@ -0,0 +1,12 @@ +--- +title: "DPO Customization" +description: "" +--- + +import { NotebookViewer } from "@/components/NotebookViewer"; +import notebook from "@/components/notebooks/dpo-customization-job"; + + diff --git a/docs/customizer/tutorials/embedding-customization-job.mdx b/docs/customizer/tutorials/embedding-customization-job.mdx new file mode 100644 index 00000000000..1b9116dd092 --- /dev/null +++ b/docs/customizer/tutorials/embedding-customization-job.mdx @@ -0,0 +1,12 @@ +--- +title: "Embedding Model Customization" +description: "" +--- + +import { NotebookViewer } from "@/components/NotebookViewer"; +import notebook from "@/components/notebooks/embedding-customization-job"; + + diff --git a/docs/customizer/tutorials/lora-customization-job.mdx b/docs/customizer/tutorials/lora-customization-job.mdx new file mode 100644 index 00000000000..2378fc3e1ba --- /dev/null +++ b/docs/customizer/tutorials/lora-customization-job.mdx @@ -0,0 +1,12 @@ +--- +title: "LoRA Model Customization" +description: "" +--- + +import { NotebookViewer } from "@/components/NotebookViewer"; +import notebook from "@/components/notebooks/lora-customization-job"; + + diff --git a/docs/customizer/tutorials/optimize-throughput.mdx b/docs/customizer/tutorials/optimize-throughput.mdx new file mode 100644 index 00000000000..7efe459cf47 --- /dev/null +++ b/docs/customizer/tutorials/optimize-throughput.mdx @@ -0,0 +1,12 @@ +--- +title: "Optimize for Tokens/GPU Throughput" +description: "" +--- + +import { NotebookViewer } from "@/components/NotebookViewer"; +import notebook from "@/components/notebooks/optimize-throughput"; + + diff --git a/docs/customizer/tutorials/sft-customization-job.mdx b/docs/customizer/tutorials/sft-customization-job.mdx new file mode 100644 index 00000000000..5794c40c231 --- /dev/null +++ b/docs/customizer/tutorials/sft-customization-job.mdx @@ -0,0 +1,12 @@ +--- +title: "Full SFT Customization" +description: "" +--- + +import { NotebookViewer } from "@/components/NotebookViewer"; +import notebook from "@/components/notebooks/sft-customization-job"; + + diff --git a/docs/example-applications/tool-calling.mdx b/docs/example-applications/tool-calling.mdx new file mode 100644 index 00000000000..c5b4d723d59 --- /dev/null +++ b/docs/example-applications/tool-calling.mdx @@ -0,0 +1,12 @@ +--- +title: "Tool-Calling Fine-Tuning with Synthetic Data" +description: "" +--- + +import { NotebookViewer } from "@/components/NotebookViewer"; +import notebook from "@/components/notebooks/tool-calling"; + + diff --git a/docs/fern/components/notebooks/distillation-customization-job.json b/docs/fern/components/notebooks/distillation-customization-job.json new file mode 100644 index 00000000000..c072c5276ff --- /dev/null +++ b/docs/fern/components/notebooks/distillation-customization-job.json @@ -0,0 +1,203 @@ +{ + "cells": [ + { + "type": "markdown", + "source": "\n\n\n# Knowledge Distillation Customization\n\nLearn how to train a smaller student model to mimic a larger teacher model using knowledge distillation (KD).\n\n## About\n\nKnowledge Distillation transfers knowledge from a large **teacher** model to a smaller **student** model. During training, the student learns to match the teacher's output probability distribution, producing a compact model that retains much of the teacher's capability.\n\n**What you can achieve with KD:**\n\n- **Compress models:** Distill a 3B model into a 1B model for faster inference and lower deployment costs\n- **Reduce latency:** Deploy a smaller model that responds faster while preserving quality\n- **Lower resource requirements:** Serve a distilled model on fewer GPUs\n\n### KD vs SFT: Understanding the Trade-offs\n\n| Aspect | Full SFT | Knowledge Distillation |\n| --- | --- | --- |\n| **Training signal** | Ground-truth labels only | Teacher's soft probability distribution + labels |\n| **Knowledge source** | Dataset examples | Teacher model's learned representations |\n| **Output model size** | Same as input model | Typically a smaller student model |\n| **GPU requirements** | Needs to fit one model | Needs to fit both teacher and student in memory |\n| **Best for** | Domain adaptation, new knowledge injection | Model compression, latency reduction |\n\n### Key Parameters\n\n| Parameter | Default | Description |\n| --- | --- | --- |\n| `teacher_model` | *(required)* | Teacher model entity URN (e.g., `default/llama-3-2-3b-teacher`) |\n| `teacher_precision` | `bf16` | Precision for the frozen teacher (`bf16`, `fp16`, `fp32`). Lower = less memory |\n| `distillation_ratio` | `0.5` | Balance between CE loss and KD loss. `0.0` = CE only, `1.0` = KD only |\n| `distillation_temperature` | `1.0` | Softmax temperature. Higher = softer distributions, more knowledge transfer |\n\n### Workflow Overview\n\nThis tutorial follows a complete distillation pipeline:\n\n1. **Fine-tune the teacher** (SFT on the task dataset) so it learns the domain\n2. **Establish a baseline** by deploying the base student model and measuring ROUGE scores\n3. **Distill into the student** using the fine-tuned teacher's soft targets\n4. **Evaluate the distilled student** and compare ROUGE scores against the baseline\n\n**When to choose KD:**\n\n- You have a high-quality large model and want a smaller, faster version\n- Deployment latency or cost is a constraint\n- The teacher and student share the same vocabulary (e.g., both are Llama models)\n\n**When to choose SFT instead:** Refer to the [Full SFT tutorial](./sft-customization-job) when you want to train a model directly on labeled data without a teacher.", + "source_html": "\n\n

Knowledge Distillation Customization

\n

Learn how to train a smaller student model to mimic a larger teacher model using knowledge distillation (KD).

\n

About

\n

Knowledge Distillation transfers knowledge from a large teacher model to a smaller student model. During training, the student learns to match the teacher's output probability distribution, producing a compact model that retains much of the teacher's capability.

\n

What you can achieve with KD:

\n
    \n
  • Compress models: Distill a 3B model into a 1B model for faster inference and lower deployment costs
    • \n
  • Reduce latency: Deploy a smaller model that responds faster while preserving quality
    • \n
  • Lower resource requirements: Serve a distilled model on fewer GPUs
    • \n
\n

KD vs SFT: Understanding the Trade-offs

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
AspectFull SFTKnowledge Distillation
Training signalGround-truth labels onlyTeacher's soft probability distribution + labels
Knowledge sourceDataset examplesTeacher model's learned representations
Output model sizeSame as input modelTypically a smaller student model
GPU requirementsNeeds to fit one modelNeeds to fit both teacher and student in memory
Best forDomain adaptation, new knowledge injectionModel compression, latency reduction
\n

Key Parameters

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ParameterDefaultDescription
teacher_model(required)Teacher model entity URN (e.g., default/llama-3-2-3b-teacher)
teacher_precisionbf16Precision for the frozen teacher (bf16, fp16, fp32). Lower = less memory
distillation_ratio0.5Balance between CE loss and KD loss. 0.0 = CE only, 1.0 = KD only
distillation_temperature1.0Softmax temperature. Higher = softer distributions, more knowledge transfer
\n

Workflow Overview

\n

This tutorial follows a complete distillation pipeline:

\n
    \n
  1. Fine-tune the teacher (SFT on the task dataset) so it learns the domain
    2. \n
  2. Establish a baseline by deploying the base student model and measuring ROUGE scores
    3. \n
  3. Distill into the student using the fine-tuned teacher's soft targets
    4. \n
  4. Evaluate the distilled student and compare ROUGE scores against the baseline
    5. \n
\n

When to choose KD:

\n
    \n
  • You have a high-quality large model and want a smaller, faster version
    • \n
  • Deployment latency or cost is a constraint
    • \n
  • The teacher and student share the same vocabulary (e.g., both are Llama models)
    • \n
\n

When to choose SFT instead: Refer to the Full SFT tutorial when you want to train a model directly on labeled data without a teacher.

\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n2. **Installed the Python SDK** (included with `pip install nemo-platform`)\n3. **Installed evaluation dependencies:**\n\n```sh\npip install evaluate rouge_score datasets\n```", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Completed the Quickstart to install and deploy NeMo Platform locally
    2. \n
  2. Installed the Python SDK (included with pip install nemo-platform)
    3. \n
  3. Installed evaluation dependencies:
    4. \n
\n
pip install evaluate rouge_score datasets\n
\n" + }, + { + "type": "markdown", + "source": "## Quick Start\n\n### 1. Initialize SDK\n\nThe SDK needs to know your NeMo Platform server URL. By default, `http://localhost:8080` is used in accordance with the [Quickstart](../../get-started/quickstart.md) guide. If NeMo Platform is running at a custom location, you can override the URL by setting the `NMP_BASE_URL` environment variable:\n\n```sh\nexport NMP_BASE_URL=\n```", + "source_html": "

Quick Start

\n

1. Initialize SDK

\n

The SDK needs to know your NeMo Platform server URL. By default, http://localhost:8080 is used in accordance with the Quickstart guide. If NeMo Platform is running at a custom location, you can override the URL by setting the NMP_BASE_URL environment variable:

\n
export NMP_BASE_URL=<YOUR_NMP_BASE_URL>\n
\n" + }, + { + "type": "code", + "source": "import json\nimport os\nimport time\nimport uuid\nfrom pathlib import Path\n\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace=\"default\"\n)", + "language": "python", + "source_html": "import json\nimport os\nimport time\nimport uuid\nfrom pathlib import Path\n\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace="default"\n)\n" + }, + { + "type": "markdown", + "source": "### 2. Prepare Dataset\n\nKnowledge distillation uses the same dataset formats as SFT. We use the SQuAD dataset for both teacher training and distillation so that the teacher first learns the task, then transfers that knowledge to the student.\n\nWe also hold out a small **test split** for ROUGE evaluation at the end.", + "source_html": "

2. Prepare Dataset

\n

Knowledge distillation uses the same dataset formats as SFT. We use the SQuAD dataset for both teacher training and distillation so that the teacher first learns the task, then transfers that knowledge to the student.

\n

We also hold out a small test split for ROUGE evaluation at the end.

\n" + }, + { + "type": "code", + "source": "from datasets import load_dataset, DatasetDict\n\nprint(\"Loading dataset rajpurkar/squad\")\nraw_dataset = load_dataset(\"rajpurkar/squad\")\nif not isinstance(raw_dataset, DatasetDict):\n raise ValueError(\"Dataset does not contain expected splits\")\n\nprint(\"Loaded dataset\")\n\nSEED = 1234\nTRAINING_SIZE = 3000\nVALIDATION_SIZE = 300\nTEST_SIZE = 100\nDATASET_PATH = Path(\"kd-dataset\").absolute()\n\nos.makedirs(DATASET_PATH, exist_ok=True)\n\ntrain_set = raw_dataset.get('train')\nsplit = train_set.train_test_split(test_size=0.05, seed=SEED)\n\ntrain_ds = split['train'].select(range(min(TRAINING_SIZE, len(split['train']))))\nval_ds = split['test'].select(range(min(VALIDATION_SIZE, len(split['test']))))\ntest_ds = split['test'].select(range(VALIDATION_SIZE, min(VALIDATION_SIZE + TEST_SIZE, len(split['test']))))\n\n\ndef convert_squad(example):\n \"\"\"Convert SQuAD format to prompt/completion format.\"\"\"\n prompt = f\"Context: {example['context']} Question: {example['question']} Answer:\"\n completion = example[\"answers\"][\"text\"][0]\n return {\"prompt\": prompt, \"completion\": completion}\n\n\ndef write_jsonl(dataset, path):\n with open(path, \"w\", encoding=\"utf-8\") as f:\n for example in dataset:\n f.write(json.dumps(convert_squad(example)) + \"\\n\")\n\n\ndef write_test_jsonl(dataset, path):\n \"\"\"Save test split with raw context/question for chat-style evaluation.\"\"\"\n with open(path, \"w\", encoding=\"utf-8\") as f:\n for example in dataset:\n f.write(json.dumps({\n \"context\": example[\"context\"],\n \"question\": example[\"question\"],\n \"completion\": example[\"answers\"][\"text\"][0],\n }) + \"\\n\")\n\n\nwrite_jsonl(train_ds, f\"{DATASET_PATH}/training.jsonl\")\nwrite_jsonl(val_ds, f\"{DATASET_PATH}/validation.jsonl\")\nwrite_test_jsonl(test_ds, f\"{DATASET_PATH}/testing.jsonl\")\n\nprint(f\"Training: {len(train_ds)} rows\")\nprint(f\"Validation: {len(val_ds)} rows\")\nprint(f\"Test: {len(test_ds)} rows\")\n\nwith open(f\"{DATASET_PATH}/training.jsonl\", 'r') as f:\n sample = json.loads(f.readline())\n print(f\"\\nSample prompt: {sample['prompt'][:150]}...\")\n print(f\"Sample completion: {sample['completion']}\")", + "language": "python", + "source_html": "from datasets import load_dataset, DatasetDict\n\nprint("Loading dataset rajpurkar/squad")\nraw_dataset = load_dataset("rajpurkar/squad")\nif not isinstance(raw_dataset, DatasetDict):\n raise ValueError("Dataset does not contain expected splits")\n\nprint("Loaded dataset")\n\nSEED = 1234\nTRAINING_SIZE = 3000\nVALIDATION_SIZE = 300\nTEST_SIZE = 100\nDATASET_PATH = Path("kd-dataset").absolute()\n\nos.makedirs(DATASET_PATH, exist_ok=True)\n\ntrain_set = raw_dataset.get('train')\nsplit = train_set.train_test_split(test_size=0.05, seed=SEED)\n\ntrain_ds = split['train'].select(range(min(TRAINING_SIZE, len(split['train']))))\nval_ds = split['test'].select(range(min(VALIDATION_SIZE, len(split['test']))))\ntest_ds = split['test'].select(range(VALIDATION_SIZE, min(VALIDATION_SIZE + TEST_SIZE, len(split['test']))))\n\n\ndef convert_squad(example):\n """Convert SQuAD format to prompt/completion format."""\n prompt = f"Context: {example['context']} Question: {example['question']} Answer:"\n completion = example["answers"]["text"][0]\n return {"prompt": prompt, "completion": completion}\n\n\ndef write_jsonl(dataset, path):\n with open(path, "w", encoding="utf-8") as f:\n for example in dataset:\n f.write(json.dumps(convert_squad(example)) + "\\n")\n\n\ndef write_test_jsonl(dataset, path):\n """Save test split with raw context/question for chat-style evaluation."""\n with open(path, "w", encoding="utf-8") as f:\n for example in dataset:\n f.write(json.dumps({\n "context": example["context"],\n "question": example["question"],\n "completion": example["answers"]["text"][0],\n }) + "\\n")\n\n\nwrite_jsonl(train_ds, f"{DATASET_PATH}/training.jsonl")\nwrite_jsonl(val_ds, f"{DATASET_PATH}/validation.jsonl")\nwrite_test_jsonl(test_ds, f"{DATASET_PATH}/testing.jsonl")\n\nprint(f"Training: {len(train_ds)} rows")\nprint(f"Validation: {len(val_ds)} rows")\nprint(f"Test: {len(test_ds)} rows")\n\nwith open(f"{DATASET_PATH}/training.jsonl", 'r') as f:\n sample = json.loads(f.readline())\n print(f"\\nSample prompt: {sample['prompt'][:150]}...")\n print(f"Sample completion: {sample['completion']}")\n" + }, + { + "type": "code", + "source": "DATASET_NAME = \"kd-dataset\"\n\ntry:\n client.files.filesets.create(\n workspace=\"default\",\n name=DATASET_NAME,\n description=\"Knowledge distillation training data\"\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\nclient.files.upload(\n local_path=DATASET_PATH,\n remote_path=\"\",\n fileset=DATASET_NAME,\n workspace=\"default\"\n)\n\nprint(\"Uploaded files:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace=\"default\").data], indent=2))", + "language": "python", + "source_html": "DATASET_NAME = "kd-dataset"\n\ntry:\n client.files.filesets.create(\n workspace="default",\n name=DATASET_NAME,\n description="Knowledge distillation training data"\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\nclient.files.upload(\n local_path=DATASET_PATH,\n remote_path="",\n fileset=DATASET_NAME,\n workspace="default"\n)\n\nprint("Uploaded files:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace="default").data], indent=2))\n" + }, + { + "type": "markdown", + "source": "### 3. Secrets Setup\n\nIn this tutorial we use two Llama 3.2 Instruct models from HuggingFace:\n- **Teacher:** [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) (3B parameters)\n- **Student:** [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) (1B parameters)\n\nBoth models share the same tokenizer/vocabulary (required for knowledge distillation) and include a chat template for deployment with `/chat/completions`.\n\n**HuggingFace Authentication:**\n- For gated models (Llama, Gemma), you must provide a HuggingFace token via the `token_secret` parameter\n- Get your token from [HuggingFace Settings](https://huggingface.co/settings/tokens) (requires Read access)\n- Accept the model's terms on the HuggingFace model page before using it:\n - [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct)\n - [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct)", + "source_html": "

3. Secrets Setup

\n

In this tutorial we use two Llama 3.2 Instruct models from HuggingFace:

\n\n

Both models share the same tokenizer/vocabulary (required for knowledge distillation) and include a chat template for deployment with /chat/completions.

\n

HuggingFace Authentication:

\n\n" + }, + { + "type": "code", + "source": "HF_TOKEN = os.getenv(\"HF_TOKEN\")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f\"{label} is not set\")\n try:\n secret = client.secrets.create(\n name=name,\n workspace=\"default\",\n value=value,\n )\n print(f\"Created secret: {name}\")\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n return client.secrets.retrieve(name=name, workspace=\"default\")\n\n\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")\nprint(\"HF_TOKEN secret:\")\nprint(hf_secret.model_dump_json(indent=2))", + "language": "python", + "source_html": "HF_TOKEN = os.getenv("HF_TOKEN")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f"{label} is not set")\n try:\n secret = client.secrets.create(\n name=name,\n workspace="default",\n value=value,\n )\n print(f"Created secret: {name}")\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n return client.secrets.retrieve(name=name, workspace="default")\n\n\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\nprint("HF_TOKEN secret:")\nprint(hf_secret.model_dump_json(indent=2))\n" + }, + { + "type": "markdown", + "source": "### 4. Create Model FileSets and Model Entities\n\nKnowledge distillation requires **two** model entities:\n1. **Student model** — the smaller model that will be trained ([meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct))\n2. **Teacher model** — the larger model that provides soft targets ([meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct))\n\nUsing the Instruct variants ensures the output model includes a chat template, which is required for the `/chat/completions` inference endpoint.", + "source_html": "

4. Create Model FileSets and Model Entities

\n

Knowledge distillation requires two model entities:

\n
    \n
  1. Student model — the smaller model that will be trained (meta-llama/Llama-3.2-1B-Instruct)
    2. \n
  2. Teacher model — the larger model that provides soft targets (meta-llama/Llama-3.2-3B-Instruct)
    3. \n
\n

Using the Instruct variants ensures the output model includes a chat template, which is required for the /chat/completions inference endpoint.

\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nSPEC_TIMEOUT_SECONDS = 120\n\n\ndef create_model(hf_repo: str, model_name: str, description: str):\n \"\"\"Create a fileset + model entity and wait for ModelSpec.\"\"\"\n try:\n client.files.filesets.create(\n workspace=\"default\",\n name=model_name,\n description=description,\n storage=HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n repo_id=hf_repo,\n repo_type=\"model\",\n token_secret=hf_secret.name\n )\n )\n print(f\"Created fileset: {model_name}\")\n except ConflictError:\n print(f\"Fileset '{model_name}' already exists.\")\n\n try:\n model = client.models.create(\n workspace=\"default\",\n name=model_name,\n fileset=f\"default/{model_name}\",\n )\n print(f\"Created Model Entity: {model_name}\")\n except ConflictError:\n print(f\"Model '{model_name}' already exists. Updating fileset.\")\n model = client.models.update(\n workspace=\"default\",\n name=model_name,\n fileset=f\"default/{model_name}\",\n )\n\n print(f\"Waiting for ModelSpec on {model_name}...\")\n spec_start = time.time()\n while not model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f\"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS}s\")\n time.sleep(2)\n model = client.models.retrieve(workspace=\"default\", name=model_name)\n print(f\"ModelSpec populated: {model.spec}\")\n return model\n\n\nstudent_model = create_model(\n hf_repo=\"meta-llama/Llama-3.2-1B-Instruct\",\n model_name=\"llama-3-2-1b-student\",\n description=\"Llama 3.2 1B Instruct student model\",\n)\n\nprint()\n\nteacher_model = create_model(\n hf_repo=\"meta-llama/Llama-3.2-3B-Instruct\",\n model_name=\"llama-3-2-3b-teacher\",\n description=\"Llama 3.2 3B Instruct teacher model\",\n)", + "language": "python", + "source_html": "from nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nSPEC_TIMEOUT_SECONDS = 120\n\n\ndef create_model(hf_repo: str, model_name: str, description: str):\n """Create a fileset + model entity and wait for ModelSpec."""\n try:\n client.files.filesets.create(\n workspace="default",\n name=model_name,\n description=description,\n storage=HuggingfaceStorageConfigParam(\n type="huggingface",\n repo_id=hf_repo,\n repo_type="model",\n token_secret=hf_secret.name\n )\n )\n print(f"Created fileset: {model_name}")\n except ConflictError:\n print(f"Fileset '{model_name}' already exists.")\n\n try:\n model = client.models.create(\n workspace="default",\n name=model_name,\n fileset=f"default/{model_name}",\n )\n print(f"Created Model Entity: {model_name}")\n except ConflictError:\n print(f"Model '{model_name}' already exists. Updating fileset.")\n model = client.models.update(\n workspace="default",\n name=model_name,\n fileset=f"default/{model_name}",\n )\n\n print(f"Waiting for ModelSpec on {model_name}...")\n spec_start = time.time()\n while not model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS}s")\n time.sleep(2)\n model = client.models.retrieve(workspace="default", name=model_name)\n print(f"ModelSpec populated: {model.spec}")\n return model\n\n\nstudent_model = create_model(\n hf_repo="meta-llama/Llama-3.2-1B-Instruct",\n model_name="llama-3-2-1b-student",\n description="Llama 3.2 1B Instruct student model",\n)\n\nprint()\n\nteacher_model = create_model(\n hf_repo="meta-llama/Llama-3.2-3B-Instruct",\n model_name="llama-3-2-3b-teacher",\n description="Llama 3.2 3B Instruct teacher model",\n)\n" + }, + { + "type": "markdown", + "source": "---\n\n## Phase 1: Fine-Tune the Teacher\n\n### 5. Train Teacher with Full SFT\n\nFor best distillation results, fine-tune the teacher on the **same dataset** that will be used for distillation. This ensures the teacher has learned the task-specific knowledge that the student will inherit.\n\nWe train the 3B Instruct model with Full SFT on the SQuAD dataset.", + "source_html": "
\n

Phase 1: Fine-Tune the Teacher

\n

5. Train Teacher with Full SFT

\n

For best distillation results, fine-tune the teacher on the same dataset that will be used for distillation. This ensures the teacher has learned the task-specific knowledge that the student will inherit.

\n

We train the 3B Instruct model with Full SFT on the SQuAD dataset.

\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n DistillationTrainingParam,\n ParallelismParamsParam,\n)\n\njob_suffix = uuid.uuid4().hex[:4]\n\nTEACHER_JOB_NAME = f\"teacher-sft-job-{job_suffix}\"\n\nteacher_job = client.customization.jobs.create(\n name=TEACHER_JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{teacher_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n )\n)\n\nTRAINED_TEACHER_NAME = teacher_job.spec.output.name\nprint(f\"Teacher training job: {teacher_job.name}\")\nprint(f\"Output teacher model: {TRAINED_TEACHER_NAME}\")", + "language": "python", + "source_html": "from nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n DistillationTrainingParam,\n ParallelismParamsParam,\n)\n\njob_suffix = uuid.uuid4().hex[:4]\n\nTEACHER_JOB_NAME = f"teacher-sft-job-{job_suffix}"\n\nteacher_job = client.customization.jobs.create(\n name=TEACHER_JOB_NAME,\n workspace="default",\n spec=CustomizationJobInputParam(\n model=f"default/{teacher_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n )\n)\n\nTRAINED_TEACHER_NAME = teacher_job.spec.output.name\nprint(f"Teacher training job: {teacher_job.name}")\nprint(f"Output teacher model: {TRAINED_TEACHER_NAME}")\n" + }, + { + "type": "code", + "source": "from IPython.display import clear_output\n\n\ndef wait_for_job(job_name: str):\n \"\"\"Poll job status until completion.\"\"\"\n while True:\n status = client.customization.jobs.get_status(name=job_name, workspace=\"default\")\n clear_output(wait=True)\n print(f\"Job: {job_name}\")\n print(f\"Status: {status.status}\")\n\n for job_step in status.steps or []:\n if job_step.name == \"customization-training-job\":\n for task in job_step.tasks or []:\n details = task.status_details or {}\n step = details.get(\"step\")\n max_steps = details.get(\"max_steps\")\n if step is not None and max_steps is not None:\n print(f\"Progress: Step {step}/{max_steps} ({step / max_steps * 100:.1f}%)\")\n phase = details.get(\"phase\")\n if phase:\n print(f\"Phase: {phase}\")\n break\n break\n\n if status.status in (\"completed\", \"failed\", \"cancelled\", \"error\"):\n print(f\"\\nJob finished: {status.status}\")\n return status\n\n time.sleep(10)\n\n\nteacher_status = wait_for_job(TEACHER_JOB_NAME)", + "language": "python", + "source_html": "from IPython.display import clear_output\n\n\ndef wait_for_job(job_name: str):\n """Poll job status until completion."""\n while True:\n status = client.customization.jobs.get_status(name=job_name, workspace="default")\n clear_output(wait=True)\n print(f"Job: {job_name}")\n print(f"Status: {status.status}")\n\n for job_step in status.steps or []:\n if job_step.name == "customization-training-job":\n for task in job_step.tasks or []:\n details = task.status_details or {}\n step = details.get("step")\n max_steps = details.get("max_steps")\n if step is not None and max_steps is not None:\n print(f"Progress: Step {step}/{max_steps} ({step / max_steps * 100:.1f}%)")\n phase = details.get("phase")\n if phase:\n print(f"Phase: {phase}")\n break\n break\n\n if status.status in ("completed", "failed", "cancelled", "error"):\n print(f"\\nJob finished: {status.status}")\n return status\n\n time.sleep(10)\n\n\nteacher_status = wait_for_job(TEACHER_JOB_NAME)\n" + }, + { + "type": "markdown", + "source": "---\n\n## Phase 2: Establish Baseline (Base Student)\n\n### 6. Deploy the Base Student Model\n\nBefore distillation, deploy the base student model (1B Instruct, without any fine-tuning) to establish a baseline ROUGE score. After distillation, we compare the distilled student against this baseline to measure improvement.", + "source_html": "
\n

Phase 2: Establish Baseline (Base Student)

\n

6. Deploy the Base Student Model

\n

Before distillation, deploy the base student model (1B Instruct, without any fine-tuning) to establish a baseline ROUGE score. After distillation, we compare the distilled student against this baseline to measure improvement.

\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.inference import NIMDeploymentParam\n\nbaseline_suffix = uuid.uuid4().hex[:4]\nBASELINE_DEPLOYMENT_CONFIG = f\"baseline-student-cfg-{baseline_suffix}\"\nBASELINE_DEPLOYMENT_NAME = f\"baseline-student-{baseline_suffix}\"\n\nbaseline_deployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=BASELINE_DEPLOYMENT_CONFIG,\n nim_deployment=NIMDeploymentParam(\n image_name=\"nvcr.io/nim/nvidia/llm-nim\",\n image_tag=\"1.15.5\",\n gpu=1,\n model_name=student_model.name,\n model_namespace=\"default\",\n additional_envs={\"NIM_MODEL_PROFILE\": \"vllm\"}\n ),\n)\n\nbaseline_deployment = client.inference.deployments.create(\n workspace=\"default\",\n name=BASELINE_DEPLOYMENT_NAME,\n config=baseline_deployment_config.name\n)\n\nprint(f\"Baseline student deployment: {baseline_deployment.name}\")", + "language": "python", + "source_html": "from nemo_platform.types.inference import NIMDeploymentParam\n\nbaseline_suffix = uuid.uuid4().hex[:4]\nBASELINE_DEPLOYMENT_CONFIG = f"baseline-student-cfg-{baseline_suffix}"\nBASELINE_DEPLOYMENT_NAME = f"baseline-student-{baseline_suffix}"\n\nbaseline_deployment_config = client.inference.deployment_configs.create(\n workspace="default",\n name=BASELINE_DEPLOYMENT_CONFIG,\n nim_deployment=NIMDeploymentParam(\n image_name="nvcr.io/nim/nvidia/llm-nim",\n image_tag="1.15.5",\n gpu=1,\n model_name=student_model.name,\n model_namespace="default",\n additional_envs={"NIM_MODEL_PROFILE": "vllm"}\n ),\n)\n\nbaseline_deployment = client.inference.deployments.create(\n workspace="default",\n name=BASELINE_DEPLOYMENT_NAME,\n config=baseline_deployment_config.name\n)\n\nprint(f"Baseline student deployment: {baseline_deployment.name}")\n" + }, + { + "type": "code", + "source": "def wait_for_deployment(deployment_name: str, timeout_minutes: int = 30):\n \"\"\"Poll deployment until ready.\"\"\"\n start = time.time()\n timeout = timeout_minutes * 60\n while True:\n dep = client.inference.deployments.retrieve(name=deployment_name, workspace=\"default\")\n elapsed = time.time() - start\n clear_output(wait=True)\n print(f\"Deployment: {deployment_name}\")\n print(f\"Status: {dep.status}\")\n print(f\"Elapsed: {int(elapsed // 60)}m {int(elapsed % 60)}s\")\n\n if dep.status == \"READY\":\n print(\"\\nDeployment is ready!\")\n return dep\n if dep.status in (\"FAILED\", \"ERROR\", \"TERMINATED\", \"LOST\"):\n print(f\"\\nDeployment failed: {dep.status}\")\n return dep\n if elapsed > timeout:\n print(f\"\\nTimeout ({timeout_minutes}m). Check status manually.\")\n return dep\n time.sleep(15)\n\n\nwait_for_deployment(BASELINE_DEPLOYMENT_NAME)", + "language": "python", + "source_html": "def wait_for_deployment(deployment_name: str, timeout_minutes: int = 30):\n """Poll deployment until ready."""\n start = time.time()\n timeout = timeout_minutes * 60\n while True:\n dep = client.inference.deployments.retrieve(name=deployment_name, workspace="default")\n elapsed = time.time() - start\n clear_output(wait=True)\n print(f"Deployment: {deployment_name}")\n print(f"Status: {dep.status}")\n print(f"Elapsed: {int(elapsed // 60)}m {int(elapsed % 60)}s")\n\n if dep.status == "READY":\n print("\\nDeployment is ready!")\n return dep\n if dep.status in ("FAILED", "ERROR", "TERMINATED", "LOST"):\n print(f"\\nDeployment failed: {dep.status}")\n return dep\n if elapsed > timeout:\n print(f"\\nTimeout ({timeout_minutes}m). Check status manually.")\n return dep\n time.sleep(15)\n\n\nwait_for_deployment(BASELINE_DEPLOYMENT_NAME)\n" + }, + { + "type": "markdown", + "source": "### 7. Generate Baseline Predictions on Test Set", + "source_html": "

7. Generate Baseline Predictions on Test Set

\n" + }, + { + "type": "code", + "source": "with open(f\"{DATASET_PATH}/testing.jsonl\", \"r\", encoding=\"utf-8\") as f:\n test_data = [json.loads(line) for line in f]\n\ncontexts = [row[\"context\"] for row in test_data]\nquestions = [row[\"question\"] for row in test_data]\nreference_completions = [row[\"completion\"] for row in test_data]\n\nprint(f\"Test samples: {len(contexts)}\")\nprint(f\"Sample context: {contexts[0]}\")\nprint(f\"Sample question: {questions[0]}\")\nprint(f\"Sample reference: {reference_completions[0]}\")", + "language": "python", + "source_html": "with open(f"{DATASET_PATH}/testing.jsonl", "r", encoding="utf-8") as f:\n test_data = [json.loads(line) for line in f]\n\ncontexts = [row["context"] for row in test_data]\nquestions = [row["question"] for row in test_data]\nreference_completions = [row["completion"] for row in test_data]\n\nprint(f"Test samples: {len(contexts)}")\nprint(f"Sample context: {contexts[0]}")\nprint(f"Sample question: {questions[0]}")\nprint(f"Sample reference: {reference_completions[0]}")\n" + }, + { + "type": "code", + "source": "def generate_completions(\n deployment_name: str,\n output_model_name: str,\n contexts: list[str],\n questions: list[str],\n) -> list[str]:\n \"\"\"Generate completions for a list of context/question pairs using a deployed model.\"\"\"\n completions = []\n for context, question in zip(contexts, questions):\n messages = [\n {\n \"role\": \"user\",\n \"content\": f\"Based on the following context, answer the question.\\n\\nContext: {context}\\n\\nQuestion: {question}\",\n }\n ]\n response = client.inference.gateway.provider.post(\n \"v1/chat/completions\",\n name=deployment_name,\n workspace=\"default\",\n body={\n \"model\": f\"default/{output_model_name}\",\n \"messages\": messages,\n \"temperature\": 0,\n \"max_tokens\": 128,\n }\n )\n completions.append(response[\"choices\"][0][\"message\"][\"content\"])\n return completions\n\n\nprint(\"Generating baseline (base student) predictions...\")\nbaseline_completions = generate_completions(BASELINE_DEPLOYMENT_NAME, student_model.name, contexts, questions)\nprint(f\"Generated {len(baseline_completions)} baseline predictions\")\nprint(f\"\\nSample baseline output: {baseline_completions[0]}\")", + "language": "python", + "source_html": "def generate_completions(\n deployment_name: str,\n output_model_name: str,\n contexts: list[str],\n questions: list[str],\n) -> list[str]:\n """Generate completions for a list of context/question pairs using a deployed model."""\n completions = []\n for context, question in zip(contexts, questions):\n messages = [\n {\n "role": "user",\n "content": f"Based on the following context, answer the question.\\n\\nContext: {context}\\n\\nQuestion: {question}",\n }\n ]\n response = client.inference.gateway.provider.post(\n "v1/chat/completions",\n name=deployment_name,\n workspace="default",\n body={\n "model": f"default/{output_model_name}",\n "messages": messages,\n "temperature": 0,\n "max_tokens": 128,\n }\n )\n completions.append(response["choices"][0]["message"]["content"])\n return completions\n\n\nprint("Generating baseline (base student) predictions...")\nbaseline_completions = generate_completions(BASELINE_DEPLOYMENT_NAME, student_model.name, contexts, questions)\nprint(f"Generated {len(baseline_completions)} baseline predictions")\nprint(f"\\nSample baseline output: {baseline_completions[0]}")\n" + }, + { + "type": "markdown", + "source": "### 8. Delete Baseline Deployment\n\nDelete the baseline student deployment to free GPU resources for the distillation training job and subsequent distilled model deployment.", + "source_html": "

8. Delete Baseline Deployment

\n

Delete the baseline student deployment to free GPU resources for the distillation training job and subsequent distilled model deployment.

\n" + }, + { + "type": "code", + "source": "client.inference.deployments.delete(name=BASELINE_DEPLOYMENT_NAME, workspace=\"default\")\nprint(f\"Deleted baseline deployment: {BASELINE_DEPLOYMENT_NAME}\")\n\n# wait for deployment to be deleted\ntime.sleep(60)\n\nclient.inference.deployment_configs.delete(name=BASELINE_DEPLOYMENT_CONFIG, workspace=\"default\")\nprint(f\"Deleted baseline deployment config: {BASELINE_DEPLOYMENT_CONFIG}\")", + "language": "python", + "source_html": "client.inference.deployments.delete(name=BASELINE_DEPLOYMENT_NAME, workspace="default")\nprint(f"Deleted baseline deployment: {BASELINE_DEPLOYMENT_NAME}")\n\n# wait for deployment to be deleted\ntime.sleep(60)\n\nclient.inference.deployment_configs.delete(name=BASELINE_DEPLOYMENT_CONFIG, workspace="default")\nprint(f"Deleted baseline deployment config: {BASELINE_DEPLOYMENT_CONFIG}")\n" + }, + { + "type": "markdown", + "source": "---\n\n## Phase 3: Distill into Student\n\n### 9. Create Knowledge Distillation Job\n\nNow create a distillation job that trains the 1B student using the **fine-tuned** 3B teacher's output distribution. The `model` field specifies the student, and `teacher_model` references the trained teacher model entity from Phase 1.", + "source_html": "
\n

Phase 3: Distill into Student

\n

9. Create Knowledge Distillation Job

\n

Now create a distillation job that trains the 1B student using the fine-tuned 3B teacher's output distribution. The model field specifies the student, and teacher_model references the trained teacher model entity from Phase 1.

\n" + }, + { + "type": "markdown", + "source": "**GPU Requirements:**\n\nKD requires loading both student and teacher models, so plan GPU memory accordingly:\n- 1B student + 3B teacher: 1 GPU (24GB+ VRAM each)\n- 3B student + 8B teacher: 4 GPUs\n- 8B student + 70B teacher: 8+ GPUs\n\nUse `teacher_precision=\"bf16\"` (default) to reduce teacher memory footprint.", + "source_html": "

GPU Requirements:

\n

KD requires loading both student and teacher models, so plan GPU memory accordingly:

\n
    \n
  • 1B student + 3B teacher: 1 GPU (24GB+ VRAM each)
    • \n
  • 3B student + 8B teacher: 4 GPUs
    • \n
  • 8B student + 70B teacher: 8+ GPUs
    • \n
\n

Use teacher_precision="bf16" (default) to reduce teacher memory footprint.

\n" + }, + { + "type": "code", + "source": "KD_JOB_NAME = f\"my-kd-job-{job_suffix}\"\n\nkd_job = client.customization.jobs.create(\n name=KD_JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{student_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=DistillationTrainingParam(\n type=\"distillation\",\n teacher_model=f\"default/{TRAINED_TEACHER_NAME}\",\n teacher_precision=\"bf16\",\n distillation_ratio=0.5,\n distillation_temperature=2.0,\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n )\n)\n\nDISTILLED_STUDENT_NAME = kd_job.spec.output.name\nprint(f\"Distillation job: {kd_job.name}\")\nprint(f\"Output student model: {DISTILLED_STUDENT_NAME}\")", + "language": "python", + "source_html": "KD_JOB_NAME = f"my-kd-job-{job_suffix}"\n\nkd_job = client.customization.jobs.create(\n name=KD_JOB_NAME,\n workspace="default",\n spec=CustomizationJobInputParam(\n model=f"default/{student_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=DistillationTrainingParam(\n type="distillation",\n teacher_model=f"default/{TRAINED_TEACHER_NAME}",\n teacher_precision="bf16",\n distillation_ratio=0.5,\n distillation_temperature=2.0,\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n )\n)\n\nDISTILLED_STUDENT_NAME = kd_job.spec.output.name\nprint(f"Distillation job: {kd_job.name}")\nprint(f"Output student model: {DISTILLED_STUDENT_NAME}")\n" + }, + { + "type": "markdown", + "source": "### 10. Track Distillation Progress", + "source_html": "

10. Track Distillation Progress

\n" + }, + { + "type": "code", + "source": "kd_status = wait_for_job(KD_JOB_NAME)", + "language": "python", + "source_html": "kd_status = wait_for_job(KD_JOB_NAME)\n" + }, + { + "type": "markdown", + "source": "---\n\n## Phase 4: Evaluate the Distilled Student Model\n\n### 11. Deploy the Distilled Student Model\n\nThe output model has the same architecture as the 1B student—only its weights have been updated via distillation. It requires just 1 GPU to deploy.", + "source_html": "
\n

Phase 4: Evaluate the Distilled Student Model

\n

11. Deploy the Distilled Student Model

\n

The output model has the same architecture as the 1B student—only its weights have been updated via distillation. It requires just 1 GPU to deploy.

\n" + }, + { + "type": "code", + "source": "deploy_suffix_2 = uuid.uuid4().hex[:4]\nSTUDENT_DEPLOYMENT_CONFIG = f\"kd-student-deploy-cfg-{deploy_suffix_2}\"\nSTUDENT_DEPLOYMENT_NAME = f\"kd-student-deploy-{deploy_suffix_2}\"\n\nstudent_deployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=STUDENT_DEPLOYMENT_CONFIG,\n nim_deployment=NIMDeploymentParam(\n image_name=\"nvcr.io/nim/nvidia/llm-nim\",\n image_tag=\"1.15.5\",\n gpu=1,\n model_name=DISTILLED_STUDENT_NAME,\n model_namespace=\"default\",\n additional_envs={\"NIM_MODEL_PROFILE\": \"vllm\"},\n ),\n)\n\nstudent_deployment = client.inference.deployments.create(\n workspace=\"default\",\n name=STUDENT_DEPLOYMENT_NAME,\n config=student_deployment_config.name\n)\n\nprint(f\"Student deployment: {student_deployment.name}\")", + "language": "python", + "source_html": "deploy_suffix_2 = uuid.uuid4().hex[:4]\nSTUDENT_DEPLOYMENT_CONFIG = f"kd-student-deploy-cfg-{deploy_suffix_2}"\nSTUDENT_DEPLOYMENT_NAME = f"kd-student-deploy-{deploy_suffix_2}"\n\nstudent_deployment_config = client.inference.deployment_configs.create(\n workspace="default",\n name=STUDENT_DEPLOYMENT_CONFIG,\n nim_deployment=NIMDeploymentParam(\n image_name="nvcr.io/nim/nvidia/llm-nim",\n image_tag="1.15.5",\n gpu=1,\n model_name=DISTILLED_STUDENT_NAME,\n model_namespace="default",\n additional_envs={"NIM_MODEL_PROFILE": "vllm"},\n ),\n)\n\nstudent_deployment = client.inference.deployments.create(\n workspace="default",\n name=STUDENT_DEPLOYMENT_NAME,\n config=student_deployment_config.name\n)\n\nprint(f"Student deployment: {student_deployment.name}")\n" + }, + { + "type": "code", + "source": "wait_for_deployment(STUDENT_DEPLOYMENT_NAME)", + "language": "python", + "source_html": "wait_for_deployment(STUDENT_DEPLOYMENT_NAME)\n" + }, + { + "type": "markdown", + "source": "### 12. Generate Student Predictions on Test Set", + "source_html": "

12. Generate Student Predictions on Test Set

\n" + }, + { + "type": "code", + "source": "print(\"Generating distilled student predictions...\")\nstudent_completions = generate_completions(STUDENT_DEPLOYMENT_NAME, DISTILLED_STUDENT_NAME, contexts, questions)\nprint(f\"Generated {len(student_completions)} student predictions\")\nprint(f\"\\nSample student output: {student_completions[0]}\")", + "language": "python", + "source_html": "print("Generating distilled student predictions...")\nstudent_completions = generate_completions(STUDENT_DEPLOYMENT_NAME, DISTILLED_STUDENT_NAME, contexts, questions)\nprint(f"Generated {len(student_completions)} student predictions")\nprint(f"\\nSample student output: {student_completions[0]}")\n" + }, + { + "type": "markdown", + "source": "### 13. Compute ROUGE Scores\n\nCompare the base student (before distillation) and the distilled student against the ground-truth reference completions using ROUGE metrics.", + "source_html": "

13. Compute ROUGE Scores

\n

Compare the base student (before distillation) and the distilled student against the ground-truth reference completions using ROUGE metrics.

\n" + }, + { + "type": "code", + "source": "import evaluate\n\nrouge = evaluate.load(\"rouge\")\n\nbaseline_scores = rouge.compute(predictions=baseline_completions, references=reference_completions)\nstudent_scores = rouge.compute(predictions=student_completions, references=reference_completions)\n\nmetrics = list(baseline_scores.keys())\nheader = f\"{'Model':<35} \" + \" \".join(f\"{m:>10}\" for m in metrics)\nseparator = \"-\" * len(header)\n\nprint(\"=\" * 60)\nprint(\"ROUGE SCORE COMPARISON\")\nprint(\"=\" * 60)\nprint(header)\nprint(separator)\nprint(f\"{'Base Student (1B, no training)':<35} \" + \" \".join(f\"{baseline_scores[m]:>10.4f}\" for m in metrics))\nprint(f\"{'Distilled Student (1B, KD)':<35} \" + \" \".join(f\"{student_scores[m]:>10.4f}\" for m in metrics))", + "language": "python", + "source_html": "import evaluate\n\nrouge = evaluate.load("rouge")\n\nbaseline_scores = rouge.compute(predictions=baseline_completions, references=reference_completions)\nstudent_scores = rouge.compute(predictions=student_completions, references=reference_completions)\n\nmetrics = list(baseline_scores.keys())\nheader = f"{'Model':<35} " + " ".join(f"{m:>10}" for m in metrics)\nseparator = "-" * len(header)\n\nprint("=" * 60)\nprint("ROUGE SCORE COMPARISON")\nprint("=" * 60)\nprint(header)\nprint(separator)\nprint(f"{'Base Student (1B, no training)':<35} " + " ".join(f"{baseline_scores[m]:>10.4f}" for m in metrics))\nprint(f"{'Distilled Student (1B, KD)':<35} " + " ".join(f"{student_scores[m]:>10.4f}" for m in metrics))\n" + }, + { + "type": "code", + "source": "print(\"Sample predictions (first 3):\\n\")\nfor i in range(min(3, len(contexts))):\n print(f\"--- Sample {i + 1} ---\")\n print(f\"Question: {questions[i]}\")\n print(f\"Reference: {reference_completions[i]}\")\n print(f\"Baseline: {baseline_completions[i][:200]}\")\n print(f\"Distilled: {student_completions[i][:200]}\")\n print()", + "language": "python", + "source_html": "print("Sample predictions (first 3):\\n")\nfor i in range(min(3, len(contexts))):\n print(f"--- Sample {i + 1} ---")\n print(f"Question: {questions[i]}")\n print(f"Reference: {reference_completions[i]}")\n print(f"Baseline: {baseline_completions[i][:200]}")\n print(f"Distilled: {student_completions[i][:200]}")\n print()\n" + }, + { + "type": "markdown", + "source": "**Interpreting ROUGE Scores:**\n\n| Metric | Measures |\n|--------|----------|\n| **ROUGE-1** | Unigram overlap between prediction and reference |\n| **ROUGE-2** | Bigram overlap (captures phrase-level similarity) |\n| **ROUGE-L** | Longest common subsequence (captures sentence structure) |\n| **ROUGE-Lsum** | ROUGE-L computed over full summaries |\n\n**What to expect:**\n- The base student (1B, no training) provides a lower bound since it has not seen the task data\n- The distilled student (1B, KD) should significantly outperform the base student, demonstrating the knowledge transferred from the 3B teacher\n- If the distilled student scores are not much higher than the baseline, try increasing `distillation_temperature`, adjusting `distillation_ratio`, or training for more epochs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated HuggingFace models (Llama, Gemma), accept the license on the model page\n- Check both `model` (student) and `teacher_model` URNs are correct\n- Ensure both model entities exist: `client.models.retrieve(name=..., workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n\nKD loads both models, so OOM is more likely than with SFT:\n1. **First try:** Use `teacher_precision=\"bf16\"` to reduce teacher memory\n2. **Still OOM:** Reduce `micro_batch_size` to 1\n3. **Still OOM:** Reduce `batch_size` and `max_seq_length`\n4. **Last resort:** Increase `num_gpus_per_node`\n\n**No chat template / `/chat/completions` fails:**\n- Use Instruct model variants (e.g., `Llama-3.2-1B-Instruct`) instead of base models (`Llama-3.2-1B`). Base models do not include a chat template in their tokenizer, so the output model will also lack one.\n\n**Distilled model quality is poor:**\n- Increase `distillation_temperature` (try 2.0–5.0) to transfer more nuanced knowledge\n- Adjust `distillation_ratio`—if dataset labels are high-quality, lower the ratio; if the teacher is strong, raise it\n- Increase `epochs` or `max_steps` for more training\n- Verify teacher and student share the same vocabulary\n\n**Vocabulary mismatch error:**\n- Teacher and student must use the same tokenizer. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct)\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=job.spec.output.name, workspace=\"default\")`\n- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace=\"default\")`\n- The distilled model has the same size as the student, so GPU requirements match the student model\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning\n- Learn about [Full SFT](./sft-customization-job) for direct supervised fine-tuning", + "source_html": "

Interpreting ROUGE Scores:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
MetricMeasures
ROUGE-1Unigram overlap between prediction and reference
ROUGE-2Bigram overlap (captures phrase-level similarity)
ROUGE-LLongest common subsequence (captures sentence structure)
ROUGE-LsumROUGE-L computed over full summaries
\n

What to expect:

\n
    \n
  • The base student (1B, no training) provides a lower bound since it has not seen the task data
    • \n
  • The distilled student (1B, KD) should significantly outperform the base student, demonstrating the knowledge transferred from the 3B teacher
    • \n
  • If the distilled student scores are not much higher than the baseline, try increasing distillation_temperature, adjusting distillation_ratio, or training for more epochs
    • \n
\n
\n

Hyperparameters

\n

For detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.

\n
\n

Troubleshooting

\n

Job fails during model download:

\n
    \n
  • Verify authentication secrets are configured (refer to Managing Secrets)
    • \n
  • For gated HuggingFace models (Llama, Gemma), accept the license on the model page
    • \n
  • Check both model (student) and teacher_model URNs are correct
    • \n
  • Ensure both model entities exist: client.models.retrieve(name=..., workspace="default")
    • \n
\n

Job fails with OOM (Out of Memory) error:

\n

KD loads both models, so OOM is more likely than with SFT:

\n
    \n
  1. First try: Use teacher_precision="bf16" to reduce teacher memory
    2. \n
  2. Still OOM: Reduce micro_batch_size to 1
    3. \n
  3. Still OOM: Reduce batch_size and max_seq_length
    4. \n
  4. Last resort: Increase num_gpus_per_node
    5. \n
\n

No chat template / /chat/completions fails:

\n
    \n
  • Use Instruct model variants (e.g., Llama-3.2-1B-Instruct) instead of base models (Llama-3.2-1B). Base models do not include a chat template in their tokenizer, so the output model will also lack one.
    • \n
\n

Distilled model quality is poor:

\n
    \n
  • Increase distillation_temperature (try 2.0–5.0) to transfer more nuanced knowledge
    • \n
  • Adjust distillation_ratio—if dataset labels are high-quality, lower the ratio; if the teacher is strong, raise it
    • \n
  • Increase epochs or max_steps for more training
    • \n
  • Verify teacher and student share the same vocabulary
    • \n
\n

Vocabulary mismatch error:

\n
    \n
  • Teacher and student must use the same tokenizer. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct)
    • \n
\n

Deployment fails:

\n
    \n
  • Verify output model exists: client.models.retrieve(name=job.spec.output.name, workspace="default")
    • \n
  • Check deployment logs: client.inference.deployments.get_logs(name=deployment.name, workspace="default")
    • \n
  • The distilled model has the same size as the student, so GPU requirements match the student model
    • \n
\n

Next Steps

\n\n" + } + ] +} \ No newline at end of file diff --git a/docs/fern/components/notebooks/distillation-customization-job.ts b/docs/fern/components/notebooks/distillation-customization-job.ts new file mode 100644 index 00000000000..215e9f1b209 --- /dev/null +++ b/docs/fern/components/notebooks/distillation-customization-job.ts @@ -0,0 +1,202 @@ +/** Auto-generated by ipynb-to-fern-json.py - do not edit */ +export default { cells: [ + { + "type": "markdown", + "source": "\n\n\n# Knowledge Distillation Customization\n\nLearn how to train a smaller student model to mimic a larger teacher model using knowledge distillation (KD).\n\n## About\n\nKnowledge Distillation transfers knowledge from a large **teacher** model to a smaller **student** model. During training, the student learns to match the teacher's output probability distribution, producing a compact model that retains much of the teacher's capability.\n\n**What you can achieve with KD:**\n\n- **Compress models:** Distill a 3B model into a 1B model for faster inference and lower deployment costs\n- **Reduce latency:** Deploy a smaller model that responds faster while preserving quality\n- **Lower resource requirements:** Serve a distilled model on fewer GPUs\n\n### KD vs SFT: Understanding the Trade-offs\n\n| Aspect | Full SFT | Knowledge Distillation |\n| --- | --- | --- |\n| **Training signal** | Ground-truth labels only | Teacher's soft probability distribution + labels |\n| **Knowledge source** | Dataset examples | Teacher model's learned representations |\n| **Output model size** | Same as input model | Typically a smaller student model |\n| **GPU requirements** | Needs to fit one model | Needs to fit both teacher and student in memory |\n| **Best for** | Domain adaptation, new knowledge injection | Model compression, latency reduction |\n\n### Key Parameters\n\n| Parameter | Default | Description |\n| --- | --- | --- |\n| `teacher_model` | *(required)* | Teacher model entity URN (e.g., `default/llama-3-2-3b-teacher`) |\n| `teacher_precision` | `bf16` | Precision for the frozen teacher (`bf16`, `fp16`, `fp32`). Lower = less memory |\n| `distillation_ratio` | `0.5` | Balance between CE loss and KD loss. `0.0` = CE only, `1.0` = KD only |\n| `distillation_temperature` | `1.0` | Softmax temperature. Higher = softer distributions, more knowledge transfer |\n\n### Workflow Overview\n\nThis tutorial follows a complete distillation pipeline:\n\n1. **Fine-tune the teacher** (SFT on the task dataset) so it learns the domain\n2. **Establish a baseline** by deploying the base student model and measuring ROUGE scores\n3. **Distill into the student** using the fine-tuned teacher's soft targets\n4. **Evaluate the distilled student** and compare ROUGE scores against the baseline\n\n**When to choose KD:**\n\n- You have a high-quality large model and want a smaller, faster version\n- Deployment latency or cost is a constraint\n- The teacher and student share the same vocabulary (e.g., both are Llama models)\n\n**When to choose SFT instead:** Refer to the [Full SFT tutorial](./sft-customization-job) when you want to train a model directly on labeled data without a teacher.", + "source_html": "\n\n

Knowledge Distillation Customization

\n

Learn how to train a smaller student model to mimic a larger teacher model using knowledge distillation (KD).

\n

About

\n

Knowledge Distillation transfers knowledge from a large teacher model to a smaller student model. During training, the student learns to match the teacher's output probability distribution, producing a compact model that retains much of the teacher's capability.

\n

What you can achieve with KD:

\n
    \n
  • Compress models: Distill a 3B model into a 1B model for faster inference and lower deployment costs
    • \n
  • Reduce latency: Deploy a smaller model that responds faster while preserving quality
    • \n
  • Lower resource requirements: Serve a distilled model on fewer GPUs
    • \n
\n

KD vs SFT: Understanding the Trade-offs

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
AspectFull SFTKnowledge Distillation
Training signalGround-truth labels onlyTeacher's soft probability distribution + labels
Knowledge sourceDataset examplesTeacher model's learned representations
Output model sizeSame as input modelTypically a smaller student model
GPU requirementsNeeds to fit one modelNeeds to fit both teacher and student in memory
Best forDomain adaptation, new knowledge injectionModel compression, latency reduction
\n

Key Parameters

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ParameterDefaultDescription
teacher_model(required)Teacher model entity URN (e.g., default/llama-3-2-3b-teacher)
teacher_precisionbf16Precision for the frozen teacher (bf16, fp16, fp32). Lower = less memory
distillation_ratio0.5Balance between CE loss and KD loss. 0.0 = CE only, 1.0 = KD only
distillation_temperature1.0Softmax temperature. Higher = softer distributions, more knowledge transfer
\n

Workflow Overview

\n

This tutorial follows a complete distillation pipeline:

\n
    \n
  1. Fine-tune the teacher (SFT on the task dataset) so it learns the domain
    2. \n
  2. Establish a baseline by deploying the base student model and measuring ROUGE scores
    3. \n
  3. Distill into the student using the fine-tuned teacher's soft targets
    4. \n
  4. Evaluate the distilled student and compare ROUGE scores against the baseline
    5. \n
\n

When to choose KD:

\n
    \n
  • You have a high-quality large model and want a smaller, faster version
    • \n
  • Deployment latency or cost is a constraint
    • \n
  • The teacher and student share the same vocabulary (e.g., both are Llama models)
    • \n
\n

When to choose SFT instead: Refer to the Full SFT tutorial when you want to train a model directly on labeled data without a teacher.

\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n2. **Installed the Python SDK** (included with `pip install nemo-platform`)\n3. **Installed evaluation dependencies:**\n\n```sh\npip install evaluate rouge_score datasets\n```", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Completed the Quickstart to install and deploy NeMo Platform locally
    2. \n
  2. Installed the Python SDK (included with pip install nemo-platform)
    3. \n
  3. Installed evaluation dependencies:
    4. \n
\n
pip install evaluate rouge_score datasets\n
\n" + }, + { + "type": "markdown", + "source": "## Quick Start\n\n### 1. Initialize SDK\n\nThe SDK needs to know your NeMo Platform server URL. By default, `http://localhost:8080` is used in accordance with the [Quickstart](../../get-started/quickstart.md) guide. If NeMo Platform is running at a custom location, you can override the URL by setting the `NMP_BASE_URL` environment variable:\n\n```sh\nexport NMP_BASE_URL=\n```", + "source_html": "

Quick Start

\n

1. Initialize SDK

\n

The SDK needs to know your NeMo Platform server URL. By default, http://localhost:8080 is used in accordance with the Quickstart guide. If NeMo Platform is running at a custom location, you can override the URL by setting the NMP_BASE_URL environment variable:

\n
export NMP_BASE_URL=<YOUR_NMP_BASE_URL>\n
\n" + }, + { + "type": "code", + "source": "import json\nimport os\nimport time\nimport uuid\nfrom pathlib import Path\n\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace=\"default\"\n)", + "language": "python", + "source_html": "import json\nimport os\nimport time\nimport uuid\nfrom pathlib import Path\n\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace="default"\n)\n" + }, + { + "type": "markdown", + "source": "### 2. Prepare Dataset\n\nKnowledge distillation uses the same dataset formats as SFT. We use the SQuAD dataset for both teacher training and distillation so that the teacher first learns the task, then transfers that knowledge to the student.\n\nWe also hold out a small **test split** for ROUGE evaluation at the end.", + "source_html": "

2. Prepare Dataset

\n

Knowledge distillation uses the same dataset formats as SFT. We use the SQuAD dataset for both teacher training and distillation so that the teacher first learns the task, then transfers that knowledge to the student.

\n

We also hold out a small test split for ROUGE evaluation at the end.

\n" + }, + { + "type": "code", + "source": "from datasets import load_dataset, DatasetDict\n\nprint(\"Loading dataset rajpurkar/squad\")\nraw_dataset = load_dataset(\"rajpurkar/squad\")\nif not isinstance(raw_dataset, DatasetDict):\n raise ValueError(\"Dataset does not contain expected splits\")\n\nprint(\"Loaded dataset\")\n\nSEED = 1234\nTRAINING_SIZE = 3000\nVALIDATION_SIZE = 300\nTEST_SIZE = 100\nDATASET_PATH = Path(\"kd-dataset\").absolute()\n\nos.makedirs(DATASET_PATH, exist_ok=True)\n\ntrain_set = raw_dataset.get('train')\nsplit = train_set.train_test_split(test_size=0.05, seed=SEED)\n\ntrain_ds = split['train'].select(range(min(TRAINING_SIZE, len(split['train']))))\nval_ds = split['test'].select(range(min(VALIDATION_SIZE, len(split['test']))))\ntest_ds = split['test'].select(range(VALIDATION_SIZE, min(VALIDATION_SIZE + TEST_SIZE, len(split['test']))))\n\n\ndef convert_squad(example):\n \"\"\"Convert SQuAD format to prompt/completion format.\"\"\"\n prompt = f\"Context: {example['context']} Question: {example['question']} Answer:\"\n completion = example[\"answers\"][\"text\"][0]\n return {\"prompt\": prompt, \"completion\": completion}\n\n\ndef write_jsonl(dataset, path):\n with open(path, \"w\", encoding=\"utf-8\") as f:\n for example in dataset:\n f.write(json.dumps(convert_squad(example)) + \"\\n\")\n\n\ndef write_test_jsonl(dataset, path):\n \"\"\"Save test split with raw context/question for chat-style evaluation.\"\"\"\n with open(path, \"w\", encoding=\"utf-8\") as f:\n for example in dataset:\n f.write(json.dumps({\n \"context\": example[\"context\"],\n \"question\": example[\"question\"],\n \"completion\": example[\"answers\"][\"text\"][0],\n }) + \"\\n\")\n\n\nwrite_jsonl(train_ds, f\"{DATASET_PATH}/training.jsonl\")\nwrite_jsonl(val_ds, f\"{DATASET_PATH}/validation.jsonl\")\nwrite_test_jsonl(test_ds, f\"{DATASET_PATH}/testing.jsonl\")\n\nprint(f\"Training: {len(train_ds)} rows\")\nprint(f\"Validation: {len(val_ds)} rows\")\nprint(f\"Test: {len(test_ds)} rows\")\n\nwith open(f\"{DATASET_PATH}/training.jsonl\", 'r') as f:\n sample = json.loads(f.readline())\n print(f\"\\nSample prompt: {sample['prompt'][:150]}...\")\n print(f\"Sample completion: {sample['completion']}\")", + "language": "python", + "source_html": "from datasets import load_dataset, DatasetDict\n\nprint("Loading dataset rajpurkar/squad")\nraw_dataset = load_dataset("rajpurkar/squad")\nif not isinstance(raw_dataset, DatasetDict):\n raise ValueError("Dataset does not contain expected splits")\n\nprint("Loaded dataset")\n\nSEED = 1234\nTRAINING_SIZE = 3000\nVALIDATION_SIZE = 300\nTEST_SIZE = 100\nDATASET_PATH = Path("kd-dataset").absolute()\n\nos.makedirs(DATASET_PATH, exist_ok=True)\n\ntrain_set = raw_dataset.get('train')\nsplit = train_set.train_test_split(test_size=0.05, seed=SEED)\n\ntrain_ds = split['train'].select(range(min(TRAINING_SIZE, len(split['train']))))\nval_ds = split['test'].select(range(min(VALIDATION_SIZE, len(split['test']))))\ntest_ds = split['test'].select(range(VALIDATION_SIZE, min(VALIDATION_SIZE + TEST_SIZE, len(split['test']))))\n\n\ndef convert_squad(example):\n """Convert SQuAD format to prompt/completion format."""\n prompt = f"Context: {example['context']} Question: {example['question']} Answer:"\n completion = example["answers"]["text"][0]\n return {"prompt": prompt, "completion": completion}\n\n\ndef write_jsonl(dataset, path):\n with open(path, "w", encoding="utf-8") as f:\n for example in dataset:\n f.write(json.dumps(convert_squad(example)) + "\\n")\n\n\ndef write_test_jsonl(dataset, path):\n """Save test split with raw context/question for chat-style evaluation."""\n with open(path, "w", encoding="utf-8") as f:\n for example in dataset:\n f.write(json.dumps({\n "context": example["context"],\n "question": example["question"],\n "completion": example["answers"]["text"][0],\n }) + "\\n")\n\n\nwrite_jsonl(train_ds, f"{DATASET_PATH}/training.jsonl")\nwrite_jsonl(val_ds, f"{DATASET_PATH}/validation.jsonl")\nwrite_test_jsonl(test_ds, f"{DATASET_PATH}/testing.jsonl")\n\nprint(f"Training: {len(train_ds)} rows")\nprint(f"Validation: {len(val_ds)} rows")\nprint(f"Test: {len(test_ds)} rows")\n\nwith open(f"{DATASET_PATH}/training.jsonl", 'r') as f:\n sample = json.loads(f.readline())\n print(f"\\nSample prompt: {sample['prompt'][:150]}...")\n print(f"Sample completion: {sample['completion']}")\n" + }, + { + "type": "code", + "source": "DATASET_NAME = \"kd-dataset\"\n\ntry:\n client.files.filesets.create(\n workspace=\"default\",\n name=DATASET_NAME,\n description=\"Knowledge distillation training data\"\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\nclient.files.upload(\n local_path=DATASET_PATH,\n remote_path=\"\",\n fileset=DATASET_NAME,\n workspace=\"default\"\n)\n\nprint(\"Uploaded files:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace=\"default\").data], indent=2))", + "language": "python", + "source_html": "DATASET_NAME = "kd-dataset"\n\ntry:\n client.files.filesets.create(\n workspace="default",\n name=DATASET_NAME,\n description="Knowledge distillation training data"\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\nclient.files.upload(\n local_path=DATASET_PATH,\n remote_path="",\n fileset=DATASET_NAME,\n workspace="default"\n)\n\nprint("Uploaded files:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace="default").data], indent=2))\n" + }, + { + "type": "markdown", + "source": "### 3. Secrets Setup\n\nIn this tutorial we use two Llama 3.2 Instruct models from HuggingFace:\n- **Teacher:** [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) (3B parameters)\n- **Student:** [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) (1B parameters)\n\nBoth models share the same tokenizer/vocabulary (required for knowledge distillation) and include a chat template for deployment with `/chat/completions`.\n\n**HuggingFace Authentication:**\n- For gated models (Llama, Gemma), you must provide a HuggingFace token via the `token_secret` parameter\n- Get your token from [HuggingFace Settings](https://huggingface.co/settings/tokens) (requires Read access)\n- Accept the model's terms on the HuggingFace model page before using it:\n - [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct)\n - [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct)", + "source_html": "

3. Secrets Setup

\n

In this tutorial we use two Llama 3.2 Instruct models from HuggingFace:

\n\n

Both models share the same tokenizer/vocabulary (required for knowledge distillation) and include a chat template for deployment with /chat/completions.

\n

HuggingFace Authentication:

\n\n" + }, + { + "type": "code", + "source": "HF_TOKEN = os.getenv(\"HF_TOKEN\")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f\"{label} is not set\")\n try:\n secret = client.secrets.create(\n name=name,\n workspace=\"default\",\n value=value,\n )\n print(f\"Created secret: {name}\")\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n return client.secrets.retrieve(name=name, workspace=\"default\")\n\n\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")\nprint(\"HF_TOKEN secret:\")\nprint(hf_secret.model_dump_json(indent=2))", + "language": "python", + "source_html": "HF_TOKEN = os.getenv("HF_TOKEN")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f"{label} is not set")\n try:\n secret = client.secrets.create(\n name=name,\n workspace="default",\n value=value,\n )\n print(f"Created secret: {name}")\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n return client.secrets.retrieve(name=name, workspace="default")\n\n\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\nprint("HF_TOKEN secret:")\nprint(hf_secret.model_dump_json(indent=2))\n" + }, + { + "type": "markdown", + "source": "### 4. Create Model FileSets and Model Entities\n\nKnowledge distillation requires **two** model entities:\n1. **Student model** — the smaller model that will be trained ([meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct))\n2. **Teacher model** — the larger model that provides soft targets ([meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct))\n\nUsing the Instruct variants ensures the output model includes a chat template, which is required for the `/chat/completions` inference endpoint.", + "source_html": "

4. Create Model FileSets and Model Entities

\n

Knowledge distillation requires two model entities:

\n
    \n
  1. Student model — the smaller model that will be trained (meta-llama/Llama-3.2-1B-Instruct)
    2. \n
  2. Teacher model — the larger model that provides soft targets (meta-llama/Llama-3.2-3B-Instruct)
    3. \n
\n

Using the Instruct variants ensures the output model includes a chat template, which is required for the /chat/completions inference endpoint.

\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nSPEC_TIMEOUT_SECONDS = 120\n\n\ndef create_model(hf_repo: str, model_name: str, description: str):\n \"\"\"Create a fileset + model entity and wait for ModelSpec.\"\"\"\n try:\n client.files.filesets.create(\n workspace=\"default\",\n name=model_name,\n description=description,\n storage=HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n repo_id=hf_repo,\n repo_type=\"model\",\n token_secret=hf_secret.name\n )\n )\n print(f\"Created fileset: {model_name}\")\n except ConflictError:\n print(f\"Fileset '{model_name}' already exists.\")\n\n try:\n model = client.models.create(\n workspace=\"default\",\n name=model_name,\n fileset=f\"default/{model_name}\",\n )\n print(f\"Created Model Entity: {model_name}\")\n except ConflictError:\n print(f\"Model '{model_name}' already exists. Updating fileset.\")\n model = client.models.update(\n workspace=\"default\",\n name=model_name,\n fileset=f\"default/{model_name}\",\n )\n\n print(f\"Waiting for ModelSpec on {model_name}...\")\n spec_start = time.time()\n while not model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f\"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS}s\")\n time.sleep(2)\n model = client.models.retrieve(workspace=\"default\", name=model_name)\n print(f\"ModelSpec populated: {model.spec}\")\n return model\n\n\nstudent_model = create_model(\n hf_repo=\"meta-llama/Llama-3.2-1B-Instruct\",\n model_name=\"llama-3-2-1b-student\",\n description=\"Llama 3.2 1B Instruct student model\",\n)\n\nprint()\n\nteacher_model = create_model(\n hf_repo=\"meta-llama/Llama-3.2-3B-Instruct\",\n model_name=\"llama-3-2-3b-teacher\",\n description=\"Llama 3.2 3B Instruct teacher model\",\n)", + "language": "python", + "source_html": "from nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nSPEC_TIMEOUT_SECONDS = 120\n\n\ndef create_model(hf_repo: str, model_name: str, description: str):\n """Create a fileset + model entity and wait for ModelSpec."""\n try:\n client.files.filesets.create(\n workspace="default",\n name=model_name,\n description=description,\n storage=HuggingfaceStorageConfigParam(\n type="huggingface",\n repo_id=hf_repo,\n repo_type="model",\n token_secret=hf_secret.name\n )\n )\n print(f"Created fileset: {model_name}")\n except ConflictError:\n print(f"Fileset '{model_name}' already exists.")\n\n try:\n model = client.models.create(\n workspace="default",\n name=model_name,\n fileset=f"default/{model_name}",\n )\n print(f"Created Model Entity: {model_name}")\n except ConflictError:\n print(f"Model '{model_name}' already exists. Updating fileset.")\n model = client.models.update(\n workspace="default",\n name=model_name,\n fileset=f"default/{model_name}",\n )\n\n print(f"Waiting for ModelSpec on {model_name}...")\n spec_start = time.time()\n while not model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS}s")\n time.sleep(2)\n model = client.models.retrieve(workspace="default", name=model_name)\n print(f"ModelSpec populated: {model.spec}")\n return model\n\n\nstudent_model = create_model(\n hf_repo="meta-llama/Llama-3.2-1B-Instruct",\n model_name="llama-3-2-1b-student",\n description="Llama 3.2 1B Instruct student model",\n)\n\nprint()\n\nteacher_model = create_model(\n hf_repo="meta-llama/Llama-3.2-3B-Instruct",\n model_name="llama-3-2-3b-teacher",\n description="Llama 3.2 3B Instruct teacher model",\n)\n" + }, + { + "type": "markdown", + "source": "---\n\n## Phase 1: Fine-Tune the Teacher\n\n### 5. Train Teacher with Full SFT\n\nFor best distillation results, fine-tune the teacher on the **same dataset** that will be used for distillation. This ensures the teacher has learned the task-specific knowledge that the student will inherit.\n\nWe train the 3B Instruct model with Full SFT on the SQuAD dataset.", + "source_html": "
\n

Phase 1: Fine-Tune the Teacher

\n

5. Train Teacher with Full SFT

\n

For best distillation results, fine-tune the teacher on the same dataset that will be used for distillation. This ensures the teacher has learned the task-specific knowledge that the student will inherit.

\n

We train the 3B Instruct model with Full SFT on the SQuAD dataset.

\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n DistillationTrainingParam,\n ParallelismParamsParam,\n)\n\njob_suffix = uuid.uuid4().hex[:4]\n\nTEACHER_JOB_NAME = f\"teacher-sft-job-{job_suffix}\"\n\nteacher_job = client.customization.jobs.create(\n name=TEACHER_JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{teacher_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n )\n)\n\nTRAINED_TEACHER_NAME = teacher_job.spec.output.name\nprint(f\"Teacher training job: {teacher_job.name}\")\nprint(f\"Output teacher model: {TRAINED_TEACHER_NAME}\")", + "language": "python", + "source_html": "from nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n DistillationTrainingParam,\n ParallelismParamsParam,\n)\n\njob_suffix = uuid.uuid4().hex[:4]\n\nTEACHER_JOB_NAME = f"teacher-sft-job-{job_suffix}"\n\nteacher_job = client.customization.jobs.create(\n name=TEACHER_JOB_NAME,\n workspace="default",\n spec=CustomizationJobInputParam(\n model=f"default/{teacher_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n )\n)\n\nTRAINED_TEACHER_NAME = teacher_job.spec.output.name\nprint(f"Teacher training job: {teacher_job.name}")\nprint(f"Output teacher model: {TRAINED_TEACHER_NAME}")\n" + }, + { + "type": "code", + "source": "from IPython.display import clear_output\n\n\ndef wait_for_job(job_name: str):\n \"\"\"Poll job status until completion.\"\"\"\n while True:\n status = client.customization.jobs.get_status(name=job_name, workspace=\"default\")\n clear_output(wait=True)\n print(f\"Job: {job_name}\")\n print(f\"Status: {status.status}\")\n\n for job_step in status.steps or []:\n if job_step.name == \"customization-training-job\":\n for task in job_step.tasks or []:\n details = task.status_details or {}\n step = details.get(\"step\")\n max_steps = details.get(\"max_steps\")\n if step is not None and max_steps is not None:\n print(f\"Progress: Step {step}/{max_steps} ({step / max_steps * 100:.1f}%)\")\n phase = details.get(\"phase\")\n if phase:\n print(f\"Phase: {phase}\")\n break\n break\n\n if status.status in (\"completed\", \"failed\", \"cancelled\", \"error\"):\n print(f\"\\nJob finished: {status.status}\")\n return status\n\n time.sleep(10)\n\n\nteacher_status = wait_for_job(TEACHER_JOB_NAME)", + "language": "python", + "source_html": "from IPython.display import clear_output\n\n\ndef wait_for_job(job_name: str):\n """Poll job status until completion."""\n while True:\n status = client.customization.jobs.get_status(name=job_name, workspace="default")\n clear_output(wait=True)\n print(f"Job: {job_name}")\n print(f"Status: {status.status}")\n\n for job_step in status.steps or []:\n if job_step.name == "customization-training-job":\n for task in job_step.tasks or []:\n details = task.status_details or {}\n step = details.get("step")\n max_steps = details.get("max_steps")\n if step is not None and max_steps is not None:\n print(f"Progress: Step {step}/{max_steps} ({step / max_steps * 100:.1f}%)")\n phase = details.get("phase")\n if phase:\n print(f"Phase: {phase}")\n break\n break\n\n if status.status in ("completed", "failed", "cancelled", "error"):\n print(f"\\nJob finished: {status.status}")\n return status\n\n time.sleep(10)\n\n\nteacher_status = wait_for_job(TEACHER_JOB_NAME)\n" + }, + { + "type": "markdown", + "source": "---\n\n## Phase 2: Establish Baseline (Base Student)\n\n### 6. Deploy the Base Student Model\n\nBefore distillation, deploy the base student model (1B Instruct, without any fine-tuning) to establish a baseline ROUGE score. After distillation, we compare the distilled student against this baseline to measure improvement.", + "source_html": "
\n

Phase 2: Establish Baseline (Base Student)

\n

6. Deploy the Base Student Model

\n

Before distillation, deploy the base student model (1B Instruct, without any fine-tuning) to establish a baseline ROUGE score. After distillation, we compare the distilled student against this baseline to measure improvement.

\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.inference import NIMDeploymentParam\n\nbaseline_suffix = uuid.uuid4().hex[:4]\nBASELINE_DEPLOYMENT_CONFIG = f\"baseline-student-cfg-{baseline_suffix}\"\nBASELINE_DEPLOYMENT_NAME = f\"baseline-student-{baseline_suffix}\"\n\nbaseline_deployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=BASELINE_DEPLOYMENT_CONFIG,\n nim_deployment=NIMDeploymentParam(\n image_name=\"nvcr.io/nim/nvidia/llm-nim\",\n image_tag=\"1.15.5\",\n gpu=1,\n model_name=student_model.name,\n model_namespace=\"default\",\n additional_envs={\"NIM_MODEL_PROFILE\": \"vllm\"}\n ),\n)\n\nbaseline_deployment = client.inference.deployments.create(\n workspace=\"default\",\n name=BASELINE_DEPLOYMENT_NAME,\n config=baseline_deployment_config.name\n)\n\nprint(f\"Baseline student deployment: {baseline_deployment.name}\")", + "language": "python", + "source_html": "from nemo_platform.types.inference import NIMDeploymentParam\n\nbaseline_suffix = uuid.uuid4().hex[:4]\nBASELINE_DEPLOYMENT_CONFIG = f"baseline-student-cfg-{baseline_suffix}"\nBASELINE_DEPLOYMENT_NAME = f"baseline-student-{baseline_suffix}"\n\nbaseline_deployment_config = client.inference.deployment_configs.create(\n workspace="default",\n name=BASELINE_DEPLOYMENT_CONFIG,\n nim_deployment=NIMDeploymentParam(\n image_name="nvcr.io/nim/nvidia/llm-nim",\n image_tag="1.15.5",\n gpu=1,\n model_name=student_model.name,\n model_namespace="default",\n additional_envs={"NIM_MODEL_PROFILE": "vllm"}\n ),\n)\n\nbaseline_deployment = client.inference.deployments.create(\n workspace="default",\n name=BASELINE_DEPLOYMENT_NAME,\n config=baseline_deployment_config.name\n)\n\nprint(f"Baseline student deployment: {baseline_deployment.name}")\n" + }, + { + "type": "code", + "source": "def wait_for_deployment(deployment_name: str, timeout_minutes: int = 30):\n \"\"\"Poll deployment until ready.\"\"\"\n start = time.time()\n timeout = timeout_minutes * 60\n while True:\n dep = client.inference.deployments.retrieve(name=deployment_name, workspace=\"default\")\n elapsed = time.time() - start\n clear_output(wait=True)\n print(f\"Deployment: {deployment_name}\")\n print(f\"Status: {dep.status}\")\n print(f\"Elapsed: {int(elapsed // 60)}m {int(elapsed % 60)}s\")\n\n if dep.status == \"READY\":\n print(\"\\nDeployment is ready!\")\n return dep\n if dep.status in (\"FAILED\", \"ERROR\", \"TERMINATED\", \"LOST\"):\n print(f\"\\nDeployment failed: {dep.status}\")\n return dep\n if elapsed > timeout:\n print(f\"\\nTimeout ({timeout_minutes}m). Check status manually.\")\n return dep\n time.sleep(15)\n\n\nwait_for_deployment(BASELINE_DEPLOYMENT_NAME)", + "language": "python", + "source_html": "def wait_for_deployment(deployment_name: str, timeout_minutes: int = 30):\n """Poll deployment until ready."""\n start = time.time()\n timeout = timeout_minutes * 60\n while True:\n dep = client.inference.deployments.retrieve(name=deployment_name, workspace="default")\n elapsed = time.time() - start\n clear_output(wait=True)\n print(f"Deployment: {deployment_name}")\n print(f"Status: {dep.status}")\n print(f"Elapsed: {int(elapsed // 60)}m {int(elapsed % 60)}s")\n\n if dep.status == "READY":\n print("\\nDeployment is ready!")\n return dep\n if dep.status in ("FAILED", "ERROR", "TERMINATED", "LOST"):\n print(f"\\nDeployment failed: {dep.status}")\n return dep\n if elapsed > timeout:\n print(f"\\nTimeout ({timeout_minutes}m). Check status manually.")\n return dep\n time.sleep(15)\n\n\nwait_for_deployment(BASELINE_DEPLOYMENT_NAME)\n" + }, + { + "type": "markdown", + "source": "### 7. Generate Baseline Predictions on Test Set", + "source_html": "

7. Generate Baseline Predictions on Test Set

\n" + }, + { + "type": "code", + "source": "with open(f\"{DATASET_PATH}/testing.jsonl\", \"r\", encoding=\"utf-8\") as f:\n test_data = [json.loads(line) for line in f]\n\ncontexts = [row[\"context\"] for row in test_data]\nquestions = [row[\"question\"] for row in test_data]\nreference_completions = [row[\"completion\"] for row in test_data]\n\nprint(f\"Test samples: {len(contexts)}\")\nprint(f\"Sample context: {contexts[0]}\")\nprint(f\"Sample question: {questions[0]}\")\nprint(f\"Sample reference: {reference_completions[0]}\")", + "language": "python", + "source_html": "with open(f"{DATASET_PATH}/testing.jsonl", "r", encoding="utf-8") as f:\n test_data = [json.loads(line) for line in f]\n\ncontexts = [row["context"] for row in test_data]\nquestions = [row["question"] for row in test_data]\nreference_completions = [row["completion"] for row in test_data]\n\nprint(f"Test samples: {len(contexts)}")\nprint(f"Sample context: {contexts[0]}")\nprint(f"Sample question: {questions[0]}")\nprint(f"Sample reference: {reference_completions[0]}")\n" + }, + { + "type": "code", + "source": "def generate_completions(\n deployment_name: str,\n output_model_name: str,\n contexts: list[str],\n questions: list[str],\n) -> list[str]:\n \"\"\"Generate completions for a list of context/question pairs using a deployed model.\"\"\"\n completions = []\n for context, question in zip(contexts, questions):\n messages = [\n {\n \"role\": \"user\",\n \"content\": f\"Based on the following context, answer the question.\\n\\nContext: {context}\\n\\nQuestion: {question}\",\n }\n ]\n response = client.inference.gateway.provider.post(\n \"v1/chat/completions\",\n name=deployment_name,\n workspace=\"default\",\n body={\n \"model\": f\"default/{output_model_name}\",\n \"messages\": messages,\n \"temperature\": 0,\n \"max_tokens\": 128,\n }\n )\n completions.append(response[\"choices\"][0][\"message\"][\"content\"])\n return completions\n\n\nprint(\"Generating baseline (base student) predictions...\")\nbaseline_completions = generate_completions(BASELINE_DEPLOYMENT_NAME, student_model.name, contexts, questions)\nprint(f\"Generated {len(baseline_completions)} baseline predictions\")\nprint(f\"\\nSample baseline output: {baseline_completions[0]}\")", + "language": "python", + "source_html": "def generate_completions(\n deployment_name: str,\n output_model_name: str,\n contexts: list[str],\n questions: list[str],\n) -> list[str]:\n """Generate completions for a list of context/question pairs using a deployed model."""\n completions = []\n for context, question in zip(contexts, questions):\n messages = [\n {\n "role": "user",\n "content": f"Based on the following context, answer the question.\\n\\nContext: {context}\\n\\nQuestion: {question}",\n }\n ]\n response = client.inference.gateway.provider.post(\n "v1/chat/completions",\n name=deployment_name,\n workspace="default",\n body={\n "model": f"default/{output_model_name}",\n "messages": messages,\n "temperature": 0,\n "max_tokens": 128,\n }\n )\n completions.append(response["choices"][0]["message"]["content"])\n return completions\n\n\nprint("Generating baseline (base student) predictions...")\nbaseline_completions = generate_completions(BASELINE_DEPLOYMENT_NAME, student_model.name, contexts, questions)\nprint(f"Generated {len(baseline_completions)} baseline predictions")\nprint(f"\\nSample baseline output: {baseline_completions[0]}")\n" + }, + { + "type": "markdown", + "source": "### 8. Delete Baseline Deployment\n\nDelete the baseline student deployment to free GPU resources for the distillation training job and subsequent distilled model deployment.", + "source_html": "

8. Delete Baseline Deployment

\n

Delete the baseline student deployment to free GPU resources for the distillation training job and subsequent distilled model deployment.

\n" + }, + { + "type": "code", + "source": "client.inference.deployments.delete(name=BASELINE_DEPLOYMENT_NAME, workspace=\"default\")\nprint(f\"Deleted baseline deployment: {BASELINE_DEPLOYMENT_NAME}\")\n\n# wait for deployment to be deleted\ntime.sleep(60)\n\nclient.inference.deployment_configs.delete(name=BASELINE_DEPLOYMENT_CONFIG, workspace=\"default\")\nprint(f\"Deleted baseline deployment config: {BASELINE_DEPLOYMENT_CONFIG}\")", + "language": "python", + "source_html": "client.inference.deployments.delete(name=BASELINE_DEPLOYMENT_NAME, workspace="default")\nprint(f"Deleted baseline deployment: {BASELINE_DEPLOYMENT_NAME}")\n\n# wait for deployment to be deleted\ntime.sleep(60)\n\nclient.inference.deployment_configs.delete(name=BASELINE_DEPLOYMENT_CONFIG, workspace="default")\nprint(f"Deleted baseline deployment config: {BASELINE_DEPLOYMENT_CONFIG}")\n" + }, + { + "type": "markdown", + "source": "---\n\n## Phase 3: Distill into Student\n\n### 9. Create Knowledge Distillation Job\n\nNow create a distillation job that trains the 1B student using the **fine-tuned** 3B teacher's output distribution. The `model` field specifies the student, and `teacher_model` references the trained teacher model entity from Phase 1.", + "source_html": "
\n

Phase 3: Distill into Student

\n

9. Create Knowledge Distillation Job

\n

Now create a distillation job that trains the 1B student using the fine-tuned 3B teacher's output distribution. The model field specifies the student, and teacher_model references the trained teacher model entity from Phase 1.

\n" + }, + { + "type": "markdown", + "source": "**GPU Requirements:**\n\nKD requires loading both student and teacher models, so plan GPU memory accordingly:\n- 1B student + 3B teacher: 1 GPU (24GB+ VRAM each)\n- 3B student + 8B teacher: 4 GPUs\n- 8B student + 70B teacher: 8+ GPUs\n\nUse `teacher_precision=\"bf16\"` (default) to reduce teacher memory footprint.", + "source_html": "

GPU Requirements:

\n

KD requires loading both student and teacher models, so plan GPU memory accordingly:

\n
    \n
  • 1B student + 3B teacher: 1 GPU (24GB+ VRAM each)
    • \n
  • 3B student + 8B teacher: 4 GPUs
    • \n
  • 8B student + 70B teacher: 8+ GPUs
    • \n
\n

Use teacher_precision="bf16" (default) to reduce teacher memory footprint.

\n" + }, + { + "type": "code", + "source": "KD_JOB_NAME = f\"my-kd-job-{job_suffix}\"\n\nkd_job = client.customization.jobs.create(\n name=KD_JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{student_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=DistillationTrainingParam(\n type=\"distillation\",\n teacher_model=f\"default/{TRAINED_TEACHER_NAME}\",\n teacher_precision=\"bf16\",\n distillation_ratio=0.5,\n distillation_temperature=2.0,\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n )\n)\n\nDISTILLED_STUDENT_NAME = kd_job.spec.output.name\nprint(f\"Distillation job: {kd_job.name}\")\nprint(f\"Output student model: {DISTILLED_STUDENT_NAME}\")", + "language": "python", + "source_html": "KD_JOB_NAME = f"my-kd-job-{job_suffix}"\n\nkd_job = client.customization.jobs.create(\n name=KD_JOB_NAME,\n workspace="default",\n spec=CustomizationJobInputParam(\n model=f"default/{student_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=DistillationTrainingParam(\n type="distillation",\n teacher_model=f"default/{TRAINED_TEACHER_NAME}",\n teacher_precision="bf16",\n distillation_ratio=0.5,\n distillation_temperature=2.0,\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n )\n)\n\nDISTILLED_STUDENT_NAME = kd_job.spec.output.name\nprint(f"Distillation job: {kd_job.name}")\nprint(f"Output student model: {DISTILLED_STUDENT_NAME}")\n" + }, + { + "type": "markdown", + "source": "### 10. Track Distillation Progress", + "source_html": "

10. Track Distillation Progress

\n" + }, + { + "type": "code", + "source": "kd_status = wait_for_job(KD_JOB_NAME)", + "language": "python", + "source_html": "kd_status = wait_for_job(KD_JOB_NAME)\n" + }, + { + "type": "markdown", + "source": "---\n\n## Phase 4: Evaluate the Distilled Student Model\n\n### 11. Deploy the Distilled Student Model\n\nThe output model has the same architecture as the 1B student—only its weights have been updated via distillation. It requires just 1 GPU to deploy.", + "source_html": "
\n

Phase 4: Evaluate the Distilled Student Model

\n

11. Deploy the Distilled Student Model

\n

The output model has the same architecture as the 1B student—only its weights have been updated via distillation. It requires just 1 GPU to deploy.

\n" + }, + { + "type": "code", + "source": "deploy_suffix_2 = uuid.uuid4().hex[:4]\nSTUDENT_DEPLOYMENT_CONFIG = f\"kd-student-deploy-cfg-{deploy_suffix_2}\"\nSTUDENT_DEPLOYMENT_NAME = f\"kd-student-deploy-{deploy_suffix_2}\"\n\nstudent_deployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=STUDENT_DEPLOYMENT_CONFIG,\n nim_deployment=NIMDeploymentParam(\n image_name=\"nvcr.io/nim/nvidia/llm-nim\",\n image_tag=\"1.15.5\",\n gpu=1,\n model_name=DISTILLED_STUDENT_NAME,\n model_namespace=\"default\",\n additional_envs={\"NIM_MODEL_PROFILE\": \"vllm\"},\n ),\n)\n\nstudent_deployment = client.inference.deployments.create(\n workspace=\"default\",\n name=STUDENT_DEPLOYMENT_NAME,\n config=student_deployment_config.name\n)\n\nprint(f\"Student deployment: {student_deployment.name}\")", + "language": "python", + "source_html": "deploy_suffix_2 = uuid.uuid4().hex[:4]\nSTUDENT_DEPLOYMENT_CONFIG = f"kd-student-deploy-cfg-{deploy_suffix_2}"\nSTUDENT_DEPLOYMENT_NAME = f"kd-student-deploy-{deploy_suffix_2}"\n\nstudent_deployment_config = client.inference.deployment_configs.create(\n workspace="default",\n name=STUDENT_DEPLOYMENT_CONFIG,\n nim_deployment=NIMDeploymentParam(\n image_name="nvcr.io/nim/nvidia/llm-nim",\n image_tag="1.15.5",\n gpu=1,\n model_name=DISTILLED_STUDENT_NAME,\n model_namespace="default",\n additional_envs={"NIM_MODEL_PROFILE": "vllm"},\n ),\n)\n\nstudent_deployment = client.inference.deployments.create(\n workspace="default",\n name=STUDENT_DEPLOYMENT_NAME,\n config=student_deployment_config.name\n)\n\nprint(f"Student deployment: {student_deployment.name}")\n" + }, + { + "type": "code", + "source": "wait_for_deployment(STUDENT_DEPLOYMENT_NAME)", + "language": "python", + "source_html": "wait_for_deployment(STUDENT_DEPLOYMENT_NAME)\n" + }, + { + "type": "markdown", + "source": "### 12. Generate Student Predictions on Test Set", + "source_html": "

12. Generate Student Predictions on Test Set

\n" + }, + { + "type": "code", + "source": "print(\"Generating distilled student predictions...\")\nstudent_completions = generate_completions(STUDENT_DEPLOYMENT_NAME, DISTILLED_STUDENT_NAME, contexts, questions)\nprint(f\"Generated {len(student_completions)} student predictions\")\nprint(f\"\\nSample student output: {student_completions[0]}\")", + "language": "python", + "source_html": "print("Generating distilled student predictions...")\nstudent_completions = generate_completions(STUDENT_DEPLOYMENT_NAME, DISTILLED_STUDENT_NAME, contexts, questions)\nprint(f"Generated {len(student_completions)} student predictions")\nprint(f"\\nSample student output: {student_completions[0]}")\n" + }, + { + "type": "markdown", + "source": "### 13. Compute ROUGE Scores\n\nCompare the base student (before distillation) and the distilled student against the ground-truth reference completions using ROUGE metrics.", + "source_html": "

13. Compute ROUGE Scores

\n

Compare the base student (before distillation) and the distilled student against the ground-truth reference completions using ROUGE metrics.

\n" + }, + { + "type": "code", + "source": "import evaluate\n\nrouge = evaluate.load(\"rouge\")\n\nbaseline_scores = rouge.compute(predictions=baseline_completions, references=reference_completions)\nstudent_scores = rouge.compute(predictions=student_completions, references=reference_completions)\n\nmetrics = list(baseline_scores.keys())\nheader = f\"{'Model':<35} \" + \" \".join(f\"{m:>10}\" for m in metrics)\nseparator = \"-\" * len(header)\n\nprint(\"=\" * 60)\nprint(\"ROUGE SCORE COMPARISON\")\nprint(\"=\" * 60)\nprint(header)\nprint(separator)\nprint(f\"{'Base Student (1B, no training)':<35} \" + \" \".join(f\"{baseline_scores[m]:>10.4f}\" for m in metrics))\nprint(f\"{'Distilled Student (1B, KD)':<35} \" + \" \".join(f\"{student_scores[m]:>10.4f}\" for m in metrics))", + "language": "python", + "source_html": "import evaluate\n\nrouge = evaluate.load("rouge")\n\nbaseline_scores = rouge.compute(predictions=baseline_completions, references=reference_completions)\nstudent_scores = rouge.compute(predictions=student_completions, references=reference_completions)\n\nmetrics = list(baseline_scores.keys())\nheader = f"{'Model':<35} " + " ".join(f"{m:>10}" for m in metrics)\nseparator = "-" * len(header)\n\nprint("=" * 60)\nprint("ROUGE SCORE COMPARISON")\nprint("=" * 60)\nprint(header)\nprint(separator)\nprint(f"{'Base Student (1B, no training)':<35} " + " ".join(f"{baseline_scores[m]:>10.4f}" for m in metrics))\nprint(f"{'Distilled Student (1B, KD)':<35} " + " ".join(f"{student_scores[m]:>10.4f}" for m in metrics))\n" + }, + { + "type": "code", + "source": "print(\"Sample predictions (first 3):\\n\")\nfor i in range(min(3, len(contexts))):\n print(f\"--- Sample {i + 1} ---\")\n print(f\"Question: {questions[i]}\")\n print(f\"Reference: {reference_completions[i]}\")\n print(f\"Baseline: {baseline_completions[i][:200]}\")\n print(f\"Distilled: {student_completions[i][:200]}\")\n print()", + "language": "python", + "source_html": "print("Sample predictions (first 3):\\n")\nfor i in range(min(3, len(contexts))):\n print(f"--- Sample {i + 1} ---")\n print(f"Question: {questions[i]}")\n print(f"Reference: {reference_completions[i]}")\n print(f"Baseline: {baseline_completions[i][:200]}")\n print(f"Distilled: {student_completions[i][:200]}")\n print()\n" + }, + { + "type": "markdown", + "source": "**Interpreting ROUGE Scores:**\n\n| Metric | Measures |\n|--------|----------|\n| **ROUGE-1** | Unigram overlap between prediction and reference |\n| **ROUGE-2** | Bigram overlap (captures phrase-level similarity) |\n| **ROUGE-L** | Longest common subsequence (captures sentence structure) |\n| **ROUGE-Lsum** | ROUGE-L computed over full summaries |\n\n**What to expect:**\n- The base student (1B, no training) provides a lower bound since it has not seen the task data\n- The distilled student (1B, KD) should significantly outperform the base student, demonstrating the knowledge transferred from the 3B teacher\n- If the distilled student scores are not much higher than the baseline, try increasing `distillation_temperature`, adjusting `distillation_ratio`, or training for more epochs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated HuggingFace models (Llama, Gemma), accept the license on the model page\n- Check both `model` (student) and `teacher_model` URNs are correct\n- Ensure both model entities exist: `client.models.retrieve(name=..., workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n\nKD loads both models, so OOM is more likely than with SFT:\n1. **First try:** Use `teacher_precision=\"bf16\"` to reduce teacher memory\n2. **Still OOM:** Reduce `micro_batch_size` to 1\n3. **Still OOM:** Reduce `batch_size` and `max_seq_length`\n4. **Last resort:** Increase `num_gpus_per_node`\n\n**No chat template / `/chat/completions` fails:**\n- Use Instruct model variants (e.g., `Llama-3.2-1B-Instruct`) instead of base models (`Llama-3.2-1B`). Base models do not include a chat template in their tokenizer, so the output model will also lack one.\n\n**Distilled model quality is poor:**\n- Increase `distillation_temperature` (try 2.0–5.0) to transfer more nuanced knowledge\n- Adjust `distillation_ratio`—if dataset labels are high-quality, lower the ratio; if the teacher is strong, raise it\n- Increase `epochs` or `max_steps` for more training\n- Verify teacher and student share the same vocabulary\n\n**Vocabulary mismatch error:**\n- Teacher and student must use the same tokenizer. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct)\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=job.spec.output.name, workspace=\"default\")`\n- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace=\"default\")`\n- The distilled model has the same size as the student, so GPU requirements match the student model\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning\n- Learn about [Full SFT](./sft-customization-job) for direct supervised fine-tuning", + "source_html": "

Interpreting ROUGE Scores:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
MetricMeasures
ROUGE-1Unigram overlap between prediction and reference
ROUGE-2Bigram overlap (captures phrase-level similarity)
ROUGE-LLongest common subsequence (captures sentence structure)
ROUGE-LsumROUGE-L computed over full summaries
\n

What to expect:

\n
    \n
  • The base student (1B, no training) provides a lower bound since it has not seen the task data
    • \n
  • The distilled student (1B, KD) should significantly outperform the base student, demonstrating the knowledge transferred from the 3B teacher
    • \n
  • If the distilled student scores are not much higher than the baseline, try increasing distillation_temperature, adjusting distillation_ratio, or training for more epochs
    • \n
\n
\n

Hyperparameters

\n

For detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.

\n
\n

Troubleshooting

\n

Job fails during model download:

\n
    \n
  • Verify authentication secrets are configured (refer to Managing Secrets)
    • \n
  • For gated HuggingFace models (Llama, Gemma), accept the license on the model page
    • \n
  • Check both model (student) and teacher_model URNs are correct
    • \n
  • Ensure both model entities exist: client.models.retrieve(name=..., workspace="default")
    • \n
\n

Job fails with OOM (Out of Memory) error:

\n

KD loads both models, so OOM is more likely than with SFT:

\n
    \n
  1. First try: Use teacher_precision="bf16" to reduce teacher memory
    2. \n
  2. Still OOM: Reduce micro_batch_size to 1
    3. \n
  3. Still OOM: Reduce batch_size and max_seq_length
    4. \n
  4. Last resort: Increase num_gpus_per_node
    5. \n
\n

No chat template / /chat/completions fails:

\n
    \n
  • Use Instruct model variants (e.g., Llama-3.2-1B-Instruct) instead of base models (Llama-3.2-1B). Base models do not include a chat template in their tokenizer, so the output model will also lack one.
    • \n
\n

Distilled model quality is poor:

\n
    \n
  • Increase distillation_temperature (try 2.0–5.0) to transfer more nuanced knowledge
    • \n
  • Adjust distillation_ratio—if dataset labels are high-quality, lower the ratio; if the teacher is strong, raise it
    • \n
  • Increase epochs or max_steps for more training
    • \n
  • Verify teacher and student share the same vocabulary
    • \n
\n

Vocabulary mismatch error:

\n
    \n
  • Teacher and student must use the same tokenizer. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct)
    • \n
\n

Deployment fails:

\n
    \n
  • Verify output model exists: client.models.retrieve(name=job.spec.output.name, workspace="default")
    • \n
  • Check deployment logs: client.inference.deployments.get_logs(name=deployment.name, workspace="default")
    • \n
  • The distilled model has the same size as the student, so GPU requirements match the student model
    • \n
\n

Next Steps

\n\n" + } +] }; diff --git a/docs/fern/components/notebooks/dpo-customization-job.json b/docs/fern/components/notebooks/dpo-customization-job.json new file mode 100644 index 00000000000..c1ffcb3c495 --- /dev/null +++ b/docs/fern/components/notebooks/dpo-customization-job.json @@ -0,0 +1,203 @@ +{ + "cells": [ + { + "type": "markdown", + "source": "\n\n\n# DPO Customization\n\nLearn how to use the NeMo Platform to create a DPO (Direct Preference Optimization) job using a custom dataset.\n\n## About\n\nDPO is an advanced fine-tuning technique for preference-based alignment. If you're new to fine-tuning, consider starting with [LoRA](./lora-customization-job) or [Full SFT](./sft-customization-job) tutorials first.\n\nDirect Preference Optimization (DPO) is an RL-free alignment algorithm that operates on preference data. Given a prompt and a pair of chosen and rejected responses, DPO aims to increase the probability of the chosen response and decrease the probability of the rejected response relative to a frozen reference model. The actor is initialized using the reference model. For more details, refer to the [DPO paper](https://arxiv.org/pdf/2305.18290).\n\nDPO shares similarities with Full SFT training workflows but differs in a few key ways:\n\n| Aspect | SFT (Supervised Fine-Tuning) | DPO (Direct Preference Optimization) |\n| --- | --- | --- |\n| Data Requirements | Labeled instruction-response pairs where the desired output is explicitly provided | Pairwise preference data, where for a given input, one response is explicitly preferred over another |\n| Learning Objective | Directly teaches the model to generate a specific \"correct\" response | Directly optimizes the model to align with human preferences by maximizing the probability of preferred responses and minimizing rejected ones, without needing an explicit reward model |\n| Alignment Focus | Aligns the model with the specific examples present in its training data | Aligns the model with broader human preferences, which can be more effective for subjective tasks or those without a single \"correct\" answer |\n| Computational Efficiency | Standard fine-tuning efficiency | More computationally efficient than SFT (especially when compared to full RLHF methods) as it bypasses the need to train a separate reward model |\n\n**What you can achieve with DPO:**\n- **Align with human preferences**: Directly optimize your model to produce outputs that align with subjective human preferences without requiring explicit reward modeling\n- **Refine response quality**: Improve helpfulness, harmlessness, honesty, and other nuanced qualities that are easier to compare than to define\n- **Control tone and style**: Adjust the model's communication style, verbosity, formality, and other subjective characteristics\n- **Implement safety guardrails**: Teach the model to avoid harmful or undesirable responses by training on preferred vs. rejected response pairs\n- **Optimize subjective tasks**: Excel at tasks where there are multiple acceptable answers but clear preferences exist (creative writing, dialogue, explanations)\n\n**When to choose DPO:**\n- **Subjective quality matters**: Your task involves style, tone, or other qualities where there's no single \"correct\" answer but clear preferences exist\n- **You have preference data**: You can collect pairwise comparisons (preferred vs. rejected responses) more easily than perfect labeled examples\n- **Refining existing capabilities**: You want to make targeted improvements to an already-trained model without major capability changes\n- **Complex evaluation**: Humans find it easier to compare which of two responses is better than to create the ideal response themselves (especially for multi-turn conversations, creative tasks, or nuanced outputs)\n- **Robust behavior changes**: You need more reliable behavior modification than prompting can provide, without the complexity of full RLHF\n- **Lower compute than RLHF**: You want human preference alignment but with simpler training that doesn't require reinforcement learning infrastructure\n\n**When to choose SFT:**\n- **Clear correct answers**: Your task has objectively correct outputs (code generation, structured data extraction, following specific formats)\n- **High-quality examples**: You have well-labeled input-output pairs that demonstrate exactly what the model should produce\n- **Imitation learning**: You want the model to closely mimic a specific style, format, or knowledge base from expert demonstrations\n- **Foundational capabilities**: You're establishing new task-specific capabilities before fine-tuning preferences (SFT is often done before DPO)\n- **Stable, predictable outputs**: You need consistent formatting or structure that's well-defined in your training examples\n- **Traditional NLP tasks**: Instruction following, translation, summarization, or classification where gold-standard labels exist", + "source_html": "\n\n

DPO Customization

\n

Learn how to use the NeMo Platform to create a DPO (Direct Preference Optimization) job using a custom dataset.

\n

About

\n

DPO is an advanced fine-tuning technique for preference-based alignment. If you're new to fine-tuning, consider starting with LoRA or Full SFT tutorials first.

\n

Direct Preference Optimization (DPO) is an RL-free alignment algorithm that operates on preference data. Given a prompt and a pair of chosen and rejected responses, DPO aims to increase the probability of the chosen response and decrease the probability of the rejected response relative to a frozen reference model. The actor is initialized using the reference model. For more details, refer to the DPO paper.

\n

DPO shares similarities with Full SFT training workflows but differs in a few key ways:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
AspectSFT (Supervised Fine-Tuning)DPO (Direct Preference Optimization)
Data RequirementsLabeled instruction-response pairs where the desired output is explicitly providedPairwise preference data, where for a given input, one response is explicitly preferred over another
Learning ObjectiveDirectly teaches the model to generate a specific "correct" responseDirectly optimizes the model to align with human preferences by maximizing the probability of preferred responses and minimizing rejected ones, without needing an explicit reward model
Alignment FocusAligns the model with the specific examples present in its training dataAligns the model with broader human preferences, which can be more effective for subjective tasks or those without a single "correct" answer
Computational EfficiencyStandard fine-tuning efficiencyMore computationally efficient than SFT (especially when compared to full RLHF methods) as it bypasses the need to train a separate reward model
\n

What you can achieve with DPO:

\n
    \n
  • Align with human preferences: Directly optimize your model to produce outputs that align with subjective human preferences without requiring explicit reward modeling
    • \n
  • Refine response quality: Improve helpfulness, harmlessness, honesty, and other nuanced qualities that are easier to compare than to define
    • \n
  • Control tone and style: Adjust the model's communication style, verbosity, formality, and other subjective characteristics
    • \n
  • Implement safety guardrails: Teach the model to avoid harmful or undesirable responses by training on preferred vs. rejected response pairs
    • \n
  • Optimize subjective tasks: Excel at tasks where there are multiple acceptable answers but clear preferences exist (creative writing, dialogue, explanations)
    • \n
\n

When to choose DPO:

\n
    \n
  • Subjective quality matters: Your task involves style, tone, or other qualities where there's no single "correct" answer but clear preferences exist
    • \n
  • You have preference data: You can collect pairwise comparisons (preferred vs. rejected responses) more easily than perfect labeled examples
    • \n
  • Refining existing capabilities: You want to make targeted improvements to an already-trained model without major capability changes
    • \n
  • Complex evaluation: Humans find it easier to compare which of two responses is better than to create the ideal response themselves (especially for multi-turn conversations, creative tasks, or nuanced outputs)
    • \n
  • Robust behavior changes: You need more reliable behavior modification than prompting can provide, without the complexity of full RLHF
    • \n
  • Lower compute than RLHF: You want human preference alignment but with simpler training that doesn't require reinforcement learning infrastructure
    • \n
\n

When to choose SFT:

\n
    \n
  • Clear correct answers: Your task has objectively correct outputs (code generation, structured data extraction, following specific formats)
    • \n
  • High-quality examples: You have well-labeled input-output pairs that demonstrate exactly what the model should produce
    • \n
  • Imitation learning: You want the model to closely mimic a specific style, format, or knowledge base from expert demonstrations
    • \n
  • Foundational capabilities: You're establishing new task-specific capabilities before fine-tuning preferences (SFT is often done before DPO)
    • \n
  • Stable, predictable outputs: You need consistent formatting or structure that's well-defined in your training examples
    • \n
  • Traditional NLP tasks: Instruction following, translation, summarization, or classification where gold-standard labels exist
    • \n
\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n2. **Installed the Python SDK** (included with `pip install nemo-platform`)", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Completed the Quickstart to install and deploy NeMo Platform locally
    2. \n
  2. Installed the Python SDK (included with pip install nemo-platform)
    3. \n
\n" + }, + { + "type": "markdown", + "source": "## Quick Start\n\n### 1. Initialize SDK\n\nThe SDK needs to know your NeMo Platform server URL. By default, `http://localhost:8080` is used in accordance with the [Quickstart](../../get-started/quickstart.md) guide. If NeMo Platform is running at a custom location, you can override the URL by setting the `NMP_BASE_URL` environment variable:\n\n```sh\nexport NMP_BASE_URL=\n```", + "source_html": "

Quick Start

\n

1. Initialize SDK

\n

The SDK needs to know your NeMo Platform server URL. By default, http://localhost:8080 is used in accordance with the Quickstart guide. If NeMo Platform is running at a custom location, you can override the URL by setting the NMP_BASE_URL environment variable:

\n
export NMP_BASE_URL=<YOUR_NMP_BASE_URL>\n
\n" + }, + { + "type": "code", + "source": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n DpoTrainingParam,\n ParallelismParamsParam,\n)\n\nNMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace=\"default\"\n)", + "language": "python", + "source_html": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n DpoTrainingParam,\n ParallelismParamsParam,\n)\n\nNMP_BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace="default"\n)\n" + }, + { + "type": "markdown", + "source": "### 2. Prepare Dataset\n\nCreate your data in JSONL format - one JSON object per line. The platform auto-detects your data format. Supported dataset formats are listed below.\n\n**Flexible Data Setup:**\n- **No validation file?** The platform automatically creates a 10% validation split\n- **Multiple files?** Upload to `training/` or `validation/` subdirectories—they'll be automatically merged\n- **Format detection:** Your data format is auto-detected at training time\n\nIn this tutorial the following dataset directory structure will be used:\n```\nmy_dataset\n`-- training.jsonl\n`-- validation.jsonl\n```", + "source_html": "

2. Prepare Dataset

\n

Create your data in JSONL format - one JSON object per line. The platform auto-detects your data format. Supported dataset formats are listed below.

\n

Flexible Data Setup:

\n
    \n
  • No validation file? The platform automatically creates a 10% validation split
    • \n
  • Multiple files? Upload to training/ or validation/ subdirectories—they'll be automatically merged
    • \n
  • Format detection: Your data format is auto-detected at training time
    • \n
\n

In this tutorial the following dataset directory structure will be used:

\n
my_dataset\n`-- training.jsonl\n`-- validation.jsonl\n
\n" + }, + { + "type": "markdown", + "source": "#### Binary Preference Format\nDPO training requires preference pairs with three fields:\n- **`prompt`**: The input prompt (can be a string or array of message objects)\n- **`chosen`**: The preferred response\n- **`rejected`**: The less preferred response", + "source_html": "

Binary Preference Format

\n

DPO training requires preference pairs with three fields:

\n
    \n
  • prompt: The input prompt (can be a string or array of message objects)
    • \n
  • chosen: The preferred response
    • \n
  • rejected: The less preferred response
    • \n
\n" + }, + { + "type": "code", + "source": "{\"prompt\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}], \"chosen\": \"The capital of France is Paris. It is the largest city in France and serves as the country's political, economic, and cultural center.\", \"rejected\": \"I think the capital of France might be London or Paris, I'm not entirely sure.\"}", + "language": "python", + "source_html": "{"prompt": [{"role": "user", "content": "What is the capital of France?"}], "chosen": "The capital of France is Paris. It is the largest city in France and serves as the country's political, economic, and cultural center.", "rejected": "I think the capital of France might be London or Paris, I'm not entirely sure."}\n" + }, + { + "type": "markdown", + "source": "#### Tulu3 Preference Dataset Format\nThis format contains complete conversation histories for both the chosen (preferred) and rejected responses.\n\nRequired fields:\n- **`chosen`**: Full conversation with the preferred response (list of message objects, last must be assistant)\n- **`rejected`**: Full conversation with the rejected response (list of message objects, last must be assistant)", + "source_html": "

Tulu3 Preference Dataset Format

\n

This format contains complete conversation histories for both the chosen (preferred) and rejected responses.

\n

Required fields:

\n
    \n
  • chosen: Full conversation with the preferred response (list of message objects, last must be assistant)
    • \n
  • rejected: Full conversation with the rejected response (list of message objects, last must be assistant)
    • \n
\n" + }, + { + "type": "code", + "source": "{\"chosen\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}, {\"role\": \"assistant\", \"content\": \"The capital of France is Paris.\"}], \"rejected\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}, {\"role\": \"assistant\", \"content\": \"I'm not sure, but I think it might be London or Paris.\"}]}", + "language": "python", + "source_html": "{"chosen": [{"role": "user", "content": "What is the capital of France?"}, {"role": "assistant", "content": "The capital of France is Paris."}], "rejected": [{"role": "user", "content": "What is the capital of France?"}, {"role": "assistant", "content": "I'm not sure, but I think it might be London or Paris."}]}\n" + }, + { + "type": "markdown", + "source": "#### HelpSteer Dataset Format\nThis format uses numeric preference scores to indicate which response is better. The context can be either a simple string or an array of message objects.\n\nRequired fields:\n- **`context`**: The input context (can be a string or array of message objects)\n- **`response1`**: First response option\n- **`response2`**: Second response option\n- **`overall_preference`**: Preference score where negative values mean response1 is preferred, positive values mean response2 is preferred, and 0 indicates a tie", + "source_html": "

HelpSteer Dataset Format

\n

This format uses numeric preference scores to indicate which response is better. The context can be either a simple string or an array of message objects.

\n

Required fields:

\n
    \n
  • context: The input context (can be a string or array of message objects)
    • \n
  • response1: First response option
    • \n
  • response2: Second response option
    • \n
  • overall_preference: Preference score where negative values mean response1 is preferred, positive values mean response2 is preferred, and 0 indicates a tie
    • \n
\n" + }, + { + "type": "code", + "source": "{\"context\": \"Explain how to use git rebase\", \"response1\": \"Git rebase is a command that rewrites commit history by moving or combining commits. Use 'git rebase main' to reapply your branch commits on top of main. This creates a linear history and avoids merge commits.\", \"response2\": \"Use git rebase to change commits. Just type git rebase and it will work.\", \"overall_preference\": -2}", + "language": "python", + "source_html": "{"context": "Explain how to use git rebase", "response1": "Git rebase is a command that rewrites commit history by moving or combining commits. Use 'git rebase main' to reapply your branch commits on top of main. This creates a linear history and avoids merge commits.", "response2": "Use git rebase to change commits. Just type git rebase and it will work.", "overall_preference": -2}\n" + }, + { + "type": "markdown", + "source": "### 3. Create Dataset FileSet and Upload Training Data", + "source_html": "

3. Create Dataset FileSet and Upload Training Data

\n" + }, + { + "type": "markdown", + "source": "Install huggingface datasets package to download public [nvidia/HelpSteer3](https://huggingface.co/datasets/nvidia/HelpSteer3) dataset if it's not installed in your Python environment:\n\n```sh\npip install datasets\n```", + "source_html": "

Install huggingface datasets package to download public nvidia/HelpSteer3 dataset if it's not installed in your Python environment:

\n
pip install datasets\n
\n" + }, + { + "type": "markdown", + "source": "#### Download nvidia/HelpSteer3 Dataset", + "source_html": "

Download nvidia/HelpSteer3 Dataset

\n" + }, + { + "type": "code", + "source": "from pathlib import Path\nfrom datasets import load_dataset, Dataset\nds = load_dataset(\"nvidia/HelpSteer3\", \"preference\")\n\n# Adjust these values to change the size of the training and validation sets\n# The larger the datasets, the better the model will perform but longer the training will take\n# For the purpose of this tutorial, we'll use a small subset of the dataset\ntraining_size = 3000\nvalidation_size = 300\nDATASET_PATH = Path(\"dpo-dataset\").absolute()\n\n# Get training split and verify it's a Dataset (not IterableDataset)\ntrain_dataset = ds[\"train\"]\nvalidation_dataset = ds[\"validation\"]\nassert isinstance(train_dataset, Dataset), \"Expected Dataset type\"\nassert isinstance(validation_dataset, Dataset), \"Expected Dataset type\"\n\n# Select subsets and save to JSONL files\ntesting_ds = train_dataset.select(range(training_size))\nvalidation_ds = validation_dataset.select(range(validation_size))\n\n# Create directory if it doesn't exist\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Save subsets to JSONL files\ntesting_ds.to_json(f\"{DATASET_PATH}/training.jsonl\")\nvalidation_ds.to_json(f\"{DATASET_PATH}/validation.jsonl\")\n\nprint(f\"Saved training.jsonl with {len(testing_ds)} rows\")\nprint(f\"Saved validation.jsonl with {len(validation_ds)} rows\")", + "language": "python", + "source_html": "from pathlib import Path\nfrom datasets import load_dataset, Dataset\nds = load_dataset("nvidia/HelpSteer3", "preference")\n\n# Adjust these values to change the size of the training and validation sets\n# The larger the datasets, the better the model will perform but longer the training will take\n# For the purpose of this tutorial, we'll use a small subset of the dataset\ntraining_size = 3000\nvalidation_size = 300\nDATASET_PATH = Path("dpo-dataset").absolute()\n\n# Get training split and verify it's a Dataset (not IterableDataset)\ntrain_dataset = ds["train"]\nvalidation_dataset = ds["validation"]\nassert isinstance(train_dataset, Dataset), "Expected Dataset type"\nassert isinstance(validation_dataset, Dataset), "Expected Dataset type"\n\n# Select subsets and save to JSONL files\ntesting_ds = train_dataset.select(range(training_size))\nvalidation_ds = validation_dataset.select(range(validation_size))\n\n# Create directory if it doesn't exist\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Save subsets to JSONL files\ntesting_ds.to_json(f"{DATASET_PATH}/training.jsonl")\nvalidation_ds.to_json(f"{DATASET_PATH}/validation.jsonl")\n\nprint(f"Saved training.jsonl with {len(testing_ds)} rows")\nprint(f"Saved validation.jsonl with {len(validation_ds)} rows")\n" + }, + { + "type": "markdown", + "source": "#### Upload Training Data", + "source_html": "

Upload Training Data

\n" + }, + { + "type": "code", + "source": "# Create fileset to store DPO training data\nDATASET_NAME = \"dpo-dataset\"\n\ntry:\n client.files.filesets.create(\n workspace=\"default\",\n name=DATASET_NAME,\n description=\"dpo training data\"\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\n# Upload training data files individually to ensure correct structure\nclient.files.upload(\n local_path=DATASET_PATH, # Local directory with your JSONL files\n remote_path=\"\",\n fileset=DATASET_NAME,\n workspace=\"default\"\n)\n\n# Validate training data is uploaded correctly\nprint(\"Training data:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace=\"default\").data], indent=2))", + "language": "python", + "source_html": "# Create fileset to store DPO training data\nDATASET_NAME = "dpo-dataset"\n\ntry:\n client.files.filesets.create(\n workspace="default",\n name=DATASET_NAME,\n description="dpo training data"\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\n# Upload training data files individually to ensure correct structure\nclient.files.upload(\n local_path=DATASET_PATH, # Local directory with your JSONL files\n remote_path="",\n fileset=DATASET_NAME,\n workspace="default"\n)\n\n# Validate training data is uploaded correctly\nprint("Training data:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace="default").data], indent=2))\n" + }, + { + "type": "markdown", + "source": "### 4. Secrets Setup\n\nIf you plan to use NGC or HuggingFace models, you'll need to configure authentication:\n\n- **NGC models** (`ngc://` URIs): Requires NGC API key\n- **HuggingFace models** (`hf://` URIs): Requires HF token for gated/private models\n\n\nConfigure these as secrets in your platform. See [Managing Secrets](../../get-started/concepts/manage-secrets.md) for detailed instructions.\n\nGet your credentials to access base models:\n- [NGC API Key](https://ngc.nvidia.com/) (Setup → Generate API Key)\n- [HuggingFace Token](https://huggingface.co/settings/tokens) (Create token with Read access)\n\n\n---\n\n#### Quick Setup Example\n\nIn this tutorial we are going to work with [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) model from HuggingFace. Ensure that you have sufficient permissions to download the model. If you cannot see the files in the [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) Hugging Face page, request access\n\n**HuggingFace Authentication:**\n- For gated models (Llama, Gemma), you must provide a HuggingFace token via the `token_secret` parameter\n- Get your token from [HuggingFace Settings](https://huggingface.co/settings/tokens) (requires Read access)\n- Accept the model's terms on the HuggingFace model page before using it. Example: [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main)\n- For public models, you can omit the `token_secret` parameter when creating a fileset for model in the next step", + "source_html": "

4. Secrets Setup

\n

If you plan to use NGC or HuggingFace models, you'll need to configure authentication:

\n
    \n
  • NGC models (ngc:// URIs): Requires NGC API key
    • \n
  • HuggingFace models (hf:// URIs): Requires HF token for gated/private models
    • \n
\n

Configure these as secrets in your platform. See Managing Secrets for detailed instructions.

\n

Get your credentials to access base models:

\n\n
\n

Quick Setup Example

\n

In this tutorial we are going to work with meta-llama/Llama-3.2-1B-Instruct model from HuggingFace. Ensure that you have sufficient permissions to download the model. If you cannot see the files in the meta-llama/Llama-3.2-1B-Instruct Hugging Face page, request access

\n

HuggingFace Authentication:

\n
    \n
  • For gated models (Llama, Gemma), you must provide a HuggingFace token via the token_secret parameter
    • \n
  • Get your token from HuggingFace Settings (requires Read access)
    • \n
  • Accept the model's terms on the HuggingFace model page before using it. Example: meta-llama/Llama-3.2-1B-Instruct
    • \n
  • For public models, you can omit the token_secret parameter when creating a fileset for model in the next step
    • \n
\n" + }, + { + "type": "code", + "source": "# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set\nHF_TOKEN = os.getenv(\"HF_TOKEN\")\nNGC_API_KEY = os.getenv(\"NGC_API_KEY\")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f\"{label} is not set\")\n try:\n secret = client.secrets.create(\n name=name,\n workspace=\"default\",\n value=value,\n )\n print(f\"Created secret: {name}\")\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n return client.secrets.retrieve(name=name, workspace=\"default\")\n\n\n# Create HuggingFace token secret\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")\nprint(\"HF_TOKEN secret:\")\nprint(hf_secret.model_dump_json(indent=2))\n\n# Create NGC API key secret\n# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n# ngc_api_key = create_or_get_secret(\"ngc-api-key\", NGC_API_KEY, \"NGC_API_KEY\")", + "language": "python", + "source_html": "# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set\nHF_TOKEN = os.getenv("HF_TOKEN")\nNGC_API_KEY = os.getenv("NGC_API_KEY")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f"{label} is not set")\n try:\n secret = client.secrets.create(\n name=name,\n workspace="default",\n value=value,\n )\n print(f"Created secret: {name}")\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n return client.secrets.retrieve(name=name, workspace="default")\n\n\n# Create HuggingFace token secret\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\nprint("HF_TOKEN secret:")\nprint(hf_secret.model_dump_json(indent=2))\n\n# Create NGC API key secret\n# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n# ngc_api_key = create_or_get_secret("ngc-api-key", NGC_API_KEY, "NGC_API_KEY")\n" + }, + { + "type": "markdown", + "source": "### 5. Create Base Model FileSet and Model Entity\n\nCreate a fileset pointing to [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) model in HuggingFace that we will train with DPO. Then create a Model Entity that references this fileset. Model downloading will take place at the DPO finetuning job creation time.\n\nNote: for public models, you can omit the `token_secret` parameter when creating a model fileset.", + "source_html": "

5. Create Base Model FileSet and Model Entity

\n

Create a fileset pointing to meta-llama/Llama-3.2-1B-Instruct model in HuggingFace that we will train with DPO. Then create a Model Entity that references this fileset. Model downloading will take place at the DPO finetuning job creation time.

\n

Note: for public models, you can omit the token_secret parameter when creating a model fileset.

\n" + }, + { + "type": "code", + "source": "import time\n\n# Create a fileset pointing to the desired HuggingFace model\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = \"meta-llama/Llama-3.2-1B-Instruct\"\nMODEL_NAME = \"llama-3-2-1b-base\"\n\n# Ensure you have a HuggingFace token secret created\ntry:\n base_model_fs = client.files.filesets.create(\n workspace=\"default\",\n name=MODEL_NAME,\n description=\"Llama 3.2 1B base model from HuggingFace\",\n storage=HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type=\"model\",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\n print(f\"Created base model fileset: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model fileset already exists. Skipping creation.\")\n base_model_fs = client.files.filesets.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\n# Create Model Entity referencing the FileSet\ntry:\n base_model = client.models.create(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n )\n print(f\"Created Model Entity: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model already exists. Updating fileset if different.\")\n base_model = client.models.update(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n )\n\nprint(f\"\\nBase model fileset: fileset://default/{base_model.name}\")\nprint(\"Base model fileset files list:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace=\"default\").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint(\"\\nWaiting for ModelSpec to be populated...\")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f\"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds\")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\nprint(f\"ModelSpec populated: {base_model.spec}\")", + "language": "python", + "source_html": "import time\n\n# Create a fileset pointing to the desired HuggingFace model\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = "meta-llama/Llama-3.2-1B-Instruct"\nMODEL_NAME = "llama-3-2-1b-base"\n\n# Ensure you have a HuggingFace token secret created\ntry:\n base_model_fs = client.files.filesets.create(\n workspace="default",\n name=MODEL_NAME,\n description="Llama 3.2 1B base model from HuggingFace",\n storage=HuggingfaceStorageConfigParam(\n type="huggingface",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type="model",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\n print(f"Created base model fileset: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model fileset already exists. Skipping creation.")\n base_model_fs = client.files.filesets.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\n# Create Model Entity referencing the FileSet\ntry:\n base_model = client.models.create(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n )\n print(f"Created Model Entity: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model already exists. Updating fileset if different.")\n base_model = client.models.update(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n )\n\nprint(f"\\nBase model fileset: fileset://default/{base_model.name}")\nprint("Base model fileset files list:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace="default").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint("\\nWaiting for ModelSpec to be populated...")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\nprint(f"ModelSpec populated: {base_model.spec}")\n" + }, + { + "type": "markdown", + "source": "### 6. Create DPO Finetuning Job\nCreate a customization job with an inline target referencing the base model and dataset filesets created in previous steps.", + "source_html": "

6. Create DPO Finetuning Job

\n

Create a customization job with an inline target referencing the base model and dataset filesets created in previous steps.

\n" + }, + { + "type": "markdown", + "source": "**Target `model_uri` Format:**\n\nCurrently, `model_uri` must reference a FileSet:\n- **FileSet:** `fileset://{workspace}/{fileset-name}`\n\nSupport for direct HuggingFace (`hf://`) and NGC (`ngc://`) URIs is coming soon. For now, create a fileset as shown in the previous step, and the HuggingFace model will be downloaded at the beginning the finetuning job.\n\n**GPU Requirements:**\n- 1B models: 1 GPU (24GB+ VRAM)\n- 3B models: 1-2 GPUs \n- 8B models: 2-4 GPUs\n- 70B models: 8+ GPUs \n\nAdjust `num_gpus_per_node` and `tensor_parallel_size` based on your model size.\n\n**Important**\n\nWhen setting `val_check_interval` for DPO, use a fractional value (e.g., `0.5` for twice per epoch) or omit it entirely (validates once at end of epoch). Avoid integer step counts — they may not divide evenly into the total training steps, which can prevent validation from running on the final step.", + "source_html": "

Target model_uri Format:

\n

Currently, model_uri must reference a FileSet:

\n
    \n
  • FileSet: fileset://{workspace}/{fileset-name}
    • \n
\n

Support for direct HuggingFace (hf://) and NGC (ngc://) URIs is coming soon. For now, create a fileset as shown in the previous step, and the HuggingFace model will be downloaded at the beginning the finetuning job.

\n

GPU Requirements:

\n
    \n
  • 1B models: 1 GPU (24GB+ VRAM)
    • \n
  • 3B models: 1-2 GPUs
    • \n
  • 8B models: 2-4 GPUs
    • \n
  • 70B models: 8+ GPUs
    • \n
\n

Adjust num_gpus_per_node and tensor_parallel_size based on your model size.

\n

Important

\n

When setting val_check_interval for DPO, use a fractional value (e.g., 0.5 for twice per epoch) or omit it entirely (validates once at end of epoch). Avoid integer step counts — they may not divide evenly into the total training steps, which can prevent validation from running on the final step.

\n" + }, + { + "type": "code", + "source": "import uuid\njob_suffix = uuid.uuid4().hex[:4]\n\nJOB_NAME = f\"my-dpo-job-{job_suffix}\"\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=DpoTrainingParam(\n type=\"dpo\",\n epochs=1,\n batch_size=16,\n learning_rate=0.00005,\n max_seq_length=4096,\n ref_policy_kl_penalty=0.1,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n )\n)\n\nprint(f\"Job ID: {job.name}\")\nprint(f\"Output model: {job.spec.output.name}\")", + "language": "python", + "source_html": "import uuid\njob_suffix = uuid.uuid4().hex[:4]\n\nJOB_NAME = f"my-dpo-job-{job_suffix}"\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace="default",\n spec=CustomizationJobInputParam(\n model=f"default/{base_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=DpoTrainingParam(\n type="dpo",\n epochs=1,\n batch_size=16,\n learning_rate=0.00005,\n max_seq_length=4096,\n ref_policy_kl_penalty=0.1,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n )\n)\n\nprint(f"Job ID: {job.name}")\nprint(f"Output model: {job.spec.output.name}")\n" + }, + { + "type": "markdown", + "source": "### 7. Track Training Progress", + "source_html": "

7. Track Training Progress

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Poll job status every 10 seconds until completed\nwhile True:\n status = client.customization.jobs.get_status(\n name=job.name,\n workspace=\"default\"\n )\n \n clear_output(wait=True)\n print(f\"Job Status: {status.model_dump_json(indent=2)}\")\n\n # Extract training progress from nested steps structure\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n\n for job_step in status.steps or []:\n if job_step.name == \"customization-training-job\":\n for task in job_step.tasks or []:\n task_details = task.status_details or {}\n step = task_details.get(\"step\")\n max_steps = task_details.get(\"max_steps\")\n training_phase = task_details.get(\"phase\")\n break\n break\n\n if step is not None and max_steps is not None:\n progress_pct = (step / max_steps) * 100\n print(f\"Training Progress: Step {step}/{max_steps} ({progress_pct:.1f}%)\")\n if training_phase:\n print(f\"Training Phase: {training_phase}\")\n else:\n print(\"Training step not started yet or progress info not available\")\n \n # Exit loop when job is completed (or failed/cancelled)\n if status.status in (\"completed\", \"failed\", \"cancelled\", \"error\"):\n print(f\"\\nJob finished with status: {status.status}\")\n break\n \n time.sleep(10)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Poll job status every 10 seconds until completed\nwhile True:\n status = client.customization.jobs.get_status(\n name=job.name,\n workspace="default"\n )\n \n clear_output(wait=True)\n print(f"Job Status: {status.model_dump_json(indent=2)}")\n\n # Extract training progress from nested steps structure\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n\n for job_step in status.steps or []:\n if job_step.name == "customization-training-job":\n for task in job_step.tasks or []:\n task_details = task.status_details or {}\n step = task_details.get("step")\n max_steps = task_details.get("max_steps")\n training_phase = task_details.get("phase")\n break\n break\n\n if step is not None and max_steps is not None:\n progress_pct = (step / max_steps) * 100\n print(f"Training Progress: Step {step}/{max_steps} ({progress_pct:.1f}%)")\n if training_phase:\n print(f"Training Phase: {training_phase}")\n else:\n print("Training step not started yet or progress info not available")\n \n # Exit loop when job is completed (or failed/cancelled)\n if status.status in ("completed", "failed", "cancelled", "error"):\n print(f"\\nJob finished with status: {status.status}")\n break\n \n time.sleep(10)\n" + }, + { + "type": "markdown", + "source": "**Interpreting DPO Training Metrics:**\n\nDPO training produces several key metrics:\n\n| Metric | Description | What to Look For |\n|--------|-------------|------------------|\n| **loss** | Total training loss (preference_loss + sft_loss) | Should decrease over training |\n| **preference_loss** | Core DPO loss measuring preference learning | Starts near ln(2) ≈ 0.693, should decrease |\n| **sft_loss** | SFT regularization term (often 0 for pure DPO) | Depends on configuration |\n| **accuracy** | Fraction of samples where chosen > rejected | Should increase toward 80-95%+ |\n| **rewards_chosen_mean** | Average implicit reward for chosen responses | Should be positive |\n| **rewards_rejected_mean** | Average implicit reward for rejected responses | Should be negative |\n\n**Key Indicators:**\n\n- **Reward Margin** = `rewards_chosen_mean - rewards_rejected_mean`\n - Should be positive and increasing\n - Indicates the model is learning to distinguish preferences\n\n- **Accuracy Interpretation:**\n - 50% = random chance (no learning)\n - 66-75% = early/moderate learning\n - 80%+ = good preference learning\n - 95%+ = strong preference alignment\n\n**Troubleshooting:**\n\n- **Loss near ln(2) ≈ 0.693**: Model is at random chance level, training just starting or not learning\n- **Accuracy stuck at ~50%**: Check data quality, increase learning rate, or verify preference labels\n- **Negative reward margin**: Model is learning the wrong direction—check chosen/rejected labels\n- **Loss increasing**: Learning rate too high or data quality issues\n\n**Note:** Training metrics measure optimization progress, not final model quality. Always evaluate the deployed model on your specific use case.", + "source_html": "

Interpreting DPO Training Metrics:

\n

DPO training produces several key metrics:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
MetricDescriptionWhat to Look For
lossTotal training loss (preference_loss + sft_loss)Should decrease over training
preference_lossCore DPO loss measuring preference learningStarts near ln(2) ≈ 0.693, should decrease
sft_lossSFT regularization term (often 0 for pure DPO)Depends on configuration
accuracyFraction of samples where chosen > rejectedShould increase toward 80-95%+
rewards_chosen_meanAverage implicit reward for chosen responsesShould be positive
rewards_rejected_meanAverage implicit reward for rejected responsesShould be negative
\n

Key Indicators:

\n
    \n
  • \n

    Reward Margin = rewards_chosen_mean - rewards_rejected_mean

    \n
      \n
    • Should be positive and increasing
      • \n
    • Indicates the model is learning to distinguish preferences
      • \n
    \n
    • \n
  • \n

    Accuracy Interpretation:

    \n
      \n
    • 50% = random chance (no learning)
      • \n
    • 66-75% = early/moderate learning
      • \n
    • 80%+ = good preference learning
      • \n
    • 95%+ = strong preference alignment
      • \n
    \n
    • \n
\n

Troubleshooting:

\n
    \n
  • Loss near ln(2) ≈ 0.693: Model is at random chance level, training just starting or not learning
    • \n
  • Accuracy stuck at ~50%: Check data quality, increase learning rate, or verify preference labels
    • \n
  • Negative reward margin: Model is learning the wrong direction—check chosen/rejected labels
    • \n
  • Loss increasing: Learning rate too high or data quality issues
    • \n
\n

Note: Training metrics measure optimization progress, not final model quality. Always evaluate the deployed model on your specific use case.

\n" + }, + { + "type": "markdown", + "source": "### 8. Deploy Fine-Tuned Model\n\nOnce training completes, deploy using the Deployment Management Service:", + "source_html": "

8. Deploy Fine-Tuned Model

\n

Once training completes, deploy using the Deployment Management Service:

\n" + }, + { + "type": "code", + "source": "# Validate model entity exists\nmodel_entity = client.models.retrieve(workspace='default', name=job.spec.output.name)\nprint(model_entity.model_dump_json(indent=2))", + "language": "python", + "source_html": "# Validate model entity exists\nmodel_entity = client.models.retrieve(workspace='default', name=job.spec.output.name)\nprint(model_entity.model_dump_json(indent=2))\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# Create deployment config\ndeploy_suffix = uuid.uuid4().hex[:4]\nDEPLOYMENT_CONFIG_NAME = f\"dpo-model-deployment-cfg-{deploy_suffix}\"\nDEPLOYMENT_NAME = f\"dpo-model-deployment-{deploy_suffix}\"\n\ndeployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=DEPLOYMENT_CONFIG_NAME,\n nim_deployment=NIMDeploymentParam(\n image_name=\"nvcr.io/nim/nvidia/llm-nim\",\n image_tag=\"1.13.1\",\n gpu=1,\n model_name=job.spec.output.name, # ModelEntity name from training,\n model_namespace=\"default\", # Workspace where ModelEntity lives\n )\n)\n\n# Deploy model using deployment_config created above\ndeployment = client.inference.deployments.create(\n workspace=\"default\",\n name=DEPLOYMENT_NAME,\n config=deployment_config.name\n)\n\n\n# Check deployment status\ndeployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace=\"default\"\n)\n\nprint(f\"Deployment name: {deployment.name}\")\nprint(f\"Deployment status: {deployment_status.status}\")", + "language": "python", + "source_html": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# Create deployment config\ndeploy_suffix = uuid.uuid4().hex[:4]\nDEPLOYMENT_CONFIG_NAME = f"dpo-model-deployment-cfg-{deploy_suffix}"\nDEPLOYMENT_NAME = f"dpo-model-deployment-{deploy_suffix}"\n\ndeployment_config = client.inference.deployment_configs.create(\n workspace="default",\n name=DEPLOYMENT_CONFIG_NAME,\n nim_deployment=NIMDeploymentParam(\n image_name="nvcr.io/nim/nvidia/llm-nim",\n image_tag="1.13.1",\n gpu=1,\n model_name=job.spec.output.name, # ModelEntity name from training,\n model_namespace="default", # Workspace where ModelEntity lives\n )\n)\n\n# Deploy model using deployment_config created above\ndeployment = client.inference.deployments.create(\n workspace="default",\n name=DEPLOYMENT_NAME,\n config=deployment_config.name\n)\n\n\n# Check deployment status\ndeployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace="default"\n)\n\nprint(f"Deployment name: {deployment.name}")\nprint(f"Deployment status: {deployment_status.status}")\n" + }, + { + "type": "markdown", + "source": "### Monitor status of deployment", + "source_html": "

Monitor status of deployment

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Poll deployment status every 15 seconds until ready\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f\"Monitoring deployment '{deployment.name}'...\")\nprint(f\"Timeout: {TIMEOUT_MINUTES} minutes\\n\")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace=\"default\"\n )\n \n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n \n clear_output(wait=True)\n print(f\"Deployment: {deployment.name}\")\n print(f\"Status: {deployment_status.status}\")\n print(f\"Elapsed time: {elapsed_min}m {elapsed_sec}s\")\n \n # Check if deployment is ready\n if deployment_status.status == \"READY\":\n print(\"\\nDeployment is ready!\")\n if not client.models.wait_for_gateway(deployment.name, workspace=\"default\", timeout=60):\n raise RuntimeError(\"Inference gateway did not become ready\")\n break\n \n # Check for failure states\n if deployment_status.status in (\"FAILED\", \"ERROR\", \"TERMINATED\", \"LOST\"):\n raise RuntimeError(f\"Deployment failed with status: {deployment_status.status}\")\n \n # Check timeout\n if elapsed > timeout_seconds:\n raise TimeoutError(f\"Deployment timeout after {TIMEOUT_MINUTES} minutes\")\n \n time.sleep(15)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Poll deployment status every 15 seconds until ready\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f"Monitoring deployment '{deployment.name}'...")\nprint(f"Timeout: {TIMEOUT_MINUTES} minutes\\n")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace="default"\n )\n \n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n \n clear_output(wait=True)\n print(f"Deployment: {deployment.name}")\n print(f"Status: {deployment_status.status}")\n print(f"Elapsed time: {elapsed_min}m {elapsed_sec}s")\n \n # Check if deployment is ready\n if deployment_status.status == "READY":\n print("\\nDeployment is ready!")\n if not client.models.wait_for_gateway(deployment.name, workspace="default", timeout=60):\n raise RuntimeError("Inference gateway did not become ready")\n break\n \n # Check for failure states\n if deployment_status.status in ("FAILED", "ERROR", "TERMINATED", "LOST"):\n raise RuntimeError(f"Deployment failed with status: {deployment_status.status}")\n \n # Check timeout\n if elapsed > timeout_seconds:\n raise TimeoutError(f"Deployment timeout after {TIMEOUT_MINUTES} minutes")\n \n time.sleep(15)\n" + }, + { + "type": "markdown", + "source": "The deployment service automatically:\n- Downloads model weights from the Files service\n- Provisions storage (PVC) for the weights\n- Configures and starts the NIM container\n\n**Multi-GPU Deployment:**\n\nFor larger models requiring multiple GPUs, configure parallelism with environment variables:\n\n```python\ndeployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=\"sft-model-config-multigpu\",\n \n nim_deployment={\n \"image_name\": \"nvcr.io/nim/nvidia/llm-nim\",\n \"image_tag\": \"1.13.1\",\n \"gpu\": 2, # Total GPUs\n \"additional_envs\": {\n \"NIM_TENSOR_PARALLEL_SIZE\": \"2\", # Tensor parallelism\n \"NIM_PIPELINE_PARALLEL_SIZE\": \"1\" # Pipeline parallelism\n }\n }\n)\n```", + "source_html": "

The deployment service automatically:

\n
    \n
  • Downloads model weights from the Files service
    • \n
  • Provisions storage (PVC) for the weights
    • \n
  • Configures and starts the NIM container
    • \n
\n

Multi-GPU Deployment:

\n

For larger models requiring multiple GPUs, configure parallelism with environment variables:

\n
deployment_config = client.inference.deployment_configs.create(\n    workspace="default",\n    name="sft-model-config-multigpu",\n    \n    nim_deployment={\n        "image_name": "nvcr.io/nim/nvidia/llm-nim",\n        "image_tag": "1.13.1",\n        "gpu": 2,  # Total GPUs\n        "additional_envs": {\n            "NIM_TENSOR_PARALLEL_SIZE": "2",  # Tensor parallelism\n            "NIM_PIPELINE_PARALLEL_SIZE": "1"  # Pipeline parallelism\n        }\n    }\n)\n
\n" + }, + { + "type": "markdown", + "source": "**Single-Node Constraint:** Model deployments are limited to a single node. The maximum `gpu` value depends on the total GPUs available on a single node in your cluster. Multi-node deployments are not supported.\n\n---\n\n#### GPU Parallelism\n\nBy default, NIM uses all GPUs for tensor parallelism (TP). You can customize this behavior using the `NIM_TENSOR_PARALLEL_SIZE` and `NIM_PIPELINE_PARALLEL_SIZE` environment variables.\n\n| Strategy | Description | Best For |\n|----------|-------------|----------|\n| **Tensor Parallel (TP)** | Splits model layers across GPUs | Lowest latency |\n| **Pipeline Parallel (PP)** | Splits model depth across GPUs | Highest throughput |\n\n**Formula:** `gpu` = `NIM_TENSOR_PARALLEL_SIZE` × `NIM_PIPELINE_PARALLEL_SIZE`\n\n---\n\n#### Example Configurations\n\n**Default (TP=8, PP=1) — Lowest Latency**\n```\n\"gpu\": 8\n# NIM automatically sets NIM_TENSOR_PARALLEL_SIZE=8\n```\n\n**Balanced (TP=4, PP=2)**\n```\n\"gpu\": 8,\n\"additional_envs\": {\n \"NIM_TENSOR_PARALLEL_SIZE\": \"4\",\n \"NIM_PIPELINE_PARALLEL_SIZE\": \"2\"\n}\n```\n\n**Throughput Optimized (TP=2, PP=4)**\n```\n\"gpu\": 8,\n\"additional_envs\": {\n \"NIM_TENSOR_PARALLEL_SIZE\": \"2\",\n \"NIM_PIPELINE_PARALLEL_SIZE\": \"4\"\n}\n```", + "source_html": "

Single-Node Constraint: Model deployments are limited to a single node. The maximum gpu value depends on the total GPUs available on a single node in your cluster. Multi-node deployments are not supported.

\n
\n

GPU Parallelism

\n

By default, NIM uses all GPUs for tensor parallelism (TP). You can customize this behavior using the NIM_TENSOR_PARALLEL_SIZE and NIM_PIPELINE_PARALLEL_SIZE environment variables.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
StrategyDescriptionBest For
Tensor Parallel (TP)Splits model layers across GPUsLowest latency
Pipeline Parallel (PP)Splits model depth across GPUsHighest throughput
\n

Formula: gpu = NIM_TENSOR_PARALLEL_SIZE × NIM_PIPELINE_PARALLEL_SIZE

\n
\n

Example Configurations

\n

Default (TP=8, PP=1) — Lowest Latency

\n
"gpu": 8\n# NIM automatically sets NIM_TENSOR_PARALLEL_SIZE=8\n
\n

Balanced (TP=4, PP=2)

\n
"gpu": 8,\n"additional_envs": {\n    "NIM_TENSOR_PARALLEL_SIZE": "4",\n    "NIM_PIPELINE_PARALLEL_SIZE": "2"\n}\n
\n

Throughput Optimized (TP=2, PP=4)

\n
"gpu": 8,\n"additional_envs": {\n    "NIM_TENSOR_PARALLEL_SIZE": "2",\n    "NIM_PIPELINE_PARALLEL_SIZE": "4"\n}\n
\n" + }, + { + "type": "markdown", + "source": "### 9. Evaluate Your Model\n\nAfter training, evaluate whether your model meets your requirements:\n\n#### Quick Manual Evaluation", + "source_html": "

9. Evaluate Your Model

\n

After training, evaluate whether your model meets your requirements:

\n

Quick Manual Evaluation

\n" + }, + { + "type": "code", + "source": "# Wait for deployment to be ready, then test\nmessages = [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": \"Write a short email to my colleague.\"}\n]\n\nresponse = client.inference.gateway.provider.post(\n \"v1/chat/completions\",\n name=deployment.name,\n workspace=\"default\",\n body={\n \"model\": f\"default/{job.spec.output.name}\", # Match the model_name from deployment config\n \"messages\": messages,\n \"temperature\": 0.7,\n \"max_tokens\": 256\n }\n)\n\n# Display prompt and completion\nprint(\"=\" * 60)\nprint(\"PROMPT\")\nprint(\"=\" * 60)\nfor msg in messages:\n print(f\"[{msg['role'].upper()}]\")\n print(msg[\"content\"])\n print()\n\nprint(\"=\" * 60)\nprint(\"COMPLETION\")\nprint(\"=\" * 60)\nprint(\"[ASSISTANT]\")\ncompletion = response[\"choices\"][0][\"message\"][\"content\"]\nprint(completion)", + "language": "python", + "source_html": "# Wait for deployment to be ready, then test\nmessages = [\n {"role": "system", "content": "You are a helpful assistant."},\n {"role": "user", "content": "Write a short email to my colleague."}\n]\n\nresponse = client.inference.gateway.provider.post(\n "v1/chat/completions",\n name=deployment.name,\n workspace="default",\n body={\n "model": f"default/{job.spec.output.name}", # Match the model_name from deployment config\n "messages": messages,\n "temperature": 0.7,\n "max_tokens": 256\n }\n)\n\n# Display prompt and completion\nprint("=" * 60)\nprint("PROMPT")\nprint("=" * 60)\nfor msg in messages:\n print(f"[{msg['role'].upper()}]")\n print(msg["content"])\n print()\n\nprint("=" * 60)\nprint("COMPLETION")\nprint("=" * 60)\nprint("[ASSISTANT]")\ncompletion = response["choices"][0]["message"]["content"]\nprint(completion)\n" + }, + { + "type": "markdown", + "source": "#### Evaluation Best Practices\n\n**Manual Evaluation** (Recommended)\n- Test with real-world examples from your use case\n- Compare responses to base model and expected outputs\n- Verify the model exhibits desired behavior changes\n- Check edge cases and error handling\n\n**What to look for:**\n- ✅ Model follows your desired output format\n- ✅ Applies domain knowledge correctly\n- ✅ Maintains general language capabilities\n- ✅ Avoids unwanted behaviors or biases\n- ❌ Doesn't hallucinate facts not in training data\n- ❌ Doesn't produce repetitive or nonsensical outputs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, see the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (see [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated HuggingFace models (Llama, Gemma), accept the license on the model page\n- Check the `model_uri` format is correct (`fileset://`)\n- Ensure you have accepted the model's terms of service on HuggingFace\n- Check job status and logs: `client.customization.jobs.retrieve(name=job.name, workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n1. **First try:** Reduce `micro_batch_size` from 2 to 1\n2. **Still OOM:** Reduce `batch_size` from 16 to 8\n3. **Still OOM:** Reduce `max_seq_length` from 2048 to 1024 or 512\n4. **Last resort:** Increase GPU count and use `tensor_parallel_size` for model sharding\n\n**Loss curves not decreasing (underfitting):**\n- Increase training duration: `epochs: 5-10` instead of 3\n- Adjust learning rate: Try `1e-5` to `1e-4`\n- Add warmup: Set `warmup_steps` to ~10% of total training steps\n- Check data quality: Verify formatting, remove duplicates, ensure diversity\n\n**Training loss decreases but validation loss increases (overfitting):**\n- Reduce epochs: Try `epochs: 1-2` instead of 5+\n- Lower learning rate: Use `2e-5` or `1e-5`\n- Increase dataset size and diversity\n- Verify train/validation split has no data leakage\n\n**Model output quality is poor despite good training metrics:**\n- Training metrics optimize for loss, not your actual task—evaluate on real use cases\n- Review data quality, format, and diversity—metrics can be misleading with poor data\n- Try a different base model size or architecture\n- Adjust learning rate and batch size\n- Compare to baseline: Test base model to ensure fine-tuning improved performance\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=job.spec.output.name, workspace=\"default\")`\n- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace=\"default\")`\n- Ensure sufficient GPU resources available for model size\n- Verify NIM image tag `1.13.1` is compatible with your model\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning", + "source_html": "

Evaluation Best Practices

\n

Manual Evaluation (Recommended)

\n
    \n
  • Test with real-world examples from your use case
    • \n
  • Compare responses to base model and expected outputs
    • \n
  • Verify the model exhibits desired behavior changes
    • \n
  • Check edge cases and error handling
    • \n
\n

What to look for:

\n
    \n
  • ✅ Model follows your desired output format
    • \n
  • ✅ Applies domain knowledge correctly
    • \n
  • ✅ Maintains general language capabilities
    • \n
  • ✅ Avoids unwanted behaviors or biases
    • \n
  • ❌ Doesn't hallucinate facts not in training data
    • \n
  • ❌ Doesn't produce repetitive or nonsensical outputs
    • \n
\n
\n

Hyperparameters

\n

For detailed information on all available hyperparameters, recommended values, and tuning guidance, see the Hyperparameter Reference.

\n
\n

Troubleshooting

\n

Job fails during model download:

\n
    \n
  • Verify authentication secrets are configured (see Managing Secrets)
    • \n
  • For gated HuggingFace models (Llama, Gemma), accept the license on the model page
    • \n
  • Check the model_uri format is correct (fileset://)
    • \n
  • Ensure you have accepted the model's terms of service on HuggingFace
    • \n
  • Check job status and logs: client.customization.jobs.retrieve(name=job.name, workspace="default")
    • \n
\n

Job fails with OOM (Out of Memory) error:

\n
    \n
  1. First try: Reduce micro_batch_size from 2 to 1
    2. \n
  2. Still OOM: Reduce batch_size from 16 to 8
    3. \n
  3. Still OOM: Reduce max_seq_length from 2048 to 1024 or 512
    4. \n
  4. Last resort: Increase GPU count and use tensor_parallel_size for model sharding
    5. \n
\n

Loss curves not decreasing (underfitting):

\n
    \n
  • Increase training duration: epochs: 5-10 instead of 3
    • \n
  • Adjust learning rate: Try 1e-5 to 1e-4
    • \n
  • Add warmup: Set warmup_steps to ~10% of total training steps
    • \n
  • Check data quality: Verify formatting, remove duplicates, ensure diversity
    • \n
\n

Training loss decreases but validation loss increases (overfitting):

\n
    \n
  • Reduce epochs: Try epochs: 1-2 instead of 5+
    • \n
  • Lower learning rate: Use 2e-5 or 1e-5
    • \n
  • Increase dataset size and diversity
    • \n
  • Verify train/validation split has no data leakage
    • \n
\n

Model output quality is poor despite good training metrics:

\n
    \n
  • Training metrics optimize for loss, not your actual task—evaluate on real use cases
    • \n
  • Review data quality, format, and diversity—metrics can be misleading with poor data
    • \n
  • Try a different base model size or architecture
    • \n
  • Adjust learning rate and batch size
    • \n
  • Compare to baseline: Test base model to ensure fine-tuning improved performance
    • \n
\n

Deployment fails:

\n
    \n
  • Verify output model exists: client.models.retrieve(name=job.spec.output.name, workspace="default")
    • \n
  • Check deployment logs: client.inference.deployments.get_logs(name=deployment.name, workspace="default")
    • \n
  • Ensure sufficient GPU resources available for model size
    • \n
  • Verify NIM image tag 1.13.1 is compatible with your model
    • \n
\n

Next Steps

\n\n" + } + ] +} \ No newline at end of file diff --git a/docs/fern/components/notebooks/dpo-customization-job.ts b/docs/fern/components/notebooks/dpo-customization-job.ts new file mode 100644 index 00000000000..a07e712147c --- /dev/null +++ b/docs/fern/components/notebooks/dpo-customization-job.ts @@ -0,0 +1,202 @@ +/** Auto-generated by ipynb-to-fern-json.py - do not edit */ +export default { cells: [ + { + "type": "markdown", + "source": "\n\n\n# DPO Customization\n\nLearn how to use the NeMo Platform to create a DPO (Direct Preference Optimization) job using a custom dataset.\n\n## About\n\nDPO is an advanced fine-tuning technique for preference-based alignment. If you're new to fine-tuning, consider starting with [LoRA](./lora-customization-job) or [Full SFT](./sft-customization-job) tutorials first.\n\nDirect Preference Optimization (DPO) is an RL-free alignment algorithm that operates on preference data. Given a prompt and a pair of chosen and rejected responses, DPO aims to increase the probability of the chosen response and decrease the probability of the rejected response relative to a frozen reference model. The actor is initialized using the reference model. For more details, refer to the [DPO paper](https://arxiv.org/pdf/2305.18290).\n\nDPO shares similarities with Full SFT training workflows but differs in a few key ways:\n\n| Aspect | SFT (Supervised Fine-Tuning) | DPO (Direct Preference Optimization) |\n| --- | --- | --- |\n| Data Requirements | Labeled instruction-response pairs where the desired output is explicitly provided | Pairwise preference data, where for a given input, one response is explicitly preferred over another |\n| Learning Objective | Directly teaches the model to generate a specific \"correct\" response | Directly optimizes the model to align with human preferences by maximizing the probability of preferred responses and minimizing rejected ones, without needing an explicit reward model |\n| Alignment Focus | Aligns the model with the specific examples present in its training data | Aligns the model with broader human preferences, which can be more effective for subjective tasks or those without a single \"correct\" answer |\n| Computational Efficiency | Standard fine-tuning efficiency | More computationally efficient than SFT (especially when compared to full RLHF methods) as it bypasses the need to train a separate reward model |\n\n**What you can achieve with DPO:**\n- **Align with human preferences**: Directly optimize your model to produce outputs that align with subjective human preferences without requiring explicit reward modeling\n- **Refine response quality**: Improve helpfulness, harmlessness, honesty, and other nuanced qualities that are easier to compare than to define\n- **Control tone and style**: Adjust the model's communication style, verbosity, formality, and other subjective characteristics\n- **Implement safety guardrails**: Teach the model to avoid harmful or undesirable responses by training on preferred vs. rejected response pairs\n- **Optimize subjective tasks**: Excel at tasks where there are multiple acceptable answers but clear preferences exist (creative writing, dialogue, explanations)\n\n**When to choose DPO:**\n- **Subjective quality matters**: Your task involves style, tone, or other qualities where there's no single \"correct\" answer but clear preferences exist\n- **You have preference data**: You can collect pairwise comparisons (preferred vs. rejected responses) more easily than perfect labeled examples\n- **Refining existing capabilities**: You want to make targeted improvements to an already-trained model without major capability changes\n- **Complex evaluation**: Humans find it easier to compare which of two responses is better than to create the ideal response themselves (especially for multi-turn conversations, creative tasks, or nuanced outputs)\n- **Robust behavior changes**: You need more reliable behavior modification than prompting can provide, without the complexity of full RLHF\n- **Lower compute than RLHF**: You want human preference alignment but with simpler training that doesn't require reinforcement learning infrastructure\n\n**When to choose SFT:**\n- **Clear correct answers**: Your task has objectively correct outputs (code generation, structured data extraction, following specific formats)\n- **High-quality examples**: You have well-labeled input-output pairs that demonstrate exactly what the model should produce\n- **Imitation learning**: You want the model to closely mimic a specific style, format, or knowledge base from expert demonstrations\n- **Foundational capabilities**: You're establishing new task-specific capabilities before fine-tuning preferences (SFT is often done before DPO)\n- **Stable, predictable outputs**: You need consistent formatting or structure that's well-defined in your training examples\n- **Traditional NLP tasks**: Instruction following, translation, summarization, or classification where gold-standard labels exist", + "source_html": "\n\n

DPO Customization

\n

Learn how to use the NeMo Platform to create a DPO (Direct Preference Optimization) job using a custom dataset.

\n

About

\n

DPO is an advanced fine-tuning technique for preference-based alignment. If you're new to fine-tuning, consider starting with LoRA or Full SFT tutorials first.

\n

Direct Preference Optimization (DPO) is an RL-free alignment algorithm that operates on preference data. Given a prompt and a pair of chosen and rejected responses, DPO aims to increase the probability of the chosen response and decrease the probability of the rejected response relative to a frozen reference model. The actor is initialized using the reference model. For more details, refer to the DPO paper.

\n

DPO shares similarities with Full SFT training workflows but differs in a few key ways:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
AspectSFT (Supervised Fine-Tuning)DPO (Direct Preference Optimization)
Data RequirementsLabeled instruction-response pairs where the desired output is explicitly providedPairwise preference data, where for a given input, one response is explicitly preferred over another
Learning ObjectiveDirectly teaches the model to generate a specific "correct" responseDirectly optimizes the model to align with human preferences by maximizing the probability of preferred responses and minimizing rejected ones, without needing an explicit reward model
Alignment FocusAligns the model with the specific examples present in its training dataAligns the model with broader human preferences, which can be more effective for subjective tasks or those without a single "correct" answer
Computational EfficiencyStandard fine-tuning efficiencyMore computationally efficient than SFT (especially when compared to full RLHF methods) as it bypasses the need to train a separate reward model
\n

What you can achieve with DPO:

\n
    \n
  • Align with human preferences: Directly optimize your model to produce outputs that align with subjective human preferences without requiring explicit reward modeling
    • \n
  • Refine response quality: Improve helpfulness, harmlessness, honesty, and other nuanced qualities that are easier to compare than to define
    • \n
  • Control tone and style: Adjust the model's communication style, verbosity, formality, and other subjective characteristics
    • \n
  • Implement safety guardrails: Teach the model to avoid harmful or undesirable responses by training on preferred vs. rejected response pairs
    • \n
  • Optimize subjective tasks: Excel at tasks where there are multiple acceptable answers but clear preferences exist (creative writing, dialogue, explanations)
    • \n
\n

When to choose DPO:

\n
    \n
  • Subjective quality matters: Your task involves style, tone, or other qualities where there's no single "correct" answer but clear preferences exist
    • \n
  • You have preference data: You can collect pairwise comparisons (preferred vs. rejected responses) more easily than perfect labeled examples
    • \n
  • Refining existing capabilities: You want to make targeted improvements to an already-trained model without major capability changes
    • \n
  • Complex evaluation: Humans find it easier to compare which of two responses is better than to create the ideal response themselves (especially for multi-turn conversations, creative tasks, or nuanced outputs)
    • \n
  • Robust behavior changes: You need more reliable behavior modification than prompting can provide, without the complexity of full RLHF
    • \n
  • Lower compute than RLHF: You want human preference alignment but with simpler training that doesn't require reinforcement learning infrastructure
    • \n
\n

When to choose SFT:

\n
    \n
  • Clear correct answers: Your task has objectively correct outputs (code generation, structured data extraction, following specific formats)
    • \n
  • High-quality examples: You have well-labeled input-output pairs that demonstrate exactly what the model should produce
    • \n
  • Imitation learning: You want the model to closely mimic a specific style, format, or knowledge base from expert demonstrations
    • \n
  • Foundational capabilities: You're establishing new task-specific capabilities before fine-tuning preferences (SFT is often done before DPO)
    • \n
  • Stable, predictable outputs: You need consistent formatting or structure that's well-defined in your training examples
    • \n
  • Traditional NLP tasks: Instruction following, translation, summarization, or classification where gold-standard labels exist
    • \n
\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n2. **Installed the Python SDK** (included with `pip install nemo-platform`)", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Completed the Quickstart to install and deploy NeMo Platform locally
    2. \n
  2. Installed the Python SDK (included with pip install nemo-platform)
    3. \n
\n" + }, + { + "type": "markdown", + "source": "## Quick Start\n\n### 1. Initialize SDK\n\nThe SDK needs to know your NeMo Platform server URL. By default, `http://localhost:8080` is used in accordance with the [Quickstart](../../get-started/quickstart.md) guide. If NeMo Platform is running at a custom location, you can override the URL by setting the `NMP_BASE_URL` environment variable:\n\n```sh\nexport NMP_BASE_URL=\n```", + "source_html": "

Quick Start

\n

1. Initialize SDK

\n

The SDK needs to know your NeMo Platform server URL. By default, http://localhost:8080 is used in accordance with the Quickstart guide. If NeMo Platform is running at a custom location, you can override the URL by setting the NMP_BASE_URL environment variable:

\n
export NMP_BASE_URL=<YOUR_NMP_BASE_URL>\n
\n" + }, + { + "type": "code", + "source": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n DpoTrainingParam,\n ParallelismParamsParam,\n)\n\nNMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace=\"default\"\n)", + "language": "python", + "source_html": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n DpoTrainingParam,\n ParallelismParamsParam,\n)\n\nNMP_BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace="default"\n)\n" + }, + { + "type": "markdown", + "source": "### 2. Prepare Dataset\n\nCreate your data in JSONL format - one JSON object per line. The platform auto-detects your data format. Supported dataset formats are listed below.\n\n**Flexible Data Setup:**\n- **No validation file?** The platform automatically creates a 10% validation split\n- **Multiple files?** Upload to `training/` or `validation/` subdirectories—they'll be automatically merged\n- **Format detection:** Your data format is auto-detected at training time\n\nIn this tutorial the following dataset directory structure will be used:\n```\nmy_dataset\n`-- training.jsonl\n`-- validation.jsonl\n```", + "source_html": "

2. Prepare Dataset

\n

Create your data in JSONL format - one JSON object per line. The platform auto-detects your data format. Supported dataset formats are listed below.

\n

Flexible Data Setup:

\n
    \n
  • No validation file? The platform automatically creates a 10% validation split
    • \n
  • Multiple files? Upload to training/ or validation/ subdirectories—they'll be automatically merged
    • \n
  • Format detection: Your data format is auto-detected at training time
    • \n
\n

In this tutorial the following dataset directory structure will be used:

\n
my_dataset\n`-- training.jsonl\n`-- validation.jsonl\n
\n" + }, + { + "type": "markdown", + "source": "#### Binary Preference Format\nDPO training requires preference pairs with three fields:\n- **`prompt`**: The input prompt (can be a string or array of message objects)\n- **`chosen`**: The preferred response\n- **`rejected`**: The less preferred response", + "source_html": "

Binary Preference Format

\n

DPO training requires preference pairs with three fields:

\n
    \n
  • prompt: The input prompt (can be a string or array of message objects)
    • \n
  • chosen: The preferred response
    • \n
  • rejected: The less preferred response
    • \n
\n" + }, + { + "type": "code", + "source": "{\"prompt\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}], \"chosen\": \"The capital of France is Paris. It is the largest city in France and serves as the country's political, economic, and cultural center.\", \"rejected\": \"I think the capital of France might be London or Paris, I'm not entirely sure.\"}", + "language": "python", + "source_html": "{"prompt": [{"role": "user", "content": "What is the capital of France?"}], "chosen": "The capital of France is Paris. It is the largest city in France and serves as the country's political, economic, and cultural center.", "rejected": "I think the capital of France might be London or Paris, I'm not entirely sure."}\n" + }, + { + "type": "markdown", + "source": "#### Tulu3 Preference Dataset Format\nThis format contains complete conversation histories for both the chosen (preferred) and rejected responses.\n\nRequired fields:\n- **`chosen`**: Full conversation with the preferred response (list of message objects, last must be assistant)\n- **`rejected`**: Full conversation with the rejected response (list of message objects, last must be assistant)", + "source_html": "

Tulu3 Preference Dataset Format

\n

This format contains complete conversation histories for both the chosen (preferred) and rejected responses.

\n

Required fields:

\n
    \n
  • chosen: Full conversation with the preferred response (list of message objects, last must be assistant)
    • \n
  • rejected: Full conversation with the rejected response (list of message objects, last must be assistant)
    • \n
\n" + }, + { + "type": "code", + "source": "{\"chosen\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}, {\"role\": \"assistant\", \"content\": \"The capital of France is Paris.\"}], \"rejected\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}, {\"role\": \"assistant\", \"content\": \"I'm not sure, but I think it might be London or Paris.\"}]}", + "language": "python", + "source_html": "{"chosen": [{"role": "user", "content": "What is the capital of France?"}, {"role": "assistant", "content": "The capital of France is Paris."}], "rejected": [{"role": "user", "content": "What is the capital of France?"}, {"role": "assistant", "content": "I'm not sure, but I think it might be London or Paris."}]}\n" + }, + { + "type": "markdown", + "source": "#### HelpSteer Dataset Format\nThis format uses numeric preference scores to indicate which response is better. The context can be either a simple string or an array of message objects.\n\nRequired fields:\n- **`context`**: The input context (can be a string or array of message objects)\n- **`response1`**: First response option\n- **`response2`**: Second response option\n- **`overall_preference`**: Preference score where negative values mean response1 is preferred, positive values mean response2 is preferred, and 0 indicates a tie", + "source_html": "

HelpSteer Dataset Format

\n

This format uses numeric preference scores to indicate which response is better. The context can be either a simple string or an array of message objects.

\n

Required fields:

\n
    \n
  • context: The input context (can be a string or array of message objects)
    • \n
  • response1: First response option
    • \n
  • response2: Second response option
    • \n
  • overall_preference: Preference score where negative values mean response1 is preferred, positive values mean response2 is preferred, and 0 indicates a tie
    • \n
\n" + }, + { + "type": "code", + "source": "{\"context\": \"Explain how to use git rebase\", \"response1\": \"Git rebase is a command that rewrites commit history by moving or combining commits. Use 'git rebase main' to reapply your branch commits on top of main. This creates a linear history and avoids merge commits.\", \"response2\": \"Use git rebase to change commits. Just type git rebase and it will work.\", \"overall_preference\": -2}", + "language": "python", + "source_html": "{"context": "Explain how to use git rebase", "response1": "Git rebase is a command that rewrites commit history by moving or combining commits. Use 'git rebase main' to reapply your branch commits on top of main. This creates a linear history and avoids merge commits.", "response2": "Use git rebase to change commits. Just type git rebase and it will work.", "overall_preference": -2}\n" + }, + { + "type": "markdown", + "source": "### 3. Create Dataset FileSet and Upload Training Data", + "source_html": "

3. Create Dataset FileSet and Upload Training Data

\n" + }, + { + "type": "markdown", + "source": "Install huggingface datasets package to download public [nvidia/HelpSteer3](https://huggingface.co/datasets/nvidia/HelpSteer3) dataset if it's not installed in your Python environment:\n\n```sh\npip install datasets\n```", + "source_html": "

Install huggingface datasets package to download public nvidia/HelpSteer3 dataset if it's not installed in your Python environment:

\n
pip install datasets\n
\n" + }, + { + "type": "markdown", + "source": "#### Download nvidia/HelpSteer3 Dataset", + "source_html": "

Download nvidia/HelpSteer3 Dataset

\n" + }, + { + "type": "code", + "source": "from pathlib import Path\nfrom datasets import load_dataset, Dataset\nds = load_dataset(\"nvidia/HelpSteer3\", \"preference\")\n\n# Adjust these values to change the size of the training and validation sets\n# The larger the datasets, the better the model will perform but longer the training will take\n# For the purpose of this tutorial, we'll use a small subset of the dataset\ntraining_size = 3000\nvalidation_size = 300\nDATASET_PATH = Path(\"dpo-dataset\").absolute()\n\n# Get training split and verify it's a Dataset (not IterableDataset)\ntrain_dataset = ds[\"train\"]\nvalidation_dataset = ds[\"validation\"]\nassert isinstance(train_dataset, Dataset), \"Expected Dataset type\"\nassert isinstance(validation_dataset, Dataset), \"Expected Dataset type\"\n\n# Select subsets and save to JSONL files\ntesting_ds = train_dataset.select(range(training_size))\nvalidation_ds = validation_dataset.select(range(validation_size))\n\n# Create directory if it doesn't exist\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Save subsets to JSONL files\ntesting_ds.to_json(f\"{DATASET_PATH}/training.jsonl\")\nvalidation_ds.to_json(f\"{DATASET_PATH}/validation.jsonl\")\n\nprint(f\"Saved training.jsonl with {len(testing_ds)} rows\")\nprint(f\"Saved validation.jsonl with {len(validation_ds)} rows\")", + "language": "python", + "source_html": "from pathlib import Path\nfrom datasets import load_dataset, Dataset\nds = load_dataset("nvidia/HelpSteer3", "preference")\n\n# Adjust these values to change the size of the training and validation sets\n# The larger the datasets, the better the model will perform but longer the training will take\n# For the purpose of this tutorial, we'll use a small subset of the dataset\ntraining_size = 3000\nvalidation_size = 300\nDATASET_PATH = Path("dpo-dataset").absolute()\n\n# Get training split and verify it's a Dataset (not IterableDataset)\ntrain_dataset = ds["train"]\nvalidation_dataset = ds["validation"]\nassert isinstance(train_dataset, Dataset), "Expected Dataset type"\nassert isinstance(validation_dataset, Dataset), "Expected Dataset type"\n\n# Select subsets and save to JSONL files\ntesting_ds = train_dataset.select(range(training_size))\nvalidation_ds = validation_dataset.select(range(validation_size))\n\n# Create directory if it doesn't exist\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Save subsets to JSONL files\ntesting_ds.to_json(f"{DATASET_PATH}/training.jsonl")\nvalidation_ds.to_json(f"{DATASET_PATH}/validation.jsonl")\n\nprint(f"Saved training.jsonl with {len(testing_ds)} rows")\nprint(f"Saved validation.jsonl with {len(validation_ds)} rows")\n" + }, + { + "type": "markdown", + "source": "#### Upload Training Data", + "source_html": "

Upload Training Data

\n" + }, + { + "type": "code", + "source": "# Create fileset to store DPO training data\nDATASET_NAME = \"dpo-dataset\"\n\ntry:\n client.files.filesets.create(\n workspace=\"default\",\n name=DATASET_NAME,\n description=\"dpo training data\"\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\n# Upload training data files individually to ensure correct structure\nclient.files.upload(\n local_path=DATASET_PATH, # Local directory with your JSONL files\n remote_path=\"\",\n fileset=DATASET_NAME,\n workspace=\"default\"\n)\n\n# Validate training data is uploaded correctly\nprint(\"Training data:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace=\"default\").data], indent=2))", + "language": "python", + "source_html": "# Create fileset to store DPO training data\nDATASET_NAME = "dpo-dataset"\n\ntry:\n client.files.filesets.create(\n workspace="default",\n name=DATASET_NAME,\n description="dpo training data"\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\n# Upload training data files individually to ensure correct structure\nclient.files.upload(\n local_path=DATASET_PATH, # Local directory with your JSONL files\n remote_path="",\n fileset=DATASET_NAME,\n workspace="default"\n)\n\n# Validate training data is uploaded correctly\nprint("Training data:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace="default").data], indent=2))\n" + }, + { + "type": "markdown", + "source": "### 4. Secrets Setup\n\nIf you plan to use NGC or HuggingFace models, you'll need to configure authentication:\n\n- **NGC models** (`ngc://` URIs): Requires NGC API key\n- **HuggingFace models** (`hf://` URIs): Requires HF token for gated/private models\n\n\nConfigure these as secrets in your platform. See [Managing Secrets](../../get-started/concepts/manage-secrets.md) for detailed instructions.\n\nGet your credentials to access base models:\n- [NGC API Key](https://ngc.nvidia.com/) (Setup → Generate API Key)\n- [HuggingFace Token](https://huggingface.co/settings/tokens) (Create token with Read access)\n\n\n---\n\n#### Quick Setup Example\n\nIn this tutorial we are going to work with [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) model from HuggingFace. Ensure that you have sufficient permissions to download the model. If you cannot see the files in the [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) Hugging Face page, request access\n\n**HuggingFace Authentication:**\n- For gated models (Llama, Gemma), you must provide a HuggingFace token via the `token_secret` parameter\n- Get your token from [HuggingFace Settings](https://huggingface.co/settings/tokens) (requires Read access)\n- Accept the model's terms on the HuggingFace model page before using it. Example: [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main)\n- For public models, you can omit the `token_secret` parameter when creating a fileset for model in the next step", + "source_html": "

4. Secrets Setup

\n

If you plan to use NGC or HuggingFace models, you'll need to configure authentication:

\n
    \n
  • NGC models (ngc:// URIs): Requires NGC API key
    • \n
  • HuggingFace models (hf:// URIs): Requires HF token for gated/private models
    • \n
\n

Configure these as secrets in your platform. See Managing Secrets for detailed instructions.

\n

Get your credentials to access base models:

\n\n
\n

Quick Setup Example

\n

In this tutorial we are going to work with meta-llama/Llama-3.2-1B-Instruct model from HuggingFace. Ensure that you have sufficient permissions to download the model. If you cannot see the files in the meta-llama/Llama-3.2-1B-Instruct Hugging Face page, request access

\n

HuggingFace Authentication:

\n
    \n
  • For gated models (Llama, Gemma), you must provide a HuggingFace token via the token_secret parameter
    • \n
  • Get your token from HuggingFace Settings (requires Read access)
    • \n
  • Accept the model's terms on the HuggingFace model page before using it. Example: meta-llama/Llama-3.2-1B-Instruct
    • \n
  • For public models, you can omit the token_secret parameter when creating a fileset for model in the next step
    • \n
\n" + }, + { + "type": "code", + "source": "# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set\nHF_TOKEN = os.getenv(\"HF_TOKEN\")\nNGC_API_KEY = os.getenv(\"NGC_API_KEY\")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f\"{label} is not set\")\n try:\n secret = client.secrets.create(\n name=name,\n workspace=\"default\",\n value=value,\n )\n print(f\"Created secret: {name}\")\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n return client.secrets.retrieve(name=name, workspace=\"default\")\n\n\n# Create HuggingFace token secret\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")\nprint(\"HF_TOKEN secret:\")\nprint(hf_secret.model_dump_json(indent=2))\n\n# Create NGC API key secret\n# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n# ngc_api_key = create_or_get_secret(\"ngc-api-key\", NGC_API_KEY, \"NGC_API_KEY\")", + "language": "python", + "source_html": "# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set\nHF_TOKEN = os.getenv("HF_TOKEN")\nNGC_API_KEY = os.getenv("NGC_API_KEY")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f"{label} is not set")\n try:\n secret = client.secrets.create(\n name=name,\n workspace="default",\n value=value,\n )\n print(f"Created secret: {name}")\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n return client.secrets.retrieve(name=name, workspace="default")\n\n\n# Create HuggingFace token secret\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\nprint("HF_TOKEN secret:")\nprint(hf_secret.model_dump_json(indent=2))\n\n# Create NGC API key secret\n# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n# ngc_api_key = create_or_get_secret("ngc-api-key", NGC_API_KEY, "NGC_API_KEY")\n" + }, + { + "type": "markdown", + "source": "### 5. Create Base Model FileSet and Model Entity\n\nCreate a fileset pointing to [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) model in HuggingFace that we will train with DPO. Then create a Model Entity that references this fileset. Model downloading will take place at the DPO finetuning job creation time.\n\nNote: for public models, you can omit the `token_secret` parameter when creating a model fileset.", + "source_html": "

5. Create Base Model FileSet and Model Entity

\n

Create a fileset pointing to meta-llama/Llama-3.2-1B-Instruct model in HuggingFace that we will train with DPO. Then create a Model Entity that references this fileset. Model downloading will take place at the DPO finetuning job creation time.

\n

Note: for public models, you can omit the token_secret parameter when creating a model fileset.

\n" + }, + { + "type": "code", + "source": "import time\n\n# Create a fileset pointing to the desired HuggingFace model\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = \"meta-llama/Llama-3.2-1B-Instruct\"\nMODEL_NAME = \"llama-3-2-1b-base\"\n\n# Ensure you have a HuggingFace token secret created\ntry:\n base_model_fs = client.files.filesets.create(\n workspace=\"default\",\n name=MODEL_NAME,\n description=\"Llama 3.2 1B base model from HuggingFace\",\n storage=HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type=\"model\",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\n print(f\"Created base model fileset: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model fileset already exists. Skipping creation.\")\n base_model_fs = client.files.filesets.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\n# Create Model Entity referencing the FileSet\ntry:\n base_model = client.models.create(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n )\n print(f\"Created Model Entity: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model already exists. Updating fileset if different.\")\n base_model = client.models.update(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n )\n\nprint(f\"\\nBase model fileset: fileset://default/{base_model.name}\")\nprint(\"Base model fileset files list:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace=\"default\").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint(\"\\nWaiting for ModelSpec to be populated...\")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f\"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds\")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\nprint(f\"ModelSpec populated: {base_model.spec}\")", + "language": "python", + "source_html": "import time\n\n# Create a fileset pointing to the desired HuggingFace model\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = "meta-llama/Llama-3.2-1B-Instruct"\nMODEL_NAME = "llama-3-2-1b-base"\n\n# Ensure you have a HuggingFace token secret created\ntry:\n base_model_fs = client.files.filesets.create(\n workspace="default",\n name=MODEL_NAME,\n description="Llama 3.2 1B base model from HuggingFace",\n storage=HuggingfaceStorageConfigParam(\n type="huggingface",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type="model",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\n print(f"Created base model fileset: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model fileset already exists. Skipping creation.")\n base_model_fs = client.files.filesets.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\n# Create Model Entity referencing the FileSet\ntry:\n base_model = client.models.create(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n )\n print(f"Created Model Entity: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model already exists. Updating fileset if different.")\n base_model = client.models.update(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n )\n\nprint(f"\\nBase model fileset: fileset://default/{base_model.name}")\nprint("Base model fileset files list:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace="default").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint("\\nWaiting for ModelSpec to be populated...")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\nprint(f"ModelSpec populated: {base_model.spec}")\n" + }, + { + "type": "markdown", + "source": "### 6. Create DPO Finetuning Job\nCreate a customization job with an inline target referencing the base model and dataset filesets created in previous steps.", + "source_html": "

6. Create DPO Finetuning Job

\n

Create a customization job with an inline target referencing the base model and dataset filesets created in previous steps.

\n" + }, + { + "type": "markdown", + "source": "**Target `model_uri` Format:**\n\nCurrently, `model_uri` must reference a FileSet:\n- **FileSet:** `fileset://{workspace}/{fileset-name}`\n\nSupport for direct HuggingFace (`hf://`) and NGC (`ngc://`) URIs is coming soon. For now, create a fileset as shown in the previous step, and the HuggingFace model will be downloaded at the beginning the finetuning job.\n\n**GPU Requirements:**\n- 1B models: 1 GPU (24GB+ VRAM)\n- 3B models: 1-2 GPUs \n- 8B models: 2-4 GPUs\n- 70B models: 8+ GPUs \n\nAdjust `num_gpus_per_node` and `tensor_parallel_size` based on your model size.\n\n**Important**\n\nWhen setting `val_check_interval` for DPO, use a fractional value (e.g., `0.5` for twice per epoch) or omit it entirely (validates once at end of epoch). Avoid integer step counts — they may not divide evenly into the total training steps, which can prevent validation from running on the final step.", + "source_html": "

Target model_uri Format:

\n

Currently, model_uri must reference a FileSet:

\n
    \n
  • FileSet: fileset://{workspace}/{fileset-name}
    • \n
\n

Support for direct HuggingFace (hf://) and NGC (ngc://) URIs is coming soon. For now, create a fileset as shown in the previous step, and the HuggingFace model will be downloaded at the beginning the finetuning job.

\n

GPU Requirements:

\n
    \n
  • 1B models: 1 GPU (24GB+ VRAM)
    • \n
  • 3B models: 1-2 GPUs
    • \n
  • 8B models: 2-4 GPUs
    • \n
  • 70B models: 8+ GPUs
    • \n
\n

Adjust num_gpus_per_node and tensor_parallel_size based on your model size.

\n

Important

\n

When setting val_check_interval for DPO, use a fractional value (e.g., 0.5 for twice per epoch) or omit it entirely (validates once at end of epoch). Avoid integer step counts — they may not divide evenly into the total training steps, which can prevent validation from running on the final step.

\n" + }, + { + "type": "code", + "source": "import uuid\njob_suffix = uuid.uuid4().hex[:4]\n\nJOB_NAME = f\"my-dpo-job-{job_suffix}\"\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=DpoTrainingParam(\n type=\"dpo\",\n epochs=1,\n batch_size=16,\n learning_rate=0.00005,\n max_seq_length=4096,\n ref_policy_kl_penalty=0.1,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n )\n)\n\nprint(f\"Job ID: {job.name}\")\nprint(f\"Output model: {job.spec.output.name}\")", + "language": "python", + "source_html": "import uuid\njob_suffix = uuid.uuid4().hex[:4]\n\nJOB_NAME = f"my-dpo-job-{job_suffix}"\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace="default",\n spec=CustomizationJobInputParam(\n model=f"default/{base_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=DpoTrainingParam(\n type="dpo",\n epochs=1,\n batch_size=16,\n learning_rate=0.00005,\n max_seq_length=4096,\n ref_policy_kl_penalty=0.1,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n )\n)\n\nprint(f"Job ID: {job.name}")\nprint(f"Output model: {job.spec.output.name}")\n" + }, + { + "type": "markdown", + "source": "### 7. Track Training Progress", + "source_html": "

7. Track Training Progress

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Poll job status every 10 seconds until completed\nwhile True:\n status = client.customization.jobs.get_status(\n name=job.name,\n workspace=\"default\"\n )\n \n clear_output(wait=True)\n print(f\"Job Status: {status.model_dump_json(indent=2)}\")\n\n # Extract training progress from nested steps structure\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n\n for job_step in status.steps or []:\n if job_step.name == \"customization-training-job\":\n for task in job_step.tasks or []:\n task_details = task.status_details or {}\n step = task_details.get(\"step\")\n max_steps = task_details.get(\"max_steps\")\n training_phase = task_details.get(\"phase\")\n break\n break\n\n if step is not None and max_steps is not None:\n progress_pct = (step / max_steps) * 100\n print(f\"Training Progress: Step {step}/{max_steps} ({progress_pct:.1f}%)\")\n if training_phase:\n print(f\"Training Phase: {training_phase}\")\n else:\n print(\"Training step not started yet or progress info not available\")\n \n # Exit loop when job is completed (or failed/cancelled)\n if status.status in (\"completed\", \"failed\", \"cancelled\", \"error\"):\n print(f\"\\nJob finished with status: {status.status}\")\n break\n \n time.sleep(10)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Poll job status every 10 seconds until completed\nwhile True:\n status = client.customization.jobs.get_status(\n name=job.name,\n workspace="default"\n )\n \n clear_output(wait=True)\n print(f"Job Status: {status.model_dump_json(indent=2)}")\n\n # Extract training progress from nested steps structure\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n\n for job_step in status.steps or []:\n if job_step.name == "customization-training-job":\n for task in job_step.tasks or []:\n task_details = task.status_details or {}\n step = task_details.get("step")\n max_steps = task_details.get("max_steps")\n training_phase = task_details.get("phase")\n break\n break\n\n if step is not None and max_steps is not None:\n progress_pct = (step / max_steps) * 100\n print(f"Training Progress: Step {step}/{max_steps} ({progress_pct:.1f}%)")\n if training_phase:\n print(f"Training Phase: {training_phase}")\n else:\n print("Training step not started yet or progress info not available")\n \n # Exit loop when job is completed (or failed/cancelled)\n if status.status in ("completed", "failed", "cancelled", "error"):\n print(f"\\nJob finished with status: {status.status}")\n break\n \n time.sleep(10)\n" + }, + { + "type": "markdown", + "source": "**Interpreting DPO Training Metrics:**\n\nDPO training produces several key metrics:\n\n| Metric | Description | What to Look For |\n|--------|-------------|------------------|\n| **loss** | Total training loss (preference_loss + sft_loss) | Should decrease over training |\n| **preference_loss** | Core DPO loss measuring preference learning | Starts near ln(2) ≈ 0.693, should decrease |\n| **sft_loss** | SFT regularization term (often 0 for pure DPO) | Depends on configuration |\n| **accuracy** | Fraction of samples where chosen > rejected | Should increase toward 80-95%+ |\n| **rewards_chosen_mean** | Average implicit reward for chosen responses | Should be positive |\n| **rewards_rejected_mean** | Average implicit reward for rejected responses | Should be negative |\n\n**Key Indicators:**\n\n- **Reward Margin** = `rewards_chosen_mean - rewards_rejected_mean`\n - Should be positive and increasing\n - Indicates the model is learning to distinguish preferences\n\n- **Accuracy Interpretation:**\n - 50% = random chance (no learning)\n - 66-75% = early/moderate learning\n - 80%+ = good preference learning\n - 95%+ = strong preference alignment\n\n**Troubleshooting:**\n\n- **Loss near ln(2) ≈ 0.693**: Model is at random chance level, training just starting or not learning\n- **Accuracy stuck at ~50%**: Check data quality, increase learning rate, or verify preference labels\n- **Negative reward margin**: Model is learning the wrong direction—check chosen/rejected labels\n- **Loss increasing**: Learning rate too high or data quality issues\n\n**Note:** Training metrics measure optimization progress, not final model quality. Always evaluate the deployed model on your specific use case.", + "source_html": "

Interpreting DPO Training Metrics:

\n

DPO training produces several key metrics:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
MetricDescriptionWhat to Look For
lossTotal training loss (preference_loss + sft_loss)Should decrease over training
preference_lossCore DPO loss measuring preference learningStarts near ln(2) ≈ 0.693, should decrease
sft_lossSFT regularization term (often 0 for pure DPO)Depends on configuration
accuracyFraction of samples where chosen > rejectedShould increase toward 80-95%+
rewards_chosen_meanAverage implicit reward for chosen responsesShould be positive
rewards_rejected_meanAverage implicit reward for rejected responsesShould be negative
\n

Key Indicators:

\n
    \n
  • \n

    Reward Margin = rewards_chosen_mean - rewards_rejected_mean

    \n
      \n
    • Should be positive and increasing
      • \n
    • Indicates the model is learning to distinguish preferences
      • \n
    \n
    • \n
  • \n

    Accuracy Interpretation:

    \n
      \n
    • 50% = random chance (no learning)
      • \n
    • 66-75% = early/moderate learning
      • \n
    • 80%+ = good preference learning
      • \n
    • 95%+ = strong preference alignment
      • \n
    \n
    • \n
\n

Troubleshooting:

\n
    \n
  • Loss near ln(2) ≈ 0.693: Model is at random chance level, training just starting or not learning
    • \n
  • Accuracy stuck at ~50%: Check data quality, increase learning rate, or verify preference labels
    • \n
  • Negative reward margin: Model is learning the wrong direction—check chosen/rejected labels
    • \n
  • Loss increasing: Learning rate too high or data quality issues
    • \n
\n

Note: Training metrics measure optimization progress, not final model quality. Always evaluate the deployed model on your specific use case.

\n" + }, + { + "type": "markdown", + "source": "### 8. Deploy Fine-Tuned Model\n\nOnce training completes, deploy using the Deployment Management Service:", + "source_html": "

8. Deploy Fine-Tuned Model

\n

Once training completes, deploy using the Deployment Management Service:

\n" + }, + { + "type": "code", + "source": "# Validate model entity exists\nmodel_entity = client.models.retrieve(workspace='default', name=job.spec.output.name)\nprint(model_entity.model_dump_json(indent=2))", + "language": "python", + "source_html": "# Validate model entity exists\nmodel_entity = client.models.retrieve(workspace='default', name=job.spec.output.name)\nprint(model_entity.model_dump_json(indent=2))\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# Create deployment config\ndeploy_suffix = uuid.uuid4().hex[:4]\nDEPLOYMENT_CONFIG_NAME = f\"dpo-model-deployment-cfg-{deploy_suffix}\"\nDEPLOYMENT_NAME = f\"dpo-model-deployment-{deploy_suffix}\"\n\ndeployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=DEPLOYMENT_CONFIG_NAME,\n nim_deployment=NIMDeploymentParam(\n image_name=\"nvcr.io/nim/nvidia/llm-nim\",\n image_tag=\"1.13.1\",\n gpu=1,\n model_name=job.spec.output.name, # ModelEntity name from training,\n model_namespace=\"default\", # Workspace where ModelEntity lives\n )\n)\n\n# Deploy model using deployment_config created above\ndeployment = client.inference.deployments.create(\n workspace=\"default\",\n name=DEPLOYMENT_NAME,\n config=deployment_config.name\n)\n\n\n# Check deployment status\ndeployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace=\"default\"\n)\n\nprint(f\"Deployment name: {deployment.name}\")\nprint(f\"Deployment status: {deployment_status.status}\")", + "language": "python", + "source_html": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# Create deployment config\ndeploy_suffix = uuid.uuid4().hex[:4]\nDEPLOYMENT_CONFIG_NAME = f"dpo-model-deployment-cfg-{deploy_suffix}"\nDEPLOYMENT_NAME = f"dpo-model-deployment-{deploy_suffix}"\n\ndeployment_config = client.inference.deployment_configs.create(\n workspace="default",\n name=DEPLOYMENT_CONFIG_NAME,\n nim_deployment=NIMDeploymentParam(\n image_name="nvcr.io/nim/nvidia/llm-nim",\n image_tag="1.13.1",\n gpu=1,\n model_name=job.spec.output.name, # ModelEntity name from training,\n model_namespace="default", # Workspace where ModelEntity lives\n )\n)\n\n# Deploy model using deployment_config created above\ndeployment = client.inference.deployments.create(\n workspace="default",\n name=DEPLOYMENT_NAME,\n config=deployment_config.name\n)\n\n\n# Check deployment status\ndeployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace="default"\n)\n\nprint(f"Deployment name: {deployment.name}")\nprint(f"Deployment status: {deployment_status.status}")\n" + }, + { + "type": "markdown", + "source": "### Monitor status of deployment", + "source_html": "

Monitor status of deployment

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Poll deployment status every 15 seconds until ready\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f\"Monitoring deployment '{deployment.name}'...\")\nprint(f\"Timeout: {TIMEOUT_MINUTES} minutes\\n\")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace=\"default\"\n )\n \n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n \n clear_output(wait=True)\n print(f\"Deployment: {deployment.name}\")\n print(f\"Status: {deployment_status.status}\")\n print(f\"Elapsed time: {elapsed_min}m {elapsed_sec}s\")\n \n # Check if deployment is ready\n if deployment_status.status == \"READY\":\n print(\"\\nDeployment is ready!\")\n if not client.models.wait_for_gateway(deployment.name, workspace=\"default\", timeout=60):\n raise RuntimeError(\"Inference gateway did not become ready\")\n break\n \n # Check for failure states\n if deployment_status.status in (\"FAILED\", \"ERROR\", \"TERMINATED\", \"LOST\"):\n raise RuntimeError(f\"Deployment failed with status: {deployment_status.status}\")\n \n # Check timeout\n if elapsed > timeout_seconds:\n raise TimeoutError(f\"Deployment timeout after {TIMEOUT_MINUTES} minutes\")\n \n time.sleep(15)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Poll deployment status every 15 seconds until ready\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f"Monitoring deployment '{deployment.name}'...")\nprint(f"Timeout: {TIMEOUT_MINUTES} minutes\\n")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace="default"\n )\n \n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n \n clear_output(wait=True)\n print(f"Deployment: {deployment.name}")\n print(f"Status: {deployment_status.status}")\n print(f"Elapsed time: {elapsed_min}m {elapsed_sec}s")\n \n # Check if deployment is ready\n if deployment_status.status == "READY":\n print("\\nDeployment is ready!")\n if not client.models.wait_for_gateway(deployment.name, workspace="default", timeout=60):\n raise RuntimeError("Inference gateway did not become ready")\n break\n \n # Check for failure states\n if deployment_status.status in ("FAILED", "ERROR", "TERMINATED", "LOST"):\n raise RuntimeError(f"Deployment failed with status: {deployment_status.status}")\n \n # Check timeout\n if elapsed > timeout_seconds:\n raise TimeoutError(f"Deployment timeout after {TIMEOUT_MINUTES} minutes")\n \n time.sleep(15)\n" + }, + { + "type": "markdown", + "source": "The deployment service automatically:\n- Downloads model weights from the Files service\n- Provisions storage (PVC) for the weights\n- Configures and starts the NIM container\n\n**Multi-GPU Deployment:**\n\nFor larger models requiring multiple GPUs, configure parallelism with environment variables:\n\n```python\ndeployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=\"sft-model-config-multigpu\",\n \n nim_deployment={\n \"image_name\": \"nvcr.io/nim/nvidia/llm-nim\",\n \"image_tag\": \"1.13.1\",\n \"gpu\": 2, # Total GPUs\n \"additional_envs\": {\n \"NIM_TENSOR_PARALLEL_SIZE\": \"2\", # Tensor parallelism\n \"NIM_PIPELINE_PARALLEL_SIZE\": \"1\" # Pipeline parallelism\n }\n }\n)\n```", + "source_html": "

The deployment service automatically:

\n
    \n
  • Downloads model weights from the Files service
    • \n
  • Provisions storage (PVC) for the weights
    • \n
  • Configures and starts the NIM container
    • \n
\n

Multi-GPU Deployment:

\n

For larger models requiring multiple GPUs, configure parallelism with environment variables:

\n
deployment_config = client.inference.deployment_configs.create(\n    workspace="default",\n    name="sft-model-config-multigpu",\n    \n    nim_deployment={\n        "image_name": "nvcr.io/nim/nvidia/llm-nim",\n        "image_tag": "1.13.1",\n        "gpu": 2,  # Total GPUs\n        "additional_envs": {\n            "NIM_TENSOR_PARALLEL_SIZE": "2",  # Tensor parallelism\n            "NIM_PIPELINE_PARALLEL_SIZE": "1"  # Pipeline parallelism\n        }\n    }\n)\n
\n" + }, + { + "type": "markdown", + "source": "**Single-Node Constraint:** Model deployments are limited to a single node. The maximum `gpu` value depends on the total GPUs available on a single node in your cluster. Multi-node deployments are not supported.\n\n---\n\n#### GPU Parallelism\n\nBy default, NIM uses all GPUs for tensor parallelism (TP). You can customize this behavior using the `NIM_TENSOR_PARALLEL_SIZE` and `NIM_PIPELINE_PARALLEL_SIZE` environment variables.\n\n| Strategy | Description | Best For |\n|----------|-------------|----------|\n| **Tensor Parallel (TP)** | Splits model layers across GPUs | Lowest latency |\n| **Pipeline Parallel (PP)** | Splits model depth across GPUs | Highest throughput |\n\n**Formula:** `gpu` = `NIM_TENSOR_PARALLEL_SIZE` × `NIM_PIPELINE_PARALLEL_SIZE`\n\n---\n\n#### Example Configurations\n\n**Default (TP=8, PP=1) — Lowest Latency**\n```\n\"gpu\": 8\n# NIM automatically sets NIM_TENSOR_PARALLEL_SIZE=8\n```\n\n**Balanced (TP=4, PP=2)**\n```\n\"gpu\": 8,\n\"additional_envs\": {\n \"NIM_TENSOR_PARALLEL_SIZE\": \"4\",\n \"NIM_PIPELINE_PARALLEL_SIZE\": \"2\"\n}\n```\n\n**Throughput Optimized (TP=2, PP=4)**\n```\n\"gpu\": 8,\n\"additional_envs\": {\n \"NIM_TENSOR_PARALLEL_SIZE\": \"2\",\n \"NIM_PIPELINE_PARALLEL_SIZE\": \"4\"\n}\n```", + "source_html": "

Single-Node Constraint: Model deployments are limited to a single node. The maximum gpu value depends on the total GPUs available on a single node in your cluster. Multi-node deployments are not supported.

\n
\n

GPU Parallelism

\n

By default, NIM uses all GPUs for tensor parallelism (TP). You can customize this behavior using the NIM_TENSOR_PARALLEL_SIZE and NIM_PIPELINE_PARALLEL_SIZE environment variables.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
StrategyDescriptionBest For
Tensor Parallel (TP)Splits model layers across GPUsLowest latency
Pipeline Parallel (PP)Splits model depth across GPUsHighest throughput
\n

Formula: gpu = NIM_TENSOR_PARALLEL_SIZE × NIM_PIPELINE_PARALLEL_SIZE

\n
\n

Example Configurations

\n

Default (TP=8, PP=1) — Lowest Latency

\n
"gpu": 8\n# NIM automatically sets NIM_TENSOR_PARALLEL_SIZE=8\n
\n

Balanced (TP=4, PP=2)

\n
"gpu": 8,\n"additional_envs": {\n    "NIM_TENSOR_PARALLEL_SIZE": "4",\n    "NIM_PIPELINE_PARALLEL_SIZE": "2"\n}\n
\n

Throughput Optimized (TP=2, PP=4)

\n
"gpu": 8,\n"additional_envs": {\n    "NIM_TENSOR_PARALLEL_SIZE": "2",\n    "NIM_PIPELINE_PARALLEL_SIZE": "4"\n}\n
\n" + }, + { + "type": "markdown", + "source": "### 9. Evaluate Your Model\n\nAfter training, evaluate whether your model meets your requirements:\n\n#### Quick Manual Evaluation", + "source_html": "

9. Evaluate Your Model

\n

After training, evaluate whether your model meets your requirements:

\n

Quick Manual Evaluation

\n" + }, + { + "type": "code", + "source": "# Wait for deployment to be ready, then test\nmessages = [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": \"Write a short email to my colleague.\"}\n]\n\nresponse = client.inference.gateway.provider.post(\n \"v1/chat/completions\",\n name=deployment.name,\n workspace=\"default\",\n body={\n \"model\": f\"default/{job.spec.output.name}\", # Match the model_name from deployment config\n \"messages\": messages,\n \"temperature\": 0.7,\n \"max_tokens\": 256\n }\n)\n\n# Display prompt and completion\nprint(\"=\" * 60)\nprint(\"PROMPT\")\nprint(\"=\" * 60)\nfor msg in messages:\n print(f\"[{msg['role'].upper()}]\")\n print(msg[\"content\"])\n print()\n\nprint(\"=\" * 60)\nprint(\"COMPLETION\")\nprint(\"=\" * 60)\nprint(\"[ASSISTANT]\")\ncompletion = response[\"choices\"][0][\"message\"][\"content\"]\nprint(completion)", + "language": "python", + "source_html": "# Wait for deployment to be ready, then test\nmessages = [\n {"role": "system", "content": "You are a helpful assistant."},\n {"role": "user", "content": "Write a short email to my colleague."}\n]\n\nresponse = client.inference.gateway.provider.post(\n "v1/chat/completions",\n name=deployment.name,\n workspace="default",\n body={\n "model": f"default/{job.spec.output.name}", # Match the model_name from deployment config\n "messages": messages,\n "temperature": 0.7,\n "max_tokens": 256\n }\n)\n\n# Display prompt and completion\nprint("=" * 60)\nprint("PROMPT")\nprint("=" * 60)\nfor msg in messages:\n print(f"[{msg['role'].upper()}]")\n print(msg["content"])\n print()\n\nprint("=" * 60)\nprint("COMPLETION")\nprint("=" * 60)\nprint("[ASSISTANT]")\ncompletion = response["choices"][0]["message"]["content"]\nprint(completion)\n" + }, + { + "type": "markdown", + "source": "#### Evaluation Best Practices\n\n**Manual Evaluation** (Recommended)\n- Test with real-world examples from your use case\n- Compare responses to base model and expected outputs\n- Verify the model exhibits desired behavior changes\n- Check edge cases and error handling\n\n**What to look for:**\n- ✅ Model follows your desired output format\n- ✅ Applies domain knowledge correctly\n- ✅ Maintains general language capabilities\n- ✅ Avoids unwanted behaviors or biases\n- ❌ Doesn't hallucinate facts not in training data\n- ❌ Doesn't produce repetitive or nonsensical outputs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, see the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (see [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated HuggingFace models (Llama, Gemma), accept the license on the model page\n- Check the `model_uri` format is correct (`fileset://`)\n- Ensure you have accepted the model's terms of service on HuggingFace\n- Check job status and logs: `client.customization.jobs.retrieve(name=job.name, workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n1. **First try:** Reduce `micro_batch_size` from 2 to 1\n2. **Still OOM:** Reduce `batch_size` from 16 to 8\n3. **Still OOM:** Reduce `max_seq_length` from 2048 to 1024 or 512\n4. **Last resort:** Increase GPU count and use `tensor_parallel_size` for model sharding\n\n**Loss curves not decreasing (underfitting):**\n- Increase training duration: `epochs: 5-10` instead of 3\n- Adjust learning rate: Try `1e-5` to `1e-4`\n- Add warmup: Set `warmup_steps` to ~10% of total training steps\n- Check data quality: Verify formatting, remove duplicates, ensure diversity\n\n**Training loss decreases but validation loss increases (overfitting):**\n- Reduce epochs: Try `epochs: 1-2` instead of 5+\n- Lower learning rate: Use `2e-5` or `1e-5`\n- Increase dataset size and diversity\n- Verify train/validation split has no data leakage\n\n**Model output quality is poor despite good training metrics:**\n- Training metrics optimize for loss, not your actual task—evaluate on real use cases\n- Review data quality, format, and diversity—metrics can be misleading with poor data\n- Try a different base model size or architecture\n- Adjust learning rate and batch size\n- Compare to baseline: Test base model to ensure fine-tuning improved performance\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=job.spec.output.name, workspace=\"default\")`\n- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace=\"default\")`\n- Ensure sufficient GPU resources available for model size\n- Verify NIM image tag `1.13.1` is compatible with your model\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning", + "source_html": "

Evaluation Best Practices

\n

Manual Evaluation (Recommended)

\n
    \n
  • Test with real-world examples from your use case
    • \n
  • Compare responses to base model and expected outputs
    • \n
  • Verify the model exhibits desired behavior changes
    • \n
  • Check edge cases and error handling
    • \n
\n

What to look for:

\n
    \n
  • ✅ Model follows your desired output format
    • \n
  • ✅ Applies domain knowledge correctly
    • \n
  • ✅ Maintains general language capabilities
    • \n
  • ✅ Avoids unwanted behaviors or biases
    • \n
  • ❌ Doesn't hallucinate facts not in training data
    • \n
  • ❌ Doesn't produce repetitive or nonsensical outputs
    • \n
\n
\n

Hyperparameters

\n

For detailed information on all available hyperparameters, recommended values, and tuning guidance, see the Hyperparameter Reference.

\n
\n

Troubleshooting

\n

Job fails during model download:

\n
    \n
  • Verify authentication secrets are configured (see Managing Secrets)
    • \n
  • For gated HuggingFace models (Llama, Gemma), accept the license on the model page
    • \n
  • Check the model_uri format is correct (fileset://)
    • \n
  • Ensure you have accepted the model's terms of service on HuggingFace
    • \n
  • Check job status and logs: client.customization.jobs.retrieve(name=job.name, workspace="default")
    • \n
\n

Job fails with OOM (Out of Memory) error:

\n
    \n
  1. First try: Reduce micro_batch_size from 2 to 1
    2. \n
  2. Still OOM: Reduce batch_size from 16 to 8
    3. \n
  3. Still OOM: Reduce max_seq_length from 2048 to 1024 or 512
    4. \n
  4. Last resort: Increase GPU count and use tensor_parallel_size for model sharding
    5. \n
\n

Loss curves not decreasing (underfitting):

\n
    \n
  • Increase training duration: epochs: 5-10 instead of 3
    • \n
  • Adjust learning rate: Try 1e-5 to 1e-4
    • \n
  • Add warmup: Set warmup_steps to ~10% of total training steps
    • \n
  • Check data quality: Verify formatting, remove duplicates, ensure diversity
    • \n
\n

Training loss decreases but validation loss increases (overfitting):

\n
    \n
  • Reduce epochs: Try epochs: 1-2 instead of 5+
    • \n
  • Lower learning rate: Use 2e-5 or 1e-5
    • \n
  • Increase dataset size and diversity
    • \n
  • Verify train/validation split has no data leakage
    • \n
\n

Model output quality is poor despite good training metrics:

\n
    \n
  • Training metrics optimize for loss, not your actual task—evaluate on real use cases
    • \n
  • Review data quality, format, and diversity—metrics can be misleading with poor data
    • \n
  • Try a different base model size or architecture
    • \n
  • Adjust learning rate and batch size
    • \n
  • Compare to baseline: Test base model to ensure fine-tuning improved performance
    • \n
\n

Deployment fails:

\n
    \n
  • Verify output model exists: client.models.retrieve(name=job.spec.output.name, workspace="default")
    • \n
  • Check deployment logs: client.inference.deployments.get_logs(name=deployment.name, workspace="default")
    • \n
  • Ensure sufficient GPU resources available for model size
    • \n
  • Verify NIM image tag 1.13.1 is compatible with your model
    • \n
\n

Next Steps

\n\n" + } +] }; diff --git a/docs/fern/components/notebooks/embedding-customization-job.json b/docs/fern/components/notebooks/embedding-customization-job.json new file mode 100644 index 00000000000..21b42099e26 --- /dev/null +++ b/docs/fern/components/notebooks/embedding-customization-job.json @@ -0,0 +1,186 @@ +{ + "cells": [ + { + "type": "markdown", + "source": "\n\n\n# Embedding Model Customization\n\nLearn how to fine-tune an embedding model to improve retrieval accuracy for your specific domain.\n\n## About\n\nEmbedding models convert text into dense vector representations that capture semantic meaning. Fine-tuning these models on your domain data significantly improves retrieval accuracy—in RAG pipelines, this means the LLM receives more relevant context and produces better answers.\n\n**What you will achieve with embedding fine-tuning:**\n\n- 🎯 **Domain specialization:** Adapt general embeddings for legal, medical, scientific, or financial content\n- 📈 **Improved retrieval:** Achieve 6-10% better recall on domain-specific benchmarks\n- 🔍 **Semantic understanding:** Teach the model your domain's vocabulary and relationships\n\n**Recall@5** measures the fraction of relevant documents that appear in the top 5 search results.\n\n**About the baseline:** In retrieval benchmarks like SciDocs, the pretrained model achieves ~0.159 Recall@5. After fine-tuning on scientific paper triplets, you can expect 6-10% improvement (~0.17 Recall@5).\n\n### Dataset Format for Embedding Models\n\nEmbedding models require **triplet format** for contrastive learning:\n\n```json\n{\"query\": \"What is machine learning?\", \"pos_doc\": \"Machine learning is a subset of AI...\", \"neg_doc\": [\"Gardening tips for beginners...\"]}\n```\n\n- **`query`**: The search query or question\n- **`pos_doc`**: A document relevant to the query (positive example)\n- **`neg_doc`**: List of hard negatives—documents that share some overlap with the query but are not actually relevant (negative example)\n\nThe model learns to maximize similarity between query and positive document while minimizing similarity with negative documents.", + "source_html": "\n\n

Embedding Model Customization

\n

Learn how to fine-tune an embedding model to improve retrieval accuracy for your specific domain.

\n

About

\n

Embedding models convert text into dense vector representations that capture semantic meaning. Fine-tuning these models on your domain data significantly improves retrieval accuracy—in RAG pipelines, this means the LLM receives more relevant context and produces better answers.

\n

What you will achieve with embedding fine-tuning:

\n
    \n
  • 🎯 Domain specialization: Adapt general embeddings for legal, medical, scientific, or financial content
    • \n
  • 📈 Improved retrieval: Achieve 6-10% better recall on domain-specific benchmarks
    • \n
  • 🔍 Semantic understanding: Teach the model your domain's vocabulary and relationships
    • \n
\n

Recall@5 measures the fraction of relevant documents that appear in the top 5 search results.

\n

About the baseline: In retrieval benchmarks like SciDocs, the pretrained model achieves ~0.159 Recall@5. After fine-tuning on scientific paper triplets, you can expect 6-10% improvement (~0.17 Recall@5).

\n

Dataset Format for Embedding Models

\n

Embedding models require triplet format for contrastive learning:

\n
{"query": "What is machine learning?", "pos_doc": "Machine learning is a subset of AI...", "neg_doc": ["Gardening tips for beginners..."]}\n
\n
    \n
  • query: The search query or question
    • \n
  • pos_doc: A document relevant to the query (positive example)
    • \n
  • neg_doc: List of hard negatives—documents that share some overlap with the query but are not actually relevant (negative example)
    • \n
\n

The model learns to maximize similarity between query and positive document while minimizing similarity with negative documents.

\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n2. **Installed the Python SDK** (included with `pip install nemo-platform`)\n3. **HuggingFace token** with read access to download the SPECTER dataset (get one at [huggingface.co/settings/tokens](https://huggingface.co/settings/tokens))\n4. **NGC API key** to pull NIM container images from nvcr.io (get one at [ngc.nvidia.com](https://ngc.nvidia.com/) → Setup → Generate API Key)", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Completed the Quickstart to install and deploy NeMo Platform locally
    2. \n
  2. Installed the Python SDK (included with pip install nemo-platform)
    3. \n
  3. HuggingFace token with read access to download the SPECTER dataset (get one at huggingface.co/settings/tokens)
    4. \n
  4. NGC API key to pull NIM container images from nvcr.io (get one at ngc.nvidia.com → Setup → Generate API Key)
    5. \n
\n" + }, + { + "type": "markdown", + "source": "## Quick Start\n\n### 1. Initialize SDK\n\nThe SDK needs to know your NeMo Platform server URL. By default, `http://localhost:8080` is used in accordance with the [Quickstart](../../get-started/quickstart.md) guide. If NeMo Platform is running at a custom location, you can override the URL by setting the `NMP_BASE_URL` environment variable:\n\n```sh\nexport NMP_BASE_URL=\n```", + "source_html": "

Quick Start

\n

1. Initialize SDK

\n

The SDK needs to know your NeMo Platform server URL. By default, http://localhost:8080 is used in accordance with the Quickstart guide. If NeMo Platform is running at a custom location, you can override the URL by setting the NMP_BASE_URL environment variable:

\n
export NMP_BASE_URL=<YOUR_NMP_BASE_URL>\n
\n" + }, + { + "type": "code", + "source": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace=\"default\"\n)", + "language": "python", + "source_html": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace="default"\n)\n" + }, + { + "type": "markdown", + "source": "### 2. Establish Baseline Performance\n\nBefore fine-tuning, establish baseline performance with the pretrained model. Deploy it, run a test query, and observe where it struggles. Following fine-tuning, compare the results.\n\n**Scenario:** Searching scientific papers by meaning, not keywords.\n\n**Demo setup:**\n- **Query:** \"Conditional Random Fields\" (CRFs) - a method for sequence labeling in NLP\n- **Trap:** \"Random Forests\" shares the word \"random\" but is an unrelated tree-based algorithm\n- **Goal:** The model should distinguish between them.", + "source_html": "

2. Establish Baseline Performance

\n

Before fine-tuning, establish baseline performance with the pretrained model. Deploy it, run a test query, and observe where it struggles. Following fine-tuning, compare the results.

\n

Scenario: Searching scientific papers by meaning, not keywords.

\n

Demo setup:

\n
    \n
  • Query: "Conditional Random Fields" (CRFs) - a method for sequence labeling in NLP
    • \n
  • Trap: "Random Forests" shares the word "random" but is an unrelated tree-based algorithm
    • \n
  • Goal: The model should distinguish between them.
    • \n
\n" + }, + { + "type": "code", + "source": "# Install required packages for dataset preparation\n%pip install -q datasets huggingface_hub", + "language": "python", + "source_html": "# Install required packages for dataset preparation\n%pip install -q datasets huggingface_hub\n" + }, + { + "type": "code", + "source": "import uuid\nimport numpy as np\n\n# Demo query and documents for baseline comparison\nDEMO_QUERY = \"Conditional Random Fields: Probabilistic Models for Segmenting and Labeling Sequence Data\"\n\nDEMO_DOCS = [\n \"Bidirectional LSTM-CRF Models for Sequence Tagging\", # CRF-based paper\n \"An Introduction to Conditional Random Fields\", # CRF tutorial \n \"Random Forests\", # Keyword trap! Unrelated.\n \"Neural Architectures for Named Entity Recognition\", # Related to sequence labeling; may use CRFs\n \"Support Vector Machines for Classification\", # Unrelated ML method\n]\n\nDEMO_LABELS = [\"BiLSTM-CRF\", \"CRF Tutorial\", \"Random Forest\", \"NER\", \"SVM\"]\nDEMO_RELEVANT = {0, 1, 3} # Papers actually relevant to CRFs\n\ndef cosine_similarity(a, b):\n \"\"\"Calculate cosine similarity between two vectors.\"\"\"\n return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))", + "language": "python", + "source_html": "import uuid\nimport numpy as np\n\n# Demo query and documents for baseline comparison\nDEMO_QUERY = "Conditional Random Fields: Probabilistic Models for Segmenting and Labeling Sequence Data"\n\nDEMO_DOCS = [\n "Bidirectional LSTM-CRF Models for Sequence Tagging", # CRF-based paper\n "An Introduction to Conditional Random Fields", # CRF tutorial \n "Random Forests", # Keyword trap! Unrelated.\n "Neural Architectures for Named Entity Recognition", # Related to sequence labeling; may use CRFs\n "Support Vector Machines for Classification", # Unrelated ML method\n]\n\nDEMO_LABELS = ["BiLSTM-CRF", "CRF Tutorial", "Random Forest", "NER", "SVM"]\nDEMO_RELEVANT = {0, 1, 3} # Papers actually relevant to CRFs\n\ndef cosine_similarity(a, b):\n """Calculate cosine similarity between two vectors."""\n return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# NGC API key is required to pull NIM images from nvcr.io\nNGC_API_KEY = os.environ.get(\"NGC_API_KEY\")\nif not NGC_API_KEY:\n raise ValueError(\"NGC_API_KEY environment variable is required. Get one at https://ngc.nvidia.com/ → Setup → Generate API Key\")\n\n# Create NGC secret for pulling NIM images\nNGC_SECRET_NAME = \"ngc-api-key\"\ntry:\n client.secrets.create(name=NGC_SECRET_NAME, workspace=\"default\", value=NGC_API_KEY)\n print(f\"Created secret: {NGC_SECRET_NAME}\")\nexcept ConflictError:\n print(f\"Secret '{NGC_SECRET_NAME}' already exists, continuing...\")\n\n# Deploy base model for baseline comparison\nBASE_MODEL_HF = \"nvidia/llama-nemotron-embed-1b-v2\"\nNIM_IMAGE = \"nvcr.io/nim/nvidia/llama-nemotron-embed-1b-v2\"\nNIM_TAG = \"1.13.0\"\n\nbaseline_suffix = uuid.uuid4().hex[:4]\nBASELINE_DEPLOYMENT_CONFIG = f\"baseline-embedding-cfg-{baseline_suffix}\"\nBASELINE_DEPLOYMENT_NAME = f\"baseline-embedding-{baseline_suffix}\"\n\nprint(\"Creating baseline deployment config...\")\nbaseline_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=BASELINE_DEPLOYMENT_CONFIG,\n nim_deployment=NIMDeploymentParam(\n image_name=NIM_IMAGE,\n image_tag=NIM_TAG,\n gpu=1,\n image_pull_secret=NGC_SECRET_NAME,\n )\n)\n\nprint(\"Deploying base model...\")\nbaseline_deployment = client.inference.deployments.create(\n workspace=\"default\",\n name=BASELINE_DEPLOYMENT_NAME,\n config=baseline_config.name\n)\nprint(f\"Baseline deployment: {baseline_deployment.name}\")", + "language": "python", + "source_html": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# NGC API key is required to pull NIM images from nvcr.io\nNGC_API_KEY = os.environ.get("NGC_API_KEY")\nif not NGC_API_KEY:\n raise ValueError("NGC_API_KEY environment variable is required. Get one at https://ngc.nvidia.com/ → Setup → Generate API Key")\n\n# Create NGC secret for pulling NIM images\nNGC_SECRET_NAME = "ngc-api-key"\ntry:\n client.secrets.create(name=NGC_SECRET_NAME, workspace="default", value=NGC_API_KEY)\n print(f"Created secret: {NGC_SECRET_NAME}")\nexcept ConflictError:\n print(f"Secret '{NGC_SECRET_NAME}' already exists, continuing...")\n\n# Deploy base model for baseline comparison\nBASE_MODEL_HF = "nvidia/llama-nemotron-embed-1b-v2"\nNIM_IMAGE = "nvcr.io/nim/nvidia/llama-nemotron-embed-1b-v2"\nNIM_TAG = "1.13.0"\n\nbaseline_suffix = uuid.uuid4().hex[:4]\nBASELINE_DEPLOYMENT_CONFIG = f"baseline-embedding-cfg-{baseline_suffix}"\nBASELINE_DEPLOYMENT_NAME = f"baseline-embedding-{baseline_suffix}"\n\nprint("Creating baseline deployment config...")\nbaseline_config = client.inference.deployment_configs.create(\n workspace="default",\n name=BASELINE_DEPLOYMENT_CONFIG,\n nim_deployment=NIMDeploymentParam(\n image_name=NIM_IMAGE,\n image_tag=NIM_TAG,\n gpu=1,\n image_pull_secret=NGC_SECRET_NAME,\n )\n)\n\nprint("Deploying base model...")\nbaseline_deployment = client.inference.deployments.create(\n workspace="default",\n name=BASELINE_DEPLOYMENT_NAME,\n config=baseline_config.name\n)\nprint(f"Baseline deployment: {baseline_deployment.name}")\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Wait for baseline deployment\nTIMEOUT_MINUTES = 15\nstart_time = time.time()\n\nprint(f\"Waiting for baseline deployment...\")\nwhile True:\n status = client.inference.deployments.retrieve(\n name=BASELINE_DEPLOYMENT_NAME,\n workspace=\"default\"\n )\n \n elapsed = time.time() - start_time\n elapsed_str = f\"{int(elapsed//60)}m {int(elapsed%60)}s\"\n \n clear_output(wait=True)\n print(f\"Baseline deployment: {status.status} | {elapsed_str}\")\n \n if status.status == \"READY\":\n print(\"Baseline model ready!\")\n if not client.models.wait_for_gateway(BASELINE_DEPLOYMENT_NAME, workspace=\"default\", timeout=60):\n raise RuntimeError(\"Inference gateway did not become ready\")\n break\n if status.status in (\"FAILED\", \"ERROR\", \"TERMINATED\", \"LOST\"):\n raise RuntimeError(f\"Baseline deployment failed: {status.status}\")\n if elapsed > TIMEOUT_MINUTES * 60:\n raise TimeoutError(\"Baseline deployment timeout\")\n \n time.sleep(10)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Wait for baseline deployment\nTIMEOUT_MINUTES = 15\nstart_time = time.time()\n\nprint(f"Waiting for baseline deployment...")\nwhile True:\n status = client.inference.deployments.retrieve(\n name=BASELINE_DEPLOYMENT_NAME,\n workspace="default"\n )\n \n elapsed = time.time() - start_time\n elapsed_str = f"{int(elapsed//60)}m {int(elapsed%60)}s"\n \n clear_output(wait=True)\n print(f"Baseline deployment: {status.status} | {elapsed_str}")\n \n if status.status == "READY":\n print("Baseline model ready!")\n if not client.models.wait_for_gateway(BASELINE_DEPLOYMENT_NAME, workspace="default", timeout=60):\n raise RuntimeError("Inference gateway did not become ready")\n break\n if status.status in ("FAILED", "ERROR", "TERMINATED", "LOST"):\n raise RuntimeError(f"Baseline deployment failed: {status.status}")\n if elapsed > TIMEOUT_MINUTES * 60:\n raise TimeoutError("Baseline deployment timeout")\n \n time.sleep(10)\n" + }, + { + "type": "code", + "source": "# Run baseline ranking with the base model\nBASE_MODEL_ID = \"nvidia/llama-nemotron-embed-1b-v2\"\n\n# Get query embedding\nquery_response = client.inference.gateway.provider.post(\n \"v1/embeddings\",\n name=BASELINE_DEPLOYMENT_NAME,\n workspace=\"default\",\n body={\n \"model\": BASE_MODEL_ID,\n \"input\": [DEMO_QUERY],\n \"input_type\": \"query\"\n }\n)\nbase_query_emb = query_response[\"data\"][0][\"embedding\"]\n\n# Get document embeddings\ndoc_response = client.inference.gateway.provider.post(\n \"v1/embeddings\",\n name=BASELINE_DEPLOYMENT_NAME,\n workspace=\"default\",\n body={\n \"model\": BASE_MODEL_ID,\n \"input\": DEMO_DOCS,\n \"input_type\": \"passage\"\n }\n)\nbase_doc_embs = [d[\"embedding\"] for d in doc_response[\"data\"]]\n\n# Calculate similarities and rank\nscores = [(i, cosine_similarity(base_query_emb, base_doc_embs[i])) for i in range(len(DEMO_DOCS))]\nBASELINE_RANKING = sorted(scores, key=lambda x: -x[1])\n\n# Display baseline results\nprint(f\"Query: \\\"{DEMO_QUERY}\\\"\\n\")\nprint(\"Base Model Ranking:\")\nprint(\"-\" * 55)\nfor rank, (idx, score) in enumerate(BASELINE_RANKING, 1):\n marker = \" <-- relevant\" if idx in DEMO_RELEVANT else \"\"\n print(f\" #{rank} [{score:.3f}] {DEMO_LABELS[idx]}{marker}\")", + "language": "python", + "source_html": "# Run baseline ranking with the base model\nBASE_MODEL_ID = "nvidia/llama-nemotron-embed-1b-v2"\n\n# Get query embedding\nquery_response = client.inference.gateway.provider.post(\n "v1/embeddings",\n name=BASELINE_DEPLOYMENT_NAME,\n workspace="default",\n body={\n "model": BASE_MODEL_ID,\n "input": [DEMO_QUERY],\n "input_type": "query"\n }\n)\nbase_query_emb = query_response["data"][0]["embedding"]\n\n# Get document embeddings\ndoc_response = client.inference.gateway.provider.post(\n "v1/embeddings",\n name=BASELINE_DEPLOYMENT_NAME,\n workspace="default",\n body={\n "model": BASE_MODEL_ID,\n "input": DEMO_DOCS,\n "input_type": "passage"\n }\n)\nbase_doc_embs = [d["embedding"] for d in doc_response["data"]]\n\n# Calculate similarities and rank\nscores = [(i, cosine_similarity(base_query_emb, base_doc_embs[i])) for i in range(len(DEMO_DOCS))]\nBASELINE_RANKING = sorted(scores, key=lambda x: -x[1])\n\n# Display baseline results\nprint(f"Query: \\"{DEMO_QUERY}\\"\\n")\nprint("Base Model Ranking:")\nprint("-" * 55)\nfor rank, (idx, score) in enumerate(BASELINE_RANKING, 1):\n marker = " <-- relevant" if idx in DEMO_RELEVANT else ""\n print(f" #{rank} [{score:.3f}] {DEMO_LABELS[idx]}{marker}")\n" + }, + { + "type": "code", + "source": "# Delete baseline deployment to free GPU for training\nprint(\"Deleting baseline deployment to free GPU...\")\nclient.inference.deployments.delete(name=BASELINE_DEPLOYMENT_NAME, workspace=\"default\")\n\n# Wait for deployment to be fully deleted before deleting config\nprint(\"Waiting for deployment deletion...\")\nwhile True:\n try:\n status = client.inference.deployments.retrieve(name=BASELINE_DEPLOYMENT_NAME, workspace=\"default\")\n if status.status == \"DELETED\":\n break\n print(f\" Status: {status.status}\")\n time.sleep(5)\n except Exception:\n # Deployment no longer exists\n break\n\n# Now safe to delete the config\nclient.inference.deployment_configs.delete(name=BASELINE_DEPLOYMENT_CONFIG, workspace=\"default\")\nprint(\"GPU freed. Proceed to fine-tune and improve these rankings.\")", + "language": "python", + "source_html": "# Delete baseline deployment to free GPU for training\nprint("Deleting baseline deployment to free GPU...")\nclient.inference.deployments.delete(name=BASELINE_DEPLOYMENT_NAME, workspace="default")\n\n# Wait for deployment to be fully deleted before deleting config\nprint("Waiting for deployment deletion...")\nwhile True:\n try:\n status = client.inference.deployments.retrieve(name=BASELINE_DEPLOYMENT_NAME, workspace="default")\n if status.status == "DELETED":\n break\n print(f" Status: {status.status}")\n time.sleep(5)\n except Exception:\n # Deployment no longer exists\n break\n\n# Now safe to delete the config\nclient.inference.deployment_configs.delete(name=BASELINE_DEPLOYMENT_CONFIG, workspace="default")\nprint("GPU freed. Proceed to fine-tune and improve these rankings.")\n" + }, + { + "type": "markdown", + "source": "### 3. Prepare Dataset\n\nUse the [SPECTER dataset](https://huggingface.co/datasets/embedding-data/SPECTER) from HuggingFace, a collection of scientific paper triplets where papers that cite each other are considered related.\n\n**Dataset structure:**\n- ~684K scientific paper triplets (this tutorial uses 10%)\n- Each triplet: (query paper, positive/related paper, negative/unrelated paper)\n- Papers that cite each other are marked as \"related\"\n\nIn this tutorial the following dataset directory structure will be used:\n```\nembedding-dataset\n`-- training.jsonl\n`-- validation.jsonl\n```", + "source_html": "

3. Prepare Dataset

\n

Use the SPECTER dataset from HuggingFace, a collection of scientific paper triplets where papers that cite each other are considered related.

\n

Dataset structure:

\n
    \n
  • ~684K scientific paper triplets (this tutorial uses 10%)
    • \n
  • Each triplet: (query paper, positive/related paper, negative/unrelated paper)
    • \n
  • Papers that cite each other are marked as "related"
    • \n
\n

In this tutorial the following dataset directory structure will be used:

\n
embedding-dataset\n`-- training.jsonl\n`-- validation.jsonl\n
\n" + }, + { + "type": "markdown", + "source": "### 4. Download and Format SPECTER Dataset\n\nThe SPECTER dataset requires conversion to the triplet format required for fine-tuning.", + "source_html": "

4. Download and Format SPECTER Dataset

\n

The SPECTER dataset requires conversion to the triplet format required for fine-tuning.

\n" + }, + { + "type": "code", + "source": "from pathlib import Path\nfrom datasets import load_dataset\nimport json\n\n# HuggingFace token for dataset access\nHF_TOKEN = os.environ.get(\"HF_TOKEN\")\nif not HF_TOKEN:\n raise ValueError(\"HF_TOKEN environment variable is required. Get one at https://huggingface.co/settings/tokens\")\nos.environ[\"HF_TOKEN\"] = HF_TOKEN\n\n# Configuration\nDATASET_SIZE = 3000 # Number of triplets (increase for better results, max ~684K)\nVALIDATION_SPLIT = 0.05 # 5% held out for validation\nSEED = 42\nDATASET_PATH = Path(\"embedding-dataset\").absolute()\n\n# Create directory\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Download SPECTER dataset\nprint(\"Downloading SPECTER dataset...\")\ndata = load_dataset(\"embedding-data/SPECTER\")[\"train\"].shuffle(seed=SEED).select(range(DATASET_SIZE))\n\n# Split into train/validation\nprint(\"Splitting into train/validation...\")\nsplits = data.train_test_split(test_size=VALIDATION_SPLIT, seed=SEED)\ntrain_data = splits[\"train\"]\nvalidation_data = splits[\"test\"]\n\n# Convert to triplet JSONL format\nprint(\"Saving to JSONL...\")\nfor name, dataset in [(\"training\", train_data), (\"validation\", validation_data)]:\n with open(f\"{DATASET_PATH}/{name}.jsonl\", \"w\") as f:\n for row in dataset:\n # SPECTER format: row['set'] = [query, positive, negative]\n triplet = {\n \"query\": row[\"set\"][0],\n \"pos_doc\": row[\"set\"][1],\n \"neg_doc\": [row[\"set\"][2]] # List of negative documents\n }\n f.write(json.dumps(triplet) + \"\\n\")\n\nprint(f\"\\nPrepared {len(train_data):,} training, {len(validation_data):,} validation samples\")\nprint(f\"\\nExample triplet:\")\nprint(f\" Query: {train_data[0]['set'][0][:100]}...\")\nprint(f\" Positive: {train_data[0]['set'][1][:100]}...\")\nprint(f\" Negative: {train_data[0]['set'][2][:100]}...\")", + "language": "python", + "source_html": "from pathlib import Path\nfrom datasets import load_dataset\nimport json\n\n# HuggingFace token for dataset access\nHF_TOKEN = os.environ.get("HF_TOKEN")\nif not HF_TOKEN:\n raise ValueError("HF_TOKEN environment variable is required. Get one at https://huggingface.co/settings/tokens")\nos.environ["HF_TOKEN"] = HF_TOKEN\n\n# Configuration\nDATASET_SIZE = 3000 # Number of triplets (increase for better results, max ~684K)\nVALIDATION_SPLIT = 0.05 # 5% held out for validation\nSEED = 42\nDATASET_PATH = Path("embedding-dataset").absolute()\n\n# Create directory\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Download SPECTER dataset\nprint("Downloading SPECTER dataset...")\ndata = load_dataset("embedding-data/SPECTER")["train"].shuffle(seed=SEED).select(range(DATASET_SIZE))\n\n# Split into train/validation\nprint("Splitting into train/validation...")\nsplits = data.train_test_split(test_size=VALIDATION_SPLIT, seed=SEED)\ntrain_data = splits["train"]\nvalidation_data = splits["test"]\n\n# Convert to triplet JSONL format\nprint("Saving to JSONL...")\nfor name, dataset in [("training", train_data), ("validation", validation_data)]:\n with open(f"{DATASET_PATH}/{name}.jsonl", "w") as f:\n for row in dataset:\n # SPECTER format: row['set'] = [query, positive, negative]\n triplet = {\n "query": row["set"][0],\n "pos_doc": row["set"][1],\n "neg_doc": [row["set"][2]] # List of negative documents\n }\n f.write(json.dumps(triplet) + "\\n")\n\nprint(f"\\nPrepared {len(train_data):,} training, {len(validation_data):,} validation samples")\nprint(f"\\nExample triplet:")\nprint(f" Query: {train_data[0]['set'][0][:100]}...")\nprint(f" Positive: {train_data[0]['set'][1][:100]}...")\nprint(f" Negative: {train_data[0]['set'][2][:100]}...")\n" + }, + { + "type": "markdown", + "source": "### 5. Create Dataset FileSet and Upload Training Data", + "source_html": "

5. Create Dataset FileSet and Upload Training Data

\n" + }, + { + "type": "code", + "source": "# Create fileset to store embedding training data\nDATASET_NAME = \"embedding-dataset\"\n\ntry:\n client.files.filesets.create(\n workspace=\"default\",\n name=DATASET_NAME,\n description=\"SPECTER embedding training data (scientific paper triplets)\"\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\n# Upload training data files\nclient.files.upload(\n local_path=DATASET_PATH,\n remote_path=\"\",\n fileset=DATASET_NAME,\n workspace=\"default\"\n)\n\n# Validate upload\nprint(\"\\nUploaded files:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace=\"default\").data], indent=2))", + "language": "python", + "source_html": "# Create fileset to store embedding training data\nDATASET_NAME = "embedding-dataset"\n\ntry:\n client.files.filesets.create(\n workspace="default",\n name=DATASET_NAME,\n description="SPECTER embedding training data (scientific paper triplets)"\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\n# Upload training data files\nclient.files.upload(\n local_path=DATASET_PATH,\n remote_path="",\n fileset=DATASET_NAME,\n workspace="default"\n)\n\n# Validate upload\nprint("\\nUploaded files:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace="default").data], indent=2))\n" + }, + { + "type": "markdown", + "source": "### 6. Secrets Setup\n\nConfigure authentication for accessing base models:\n\n- **NGC models** (`ngc://` URIs): Requires NGC API key\n- **HuggingFace models** (`hf://` URIs): Requires HF token for gated/private models\n\nGet your credentials:\n- [NGC API Key](https://ngc.nvidia.com/) (Setup → Generate API Key)\n- [HuggingFace Token](https://huggingface.co/settings/tokens) (Create token with Read access)\n\n---\n\n#### Quick Setup Example\n\nThis tutorial fine-tunes [nvidia/llama-3.2-nv-embedqa-1b-v2](https://huggingface.co/nvidia/llama-3.2-nv-embedqa-1b-v2), NVIDIA embedding model optimized for question-answering and retrieval tasks.", + "source_html": "

6. Secrets Setup

\n

Configure authentication for accessing base models:

\n
    \n
  • NGC models (ngc:// URIs): Requires NGC API key
    • \n
  • HuggingFace models (hf:// URIs): Requires HF token for gated/private models
    • \n
\n

Get your credentials:

\n\n
\n

Quick Setup Example

\n

This tutorial fine-tunes nvidia/llama-3.2-nv-embedqa-1b-v2, NVIDIA embedding model optimized for question-answering and retrieval tasks.

\n" + }, + { + "type": "code", + "source": "# Create secrets for model access\n# Note: NGC_API_KEY secret was already created in the baseline step (Step 2)\nHF_TOKEN = os.getenv(\"HF_TOKEN\")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f\"{label} is not set\")\n try:\n secret = client.secrets.create(\n name=name,\n workspace=\"default\",\n value=value,\n )\n print(f\"Created secret: {name}\")\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n return client.secrets.retrieve(name=name, workspace=\"default\")\n\n\n# Create HuggingFace token secret (for downloading model from HF during training)\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")\nprint(f\"HF_TOKEN secret: {hf_secret.name}\")\n\n# NGC secret was already created in baseline step\nprint(f\"NGC_API_KEY secret: {NGC_SECRET_NAME} (created in Step 2)\")", + "language": "python", + "source_html": "# Create secrets for model access\n# Note: NGC_API_KEY secret was already created in the baseline step (Step 2)\nHF_TOKEN = os.getenv("HF_TOKEN")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f"{label} is not set")\n try:\n secret = client.secrets.create(\n name=name,\n workspace="default",\n value=value,\n )\n print(f"Created secret: {name}")\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n return client.secrets.retrieve(name=name, workspace="default")\n\n\n# Create HuggingFace token secret (for downloading model from HF during training)\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\nprint(f"HF_TOKEN secret: {hf_secret.name}")\n\n# NGC secret was already created in baseline step\nprint(f"NGC_API_KEY secret: {NGC_SECRET_NAME} (created in Step 2)")\n" + }, + { + "type": "markdown", + "source": "### 7. Create Base Model FileSet and Model Entity\n\nCreate a fileset pointing to the [nvidia/llama-3.2-nv-embedqa-1b-v2](https://huggingface.co/nvidia/llama-3.2-nv-embedqa-1b-v2) embedding model from HuggingFace, then create a Model Entity that references this fileset. Model downloading will take place at training time.\n\n---\n*Note*: Either `MODEL_NAME` or `HF_REPO_ID` below must contain the substring embed to indicate that this is an embedding model.", + "source_html": "

7. Create Base Model FileSet and Model Entity

\n

Create a fileset pointing to the nvidia/llama-3.2-nv-embedqa-1b-v2 embedding model from HuggingFace, then create a Model Entity that references this fileset. Model downloading will take place at training time.

\n
\n

Note: Either MODEL_NAME or HF_REPO_ID below must contain the substring embed to indicate that this is an embedding model.

\n" + }, + { + "type": "code", + "source": "import time\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = \"nvidia/llama-nemotron-embed-1b-v2\"\nMODEL_NAME = \"nv-nemotron-embed-1b-base\"\n\n# Ensure you have a HuggingFace token secret created\ntry:\n base_model_fs = client.files.filesets.create(\n workspace=\"default\",\n name=MODEL_NAME,\n description=\"NVIDIA Llama 3.2 NV EmbedQA 1B v2 embedding model\",\n storage=HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type=\"model\",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\nexcept ConflictError as e:\n print(f\"Base model fileset already exists. Skipping creation.\")\n base_model_fs = client.files.filesets.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\n# Create Model Entity referencing the FileSet\ntry:\n base_model = client.models.create(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n trust_remote_code=True,\n )\n print(f\"Created Model Entity: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model already exists. Updating fileset if different.\")\n base_model = client.models.update(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n trust_remote_code=True,\n )\n\nprint(f\"\\nBase model fileset: fileset://default/{base_model.name}\")\nprint(\"\\nBase model files:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace=\"default\").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint(\"\\nWaiting for ModelSpec to be populated...\")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f\"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds\")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\nprint(f\"ModelSpec populated: {base_model.spec}\")", + "language": "python", + "source_html": "import time\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = "nvidia/llama-nemotron-embed-1b-v2"\nMODEL_NAME = "nv-nemotron-embed-1b-base"\n\n# Ensure you have a HuggingFace token secret created\ntry:\n base_model_fs = client.files.filesets.create(\n workspace="default",\n name=MODEL_NAME,\n description="NVIDIA Llama 3.2 NV EmbedQA 1B v2 embedding model",\n storage=HuggingfaceStorageConfigParam(\n type="huggingface",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type="model",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\nexcept ConflictError as e:\n print(f"Base model fileset already exists. Skipping creation.")\n base_model_fs = client.files.filesets.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\n# Create Model Entity referencing the FileSet\ntry:\n base_model = client.models.create(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n trust_remote_code=True,\n )\n print(f"Created Model Entity: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model already exists. Updating fileset if different.")\n base_model = client.models.update(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n trust_remote_code=True,\n )\n\nprint(f"\\nBase model fileset: fileset://default/{base_model.name}")\nprint("\\nBase model files:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace="default").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint("\\nWaiting for ModelSpec to be populated...")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\nprint(f"ModelSpec populated: {base_model.spec}")\n" + }, + { + "type": "markdown", + "source": "### 8. Create Embedding Fine-tuning Job\n\nCreate a customization job to fine-tune the embedding model using contrastive learning on the SPECTER dataset.\n\n**Key hyperparameters for embedding fine-tuning:**\n- **`training_type`**: `sft` (supervised fine-tuning)\n- **Full fine-tuning**: No `peft` config needed (omit for all-weights training)\n- **`learning_rate`**: Lower values (1e-6 to 5e-6) work well for embedding models\n- **`batch_size`**: Larger batches improve contrastive learning (128-256 recommended)\n\n**NOTE:**\n\nNeMo Platform does not support unmerged LoRA adapters for embedding models because the embedding NIM requires ONNX format, which cannot represent standalone adapters. This notebook creates a job with all-weights finetuning but you can also run LoRA with `merge=True`, which trains a LoRA adapter and then merges it back into the base model after training. The final output is a standard full-weight checkpoint, identical in format to an all-weights fine-tuned model, but LoRA training is faster, uses less memory, and is more lenient in hyperparameter tuning.\n\nTo do that update the job request like so\n\n```\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=EPOCHS,\n batch_size=BATCH_SIZE,\n learning_rate=LEARNING_RATE,\n max_seq_length=MAX_SEQ_LENGTH,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n peft=LoRaParamsParam(\n type= \"lora\",\n merge=True\n )\n )\n )\n)\n```", + "source_html": "

8. Create Embedding Fine-tuning Job

\n

Create a customization job to fine-tune the embedding model using contrastive learning on the SPECTER dataset.

\n

Key hyperparameters for embedding fine-tuning:

\n
    \n
  • training_type: sft (supervised fine-tuning)
    • \n
  • Full fine-tuning: No peft config needed (omit for all-weights training)
    • \n
  • learning_rate: Lower values (1e-6 to 5e-6) work well for embedding models
    • \n
  • batch_size: Larger batches improve contrastive learning (128-256 recommended)
    • \n
\n

NOTE:

\n

NeMo Platform does not support unmerged LoRA adapters for embedding models because the embedding NIM requires ONNX format, which cannot represent standalone adapters. This notebook creates a job with all-weights finetuning but you can also run LoRA with merge=True, which trains a LoRA adapter and then merges it back into the base model after training. The final output is a standard full-weight checkpoint, identical in format to an all-weights fine-tuned model, but LoRA training is faster, uses less memory, and is more lenient in hyperparameter tuning.

\n

To do that update the job request like so

\n
job = client.customization.jobs.create(\n    name=JOB_NAME,\n    workspace="default",\n    spec=CustomizationJobInputParam(\n        model=f"default/{base_model.name}",\n        dataset=f"fileset://default/{DATASET_NAME}",\n        training=SftTrainingParam(\n            type="sft",\n            epochs=EPOCHS,\n            batch_size=BATCH_SIZE,\n            learning_rate=LEARNING_RATE,\n            max_seq_length=MAX_SEQ_LENGTH,\n            micro_batch_size=1,\n            parallelism=ParallelismParamsParam(\n                num_gpus_per_node=1,\n                num_nodes=1,\n                tensor_parallel_size=1,\n                pipeline_parallel_size=1,\n            ),\n            peft=LoRaParamsParam(\n                type= "lora",\n                merge=True\n            )\n        )\n    )\n)\n
\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam\n)\n\njob_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f\"embedding-finetune-job-{job_suffix}\"\n\n# Hyperparameters optimized for embedding fine-tuning\nEPOCHS = 1\nBATCH_SIZE = 128 # Larger batches help contrastive learning\nLEARNING_RATE = 5e-6 # Lower LR for embedding models\nMAX_SEQ_LENGTH = 512 # Typical for embedding models\n\n# Note: The 'name' field must contain 'embed' for the customizer to detect this as an embedding model\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=EPOCHS,\n batch_size=BATCH_SIZE,\n learning_rate=LEARNING_RATE,\n max_seq_length=MAX_SEQ_LENGTH,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n )\n)\n\nprint(f\"Job ID: {job.name}\")\nprint(f\"Output model: {job.spec.output.name}\")", + "language": "python", + "source_html": "from nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam\n)\n\njob_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f"embedding-finetune-job-{job_suffix}"\n\n# Hyperparameters optimized for embedding fine-tuning\nEPOCHS = 1\nBATCH_SIZE = 128 # Larger batches help contrastive learning\nLEARNING_RATE = 5e-6 # Lower LR for embedding models\nMAX_SEQ_LENGTH = 512 # Typical for embedding models\n\n# Note: The 'name' field must contain 'embed' for the customizer to detect this as an embedding model\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace="default",\n spec=CustomizationJobInputParam(\n model=f"default/{base_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=EPOCHS,\n batch_size=BATCH_SIZE,\n learning_rate=LEARNING_RATE,\n max_seq_length=MAX_SEQ_LENGTH,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n )\n)\n\nprint(f"Job ID: {job.name}")\nprint(f"Output model: {job.spec.output.name}")\n" + }, + { + "type": "markdown", + "source": "### 9. Track Training Progress", + "source_html": "

9. Track Training Progress

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Poll job status every 10 seconds until completed\nwhile True:\n status = client.customization.jobs.get_status(\n name=job.name,\n workspace=\"default\"\n )\n \n clear_output(wait=True)\n print(f\"Job Status: {status.model_dump_json(indent=2)}\")\n\n # Extract training progress from nested steps structure\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n\n for job_step in status.steps or []:\n if job_step.name == \"customization-training-job\":\n for task in job_step.tasks or []:\n task_details = task.status_details or {}\n step = task_details.get(\"step\")\n max_steps = task_details.get(\"max_steps\")\n training_phase = task_details.get(\"phase\")\n break\n break\n\n if step is not None and max_steps is not None:\n progress_pct = (step / max_steps) * 100\n print(f\"Training Progress: Step {step}/{max_steps} ({progress_pct:.1f}%)\")\n if training_phase:\n print(f\"Training Phase: {training_phase}\")\n else:\n print(\"Training step not started yet or progress info not available\")\n \n # Exit loop when job is completed (or failed/cancelled)\n if status.status in (\"completed\", \"failed\", \"cancelled\", \"error\"):\n print(f\"\\nJob finished with status: {status.status}\")\n break\n \n time.sleep(10)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Poll job status every 10 seconds until completed\nwhile True:\n status = client.customization.jobs.get_status(\n name=job.name,\n workspace="default"\n )\n \n clear_output(wait=True)\n print(f"Job Status: {status.model_dump_json(indent=2)}")\n\n # Extract training progress from nested steps structure\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n\n for job_step in status.steps or []:\n if job_step.name == "customization-training-job":\n for task in job_step.tasks or []:\n task_details = task.status_details or {}\n step = task_details.get("step")\n max_steps = task_details.get("max_steps")\n training_phase = task_details.get("phase")\n break\n break\n\n if step is not None and max_steps is not None:\n progress_pct = (step / max_steps) * 100\n print(f"Training Progress: Step {step}/{max_steps} ({progress_pct:.1f}%)")\n if training_phase:\n print(f"Training Phase: {training_phase}")\n else:\n print("Training step not started yet or progress info not available")\n \n # Exit loop when job is completed (or failed/cancelled)\n if status.status in ("completed", "failed", "cancelled", "error"):\n print(f"\\nJob finished with status: {status.status}")\n break\n \n time.sleep(10)\n" + }, + { + "type": "markdown", + "source": "**Interpreting Embedding Training Metrics:**\n\nEmbedding models use contrastive loss—lower values indicate better separation between similar and dissimilar pairs:\n\n| Scenario | Interpretation | Action |\n|----------|----------------|--------|\n| **Loss steadily decreasing** | Model learning semantic relationships | Continue training |\n| **Loss plateaus early** | May need more data or epochs | Increase dataset/epochs |\n| **Loss spikes** | Training instability | Lower learning rate |\n| **Validation loss increasing** | Overfitting | Reduce epochs, add data |", + "source_html": "

Interpreting Embedding Training Metrics:

\n

Embedding models use contrastive loss—lower values indicate better separation between similar and dissimilar pairs:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ScenarioInterpretationAction
Loss steadily decreasingModel learning semantic relationshipsContinue training
Loss plateaus earlyMay need more data or epochsIncrease dataset/epochs
Loss spikesTraining instabilityLower learning rate
Validation loss increasingOverfittingReduce epochs, add data
\n" + }, + { + "type": "markdown", + "source": "### 10. Deploy Fine-Tuned Embedding Model\n\nAfter training completes, deploy the embedding model using the Deployment Management Service:", + "source_html": "

10. Deploy Fine-Tuned Embedding Model

\n

After training completes, deploy the embedding model using the Deployment Management Service:

\n" + }, + { + "type": "code", + "source": "# Validate model entity exists\nmodel_entity = client.models.retrieve(workspace=\"default\", name=job.spec.output.name)\nprint(model_entity.model_dump_json(indent=2))", + "language": "python", + "source_html": "# Validate model entity exists\nmodel_entity = client.models.retrieve(workspace="default", name=job.spec.output.name)\nprint(model_entity.model_dump_json(indent=2))\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# Create deployment config for embedding model\ndeploy_suffix = uuid.uuid4().hex[:4]\nDEPLOYMENT_CONFIG_NAME = f\"embedding-model-deployment-cfg-{deploy_suffix}\"\nDEPLOYMENT_NAME = f\"embedding-model-deployment-{deploy_suffix}\"\n\n# Embedding NIM image\nNIM_IMAGE = \"nvcr.io/nim/nvidia/llama-nemotron-embed-1b-v2\"\nNIM_TAG = \"1.13.0\" # Update if using newer NIM release\n\ndeployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=DEPLOYMENT_CONFIG_NAME,\n nim_deployment=NIMDeploymentParam(\n image_name=NIM_IMAGE,\n image_tag=NIM_TAG,\n gpu=1,\n model_name=job.spec.output.name,\n model_namespace=\"default\",\n )\n)\n\n# Deploy model\ndeployment = client.inference.deployments.create(\n workspace=\"default\",\n name=DEPLOYMENT_NAME,\n config=deployment_config.name\n)\n\nprint(f\"Deployment name: {deployment.name}\")\nprint(f\"Deployment status: {client.inference.deployments.retrieve(name=deployment.name, workspace='default').status}\")", + "language": "python", + "source_html": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# Create deployment config for embedding model\ndeploy_suffix = uuid.uuid4().hex[:4]\nDEPLOYMENT_CONFIG_NAME = f"embedding-model-deployment-cfg-{deploy_suffix}"\nDEPLOYMENT_NAME = f"embedding-model-deployment-{deploy_suffix}"\n\n# Embedding NIM image\nNIM_IMAGE = "nvcr.io/nim/nvidia/llama-nemotron-embed-1b-v2"\nNIM_TAG = "1.13.0" # Update if using newer NIM release\n\ndeployment_config = client.inference.deployment_configs.create(\n workspace="default",\n name=DEPLOYMENT_CONFIG_NAME,\n nim_deployment=NIMDeploymentParam(\n image_name=NIM_IMAGE,\n image_tag=NIM_TAG,\n gpu=1,\n model_name=job.spec.output.name,\n model_namespace="default",\n )\n)\n\n# Deploy model\ndeployment = client.inference.deployments.create(\n workspace="default",\n name=DEPLOYMENT_NAME,\n config=deployment_config.name\n)\n\nprint(f"Deployment name: {deployment.name}")\nprint(f"Deployment status: {client.inference.deployments.retrieve(name=deployment.name, workspace='default').status}")\n" + }, + { + "type": "markdown", + "source": "#### Track Deployment Status", + "source_html": "

Track Deployment Status

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Poll deployment status every 15 seconds until ready\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f\"Monitoring deployment '{deployment.name}'...\")\nprint(f\"Timeout: {TIMEOUT_MINUTES} minutes\\n\")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace=\"default\"\n )\n \n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n \n clear_output(wait=True)\n print(f\"Deployment: {deployment.name}\")\n print(f\"Status: {deployment_status.status}\")\n print(f\"Elapsed time: {elapsed_min}m {elapsed_sec}s\")\n \n # Check if deployment is ready\n if deployment_status.status == \"READY\":\n print(\"\\nDeployment is ready!\")\n if not client.models.wait_for_gateway(deployment.name, workspace=\"default\", timeout=60):\n raise RuntimeError(\"Inference gateway did not become ready\")\n break\n \n # Check for failure states\n if deployment_status.status in (\"FAILED\", \"ERROR\", \"TERMINATED\", \"LOST\"):\n raise RuntimeError(f\"Deployment failed with status: {deployment_status.status}\")\n \n # Check timeout\n if elapsed > timeout_seconds:\n raise TimeoutError(f\"Deployment timeout after {TIMEOUT_MINUTES} minutes\")\n \n time.sleep(15)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Poll deployment status every 15 seconds until ready\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f"Monitoring deployment '{deployment.name}'...")\nprint(f"Timeout: {TIMEOUT_MINUTES} minutes\\n")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace="default"\n )\n \n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n \n clear_output(wait=True)\n print(f"Deployment: {deployment.name}")\n print(f"Status: {deployment_status.status}")\n print(f"Elapsed time: {elapsed_min}m {elapsed_sec}s")\n \n # Check if deployment is ready\n if deployment_status.status == "READY":\n print("\\nDeployment is ready!")\n if not client.models.wait_for_gateway(deployment.name, workspace="default", timeout=60):\n raise RuntimeError("Inference gateway did not become ready")\n break\n \n # Check for failure states\n if deployment_status.status in ("FAILED", "ERROR", "TERMINATED", "LOST"):\n raise RuntimeError(f"Deployment failed with status: {deployment_status.status}")\n \n # Check timeout\n if elapsed > timeout_seconds:\n raise TimeoutError(f"Deployment timeout after {TIMEOUT_MINUTES} minutes")\n \n time.sleep(15)\n" + }, + { + "type": "markdown", + "source": "### 11. View the Improvement\n\nRun the same query against the fine-tuned model and compare to the baseline from earlier.", + "source_html": "

11. View the Improvement

\n

Run the same query against the fine-tuned model and compare to the baseline from earlier.

\n" + }, + { + "type": "code", + "source": "# Compare: same query, base model vs fine-tuned\n# Using the same DEMO_QUERY and DEMO_DOCS from the baseline test\nMODEL_ID = f\"default/{job.spec.output.name}\"\n\n# Get query embedding from fine-tuned model\nquery_response = client.inference.gateway.provider.post(\n \"v1/embeddings\",\n name=deployment.name,\n workspace=\"default\",\n body={\n \"model\": MODEL_ID,\n \"input\": [DEMO_QUERY],\n \"input_type\": \"query\"\n }\n)\nquery_embedding = query_response[\"data\"][0][\"embedding\"]\n\n# Get document embeddings from fine-tuned model\ndoc_response = client.inference.gateway.provider.post(\n \"v1/embeddings\",\n name=deployment.name,\n workspace=\"default\",\n body={\n \"model\": MODEL_ID,\n \"input\": DEMO_DOCS,\n \"input_type\": \"passage\"\n }\n)\ndoc_embeddings = [d[\"embedding\"] for d in doc_response[\"data\"]]\n\n# Calculate similarities and rank\nscores = [(i, cosine_similarity(query_embedding, doc_embeddings[i])) for i in range(len(DEMO_DOCS))]\nFINETUNED_RANKING = sorted(scores, key=lambda x: -x[1])\n\n# Display side-by-side comparison\nprint(f\"Query: \\\"{DEMO_QUERY}\\\"\\n\")\nprint(f\"{'Rank':<6} {'Base Model':<30} {'Fine-tuned Model':<30}\")\nprint(\"-\" * 66)\n\nfor rank in range(len(DEMO_DOCS)):\n b_idx, b_score = BASELINE_RANKING[rank]\n f_idx, f_score = FINETUNED_RANKING[rank]\n \n b_label = f\"{DEMO_LABELS[b_idx]} [{b_score:.3f}]\" + (\" *\" if b_idx in DEMO_RELEVANT else \"\")\n f_label = f\"{DEMO_LABELS[f_idx]} [{f_score:.3f}]\" + (\" *\" if f_idx in DEMO_RELEVANT else \"\")\n \n print(f\"#{rank+1:<5} {b_label:<30} {f_label:<30}\")\n\nprint(\"\\n* = relevant paper\")\nprint(\"\\nThe fine-tuned model pushes 'Random Forest' down and ranks CRF papers higher.\")", + "language": "python", + "source_html": "# Compare: same query, base model vs fine-tuned\n# Using the same DEMO_QUERY and DEMO_DOCS from the baseline test\nMODEL_ID = f"default/{job.spec.output.name}"\n\n# Get query embedding from fine-tuned model\nquery_response = client.inference.gateway.provider.post(\n "v1/embeddings",\n name=deployment.name,\n workspace="default",\n body={\n "model": MODEL_ID,\n "input": [DEMO_QUERY],\n "input_type": "query"\n }\n)\nquery_embedding = query_response["data"][0]["embedding"]\n\n# Get document embeddings from fine-tuned model\ndoc_response = client.inference.gateway.provider.post(\n "v1/embeddings",\n name=deployment.name,\n workspace="default",\n body={\n "model": MODEL_ID,\n "input": DEMO_DOCS,\n "input_type": "passage"\n }\n)\ndoc_embeddings = [d["embedding"] for d in doc_response["data"]]\n\n# Calculate similarities and rank\nscores = [(i, cosine_similarity(query_embedding, doc_embeddings[i])) for i in range(len(DEMO_DOCS))]\nFINETUNED_RANKING = sorted(scores, key=lambda x: -x[1])\n\n# Display side-by-side comparison\nprint(f"Query: \\"{DEMO_QUERY}\\"\\n")\nprint(f"{'Rank':<6} {'Base Model':<30} {'Fine-tuned Model':<30}")\nprint("-" * 66)\n\nfor rank in range(len(DEMO_DOCS)):\n b_idx, b_score = BASELINE_RANKING[rank]\n f_idx, f_score = FINETUNED_RANKING[rank]\n \n b_label = f"{DEMO_LABELS[b_idx]} [{b_score:.3f}]" + (" *" if b_idx in DEMO_RELEVANT else "")\n f_label = f"{DEMO_LABELS[f_idx]} [{f_score:.3f}]" + (" *" if f_idx in DEMO_RELEVANT else "")\n \n print(f"#{rank+1:<5} {b_label:<30} {f_label:<30}")\n\nprint("\\n* = relevant paper")\nprint("\\nThe fine-tuned model pushes 'Random Forest' down and ranks CRF papers higher.")\n" + }, + { + "type": "markdown", + "source": "### Evaluation Best Practices\n\n**Manual Evaluation** (Recommended)\n- Test with real-world queries from your domain\n- Compare retrieval rankings before and after fine-tuning\n- Check that semantically similar items rank higher than keyword matches\n\n**What to look for:**\n- ✅ Relevant documents consistently rank in top positions\n- ✅ Keyword traps (like \"Random Forest\" vs \"Random Fields\") are handled correctly\n- ✅ Domain-specific terminology is understood\n- ❌ Unrelated documents with matching keywords do not rank high\n\n**Benchmark Evaluation**\n\nFor systematic evaluation, use the NeMo Evaluator service with retrieval benchmarks like SciDocs, BEIR, or MTEB. Refer to the [Evaluator documentation](../../evaluator/index.md) for details.\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n**Embedding-Specific Recommendations:**\n\n| Parameter | Recommended | Notes |\n|-----------|-------------|-------|\n| `learning_rate` | 1e-6 to 5e-6 | Lower than standard SFT |\n| `batch_size` | 128-256 | Larger batches improve contrastive learning |\n| `max_seq_length` | 512 | Typical for embedding models |\n| `epochs` | 1-3 | Start small, increase if needed |\n\n---\n\n## Troubleshooting\n\n**Embeddings do not show improved retrieval:**\n- Verify dataset quality: triplets should have clear positive/negative distinctions\n- Use hard negatives: negatives should share some overlap with the query but not be relevant (easy negatives do not teach the model much)\n- Increase dataset size: 10K+ triplets recommended for meaningful improvement\n- Try more epochs: embedding models often need multiple passes\n- Lower learning rate: embedding models are sensitive to LR\n\n**Training loss not decreasing:**\n- Check triplet format: ensure `neg_doc` is a list even for single negatives\n- Verify hard negative quality: negatives should be challenging but clearly non-relevant\n- Increase batch size: contrastive learning benefits from larger batches\n\n**Deployment fails:**\n- Ensure you use the correct NIM image for embedding models\n- Verify sufficient GPU memory for the model size\n- Check deployment status: `client.inference.deployments.retrieve(name=deployment.name, workspace=\"default\")` and refer to platform logs for debugging\n\n## Next Steps\n\n- [Monitor training metrics](../manage-customization-jobs/get-job-status.md) in detail\n- [Evaluate your model](../../evaluator/index.md) with retrieval benchmarks\n- Integrate the fine-tuned embedding model into your RAG pipeline\n- Scale up training with the full SPECTER dataset (~684K triplets) for better results", + "source_html": "

Evaluation Best Practices

\n

Manual Evaluation (Recommended)

\n
    \n
  • Test with real-world queries from your domain
    • \n
  • Compare retrieval rankings before and after fine-tuning
    • \n
  • Check that semantically similar items rank higher than keyword matches
    • \n
\n

What to look for:

\n
    \n
  • ✅ Relevant documents consistently rank in top positions
    • \n
  • ✅ Keyword traps (like "Random Forest" vs "Random Fields") are handled correctly
    • \n
  • ✅ Domain-specific terminology is understood
    • \n
  • ❌ Unrelated documents with matching keywords do not rank high
    • \n
\n

Benchmark Evaluation

\n

For systematic evaluation, use the NeMo Evaluator service with retrieval benchmarks like SciDocs, BEIR, or MTEB. Refer to the Evaluator documentation for details.

\n
\n

Hyperparameters

\n

For detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.

\n

Embedding-Specific Recommendations:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ParameterRecommendedNotes
learning_rate1e-6 to 5e-6Lower than standard SFT
batch_size128-256Larger batches improve contrastive learning
max_seq_length512Typical for embedding models
epochs1-3Start small, increase if needed
\n
\n

Troubleshooting

\n

Embeddings do not show improved retrieval:

\n
    \n
  • Verify dataset quality: triplets should have clear positive/negative distinctions
    • \n
  • Use hard negatives: negatives should share some overlap with the query but not be relevant (easy negatives do not teach the model much)
    • \n
  • Increase dataset size: 10K+ triplets recommended for meaningful improvement
    • \n
  • Try more epochs: embedding models often need multiple passes
    • \n
  • Lower learning rate: embedding models are sensitive to LR
    • \n
\n

Training loss not decreasing:

\n
    \n
  • Check triplet format: ensure neg_doc is a list even for single negatives
    • \n
  • Verify hard negative quality: negatives should be challenging but clearly non-relevant
    • \n
  • Increase batch size: contrastive learning benefits from larger batches
    • \n
\n

Deployment fails:

\n
    \n
  • Ensure you use the correct NIM image for embedding models
    • \n
  • Verify sufficient GPU memory for the model size
    • \n
  • Check deployment status: client.inference.deployments.retrieve(name=deployment.name, workspace="default") and refer to platform logs for debugging
    • \n
\n

Next Steps

\n
    \n
  • Monitor training metrics in detail
    • \n
  • Evaluate your model with retrieval benchmarks
    • \n
  • Integrate the fine-tuned embedding model into your RAG pipeline
    • \n
  • Scale up training with the full SPECTER dataset (~684K triplets) for better results
    • \n
\n" + } + ] +} \ No newline at end of file diff --git a/docs/fern/components/notebooks/embedding-customization-job.ts b/docs/fern/components/notebooks/embedding-customization-job.ts new file mode 100644 index 00000000000..1b174b2a096 --- /dev/null +++ b/docs/fern/components/notebooks/embedding-customization-job.ts @@ -0,0 +1,185 @@ +/** Auto-generated by ipynb-to-fern-json.py - do not edit */ +export default { cells: [ + { + "type": "markdown", + "source": "\n\n\n# Embedding Model Customization\n\nLearn how to fine-tune an embedding model to improve retrieval accuracy for your specific domain.\n\n## About\n\nEmbedding models convert text into dense vector representations that capture semantic meaning. Fine-tuning these models on your domain data significantly improves retrieval accuracy—in RAG pipelines, this means the LLM receives more relevant context and produces better answers.\n\n**What you will achieve with embedding fine-tuning:**\n\n- 🎯 **Domain specialization:** Adapt general embeddings for legal, medical, scientific, or financial content\n- 📈 **Improved retrieval:** Achieve 6-10% better recall on domain-specific benchmarks\n- 🔍 **Semantic understanding:** Teach the model your domain's vocabulary and relationships\n\n**Recall@5** measures the fraction of relevant documents that appear in the top 5 search results.\n\n**About the baseline:** In retrieval benchmarks like SciDocs, the pretrained model achieves ~0.159 Recall@5. After fine-tuning on scientific paper triplets, you can expect 6-10% improvement (~0.17 Recall@5).\n\n### Dataset Format for Embedding Models\n\nEmbedding models require **triplet format** for contrastive learning:\n\n```json\n{\"query\": \"What is machine learning?\", \"pos_doc\": \"Machine learning is a subset of AI...\", \"neg_doc\": [\"Gardening tips for beginners...\"]}\n```\n\n- **`query`**: The search query or question\n- **`pos_doc`**: A document relevant to the query (positive example)\n- **`neg_doc`**: List of hard negatives—documents that share some overlap with the query but are not actually relevant (negative example)\n\nThe model learns to maximize similarity between query and positive document while minimizing similarity with negative documents.", + "source_html": "\n\n

Embedding Model Customization

\n

Learn how to fine-tune an embedding model to improve retrieval accuracy for your specific domain.

\n

About

\n

Embedding models convert text into dense vector representations that capture semantic meaning. Fine-tuning these models on your domain data significantly improves retrieval accuracy—in RAG pipelines, this means the LLM receives more relevant context and produces better answers.

\n

What you will achieve with embedding fine-tuning:

\n
    \n
  • 🎯 Domain specialization: Adapt general embeddings for legal, medical, scientific, or financial content
    • \n
  • 📈 Improved retrieval: Achieve 6-10% better recall on domain-specific benchmarks
    • \n
  • 🔍 Semantic understanding: Teach the model your domain's vocabulary and relationships
    • \n
\n

Recall@5 measures the fraction of relevant documents that appear in the top 5 search results.

\n

About the baseline: In retrieval benchmarks like SciDocs, the pretrained model achieves ~0.159 Recall@5. After fine-tuning on scientific paper triplets, you can expect 6-10% improvement (~0.17 Recall@5).

\n

Dataset Format for Embedding Models

\n

Embedding models require triplet format for contrastive learning:

\n
{"query": "What is machine learning?", "pos_doc": "Machine learning is a subset of AI...", "neg_doc": ["Gardening tips for beginners..."]}\n
\n
    \n
  • query: The search query or question
    • \n
  • pos_doc: A document relevant to the query (positive example)
    • \n
  • neg_doc: List of hard negatives—documents that share some overlap with the query but are not actually relevant (negative example)
    • \n
\n

The model learns to maximize similarity between query and positive document while minimizing similarity with negative documents.

\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n2. **Installed the Python SDK** (included with `pip install nemo-platform`)\n3. **HuggingFace token** with read access to download the SPECTER dataset (get one at [huggingface.co/settings/tokens](https://huggingface.co/settings/tokens))\n4. **NGC API key** to pull NIM container images from nvcr.io (get one at [ngc.nvidia.com](https://ngc.nvidia.com/) → Setup → Generate API Key)", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Completed the Quickstart to install and deploy NeMo Platform locally
    2. \n
  2. Installed the Python SDK (included with pip install nemo-platform)
    3. \n
  3. HuggingFace token with read access to download the SPECTER dataset (get one at huggingface.co/settings/tokens)
    4. \n
  4. NGC API key to pull NIM container images from nvcr.io (get one at ngc.nvidia.com → Setup → Generate API Key)
    5. \n
\n" + }, + { + "type": "markdown", + "source": "## Quick Start\n\n### 1. Initialize SDK\n\nThe SDK needs to know your NeMo Platform server URL. By default, `http://localhost:8080` is used in accordance with the [Quickstart](../../get-started/quickstart.md) guide. If NeMo Platform is running at a custom location, you can override the URL by setting the `NMP_BASE_URL` environment variable:\n\n```sh\nexport NMP_BASE_URL=\n```", + "source_html": "

Quick Start

\n

1. Initialize SDK

\n

The SDK needs to know your NeMo Platform server URL. By default, http://localhost:8080 is used in accordance with the Quickstart guide. If NeMo Platform is running at a custom location, you can override the URL by setting the NMP_BASE_URL environment variable:

\n
export NMP_BASE_URL=<YOUR_NMP_BASE_URL>\n
\n" + }, + { + "type": "code", + "source": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace=\"default\"\n)", + "language": "python", + "source_html": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace="default"\n)\n" + }, + { + "type": "markdown", + "source": "### 2. Establish Baseline Performance\n\nBefore fine-tuning, establish baseline performance with the pretrained model. Deploy it, run a test query, and observe where it struggles. Following fine-tuning, compare the results.\n\n**Scenario:** Searching scientific papers by meaning, not keywords.\n\n**Demo setup:**\n- **Query:** \"Conditional Random Fields\" (CRFs) - a method for sequence labeling in NLP\n- **Trap:** \"Random Forests\" shares the word \"random\" but is an unrelated tree-based algorithm\n- **Goal:** The model should distinguish between them.", + "source_html": "

2. Establish Baseline Performance

\n

Before fine-tuning, establish baseline performance with the pretrained model. Deploy it, run a test query, and observe where it struggles. Following fine-tuning, compare the results.

\n

Scenario: Searching scientific papers by meaning, not keywords.

\n

Demo setup:

\n
    \n
  • Query: "Conditional Random Fields" (CRFs) - a method for sequence labeling in NLP
    • \n
  • Trap: "Random Forests" shares the word "random" but is an unrelated tree-based algorithm
    • \n
  • Goal: The model should distinguish between them.
    • \n
\n" + }, + { + "type": "code", + "source": "# Install required packages for dataset preparation\n%pip install -q datasets huggingface_hub", + "language": "python", + "source_html": "# Install required packages for dataset preparation\n%pip install -q datasets huggingface_hub\n" + }, + { + "type": "code", + "source": "import uuid\nimport numpy as np\n\n# Demo query and documents for baseline comparison\nDEMO_QUERY = \"Conditional Random Fields: Probabilistic Models for Segmenting and Labeling Sequence Data\"\n\nDEMO_DOCS = [\n \"Bidirectional LSTM-CRF Models for Sequence Tagging\", # CRF-based paper\n \"An Introduction to Conditional Random Fields\", # CRF tutorial \n \"Random Forests\", # Keyword trap! Unrelated.\n \"Neural Architectures for Named Entity Recognition\", # Related to sequence labeling; may use CRFs\n \"Support Vector Machines for Classification\", # Unrelated ML method\n]\n\nDEMO_LABELS = [\"BiLSTM-CRF\", \"CRF Tutorial\", \"Random Forest\", \"NER\", \"SVM\"]\nDEMO_RELEVANT = {0, 1, 3} # Papers actually relevant to CRFs\n\ndef cosine_similarity(a, b):\n \"\"\"Calculate cosine similarity between two vectors.\"\"\"\n return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))", + "language": "python", + "source_html": "import uuid\nimport numpy as np\n\n# Demo query and documents for baseline comparison\nDEMO_QUERY = "Conditional Random Fields: Probabilistic Models for Segmenting and Labeling Sequence Data"\n\nDEMO_DOCS = [\n "Bidirectional LSTM-CRF Models for Sequence Tagging", # CRF-based paper\n "An Introduction to Conditional Random Fields", # CRF tutorial \n "Random Forests", # Keyword trap! Unrelated.\n "Neural Architectures for Named Entity Recognition", # Related to sequence labeling; may use CRFs\n "Support Vector Machines for Classification", # Unrelated ML method\n]\n\nDEMO_LABELS = ["BiLSTM-CRF", "CRF Tutorial", "Random Forest", "NER", "SVM"]\nDEMO_RELEVANT = {0, 1, 3} # Papers actually relevant to CRFs\n\ndef cosine_similarity(a, b):\n """Calculate cosine similarity between two vectors."""\n return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# NGC API key is required to pull NIM images from nvcr.io\nNGC_API_KEY = os.environ.get(\"NGC_API_KEY\")\nif not NGC_API_KEY:\n raise ValueError(\"NGC_API_KEY environment variable is required. Get one at https://ngc.nvidia.com/ → Setup → Generate API Key\")\n\n# Create NGC secret for pulling NIM images\nNGC_SECRET_NAME = \"ngc-api-key\"\ntry:\n client.secrets.create(name=NGC_SECRET_NAME, workspace=\"default\", value=NGC_API_KEY)\n print(f\"Created secret: {NGC_SECRET_NAME}\")\nexcept ConflictError:\n print(f\"Secret '{NGC_SECRET_NAME}' already exists, continuing...\")\n\n# Deploy base model for baseline comparison\nBASE_MODEL_HF = \"nvidia/llama-nemotron-embed-1b-v2\"\nNIM_IMAGE = \"nvcr.io/nim/nvidia/llama-nemotron-embed-1b-v2\"\nNIM_TAG = \"1.13.0\"\n\nbaseline_suffix = uuid.uuid4().hex[:4]\nBASELINE_DEPLOYMENT_CONFIG = f\"baseline-embedding-cfg-{baseline_suffix}\"\nBASELINE_DEPLOYMENT_NAME = f\"baseline-embedding-{baseline_suffix}\"\n\nprint(\"Creating baseline deployment config...\")\nbaseline_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=BASELINE_DEPLOYMENT_CONFIG,\n nim_deployment=NIMDeploymentParam(\n image_name=NIM_IMAGE,\n image_tag=NIM_TAG,\n gpu=1,\n image_pull_secret=NGC_SECRET_NAME,\n )\n)\n\nprint(\"Deploying base model...\")\nbaseline_deployment = client.inference.deployments.create(\n workspace=\"default\",\n name=BASELINE_DEPLOYMENT_NAME,\n config=baseline_config.name\n)\nprint(f\"Baseline deployment: {baseline_deployment.name}\")", + "language": "python", + "source_html": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# NGC API key is required to pull NIM images from nvcr.io\nNGC_API_KEY = os.environ.get("NGC_API_KEY")\nif not NGC_API_KEY:\n raise ValueError("NGC_API_KEY environment variable is required. Get one at https://ngc.nvidia.com/ → Setup → Generate API Key")\n\n# Create NGC secret for pulling NIM images\nNGC_SECRET_NAME = "ngc-api-key"\ntry:\n client.secrets.create(name=NGC_SECRET_NAME, workspace="default", value=NGC_API_KEY)\n print(f"Created secret: {NGC_SECRET_NAME}")\nexcept ConflictError:\n print(f"Secret '{NGC_SECRET_NAME}' already exists, continuing...")\n\n# Deploy base model for baseline comparison\nBASE_MODEL_HF = "nvidia/llama-nemotron-embed-1b-v2"\nNIM_IMAGE = "nvcr.io/nim/nvidia/llama-nemotron-embed-1b-v2"\nNIM_TAG = "1.13.0"\n\nbaseline_suffix = uuid.uuid4().hex[:4]\nBASELINE_DEPLOYMENT_CONFIG = f"baseline-embedding-cfg-{baseline_suffix}"\nBASELINE_DEPLOYMENT_NAME = f"baseline-embedding-{baseline_suffix}"\n\nprint("Creating baseline deployment config...")\nbaseline_config = client.inference.deployment_configs.create(\n workspace="default",\n name=BASELINE_DEPLOYMENT_CONFIG,\n nim_deployment=NIMDeploymentParam(\n image_name=NIM_IMAGE,\n image_tag=NIM_TAG,\n gpu=1,\n image_pull_secret=NGC_SECRET_NAME,\n )\n)\n\nprint("Deploying base model...")\nbaseline_deployment = client.inference.deployments.create(\n workspace="default",\n name=BASELINE_DEPLOYMENT_NAME,\n config=baseline_config.name\n)\nprint(f"Baseline deployment: {baseline_deployment.name}")\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Wait for baseline deployment\nTIMEOUT_MINUTES = 15\nstart_time = time.time()\n\nprint(f\"Waiting for baseline deployment...\")\nwhile True:\n status = client.inference.deployments.retrieve(\n name=BASELINE_DEPLOYMENT_NAME,\n workspace=\"default\"\n )\n \n elapsed = time.time() - start_time\n elapsed_str = f\"{int(elapsed//60)}m {int(elapsed%60)}s\"\n \n clear_output(wait=True)\n print(f\"Baseline deployment: {status.status} | {elapsed_str}\")\n \n if status.status == \"READY\":\n print(\"Baseline model ready!\")\n if not client.models.wait_for_gateway(BASELINE_DEPLOYMENT_NAME, workspace=\"default\", timeout=60):\n raise RuntimeError(\"Inference gateway did not become ready\")\n break\n if status.status in (\"FAILED\", \"ERROR\", \"TERMINATED\", \"LOST\"):\n raise RuntimeError(f\"Baseline deployment failed: {status.status}\")\n if elapsed > TIMEOUT_MINUTES * 60:\n raise TimeoutError(\"Baseline deployment timeout\")\n \n time.sleep(10)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Wait for baseline deployment\nTIMEOUT_MINUTES = 15\nstart_time = time.time()\n\nprint(f"Waiting for baseline deployment...")\nwhile True:\n status = client.inference.deployments.retrieve(\n name=BASELINE_DEPLOYMENT_NAME,\n workspace="default"\n )\n \n elapsed = time.time() - start_time\n elapsed_str = f"{int(elapsed//60)}m {int(elapsed%60)}s"\n \n clear_output(wait=True)\n print(f"Baseline deployment: {status.status} | {elapsed_str}")\n \n if status.status == "READY":\n print("Baseline model ready!")\n if not client.models.wait_for_gateway(BASELINE_DEPLOYMENT_NAME, workspace="default", timeout=60):\n raise RuntimeError("Inference gateway did not become ready")\n break\n if status.status in ("FAILED", "ERROR", "TERMINATED", "LOST"):\n raise RuntimeError(f"Baseline deployment failed: {status.status}")\n if elapsed > TIMEOUT_MINUTES * 60:\n raise TimeoutError("Baseline deployment timeout")\n \n time.sleep(10)\n" + }, + { + "type": "code", + "source": "# Run baseline ranking with the base model\nBASE_MODEL_ID = \"nvidia/llama-nemotron-embed-1b-v2\"\n\n# Get query embedding\nquery_response = client.inference.gateway.provider.post(\n \"v1/embeddings\",\n name=BASELINE_DEPLOYMENT_NAME,\n workspace=\"default\",\n body={\n \"model\": BASE_MODEL_ID,\n \"input\": [DEMO_QUERY],\n \"input_type\": \"query\"\n }\n)\nbase_query_emb = query_response[\"data\"][0][\"embedding\"]\n\n# Get document embeddings\ndoc_response = client.inference.gateway.provider.post(\n \"v1/embeddings\",\n name=BASELINE_DEPLOYMENT_NAME,\n workspace=\"default\",\n body={\n \"model\": BASE_MODEL_ID,\n \"input\": DEMO_DOCS,\n \"input_type\": \"passage\"\n }\n)\nbase_doc_embs = [d[\"embedding\"] for d in doc_response[\"data\"]]\n\n# Calculate similarities and rank\nscores = [(i, cosine_similarity(base_query_emb, base_doc_embs[i])) for i in range(len(DEMO_DOCS))]\nBASELINE_RANKING = sorted(scores, key=lambda x: -x[1])\n\n# Display baseline results\nprint(f\"Query: \\\"{DEMO_QUERY}\\\"\\n\")\nprint(\"Base Model Ranking:\")\nprint(\"-\" * 55)\nfor rank, (idx, score) in enumerate(BASELINE_RANKING, 1):\n marker = \" <-- relevant\" if idx in DEMO_RELEVANT else \"\"\n print(f\" #{rank} [{score:.3f}] {DEMO_LABELS[idx]}{marker}\")", + "language": "python", + "source_html": "# Run baseline ranking with the base model\nBASE_MODEL_ID = "nvidia/llama-nemotron-embed-1b-v2"\n\n# Get query embedding\nquery_response = client.inference.gateway.provider.post(\n "v1/embeddings",\n name=BASELINE_DEPLOYMENT_NAME,\n workspace="default",\n body={\n "model": BASE_MODEL_ID,\n "input": [DEMO_QUERY],\n "input_type": "query"\n }\n)\nbase_query_emb = query_response["data"][0]["embedding"]\n\n# Get document embeddings\ndoc_response = client.inference.gateway.provider.post(\n "v1/embeddings",\n name=BASELINE_DEPLOYMENT_NAME,\n workspace="default",\n body={\n "model": BASE_MODEL_ID,\n "input": DEMO_DOCS,\n "input_type": "passage"\n }\n)\nbase_doc_embs = [d["embedding"] for d in doc_response["data"]]\n\n# Calculate similarities and rank\nscores = [(i, cosine_similarity(base_query_emb, base_doc_embs[i])) for i in range(len(DEMO_DOCS))]\nBASELINE_RANKING = sorted(scores, key=lambda x: -x[1])\n\n# Display baseline results\nprint(f"Query: \\"{DEMO_QUERY}\\"\\n")\nprint("Base Model Ranking:")\nprint("-" * 55)\nfor rank, (idx, score) in enumerate(BASELINE_RANKING, 1):\n marker = " <-- relevant" if idx in DEMO_RELEVANT else ""\n print(f" #{rank} [{score:.3f}] {DEMO_LABELS[idx]}{marker}")\n" + }, + { + "type": "code", + "source": "# Delete baseline deployment to free GPU for training\nprint(\"Deleting baseline deployment to free GPU...\")\nclient.inference.deployments.delete(name=BASELINE_DEPLOYMENT_NAME, workspace=\"default\")\n\n# Wait for deployment to be fully deleted before deleting config\nprint(\"Waiting for deployment deletion...\")\nwhile True:\n try:\n status = client.inference.deployments.retrieve(name=BASELINE_DEPLOYMENT_NAME, workspace=\"default\")\n if status.status == \"DELETED\":\n break\n print(f\" Status: {status.status}\")\n time.sleep(5)\n except Exception:\n # Deployment no longer exists\n break\n\n# Now safe to delete the config\nclient.inference.deployment_configs.delete(name=BASELINE_DEPLOYMENT_CONFIG, workspace=\"default\")\nprint(\"GPU freed. Proceed to fine-tune and improve these rankings.\")", + "language": "python", + "source_html": "# Delete baseline deployment to free GPU for training\nprint("Deleting baseline deployment to free GPU...")\nclient.inference.deployments.delete(name=BASELINE_DEPLOYMENT_NAME, workspace="default")\n\n# Wait for deployment to be fully deleted before deleting config\nprint("Waiting for deployment deletion...")\nwhile True:\n try:\n status = client.inference.deployments.retrieve(name=BASELINE_DEPLOYMENT_NAME, workspace="default")\n if status.status == "DELETED":\n break\n print(f" Status: {status.status}")\n time.sleep(5)\n except Exception:\n # Deployment no longer exists\n break\n\n# Now safe to delete the config\nclient.inference.deployment_configs.delete(name=BASELINE_DEPLOYMENT_CONFIG, workspace="default")\nprint("GPU freed. Proceed to fine-tune and improve these rankings.")\n" + }, + { + "type": "markdown", + "source": "### 3. Prepare Dataset\n\nUse the [SPECTER dataset](https://huggingface.co/datasets/embedding-data/SPECTER) from HuggingFace, a collection of scientific paper triplets where papers that cite each other are considered related.\n\n**Dataset structure:**\n- ~684K scientific paper triplets (this tutorial uses 10%)\n- Each triplet: (query paper, positive/related paper, negative/unrelated paper)\n- Papers that cite each other are marked as \"related\"\n\nIn this tutorial the following dataset directory structure will be used:\n```\nembedding-dataset\n`-- training.jsonl\n`-- validation.jsonl\n```", + "source_html": "

3. Prepare Dataset

\n

Use the SPECTER dataset from HuggingFace, a collection of scientific paper triplets where papers that cite each other are considered related.

\n

Dataset structure:

\n
    \n
  • ~684K scientific paper triplets (this tutorial uses 10%)
    • \n
  • Each triplet: (query paper, positive/related paper, negative/unrelated paper)
    • \n
  • Papers that cite each other are marked as "related"
    • \n
\n

In this tutorial the following dataset directory structure will be used:

\n
embedding-dataset\n`-- training.jsonl\n`-- validation.jsonl\n
\n" + }, + { + "type": "markdown", + "source": "### 4. Download and Format SPECTER Dataset\n\nThe SPECTER dataset requires conversion to the triplet format required for fine-tuning.", + "source_html": "

4. Download and Format SPECTER Dataset

\n

The SPECTER dataset requires conversion to the triplet format required for fine-tuning.

\n" + }, + { + "type": "code", + "source": "from pathlib import Path\nfrom datasets import load_dataset\nimport json\n\n# HuggingFace token for dataset access\nHF_TOKEN = os.environ.get(\"HF_TOKEN\")\nif not HF_TOKEN:\n raise ValueError(\"HF_TOKEN environment variable is required. Get one at https://huggingface.co/settings/tokens\")\nos.environ[\"HF_TOKEN\"] = HF_TOKEN\n\n# Configuration\nDATASET_SIZE = 3000 # Number of triplets (increase for better results, max ~684K)\nVALIDATION_SPLIT = 0.05 # 5% held out for validation\nSEED = 42\nDATASET_PATH = Path(\"embedding-dataset\").absolute()\n\n# Create directory\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Download SPECTER dataset\nprint(\"Downloading SPECTER dataset...\")\ndata = load_dataset(\"embedding-data/SPECTER\")[\"train\"].shuffle(seed=SEED).select(range(DATASET_SIZE))\n\n# Split into train/validation\nprint(\"Splitting into train/validation...\")\nsplits = data.train_test_split(test_size=VALIDATION_SPLIT, seed=SEED)\ntrain_data = splits[\"train\"]\nvalidation_data = splits[\"test\"]\n\n# Convert to triplet JSONL format\nprint(\"Saving to JSONL...\")\nfor name, dataset in [(\"training\", train_data), (\"validation\", validation_data)]:\n with open(f\"{DATASET_PATH}/{name}.jsonl\", \"w\") as f:\n for row in dataset:\n # SPECTER format: row['set'] = [query, positive, negative]\n triplet = {\n \"query\": row[\"set\"][0],\n \"pos_doc\": row[\"set\"][1],\n \"neg_doc\": [row[\"set\"][2]] # List of negative documents\n }\n f.write(json.dumps(triplet) + \"\\n\")\n\nprint(f\"\\nPrepared {len(train_data):,} training, {len(validation_data):,} validation samples\")\nprint(f\"\\nExample triplet:\")\nprint(f\" Query: {train_data[0]['set'][0][:100]}...\")\nprint(f\" Positive: {train_data[0]['set'][1][:100]}...\")\nprint(f\" Negative: {train_data[0]['set'][2][:100]}...\")", + "language": "python", + "source_html": "from pathlib import Path\nfrom datasets import load_dataset\nimport json\n\n# HuggingFace token for dataset access\nHF_TOKEN = os.environ.get("HF_TOKEN")\nif not HF_TOKEN:\n raise ValueError("HF_TOKEN environment variable is required. Get one at https://huggingface.co/settings/tokens")\nos.environ["HF_TOKEN"] = HF_TOKEN\n\n# Configuration\nDATASET_SIZE = 3000 # Number of triplets (increase for better results, max ~684K)\nVALIDATION_SPLIT = 0.05 # 5% held out for validation\nSEED = 42\nDATASET_PATH = Path("embedding-dataset").absolute()\n\n# Create directory\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Download SPECTER dataset\nprint("Downloading SPECTER dataset...")\ndata = load_dataset("embedding-data/SPECTER")["train"].shuffle(seed=SEED).select(range(DATASET_SIZE))\n\n# Split into train/validation\nprint("Splitting into train/validation...")\nsplits = data.train_test_split(test_size=VALIDATION_SPLIT, seed=SEED)\ntrain_data = splits["train"]\nvalidation_data = splits["test"]\n\n# Convert to triplet JSONL format\nprint("Saving to JSONL...")\nfor name, dataset in [("training", train_data), ("validation", validation_data)]:\n with open(f"{DATASET_PATH}/{name}.jsonl", "w") as f:\n for row in dataset:\n # SPECTER format: row['set'] = [query, positive, negative]\n triplet = {\n "query": row["set"][0],\n "pos_doc": row["set"][1],\n "neg_doc": [row["set"][2]] # List of negative documents\n }\n f.write(json.dumps(triplet) + "\\n")\n\nprint(f"\\nPrepared {len(train_data):,} training, {len(validation_data):,} validation samples")\nprint(f"\\nExample triplet:")\nprint(f" Query: {train_data[0]['set'][0][:100]}...")\nprint(f" Positive: {train_data[0]['set'][1][:100]}...")\nprint(f" Negative: {train_data[0]['set'][2][:100]}...")\n" + }, + { + "type": "markdown", + "source": "### 5. Create Dataset FileSet and Upload Training Data", + "source_html": "

5. Create Dataset FileSet and Upload Training Data

\n" + }, + { + "type": "code", + "source": "# Create fileset to store embedding training data\nDATASET_NAME = \"embedding-dataset\"\n\ntry:\n client.files.filesets.create(\n workspace=\"default\",\n name=DATASET_NAME,\n description=\"SPECTER embedding training data (scientific paper triplets)\"\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\n# Upload training data files\nclient.files.upload(\n local_path=DATASET_PATH,\n remote_path=\"\",\n fileset=DATASET_NAME,\n workspace=\"default\"\n)\n\n# Validate upload\nprint(\"\\nUploaded files:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace=\"default\").data], indent=2))", + "language": "python", + "source_html": "# Create fileset to store embedding training data\nDATASET_NAME = "embedding-dataset"\n\ntry:\n client.files.filesets.create(\n workspace="default",\n name=DATASET_NAME,\n description="SPECTER embedding training data (scientific paper triplets)"\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\n# Upload training data files\nclient.files.upload(\n local_path=DATASET_PATH,\n remote_path="",\n fileset=DATASET_NAME,\n workspace="default"\n)\n\n# Validate upload\nprint("\\nUploaded files:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace="default").data], indent=2))\n" + }, + { + "type": "markdown", + "source": "### 6. Secrets Setup\n\nConfigure authentication for accessing base models:\n\n- **NGC models** (`ngc://` URIs): Requires NGC API key\n- **HuggingFace models** (`hf://` URIs): Requires HF token for gated/private models\n\nGet your credentials:\n- [NGC API Key](https://ngc.nvidia.com/) (Setup → Generate API Key)\n- [HuggingFace Token](https://huggingface.co/settings/tokens) (Create token with Read access)\n\n---\n\n#### Quick Setup Example\n\nThis tutorial fine-tunes [nvidia/llama-3.2-nv-embedqa-1b-v2](https://huggingface.co/nvidia/llama-3.2-nv-embedqa-1b-v2), NVIDIA embedding model optimized for question-answering and retrieval tasks.", + "source_html": "

6. Secrets Setup

\n

Configure authentication for accessing base models:

\n
    \n
  • NGC models (ngc:// URIs): Requires NGC API key
    • \n
  • HuggingFace models (hf:// URIs): Requires HF token for gated/private models
    • \n
\n

Get your credentials:

\n\n
\n

Quick Setup Example

\n

This tutorial fine-tunes nvidia/llama-3.2-nv-embedqa-1b-v2, NVIDIA embedding model optimized for question-answering and retrieval tasks.

\n" + }, + { + "type": "code", + "source": "# Create secrets for model access\n# Note: NGC_API_KEY secret was already created in the baseline step (Step 2)\nHF_TOKEN = os.getenv(\"HF_TOKEN\")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f\"{label} is not set\")\n try:\n secret = client.secrets.create(\n name=name,\n workspace=\"default\",\n value=value,\n )\n print(f\"Created secret: {name}\")\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n return client.secrets.retrieve(name=name, workspace=\"default\")\n\n\n# Create HuggingFace token secret (for downloading model from HF during training)\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")\nprint(f\"HF_TOKEN secret: {hf_secret.name}\")\n\n# NGC secret was already created in baseline step\nprint(f\"NGC_API_KEY secret: {NGC_SECRET_NAME} (created in Step 2)\")", + "language": "python", + "source_html": "# Create secrets for model access\n# Note: NGC_API_KEY secret was already created in the baseline step (Step 2)\nHF_TOKEN = os.getenv("HF_TOKEN")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f"{label} is not set")\n try:\n secret = client.secrets.create(\n name=name,\n workspace="default",\n value=value,\n )\n print(f"Created secret: {name}")\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n return client.secrets.retrieve(name=name, workspace="default")\n\n\n# Create HuggingFace token secret (for downloading model from HF during training)\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\nprint(f"HF_TOKEN secret: {hf_secret.name}")\n\n# NGC secret was already created in baseline step\nprint(f"NGC_API_KEY secret: {NGC_SECRET_NAME} (created in Step 2)")\n" + }, + { + "type": "markdown", + "source": "### 7. Create Base Model FileSet and Model Entity\n\nCreate a fileset pointing to the [nvidia/llama-3.2-nv-embedqa-1b-v2](https://huggingface.co/nvidia/llama-3.2-nv-embedqa-1b-v2) embedding model from HuggingFace, then create a Model Entity that references this fileset. Model downloading will take place at training time.\n\n---\n*Note*: Either `MODEL_NAME` or `HF_REPO_ID` below must contain the substring embed to indicate that this is an embedding model.", + "source_html": "

7. Create Base Model FileSet and Model Entity

\n

Create a fileset pointing to the nvidia/llama-3.2-nv-embedqa-1b-v2 embedding model from HuggingFace, then create a Model Entity that references this fileset. Model downloading will take place at training time.

\n
\n

Note: Either MODEL_NAME or HF_REPO_ID below must contain the substring embed to indicate that this is an embedding model.

\n" + }, + { + "type": "code", + "source": "import time\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = \"nvidia/llama-nemotron-embed-1b-v2\"\nMODEL_NAME = \"nv-nemotron-embed-1b-base\"\n\n# Ensure you have a HuggingFace token secret created\ntry:\n base_model_fs = client.files.filesets.create(\n workspace=\"default\",\n name=MODEL_NAME,\n description=\"NVIDIA Llama 3.2 NV EmbedQA 1B v2 embedding model\",\n storage=HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type=\"model\",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\nexcept ConflictError as e:\n print(f\"Base model fileset already exists. Skipping creation.\")\n base_model_fs = client.files.filesets.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\n# Create Model Entity referencing the FileSet\ntry:\n base_model = client.models.create(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n trust_remote_code=True,\n )\n print(f\"Created Model Entity: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model already exists. Updating fileset if different.\")\n base_model = client.models.update(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n trust_remote_code=True,\n )\n\nprint(f\"\\nBase model fileset: fileset://default/{base_model.name}\")\nprint(\"\\nBase model files:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace=\"default\").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint(\"\\nWaiting for ModelSpec to be populated...\")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f\"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds\")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\nprint(f\"ModelSpec populated: {base_model.spec}\")", + "language": "python", + "source_html": "import time\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = "nvidia/llama-nemotron-embed-1b-v2"\nMODEL_NAME = "nv-nemotron-embed-1b-base"\n\n# Ensure you have a HuggingFace token secret created\ntry:\n base_model_fs = client.files.filesets.create(\n workspace="default",\n name=MODEL_NAME,\n description="NVIDIA Llama 3.2 NV EmbedQA 1B v2 embedding model",\n storage=HuggingfaceStorageConfigParam(\n type="huggingface",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type="model",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\nexcept ConflictError as e:\n print(f"Base model fileset already exists. Skipping creation.")\n base_model_fs = client.files.filesets.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\n# Create Model Entity referencing the FileSet\ntry:\n base_model = client.models.create(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n trust_remote_code=True,\n )\n print(f"Created Model Entity: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model already exists. Updating fileset if different.")\n base_model = client.models.update(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n trust_remote_code=True,\n )\n\nprint(f"\\nBase model fileset: fileset://default/{base_model.name}")\nprint("\\nBase model files:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace="default").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint("\\nWaiting for ModelSpec to be populated...")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\nprint(f"ModelSpec populated: {base_model.spec}")\n" + }, + { + "type": "markdown", + "source": "### 8. Create Embedding Fine-tuning Job\n\nCreate a customization job to fine-tune the embedding model using contrastive learning on the SPECTER dataset.\n\n**Key hyperparameters for embedding fine-tuning:**\n- **`training_type`**: `sft` (supervised fine-tuning)\n- **Full fine-tuning**: No `peft` config needed (omit for all-weights training)\n- **`learning_rate`**: Lower values (1e-6 to 5e-6) work well for embedding models\n- **`batch_size`**: Larger batches improve contrastive learning (128-256 recommended)\n\n**NOTE:**\n\nNeMo Platform does not support unmerged LoRA adapters for embedding models because the embedding NIM requires ONNX format, which cannot represent standalone adapters. This notebook creates a job with all-weights finetuning but you can also run LoRA with `merge=True`, which trains a LoRA adapter and then merges it back into the base model after training. The final output is a standard full-weight checkpoint, identical in format to an all-weights fine-tuned model, but LoRA training is faster, uses less memory, and is more lenient in hyperparameter tuning.\n\nTo do that update the job request like so\n\n```\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=EPOCHS,\n batch_size=BATCH_SIZE,\n learning_rate=LEARNING_RATE,\n max_seq_length=MAX_SEQ_LENGTH,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n peft=LoRaParamsParam(\n type= \"lora\",\n merge=True\n )\n )\n )\n)\n```", + "source_html": "

8. Create Embedding Fine-tuning Job

\n

Create a customization job to fine-tune the embedding model using contrastive learning on the SPECTER dataset.

\n

Key hyperparameters for embedding fine-tuning:

\n
    \n
  • training_type: sft (supervised fine-tuning)
    • \n
  • Full fine-tuning: No peft config needed (omit for all-weights training)
    • \n
  • learning_rate: Lower values (1e-6 to 5e-6) work well for embedding models
    • \n
  • batch_size: Larger batches improve contrastive learning (128-256 recommended)
    • \n
\n

NOTE:

\n

NeMo Platform does not support unmerged LoRA adapters for embedding models because the embedding NIM requires ONNX format, which cannot represent standalone adapters. This notebook creates a job with all-weights finetuning but you can also run LoRA with merge=True, which trains a LoRA adapter and then merges it back into the base model after training. The final output is a standard full-weight checkpoint, identical in format to an all-weights fine-tuned model, but LoRA training is faster, uses less memory, and is more lenient in hyperparameter tuning.

\n

To do that update the job request like so

\n
job = client.customization.jobs.create(\n    name=JOB_NAME,\n    workspace="default",\n    spec=CustomizationJobInputParam(\n        model=f"default/{base_model.name}",\n        dataset=f"fileset://default/{DATASET_NAME}",\n        training=SftTrainingParam(\n            type="sft",\n            epochs=EPOCHS,\n            batch_size=BATCH_SIZE,\n            learning_rate=LEARNING_RATE,\n            max_seq_length=MAX_SEQ_LENGTH,\n            micro_batch_size=1,\n            parallelism=ParallelismParamsParam(\n                num_gpus_per_node=1,\n                num_nodes=1,\n                tensor_parallel_size=1,\n                pipeline_parallel_size=1,\n            ),\n            peft=LoRaParamsParam(\n                type= "lora",\n                merge=True\n            )\n        )\n    )\n)\n
\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam\n)\n\njob_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f\"embedding-finetune-job-{job_suffix}\"\n\n# Hyperparameters optimized for embedding fine-tuning\nEPOCHS = 1\nBATCH_SIZE = 128 # Larger batches help contrastive learning\nLEARNING_RATE = 5e-6 # Lower LR for embedding models\nMAX_SEQ_LENGTH = 512 # Typical for embedding models\n\n# Note: The 'name' field must contain 'embed' for the customizer to detect this as an embedding model\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=EPOCHS,\n batch_size=BATCH_SIZE,\n learning_rate=LEARNING_RATE,\n max_seq_length=MAX_SEQ_LENGTH,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n )\n)\n\nprint(f\"Job ID: {job.name}\")\nprint(f\"Output model: {job.spec.output.name}\")", + "language": "python", + "source_html": "from nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam\n)\n\njob_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f"embedding-finetune-job-{job_suffix}"\n\n# Hyperparameters optimized for embedding fine-tuning\nEPOCHS = 1\nBATCH_SIZE = 128 # Larger batches help contrastive learning\nLEARNING_RATE = 5e-6 # Lower LR for embedding models\nMAX_SEQ_LENGTH = 512 # Typical for embedding models\n\n# Note: The 'name' field must contain 'embed' for the customizer to detect this as an embedding model\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace="default",\n spec=CustomizationJobInputParam(\n model=f"default/{base_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=EPOCHS,\n batch_size=BATCH_SIZE,\n learning_rate=LEARNING_RATE,\n max_seq_length=MAX_SEQ_LENGTH,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n )\n)\n\nprint(f"Job ID: {job.name}")\nprint(f"Output model: {job.spec.output.name}")\n" + }, + { + "type": "markdown", + "source": "### 9. Track Training Progress", + "source_html": "

9. Track Training Progress

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Poll job status every 10 seconds until completed\nwhile True:\n status = client.customization.jobs.get_status(\n name=job.name,\n workspace=\"default\"\n )\n \n clear_output(wait=True)\n print(f\"Job Status: {status.model_dump_json(indent=2)}\")\n\n # Extract training progress from nested steps structure\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n\n for job_step in status.steps or []:\n if job_step.name == \"customization-training-job\":\n for task in job_step.tasks or []:\n task_details = task.status_details or {}\n step = task_details.get(\"step\")\n max_steps = task_details.get(\"max_steps\")\n training_phase = task_details.get(\"phase\")\n break\n break\n\n if step is not None and max_steps is not None:\n progress_pct = (step / max_steps) * 100\n print(f\"Training Progress: Step {step}/{max_steps} ({progress_pct:.1f}%)\")\n if training_phase:\n print(f\"Training Phase: {training_phase}\")\n else:\n print(\"Training step not started yet or progress info not available\")\n \n # Exit loop when job is completed (or failed/cancelled)\n if status.status in (\"completed\", \"failed\", \"cancelled\", \"error\"):\n print(f\"\\nJob finished with status: {status.status}\")\n break\n \n time.sleep(10)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Poll job status every 10 seconds until completed\nwhile True:\n status = client.customization.jobs.get_status(\n name=job.name,\n workspace="default"\n )\n \n clear_output(wait=True)\n print(f"Job Status: {status.model_dump_json(indent=2)}")\n\n # Extract training progress from nested steps structure\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n\n for job_step in status.steps or []:\n if job_step.name == "customization-training-job":\n for task in job_step.tasks or []:\n task_details = task.status_details or {}\n step = task_details.get("step")\n max_steps = task_details.get("max_steps")\n training_phase = task_details.get("phase")\n break\n break\n\n if step is not None and max_steps is not None:\n progress_pct = (step / max_steps) * 100\n print(f"Training Progress: Step {step}/{max_steps} ({progress_pct:.1f}%)")\n if training_phase:\n print(f"Training Phase: {training_phase}")\n else:\n print("Training step not started yet or progress info not available")\n \n # Exit loop when job is completed (or failed/cancelled)\n if status.status in ("completed", "failed", "cancelled", "error"):\n print(f"\\nJob finished with status: {status.status}")\n break\n \n time.sleep(10)\n" + }, + { + "type": "markdown", + "source": "**Interpreting Embedding Training Metrics:**\n\nEmbedding models use contrastive loss—lower values indicate better separation between similar and dissimilar pairs:\n\n| Scenario | Interpretation | Action |\n|----------|----------------|--------|\n| **Loss steadily decreasing** | Model learning semantic relationships | Continue training |\n| **Loss plateaus early** | May need more data or epochs | Increase dataset/epochs |\n| **Loss spikes** | Training instability | Lower learning rate |\n| **Validation loss increasing** | Overfitting | Reduce epochs, add data |", + "source_html": "

Interpreting Embedding Training Metrics:

\n

Embedding models use contrastive loss—lower values indicate better separation between similar and dissimilar pairs:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ScenarioInterpretationAction
Loss steadily decreasingModel learning semantic relationshipsContinue training
Loss plateaus earlyMay need more data or epochsIncrease dataset/epochs
Loss spikesTraining instabilityLower learning rate
Validation loss increasingOverfittingReduce epochs, add data
\n" + }, + { + "type": "markdown", + "source": "### 10. Deploy Fine-Tuned Embedding Model\n\nAfter training completes, deploy the embedding model using the Deployment Management Service:", + "source_html": "

10. Deploy Fine-Tuned Embedding Model

\n

After training completes, deploy the embedding model using the Deployment Management Service:

\n" + }, + { + "type": "code", + "source": "# Validate model entity exists\nmodel_entity = client.models.retrieve(workspace=\"default\", name=job.spec.output.name)\nprint(model_entity.model_dump_json(indent=2))", + "language": "python", + "source_html": "# Validate model entity exists\nmodel_entity = client.models.retrieve(workspace="default", name=job.spec.output.name)\nprint(model_entity.model_dump_json(indent=2))\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# Create deployment config for embedding model\ndeploy_suffix = uuid.uuid4().hex[:4]\nDEPLOYMENT_CONFIG_NAME = f\"embedding-model-deployment-cfg-{deploy_suffix}\"\nDEPLOYMENT_NAME = f\"embedding-model-deployment-{deploy_suffix}\"\n\n# Embedding NIM image\nNIM_IMAGE = \"nvcr.io/nim/nvidia/llama-nemotron-embed-1b-v2\"\nNIM_TAG = \"1.13.0\" # Update if using newer NIM release\n\ndeployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=DEPLOYMENT_CONFIG_NAME,\n nim_deployment=NIMDeploymentParam(\n image_name=NIM_IMAGE,\n image_tag=NIM_TAG,\n gpu=1,\n model_name=job.spec.output.name,\n model_namespace=\"default\",\n )\n)\n\n# Deploy model\ndeployment = client.inference.deployments.create(\n workspace=\"default\",\n name=DEPLOYMENT_NAME,\n config=deployment_config.name\n)\n\nprint(f\"Deployment name: {deployment.name}\")\nprint(f\"Deployment status: {client.inference.deployments.retrieve(name=deployment.name, workspace='default').status}\")", + "language": "python", + "source_html": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# Create deployment config for embedding model\ndeploy_suffix = uuid.uuid4().hex[:4]\nDEPLOYMENT_CONFIG_NAME = f"embedding-model-deployment-cfg-{deploy_suffix}"\nDEPLOYMENT_NAME = f"embedding-model-deployment-{deploy_suffix}"\n\n# Embedding NIM image\nNIM_IMAGE = "nvcr.io/nim/nvidia/llama-nemotron-embed-1b-v2"\nNIM_TAG = "1.13.0" # Update if using newer NIM release\n\ndeployment_config = client.inference.deployment_configs.create(\n workspace="default",\n name=DEPLOYMENT_CONFIG_NAME,\n nim_deployment=NIMDeploymentParam(\n image_name=NIM_IMAGE,\n image_tag=NIM_TAG,\n gpu=1,\n model_name=job.spec.output.name,\n model_namespace="default",\n )\n)\n\n# Deploy model\ndeployment = client.inference.deployments.create(\n workspace="default",\n name=DEPLOYMENT_NAME,\n config=deployment_config.name\n)\n\nprint(f"Deployment name: {deployment.name}")\nprint(f"Deployment status: {client.inference.deployments.retrieve(name=deployment.name, workspace='default').status}")\n" + }, + { + "type": "markdown", + "source": "#### Track Deployment Status", + "source_html": "

Track Deployment Status

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Poll deployment status every 15 seconds until ready\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f\"Monitoring deployment '{deployment.name}'...\")\nprint(f\"Timeout: {TIMEOUT_MINUTES} minutes\\n\")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace=\"default\"\n )\n \n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n \n clear_output(wait=True)\n print(f\"Deployment: {deployment.name}\")\n print(f\"Status: {deployment_status.status}\")\n print(f\"Elapsed time: {elapsed_min}m {elapsed_sec}s\")\n \n # Check if deployment is ready\n if deployment_status.status == \"READY\":\n print(\"\\nDeployment is ready!\")\n if not client.models.wait_for_gateway(deployment.name, workspace=\"default\", timeout=60):\n raise RuntimeError(\"Inference gateway did not become ready\")\n break\n \n # Check for failure states\n if deployment_status.status in (\"FAILED\", \"ERROR\", \"TERMINATED\", \"LOST\"):\n raise RuntimeError(f\"Deployment failed with status: {deployment_status.status}\")\n \n # Check timeout\n if elapsed > timeout_seconds:\n raise TimeoutError(f\"Deployment timeout after {TIMEOUT_MINUTES} minutes\")\n \n time.sleep(15)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Poll deployment status every 15 seconds until ready\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f"Monitoring deployment '{deployment.name}'...")\nprint(f"Timeout: {TIMEOUT_MINUTES} minutes\\n")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace="default"\n )\n \n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n \n clear_output(wait=True)\n print(f"Deployment: {deployment.name}")\n print(f"Status: {deployment_status.status}")\n print(f"Elapsed time: {elapsed_min}m {elapsed_sec}s")\n \n # Check if deployment is ready\n if deployment_status.status == "READY":\n print("\\nDeployment is ready!")\n if not client.models.wait_for_gateway(deployment.name, workspace="default", timeout=60):\n raise RuntimeError("Inference gateway did not become ready")\n break\n \n # Check for failure states\n if deployment_status.status in ("FAILED", "ERROR", "TERMINATED", "LOST"):\n raise RuntimeError(f"Deployment failed with status: {deployment_status.status}")\n \n # Check timeout\n if elapsed > timeout_seconds:\n raise TimeoutError(f"Deployment timeout after {TIMEOUT_MINUTES} minutes")\n \n time.sleep(15)\n" + }, + { + "type": "markdown", + "source": "### 11. View the Improvement\n\nRun the same query against the fine-tuned model and compare to the baseline from earlier.", + "source_html": "

11. View the Improvement

\n

Run the same query against the fine-tuned model and compare to the baseline from earlier.

\n" + }, + { + "type": "code", + "source": "# Compare: same query, base model vs fine-tuned\n# Using the same DEMO_QUERY and DEMO_DOCS from the baseline test\nMODEL_ID = f\"default/{job.spec.output.name}\"\n\n# Get query embedding from fine-tuned model\nquery_response = client.inference.gateway.provider.post(\n \"v1/embeddings\",\n name=deployment.name,\n workspace=\"default\",\n body={\n \"model\": MODEL_ID,\n \"input\": [DEMO_QUERY],\n \"input_type\": \"query\"\n }\n)\nquery_embedding = query_response[\"data\"][0][\"embedding\"]\n\n# Get document embeddings from fine-tuned model\ndoc_response = client.inference.gateway.provider.post(\n \"v1/embeddings\",\n name=deployment.name,\n workspace=\"default\",\n body={\n \"model\": MODEL_ID,\n \"input\": DEMO_DOCS,\n \"input_type\": \"passage\"\n }\n)\ndoc_embeddings = [d[\"embedding\"] for d in doc_response[\"data\"]]\n\n# Calculate similarities and rank\nscores = [(i, cosine_similarity(query_embedding, doc_embeddings[i])) for i in range(len(DEMO_DOCS))]\nFINETUNED_RANKING = sorted(scores, key=lambda x: -x[1])\n\n# Display side-by-side comparison\nprint(f\"Query: \\\"{DEMO_QUERY}\\\"\\n\")\nprint(f\"{'Rank':<6} {'Base Model':<30} {'Fine-tuned Model':<30}\")\nprint(\"-\" * 66)\n\nfor rank in range(len(DEMO_DOCS)):\n b_idx, b_score = BASELINE_RANKING[rank]\n f_idx, f_score = FINETUNED_RANKING[rank]\n \n b_label = f\"{DEMO_LABELS[b_idx]} [{b_score:.3f}]\" + (\" *\" if b_idx in DEMO_RELEVANT else \"\")\n f_label = f\"{DEMO_LABELS[f_idx]} [{f_score:.3f}]\" + (\" *\" if f_idx in DEMO_RELEVANT else \"\")\n \n print(f\"#{rank+1:<5} {b_label:<30} {f_label:<30}\")\n\nprint(\"\\n* = relevant paper\")\nprint(\"\\nThe fine-tuned model pushes 'Random Forest' down and ranks CRF papers higher.\")", + "language": "python", + "source_html": "# Compare: same query, base model vs fine-tuned\n# Using the same DEMO_QUERY and DEMO_DOCS from the baseline test\nMODEL_ID = f"default/{job.spec.output.name}"\n\n# Get query embedding from fine-tuned model\nquery_response = client.inference.gateway.provider.post(\n "v1/embeddings",\n name=deployment.name,\n workspace="default",\n body={\n "model": MODEL_ID,\n "input": [DEMO_QUERY],\n "input_type": "query"\n }\n)\nquery_embedding = query_response["data"][0]["embedding"]\n\n# Get document embeddings from fine-tuned model\ndoc_response = client.inference.gateway.provider.post(\n "v1/embeddings",\n name=deployment.name,\n workspace="default",\n body={\n "model": MODEL_ID,\n "input": DEMO_DOCS,\n "input_type": "passage"\n }\n)\ndoc_embeddings = [d["embedding"] for d in doc_response["data"]]\n\n# Calculate similarities and rank\nscores = [(i, cosine_similarity(query_embedding, doc_embeddings[i])) for i in range(len(DEMO_DOCS))]\nFINETUNED_RANKING = sorted(scores, key=lambda x: -x[1])\n\n# Display side-by-side comparison\nprint(f"Query: \\"{DEMO_QUERY}\\"\\n")\nprint(f"{'Rank':<6} {'Base Model':<30} {'Fine-tuned Model':<30}")\nprint("-" * 66)\n\nfor rank in range(len(DEMO_DOCS)):\n b_idx, b_score = BASELINE_RANKING[rank]\n f_idx, f_score = FINETUNED_RANKING[rank]\n \n b_label = f"{DEMO_LABELS[b_idx]} [{b_score:.3f}]" + (" *" if b_idx in DEMO_RELEVANT else "")\n f_label = f"{DEMO_LABELS[f_idx]} [{f_score:.3f}]" + (" *" if f_idx in DEMO_RELEVANT else "")\n \n print(f"#{rank+1:<5} {b_label:<30} {f_label:<30}")\n\nprint("\\n* = relevant paper")\nprint("\\nThe fine-tuned model pushes 'Random Forest' down and ranks CRF papers higher.")\n" + }, + { + "type": "markdown", + "source": "### Evaluation Best Practices\n\n**Manual Evaluation** (Recommended)\n- Test with real-world queries from your domain\n- Compare retrieval rankings before and after fine-tuning\n- Check that semantically similar items rank higher than keyword matches\n\n**What to look for:**\n- ✅ Relevant documents consistently rank in top positions\n- ✅ Keyword traps (like \"Random Forest\" vs \"Random Fields\") are handled correctly\n- ✅ Domain-specific terminology is understood\n- ❌ Unrelated documents with matching keywords do not rank high\n\n**Benchmark Evaluation**\n\nFor systematic evaluation, use the NeMo Evaluator service with retrieval benchmarks like SciDocs, BEIR, or MTEB. Refer to the [Evaluator documentation](../../evaluator/index.md) for details.\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n**Embedding-Specific Recommendations:**\n\n| Parameter | Recommended | Notes |\n|-----------|-------------|-------|\n| `learning_rate` | 1e-6 to 5e-6 | Lower than standard SFT |\n| `batch_size` | 128-256 | Larger batches improve contrastive learning |\n| `max_seq_length` | 512 | Typical for embedding models |\n| `epochs` | 1-3 | Start small, increase if needed |\n\n---\n\n## Troubleshooting\n\n**Embeddings do not show improved retrieval:**\n- Verify dataset quality: triplets should have clear positive/negative distinctions\n- Use hard negatives: negatives should share some overlap with the query but not be relevant (easy negatives do not teach the model much)\n- Increase dataset size: 10K+ triplets recommended for meaningful improvement\n- Try more epochs: embedding models often need multiple passes\n- Lower learning rate: embedding models are sensitive to LR\n\n**Training loss not decreasing:**\n- Check triplet format: ensure `neg_doc` is a list even for single negatives\n- Verify hard negative quality: negatives should be challenging but clearly non-relevant\n- Increase batch size: contrastive learning benefits from larger batches\n\n**Deployment fails:**\n- Ensure you use the correct NIM image for embedding models\n- Verify sufficient GPU memory for the model size\n- Check deployment status: `client.inference.deployments.retrieve(name=deployment.name, workspace=\"default\")` and refer to platform logs for debugging\n\n## Next Steps\n\n- [Monitor training metrics](../manage-customization-jobs/get-job-status.md) in detail\n- [Evaluate your model](../../evaluator/index.md) with retrieval benchmarks\n- Integrate the fine-tuned embedding model into your RAG pipeline\n- Scale up training with the full SPECTER dataset (~684K triplets) for better results", + "source_html": "

Evaluation Best Practices

\n

Manual Evaluation (Recommended)

\n
    \n
  • Test with real-world queries from your domain
    • \n
  • Compare retrieval rankings before and after fine-tuning
    • \n
  • Check that semantically similar items rank higher than keyword matches
    • \n
\n

What to look for:

\n
    \n
  • ✅ Relevant documents consistently rank in top positions
    • \n
  • ✅ Keyword traps (like "Random Forest" vs "Random Fields") are handled correctly
    • \n
  • ✅ Domain-specific terminology is understood
    • \n
  • ❌ Unrelated documents with matching keywords do not rank high
    • \n
\n

Benchmark Evaluation

\n

For systematic evaluation, use the NeMo Evaluator service with retrieval benchmarks like SciDocs, BEIR, or MTEB. Refer to the Evaluator documentation for details.

\n
\n

Hyperparameters

\n

For detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.

\n

Embedding-Specific Recommendations:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ParameterRecommendedNotes
learning_rate1e-6 to 5e-6Lower than standard SFT
batch_size128-256Larger batches improve contrastive learning
max_seq_length512Typical for embedding models
epochs1-3Start small, increase if needed
\n
\n

Troubleshooting

\n

Embeddings do not show improved retrieval:

\n
    \n
  • Verify dataset quality: triplets should have clear positive/negative distinctions
    • \n
  • Use hard negatives: negatives should share some overlap with the query but not be relevant (easy negatives do not teach the model much)
    • \n
  • Increase dataset size: 10K+ triplets recommended for meaningful improvement
    • \n
  • Try more epochs: embedding models often need multiple passes
    • \n
  • Lower learning rate: embedding models are sensitive to LR
    • \n
\n

Training loss not decreasing:

\n
    \n
  • Check triplet format: ensure neg_doc is a list even for single negatives
    • \n
  • Verify hard negative quality: negatives should be challenging but clearly non-relevant
    • \n
  • Increase batch size: contrastive learning benefits from larger batches
    • \n
\n

Deployment fails:

\n
    \n
  • Ensure you use the correct NIM image for embedding models
    • \n
  • Verify sufficient GPU memory for the model size
    • \n
  • Check deployment status: client.inference.deployments.retrieve(name=deployment.name, workspace="default") and refer to platform logs for debugging
    • \n
\n

Next Steps

\n
    \n
  • Monitor training metrics in detail
    • \n
  • Evaluate your model with retrieval benchmarks
    • \n
  • Integrate the fine-tuned embedding model into your RAG pipeline
    • \n
  • Scale up training with the full SPECTER dataset (~684K triplets) for better results
    • \n
\n" + } +] }; diff --git a/docs/fern/components/notebooks/lora-customization-job.json b/docs/fern/components/notebooks/lora-customization-job.json new file mode 100644 index 00000000000..bb1ac4e94d6 --- /dev/null +++ b/docs/fern/components/notebooks/lora-customization-job.json @@ -0,0 +1,139 @@ +{ + "cells": [ + { + "type": "markdown", + "source": "\n\n\n# LoRA Model Customization Job\n\nLearn how to use the NeMo Platform to create a LoRA (Low-Rank Adaptation) customization job using a custom dataset. In this tutorial we use LoRA to fine-tune a **question-answering model** from the SQuAD dataset.\n\nLoRA is a parameter-efficient fine-tuning method that requires fewer computational resources than full fine-tuning. If you need full model fine-tuning instead, see the [Full SFT Customization Job](./sft-customization-job) tutorial.\n\n**Time to complete:** approximately 45 minutes. Job duration increases with model size and dataset size.", + "source_html": "\n\n

LoRA Model Customization Job

\n

Learn how to use the NeMo Platform to create a LoRA (Low-Rank Adaptation) customization job using a custom dataset. In this tutorial we use LoRA to fine-tune a question-answering model from the SQuAD dataset.

\n

LoRA is a parameter-efficient fine-tuning method that requires fewer computational resources than full fine-tuning. If you need full model fine-tuning instead, see the Full SFT Customization Job tutorial.

\n

Time to complete: approximately 45 minutes. Job duration increases with model size and dataset size.

\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n2. **Installed the Python SDK** (included with `pip install nemo-platform`)\n3. **Installed the `datasets` package** for loading SQuAD: `pip install datasets`\n4. **At least one GPU with CUDA 12.8+**", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Completed the Quickstart to install and deploy NeMo Platform locally
    2. \n
  2. Installed the Python SDK (included with pip install nemo-platform)
    3. \n
  3. Installed the datasets package for loading SQuAD: pip install datasets
    4. \n
  4. At least one GPU with CUDA 12.8+
    5. \n
\n" + }, + { + "type": "markdown", + "source": "## Quick Start\n\n### 1. Initialize SDK\n\nThe SDK needs to know your NeMo Platform server URL. By default, `http://localhost:8080` is used. If NeMo Platform is running elsewhere, set the `NMP_BASE_URL` environment variable:\n\n```sh\nexport NMP_BASE_URL=\n```", + "source_html": "

Quick Start

\n

1. Initialize SDK

\n

The SDK needs to know your NeMo Platform server URL. By default, http://localhost:8080 is used. If NeMo Platform is running elsewhere, set the NMP_BASE_URL environment variable:

\n
export NMP_BASE_URL=<YOUR_NMP_BASE_URL>\n
\n" + }, + { + "type": "code", + "source": "import os\nimport json\nimport re\nimport time\nimport uuid\nfrom pathlib import Path\nfrom nemo_platform import NeMoPlatform, ConflictError\nfrom nemo_platform.types.secrets import PlatformSecretResponse\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n DeploymentParamsParam,\n LoRaParamsParam,\n ParallelismParamsParam,\n SftTrainingParam,\n)\n\n\ndef sanitize_name(prefix: str, name: str):\n \"\"\"Sanitize model_name for deployment/config naming. Compatible with platform naming rules.\"\"\"\n name = name.split(\"/\")[-1]\n sanitized = re.sub(r\"[^a-z0-9@.+_-]\", \"-\", name.lower())\n sanitized = re.sub(r\"-+\", \"-\", sanitized).strip(\"-\")\n return f\"{prefix}-{sanitized}\"[:59].rstrip(\"-\")\n\n\ndef max_wait_time_checker(seconds: int, job_name: str = \"\"):\n \"\"\"Return a check() that raises TimeoutError if called after `seconds` have elapsed.\"\"\"\n start_time = time.time()\n\n def check():\n if time.time() - start_time > seconds:\n raise TimeoutError(f\"{job_name} took longer than {seconds} seconds\")\n\n return check\n\n\nNMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace=\"default\"\n)", + "language": "python", + "source_html": "import os\nimport json\nimport re\nimport time\nimport uuid\nfrom pathlib import Path\nfrom nemo_platform import NeMoPlatform, ConflictError\nfrom nemo_platform.types.secrets import PlatformSecretResponse\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n DeploymentParamsParam,\n LoRaParamsParam,\n ParallelismParamsParam,\n SftTrainingParam,\n)\n\n\ndef sanitize_name(prefix: str, name: str):\n """Sanitize model_name for deployment/config naming. Compatible with platform naming rules."""\n name = name.split("/")[-1]\n sanitized = re.sub(r"[^a-z0-9@.+_-]", "-", name.lower())\n sanitized = re.sub(r"-+", "-", sanitized).strip("-")\n return f"{prefix}-{sanitized}"[:59].rstrip("-")\n\n\ndef max_wait_time_checker(seconds: int, job_name: str = ""):\n """Return a check() that raises TimeoutError if called after `seconds` have elapsed."""\n start_time = time.time()\n\n def check():\n if time.time() - start_time > seconds:\n raise TimeoutError(f"{job_name} took longer than {seconds} seconds")\n\n return check\n\n\nNMP_BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace="default"\n)\n" + }, + { + "type": "markdown", + "source": "### 2. Prepare Dataset\n\nCreate your data in JSONL format (one JSON object per line). For SFT with LoRA, the platform expects **prompt/completion** pairs.\n\n**Dataset structure:**\n- Training files under `training/` (or root with `training.jsonl`)\n- Validation files under `validation/` (or `validation.jsonl`)\n\n**SFT format:** Each line is a JSON object with:\n- **`prompt`**: The input (e.g. context + question)\n- **`completion`**: The desired model output", + "source_html": "

2. Prepare Dataset

\n

Create your data in JSONL format (one JSON object per line). For SFT with LoRA, the platform expects prompt/completion pairs.

\n

Dataset structure:

\n
    \n
  • Training files under training/ (or root with training.jsonl)
    • \n
  • Validation files under validation/ (or validation.jsonl)
    • \n
\n

SFT format: Each line is a JSON object with:

\n
    \n
  • prompt: The input (e.g. context + question)
    • \n
  • completion: The desired model output
    • \n
\n" + }, + { + "type": "markdown", + "source": "Example record (single line in `.jsonl`):\n\n```json\n{\"prompt\": \"Context: ... Question: What is X? Answer:\", \"completion\": \"X is ...\"}\n```", + "source_html": "

Example record (single line in .jsonl):

\n
{"prompt": "Context: ... Question: What is X? Answer:", "completion": "X is ..."}\n
\n" + }, + { + "type": "markdown", + "source": "#### Download SQuAD and Convert to SFT Format\n\nWe use the [SQuAD](https://huggingface.co/datasets/rajpurkar/squad) dataset and convert it to prompt/completion JSONL.", + "source_html": "

Download SQuAD and Convert to SFT Format

\n

We use the SQuAD dataset and convert it to prompt/completion JSONL.

\n" + }, + { + "type": "code", + "source": "from datasets import load_dataset, DatasetDict\n\nprint(\"Loading dataset rajpurkar/squad\")\nraw_dataset = load_dataset(\"rajpurkar/squad\")\nif not isinstance(raw_dataset, DatasetDict):\n raise ValueError(\"Dataset does not contain expected splits\")\nprint(\"Loaded dataset\")\n\nVALIDATION_PROPORTION = 0.05\nSEED = 1234\ntraining_size = 3000\nvalidation_size = 300\nDATASET_NAME = \"sft-dataset\"\nDATASET_PATH = Path(\"sft-dataset\").absolute()\n\nos.makedirs(DATASET_PATH, exist_ok=True)\ntrain_set = raw_dataset.get(\"train\")\nsplit_dataset = train_set.train_test_split(test_size=VALIDATION_PROPORTION, seed=SEED)\ntrain_ds = split_dataset[\"train\"].select(range(min(training_size, len(split_dataset[\"train\"]))))\nvalidation_ds = split_dataset[\"test\"].select(range(min(validation_size, len(split_dataset[\"test\"]))))\n\ndef convert_squad_to_sft_format(example):\n prompt = f\"Context: {example['context']} Question: {example['question']} Answer:\"\n completion = example[\"answers\"][\"text\"][0]\n return {\"prompt\": prompt, \"completion\": completion}\n\nwith open(f\"{DATASET_PATH}/training.jsonl\", \"w\", encoding=\"utf-8\") as f:\n for example in train_ds:\n f.write(json.dumps(convert_squad_to_sft_format(example)) + \"\\n\")\nwith open(f\"{DATASET_PATH}/validation.jsonl\", \"w\", encoding=\"utf-8\") as f:\n for example in validation_ds:\n f.write(json.dumps(convert_squad_to_sft_format(example)) + \"\\n\")\n\nprint(f\"Saved training.jsonl with {len(train_ds)} rows\")\nprint(f\"Saved validation.jsonl with {len(validation_ds)} rows\")\nwith open(f\"{DATASET_PATH}/training.jsonl\", \"r\") as f:\n sample = json.loads(f.readline())\nprint(\"Sample prompt (first 200 chars):\", sample[\"prompt\"][:200] + \"...\")\nprint(\"Sample completion:\", sample[\"completion\"])", + "language": "python", + "source_html": "from datasets import load_dataset, DatasetDict\n\nprint("Loading dataset rajpurkar/squad")\nraw_dataset = load_dataset("rajpurkar/squad")\nif not isinstance(raw_dataset, DatasetDict):\n raise ValueError("Dataset does not contain expected splits")\nprint("Loaded dataset")\n\nVALIDATION_PROPORTION = 0.05\nSEED = 1234\ntraining_size = 3000\nvalidation_size = 300\nDATASET_NAME = "sft-dataset"\nDATASET_PATH = Path("sft-dataset").absolute()\n\nos.makedirs(DATASET_PATH, exist_ok=True)\ntrain_set = raw_dataset.get("train")\nsplit_dataset = train_set.train_test_split(test_size=VALIDATION_PROPORTION, seed=SEED)\ntrain_ds = split_dataset["train"].select(range(min(training_size, len(split_dataset["train"]))))\nvalidation_ds = split_dataset["test"].select(range(min(validation_size, len(split_dataset["test"]))))\n\ndef convert_squad_to_sft_format(example):\n prompt = f"Context: {example['context']} Question: {example['question']} Answer:"\n completion = example["answers"]["text"][0]\n return {"prompt": prompt, "completion": completion}\n\nwith open(f"{DATASET_PATH}/training.jsonl", "w", encoding="utf-8") as f:\n for example in train_ds:\n f.write(json.dumps(convert_squad_to_sft_format(example)) + "\\n")\nwith open(f"{DATASET_PATH}/validation.jsonl", "w", encoding="utf-8") as f:\n for example in validation_ds:\n f.write(json.dumps(convert_squad_to_sft_format(example)) + "\\n")\n\nprint(f"Saved training.jsonl with {len(train_ds)} rows")\nprint(f"Saved validation.jsonl with {len(validation_ds)} rows")\nwith open(f"{DATASET_PATH}/training.jsonl", "r") as f:\n sample = json.loads(f.readline())\nprint("Sample prompt (first 200 chars):", sample["prompt"][:200] + "...")\nprint("Sample completion:", sample["completion"])\n" + }, + { + "type": "markdown", + "source": "### 3. Create FileSet and Upload Training Data\n\nUpload the training and validation JSONL files to a FileSet so the customization job can use them.", + "source_html": "

3. Create FileSet and Upload Training Data

\n

Upload the training and validation JSONL files to a FileSet so the customization job can use them.

\n" + }, + { + "type": "code", + "source": "try:\n client.files.filesets.create(\n workspace=\"default\",\n name=DATASET_NAME,\n description=\"SFT training data\",\n cache=True,\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\nclient.files.fsspec.put(\n lpath=DATASET_PATH,\n rpath=f\"default/{DATASET_NAME}/\",\n recursive=True\n)\nprint(\"Training data:\")\nprint(client.files.list(fileset=DATASET_NAME, workspace=\"default\"))", + "language": "python", + "source_html": "try:\n client.files.filesets.create(\n workspace="default",\n name=DATASET_NAME,\n description="SFT training data",\n cache=True,\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\nclient.files.fsspec.put(\n lpath=DATASET_PATH,\n rpath=f"default/{DATASET_NAME}/",\n recursive=True\n)\nprint("Training data:")\nprint(client.files.list(fileset=DATASET_NAME, workspace="default"))\n" + }, + { + "type": "markdown", + "source": "### 4. Secrets Setup\n\nFor Huggingface models that require authentication, create a secret with your HF token. Get a token from [Huggingface Settings](https://huggingface.co/settings/tokens) and accept the model terms.\n\nThis is generally true for LLaMa based models (e.g. [Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct)).\n\n```sh\nexport HF_TOKEN=\n```", + "source_html": "

4. Secrets Setup

\n

For Huggingface models that require authentication, create a secret with your HF token. Get a token from Huggingface Settings and accept the model terms.

\n

This is generally true for LLaMa based models (e.g. Llama-3.2-1B-Instruct).

\n
export HF_TOKEN=<your-huggingface-token>\n
\n" + }, + { + "type": "code", + "source": "HF_TOKEN = os.getenv(\"HF_TOKEN\")\n\ndef create_or_get_secret(name: str, value: str | None, label: str) -> PlatformSecretResponse | None:\n if not value:\n print(f\"{label} is not set - skipping setting secret\")\n return None\n try:\n secret = client.secrets.create(name=name, workspace=\"default\", value=value)\n print(f\"Created secret: {name}\")\n print(secret.model_dump_json(indent=2))\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n secret = client.secrets.retrieve(name=name, workspace=\"default\")\n print(secret.model_dump_json(indent=2))\n return secret\n\n\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")", + "language": "python", + "source_html": "HF_TOKEN = os.getenv("HF_TOKEN")\n\ndef create_or_get_secret(name: str, value: str | None, label: str) -> PlatformSecretResponse | None:\n if not value:\n print(f"{label} is not set - skipping setting secret")\n return None\n try:\n secret = client.secrets.create(name=name, workspace="default", value=value)\n print(f"Created secret: {name}")\n print(secret.model_dump_json(indent=2))\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n secret = client.secrets.retrieve(name=name, workspace="default")\n print(secret.model_dump_json(indent=2))\n return secret\n\n\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\n" + }, + { + "type": "markdown", + "source": "### 5. Create Base Model FileSet and Model Entity\n\nCreate a fileset pointing to [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) and a Model Entity that references it. Model download happens when the customization job runs.", + "source_html": "

5. Create Base Model FileSet and Model Entity

\n

Create a fileset pointing to Qwen/Qwen3-0.6B and a Model Entity that references it. Model download happens when the customization job runs.

\n" + }, + { + "type": "code", + "source": "HF_REPO_ID = \"Qwen/Qwen3-0.6B\"\nMODEL_NAME = \"qwen3-0.6b\"\n\ntry:\n storage = HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n repo_id=HF_REPO_ID,\n repo_type=\"model\",\n )\n if hf_secret:\n storage[\"token_secret\"] = hf_secret.name\n base_model_fs = client.files.filesets.create(\n workspace=\"default\",\n name=MODEL_NAME,\n description=\"Qwen3 0.6b base model from Huggingface\",\n storage=storage,\n cache=True,\n )\nexcept ConflictError:\n base_model_fs = client.files.filesets.retrieve(workspace=\"default\", name=MODEL_NAME)\n\ntry:\n base_model = client.models.create(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n trust_remote_code=False,\n )\nexcept ConflictError:\n client.models.update(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n trust_remote_code=False,\n )\n base_model = client.models.retrieve(workspace=\"default\", name=MODEL_NAME)\n\nprint(f\"Base model fileset: fileset://default/{base_model.name}\")\nprint(client.files.list(fileset=MODEL_NAME, workspace=\"default\"))\n\ntime_check = max_wait_time_checker(600, \"Model Spec\")\nwhile not base_model.spec:\n time_check()\n time.sleep(10)\n base_model = client.models.retrieve(workspace=\"default\", name=MODEL_NAME)\n\n# Clear verbose linear_layers list for cleaner output\nbase_model.spec.linear_layers = None\nprint(f\"ModelSpec: {base_model.spec}\")", + "language": "python", + "source_html": "HF_REPO_ID = "Qwen/Qwen3-0.6B"\nMODEL_NAME = "qwen3-0.6b"\n\ntry:\n storage = HuggingfaceStorageConfigParam(\n type="huggingface",\n repo_id=HF_REPO_ID,\n repo_type="model",\n )\n if hf_secret:\n storage["token_secret"] = hf_secret.name\n base_model_fs = client.files.filesets.create(\n workspace="default",\n name=MODEL_NAME,\n description="Qwen3 0.6b base model from Huggingface",\n storage=storage,\n cache=True,\n )\nexcept ConflictError:\n base_model_fs = client.files.filesets.retrieve(workspace="default", name=MODEL_NAME)\n\ntry:\n base_model = client.models.create(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n trust_remote_code=False,\n )\nexcept ConflictError:\n client.models.update(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n trust_remote_code=False,\n )\n base_model = client.models.retrieve(workspace="default", name=MODEL_NAME)\n\nprint(f"Base model fileset: fileset://default/{base_model.name}")\nprint(client.files.list(fileset=MODEL_NAME, workspace="default"))\n\ntime_check = max_wait_time_checker(600, "Model Spec")\nwhile not base_model.spec:\n time_check()\n time.sleep(10)\n base_model = client.models.retrieve(workspace="default", name=MODEL_NAME)\n\n# Clear verbose linear_layers list for cleaner output\nbase_model.spec.linear_layers = None\nprint(f"ModelSpec: {base_model.spec}")\n" + }, + { + "type": "markdown", + "source": "### 6. Create LoRA Customization Job\n\nSubmit a customization job with `training=SftTrainingParam(type=\"sft\", peft=LoRaParamsParam(type=\"lora\"), ...)`. Set `lora_enabled=True` in the `deployment_config` so the platform can deploy the base model with LoRA support automatically.\n\nWhen LoRA Enabled is set to true for Models Deployed via the `/apis/models` endpoint or via the `deployment_config` option during the customization job, all LoRA adapters (enabled by default) will get automatically deployed in the NIM.", + "source_html": "

6. Create LoRA Customization Job

\n

Submit a customization job with training=SftTrainingParam(type="sft", peft=LoRaParamsParam(type="lora"), ...). Set lora_enabled=True in the deployment_config so the platform can deploy the base model with LoRA support automatically.

\n

When LoRA Enabled is set to true for Models Deployed via the /apis/models endpoint or via the deployment_config option during the customization job, all LoRA adapters (enabled by default) will get automatically deployed in the NIM.

\n" + }, + { + "type": "code", + "source": "job_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f\"my-sft-job-{job_suffix}\"\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=2,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n context_parallel_size=1,\n expert_parallel_size=1,\n ),\n micro_batch_size=1,\n peft=LoRaParamsParam(type=\"lora\"),\n ),\n deployment_config=DeploymentParamsParam(\n lora_enabled=True,\n gpu=1,\n additional_envs={\"NIM_MODEL_PROFILE\": \"vllm-lora\"},\n ),\n ),\n)\nprint(f\"Job ID: {job.name}\")\nprint(f\"Output model: {job.spec.output.name}\")", + "language": "python", + "source_html": "job_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f"my-sft-job-{job_suffix}"\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace="default",\n spec=CustomizationJobInputParam(\n model=f"default/{base_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=2,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n context_parallel_size=1,\n expert_parallel_size=1,\n ),\n micro_batch_size=1,\n peft=LoRaParamsParam(type="lora"),\n ),\n deployment_config=DeploymentParamsParam(\n lora_enabled=True,\n gpu=1,\n additional_envs={"NIM_MODEL_PROFILE": "vllm-lora"},\n ),\n ),\n)\nprint(f"Job ID: {job.name}")\nprint(f"Output model: {job.spec.output.name}")\n" + }, + { + "type": "markdown", + "source": "### 7. Track Training Progress\n\nPoll job status until it completes. Progress (step/max_steps) is shown when available.", + "source_html": "

7. Track Training Progress

\n

Poll job status until it completes. Progress (step/max_steps) is shown when available.

\n" + }, + { + "type": "code", + "source": "from IPython.display import clear_output\n\ntime_check = max_wait_time_checker(3600, \"Customization Job\")\nwhile True:\n time_check()\n status = client.customization.jobs.get_status(name=job.name, workspace=\"default\")\n clear_output(wait=True)\n print(f\"Job Status: {status.status}\")\n step = max_steps = training_phase = None\n for job_step in status.steps or []:\n if job_step.name == \"customization-training-job\":\n for task in job_step.tasks or []:\n d = task.status_details or {}\n step, max_steps = d.get(\"step\"), d.get(\"max_steps\")\n training_phase = d.get(\"phase\")\n break\n break\n if step is not None and max_steps is not None:\n print(f\"Training: Step {step}/{max_steps} ({100 * step / max_steps:.1f}%)\")\n if training_phase:\n print(f\"Phase: {training_phase}\")\n if status.status in (\"completed\", \"failed\", \"cancelled\", \"error\"):\n print(f\"\\nJob finished: {status.status}\")\n break\n time.sleep(10)\n\nassert status.status == \"completed\"", + "language": "python", + "source_html": "from IPython.display import clear_output\n\ntime_check = max_wait_time_checker(3600, "Customization Job")\nwhile True:\n time_check()\n status = client.customization.jobs.get_status(name=job.name, workspace="default")\n clear_output(wait=True)\n print(f"Job Status: {status.status}")\n step = max_steps = training_phase = None\n for job_step in status.steps or []:\n if job_step.name == "customization-training-job":\n for task in job_step.tasks or []:\n d = task.status_details or {}\n step, max_steps = d.get("step"), d.get("max_steps")\n training_phase = d.get("phase")\n break\n break\n if step is not None and max_steps is not None:\n print(f"Training: Step {step}/{max_steps} ({100 * step / max_steps:.1f}%)")\n if training_phase:\n print(f"Phase: {training_phase}")\n if status.status in ("completed", "failed", "cancelled", "error"):\n print(f"\\nJob finished: {status.status}")\n break\n time.sleep(10)\n\nassert status.status == "completed"\n" + }, + { + "type": "markdown", + "source": "### 8. Validate Output Model and Deployment\n\nWith `deployment_config` configured, the platform will create a NIM deployment for the base model after training. The fine-tuned LoRA adapter is enabled by default and automatically served through the deployment. Check the model entity and deployment status.", + "source_html": "

8. Validate Output Model and Deployment

\n

With deployment_config configured, the platform will create a NIM deployment for the base model after training. The fine-tuned LoRA adapter is enabled by default and automatically served through the deployment. Check the model entity and deployment status.

\n" + }, + { + "type": "code", + "source": "model_entity = client.models.retrieve(workspace=\"default\", name=MODEL_NAME)\n# Clear verbose linear_layers list for cleaner output\nmodel_entity.spec.linear_layers = None\nprint(model_entity.model_dump_json(indent=2))\n\ndeployment_name = sanitize_name(\"sft-deploy\", job.spec.model)\ndeployment_status = client.inference.deployments.retrieve(name=deployment_name, workspace=\"default\")\nprint(f\"Deployment status: {deployment_status.status}\")", + "language": "python", + "source_html": "model_entity = client.models.retrieve(workspace="default", name=MODEL_NAME)\n# Clear verbose linear_layers list for cleaner output\nmodel_entity.spec.linear_layers = None\nprint(model_entity.model_dump_json(indent=2))\n\ndeployment_name = sanitize_name("sft-deploy", job.spec.model)\ndeployment_status = client.inference.deployments.retrieve(name=deployment_name, workspace="default")\nprint(f"Deployment status: {deployment_status.status}")\n" + }, + { + "type": "markdown", + "source": "### 9. Monitor Deployment Until Ready\n\nWait for the deployment to reach RUNNING/READY before sending inference requests.", + "source_html": "

9. Monitor Deployment Until Ready

\n

Wait for the deployment to reach RUNNING/READY before sending inference requests.

\n" + }, + { + "type": "code", + "source": "TIMEOUT_MINUTES = 30\nstart_time = time.time()\ntime_check = max_wait_time_checker(TIMEOUT_MINUTES * 60, \"Deployment\")\nprint(f\"Monitoring deployment '{deployment_name}'... (timeout {TIMEOUT_MINUTES} min)\\n\")\n\nwhile True:\n time.sleep(15)\n time_check()\n deployment_status = client.inference.deployments.retrieve(name=deployment_name, workspace=\"default\")\n elapsed = time.time() - start_time\n clear_output(wait=True)\n print(f\"Deployment: {deployment_name}\")\n print(f\"Status: {deployment_status.status}\")\n print(f\"Elapsed: {int(elapsed // 60)}m {int(elapsed % 60)}s\")\n if deployment_status.status in (\"RUNNING\", \"READY\"):\n print(\"\\nDeployment is ready!\")\n if not client.models.wait_for_gateway(deployment_name, workspace=\"default\", timeout=60):\n raise RuntimeError(\"Inference gateway did not become ready\")\n break\n if deployment_status.status in (\"FAILED\", \"ERROR\", \"TERMINATED\"):\n raise RuntimeError(f\"Deployment failed with status: {deployment_status.status}\")\n\nassert deployment_status.status in (\"RUNNING\", \"READY\")", + "language": "python", + "source_html": "TIMEOUT_MINUTES = 30\nstart_time = time.time()\ntime_check = max_wait_time_checker(TIMEOUT_MINUTES * 60, "Deployment")\nprint(f"Monitoring deployment '{deployment_name}'... (timeout {TIMEOUT_MINUTES} min)\\n")\n\nwhile True:\n time.sleep(15)\n time_check()\n deployment_status = client.inference.deployments.retrieve(name=deployment_name, workspace="default")\n elapsed = time.time() - start_time\n clear_output(wait=True)\n print(f"Deployment: {deployment_name}")\n print(f"Status: {deployment_status.status}")\n print(f"Elapsed: {int(elapsed // 60)}m {int(elapsed % 60)}s")\n if deployment_status.status in ("RUNNING", "READY"):\n print("\\nDeployment is ready!")\n if not client.models.wait_for_gateway(deployment_name, workspace="default", timeout=60):\n raise RuntimeError("Inference gateway did not become ready")\n break\n if deployment_status.status in ("FAILED", "ERROR", "TERMINATED"):\n raise RuntimeError(f"Deployment failed with status: {deployment_status.status}")\n\nassert deployment_status.status in ("RUNNING", "READY")\n" + }, + { + "type": "markdown", + "source": "### 10. Check Model Output\n\nSend a chat completion request to the deployed LoRA model and compare the output to the expected answer.", + "source_html": "

10. Check Model Output

\n

Send a chat completion request to the deployed LoRA model and compare the output to the expected answer.

\n" + }, + { + "type": "code", + "source": "context = \"The Apollo 11 mission was the first manned mission to land on the Moon. It was launched on July 16, 1969, and Neil Armstrong became the first person to walk on the lunar surface on July 20, 1969. Buzz Aldrin joined him shortly after, while Michael Collins remained in lunar orbit.\"\nquestion = \"Who was the first person to walk on the Moon?\"\nmessages = [\n {\"role\": \"user\", \"content\": f\"Based on the following context, answer the question.\\n\\nContext: {context}\\n\\nQuestion: {question}\"}\n]\nresponse = client.inference.gateway.provider.post(\n \"v1/chat/completions\",\n name=deployment_name,\n workspace=\"default\",\n body={\n \"model\": job.spec.output.name,\n \"messages\": messages,\n \"temperature\": 0,\n \"max_tokens\": 256,\n }\n)\nprint(\"=\" * 60)\nprint(\"MODEL INFERENCE\")\nprint(\"=\" * 60)\nprint(f\"Question: {question}\")\nprint(f\"Expected: Neil Armstrong\")\nprint(f\"Model output: {response['choices'][0]['message']['content']}\")", + "language": "python", + "source_html": "context = "The Apollo 11 mission was the first manned mission to land on the Moon. It was launched on July 16, 1969, and Neil Armstrong became the first person to walk on the lunar surface on July 20, 1969. Buzz Aldrin joined him shortly after, while Michael Collins remained in lunar orbit."\nquestion = "Who was the first person to walk on the Moon?"\nmessages = [\n {"role": "user", "content": f"Based on the following context, answer the question.\\n\\nContext: {context}\\n\\nQuestion: {question}"}\n]\nresponse = client.inference.gateway.provider.post(\n "v1/chat/completions",\n name=deployment_name,\n workspace="default",\n body={\n "model": job.spec.output.name,\n "messages": messages,\n "temperature": 0,\n "max_tokens": 256,\n }\n)\nprint("=" * 60)\nprint("MODEL INFERENCE")\nprint("=" * 60)\nprint(f"Question: {question}")\nprint(f"Expected: Neil Armstrong")\nprint(f"Model output: {response['choices'][0]['message']['content']}")\n" + }, + { + "type": "markdown", + "source": "## Conclusion\n\nYou have started a LoRA customization job, monitored it to completion, and evaluated the fine-tuned model. Use the `output.name` to access the model for further inference or evaluation.\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Try [Full SFT](./sft-customization-job) or [DPO](./dpo-customization-job) for other customization options", + "source_html": "

Conclusion

\n

You have started a LoRA customization job, monitored it to completion, and evaluated the fine-tuned model. Use the output.name to access the model for further inference or evaluation.

\n

Next Steps

\n\n" + } + ] +} \ No newline at end of file diff --git a/docs/fern/components/notebooks/lora-customization-job.ts b/docs/fern/components/notebooks/lora-customization-job.ts new file mode 100644 index 00000000000..f3ff2f0823c --- /dev/null +++ b/docs/fern/components/notebooks/lora-customization-job.ts @@ -0,0 +1,138 @@ +/** Auto-generated by ipynb-to-fern-json.py - do not edit */ +export default { cells: [ + { + "type": "markdown", + "source": "\n\n\n# LoRA Model Customization Job\n\nLearn how to use the NeMo Platform to create a LoRA (Low-Rank Adaptation) customization job using a custom dataset. In this tutorial we use LoRA to fine-tune a **question-answering model** from the SQuAD dataset.\n\nLoRA is a parameter-efficient fine-tuning method that requires fewer computational resources than full fine-tuning. If you need full model fine-tuning instead, see the [Full SFT Customization Job](./sft-customization-job) tutorial.\n\n**Time to complete:** approximately 45 minutes. Job duration increases with model size and dataset size.", + "source_html": "\n\n

LoRA Model Customization Job

\n

Learn how to use the NeMo Platform to create a LoRA (Low-Rank Adaptation) customization job using a custom dataset. In this tutorial we use LoRA to fine-tune a question-answering model from the SQuAD dataset.

\n

LoRA is a parameter-efficient fine-tuning method that requires fewer computational resources than full fine-tuning. If you need full model fine-tuning instead, see the Full SFT Customization Job tutorial.

\n

Time to complete: approximately 45 minutes. Job duration increases with model size and dataset size.

\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n2. **Installed the Python SDK** (included with `pip install nemo-platform`)\n3. **Installed the `datasets` package** for loading SQuAD: `pip install datasets`\n4. **At least one GPU with CUDA 12.8+**", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Completed the Quickstart to install and deploy NeMo Platform locally
    2. \n
  2. Installed the Python SDK (included with pip install nemo-platform)
    3. \n
  3. Installed the datasets package for loading SQuAD: pip install datasets
    4. \n
  4. At least one GPU with CUDA 12.8+
    5. \n
\n" + }, + { + "type": "markdown", + "source": "## Quick Start\n\n### 1. Initialize SDK\n\nThe SDK needs to know your NeMo Platform server URL. By default, `http://localhost:8080` is used. If NeMo Platform is running elsewhere, set the `NMP_BASE_URL` environment variable:\n\n```sh\nexport NMP_BASE_URL=\n```", + "source_html": "

Quick Start

\n

1. Initialize SDK

\n

The SDK needs to know your NeMo Platform server URL. By default, http://localhost:8080 is used. If NeMo Platform is running elsewhere, set the NMP_BASE_URL environment variable:

\n
export NMP_BASE_URL=<YOUR_NMP_BASE_URL>\n
\n" + }, + { + "type": "code", + "source": "import os\nimport json\nimport re\nimport time\nimport uuid\nfrom pathlib import Path\nfrom nemo_platform import NeMoPlatform, ConflictError\nfrom nemo_platform.types.secrets import PlatformSecretResponse\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n DeploymentParamsParam,\n LoRaParamsParam,\n ParallelismParamsParam,\n SftTrainingParam,\n)\n\n\ndef sanitize_name(prefix: str, name: str):\n \"\"\"Sanitize model_name for deployment/config naming. Compatible with platform naming rules.\"\"\"\n name = name.split(\"/\")[-1]\n sanitized = re.sub(r\"[^a-z0-9@.+_-]\", \"-\", name.lower())\n sanitized = re.sub(r\"-+\", \"-\", sanitized).strip(\"-\")\n return f\"{prefix}-{sanitized}\"[:59].rstrip(\"-\")\n\n\ndef max_wait_time_checker(seconds: int, job_name: str = \"\"):\n \"\"\"Return a check() that raises TimeoutError if called after `seconds` have elapsed.\"\"\"\n start_time = time.time()\n\n def check():\n if time.time() - start_time > seconds:\n raise TimeoutError(f\"{job_name} took longer than {seconds} seconds\")\n\n return check\n\n\nNMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace=\"default\"\n)", + "language": "python", + "source_html": "import os\nimport json\nimport re\nimport time\nimport uuid\nfrom pathlib import Path\nfrom nemo_platform import NeMoPlatform, ConflictError\nfrom nemo_platform.types.secrets import PlatformSecretResponse\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n DeploymentParamsParam,\n LoRaParamsParam,\n ParallelismParamsParam,\n SftTrainingParam,\n)\n\n\ndef sanitize_name(prefix: str, name: str):\n """Sanitize model_name for deployment/config naming. Compatible with platform naming rules."""\n name = name.split("/")[-1]\n sanitized = re.sub(r"[^a-z0-9@.+_-]", "-", name.lower())\n sanitized = re.sub(r"-+", "-", sanitized).strip("-")\n return f"{prefix}-{sanitized}"[:59].rstrip("-")\n\n\ndef max_wait_time_checker(seconds: int, job_name: str = ""):\n """Return a check() that raises TimeoutError if called after `seconds` have elapsed."""\n start_time = time.time()\n\n def check():\n if time.time() - start_time > seconds:\n raise TimeoutError(f"{job_name} took longer than {seconds} seconds")\n\n return check\n\n\nNMP_BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace="default"\n)\n" + }, + { + "type": "markdown", + "source": "### 2. Prepare Dataset\n\nCreate your data in JSONL format (one JSON object per line). For SFT with LoRA, the platform expects **prompt/completion** pairs.\n\n**Dataset structure:**\n- Training files under `training/` (or root with `training.jsonl`)\n- Validation files under `validation/` (or `validation.jsonl`)\n\n**SFT format:** Each line is a JSON object with:\n- **`prompt`**: The input (e.g. context + question)\n- **`completion`**: The desired model output", + "source_html": "

2. Prepare Dataset

\n

Create your data in JSONL format (one JSON object per line). For SFT with LoRA, the platform expects prompt/completion pairs.

\n

Dataset structure:

\n
    \n
  • Training files under training/ (or root with training.jsonl)
    • \n
  • Validation files under validation/ (or validation.jsonl)
    • \n
\n

SFT format: Each line is a JSON object with:

\n
    \n
  • prompt: The input (e.g. context + question)
    • \n
  • completion: The desired model output
    • \n
\n" + }, + { + "type": "markdown", + "source": "Example record (single line in `.jsonl`):\n\n```json\n{\"prompt\": \"Context: ... Question: What is X? Answer:\", \"completion\": \"X is ...\"}\n```", + "source_html": "

Example record (single line in .jsonl):

\n
{"prompt": "Context: ... Question: What is X? Answer:", "completion": "X is ..."}\n
\n" + }, + { + "type": "markdown", + "source": "#### Download SQuAD and Convert to SFT Format\n\nWe use the [SQuAD](https://huggingface.co/datasets/rajpurkar/squad) dataset and convert it to prompt/completion JSONL.", + "source_html": "

Download SQuAD and Convert to SFT Format

\n

We use the SQuAD dataset and convert it to prompt/completion JSONL.

\n" + }, + { + "type": "code", + "source": "from datasets import load_dataset, DatasetDict\n\nprint(\"Loading dataset rajpurkar/squad\")\nraw_dataset = load_dataset(\"rajpurkar/squad\")\nif not isinstance(raw_dataset, DatasetDict):\n raise ValueError(\"Dataset does not contain expected splits\")\nprint(\"Loaded dataset\")\n\nVALIDATION_PROPORTION = 0.05\nSEED = 1234\ntraining_size = 3000\nvalidation_size = 300\nDATASET_NAME = \"sft-dataset\"\nDATASET_PATH = Path(\"sft-dataset\").absolute()\n\nos.makedirs(DATASET_PATH, exist_ok=True)\ntrain_set = raw_dataset.get(\"train\")\nsplit_dataset = train_set.train_test_split(test_size=VALIDATION_PROPORTION, seed=SEED)\ntrain_ds = split_dataset[\"train\"].select(range(min(training_size, len(split_dataset[\"train\"]))))\nvalidation_ds = split_dataset[\"test\"].select(range(min(validation_size, len(split_dataset[\"test\"]))))\n\ndef convert_squad_to_sft_format(example):\n prompt = f\"Context: {example['context']} Question: {example['question']} Answer:\"\n completion = example[\"answers\"][\"text\"][0]\n return {\"prompt\": prompt, \"completion\": completion}\n\nwith open(f\"{DATASET_PATH}/training.jsonl\", \"w\", encoding=\"utf-8\") as f:\n for example in train_ds:\n f.write(json.dumps(convert_squad_to_sft_format(example)) + \"\\n\")\nwith open(f\"{DATASET_PATH}/validation.jsonl\", \"w\", encoding=\"utf-8\") as f:\n for example in validation_ds:\n f.write(json.dumps(convert_squad_to_sft_format(example)) + \"\\n\")\n\nprint(f\"Saved training.jsonl with {len(train_ds)} rows\")\nprint(f\"Saved validation.jsonl with {len(validation_ds)} rows\")\nwith open(f\"{DATASET_PATH}/training.jsonl\", \"r\") as f:\n sample = json.loads(f.readline())\nprint(\"Sample prompt (first 200 chars):\", sample[\"prompt\"][:200] + \"...\")\nprint(\"Sample completion:\", sample[\"completion\"])", + "language": "python", + "source_html": "from datasets import load_dataset, DatasetDict\n\nprint("Loading dataset rajpurkar/squad")\nraw_dataset = load_dataset("rajpurkar/squad")\nif not isinstance(raw_dataset, DatasetDict):\n raise ValueError("Dataset does not contain expected splits")\nprint("Loaded dataset")\n\nVALIDATION_PROPORTION = 0.05\nSEED = 1234\ntraining_size = 3000\nvalidation_size = 300\nDATASET_NAME = "sft-dataset"\nDATASET_PATH = Path("sft-dataset").absolute()\n\nos.makedirs(DATASET_PATH, exist_ok=True)\ntrain_set = raw_dataset.get("train")\nsplit_dataset = train_set.train_test_split(test_size=VALIDATION_PROPORTION, seed=SEED)\ntrain_ds = split_dataset["train"].select(range(min(training_size, len(split_dataset["train"]))))\nvalidation_ds = split_dataset["test"].select(range(min(validation_size, len(split_dataset["test"]))))\n\ndef convert_squad_to_sft_format(example):\n prompt = f"Context: {example['context']} Question: {example['question']} Answer:"\n completion = example["answers"]["text"][0]\n return {"prompt": prompt, "completion": completion}\n\nwith open(f"{DATASET_PATH}/training.jsonl", "w", encoding="utf-8") as f:\n for example in train_ds:\n f.write(json.dumps(convert_squad_to_sft_format(example)) + "\\n")\nwith open(f"{DATASET_PATH}/validation.jsonl", "w", encoding="utf-8") as f:\n for example in validation_ds:\n f.write(json.dumps(convert_squad_to_sft_format(example)) + "\\n")\n\nprint(f"Saved training.jsonl with {len(train_ds)} rows")\nprint(f"Saved validation.jsonl with {len(validation_ds)} rows")\nwith open(f"{DATASET_PATH}/training.jsonl", "r") as f:\n sample = json.loads(f.readline())\nprint("Sample prompt (first 200 chars):", sample["prompt"][:200] + "...")\nprint("Sample completion:", sample["completion"])\n" + }, + { + "type": "markdown", + "source": "### 3. Create FileSet and Upload Training Data\n\nUpload the training and validation JSONL files to a FileSet so the customization job can use them.", + "source_html": "

3. Create FileSet and Upload Training Data

\n

Upload the training and validation JSONL files to a FileSet so the customization job can use them.

\n" + }, + { + "type": "code", + "source": "try:\n client.files.filesets.create(\n workspace=\"default\",\n name=DATASET_NAME,\n description=\"SFT training data\",\n cache=True,\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\nclient.files.fsspec.put(\n lpath=DATASET_PATH,\n rpath=f\"default/{DATASET_NAME}/\",\n recursive=True\n)\nprint(\"Training data:\")\nprint(client.files.list(fileset=DATASET_NAME, workspace=\"default\"))", + "language": "python", + "source_html": "try:\n client.files.filesets.create(\n workspace="default",\n name=DATASET_NAME,\n description="SFT training data",\n cache=True,\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\nclient.files.fsspec.put(\n lpath=DATASET_PATH,\n rpath=f"default/{DATASET_NAME}/",\n recursive=True\n)\nprint("Training data:")\nprint(client.files.list(fileset=DATASET_NAME, workspace="default"))\n" + }, + { + "type": "markdown", + "source": "### 4. Secrets Setup\n\nFor Huggingface models that require authentication, create a secret with your HF token. Get a token from [Huggingface Settings](https://huggingface.co/settings/tokens) and accept the model terms.\n\nThis is generally true for LLaMa based models (e.g. [Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct)).\n\n```sh\nexport HF_TOKEN=\n```", + "source_html": "

4. Secrets Setup

\n

For Huggingface models that require authentication, create a secret with your HF token. Get a token from Huggingface Settings and accept the model terms.

\n

This is generally true for LLaMa based models (e.g. Llama-3.2-1B-Instruct).

\n
export HF_TOKEN=<your-huggingface-token>\n
\n" + }, + { + "type": "code", + "source": "HF_TOKEN = os.getenv(\"HF_TOKEN\")\n\ndef create_or_get_secret(name: str, value: str | None, label: str) -> PlatformSecretResponse | None:\n if not value:\n print(f\"{label} is not set - skipping setting secret\")\n return None\n try:\n secret = client.secrets.create(name=name, workspace=\"default\", value=value)\n print(f\"Created secret: {name}\")\n print(secret.model_dump_json(indent=2))\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n secret = client.secrets.retrieve(name=name, workspace=\"default\")\n print(secret.model_dump_json(indent=2))\n return secret\n\n\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")", + "language": "python", + "source_html": "HF_TOKEN = os.getenv("HF_TOKEN")\n\ndef create_or_get_secret(name: str, value: str | None, label: str) -> PlatformSecretResponse | None:\n if not value:\n print(f"{label} is not set - skipping setting secret")\n return None\n try:\n secret = client.secrets.create(name=name, workspace="default", value=value)\n print(f"Created secret: {name}")\n print(secret.model_dump_json(indent=2))\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n secret = client.secrets.retrieve(name=name, workspace="default")\n print(secret.model_dump_json(indent=2))\n return secret\n\n\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\n" + }, + { + "type": "markdown", + "source": "### 5. Create Base Model FileSet and Model Entity\n\nCreate a fileset pointing to [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) and a Model Entity that references it. Model download happens when the customization job runs.", + "source_html": "

5. Create Base Model FileSet and Model Entity

\n

Create a fileset pointing to Qwen/Qwen3-0.6B and a Model Entity that references it. Model download happens when the customization job runs.

\n" + }, + { + "type": "code", + "source": "HF_REPO_ID = \"Qwen/Qwen3-0.6B\"\nMODEL_NAME = \"qwen3-0.6b\"\n\ntry:\n storage = HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n repo_id=HF_REPO_ID,\n repo_type=\"model\",\n )\n if hf_secret:\n storage[\"token_secret\"] = hf_secret.name\n base_model_fs = client.files.filesets.create(\n workspace=\"default\",\n name=MODEL_NAME,\n description=\"Qwen3 0.6b base model from Huggingface\",\n storage=storage,\n cache=True,\n )\nexcept ConflictError:\n base_model_fs = client.files.filesets.retrieve(workspace=\"default\", name=MODEL_NAME)\n\ntry:\n base_model = client.models.create(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n trust_remote_code=False,\n )\nexcept ConflictError:\n client.models.update(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n trust_remote_code=False,\n )\n base_model = client.models.retrieve(workspace=\"default\", name=MODEL_NAME)\n\nprint(f\"Base model fileset: fileset://default/{base_model.name}\")\nprint(client.files.list(fileset=MODEL_NAME, workspace=\"default\"))\n\ntime_check = max_wait_time_checker(600, \"Model Spec\")\nwhile not base_model.spec:\n time_check()\n time.sleep(10)\n base_model = client.models.retrieve(workspace=\"default\", name=MODEL_NAME)\n\n# Clear verbose linear_layers list for cleaner output\nbase_model.spec.linear_layers = None\nprint(f\"ModelSpec: {base_model.spec}\")", + "language": "python", + "source_html": "HF_REPO_ID = "Qwen/Qwen3-0.6B"\nMODEL_NAME = "qwen3-0.6b"\n\ntry:\n storage = HuggingfaceStorageConfigParam(\n type="huggingface",\n repo_id=HF_REPO_ID,\n repo_type="model",\n )\n if hf_secret:\n storage["token_secret"] = hf_secret.name\n base_model_fs = client.files.filesets.create(\n workspace="default",\n name=MODEL_NAME,\n description="Qwen3 0.6b base model from Huggingface",\n storage=storage,\n cache=True,\n )\nexcept ConflictError:\n base_model_fs = client.files.filesets.retrieve(workspace="default", name=MODEL_NAME)\n\ntry:\n base_model = client.models.create(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n trust_remote_code=False,\n )\nexcept ConflictError:\n client.models.update(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n trust_remote_code=False,\n )\n base_model = client.models.retrieve(workspace="default", name=MODEL_NAME)\n\nprint(f"Base model fileset: fileset://default/{base_model.name}")\nprint(client.files.list(fileset=MODEL_NAME, workspace="default"))\n\ntime_check = max_wait_time_checker(600, "Model Spec")\nwhile not base_model.spec:\n time_check()\n time.sleep(10)\n base_model = client.models.retrieve(workspace="default", name=MODEL_NAME)\n\n# Clear verbose linear_layers list for cleaner output\nbase_model.spec.linear_layers = None\nprint(f"ModelSpec: {base_model.spec}")\n" + }, + { + "type": "markdown", + "source": "### 6. Create LoRA Customization Job\n\nSubmit a customization job with `training=SftTrainingParam(type=\"sft\", peft=LoRaParamsParam(type=\"lora\"), ...)`. Set `lora_enabled=True` in the `deployment_config` so the platform can deploy the base model with LoRA support automatically.\n\nWhen LoRA Enabled is set to true for Models Deployed via the `/apis/models` endpoint or via the `deployment_config` option during the customization job, all LoRA adapters (enabled by default) will get automatically deployed in the NIM.", + "source_html": "

6. Create LoRA Customization Job

\n

Submit a customization job with training=SftTrainingParam(type="sft", peft=LoRaParamsParam(type="lora"), ...). Set lora_enabled=True in the deployment_config so the platform can deploy the base model with LoRA support automatically.

\n

When LoRA Enabled is set to true for Models Deployed via the /apis/models endpoint or via the deployment_config option during the customization job, all LoRA adapters (enabled by default) will get automatically deployed in the NIM.

\n" + }, + { + "type": "code", + "source": "job_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f\"my-sft-job-{job_suffix}\"\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=2,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n context_parallel_size=1,\n expert_parallel_size=1,\n ),\n micro_batch_size=1,\n peft=LoRaParamsParam(type=\"lora\"),\n ),\n deployment_config=DeploymentParamsParam(\n lora_enabled=True,\n gpu=1,\n additional_envs={\"NIM_MODEL_PROFILE\": \"vllm-lora\"},\n ),\n ),\n)\nprint(f\"Job ID: {job.name}\")\nprint(f\"Output model: {job.spec.output.name}\")", + "language": "python", + "source_html": "job_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f"my-sft-job-{job_suffix}"\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace="default",\n spec=CustomizationJobInputParam(\n model=f"default/{base_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=2,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n context_parallel_size=1,\n expert_parallel_size=1,\n ),\n micro_batch_size=1,\n peft=LoRaParamsParam(type="lora"),\n ),\n deployment_config=DeploymentParamsParam(\n lora_enabled=True,\n gpu=1,\n additional_envs={"NIM_MODEL_PROFILE": "vllm-lora"},\n ),\n ),\n)\nprint(f"Job ID: {job.name}")\nprint(f"Output model: {job.spec.output.name}")\n" + }, + { + "type": "markdown", + "source": "### 7. Track Training Progress\n\nPoll job status until it completes. Progress (step/max_steps) is shown when available.", + "source_html": "

7. Track Training Progress

\n

Poll job status until it completes. Progress (step/max_steps) is shown when available.

\n" + }, + { + "type": "code", + "source": "from IPython.display import clear_output\n\ntime_check = max_wait_time_checker(3600, \"Customization Job\")\nwhile True:\n time_check()\n status = client.customization.jobs.get_status(name=job.name, workspace=\"default\")\n clear_output(wait=True)\n print(f\"Job Status: {status.status}\")\n step = max_steps = training_phase = None\n for job_step in status.steps or []:\n if job_step.name == \"customization-training-job\":\n for task in job_step.tasks or []:\n d = task.status_details or {}\n step, max_steps = d.get(\"step\"), d.get(\"max_steps\")\n training_phase = d.get(\"phase\")\n break\n break\n if step is not None and max_steps is not None:\n print(f\"Training: Step {step}/{max_steps} ({100 * step / max_steps:.1f}%)\")\n if training_phase:\n print(f\"Phase: {training_phase}\")\n if status.status in (\"completed\", \"failed\", \"cancelled\", \"error\"):\n print(f\"\\nJob finished: {status.status}\")\n break\n time.sleep(10)\n\nassert status.status == \"completed\"", + "language": "python", + "source_html": "from IPython.display import clear_output\n\ntime_check = max_wait_time_checker(3600, "Customization Job")\nwhile True:\n time_check()\n status = client.customization.jobs.get_status(name=job.name, workspace="default")\n clear_output(wait=True)\n print(f"Job Status: {status.status}")\n step = max_steps = training_phase = None\n for job_step in status.steps or []:\n if job_step.name == "customization-training-job":\n for task in job_step.tasks or []:\n d = task.status_details or {}\n step, max_steps = d.get("step"), d.get("max_steps")\n training_phase = d.get("phase")\n break\n break\n if step is not None and max_steps is not None:\n print(f"Training: Step {step}/{max_steps} ({100 * step / max_steps:.1f}%)")\n if training_phase:\n print(f"Phase: {training_phase}")\n if status.status in ("completed", "failed", "cancelled", "error"):\n print(f"\\nJob finished: {status.status}")\n break\n time.sleep(10)\n\nassert status.status == "completed"\n" + }, + { + "type": "markdown", + "source": "### 8. Validate Output Model and Deployment\n\nWith `deployment_config` configured, the platform will create a NIM deployment for the base model after training. The fine-tuned LoRA adapter is enabled by default and automatically served through the deployment. Check the model entity and deployment status.", + "source_html": "

8. Validate Output Model and Deployment

\n

With deployment_config configured, the platform will create a NIM deployment for the base model after training. The fine-tuned LoRA adapter is enabled by default and automatically served through the deployment. Check the model entity and deployment status.

\n" + }, + { + "type": "code", + "source": "model_entity = client.models.retrieve(workspace=\"default\", name=MODEL_NAME)\n# Clear verbose linear_layers list for cleaner output\nmodel_entity.spec.linear_layers = None\nprint(model_entity.model_dump_json(indent=2))\n\ndeployment_name = sanitize_name(\"sft-deploy\", job.spec.model)\ndeployment_status = client.inference.deployments.retrieve(name=deployment_name, workspace=\"default\")\nprint(f\"Deployment status: {deployment_status.status}\")", + "language": "python", + "source_html": "model_entity = client.models.retrieve(workspace="default", name=MODEL_NAME)\n# Clear verbose linear_layers list for cleaner output\nmodel_entity.spec.linear_layers = None\nprint(model_entity.model_dump_json(indent=2))\n\ndeployment_name = sanitize_name("sft-deploy", job.spec.model)\ndeployment_status = client.inference.deployments.retrieve(name=deployment_name, workspace="default")\nprint(f"Deployment status: {deployment_status.status}")\n" + }, + { + "type": "markdown", + "source": "### 9. Monitor Deployment Until Ready\n\nWait for the deployment to reach RUNNING/READY before sending inference requests.", + "source_html": "

9. Monitor Deployment Until Ready

\n

Wait for the deployment to reach RUNNING/READY before sending inference requests.

\n" + }, + { + "type": "code", + "source": "TIMEOUT_MINUTES = 30\nstart_time = time.time()\ntime_check = max_wait_time_checker(TIMEOUT_MINUTES * 60, \"Deployment\")\nprint(f\"Monitoring deployment '{deployment_name}'... (timeout {TIMEOUT_MINUTES} min)\\n\")\n\nwhile True:\n time.sleep(15)\n time_check()\n deployment_status = client.inference.deployments.retrieve(name=deployment_name, workspace=\"default\")\n elapsed = time.time() - start_time\n clear_output(wait=True)\n print(f\"Deployment: {deployment_name}\")\n print(f\"Status: {deployment_status.status}\")\n print(f\"Elapsed: {int(elapsed // 60)}m {int(elapsed % 60)}s\")\n if deployment_status.status in (\"RUNNING\", \"READY\"):\n print(\"\\nDeployment is ready!\")\n if not client.models.wait_for_gateway(deployment_name, workspace=\"default\", timeout=60):\n raise RuntimeError(\"Inference gateway did not become ready\")\n break\n if deployment_status.status in (\"FAILED\", \"ERROR\", \"TERMINATED\"):\n raise RuntimeError(f\"Deployment failed with status: {deployment_status.status}\")\n\nassert deployment_status.status in (\"RUNNING\", \"READY\")", + "language": "python", + "source_html": "TIMEOUT_MINUTES = 30\nstart_time = time.time()\ntime_check = max_wait_time_checker(TIMEOUT_MINUTES * 60, "Deployment")\nprint(f"Monitoring deployment '{deployment_name}'... (timeout {TIMEOUT_MINUTES} min)\\n")\n\nwhile True:\n time.sleep(15)\n time_check()\n deployment_status = client.inference.deployments.retrieve(name=deployment_name, workspace="default")\n elapsed = time.time() - start_time\n clear_output(wait=True)\n print(f"Deployment: {deployment_name}")\n print(f"Status: {deployment_status.status}")\n print(f"Elapsed: {int(elapsed // 60)}m {int(elapsed % 60)}s")\n if deployment_status.status in ("RUNNING", "READY"):\n print("\\nDeployment is ready!")\n if not client.models.wait_for_gateway(deployment_name, workspace="default", timeout=60):\n raise RuntimeError("Inference gateway did not become ready")\n break\n if deployment_status.status in ("FAILED", "ERROR", "TERMINATED"):\n raise RuntimeError(f"Deployment failed with status: {deployment_status.status}")\n\nassert deployment_status.status in ("RUNNING", "READY")\n" + }, + { + "type": "markdown", + "source": "### 10. Check Model Output\n\nSend a chat completion request to the deployed LoRA model and compare the output to the expected answer.", + "source_html": "

10. Check Model Output

\n

Send a chat completion request to the deployed LoRA model and compare the output to the expected answer.

\n" + }, + { + "type": "code", + "source": "context = \"The Apollo 11 mission was the first manned mission to land on the Moon. It was launched on July 16, 1969, and Neil Armstrong became the first person to walk on the lunar surface on July 20, 1969. Buzz Aldrin joined him shortly after, while Michael Collins remained in lunar orbit.\"\nquestion = \"Who was the first person to walk on the Moon?\"\nmessages = [\n {\"role\": \"user\", \"content\": f\"Based on the following context, answer the question.\\n\\nContext: {context}\\n\\nQuestion: {question}\"}\n]\nresponse = client.inference.gateway.provider.post(\n \"v1/chat/completions\",\n name=deployment_name,\n workspace=\"default\",\n body={\n \"model\": job.spec.output.name,\n \"messages\": messages,\n \"temperature\": 0,\n \"max_tokens\": 256,\n }\n)\nprint(\"=\" * 60)\nprint(\"MODEL INFERENCE\")\nprint(\"=\" * 60)\nprint(f\"Question: {question}\")\nprint(f\"Expected: Neil Armstrong\")\nprint(f\"Model output: {response['choices'][0]['message']['content']}\")", + "language": "python", + "source_html": "context = "The Apollo 11 mission was the first manned mission to land on the Moon. It was launched on July 16, 1969, and Neil Armstrong became the first person to walk on the lunar surface on July 20, 1969. Buzz Aldrin joined him shortly after, while Michael Collins remained in lunar orbit."\nquestion = "Who was the first person to walk on the Moon?"\nmessages = [\n {"role": "user", "content": f"Based on the following context, answer the question.\\n\\nContext: {context}\\n\\nQuestion: {question}"}\n]\nresponse = client.inference.gateway.provider.post(\n "v1/chat/completions",\n name=deployment_name,\n workspace="default",\n body={\n "model": job.spec.output.name,\n "messages": messages,\n "temperature": 0,\n "max_tokens": 256,\n }\n)\nprint("=" * 60)\nprint("MODEL INFERENCE")\nprint("=" * 60)\nprint(f"Question: {question}")\nprint(f"Expected: Neil Armstrong")\nprint(f"Model output: {response['choices'][0]['message']['content']}")\n" + }, + { + "type": "markdown", + "source": "## Conclusion\n\nYou have started a LoRA customization job, monitored it to completion, and evaluated the fine-tuned model. Use the `output.name` to access the model for further inference or evaluation.\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Try [Full SFT](./sft-customization-job) or [DPO](./dpo-customization-job) for other customization options", + "source_html": "

Conclusion

\n

You have started a LoRA customization job, monitored it to completion, and evaluated the fine-tuned model. Use the output.name to access the model for further inference or evaluation.

\n

Next Steps

\n\n" + } +] }; diff --git a/docs/fern/components/notebooks/optimize-throughput.json b/docs/fern/components/notebooks/optimize-throughput.json new file mode 100644 index 00000000000..17d520aa81f --- /dev/null +++ b/docs/fern/components/notebooks/optimize-throughput.json @@ -0,0 +1,156 @@ +{ + "cells": [ + { + "type": "markdown", + "source": "\n\n\n# Optimize for Tokens/GPU Throughput\n\n## About\nLearn how to use the NeMo Platform Customizer to create a [LoRA](nemo-ms-about-concepts-customization) (Low-Rank Adaptation) customization job optimized for higher tokens/GPU throughput and lower runtime. \n\n**In this tutorial, you will:**\n1. Fine-tune [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) on the SQuAD dataset using LoRA, with [sequence packing](nemo-ms-about-concepts-customization) enabled for one run and disabled for another.\n2. Compare training runtime, GPU utilization, and memory allocation between the two runs.\n3. Verify that validation loss remains comparable, confirming that sequence packing improves throughput without sacrificing model quality.\n\n> **Note:** While this tutorial demonstrates sequence packing with LoRA, the optimization is also available for all_weights (full) SFT customization jobs.", + "source_html": "\n\n

Optimize for Tokens/GPU Throughput

\n

About

\n

Learn how to use the NeMo Platform Customizer to create a LoRA (Low-Rank Adaptation) customization job optimized for higher tokens/GPU throughput and lower runtime.

\n

In this tutorial, you will:

\n
    \n
  1. Fine-tune meta-llama/Llama-3.2-1B-Instruct on the SQuAD dataset using LoRA, with sequence packing enabled for one run and disabled for another.
    2. \n
  2. Compare training runtime, GPU utilization, and memory allocation between the two runs.
    3. \n
  3. Verify that validation loss remains comparable, confirming that sequence packing improves throughput without sacrificing model quality.
    4. \n
\n
\n

Note: While this tutorial demonstrates sequence packing with LoRA, the optimization is also available for all_weights (full) SFT customization jobs.

\n
\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n2. **Installed the Python SDK** (included with `pip install nemo-platform`)", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Completed the Quickstart to install and deploy NeMo Platform locally
    2. \n
  2. Installed the Python SDK (included with pip install nemo-platform)
    3. \n
\n" + }, + { + "type": "markdown", + "source": "## Quick Start\n\n### 1. Initialize SDK\n\nThe SDK needs to know your NeMo Platform server URL. By default, `http://localhost:8080` is used in accordance with the [Quickstart](../../get-started/quickstart.md) guide. If NeMo Platform is running at a custom location, you can override the URL by setting the `NMP_BASE_URL` environment variable:\n\n```sh\nexport NMP_BASE_URL=\n```", + "source_html": "

Quick Start

\n

1. Initialize SDK

\n

The SDK needs to know your NeMo Platform server URL. By default, http://localhost:8080 is used in accordance with the Quickstart guide. If NeMo Platform is running at a custom location, you can override the URL by setting the NMP_BASE_URL environment variable:

\n
export NMP_BASE_URL=<YOUR_NMP_BASE_URL>\n
\n" + }, + { + "type": "code", + "source": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace=\"default\"\n)", + "language": "python", + "source_html": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace="default"\n)\n" + }, + { + "type": "markdown", + "source": "### 2. Create Dataset FileSet and Upload Training Data", + "source_html": "

2. Create Dataset FileSet and Upload Training Data

\n" + }, + { + "type": "markdown", + "source": "Install additional dependencies if they are not installed in your Python environment.\n\nThe cell below automatically detects your environment and uses:\n- `uv pip install` if you're in a uv-managed virtual environment\n- `pip install` otherwise\n\nRequired packages:\n- `datasets` - Download the public [rajpurkar/squad](https://huggingface.co/datasets/rajpurkar/squad) dataset\n- `pandas` - Compare job results in table format\n- `matplotlib` - Plot live training metrics (loss curves, GPU utilization)\n- `nvidia-ml-py` - Collect GPU VRAM and compute utilization metrics during training", + "source_html": "

Install additional dependencies if they are not installed in your Python environment.

\n

The cell below automatically detects your environment and uses:

\n
    \n
  • uv pip install if you're in a uv-managed virtual environment
    • \n
  • pip install otherwise
    • \n
\n

Required packages:

\n
    \n
  • datasets - Download the public rajpurkar/squad dataset
    • \n
  • pandas - Compare job results in table format
    • \n
  • matplotlib - Plot live training metrics (loss curves, GPU utilization)
    • \n
  • nvidia-ml-py - Collect GPU VRAM and compute utilization metrics during training
    • \n
\n" + }, + { + "type": "code", + "source": "%%bash\nif command -v uv >/dev/null 2>&1 && [ -n \"$VIRTUAL_ENV\" ]; then\n uv pip install datasets pandas matplotlib nvidia-ml-py\nelse\n pip install datasets pandas matplotlib nvidia-ml-py\nfi", + "language": "python", + "source_html": "%%bash\nif command -v uv >/dev/null 2>&1 && [ -n "$VIRTUAL_ENV" ]; then\n uv pip install datasets pandas matplotlib nvidia-ml-py\nelse\n pip install datasets pandas matplotlib nvidia-ml-py\nfi\n" + }, + { + "type": "markdown", + "source": "#### Download rajpurkar/squad Dataset\n\nSQuAD (Stanford Question Answering Dataset) is a reading comprehension dataset consisting of questions posed on Wikipedia articles, where the answer is a segment of text from the corresponding passage.", + "source_html": "

Download rajpurkar/squad Dataset

\n

SQuAD (Stanford Question Answering Dataset) is a reading comprehension dataset consisting of questions posed on Wikipedia articles, where the answer is a segment of text from the corresponding passage.

\n" + }, + { + "type": "code", + "source": "import json\nimport os\nfrom pathlib import Path\nfrom datasets import load_dataset, Dataset, DatasetDict\n\n# Configuration\nSEED = 1234\nDATASET_NAME = \"sft-dataset\"\n\n# Convert SQuAD format to prompt/completion format and save to JSONL\ndef convert_squad_to_sft_format(example):\n \"\"\"Convert SQuAD format to prompt/completion format for SFT training.\"\"\"\n prompt = f\"Context: {example['context']} Question: {example['question']} Answer:\"\n completion = example[\"answers\"][\"text\"][0] # Take the first answer\n return {\"prompt\": prompt, \"completion\": completion}\n\n# Load the SQuAD dataset from Hugging Face\nprint(\"Loading dataset rajpurkar/squad\")\nds = load_dataset(\"rajpurkar/squad\")\nif not isinstance(ds, DatasetDict):\n raise ValueError(\"Dataset does not contain expected splits\")\n\nprint(\"Loaded dataset\")\n\n# For the purpose of this tutorial, we'll use a subset of the dataset\n# We use a reduced dataset size (3000 training/300 validation samples) to keep tutorial runtime manageable\n# while still demonstrating the performance benefits of sequence packing. The larger the dataset,\n# the better the model will perform but the longer the training will take.\ntraining_size = 3000\nvalidation_size = 300\nDATASET_PATH = Path(DATASET_NAME).absolute()\n\n# Get training split and verify it's a Dataset (not IterableDataset)\ntrain_dataset = ds[\"train\"]\nvalidation_dataset = ds[\"validation\"]\nassert isinstance(train_dataset, Dataset), \"Expected Dataset type\"\nassert isinstance(validation_dataset, Dataset), \"Expected Dataset type\"\n\n# Select subsets and save to JSONL files\ntraining_ds = train_dataset.select(range(training_size))\nvalidation_ds = validation_dataset.select(range(validation_size))\n\n# Transform to SFT format (prompt/completion)\ntraining_ds = training_ds.map(convert_squad_to_sft_format, remove_columns=training_ds.column_names)\nvalidation_ds = validation_ds.map(convert_squad_to_sft_format, remove_columns=validation_ds.column_names)\n\n# Create directory if it doesn't exist\n# Note: This will create a local 'sft-dataset/' directory with training.jsonl and validation.jsonl files\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Save subsets to JSONL files\ntraining_ds.to_json(f\"{DATASET_PATH}/training.jsonl\")\nvalidation_ds.to_json(f\"{DATASET_PATH}/validation.jsonl\")\n\nprint(f\"Saved training.jsonl with {len(training_ds)} rows\")\nprint(f\"Saved validation.jsonl with {len(validation_ds)} rows\")", + "language": "python", + "source_html": "import json\nimport os\nfrom pathlib import Path\nfrom datasets import load_dataset, Dataset, DatasetDict\n\n# Configuration\nSEED = 1234\nDATASET_NAME = "sft-dataset"\n\n# Convert SQuAD format to prompt/completion format and save to JSONL\ndef convert_squad_to_sft_format(example):\n """Convert SQuAD format to prompt/completion format for SFT training."""\n prompt = f"Context: {example['context']} Question: {example['question']} Answer:"\n completion = example["answers"]["text"][0] # Take the first answer\n return {"prompt": prompt, "completion": completion}\n\n# Load the SQuAD dataset from Hugging Face\nprint("Loading dataset rajpurkar/squad")\nds = load_dataset("rajpurkar/squad")\nif not isinstance(ds, DatasetDict):\n raise ValueError("Dataset does not contain expected splits")\n\nprint("Loaded dataset")\n\n# For the purpose of this tutorial, we'll use a subset of the dataset\n# We use a reduced dataset size (3000 training/300 validation samples) to keep tutorial runtime manageable\n# while still demonstrating the performance benefits of sequence packing. The larger the dataset,\n# the better the model will perform but the longer the training will take.\ntraining_size = 3000\nvalidation_size = 300\nDATASET_PATH = Path(DATASET_NAME).absolute()\n\n# Get training split and verify it's a Dataset (not IterableDataset)\ntrain_dataset = ds["train"]\nvalidation_dataset = ds["validation"]\nassert isinstance(train_dataset, Dataset), "Expected Dataset type"\nassert isinstance(validation_dataset, Dataset), "Expected Dataset type"\n\n# Select subsets and save to JSONL files\ntraining_ds = train_dataset.select(range(training_size))\nvalidation_ds = validation_dataset.select(range(validation_size))\n\n# Transform to SFT format (prompt/completion)\ntraining_ds = training_ds.map(convert_squad_to_sft_format, remove_columns=training_ds.column_names)\nvalidation_ds = validation_ds.map(convert_squad_to_sft_format, remove_columns=validation_ds.column_names)\n\n# Create directory if it doesn't exist\n# Note: This will create a local 'sft-dataset/' directory with training.jsonl and validation.jsonl files\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Save subsets to JSONL files\ntraining_ds.to_json(f"{DATASET_PATH}/training.jsonl")\nvalidation_ds.to_json(f"{DATASET_PATH}/validation.jsonl")\n\nprint(f"Saved training.jsonl with {len(training_ds)} rows")\nprint(f"Saved validation.jsonl with {len(validation_ds)} rows")\n" + }, + { + "type": "code", + "source": "# Create fileset to store SFT training data\n\ntry:\n client.files.filesets.create(\n workspace=\"default\",\n name=DATASET_NAME,\n description=\"SFT training data\"\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\n# Upload training data files individually to ensure correct structure\nclient.files.upload(\n local_path=DATASET_PATH, # Local directory with your JSONL files\n remote_path=\"\",\n fileset=DATASET_NAME,\n workspace=\"default\"\n)\n\n# Validate training data is uploaded correctly\nprint(\"Training data:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace=\"default\").data], indent=2))", + "language": "python", + "source_html": "# Create fileset to store SFT training data\n\ntry:\n client.files.filesets.create(\n workspace="default",\n name=DATASET_NAME,\n description="SFT training data"\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\n# Upload training data files individually to ensure correct structure\nclient.files.upload(\n local_path=DATASET_PATH, # Local directory with your JSONL files\n remote_path="",\n fileset=DATASET_NAME,\n workspace="default"\n)\n\n# Validate training data is uploaded correctly\nprint("Training data:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace="default").data], indent=2))\n" + }, + { + "type": "markdown", + "source": "### 3. Secrets Setup\n\nIf you plan to use NGC or HuggingFace models, you will need to configure authentication:\n\n- **NGC models** (`ngc://` URIs): Requires NGC API key\n- **HuggingFace models** (`hf://` URIs): Requires HF token for gated/private models\n\n\nConfigure these as secrets in your platform. Refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md) for detailed instructions.\n\nGet your credentials to access base models:\n- [NGC API Key](https://ngc.nvidia.com/) (Setup → Generate API Key)\n- [HuggingFace Token](https://huggingface.co/settings/tokens) (Create token with Read access)\n\n\n---\n\n#### Quick Setup Example\n\nThis tutorial uses the [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) model from HuggingFace. Ensure that you have sufficient permissions to download the model. If you cannot access the files on the [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) Hugging Face page, request access.\n\n**HuggingFace Authentication:**\n- For gated models (Llama, Gemma), you must provide a HuggingFace token via the `token_secret` parameter\n- Get your token from [HuggingFace Settings](https://huggingface.co/settings/tokens) (requires Read access)\n- Accept the model's terms on the HuggingFace model page before using it. Example: [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main)\n- For public models, you can omit the `token_secret` parameter when creating a fileset for the model in the next step.", + "source_html": "

3. Secrets Setup

\n

If you plan to use NGC or HuggingFace models, you will need to configure authentication:

\n
    \n
  • NGC models (ngc:// URIs): Requires NGC API key
    • \n
  • HuggingFace models (hf:// URIs): Requires HF token for gated/private models
    • \n
\n

Configure these as secrets in your platform. Refer to Managing Secrets for detailed instructions.

\n

Get your credentials to access base models:

\n\n
\n

Quick Setup Example

\n

This tutorial uses the meta-llama/Llama-3.2-1B-Instruct model from HuggingFace. Ensure that you have sufficient permissions to download the model. If you cannot access the files on the meta-llama/Llama-3.2-1B-Instruct Hugging Face page, request access.

\n

HuggingFace Authentication:

\n
    \n
  • For gated models (Llama, Gemma), you must provide a HuggingFace token via the token_secret parameter
    • \n
  • Get your token from HuggingFace Settings (requires Read access)
    • \n
  • Accept the model's terms on the HuggingFace model page before using it. Example: meta-llama/Llama-3.2-1B-Instruct
    • \n
  • For public models, you can omit the token_secret parameter when creating a fileset for the model in the next step.
    • \n
\n" + }, + { + "type": "code", + "source": "# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set\nHF_TOKEN = os.getenv(\"HF_TOKEN\")\nNGC_API_KEY = os.getenv(\"NGC_API_KEY\")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f\"{label} environment variable is not set. Set it and try again.\")\n try:\n secret = client.secrets.create(\n name=name,\n workspace=\"default\",\n value=value,\n )\n print(f\"Created secret: {name}\")\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n return client.secrets.retrieve(name=name, workspace=\"default\")\n\n\n# Create HuggingFace token secret\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")\nprint(\"HF_TOKEN secret:\")\nprint(hf_secret.model_dump_json(indent=2))\n\n# Create NGC API key secret\n# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n# ngc_api_key = create_or_get_secret(\"ngc-api-key\", NGC_API_KEY, \"NGC_API_KEY\")", + "language": "python", + "source_html": "# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set\nHF_TOKEN = os.getenv("HF_TOKEN")\nNGC_API_KEY = os.getenv("NGC_API_KEY")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f"{label} environment variable is not set. Set it and try again.")\n try:\n secret = client.secrets.create(\n name=name,\n workspace="default",\n value=value,\n )\n print(f"Created secret: {name}")\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n return client.secrets.retrieve(name=name, workspace="default")\n\n\n# Create HuggingFace token secret\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\nprint("HF_TOKEN secret:")\nprint(hf_secret.model_dump_json(indent=2))\n\n# Create NGC API key secret\n# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n# ngc_api_key = create_or_get_secret("ngc-api-key", NGC_API_KEY, "NGC_API_KEY")\n" + }, + { + "type": "markdown", + "source": "### 4. Create Base Model FileSet\n\nCreate a fileset pointing to the [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) model on HuggingFace. This step creates a pointer to the model on Hugging Face and does not download it. The model is downloaded at job creation time.\n\nNote: for public models, you can omit the `token_secret` parameter when creating a model fileset.", + "source_html": "

4. Create Base Model FileSet

\n

Create a fileset pointing to the meta-llama/Llama-3.2-1B-Instruct model on HuggingFace. This step creates a pointer to the model on Hugging Face and does not download it. The model is downloaded at job creation time.

\n

Note: for public models, you can omit the token_secret parameter when creating a model fileset.

\n" + }, + { + "type": "code", + "source": "import time\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = \"meta-llama/Llama-3.2-1B-Instruct\"\nMODEL_NAME = \"llama-3-2-1b-base\"\n\n# Ensure you have a HuggingFace token secret created\n# Create a fileset pointing to the desired HuggingFace model\ntry:\n base_model_fs = client.files.filesets.create(\n workspace=\"default\",\n name=MODEL_NAME,\n description=\"Llama 3.2 1B base model from HuggingFace\",\n storage=HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type=\"model\",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\n print(f\"Created base model fileset: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model fileset already exists. Skipping creation.\")\n base_model_fs = client.files.filesets.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\n# Create the Model Entity representation.\ntry:\n base_model = client.models.create(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n )\n print(f\"Created Model Entity: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model already exists. Updating fileset if different.\")\n base_model = client.models.update(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n )\n\nprint(f\"\\nBase model fileset: fileset://default/{base_model.name}\")\nprint(\"Base model fileset files list:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace=\"default\").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint(\"\\nWaiting for ModelSpec to be populated...\")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f\"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds\")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\nprint(f\"ModelSpec populated: {base_model.spec}\")", + "language": "python", + "source_html": "import time\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = "meta-llama/Llama-3.2-1B-Instruct"\nMODEL_NAME = "llama-3-2-1b-base"\n\n# Ensure you have a HuggingFace token secret created\n# Create a fileset pointing to the desired HuggingFace model\ntry:\n base_model_fs = client.files.filesets.create(\n workspace="default",\n name=MODEL_NAME,\n description="Llama 3.2 1B base model from HuggingFace",\n storage=HuggingfaceStorageConfigParam(\n type="huggingface",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type="model",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\n print(f"Created base model fileset: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model fileset already exists. Skipping creation.")\n base_model_fs = client.files.filesets.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\n# Create the Model Entity representation.\ntry:\n base_model = client.models.create(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n )\n print(f"Created Model Entity: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model already exists. Updating fileset if different.")\n base_model = client.models.update(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n )\n\nprint(f"\\nBase model fileset: fileset://default/{base_model.name}")\nprint("Base model fileset files list:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace="default").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint("\\nWaiting for ModelSpec to be populated...")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\nprint(f"ModelSpec populated: {base_model.spec}")\n" + }, + { + "type": "markdown", + "source": "### 5. Create LoRA Job with Sequence Packing\nCreate a customization job with an inline target referencing the base model and dataset filesets created in previous steps.", + "source_html": "

5. Create LoRA Job with Sequence Packing

\n

Create a customization job with an inline target referencing the base model and dataset filesets created in previous steps.

\n" + }, + { + "type": "code", + "source": "import uuid\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam,\n LoRaParamsParam,\n)\n\n# Enable sequence packing to improve throughput and GPU utilization\nSEQUENCE_PACKING_ENABLED = True\n\njob_suffix = uuid.uuid4().hex[:4]\n\nJOB_NAME = f\"my-sft-job-{job_suffix}\"\njob_spec = CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=4096,\n val_check_interval=0.1,\n micro_batch_size=1,\n sequence_packing=SEQUENCE_PACKING_ENABLED,\n peft=LoRaParamsParam(),\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n)\n\njob_with_sequence_packing = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=job_spec\n)\n\nprint(f\"Job ID: {job_with_sequence_packing.name}\")\nprint(f\"Output model: {job_with_sequence_packing.spec.output.name}\")", + "language": "python", + "source_html": "import uuid\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam,\n LoRaParamsParam,\n)\n\n# Enable sequence packing to improve throughput and GPU utilization\nSEQUENCE_PACKING_ENABLED = True\n\njob_suffix = uuid.uuid4().hex[:4]\n\nJOB_NAME = f"my-sft-job-{job_suffix}"\njob_spec = CustomizationJobInputParam(\n model=f"default/{base_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=4096,\n val_check_interval=0.1,\n micro_batch_size=1,\n sequence_packing=SEQUENCE_PACKING_ENABLED,\n peft=LoRaParamsParam(),\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n)\n\njob_with_sequence_packing = client.customization.jobs.create(\n name=JOB_NAME,\n workspace="default",\n spec=job_spec\n)\n\nprint(f"Job ID: {job_with_sequence_packing.name}")\nprint(f"Output model: {job_with_sequence_packing.spec.output.name}")\n" + }, + { + "type": "markdown", + "source": "### 6. Track Finetuning Progress\n\nA training job contains multiple steps: \n- Model and dataset downloading\n- Finetuning where LoRA adapter weights are trained\n- Creating a fileset entry for the finetuned model\n- Finetuned weights uploading\n\nThe elapsed time printed below reflects progress of the entire job. We compare the time taken by the finetuning step for both jobs in the last section of this tutorial.", + "source_html": "

6. Track Finetuning Progress

\n

A training job contains multiple steps:

\n
    \n
  • Model and dataset downloading
    • \n
  • Finetuning where LoRA adapter weights are trained
    • \n
  • Creating a fileset entry for the finetuned model
    • \n
  • Finetuned weights uploading
    • \n
\n

The elapsed time printed below reflects progress of the entire job. We compare the time taken by the finetuning step for both jobs in the last section of this tutorial.

\n" + }, + { + "type": "markdown", + "source": "#### Define Helper Functions", + "source_html": "

Define Helper Functions

\n" + }, + { + "type": "code", + "source": "\n# Helpers to draw GPU VRAM Utilization and Validation Loss\nimport matplotlib.pyplot as plt\ntry:\n import pynvml\n _PYNVML_AVAILABLE = True\nexcept ImportError:\n _PYNVML_AVAILABLE = False\n print(\"Note: Install nvidia-ml-py ('pip install nvidia-ml-py' or 'uv pip install nvidia-ml-py') to enable live GPU metrics.\")\n\n# ---------------------------------------------------------------------------\n# GPU metrics collection (nvidia-ml-py; import name is pynvml)\n# ---------------------------------------------------------------------------\n\ndef _get_gpu_snapshot() -> tuple[list[float], list[float]]:\n \"\"\"Return (vram_usage_pcts, compute_util_pcts) for each GPU.\"\"\"\n if not _PYNVML_AVAILABLE:\n return [], []\n pynvml.nvmlInit()\n try:\n vram, util = [], []\n for i in range(pynvml.nvmlDeviceGetCount()):\n h = pynvml.nvmlDeviceGetHandleByIndex(i)\n mem = pynvml.nvmlDeviceGetMemoryInfo(h)\n rates = pynvml.nvmlDeviceGetUtilizationRates(h)\n vram.append(int(mem.used) / int(mem.total) * 100)\n util.append(float(rates.gpu))\n return vram, util\n finally:\n pynvml.nvmlShutdown()\n\n\n# ---------------------------------------------------------------------------\n# Dashboard drawing helpers\n# ---------------------------------------------------------------------------\n\n_PALETTE = {\n \"val_loss\": \"#E74C3C\",\n \"train_loss\": \"#F39C12\",\n \"vram\": [\"#3498DB\", \"#9B59B6\", \"#1ABC9C\", \"#E67E22\"],\n \"util\": [\"#2ECC71\", \"#E74C3C\", \"#3498DB\", \"#F1C40F\"],\n \"grid\": \"#ECECEC\",\n \"title\": \"#2C3E50\",\n \"subtitle\": \"#7F8C8D\",\n \"spine\": \"#CCCCCC\",\n \"tick\": \"#666666\",\n}\n\n\ndef _style_axis(ax):\n \"\"\"Apply shared cosmetic styling to a subplot axis.\"\"\"\n ax.set_facecolor(\"white\")\n ax.grid(True, alpha=0.4, color=_PALETTE[\"grid\"], linewidth=0.8)\n for spine in (\"top\", \"right\"):\n ax.spines[spine].set_visible(False)\n ax.spines[\"left\"].set_color(_PALETTE[\"spine\"])\n ax.spines[\"bottom\"].set_color(_PALETTE[\"spine\"])\n ax.tick_params(colors=_PALETTE[\"tick\"], labelsize=9)\n\n\ndef _plot_line(ax, xs, ys, color, label, fill=True):\n \"\"\"Plot a time series, gracefully skipping None values.\"\"\"\n pts = [(x, y) for x, y in zip(xs, ys) if y is not None]\n if not pts:\n return\n px, py = zip(*pts)\n ax.plot(\n px, py, color=color, linewidth=2.2,\n marker=\"o\", markersize=4,\n markerfacecolor=\"white\", markeredgewidth=1.8, markeredgecolor=color,\n label=label, zorder=3,\n )\n if fill:\n ax.fill_between(px, py, alpha=0.08, color=color)\n\n\ndef _plot_gpu_panel(ax, xs, history, colors, fallback_label):\n \"\"\"Plot per-GPU time series with area fill.\"\"\"\n if not history or not history[0]:\n ax.text(\n 0.5, 0.5, \"No GPU data\", transform=ax.transAxes,\n ha=\"center\", va=\"center\", fontsize=11, color=\"#AAAAAA\",\n )\n return\n n_gpus = max(len(snap) for snap in history)\n for g in range(n_gpus):\n vals = [snap[g] if g < len(snap) else 0 for snap in history]\n c = colors[g % len(colors)]\n label = f\"GPU {g}\" if n_gpus > 1 else fallback_label\n ax.plot(xs[: len(vals)], vals, color=c, linewidth=2, label=label)\n ax.fill_between(xs[: len(vals)], vals, alpha=0.08, color=c)\n if n_gpus > 1:\n ax.legend(fontsize=9, framealpha=0.9, edgecolor=\"#DDD\")\n\n\ndef _draw_dashboard(\n elapsed_mins, val_losses, train_losses,\n vram_history, util_history,\n job_name, status_str, step_str, elapsed_str,\n):\n \"\"\"Render a live 1x3 training dashboard.\"\"\"\n fig, axes = plt.subplots(1, 3, figsize=(20, 5.5))\n fig.patch.set_facecolor(\"#FAFBFC\")\n\n fig.suptitle(\n job_name, fontsize=15, fontweight=\"bold\",\n color=_PALETTE[\"title\"], y=1.10,\n )\n fig.text(\n 0.5, 1.01,\n f\"{status_str} | {step_str} | {elapsed_str}\",\n ha=\"center\", fontsize=13, color=_PALETTE[\"subtitle\"],\n )\n\n for ax in axes:\n _style_axis(ax)\n\n # -- Panel 1: Loss curves --\n _plot_line(axes[0], elapsed_mins, val_losses, _PALETTE[\"val_loss\"], \"Val Loss\", fill=True)\n _plot_line(axes[0], elapsed_mins, train_losses, _PALETTE[\"train_loss\"], \"Train Loss\", fill=False)\n axes[0].set_title(\"Train/Validation Loss\", fontsize=13, fontweight=\"bold\", color=_PALETTE[\"title\"], pad=12)\n axes[0].set_xlabel(\"Time (min)\", fontsize=10, color=\"#666\")\n axes[0].set_ylabel(\"Loss\", fontsize=10, color=\"#666\")\n if any(v is not None for v in val_losses + train_losses):\n axes[0].legend(fontsize=9, framealpha=0.9, edgecolor=\"#DDD\")\n\n # -- Panel 2: GPU VRAM usage --\n _plot_gpu_panel(axes[1], elapsed_mins, vram_history, _PALETTE[\"vram\"], \"VRAM\")\n axes[1].set_title(\"GPU VRAM Usage\", fontsize=13, fontweight=\"bold\", color=_PALETTE[\"title\"], pad=12)\n axes[1].set_xlabel(\"Time (min)\", fontsize=10, color=\"#666\")\n axes[1].set_ylabel(\"Usage (%)\", fontsize=10, color=\"#666\")\n axes[1].set_ylim(-2, 105)\n\n # -- Panel 3: GPU utilization --\n _plot_gpu_panel(axes[2], elapsed_mins, util_history, _PALETTE[\"util\"], \"Utilization\")\n axes[2].set_title(\"GPU Utilization\", fontsize=13, fontweight=\"bold\", color=_PALETTE[\"title\"], pad=12)\n axes[2].set_xlabel(\"Time (min)\", fontsize=10, color=\"#666\")\n axes[2].set_ylabel(\"Utilization (%)\", fontsize=10, color=\"#666\")\n axes[2].set_ylim(-2, 105)\n\n plt.tight_layout(rect=[0, 0, 1, 0.98])\n plt.show()", + "language": "python", + "source_html": "# Helpers to draw GPU VRAM Utilization and Validation Loss\nimport matplotlib.pyplot as plt\ntry:\n import pynvml\n _PYNVML_AVAILABLE = True\nexcept ImportError:\n _PYNVML_AVAILABLE = False\n print("Note: Install nvidia-ml-py ('pip install nvidia-ml-py' or 'uv pip install nvidia-ml-py') to enable live GPU metrics.")\n\n# ---------------------------------------------------------------------------\n# GPU metrics collection (nvidia-ml-py; import name is pynvml)\n# ---------------------------------------------------------------------------\n\ndef _get_gpu_snapshot() -> tuple[list[float], list[float]]:\n """Return (vram_usage_pcts, compute_util_pcts) for each GPU."""\n if not _PYNVML_AVAILABLE:\n return [], []\n pynvml.nvmlInit()\n try:\n vram, util = [], []\n for i in range(pynvml.nvmlDeviceGetCount()):\n h = pynvml.nvmlDeviceGetHandleByIndex(i)\n mem = pynvml.nvmlDeviceGetMemoryInfo(h)\n rates = pynvml.nvmlDeviceGetUtilizationRates(h)\n vram.append(int(mem.used) / int(mem.total) * 100)\n util.append(float(rates.gpu))\n return vram, util\n finally:\n pynvml.nvmlShutdown()\n\n\n# ---------------------------------------------------------------------------\n# Dashboard drawing helpers\n# ---------------------------------------------------------------------------\n\n_PALETTE = {\n "val_loss": "#E74C3C",\n "train_loss": "#F39C12",\n "vram": ["#3498DB", "#9B59B6", "#1ABC9C", "#E67E22"],\n "util": ["#2ECC71", "#E74C3C", "#3498DB", "#F1C40F"],\n "grid": "#ECECEC",\n "title": "#2C3E50",\n "subtitle": "#7F8C8D",\n "spine": "#CCCCCC",\n "tick": "#666666",\n}\n\n\ndef _style_axis(ax):\n """Apply shared cosmetic styling to a subplot axis."""\n ax.set_facecolor("white")\n ax.grid(True, alpha=0.4, color=_PALETTE["grid"], linewidth=0.8)\n for spine in ("top", "right"):\n ax.spines[spine].set_visible(False)\n ax.spines["left"].set_color(_PALETTE["spine"])\n ax.spines["bottom"].set_color(_PALETTE["spine"])\n ax.tick_params(colors=_PALETTE["tick"], labelsize=9)\n\n\ndef _plot_line(ax, xs, ys, color, label, fill=True):\n """Plot a time series, gracefully skipping None values."""\n pts = [(x, y) for x, y in zip(xs, ys) if y is not None]\n if not pts:\n return\n px, py = zip(*pts)\n ax.plot(\n px, py, color=color, linewidth=2.2,\n marker="o", markersize=4,\n markerfacecolor="white", markeredgewidth=1.8, markeredgecolor=color,\n label=label, zorder=3,\n )\n if fill:\n ax.fill_between(px, py, alpha=0.08, color=color)\n\n\ndef _plot_gpu_panel(ax, xs, history, colors, fallback_label):\n """Plot per-GPU time series with area fill."""\n if not history or not history[0]:\n ax.text(\n 0.5, 0.5, "No GPU data", transform=ax.transAxes,\n ha="center", va="center", fontsize=11, color="#AAAAAA",\n )\n return\n n_gpus = max(len(snap) for snap in history)\n for g in range(n_gpus):\n vals = [snap[g] if g < len(snap) else 0 for snap in history]\n c = colors[g % len(colors)]\n label = f"GPU {g}" if n_gpus > 1 else fallback_label\n ax.plot(xs[: len(vals)], vals, color=c, linewidth=2, label=label)\n ax.fill_between(xs[: len(vals)], vals, alpha=0.08, color=c)\n if n_gpus > 1:\n ax.legend(fontsize=9, framealpha=0.9, edgecolor="#DDD")\n\n\ndef _draw_dashboard(\n elapsed_mins, val_losses, train_losses,\n vram_history, util_history,\n job_name, status_str, step_str, elapsed_str,\n):\n """Render a live 1x3 training dashboard."""\n fig, axes = plt.subplots(1, 3, figsize=(20, 5.5))\n fig.patch.set_facecolor("#FAFBFC")\n\n fig.suptitle(\n job_name, fontsize=15, fontweight="bold",\n color=_PALETTE["title"], y=1.10,\n )\n fig.text(\n 0.5, 1.01,\n f"{status_str} | {step_str} | {elapsed_str}",\n ha="center", fontsize=13, color=_PALETTE["subtitle"],\n )\n\n for ax in axes:\n _style_axis(ax)\n\n # -- Panel 1: Loss curves --\n _plot_line(axes[0], elapsed_mins, val_losses, _PALETTE["val_loss"], "Val Loss", fill=True)\n _plot_line(axes[0], elapsed_mins, train_losses, _PALETTE["train_loss"], "Train Loss", fill=False)\n axes[0].set_title("Train/Validation Loss", fontsize=13, fontweight="bold", color=_PALETTE["title"], pad=12)\n axes[0].set_xlabel("Time (min)", fontsize=10, color="#666")\n axes[0].set_ylabel("Loss", fontsize=10, color="#666")\n if any(v is not None for v in val_losses + train_losses):\n axes[0].legend(fontsize=9, framealpha=0.9, edgecolor="#DDD")\n\n # -- Panel 2: GPU VRAM usage --\n _plot_gpu_panel(axes[1], elapsed_mins, vram_history, _PALETTE["vram"], "VRAM")\n axes[1].set_title("GPU VRAM Usage", fontsize=13, fontweight="bold", color=_PALETTE["title"], pad=12)\n axes[1].set_xlabel("Time (min)", fontsize=10, color="#666")\n axes[1].set_ylabel("Usage (%)", fontsize=10, color="#666")\n axes[1].set_ylim(-2, 105)\n\n # -- Panel 3: GPU utilization --\n _plot_gpu_panel(axes[2], elapsed_mins, util_history, _PALETTE["util"], "Utilization")\n axes[2].set_title("GPU Utilization", fontsize=13, fontweight="bold", color=_PALETTE["title"], pad=12)\n axes[2].set_xlabel("Time (min)", fontsize=10, color="#666")\n axes[2].set_ylabel("Utilization (%)", fontsize=10, color="#666")\n axes[2].set_ylim(-2, 105)\n\n plt.tight_layout(rect=[0, 0, 1, 0.98])\n plt.show()\n" + }, + { + "type": "markdown", + "source": "#### Monitor the Job Until Completion\n\nThe cell below polls the job status every 10 seconds and renders a live dashboard with validation loss, GPU VRAM usage, and GPU utilization charts. The charts appear empty at first while the model and dataset download; training metrics and GPU activity populate after the finetuning step begins.\n\n> **Note:** This is additional code. You can also use the Weights & Biases or MLflow integrations.", + "source_html": "

Monitor the Job Until Completion

\n

The cell below polls the job status every 10 seconds and renders a live dashboard with validation loss, GPU VRAM usage, and GPU utilization charts. The charts appear empty at first while the model and dataset download; training metrics and GPU activity populate after the finetuning step begins.

\n
\n

Note: This is additional code. You can also use the Weights & Biases or MLflow integrations.

\n
\n" + }, + { + "type": "code", + "source": "import time\nfrom typing import cast\nfrom IPython.display import clear_output\nfrom nemo_platform.types.shared import PlatformJobStatusResponse\n\n# Timeout set to 30 minutes to accommodate typical LoRA training duration for this dataset size.\n# Actual training time will vary based on hardware, model size, and dataset complexity.\nTIMEOUT_SECONDS = 30 * 60 # 30 minutes\nVAL_LOSS_KEY = \"val_loss\"\nTRAIN_LOSS_KEY = \"loss\"\n\n# ---------------------------------------------------------------------------\n# Job polling with live dashboard\n# ---------------------------------------------------------------------------\n\ndef wait_for_job(\n workspace: str,\n job_name: str,\n timeout: int = TIMEOUT_SECONDS,\n poll_interval: int = 10,\n val_loss_key: str = VAL_LOSS_KEY,\n train_loss_key: str = TRAIN_LOSS_KEY,\n) -> PlatformJobStatusResponse:\n \"\"\"\n Poll job status until completed, failed, cancelled, or timeout.\n Displays a live dashboard with loss curves and GPU metrics.\n\n Args:\n workspace: The workspace where the job is running.\n job_name: The name of the job to monitor.\n timeout: Maximum time to wait in seconds (default: 30 minutes).\n poll_interval: Time between status checks in seconds (default: 10).\n\n Returns:\n The final job status response.\n \"\"\"\n start_time = time.time()\n\n # Time-series accumulators required for plotting\n elapsed_mins: list[float] = []\n val_losses: list[float | None] = []\n train_losses: list[float | None] = []\n vram_history: list[list[float]] = []\n util_history: list[list[float]] = []\n\n while True:\n elapsed = time.time() - start_time\n elapsed_min = elapsed / 60\n\n # Check for timeout\n if elapsed > timeout:\n error_message = f\"Timeout reached after {elapsed_min:.1f} minutes\"\n print(f\"\\n{error_message}\")\n print(\"Job did not complete within the timeout period.\")\n raise Exception(error_message)\n\n status = client.jobs.get_status(name=job_name, workspace=workspace)\n\n # -- Extract training progress from nested steps structure --\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n val_loss: float | None = None\n train_loss: float | None = None\n current_step_name: str | None = None\n current_step_phase: str | None = None\n\n for job_step in status.steps or []:\n # Track the current active step name and phase for progress display\n if job_step.tasks:\n task = job_step.tasks[0]\n td = task.status_details or {}\n phase = cast(str, td.get(\"phase\", \"\"))\n # Update current step if it's active or pending (not completed)\n if job_step.status in (\"active\", \"pending\"):\n current_step_name = job_step.name\n current_step_phase = phase or \"started\"\n\n if job_step.name == \"customization-training-job\":\n for task in job_step.tasks or []:\n td = task.status_details or {}\n step = cast(int, td[\"step\"]) if \"step\" in td else None\n max_steps = cast(int, td[\"max_steps\"]) if \"max_steps\" in td else None\n training_phase = cast(str, td[\"phase\"]) if \"phase\" in td else None\n val_loss = float(td[val_loss_key]) if val_loss_key in td else None\n train_loss = float(td[train_loss_key]) if train_loss_key in td else None\n break\n break\n\n # Fall back to top-level status_details\n if status.status_details:\n if val_loss is None and val_loss_key in status.status_details:\n val_loss = float(status.status_details[val_loss_key])\n if train_loss is None and train_loss_key in status.status_details:\n train_loss = float(status.status_details[train_loss_key])\n\n # -- Collect GPU snapshot --\n vram_pcts, util_pcts = _get_gpu_snapshot()\n\n # -- Append to accumulators used for the plots --\n elapsed_mins.append(elapsed_min)\n val_losses.append(val_loss)\n train_losses.append(train_loss)\n vram_history.append(vram_pcts)\n util_history.append(util_pcts)\n\n # -- Build status strings --\n status_str = f\"Status: {status.status}\"\n if step is not None and max_steps is not None:\n pct = step / max_steps * 100\n step_str = f\"Step {step}/{max_steps} ({pct:.0f}%)\"\n if training_phase:\n step_str += f\" - {training_phase}\"\n else:\n if current_step_name and current_step_phase:\n step_str = f\"{current_step_name} - {current_step_phase}\"\n elif current_step_name:\n step_str = f\"{current_step_name}\"\n else:\n step_str = \"Waiting for training to start...\"\n elapsed_str = f\"Elapsed: {elapsed_min:.1f} min\"\n\n # -- Redraw dashboard --\n clear_output(wait=True)\n _draw_dashboard(\n elapsed_mins, val_losses, train_losses,\n vram_history, util_history,\n job_name, status_str, step_str, elapsed_str,\n )\n\n # -- Check terminal conditions --\n if status.status.lower() == \"completed\":\n # Redraw dashboard one final time with \"completed\" status\n status_str = f\"Status: {status.status}\"\n if step is not None and max_steps is not None:\n step_str = f\"Step {max_steps}/{max_steps} (100%)\"\n clear_output(wait=True)\n _draw_dashboard(\n elapsed_mins, val_losses, train_losses,\n vram_history, util_history,\n job_name, status_str, step_str, elapsed_str,\n )\n print(f\"\\nJob completed in {elapsed_min:.1f} minutes ({elapsed:.0f}s)\")\n return status\n elif status.status.lower() in (\"failed\", \"cancelled\", \"error\"):\n print(f\"\\nJob finished with status: {status.status}\")\n print(f\"Total time elapsed: {elapsed_min:.1f} minutes ({elapsed:.0f}s)\")\n\n # Print error details from the job level\n if status.error_details:\n error_msg = status.error_details.get(\"message\", \"\")\n if error_msg:\n print(f\"\\nError: {error_msg}\")\n\n # Find and print error details from the failed step/task\n for job_step in status.steps or []:\n if job_step.status == \"error\":\n print(f\"\\nFailed step: {job_step.name}\")\n if job_step.error_details:\n step_error = job_step.error_details.get(\"message\", \"\")\n if step_error:\n print(f\"Step error: {step_error}\")\n # Get error_stack from the failed task\n for task in job_step.tasks or []:\n if task.status == \"error\" and hasattr(task, \"error_stack\") and task.error_stack:\n print(f\"\\nError stack trace:\\n{task.error_stack}\")\n elif task.status == \"error\" and task.error_details:\n task_error = task.error_details.get(\"message\", \"\")\n if task_error:\n print(f\"Task error: {task_error}\")\n break\n\n raise Exception(f\"Job finished with status: {status.status}\")\n\n time.sleep(poll_interval)\n\n\n# Wait for the job to complete\njob_with_sequence_packing_status = wait_for_job(\n workspace=\"default\",\n job_name=job_with_sequence_packing.name,\n timeout=TIMEOUT_SECONDS,\n)\n\nprint(f\"Validation loss: {job_with_sequence_packing_status.status_details['val_loss']:.2f}\")", + "language": "python", + "source_html": "import time\nfrom typing import cast\nfrom IPython.display import clear_output\nfrom nemo_platform.types.shared import PlatformJobStatusResponse\n\n# Timeout set to 30 minutes to accommodate typical LoRA training duration for this dataset size.\n# Actual training time will vary based on hardware, model size, and dataset complexity.\nTIMEOUT_SECONDS = 30 * 60 # 30 minutes\nVAL_LOSS_KEY = "val_loss"\nTRAIN_LOSS_KEY = "loss"\n\n# ---------------------------------------------------------------------------\n# Job polling with live dashboard\n# ---------------------------------------------------------------------------\n\ndef wait_for_job(\n workspace: str,\n job_name: str,\n timeout: int = TIMEOUT_SECONDS,\n poll_interval: int = 10,\n val_loss_key: str = VAL_LOSS_KEY,\n train_loss_key: str = TRAIN_LOSS_KEY,\n) -> PlatformJobStatusResponse:\n """\n Poll job status until completed, failed, cancelled, or timeout.\n Displays a live dashboard with loss curves and GPU metrics.\n\n Args:\n workspace: The workspace where the job is running.\n job_name: The name of the job to monitor.\n timeout: Maximum time to wait in seconds (default: 30 minutes).\n poll_interval: Time between status checks in seconds (default: 10).\n\n Returns:\n The final job status response.\n """\n start_time = time.time()\n\n # Time-series accumulators required for plotting\n elapsed_mins: list[float] = []\n val_losses: list[float | None] = []\n train_losses: list[float | None] = []\n vram_history: list[list[float]] = []\n util_history: list[list[float]] = []\n\n while True:\n elapsed = time.time() - start_time\n elapsed_min = elapsed / 60\n\n # Check for timeout\n if elapsed > timeout:\n error_message = f"Timeout reached after {elapsed_min:.1f} minutes"\n print(f"\\n{error_message}")\n print("Job did not complete within the timeout period.")\n raise Exception(error_message)\n\n status = client.jobs.get_status(name=job_name, workspace=workspace)\n\n # -- Extract training progress from nested steps structure --\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n val_loss: float | None = None\n train_loss: float | None = None\n current_step_name: str | None = None\n current_step_phase: str | None = None\n\n for job_step in status.steps or []:\n # Track the current active step name and phase for progress display\n if job_step.tasks:\n task = job_step.tasks[0]\n td = task.status_details or {}\n phase = cast(str, td.get("phase", ""))\n # Update current step if it's active or pending (not completed)\n if job_step.status in ("active", "pending"):\n current_step_name = job_step.name\n current_step_phase = phase or "started"\n\n if job_step.name == "customization-training-job":\n for task in job_step.tasks or []:\n td = task.status_details or {}\n step = cast(int, td["step"]) if "step" in td else None\n max_steps = cast(int, td["max_steps"]) if "max_steps" in td else None\n training_phase = cast(str, td["phase"]) if "phase" in td else None\n val_loss = float(td[val_loss_key]) if val_loss_key in td else None\n train_loss = float(td[train_loss_key]) if train_loss_key in td else None\n break\n break\n\n # Fall back to top-level status_details\n if status.status_details:\n if val_loss is None and val_loss_key in status.status_details:\n val_loss = float(status.status_details[val_loss_key])\n if train_loss is None and train_loss_key in status.status_details:\n train_loss = float(status.status_details[train_loss_key])\n\n # -- Collect GPU snapshot --\n vram_pcts, util_pcts = _get_gpu_snapshot()\n\n # -- Append to accumulators used for the plots --\n elapsed_mins.append(elapsed_min)\n val_losses.append(val_loss)\n train_losses.append(train_loss)\n vram_history.append(vram_pcts)\n util_history.append(util_pcts)\n\n # -- Build status strings --\n status_str = f"Status: {status.status}"\n if step is not None and max_steps is not None:\n pct = step / max_steps * 100\n step_str = f"Step {step}/{max_steps} ({pct:.0f}%)"\n if training_phase:\n step_str += f" - {training_phase}"\n else:\n if current_step_name and current_step_phase:\n step_str = f"{current_step_name} - {current_step_phase}"\n elif current_step_name:\n step_str = f"{current_step_name}"\n else:\n step_str = "Waiting for training to start..."\n elapsed_str = f"Elapsed: {elapsed_min:.1f} min"\n\n # -- Redraw dashboard --\n clear_output(wait=True)\n _draw_dashboard(\n elapsed_mins, val_losses, train_losses,\n vram_history, util_history,\n job_name, status_str, step_str, elapsed_str,\n )\n\n # -- Check terminal conditions --\n if status.status.lower() == "completed":\n # Redraw dashboard one final time with "completed" status\n status_str = f"Status: {status.status}"\n if step is not None and max_steps is not None:\n step_str = f"Step {max_steps}/{max_steps} (100%)"\n clear_output(wait=True)\n _draw_dashboard(\n elapsed_mins, val_losses, train_losses,\n vram_history, util_history,\n job_name, status_str, step_str, elapsed_str,\n )\n print(f"\\nJob completed in {elapsed_min:.1f} minutes ({elapsed:.0f}s)")\n return status\n elif status.status.lower() in ("failed", "cancelled", "error"):\n print(f"\\nJob finished with status: {status.status}")\n print(f"Total time elapsed: {elapsed_min:.1f} minutes ({elapsed:.0f}s)")\n\n # Print error details from the job level\n if status.error_details:\n error_msg = status.error_details.get("message", "")\n if error_msg:\n print(f"\\nError: {error_msg}")\n\n # Find and print error details from the failed step/task\n for job_step in status.steps or []:\n if job_step.status == "error":\n print(f"\\nFailed step: {job_step.name}")\n if job_step.error_details:\n step_error = job_step.error_details.get("message", "")\n if step_error:\n print(f"Step error: {step_error}")\n # Get error_stack from the failed task\n for task in job_step.tasks or []:\n if task.status == "error" and hasattr(task, "error_stack") and task.error_stack:\n print(f"\\nError stack trace:\\n{task.error_stack}")\n elif task.status == "error" and task.error_details:\n task_error = task.error_details.get("message", "")\n if task_error:\n print(f"Task error: {task_error}")\n break\n\n raise Exception(f"Job finished with status: {status.status}")\n\n time.sleep(poll_interval)\n\n\n# Wait for the job to complete\njob_with_sequence_packing_status = wait_for_job(\n workspace="default",\n job_name=job_with_sequence_packing.name,\n timeout=TIMEOUT_SECONDS,\n)\n\nprint(f"Validation loss: {job_with_sequence_packing_status.status_details['val_loss']:.2f}")\n" + }, + { + "type": "markdown", + "source": "### 7. Create LoRA Job without Sequence Packing\nCreate a customization job with sequence packing disabled. It's expected to take longer to complete.", + "source_html": "

7. Create LoRA Job without Sequence Packing

\n

Create a customization job with sequence packing disabled. It's expected to take longer to complete.

\n" + }, + { + "type": "code", + "source": "import uuid\n\njob_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f\"my-sft-job-{job_suffix}\"\n\njob_spec_no_packing = CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=4096,\n val_check_interval=0.1,\n micro_batch_size=1,\n sequence_packing=False,\n peft=LoRaParamsParam(),\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n)\n\njob_without_sequence_packing = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=job_spec_no_packing\n)\n\nprint(f\"Job ID: {job_without_sequence_packing.name}\")\nprint(f\"Output model: {job_without_sequence_packing.spec.output.name}\")", + "language": "python", + "source_html": "import uuid\n\njob_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f"my-sft-job-{job_suffix}"\n\njob_spec_no_packing = CustomizationJobInputParam(\n model=f"default/{base_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=4096,\n val_check_interval=0.1,\n micro_batch_size=1,\n sequence_packing=False,\n peft=LoRaParamsParam(),\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n)\n\njob_without_sequence_packing = client.customization.jobs.create(\n name=JOB_NAME,\n workspace="default",\n spec=job_spec_no_packing\n)\n\nprint(f"Job ID: {job_without_sequence_packing.name}")\nprint(f"Output model: {job_without_sequence_packing.spec.output.name}")\n" + }, + { + "type": "markdown", + "source": "### 8. Track Finetuning Progress for Job without Sequence Packing", + "source_html": "

8. Track Finetuning Progress for Job without Sequence Packing

\n" + }, + { + "type": "code", + "source": "# Wait for the training step to complete\njob_without_sequence_packing_status = wait_for_job(\n workspace=\"default\",\n job_name=job_without_sequence_packing.name,\n timeout=TIMEOUT_SECONDS\n)\n\nprint(f\"Validation loss: {job_without_sequence_packing_status.status_details['val_loss']:.2f}\")", + "language": "python", + "source_html": "# Wait for the training step to complete\njob_without_sequence_packing_status = wait_for_job(\n workspace="default",\n job_name=job_without_sequence_packing.name,\n timeout=TIMEOUT_SECONDS\n)\n\nprint(f"Validation loss: {job_without_sequence_packing_status.status_details['val_loss']:.2f}")\n" + }, + { + "type": "markdown", + "source": "### 9. Compare Results\n- Time to complete training should be significantly lower for the job that used sequence packing.\n- The expected validation loss for both jobs should be similar.\n- Sequence packed version should have a higher GPU utilization and higher GPU Memory Allocation.", + "source_html": "

9. Compare Results

\n
    \n
  • Time to complete training should be significantly lower for the job that used sequence packing.
    • \n
  • The expected validation loss for both jobs should be similar.
    • \n
  • Sequence packed version should have a higher GPU utilization and higher GPU Memory Allocation.
    • \n
\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.jobs import PlatformJobStep\nfrom datetime import datetime\nimport pandas as pd\n\nSTEP_NAME = \"customization-training-job\"\n\ndef get_elapsed_time(step: PlatformJobStep) -> float:\n \"\"\"Calculate elapsed time in seconds from step's created_at to updated_at.\"\"\"\n created_at = datetime.fromisoformat(step.created_at.replace(\"Z\", \"+00:00\"))\n updated_at = datetime.fromisoformat(step.updated_at.replace(\"Z\", \"+00:00\"))\n return (updated_at - created_at).total_seconds()\n\nstep_with_sequence_packing = client.jobs.steps.retrieve(\n name=STEP_NAME,\n workspace=\"default\",\n job=job_with_sequence_packing.name,\n)\n\nstep_without_sequence_packing = client.jobs.steps.retrieve(\n name=STEP_NAME,\n workspace=\"default\",\n job=job_without_sequence_packing.name,\n)\n\ntime_to_complete_with_sequence_packing = get_elapsed_time(step_with_sequence_packing)\ntime_to_complete_without_sequence_packing = get_elapsed_time(step_without_sequence_packing)\n\n# Display results as a table\nresults_df = pd.DataFrame({\n \"Seq Packing Enabled\": [True, False],\n \"Val Loss\": [\n job_with_sequence_packing_status.status_details['val_loss'],\n job_without_sequence_packing_status.status_details['val_loss']\n ],\n \"Training Step Time, sec\": [\n time_to_complete_with_sequence_packing,\n time_to_complete_without_sequence_packing\n ]\n})\n\nresults_df.style.format({\"Val Loss\": \"{:.2f}\", \"Training Step Time, sec\": \"{:.0f}\"}).hide(axis='index')", + "language": "python", + "source_html": "from nemo_platform.types.jobs import PlatformJobStep\nfrom datetime import datetime\nimport pandas as pd\n\nSTEP_NAME = "customization-training-job"\n\ndef get_elapsed_time(step: PlatformJobStep) -> float:\n """Calculate elapsed time in seconds from step's created_at to updated_at."""\n created_at = datetime.fromisoformat(step.created_at.replace("Z", "+00:00"))\n updated_at = datetime.fromisoformat(step.updated_at.replace("Z", "+00:00"))\n return (updated_at - created_at).total_seconds()\n\nstep_with_sequence_packing = client.jobs.steps.retrieve(\n name=STEP_NAME,\n workspace="default",\n job=job_with_sequence_packing.name,\n)\n\nstep_without_sequence_packing = client.jobs.steps.retrieve(\n name=STEP_NAME,\n workspace="default",\n job=job_without_sequence_packing.name,\n)\n\ntime_to_complete_with_sequence_packing = get_elapsed_time(step_with_sequence_packing)\ntime_to_complete_without_sequence_packing = get_elapsed_time(step_without_sequence_packing)\n\n# Display results as a table\nresults_df = pd.DataFrame({\n "Seq Packing Enabled": [True, False],\n "Val Loss": [\n job_with_sequence_packing_status.status_details['val_loss'],\n job_without_sequence_packing_status.status_details['val_loss']\n ],\n "Training Step Time, sec": [\n time_to_complete_with_sequence_packing,\n time_to_complete_without_sequence_packing\n ]\n})\n\nresults_df.style.format({"Val Loss": "{:.2f}", "Training Step Time, sec": "{:.0f}"}).hide(axis='index')\n" + }, + { + "type": "markdown", + "source": "#### Examples of Validation Loss\n\nThe expected validation loss curves should match closely for both jobs.\n![Validation loss comparison chart showing similar convergence patterns between sequence-packed and non-packed training runs over training steps](../_images/packed_vs_not_packed_val_loss.png)\n\nSequence packed version should complete significantly faster.\n![Runtime comparison chart demonstrating significantly reduced training time for sequence-packed job compared to non-packed baseline](../_images/runtime.png)\n\n#### GPU Utilization\nSequence packed version should have a higher GPU utilization.\n![GPU utilization chart showing higher and more consistent GPU usage with sequence packing enabled throughout the training process](../_images/gpu_utilization.png)\n\n#### GPU Memory Allocation\nSequence packed version should have a higher GPU Memory Allocation.\n![GPU memory allocation chart illustrating increased memory utilization efficiency with sequence packing enabled](../_images/gpu_memory.png)", + "source_html": "

Examples of Validation Loss

\n

The expected validation loss curves should match closely for both jobs.\n\"Validation

\n

Sequence packed version should complete significantly faster.\n\"Runtime

\n

GPU Utilization

\n

Sequence packed version should have a higher GPU utilization.\n\"GPU

\n

GPU Memory Allocation

\n

Sequence packed version should have a higher GPU Memory Allocation.\n\"GPU

\n" + } + ] +} \ No newline at end of file diff --git a/docs/fern/components/notebooks/optimize-throughput.ts b/docs/fern/components/notebooks/optimize-throughput.ts new file mode 100644 index 00000000000..cc11a591bdb --- /dev/null +++ b/docs/fern/components/notebooks/optimize-throughput.ts @@ -0,0 +1,155 @@ +/** Auto-generated by ipynb-to-fern-json.py - do not edit */ +export default { cells: [ + { + "type": "markdown", + "source": "\n\n\n# Optimize for Tokens/GPU Throughput\n\n## About\nLearn how to use the NeMo Platform Customizer to create a [LoRA](nemo-ms-about-concepts-customization) (Low-Rank Adaptation) customization job optimized for higher tokens/GPU throughput and lower runtime. \n\n**In this tutorial, you will:**\n1. Fine-tune [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) on the SQuAD dataset using LoRA, with [sequence packing](nemo-ms-about-concepts-customization) enabled for one run and disabled for another.\n2. Compare training runtime, GPU utilization, and memory allocation between the two runs.\n3. Verify that validation loss remains comparable, confirming that sequence packing improves throughput without sacrificing model quality.\n\n> **Note:** While this tutorial demonstrates sequence packing with LoRA, the optimization is also available for all_weights (full) SFT customization jobs.", + "source_html": "\n\n

Optimize for Tokens/GPU Throughput

\n

About

\n

Learn how to use the NeMo Platform Customizer to create a LoRA (Low-Rank Adaptation) customization job optimized for higher tokens/GPU throughput and lower runtime.

\n

In this tutorial, you will:

\n
    \n
  1. Fine-tune meta-llama/Llama-3.2-1B-Instruct on the SQuAD dataset using LoRA, with sequence packing enabled for one run and disabled for another.
    2. \n
  2. Compare training runtime, GPU utilization, and memory allocation between the two runs.
    3. \n
  3. Verify that validation loss remains comparable, confirming that sequence packing improves throughput without sacrificing model quality.
    4. \n
\n
\n

Note: While this tutorial demonstrates sequence packing with LoRA, the optimization is also available for all_weights (full) SFT customization jobs.

\n
\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n2. **Installed the Python SDK** (included with `pip install nemo-platform`)", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Completed the Quickstart to install and deploy NeMo Platform locally
    2. \n
  2. Installed the Python SDK (included with pip install nemo-platform)
    3. \n
\n" + }, + { + "type": "markdown", + "source": "## Quick Start\n\n### 1. Initialize SDK\n\nThe SDK needs to know your NeMo Platform server URL. By default, `http://localhost:8080` is used in accordance with the [Quickstart](../../get-started/quickstart.md) guide. If NeMo Platform is running at a custom location, you can override the URL by setting the `NMP_BASE_URL` environment variable:\n\n```sh\nexport NMP_BASE_URL=\n```", + "source_html": "

Quick Start

\n

1. Initialize SDK

\n

The SDK needs to know your NeMo Platform server URL. By default, http://localhost:8080 is used in accordance with the Quickstart guide. If NeMo Platform is running at a custom location, you can override the URL by setting the NMP_BASE_URL environment variable:

\n
export NMP_BASE_URL=<YOUR_NMP_BASE_URL>\n
\n" + }, + { + "type": "code", + "source": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace=\"default\"\n)", + "language": "python", + "source_html": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace="default"\n)\n" + }, + { + "type": "markdown", + "source": "### 2. Create Dataset FileSet and Upload Training Data", + "source_html": "

2. Create Dataset FileSet and Upload Training Data

\n" + }, + { + "type": "markdown", + "source": "Install additional dependencies if they are not installed in your Python environment.\n\nThe cell below automatically detects your environment and uses:\n- `uv pip install` if you're in a uv-managed virtual environment\n- `pip install` otherwise\n\nRequired packages:\n- `datasets` - Download the public [rajpurkar/squad](https://huggingface.co/datasets/rajpurkar/squad) dataset\n- `pandas` - Compare job results in table format\n- `matplotlib` - Plot live training metrics (loss curves, GPU utilization)\n- `nvidia-ml-py` - Collect GPU VRAM and compute utilization metrics during training", + "source_html": "

Install additional dependencies if they are not installed in your Python environment.

\n

The cell below automatically detects your environment and uses:

\n
    \n
  • uv pip install if you're in a uv-managed virtual environment
    • \n
  • pip install otherwise
    • \n
\n

Required packages:

\n
    \n
  • datasets - Download the public rajpurkar/squad dataset
    • \n
  • pandas - Compare job results in table format
    • \n
  • matplotlib - Plot live training metrics (loss curves, GPU utilization)
    • \n
  • nvidia-ml-py - Collect GPU VRAM and compute utilization metrics during training
    • \n
\n" + }, + { + "type": "code", + "source": "%%bash\nif command -v uv >/dev/null 2>&1 && [ -n \"$VIRTUAL_ENV\" ]; then\n uv pip install datasets pandas matplotlib nvidia-ml-py\nelse\n pip install datasets pandas matplotlib nvidia-ml-py\nfi", + "language": "python", + "source_html": "%%bash\nif command -v uv >/dev/null 2>&1 && [ -n "$VIRTUAL_ENV" ]; then\n uv pip install datasets pandas matplotlib nvidia-ml-py\nelse\n pip install datasets pandas matplotlib nvidia-ml-py\nfi\n" + }, + { + "type": "markdown", + "source": "#### Download rajpurkar/squad Dataset\n\nSQuAD (Stanford Question Answering Dataset) is a reading comprehension dataset consisting of questions posed on Wikipedia articles, where the answer is a segment of text from the corresponding passage.", + "source_html": "

Download rajpurkar/squad Dataset

\n

SQuAD (Stanford Question Answering Dataset) is a reading comprehension dataset consisting of questions posed on Wikipedia articles, where the answer is a segment of text from the corresponding passage.

\n" + }, + { + "type": "code", + "source": "import json\nimport os\nfrom pathlib import Path\nfrom datasets import load_dataset, Dataset, DatasetDict\n\n# Configuration\nSEED = 1234\nDATASET_NAME = \"sft-dataset\"\n\n# Convert SQuAD format to prompt/completion format and save to JSONL\ndef convert_squad_to_sft_format(example):\n \"\"\"Convert SQuAD format to prompt/completion format for SFT training.\"\"\"\n prompt = f\"Context: {example['context']} Question: {example['question']} Answer:\"\n completion = example[\"answers\"][\"text\"][0] # Take the first answer\n return {\"prompt\": prompt, \"completion\": completion}\n\n# Load the SQuAD dataset from Hugging Face\nprint(\"Loading dataset rajpurkar/squad\")\nds = load_dataset(\"rajpurkar/squad\")\nif not isinstance(ds, DatasetDict):\n raise ValueError(\"Dataset does not contain expected splits\")\n\nprint(\"Loaded dataset\")\n\n# For the purpose of this tutorial, we'll use a subset of the dataset\n# We use a reduced dataset size (3000 training/300 validation samples) to keep tutorial runtime manageable\n# while still demonstrating the performance benefits of sequence packing. The larger the dataset,\n# the better the model will perform but the longer the training will take.\ntraining_size = 3000\nvalidation_size = 300\nDATASET_PATH = Path(DATASET_NAME).absolute()\n\n# Get training split and verify it's a Dataset (not IterableDataset)\ntrain_dataset = ds[\"train\"]\nvalidation_dataset = ds[\"validation\"]\nassert isinstance(train_dataset, Dataset), \"Expected Dataset type\"\nassert isinstance(validation_dataset, Dataset), \"Expected Dataset type\"\n\n# Select subsets and save to JSONL files\ntraining_ds = train_dataset.select(range(training_size))\nvalidation_ds = validation_dataset.select(range(validation_size))\n\n# Transform to SFT format (prompt/completion)\ntraining_ds = training_ds.map(convert_squad_to_sft_format, remove_columns=training_ds.column_names)\nvalidation_ds = validation_ds.map(convert_squad_to_sft_format, remove_columns=validation_ds.column_names)\n\n# Create directory if it doesn't exist\n# Note: This will create a local 'sft-dataset/' directory with training.jsonl and validation.jsonl files\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Save subsets to JSONL files\ntraining_ds.to_json(f\"{DATASET_PATH}/training.jsonl\")\nvalidation_ds.to_json(f\"{DATASET_PATH}/validation.jsonl\")\n\nprint(f\"Saved training.jsonl with {len(training_ds)} rows\")\nprint(f\"Saved validation.jsonl with {len(validation_ds)} rows\")", + "language": "python", + "source_html": "import json\nimport os\nfrom pathlib import Path\nfrom datasets import load_dataset, Dataset, DatasetDict\n\n# Configuration\nSEED = 1234\nDATASET_NAME = "sft-dataset"\n\n# Convert SQuAD format to prompt/completion format and save to JSONL\ndef convert_squad_to_sft_format(example):\n """Convert SQuAD format to prompt/completion format for SFT training."""\n prompt = f"Context: {example['context']} Question: {example['question']} Answer:"\n completion = example["answers"]["text"][0] # Take the first answer\n return {"prompt": prompt, "completion": completion}\n\n# Load the SQuAD dataset from Hugging Face\nprint("Loading dataset rajpurkar/squad")\nds = load_dataset("rajpurkar/squad")\nif not isinstance(ds, DatasetDict):\n raise ValueError("Dataset does not contain expected splits")\n\nprint("Loaded dataset")\n\n# For the purpose of this tutorial, we'll use a subset of the dataset\n# We use a reduced dataset size (3000 training/300 validation samples) to keep tutorial runtime manageable\n# while still demonstrating the performance benefits of sequence packing. The larger the dataset,\n# the better the model will perform but the longer the training will take.\ntraining_size = 3000\nvalidation_size = 300\nDATASET_PATH = Path(DATASET_NAME).absolute()\n\n# Get training split and verify it's a Dataset (not IterableDataset)\ntrain_dataset = ds["train"]\nvalidation_dataset = ds["validation"]\nassert isinstance(train_dataset, Dataset), "Expected Dataset type"\nassert isinstance(validation_dataset, Dataset), "Expected Dataset type"\n\n# Select subsets and save to JSONL files\ntraining_ds = train_dataset.select(range(training_size))\nvalidation_ds = validation_dataset.select(range(validation_size))\n\n# Transform to SFT format (prompt/completion)\ntraining_ds = training_ds.map(convert_squad_to_sft_format, remove_columns=training_ds.column_names)\nvalidation_ds = validation_ds.map(convert_squad_to_sft_format, remove_columns=validation_ds.column_names)\n\n# Create directory if it doesn't exist\n# Note: This will create a local 'sft-dataset/' directory with training.jsonl and validation.jsonl files\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Save subsets to JSONL files\ntraining_ds.to_json(f"{DATASET_PATH}/training.jsonl")\nvalidation_ds.to_json(f"{DATASET_PATH}/validation.jsonl")\n\nprint(f"Saved training.jsonl with {len(training_ds)} rows")\nprint(f"Saved validation.jsonl with {len(validation_ds)} rows")\n" + }, + { + "type": "code", + "source": "# Create fileset to store SFT training data\n\ntry:\n client.files.filesets.create(\n workspace=\"default\",\n name=DATASET_NAME,\n description=\"SFT training data\"\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\n# Upload training data files individually to ensure correct structure\nclient.files.upload(\n local_path=DATASET_PATH, # Local directory with your JSONL files\n remote_path=\"\",\n fileset=DATASET_NAME,\n workspace=\"default\"\n)\n\n# Validate training data is uploaded correctly\nprint(\"Training data:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace=\"default\").data], indent=2))", + "language": "python", + "source_html": "# Create fileset to store SFT training data\n\ntry:\n client.files.filesets.create(\n workspace="default",\n name=DATASET_NAME,\n description="SFT training data"\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\n# Upload training data files individually to ensure correct structure\nclient.files.upload(\n local_path=DATASET_PATH, # Local directory with your JSONL files\n remote_path="",\n fileset=DATASET_NAME,\n workspace="default"\n)\n\n# Validate training data is uploaded correctly\nprint("Training data:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace="default").data], indent=2))\n" + }, + { + "type": "markdown", + "source": "### 3. Secrets Setup\n\nIf you plan to use NGC or HuggingFace models, you will need to configure authentication:\n\n- **NGC models** (`ngc://` URIs): Requires NGC API key\n- **HuggingFace models** (`hf://` URIs): Requires HF token for gated/private models\n\n\nConfigure these as secrets in your platform. Refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md) for detailed instructions.\n\nGet your credentials to access base models:\n- [NGC API Key](https://ngc.nvidia.com/) (Setup → Generate API Key)\n- [HuggingFace Token](https://huggingface.co/settings/tokens) (Create token with Read access)\n\n\n---\n\n#### Quick Setup Example\n\nThis tutorial uses the [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) model from HuggingFace. Ensure that you have sufficient permissions to download the model. If you cannot access the files on the [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) Hugging Face page, request access.\n\n**HuggingFace Authentication:**\n- For gated models (Llama, Gemma), you must provide a HuggingFace token via the `token_secret` parameter\n- Get your token from [HuggingFace Settings](https://huggingface.co/settings/tokens) (requires Read access)\n- Accept the model's terms on the HuggingFace model page before using it. Example: [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main)\n- For public models, you can omit the `token_secret` parameter when creating a fileset for the model in the next step.", + "source_html": "

3. Secrets Setup

\n

If you plan to use NGC or HuggingFace models, you will need to configure authentication:

\n
    \n
  • NGC models (ngc:// URIs): Requires NGC API key
    • \n
  • HuggingFace models (hf:// URIs): Requires HF token for gated/private models
    • \n
\n

Configure these as secrets in your platform. Refer to Managing Secrets for detailed instructions.

\n

Get your credentials to access base models:

\n\n
\n

Quick Setup Example

\n

This tutorial uses the meta-llama/Llama-3.2-1B-Instruct model from HuggingFace. Ensure that you have sufficient permissions to download the model. If you cannot access the files on the meta-llama/Llama-3.2-1B-Instruct Hugging Face page, request access.

\n

HuggingFace Authentication:

\n
    \n
  • For gated models (Llama, Gemma), you must provide a HuggingFace token via the token_secret parameter
    • \n
  • Get your token from HuggingFace Settings (requires Read access)
    • \n
  • Accept the model's terms on the HuggingFace model page before using it. Example: meta-llama/Llama-3.2-1B-Instruct
    • \n
  • For public models, you can omit the token_secret parameter when creating a fileset for the model in the next step.
    • \n
\n" + }, + { + "type": "code", + "source": "# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set\nHF_TOKEN = os.getenv(\"HF_TOKEN\")\nNGC_API_KEY = os.getenv(\"NGC_API_KEY\")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f\"{label} environment variable is not set. Set it and try again.\")\n try:\n secret = client.secrets.create(\n name=name,\n workspace=\"default\",\n value=value,\n )\n print(f\"Created secret: {name}\")\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n return client.secrets.retrieve(name=name, workspace=\"default\")\n\n\n# Create HuggingFace token secret\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")\nprint(\"HF_TOKEN secret:\")\nprint(hf_secret.model_dump_json(indent=2))\n\n# Create NGC API key secret\n# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n# ngc_api_key = create_or_get_secret(\"ngc-api-key\", NGC_API_KEY, \"NGC_API_KEY\")", + "language": "python", + "source_html": "# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set\nHF_TOKEN = os.getenv("HF_TOKEN")\nNGC_API_KEY = os.getenv("NGC_API_KEY")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f"{label} environment variable is not set. Set it and try again.")\n try:\n secret = client.secrets.create(\n name=name,\n workspace="default",\n value=value,\n )\n print(f"Created secret: {name}")\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n return client.secrets.retrieve(name=name, workspace="default")\n\n\n# Create HuggingFace token secret\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\nprint("HF_TOKEN secret:")\nprint(hf_secret.model_dump_json(indent=2))\n\n# Create NGC API key secret\n# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n# ngc_api_key = create_or_get_secret("ngc-api-key", NGC_API_KEY, "NGC_API_KEY")\n" + }, + { + "type": "markdown", + "source": "### 4. Create Base Model FileSet\n\nCreate a fileset pointing to the [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) model on HuggingFace. This step creates a pointer to the model on Hugging Face and does not download it. The model is downloaded at job creation time.\n\nNote: for public models, you can omit the `token_secret` parameter when creating a model fileset.", + "source_html": "

4. Create Base Model FileSet

\n

Create a fileset pointing to the meta-llama/Llama-3.2-1B-Instruct model on HuggingFace. This step creates a pointer to the model on Hugging Face and does not download it. The model is downloaded at job creation time.

\n

Note: for public models, you can omit the token_secret parameter when creating a model fileset.

\n" + }, + { + "type": "code", + "source": "import time\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = \"meta-llama/Llama-3.2-1B-Instruct\"\nMODEL_NAME = \"llama-3-2-1b-base\"\n\n# Ensure you have a HuggingFace token secret created\n# Create a fileset pointing to the desired HuggingFace model\ntry:\n base_model_fs = client.files.filesets.create(\n workspace=\"default\",\n name=MODEL_NAME,\n description=\"Llama 3.2 1B base model from HuggingFace\",\n storage=HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type=\"model\",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\n print(f\"Created base model fileset: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model fileset already exists. Skipping creation.\")\n base_model_fs = client.files.filesets.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\n# Create the Model Entity representation.\ntry:\n base_model = client.models.create(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n )\n print(f\"Created Model Entity: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model already exists. Updating fileset if different.\")\n base_model = client.models.update(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n )\n\nprint(f\"\\nBase model fileset: fileset://default/{base_model.name}\")\nprint(\"Base model fileset files list:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace=\"default\").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint(\"\\nWaiting for ModelSpec to be populated...\")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f\"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds\")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\nprint(f\"ModelSpec populated: {base_model.spec}\")", + "language": "python", + "source_html": "import time\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = "meta-llama/Llama-3.2-1B-Instruct"\nMODEL_NAME = "llama-3-2-1b-base"\n\n# Ensure you have a HuggingFace token secret created\n# Create a fileset pointing to the desired HuggingFace model\ntry:\n base_model_fs = client.files.filesets.create(\n workspace="default",\n name=MODEL_NAME,\n description="Llama 3.2 1B base model from HuggingFace",\n storage=HuggingfaceStorageConfigParam(\n type="huggingface",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type="model",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\n print(f"Created base model fileset: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model fileset already exists. Skipping creation.")\n base_model_fs = client.files.filesets.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\n# Create the Model Entity representation.\ntry:\n base_model = client.models.create(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n )\n print(f"Created Model Entity: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model already exists. Updating fileset if different.")\n base_model = client.models.update(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n )\n\nprint(f"\\nBase model fileset: fileset://default/{base_model.name}")\nprint("Base model fileset files list:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace="default").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint("\\nWaiting for ModelSpec to be populated...")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\nprint(f"ModelSpec populated: {base_model.spec}")\n" + }, + { + "type": "markdown", + "source": "### 5. Create LoRA Job with Sequence Packing\nCreate a customization job with an inline target referencing the base model and dataset filesets created in previous steps.", + "source_html": "

5. Create LoRA Job with Sequence Packing

\n

Create a customization job with an inline target referencing the base model and dataset filesets created in previous steps.

\n" + }, + { + "type": "code", + "source": "import uuid\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam,\n LoRaParamsParam,\n)\n\n# Enable sequence packing to improve throughput and GPU utilization\nSEQUENCE_PACKING_ENABLED = True\n\njob_suffix = uuid.uuid4().hex[:4]\n\nJOB_NAME = f\"my-sft-job-{job_suffix}\"\njob_spec = CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=4096,\n val_check_interval=0.1,\n micro_batch_size=1,\n sequence_packing=SEQUENCE_PACKING_ENABLED,\n peft=LoRaParamsParam(),\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n)\n\njob_with_sequence_packing = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=job_spec\n)\n\nprint(f\"Job ID: {job_with_sequence_packing.name}\")\nprint(f\"Output model: {job_with_sequence_packing.spec.output.name}\")", + "language": "python", + "source_html": "import uuid\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam,\n LoRaParamsParam,\n)\n\n# Enable sequence packing to improve throughput and GPU utilization\nSEQUENCE_PACKING_ENABLED = True\n\njob_suffix = uuid.uuid4().hex[:4]\n\nJOB_NAME = f"my-sft-job-{job_suffix}"\njob_spec = CustomizationJobInputParam(\n model=f"default/{base_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=4096,\n val_check_interval=0.1,\n micro_batch_size=1,\n sequence_packing=SEQUENCE_PACKING_ENABLED,\n peft=LoRaParamsParam(),\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n)\n\njob_with_sequence_packing = client.customization.jobs.create(\n name=JOB_NAME,\n workspace="default",\n spec=job_spec\n)\n\nprint(f"Job ID: {job_with_sequence_packing.name}")\nprint(f"Output model: {job_with_sequence_packing.spec.output.name}")\n" + }, + { + "type": "markdown", + "source": "### 6. Track Finetuning Progress\n\nA training job contains multiple steps: \n- Model and dataset downloading\n- Finetuning where LoRA adapter weights are trained\n- Creating a fileset entry for the finetuned model\n- Finetuned weights uploading\n\nThe elapsed time printed below reflects progress of the entire job. We compare the time taken by the finetuning step for both jobs in the last section of this tutorial.", + "source_html": "

6. Track Finetuning Progress

\n

A training job contains multiple steps:

\n
    \n
  • Model and dataset downloading
    • \n
  • Finetuning where LoRA adapter weights are trained
    • \n
  • Creating a fileset entry for the finetuned model
    • \n
  • Finetuned weights uploading
    • \n
\n

The elapsed time printed below reflects progress of the entire job. We compare the time taken by the finetuning step for both jobs in the last section of this tutorial.

\n" + }, + { + "type": "markdown", + "source": "#### Define Helper Functions", + "source_html": "

Define Helper Functions

\n" + }, + { + "type": "code", + "source": "\n# Helpers to draw GPU VRAM Utilization and Validation Loss\nimport matplotlib.pyplot as plt\ntry:\n import pynvml\n _PYNVML_AVAILABLE = True\nexcept ImportError:\n _PYNVML_AVAILABLE = False\n print(\"Note: Install nvidia-ml-py ('pip install nvidia-ml-py' or 'uv pip install nvidia-ml-py') to enable live GPU metrics.\")\n\n# ---------------------------------------------------------------------------\n# GPU metrics collection (nvidia-ml-py; import name is pynvml)\n# ---------------------------------------------------------------------------\n\ndef _get_gpu_snapshot() -> tuple[list[float], list[float]]:\n \"\"\"Return (vram_usage_pcts, compute_util_pcts) for each GPU.\"\"\"\n if not _PYNVML_AVAILABLE:\n return [], []\n pynvml.nvmlInit()\n try:\n vram, util = [], []\n for i in range(pynvml.nvmlDeviceGetCount()):\n h = pynvml.nvmlDeviceGetHandleByIndex(i)\n mem = pynvml.nvmlDeviceGetMemoryInfo(h)\n rates = pynvml.nvmlDeviceGetUtilizationRates(h)\n vram.append(int(mem.used) / int(mem.total) * 100)\n util.append(float(rates.gpu))\n return vram, util\n finally:\n pynvml.nvmlShutdown()\n\n\n# ---------------------------------------------------------------------------\n# Dashboard drawing helpers\n# ---------------------------------------------------------------------------\n\n_PALETTE = {\n \"val_loss\": \"#E74C3C\",\n \"train_loss\": \"#F39C12\",\n \"vram\": [\"#3498DB\", \"#9B59B6\", \"#1ABC9C\", \"#E67E22\"],\n \"util\": [\"#2ECC71\", \"#E74C3C\", \"#3498DB\", \"#F1C40F\"],\n \"grid\": \"#ECECEC\",\n \"title\": \"#2C3E50\",\n \"subtitle\": \"#7F8C8D\",\n \"spine\": \"#CCCCCC\",\n \"tick\": \"#666666\",\n}\n\n\ndef _style_axis(ax):\n \"\"\"Apply shared cosmetic styling to a subplot axis.\"\"\"\n ax.set_facecolor(\"white\")\n ax.grid(True, alpha=0.4, color=_PALETTE[\"grid\"], linewidth=0.8)\n for spine in (\"top\", \"right\"):\n ax.spines[spine].set_visible(False)\n ax.spines[\"left\"].set_color(_PALETTE[\"spine\"])\n ax.spines[\"bottom\"].set_color(_PALETTE[\"spine\"])\n ax.tick_params(colors=_PALETTE[\"tick\"], labelsize=9)\n\n\ndef _plot_line(ax, xs, ys, color, label, fill=True):\n \"\"\"Plot a time series, gracefully skipping None values.\"\"\"\n pts = [(x, y) for x, y in zip(xs, ys) if y is not None]\n if not pts:\n return\n px, py = zip(*pts)\n ax.plot(\n px, py, color=color, linewidth=2.2,\n marker=\"o\", markersize=4,\n markerfacecolor=\"white\", markeredgewidth=1.8, markeredgecolor=color,\n label=label, zorder=3,\n )\n if fill:\n ax.fill_between(px, py, alpha=0.08, color=color)\n\n\ndef _plot_gpu_panel(ax, xs, history, colors, fallback_label):\n \"\"\"Plot per-GPU time series with area fill.\"\"\"\n if not history or not history[0]:\n ax.text(\n 0.5, 0.5, \"No GPU data\", transform=ax.transAxes,\n ha=\"center\", va=\"center\", fontsize=11, color=\"#AAAAAA\",\n )\n return\n n_gpus = max(len(snap) for snap in history)\n for g in range(n_gpus):\n vals = [snap[g] if g < len(snap) else 0 for snap in history]\n c = colors[g % len(colors)]\n label = f\"GPU {g}\" if n_gpus > 1 else fallback_label\n ax.plot(xs[: len(vals)], vals, color=c, linewidth=2, label=label)\n ax.fill_between(xs[: len(vals)], vals, alpha=0.08, color=c)\n if n_gpus > 1:\n ax.legend(fontsize=9, framealpha=0.9, edgecolor=\"#DDD\")\n\n\ndef _draw_dashboard(\n elapsed_mins, val_losses, train_losses,\n vram_history, util_history,\n job_name, status_str, step_str, elapsed_str,\n):\n \"\"\"Render a live 1x3 training dashboard.\"\"\"\n fig, axes = plt.subplots(1, 3, figsize=(20, 5.5))\n fig.patch.set_facecolor(\"#FAFBFC\")\n\n fig.suptitle(\n job_name, fontsize=15, fontweight=\"bold\",\n color=_PALETTE[\"title\"], y=1.10,\n )\n fig.text(\n 0.5, 1.01,\n f\"{status_str} | {step_str} | {elapsed_str}\",\n ha=\"center\", fontsize=13, color=_PALETTE[\"subtitle\"],\n )\n\n for ax in axes:\n _style_axis(ax)\n\n # -- Panel 1: Loss curves --\n _plot_line(axes[0], elapsed_mins, val_losses, _PALETTE[\"val_loss\"], \"Val Loss\", fill=True)\n _plot_line(axes[0], elapsed_mins, train_losses, _PALETTE[\"train_loss\"], \"Train Loss\", fill=False)\n axes[0].set_title(\"Train/Validation Loss\", fontsize=13, fontweight=\"bold\", color=_PALETTE[\"title\"], pad=12)\n axes[0].set_xlabel(\"Time (min)\", fontsize=10, color=\"#666\")\n axes[0].set_ylabel(\"Loss\", fontsize=10, color=\"#666\")\n if any(v is not None for v in val_losses + train_losses):\n axes[0].legend(fontsize=9, framealpha=0.9, edgecolor=\"#DDD\")\n\n # -- Panel 2: GPU VRAM usage --\n _plot_gpu_panel(axes[1], elapsed_mins, vram_history, _PALETTE[\"vram\"], \"VRAM\")\n axes[1].set_title(\"GPU VRAM Usage\", fontsize=13, fontweight=\"bold\", color=_PALETTE[\"title\"], pad=12)\n axes[1].set_xlabel(\"Time (min)\", fontsize=10, color=\"#666\")\n axes[1].set_ylabel(\"Usage (%)\", fontsize=10, color=\"#666\")\n axes[1].set_ylim(-2, 105)\n\n # -- Panel 3: GPU utilization --\n _plot_gpu_panel(axes[2], elapsed_mins, util_history, _PALETTE[\"util\"], \"Utilization\")\n axes[2].set_title(\"GPU Utilization\", fontsize=13, fontweight=\"bold\", color=_PALETTE[\"title\"], pad=12)\n axes[2].set_xlabel(\"Time (min)\", fontsize=10, color=\"#666\")\n axes[2].set_ylabel(\"Utilization (%)\", fontsize=10, color=\"#666\")\n axes[2].set_ylim(-2, 105)\n\n plt.tight_layout(rect=[0, 0, 1, 0.98])\n plt.show()", + "language": "python", + "source_html": "# Helpers to draw GPU VRAM Utilization and Validation Loss\nimport matplotlib.pyplot as plt\ntry:\n import pynvml\n _PYNVML_AVAILABLE = True\nexcept ImportError:\n _PYNVML_AVAILABLE = False\n print("Note: Install nvidia-ml-py ('pip install nvidia-ml-py' or 'uv pip install nvidia-ml-py') to enable live GPU metrics.")\n\n# ---------------------------------------------------------------------------\n# GPU metrics collection (nvidia-ml-py; import name is pynvml)\n# ---------------------------------------------------------------------------\n\ndef _get_gpu_snapshot() -> tuple[list[float], list[float]]:\n """Return (vram_usage_pcts, compute_util_pcts) for each GPU."""\n if not _PYNVML_AVAILABLE:\n return [], []\n pynvml.nvmlInit()\n try:\n vram, util = [], []\n for i in range(pynvml.nvmlDeviceGetCount()):\n h = pynvml.nvmlDeviceGetHandleByIndex(i)\n mem = pynvml.nvmlDeviceGetMemoryInfo(h)\n rates = pynvml.nvmlDeviceGetUtilizationRates(h)\n vram.append(int(mem.used) / int(mem.total) * 100)\n util.append(float(rates.gpu))\n return vram, util\n finally:\n pynvml.nvmlShutdown()\n\n\n# ---------------------------------------------------------------------------\n# Dashboard drawing helpers\n# ---------------------------------------------------------------------------\n\n_PALETTE = {\n "val_loss": "#E74C3C",\n "train_loss": "#F39C12",\n "vram": ["#3498DB", "#9B59B6", "#1ABC9C", "#E67E22"],\n "util": ["#2ECC71", "#E74C3C", "#3498DB", "#F1C40F"],\n "grid": "#ECECEC",\n "title": "#2C3E50",\n "subtitle": "#7F8C8D",\n "spine": "#CCCCCC",\n "tick": "#666666",\n}\n\n\ndef _style_axis(ax):\n """Apply shared cosmetic styling to a subplot axis."""\n ax.set_facecolor("white")\n ax.grid(True, alpha=0.4, color=_PALETTE["grid"], linewidth=0.8)\n for spine in ("top", "right"):\n ax.spines[spine].set_visible(False)\n ax.spines["left"].set_color(_PALETTE["spine"])\n ax.spines["bottom"].set_color(_PALETTE["spine"])\n ax.tick_params(colors=_PALETTE["tick"], labelsize=9)\n\n\ndef _plot_line(ax, xs, ys, color, label, fill=True):\n """Plot a time series, gracefully skipping None values."""\n pts = [(x, y) for x, y in zip(xs, ys) if y is not None]\n if not pts:\n return\n px, py = zip(*pts)\n ax.plot(\n px, py, color=color, linewidth=2.2,\n marker="o", markersize=4,\n markerfacecolor="white", markeredgewidth=1.8, markeredgecolor=color,\n label=label, zorder=3,\n )\n if fill:\n ax.fill_between(px, py, alpha=0.08, color=color)\n\n\ndef _plot_gpu_panel(ax, xs, history, colors, fallback_label):\n """Plot per-GPU time series with area fill."""\n if not history or not history[0]:\n ax.text(\n 0.5, 0.5, "No GPU data", transform=ax.transAxes,\n ha="center", va="center", fontsize=11, color="#AAAAAA",\n )\n return\n n_gpus = max(len(snap) for snap in history)\n for g in range(n_gpus):\n vals = [snap[g] if g < len(snap) else 0 for snap in history]\n c = colors[g % len(colors)]\n label = f"GPU {g}" if n_gpus > 1 else fallback_label\n ax.plot(xs[: len(vals)], vals, color=c, linewidth=2, label=label)\n ax.fill_between(xs[: len(vals)], vals, alpha=0.08, color=c)\n if n_gpus > 1:\n ax.legend(fontsize=9, framealpha=0.9, edgecolor="#DDD")\n\n\ndef _draw_dashboard(\n elapsed_mins, val_losses, train_losses,\n vram_history, util_history,\n job_name, status_str, step_str, elapsed_str,\n):\n """Render a live 1x3 training dashboard."""\n fig, axes = plt.subplots(1, 3, figsize=(20, 5.5))\n fig.patch.set_facecolor("#FAFBFC")\n\n fig.suptitle(\n job_name, fontsize=15, fontweight="bold",\n color=_PALETTE["title"], y=1.10,\n )\n fig.text(\n 0.5, 1.01,\n f"{status_str} | {step_str} | {elapsed_str}",\n ha="center", fontsize=13, color=_PALETTE["subtitle"],\n )\n\n for ax in axes:\n _style_axis(ax)\n\n # -- Panel 1: Loss curves --\n _plot_line(axes[0], elapsed_mins, val_losses, _PALETTE["val_loss"], "Val Loss", fill=True)\n _plot_line(axes[0], elapsed_mins, train_losses, _PALETTE["train_loss"], "Train Loss", fill=False)\n axes[0].set_title("Train/Validation Loss", fontsize=13, fontweight="bold", color=_PALETTE["title"], pad=12)\n axes[0].set_xlabel("Time (min)", fontsize=10, color="#666")\n axes[0].set_ylabel("Loss", fontsize=10, color="#666")\n if any(v is not None for v in val_losses + train_losses):\n axes[0].legend(fontsize=9, framealpha=0.9, edgecolor="#DDD")\n\n # -- Panel 2: GPU VRAM usage --\n _plot_gpu_panel(axes[1], elapsed_mins, vram_history, _PALETTE["vram"], "VRAM")\n axes[1].set_title("GPU VRAM Usage", fontsize=13, fontweight="bold", color=_PALETTE["title"], pad=12)\n axes[1].set_xlabel("Time (min)", fontsize=10, color="#666")\n axes[1].set_ylabel("Usage (%)", fontsize=10, color="#666")\n axes[1].set_ylim(-2, 105)\n\n # -- Panel 3: GPU utilization --\n _plot_gpu_panel(axes[2], elapsed_mins, util_history, _PALETTE["util"], "Utilization")\n axes[2].set_title("GPU Utilization", fontsize=13, fontweight="bold", color=_PALETTE["title"], pad=12)\n axes[2].set_xlabel("Time (min)", fontsize=10, color="#666")\n axes[2].set_ylabel("Utilization (%)", fontsize=10, color="#666")\n axes[2].set_ylim(-2, 105)\n\n plt.tight_layout(rect=[0, 0, 1, 0.98])\n plt.show()\n" + }, + { + "type": "markdown", + "source": "#### Monitor the Job Until Completion\n\nThe cell below polls the job status every 10 seconds and renders a live dashboard with validation loss, GPU VRAM usage, and GPU utilization charts. The charts appear empty at first while the model and dataset download; training metrics and GPU activity populate after the finetuning step begins.\n\n> **Note:** This is additional code. You can also use the Weights & Biases or MLflow integrations.", + "source_html": "

Monitor the Job Until Completion

\n

The cell below polls the job status every 10 seconds and renders a live dashboard with validation loss, GPU VRAM usage, and GPU utilization charts. The charts appear empty at first while the model and dataset download; training metrics and GPU activity populate after the finetuning step begins.

\n
\n

Note: This is additional code. You can also use the Weights & Biases or MLflow integrations.

\n
\n" + }, + { + "type": "code", + "source": "import time\nfrom typing import cast\nfrom IPython.display import clear_output\nfrom nemo_platform.types.shared import PlatformJobStatusResponse\n\n# Timeout set to 30 minutes to accommodate typical LoRA training duration for this dataset size.\n# Actual training time will vary based on hardware, model size, and dataset complexity.\nTIMEOUT_SECONDS = 30 * 60 # 30 minutes\nVAL_LOSS_KEY = \"val_loss\"\nTRAIN_LOSS_KEY = \"loss\"\n\n# ---------------------------------------------------------------------------\n# Job polling with live dashboard\n# ---------------------------------------------------------------------------\n\ndef wait_for_job(\n workspace: str,\n job_name: str,\n timeout: int = TIMEOUT_SECONDS,\n poll_interval: int = 10,\n val_loss_key: str = VAL_LOSS_KEY,\n train_loss_key: str = TRAIN_LOSS_KEY,\n) -> PlatformJobStatusResponse:\n \"\"\"\n Poll job status until completed, failed, cancelled, or timeout.\n Displays a live dashboard with loss curves and GPU metrics.\n\n Args:\n workspace: The workspace where the job is running.\n job_name: The name of the job to monitor.\n timeout: Maximum time to wait in seconds (default: 30 minutes).\n poll_interval: Time between status checks in seconds (default: 10).\n\n Returns:\n The final job status response.\n \"\"\"\n start_time = time.time()\n\n # Time-series accumulators required for plotting\n elapsed_mins: list[float] = []\n val_losses: list[float | None] = []\n train_losses: list[float | None] = []\n vram_history: list[list[float]] = []\n util_history: list[list[float]] = []\n\n while True:\n elapsed = time.time() - start_time\n elapsed_min = elapsed / 60\n\n # Check for timeout\n if elapsed > timeout:\n error_message = f\"Timeout reached after {elapsed_min:.1f} minutes\"\n print(f\"\\n{error_message}\")\n print(\"Job did not complete within the timeout period.\")\n raise Exception(error_message)\n\n status = client.jobs.get_status(name=job_name, workspace=workspace)\n\n # -- Extract training progress from nested steps structure --\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n val_loss: float | None = None\n train_loss: float | None = None\n current_step_name: str | None = None\n current_step_phase: str | None = None\n\n for job_step in status.steps or []:\n # Track the current active step name and phase for progress display\n if job_step.tasks:\n task = job_step.tasks[0]\n td = task.status_details or {}\n phase = cast(str, td.get(\"phase\", \"\"))\n # Update current step if it's active or pending (not completed)\n if job_step.status in (\"active\", \"pending\"):\n current_step_name = job_step.name\n current_step_phase = phase or \"started\"\n\n if job_step.name == \"customization-training-job\":\n for task in job_step.tasks or []:\n td = task.status_details or {}\n step = cast(int, td[\"step\"]) if \"step\" in td else None\n max_steps = cast(int, td[\"max_steps\"]) if \"max_steps\" in td else None\n training_phase = cast(str, td[\"phase\"]) if \"phase\" in td else None\n val_loss = float(td[val_loss_key]) if val_loss_key in td else None\n train_loss = float(td[train_loss_key]) if train_loss_key in td else None\n break\n break\n\n # Fall back to top-level status_details\n if status.status_details:\n if val_loss is None and val_loss_key in status.status_details:\n val_loss = float(status.status_details[val_loss_key])\n if train_loss is None and train_loss_key in status.status_details:\n train_loss = float(status.status_details[train_loss_key])\n\n # -- Collect GPU snapshot --\n vram_pcts, util_pcts = _get_gpu_snapshot()\n\n # -- Append to accumulators used for the plots --\n elapsed_mins.append(elapsed_min)\n val_losses.append(val_loss)\n train_losses.append(train_loss)\n vram_history.append(vram_pcts)\n util_history.append(util_pcts)\n\n # -- Build status strings --\n status_str = f\"Status: {status.status}\"\n if step is not None and max_steps is not None:\n pct = step / max_steps * 100\n step_str = f\"Step {step}/{max_steps} ({pct:.0f}%)\"\n if training_phase:\n step_str += f\" - {training_phase}\"\n else:\n if current_step_name and current_step_phase:\n step_str = f\"{current_step_name} - {current_step_phase}\"\n elif current_step_name:\n step_str = f\"{current_step_name}\"\n else:\n step_str = \"Waiting for training to start...\"\n elapsed_str = f\"Elapsed: {elapsed_min:.1f} min\"\n\n # -- Redraw dashboard --\n clear_output(wait=True)\n _draw_dashboard(\n elapsed_mins, val_losses, train_losses,\n vram_history, util_history,\n job_name, status_str, step_str, elapsed_str,\n )\n\n # -- Check terminal conditions --\n if status.status.lower() == \"completed\":\n # Redraw dashboard one final time with \"completed\" status\n status_str = f\"Status: {status.status}\"\n if step is not None and max_steps is not None:\n step_str = f\"Step {max_steps}/{max_steps} (100%)\"\n clear_output(wait=True)\n _draw_dashboard(\n elapsed_mins, val_losses, train_losses,\n vram_history, util_history,\n job_name, status_str, step_str, elapsed_str,\n )\n print(f\"\\nJob completed in {elapsed_min:.1f} minutes ({elapsed:.0f}s)\")\n return status\n elif status.status.lower() in (\"failed\", \"cancelled\", \"error\"):\n print(f\"\\nJob finished with status: {status.status}\")\n print(f\"Total time elapsed: {elapsed_min:.1f} minutes ({elapsed:.0f}s)\")\n\n # Print error details from the job level\n if status.error_details:\n error_msg = status.error_details.get(\"message\", \"\")\n if error_msg:\n print(f\"\\nError: {error_msg}\")\n\n # Find and print error details from the failed step/task\n for job_step in status.steps or []:\n if job_step.status == \"error\":\n print(f\"\\nFailed step: {job_step.name}\")\n if job_step.error_details:\n step_error = job_step.error_details.get(\"message\", \"\")\n if step_error:\n print(f\"Step error: {step_error}\")\n # Get error_stack from the failed task\n for task in job_step.tasks or []:\n if task.status == \"error\" and hasattr(task, \"error_stack\") and task.error_stack:\n print(f\"\\nError stack trace:\\n{task.error_stack}\")\n elif task.status == \"error\" and task.error_details:\n task_error = task.error_details.get(\"message\", \"\")\n if task_error:\n print(f\"Task error: {task_error}\")\n break\n\n raise Exception(f\"Job finished with status: {status.status}\")\n\n time.sleep(poll_interval)\n\n\n# Wait for the job to complete\njob_with_sequence_packing_status = wait_for_job(\n workspace=\"default\",\n job_name=job_with_sequence_packing.name,\n timeout=TIMEOUT_SECONDS,\n)\n\nprint(f\"Validation loss: {job_with_sequence_packing_status.status_details['val_loss']:.2f}\")", + "language": "python", + "source_html": "import time\nfrom typing import cast\nfrom IPython.display import clear_output\nfrom nemo_platform.types.shared import PlatformJobStatusResponse\n\n# Timeout set to 30 minutes to accommodate typical LoRA training duration for this dataset size.\n# Actual training time will vary based on hardware, model size, and dataset complexity.\nTIMEOUT_SECONDS = 30 * 60 # 30 minutes\nVAL_LOSS_KEY = "val_loss"\nTRAIN_LOSS_KEY = "loss"\n\n# ---------------------------------------------------------------------------\n# Job polling with live dashboard\n# ---------------------------------------------------------------------------\n\ndef wait_for_job(\n workspace: str,\n job_name: str,\n timeout: int = TIMEOUT_SECONDS,\n poll_interval: int = 10,\n val_loss_key: str = VAL_LOSS_KEY,\n train_loss_key: str = TRAIN_LOSS_KEY,\n) -> PlatformJobStatusResponse:\n """\n Poll job status until completed, failed, cancelled, or timeout.\n Displays a live dashboard with loss curves and GPU metrics.\n\n Args:\n workspace: The workspace where the job is running.\n job_name: The name of the job to monitor.\n timeout: Maximum time to wait in seconds (default: 30 minutes).\n poll_interval: Time between status checks in seconds (default: 10).\n\n Returns:\n The final job status response.\n """\n start_time = time.time()\n\n # Time-series accumulators required for plotting\n elapsed_mins: list[float] = []\n val_losses: list[float | None] = []\n train_losses: list[float | None] = []\n vram_history: list[list[float]] = []\n util_history: list[list[float]] = []\n\n while True:\n elapsed = time.time() - start_time\n elapsed_min = elapsed / 60\n\n # Check for timeout\n if elapsed > timeout:\n error_message = f"Timeout reached after {elapsed_min:.1f} minutes"\n print(f"\\n{error_message}")\n print("Job did not complete within the timeout period.")\n raise Exception(error_message)\n\n status = client.jobs.get_status(name=job_name, workspace=workspace)\n\n # -- Extract training progress from nested steps structure --\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n val_loss: float | None = None\n train_loss: float | None = None\n current_step_name: str | None = None\n current_step_phase: str | None = None\n\n for job_step in status.steps or []:\n # Track the current active step name and phase for progress display\n if job_step.tasks:\n task = job_step.tasks[0]\n td = task.status_details or {}\n phase = cast(str, td.get("phase", ""))\n # Update current step if it's active or pending (not completed)\n if job_step.status in ("active", "pending"):\n current_step_name = job_step.name\n current_step_phase = phase or "started"\n\n if job_step.name == "customization-training-job":\n for task in job_step.tasks or []:\n td = task.status_details or {}\n step = cast(int, td["step"]) if "step" in td else None\n max_steps = cast(int, td["max_steps"]) if "max_steps" in td else None\n training_phase = cast(str, td["phase"]) if "phase" in td else None\n val_loss = float(td[val_loss_key]) if val_loss_key in td else None\n train_loss = float(td[train_loss_key]) if train_loss_key in td else None\n break\n break\n\n # Fall back to top-level status_details\n if status.status_details:\n if val_loss is None and val_loss_key in status.status_details:\n val_loss = float(status.status_details[val_loss_key])\n if train_loss is None and train_loss_key in status.status_details:\n train_loss = float(status.status_details[train_loss_key])\n\n # -- Collect GPU snapshot --\n vram_pcts, util_pcts = _get_gpu_snapshot()\n\n # -- Append to accumulators used for the plots --\n elapsed_mins.append(elapsed_min)\n val_losses.append(val_loss)\n train_losses.append(train_loss)\n vram_history.append(vram_pcts)\n util_history.append(util_pcts)\n\n # -- Build status strings --\n status_str = f"Status: {status.status}"\n if step is not None and max_steps is not None:\n pct = step / max_steps * 100\n step_str = f"Step {step}/{max_steps} ({pct:.0f}%)"\n if training_phase:\n step_str += f" - {training_phase}"\n else:\n if current_step_name and current_step_phase:\n step_str = f"{current_step_name} - {current_step_phase}"\n elif current_step_name:\n step_str = f"{current_step_name}"\n else:\n step_str = "Waiting for training to start..."\n elapsed_str = f"Elapsed: {elapsed_min:.1f} min"\n\n # -- Redraw dashboard --\n clear_output(wait=True)\n _draw_dashboard(\n elapsed_mins, val_losses, train_losses,\n vram_history, util_history,\n job_name, status_str, step_str, elapsed_str,\n )\n\n # -- Check terminal conditions --\n if status.status.lower() == "completed":\n # Redraw dashboard one final time with "completed" status\n status_str = f"Status: {status.status}"\n if step is not None and max_steps is not None:\n step_str = f"Step {max_steps}/{max_steps} (100%)"\n clear_output(wait=True)\n _draw_dashboard(\n elapsed_mins, val_losses, train_losses,\n vram_history, util_history,\n job_name, status_str, step_str, elapsed_str,\n )\n print(f"\\nJob completed in {elapsed_min:.1f} minutes ({elapsed:.0f}s)")\n return status\n elif status.status.lower() in ("failed", "cancelled", "error"):\n print(f"\\nJob finished with status: {status.status}")\n print(f"Total time elapsed: {elapsed_min:.1f} minutes ({elapsed:.0f}s)")\n\n # Print error details from the job level\n if status.error_details:\n error_msg = status.error_details.get("message", "")\n if error_msg:\n print(f"\\nError: {error_msg}")\n\n # Find and print error details from the failed step/task\n for job_step in status.steps or []:\n if job_step.status == "error":\n print(f"\\nFailed step: {job_step.name}")\n if job_step.error_details:\n step_error = job_step.error_details.get("message", "")\n if step_error:\n print(f"Step error: {step_error}")\n # Get error_stack from the failed task\n for task in job_step.tasks or []:\n if task.status == "error" and hasattr(task, "error_stack") and task.error_stack:\n print(f"\\nError stack trace:\\n{task.error_stack}")\n elif task.status == "error" and task.error_details:\n task_error = task.error_details.get("message", "")\n if task_error:\n print(f"Task error: {task_error}")\n break\n\n raise Exception(f"Job finished with status: {status.status}")\n\n time.sleep(poll_interval)\n\n\n# Wait for the job to complete\njob_with_sequence_packing_status = wait_for_job(\n workspace="default",\n job_name=job_with_sequence_packing.name,\n timeout=TIMEOUT_SECONDS,\n)\n\nprint(f"Validation loss: {job_with_sequence_packing_status.status_details['val_loss']:.2f}")\n" + }, + { + "type": "markdown", + "source": "### 7. Create LoRA Job without Sequence Packing\nCreate a customization job with sequence packing disabled. It's expected to take longer to complete.", + "source_html": "

7. Create LoRA Job without Sequence Packing

\n

Create a customization job with sequence packing disabled. It's expected to take longer to complete.

\n" + }, + { + "type": "code", + "source": "import uuid\n\njob_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f\"my-sft-job-{job_suffix}\"\n\njob_spec_no_packing = CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=4096,\n val_check_interval=0.1,\n micro_batch_size=1,\n sequence_packing=False,\n peft=LoRaParamsParam(),\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n)\n\njob_without_sequence_packing = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=job_spec_no_packing\n)\n\nprint(f\"Job ID: {job_without_sequence_packing.name}\")\nprint(f\"Output model: {job_without_sequence_packing.spec.output.name}\")", + "language": "python", + "source_html": "import uuid\n\njob_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f"my-sft-job-{job_suffix}"\n\njob_spec_no_packing = CustomizationJobInputParam(\n model=f"default/{base_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=1,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=4096,\n val_check_interval=0.1,\n micro_batch_size=1,\n sequence_packing=False,\n peft=LoRaParamsParam(),\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n )\n)\n\njob_without_sequence_packing = client.customization.jobs.create(\n name=JOB_NAME,\n workspace="default",\n spec=job_spec_no_packing\n)\n\nprint(f"Job ID: {job_without_sequence_packing.name}")\nprint(f"Output model: {job_without_sequence_packing.spec.output.name}")\n" + }, + { + "type": "markdown", + "source": "### 8. Track Finetuning Progress for Job without Sequence Packing", + "source_html": "

8. Track Finetuning Progress for Job without Sequence Packing

\n" + }, + { + "type": "code", + "source": "# Wait for the training step to complete\njob_without_sequence_packing_status = wait_for_job(\n workspace=\"default\",\n job_name=job_without_sequence_packing.name,\n timeout=TIMEOUT_SECONDS\n)\n\nprint(f\"Validation loss: {job_without_sequence_packing_status.status_details['val_loss']:.2f}\")", + "language": "python", + "source_html": "# Wait for the training step to complete\njob_without_sequence_packing_status = wait_for_job(\n workspace="default",\n job_name=job_without_sequence_packing.name,\n timeout=TIMEOUT_SECONDS\n)\n\nprint(f"Validation loss: {job_without_sequence_packing_status.status_details['val_loss']:.2f}")\n" + }, + { + "type": "markdown", + "source": "### 9. Compare Results\n- Time to complete training should be significantly lower for the job that used sequence packing.\n- The expected validation loss for both jobs should be similar.\n- Sequence packed version should have a higher GPU utilization and higher GPU Memory Allocation.", + "source_html": "

9. Compare Results

\n
    \n
  • Time to complete training should be significantly lower for the job that used sequence packing.
    • \n
  • The expected validation loss for both jobs should be similar.
    • \n
  • Sequence packed version should have a higher GPU utilization and higher GPU Memory Allocation.
    • \n
\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.jobs import PlatformJobStep\nfrom datetime import datetime\nimport pandas as pd\n\nSTEP_NAME = \"customization-training-job\"\n\ndef get_elapsed_time(step: PlatformJobStep) -> float:\n \"\"\"Calculate elapsed time in seconds from step's created_at to updated_at.\"\"\"\n created_at = datetime.fromisoformat(step.created_at.replace(\"Z\", \"+00:00\"))\n updated_at = datetime.fromisoformat(step.updated_at.replace(\"Z\", \"+00:00\"))\n return (updated_at - created_at).total_seconds()\n\nstep_with_sequence_packing = client.jobs.steps.retrieve(\n name=STEP_NAME,\n workspace=\"default\",\n job=job_with_sequence_packing.name,\n)\n\nstep_without_sequence_packing = client.jobs.steps.retrieve(\n name=STEP_NAME,\n workspace=\"default\",\n job=job_without_sequence_packing.name,\n)\n\ntime_to_complete_with_sequence_packing = get_elapsed_time(step_with_sequence_packing)\ntime_to_complete_without_sequence_packing = get_elapsed_time(step_without_sequence_packing)\n\n# Display results as a table\nresults_df = pd.DataFrame({\n \"Seq Packing Enabled\": [True, False],\n \"Val Loss\": [\n job_with_sequence_packing_status.status_details['val_loss'],\n job_without_sequence_packing_status.status_details['val_loss']\n ],\n \"Training Step Time, sec\": [\n time_to_complete_with_sequence_packing,\n time_to_complete_without_sequence_packing\n ]\n})\n\nresults_df.style.format({\"Val Loss\": \"{:.2f}\", \"Training Step Time, sec\": \"{:.0f}\"}).hide(axis='index')", + "language": "python", + "source_html": "from nemo_platform.types.jobs import PlatformJobStep\nfrom datetime import datetime\nimport pandas as pd\n\nSTEP_NAME = "customization-training-job"\n\ndef get_elapsed_time(step: PlatformJobStep) -> float:\n """Calculate elapsed time in seconds from step's created_at to updated_at."""\n created_at = datetime.fromisoformat(step.created_at.replace("Z", "+00:00"))\n updated_at = datetime.fromisoformat(step.updated_at.replace("Z", "+00:00"))\n return (updated_at - created_at).total_seconds()\n\nstep_with_sequence_packing = client.jobs.steps.retrieve(\n name=STEP_NAME,\n workspace="default",\n job=job_with_sequence_packing.name,\n)\n\nstep_without_sequence_packing = client.jobs.steps.retrieve(\n name=STEP_NAME,\n workspace="default",\n job=job_without_sequence_packing.name,\n)\n\ntime_to_complete_with_sequence_packing = get_elapsed_time(step_with_sequence_packing)\ntime_to_complete_without_sequence_packing = get_elapsed_time(step_without_sequence_packing)\n\n# Display results as a table\nresults_df = pd.DataFrame({\n "Seq Packing Enabled": [True, False],\n "Val Loss": [\n job_with_sequence_packing_status.status_details['val_loss'],\n job_without_sequence_packing_status.status_details['val_loss']\n ],\n "Training Step Time, sec": [\n time_to_complete_with_sequence_packing,\n time_to_complete_without_sequence_packing\n ]\n})\n\nresults_df.style.format({"Val Loss": "{:.2f}", "Training Step Time, sec": "{:.0f}"}).hide(axis='index')\n" + }, + { + "type": "markdown", + "source": "#### Examples of Validation Loss\n\nThe expected validation loss curves should match closely for both jobs.\n![Validation loss comparison chart showing similar convergence patterns between sequence-packed and non-packed training runs over training steps](../_images/packed_vs_not_packed_val_loss.png)\n\nSequence packed version should complete significantly faster.\n![Runtime comparison chart demonstrating significantly reduced training time for sequence-packed job compared to non-packed baseline](../_images/runtime.png)\n\n#### GPU Utilization\nSequence packed version should have a higher GPU utilization.\n![GPU utilization chart showing higher and more consistent GPU usage with sequence packing enabled throughout the training process](../_images/gpu_utilization.png)\n\n#### GPU Memory Allocation\nSequence packed version should have a higher GPU Memory Allocation.\n![GPU memory allocation chart illustrating increased memory utilization efficiency with sequence packing enabled](../_images/gpu_memory.png)", + "source_html": "

Examples of Validation Loss

\n

The expected validation loss curves should match closely for both jobs.\n\"Validation

\n

Sequence packed version should complete significantly faster.\n\"Runtime

\n

GPU Utilization

\n

Sequence packed version should have a higher GPU utilization.\n\"GPU

\n

GPU Memory Allocation

\n

Sequence packed version should have a higher GPU Memory Allocation.\n\"GPU

\n" + } +] }; diff --git a/docs/fern/components/notebooks/sft-customization-job.json b/docs/fern/components/notebooks/sft-customization-job.json new file mode 100644 index 00000000000..139e4d99d14 --- /dev/null +++ b/docs/fern/components/notebooks/sft-customization-job.json @@ -0,0 +1,195 @@ +{ + "cells": [ + { + "type": "markdown", + "source": "\n\n\n# Full SFT Customization\n\nLearn how to fine-tune all model weights using supervised fine-tuning (SFT) to customize LLM behavior for your specific tasks.\n\n## About\n\nSupervised Fine-Tuning (SFT) customizes model behavior, injects new knowledge, and optimizes performance for specific domains and tasks. Full SFT modifies **all model weights** during training, providing maximum customization flexibility.\n\n**What you can achieve with SFT:**\n\n- 🎯 **Specialize for domains:** Fine-tune models on legal texts, medical records, or financial data\n- 💡 **Inject knowledge:** Add new information not present in the base model\n- 📈 **Improve accuracy:** Optimize for specific tasks like sentiment analysis, summarization, or code generation\n\n### SFT vs LoRA: Understanding the Trade-offs\n\n**Full SFT** trains all model parameters (for example, all 70 billion weights in Llama 70B):\n\n- ✅ Maximum model adaptation and knowledge injection\n- ✅ Can fundamentally change model behavior\n- ✅ Best for significant domain shifts or specialized tasks\n- ❌ Requires substantial GPU resources (4-8x more than LoRA)\n- ❌ Produces full model weights (~140GB for Llama 70B)\n- ❌ Longer training time\n\n**LoRA** trains only ~1% of weights by adding thin matrices to existing weights:\n\n- ✅ 75-95% less memory required\n- ✅ Faster training (2-4x speedup)\n- ✅ Produces small adapter files (~100-500MB)\n- ✅ Multiple adapters can share one base model\n- ❌ Limited adaptation capability compared to full fine-tuning\n\n**When to choose Full SFT:**\n\n- Training small models (1B-8B) where resource cost is manageable\n- Need fundamental behavior changes (for example, medical diagnosis, legal reasoning)\n- Injecting substantial new knowledge not in the base model\n\n**When to choose LoRA:** Refer to the [LoRA tutorial](./lora-customization-job) for most use cases, especially with large models (70B+) or limited GPU resources.", + "source_html": "\n\n

Full SFT Customization

\n

Learn how to fine-tune all model weights using supervised fine-tuning (SFT) to customize LLM behavior for your specific tasks.

\n

About

\n

Supervised Fine-Tuning (SFT) customizes model behavior, injects new knowledge, and optimizes performance for specific domains and tasks. Full SFT modifies all model weights during training, providing maximum customization flexibility.

\n

What you can achieve with SFT:

\n
    \n
  • 🎯 Specialize for domains: Fine-tune models on legal texts, medical records, or financial data
    • \n
  • 💡 Inject knowledge: Add new information not present in the base model
    • \n
  • 📈 Improve accuracy: Optimize for specific tasks like sentiment analysis, summarization, or code generation
    • \n
\n

SFT vs LoRA: Understanding the Trade-offs

\n

Full SFT trains all model parameters (for example, all 70 billion weights in Llama 70B):

\n
    \n
  • ✅ Maximum model adaptation and knowledge injection
    • \n
  • ✅ Can fundamentally change model behavior
    • \n
  • ✅ Best for significant domain shifts or specialized tasks
    • \n
  • ❌ Requires substantial GPU resources (4-8x more than LoRA)
    • \n
  • ❌ Produces full model weights (~140GB for Llama 70B)
    • \n
  • ❌ Longer training time
    • \n
\n

LoRA trains only ~1% of weights by adding thin matrices to existing weights:

\n
    \n
  • ✅ 75-95% less memory required
    • \n
  • ✅ Faster training (2-4x speedup)
    • \n
  • ✅ Produces small adapter files (~100-500MB)
    • \n
  • ✅ Multiple adapters can share one base model
    • \n
  • ❌ Limited adaptation capability compared to full fine-tuning
    • \n
\n

When to choose Full SFT:

\n
    \n
  • Training small models (1B-8B) where resource cost is manageable
    • \n
  • Need fundamental behavior changes (for example, medical diagnosis, legal reasoning)
    • \n
  • Injecting substantial new knowledge not in the base model
    • \n
\n

When to choose LoRA: Refer to the LoRA tutorial for most use cases, especially with large models (70B+) or limited GPU resources.

\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n2. **Installed the Python SDK** (included with `pip install nemo-platform`)", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Completed the Quickstart to install and deploy NeMo Platform locally
    2. \n
  2. Installed the Python SDK (included with pip install nemo-platform)
    3. \n
\n" + }, + { + "type": "markdown", + "source": "## Quick Start\n\n### 1. Initialize SDK\n\nThe SDK needs to know your NeMo Platform server URL. By default, `http://localhost:8080` is used in accordance with the [Quickstart](../../get-started/quickstart.md) guide. If NeMo Platform is running at a custom location, you can override the URL by setting the `NMP_BASE_URL` environment variable:\n\n```sh\nexport NMP_BASE_URL=\n```", + "source_html": "

Quick Start

\n

1. Initialize SDK

\n

The SDK needs to know your NeMo Platform server URL. By default, http://localhost:8080 is used in accordance with the Quickstart guide. If NeMo Platform is running at a custom location, you can override the URL by setting the NMP_BASE_URL environment variable:

\n
export NMP_BASE_URL=<YOUR_NMP_BASE_URL>\n
\n" + }, + { + "type": "code", + "source": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace=\"default\"\n)", + "language": "python", + "source_html": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace="default"\n)\n" + }, + { + "type": "markdown", + "source": "### 2. Prepare Dataset\n\nCreate your data in JSONL format—one JSON object per line. The platform auto-detects your data format. Supported dataset formats are listed below.\n\n**Flexible Data Setup:**\n- **No validation file?** The platform automatically creates a 10% validation split\n- **Multiple files?** Upload to `training/` or `validation/` subdirectories—they will be automatically merged\n- **Format detection:** Your data format is auto-detected at training time\n\nIn this tutorial the following dataset directory structure will be used:\n```\nmy_dataset\n`-- training.jsonl\n`-- validation.jsonl\n```", + "source_html": "

2. Prepare Dataset

\n

Create your data in JSONL format—one JSON object per line. The platform auto-detects your data format. Supported dataset formats are listed below.

\n

Flexible Data Setup:

\n
    \n
  • No validation file? The platform automatically creates a 10% validation split
    • \n
  • Multiple files? Upload to training/ or validation/ subdirectories—they will be automatically merged
    • \n
  • Format detection: Your data format is auto-detected at training time
    • \n
\n

In this tutorial the following dataset directory structure will be used:

\n
my_dataset\n`-- training.jsonl\n`-- validation.jsonl\n
\n" + }, + { + "type": "markdown", + "source": "#### Simple Prompt/Completion Format\nThe simplest format with input prompt and expected completion:\n- **`prompt`**: The input prompt for the model\n- **`completion`**: The expected output response", + "source_html": "

Simple Prompt/Completion Format

\n

The simplest format with input prompt and expected completion:

\n
    \n
  • prompt: The input prompt for the model
    • \n
  • completion: The expected output response
    • \n
\n" + }, + { + "type": "markdown", + "source": "```json\n{\"prompt\": \"Write an email to confirm our hotel reservation.\", \"completion\": \"Dear Hotel Team, I am writing to confirm our reservation for two guests...\"}\n```", + "source_html": "
{"prompt": "Write an email to confirm our hotel reservation.", "completion": "Dear Hotel Team, I am writing to confirm our reservation for two guests..."}\n
\n" + }, + { + "type": "markdown", + "source": "#### Chat Format (for conversational models)\nFor multi-turn conversations, use the messages format:\n- **`messages`**: List of message objects with `role` and `content` fields\n- Roles: `system`, `user`, `assistant`", + "source_html": "

Chat Format (for conversational models)

\n

For multi-turn conversations, use the messages format:

\n
    \n
  • messages: List of message objects with role and content fields
    • \n
  • Roles: system, user, assistant
    • \n
\n" + }, + { + "type": "markdown", + "source": "```json\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}, {\"role\": \"user\", \"content\": \"What is AI?\"}, {\"role\": \"assistant\", \"content\": \"AI is...\"}]}\n```", + "source_html": "
{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "What is AI?"}, {"role": "assistant", "content": "AI is..."}]}\n
\n" + }, + { + "type": "markdown", + "source": "#### Custom Format (specify columns in job)\nYou can use custom field names and map them during job creation:\n- Define your own field names\n- Map them to prompt/completion in the job configuration", + "source_html": "

Custom Format (specify columns in job)

\n

You can use custom field names and map them during job creation:

\n
    \n
  • Define your own field names
    • \n
  • Map them to prompt/completion in the job configuration
    • \n
\n" + }, + { + "type": "markdown", + "source": "```json\n{\"question\": \"What is 2+2?\", \"answer\": \"4\"}\n```", + "source_html": "
{"question": "What is 2+2?", "answer": "4"}\n
\n" + }, + { + "type": "markdown", + "source": "### 3. Create Dataset FileSet and Upload Training Data", + "source_html": "

3. Create Dataset FileSet and Upload Training Data

\n" + }, + { + "type": "markdown", + "source": "Install huggingface datasets package to download public [rajpurkar/squad](https://huggingface.co/datasets/rajpurkar/squad) dataset if it is not installed in your Python environment:\n\n```sh\npip install datasets\n```", + "source_html": "

Install huggingface datasets package to download public rajpurkar/squad dataset if it is not installed in your Python environment:

\n
pip install datasets\n
\n" + }, + { + "type": "markdown", + "source": "#### Download rajpurkar/squad Dataset\n\nSQuAD (Stanford Question Answering Dataset) is a reading comprehension dataset consisting of questions posed on Wikipedia articles, where the answer is a segment of text from the corresponding passage.", + "source_html": "

Download rajpurkar/squad Dataset

\n

SQuAD (Stanford Question Answering Dataset) is a reading comprehension dataset consisting of questions posed on Wikipedia articles, where the answer is a segment of text from the corresponding passage.

\n" + }, + { + "type": "code", + "source": "from pathlib import Path\nfrom datasets import load_dataset, DatasetDict\nimport json\n\n# Load the SQuAD dataset from Hugging Face\nprint(\"Loading dataset rajpurkar/squad\")\nraw_dataset = load_dataset(\"rajpurkar/squad\")\nif not isinstance(raw_dataset, DatasetDict):\n raise ValueError(\"Dataset does not contain expected splits\")\n\nprint(\"Loaded dataset\")\n\n# Configuration\nVALIDATION_PROPORTION = 0.05\nSEED = 1234\n\n# For the purpose of this tutorial, we'll use a subset of the dataset\n# The larger the datasets, the better the model will perform but longer the training will take\ntraining_size = 3000\nvalidation_size = 300\nDATASET_PATH = Path(\"sft-dataset\").absolute()\n\n# Create directory if it doesn't exist\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Get the train split and create a validation split from it\ntrain_set = raw_dataset.get('train')\nsplit_dataset = train_set.train_test_split(test_size=VALIDATION_PROPORTION, seed=SEED)\n\n# Select subsets for the tutorial\ntrain_ds = split_dataset['train'].select(range(min(training_size, len(split_dataset['train']))))\nvalidation_ds = split_dataset['test'].select(range(min(validation_size, len(split_dataset['test']))))\n\n# Convert SQuAD format to prompt/completion format and save to JSONL\ndef convert_squad_to_sft_format(example):\n \"\"\"Convert SQuAD format to prompt/completion format for SFT training.\"\"\"\n prompt = f\"Context: {example['context']} Question: {example['question']} Answer:\"\n completion = example[\"answers\"][\"text\"][0] # Take the first answer\n return {\"prompt\": prompt, \"completion\": completion}\n\n# Save training data\nwith open(f\"{DATASET_PATH}/training.jsonl\", \"w\", encoding=\"utf-8\") as f:\n for example in train_ds:\n converted = convert_squad_to_sft_format(example)\n f.write(json.dumps(converted) + \"\\n\")\n\n# Save validation data\nwith open(f\"{DATASET_PATH}/validation.jsonl\", \"w\", encoding=\"utf-8\") as f:\n for example in validation_ds:\n converted = convert_squad_to_sft_format(example)\n f.write(json.dumps(converted) + \"\\n\")\n\nprint(f\"Saved training.jsonl with {len(train_ds)} rows\")\nprint(f\"Saved validation.jsonl with {len(validation_ds)} rows\")\n\n# Show a sample from the training data\nprint(\"\\nSample from training data:\")\nwith open(f\"{DATASET_PATH}/training.jsonl\", 'r') as f:\n first_line = f.readline()\n sample = json.loads(first_line)\n print(f\"Prompt: {sample['prompt'][:200]}...\")\n print(f\"Completion: {sample['completion']}\")", + "language": "python", + "source_html": "from pathlib import Path\nfrom datasets import load_dataset, DatasetDict\nimport json\n\n# Load the SQuAD dataset from Hugging Face\nprint("Loading dataset rajpurkar/squad")\nraw_dataset = load_dataset("rajpurkar/squad")\nif not isinstance(raw_dataset, DatasetDict):\n raise ValueError("Dataset does not contain expected splits")\n\nprint("Loaded dataset")\n\n# Configuration\nVALIDATION_PROPORTION = 0.05\nSEED = 1234\n\n# For the purpose of this tutorial, we'll use a subset of the dataset\n# The larger the datasets, the better the model will perform but longer the training will take\ntraining_size = 3000\nvalidation_size = 300\nDATASET_PATH = Path("sft-dataset").absolute()\n\n# Create directory if it doesn't exist\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Get the train split and create a validation split from it\ntrain_set = raw_dataset.get('train')\nsplit_dataset = train_set.train_test_split(test_size=VALIDATION_PROPORTION, seed=SEED)\n\n# Select subsets for the tutorial\ntrain_ds = split_dataset['train'].select(range(min(training_size, len(split_dataset['train']))))\nvalidation_ds = split_dataset['test'].select(range(min(validation_size, len(split_dataset['test']))))\n\n# Convert SQuAD format to prompt/completion format and save to JSONL\ndef convert_squad_to_sft_format(example):\n """Convert SQuAD format to prompt/completion format for SFT training."""\n prompt = f"Context: {example['context']} Question: {example['question']} Answer:"\n completion = example["answers"]["text"][0] # Take the first answer\n return {"prompt": prompt, "completion": completion}\n\n# Save training data\nwith open(f"{DATASET_PATH}/training.jsonl", "w", encoding="utf-8") as f:\n for example in train_ds:\n converted = convert_squad_to_sft_format(example)\n f.write(json.dumps(converted) + "\\n")\n\n# Save validation data\nwith open(f"{DATASET_PATH}/validation.jsonl", "w", encoding="utf-8") as f:\n for example in validation_ds:\n converted = convert_squad_to_sft_format(example)\n f.write(json.dumps(converted) + "\\n")\n\nprint(f"Saved training.jsonl with {len(train_ds)} rows")\nprint(f"Saved validation.jsonl with {len(validation_ds)} rows")\n\n# Show a sample from the training data\nprint("\\nSample from training data:")\nwith open(f"{DATASET_PATH}/training.jsonl", 'r') as f:\n first_line = f.readline()\n sample = json.loads(first_line)\n print(f"Prompt: {sample['prompt'][:200]}...")\n print(f"Completion: {sample['completion']}")\n" + }, + { + "type": "code", + "source": "# Create fileset to store SFT training data\nDATASET_NAME = \"sft-dataset\"\n\ntry:\n client.files.filesets.create(\n workspace=\"default\",\n name=DATASET_NAME,\n description=\"SFT training data\"\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\n# Upload training data files individually to ensure correct structure\nclient.files.upload(\n local_path=DATASET_PATH, # Local directory with your JSONL files\n remote_path=\"\",\n fileset=DATASET_NAME,\n workspace=\"default\"\n)\n\n# Validate training data is uploaded correctly\nprint(\"Training data:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace=\"default\").data], indent=2))", + "language": "python", + "source_html": "# Create fileset to store SFT training data\nDATASET_NAME = "sft-dataset"\n\ntry:\n client.files.filesets.create(\n workspace="default",\n name=DATASET_NAME,\n description="SFT training data"\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\n# Upload training data files individually to ensure correct structure\nclient.files.upload(\n local_path=DATASET_PATH, # Local directory with your JSONL files\n remote_path="",\n fileset=DATASET_NAME,\n workspace="default"\n)\n\n# Validate training data is uploaded correctly\nprint("Training data:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace="default").data], indent=2))\n" + }, + { + "type": "markdown", + "source": "### 4. Secrets Setup\n\nIf you plan to use NGC or HuggingFace models, you will need to configure authentication:\n\n- **NGC models** (`ngc://` URIs): Requires NGC API key\n- **HuggingFace models** (`hf://` URIs): Requires HF token for gated/private models\n\n\nConfigure these as secrets in your platform. Refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md) for detailed instructions.\n\nGet your credentials to access base models:\n- [NGC API Key](https://ngc.nvidia.com/) (Setup → Generate API Key)\n- [HuggingFace Token](https://huggingface.co/settings/tokens) (Create token with Read access)\n\n\n---\n\n#### Quick Setup Example\n\nIn this tutorial we are going to work with [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) model from HuggingFace. Ensure that you have sufficient permissions to download the model. If you cannot access the files on the [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) Hugging Face page, request access\n\n**HuggingFace Authentication:**\n- For gated models (Llama, Gemma), you must provide a HuggingFace token via the `token_secret` parameter\n- Get your token from [HuggingFace Settings](https://huggingface.co/settings/tokens) (requires Read access)\n- Accept the model's terms on the HuggingFace model page before using it. Example: [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main)\n- For public models, you can omit the `token_secret` parameter when creating a fileset for model in the next step", + "source_html": "

4. Secrets Setup

\n

If you plan to use NGC or HuggingFace models, you will need to configure authentication:

\n
    \n
  • NGC models (ngc:// URIs): Requires NGC API key
    • \n
  • HuggingFace models (hf:// URIs): Requires HF token for gated/private models
    • \n
\n

Configure these as secrets in your platform. Refer to Managing Secrets for detailed instructions.

\n

Get your credentials to access base models:

\n\n
\n

Quick Setup Example

\n

In this tutorial we are going to work with meta-llama/Llama-3.2-1B-Instruct model from HuggingFace. Ensure that you have sufficient permissions to download the model. If you cannot access the files on the meta-llama/Llama-3.2-1B-Instruct Hugging Face page, request access

\n

HuggingFace Authentication:

\n
    \n
  • For gated models (Llama, Gemma), you must provide a HuggingFace token via the token_secret parameter
    • \n
  • Get your token from HuggingFace Settings (requires Read access)
    • \n
  • Accept the model's terms on the HuggingFace model page before using it. Example: meta-llama/Llama-3.2-1B-Instruct
    • \n
  • For public models, you can omit the token_secret parameter when creating a fileset for model in the next step
    • \n
\n" + }, + { + "type": "code", + "source": "# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set\nHF_TOKEN = os.getenv(\"HF_TOKEN\")\nNGC_API_KEY = os.getenv(\"NGC_API_KEY\")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f\"{label} is not set\")\n try:\n secret = client.secrets.create(\n name=name,\n workspace=\"default\",\n value=value,\n )\n print(f\"Created secret: {name}\")\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n return client.secrets.retrieve(name=name, workspace=\"default\")\n\n\n# Create HuggingFace token secret\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")\nprint(\"HF_TOKEN secret:\")\nprint(hf_secret.model_dump_json(indent=2))\n\n# Create NGC API key secret\n# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n# ngc_api_key = create_or_get_secret(\"ngc-api-key\", NGC_API_KEY, \"NGC_API_KEY\")", + "language": "python", + "source_html": "# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set\nHF_TOKEN = os.getenv("HF_TOKEN")\nNGC_API_KEY = os.getenv("NGC_API_KEY")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f"{label} is not set")\n try:\n secret = client.secrets.create(\n name=name,\n workspace="default",\n value=value,\n )\n print(f"Created secret: {name}")\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n return client.secrets.retrieve(name=name, workspace="default")\n\n\n# Create HuggingFace token secret\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\nprint("HF_TOKEN secret:")\nprint(hf_secret.model_dump_json(indent=2))\n\n# Create NGC API key secret\n# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n# ngc_api_key = create_or_get_secret("ngc-api-key", NGC_API_KEY, "NGC_API_KEY")\n" + }, + { + "type": "markdown", + "source": "### 5. Create Base Model FileSet and Model Entity\n\nCreate a fileset pointing to [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) model in HuggingFace that we will train with SFT. Then create a Model Entity that references this fileset. Model downloading will take place at training time.\n\nNote: for public models, you can omit the `token_secret` parameter when creating a model fileset.", + "source_html": "

5. Create Base Model FileSet and Model Entity

\n

Create a fileset pointing to meta-llama/Llama-3.2-1B-Instruct model in HuggingFace that we will train with SFT. Then create a Model Entity that references this fileset. Model downloading will take place at training time.

\n

Note: for public models, you can omit the token_secret parameter when creating a model fileset.

\n" + }, + { + "type": "code", + "source": "import time\n\n# Create a fileset pointing to the desired HuggingFace model\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = \"meta-llama/Llama-3.2-1B-Instruct\"\nMODEL_NAME = \"llama-3-2-1b-base\"\n\n# Ensure you have a HuggingFace token secret created\ntry:\n base_model_fs = client.files.filesets.create(\n workspace=\"default\",\n name=MODEL_NAME,\n description=\"Llama 3.2 1B base model from HuggingFace\",\n storage=HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type=\"model\",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\n print(f\"Created base model fileset: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model fileset already exists. Skipping creation.\")\n base_model_fs = client.files.filesets.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\n# Create the Model Entity representation.\ntry:\n base_model = client.models.create(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n )\n print(f\"Created Model Entity: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model already exists. Updating fileset if different.\")\n base_model = client.models.update(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n )\n\nprint(f\"\\nBase model fileset: fileset://default/{base_model.name}\")\nprint(\"Base model fileset files list:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace=\"default\").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint(\"\\nWaiting for ModelSpec to be populated...\")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f\"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds\")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\nprint(f\"ModelSpec populated: {base_model.spec}\")", + "language": "python", + "source_html": "import time\n\n# Create a fileset pointing to the desired HuggingFace model\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = "meta-llama/Llama-3.2-1B-Instruct"\nMODEL_NAME = "llama-3-2-1b-base"\n\n# Ensure you have a HuggingFace token secret created\ntry:\n base_model_fs = client.files.filesets.create(\n workspace="default",\n name=MODEL_NAME,\n description="Llama 3.2 1B base model from HuggingFace",\n storage=HuggingfaceStorageConfigParam(\n type="huggingface",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type="model",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\n print(f"Created base model fileset: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model fileset already exists. Skipping creation.")\n base_model_fs = client.files.filesets.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\n# Create the Model Entity representation.\ntry:\n base_model = client.models.create(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n )\n print(f"Created Model Entity: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model already exists. Updating fileset if different.")\n base_model = client.models.update(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n )\n\nprint(f"\\nBase model fileset: fileset://default/{base_model.name}")\nprint("Base model fileset files list:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace="default").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint("\\nWaiting for ModelSpec to be populated...")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\nprint(f"ModelSpec populated: {base_model.spec}")\n" + }, + { + "type": "markdown", + "source": "### 6. Create SFT Finetuning Job\nCreate a customization job with an inline target referencing the base model and dataset filesets created in previous steps.", + "source_html": "

6. Create SFT Finetuning Job

\n

Create a customization job with an inline target referencing the base model and dataset filesets created in previous steps.

\n" + }, + { + "type": "markdown", + "source": "\n**GPU Requirements:**\n- 1B models: 1 GPU (24GB+ VRAM)\n- 3B models: 1-2 GPUs \n- 8B models: 2-4 GPUs\n- 70B models: 8+ GPUs \n\nAdjust `num_gpus_per_node` based on your model size.", + "source_html": "

GPU Requirements:

\n
    \n
  • 1B models: 1 GPU (24GB+ VRAM)
    • \n
  • 3B models: 1-2 GPUs
    • \n
  • 8B models: 2-4 GPUs
    • \n
  • 70B models: 8+ GPUs
    • \n
\n

Adjust num_gpus_per_node based on your model size.

\n" + }, + { + "type": "code", + "source": "import uuid\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam,\n)\n\njob_suffix = uuid.uuid4().hex[:4]\n\nJOB_NAME = f\"my-sft-job-{job_suffix}\"\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=2,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n )\n)\n\nprint(f\"Job ID: {job.name}\")\nprint(f\"Output model: {job.spec.output.name}\")", + "language": "python", + "source_html": "import uuid\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam,\n)\n\njob_suffix = uuid.uuid4().hex[:4]\n\nJOB_NAME = f"my-sft-job-{job_suffix}"\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace="default",\n spec=CustomizationJobInputParam(\n model=f"default/{base_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=2,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n )\n)\n\nprint(f"Job ID: {job.name}")\nprint(f"Output model: {job.spec.output.name}")\n" + }, + { + "type": "markdown", + "source": "### 7. Track Training Progress", + "source_html": "

7. Track Training Progress

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Poll job status every 10 seconds until completed\nwhile True:\n status = client.customization.jobs.get_status(\n name=job.name,\n workspace=\"default\"\n )\n\n clear_output(wait=True)\n print(f\"Job Status: {status.model_dump_json(indent=2)}\")\n\n # Extract training progress from nested steps structure\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n\n for job_step in status.steps or []:\n if job_step.name == \"customization-training-job\":\n for task in job_step.tasks or []:\n task_details = task.status_details or {}\n step = task_details.get(\"step\")\n max_steps = task_details.get(\"max_steps\")\n training_phase = task_details.get(\"phase\")\n break\n break\n\n if step is not None and max_steps is not None:\n progress_pct = (step / max_steps) * 100\n print(f\"Training Progress: Step {step}/{max_steps} ({progress_pct:.1f}%)\")\n if training_phase:\n print(f\"Training Phase: {training_phase}\")\n else:\n print(\"Training step not started yet or progress info not available\")\n\n # Exit loop when job is completed (or failed/cancelled)\n if status.status in (\"completed\", \"failed\", \"cancelled\", \"error\"):\n print(f\"\\nJob finished with status: {status.status}\")\n break\n\n time.sleep(10)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Poll job status every 10 seconds until completed\nwhile True:\n status = client.customization.jobs.get_status(\n name=job.name,\n workspace="default"\n )\n\n clear_output(wait=True)\n print(f"Job Status: {status.model_dump_json(indent=2)}")\n\n # Extract training progress from nested steps structure\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n\n for job_step in status.steps or []:\n if job_step.name == "customization-training-job":\n for task in job_step.tasks or []:\n task_details = task.status_details or {}\n step = task_details.get("step")\n max_steps = task_details.get("max_steps")\n training_phase = task_details.get("phase")\n break\n break\n\n if step is not None and max_steps is not None:\n progress_pct = (step / max_steps) * 100\n print(f"Training Progress: Step {step}/{max_steps} ({progress_pct:.1f}%)")\n if training_phase:\n print(f"Training Phase: {training_phase}")\n else:\n print("Training step not started yet or progress info not available")\n\n # Exit loop when job is completed (or failed/cancelled)\n if status.status in ("completed", "failed", "cancelled", "error"):\n print(f"\\nJob finished with status: {status.status}")\n break\n\n time.sleep(10)\n" + }, + { + "type": "markdown", + "source": "**Interpreting SFT Training Metrics:**\n\nMonitor the relationship between training and validation loss curves:\n\n| Scenario | Interpretation | Action |\n|----------|----------------|--------|\n| **Both decreasing together** | Model is learning well | Continue training |\n| **Training decreases, validation flat/increasing** | Overfitting | Reduce epochs, add data |\n| **Both flat/not decreasing** | Underfitting | Increase LR, check data |\n| **Sudden spikes** | Training instability | Lower learning rate |\n\n**Note:** Training metrics measure optimization progress, not final model quality. Always evaluate the deployed model on your specific use case.", + "source_html": "

Interpreting SFT Training Metrics:

\n

Monitor the relationship between training and validation loss curves:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ScenarioInterpretationAction
Both decreasing togetherModel is learning wellContinue training
Training decreases, validation flat/increasingOverfittingReduce epochs, add data
Both flat/not decreasingUnderfittingIncrease LR, check data
Sudden spikesTraining instabilityLower learning rate
\n

Note: Training metrics measure optimization progress, not final model quality. Always evaluate the deployed model on your specific use case.

\n" + }, + { + "type": "markdown", + "source": "(ft-deploy-full-weight-model)=\n\n### 8. Deploy Fine-Tuned Model\n\nAfter training completes, deploy using the Deployment Management Service:", + "source_html": "

(ft-deploy-full-weight-model)=

\n

8. Deploy Fine-Tuned Model

\n

After training completes, deploy using the Deployment Management Service:

\n" + }, + { + "type": "code", + "source": "# Validate model entity exists\nmodel_entity = client.models.retrieve(workspace='default', name=job.spec.output.name)\nprint(model_entity.model_dump_json(indent=2))", + "language": "python", + "source_html": "# Validate model entity exists\nmodel_entity = client.models.retrieve(workspace='default', name=job.spec.output.name)\nprint(model_entity.model_dump_json(indent=2))\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# Create deployment config\ndeploy_suffix = uuid.uuid4().hex[:4]\nDEPLOYMENT_CONFIG_NAME = f\"sft-model-deployment-cfg-{deploy_suffix}\"\nDEPLOYMENT_NAME = f\"sft-model-deployment-{deploy_suffix}\"\n\ndeployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=DEPLOYMENT_CONFIG_NAME,\n nim_deployment=NIMDeploymentParam(\n image_name=\"nvcr.io/nim/nvidia/llm-nim\",\n image_tag=\"1.15.5\",\n gpu=1,\n model_name=job.spec.output.name, # ModelEntity name from training,\n model_namespace=\"default\", # Workspace where ModelEntity lives\n additional_envs={\"NIM_MODEL_PROFILE\": \"vllm\"}\n ),\n)\n\n# Deploy model using deployment_config created above\ndeployment = client.inference.deployments.create(\n workspace=\"default\",\n name=DEPLOYMENT_NAME,\n config=deployment_config.name\n)\n\n\n# Check deployment status\ndeployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace=\"default\"\n)\n\nprint(f\"Deployment name: {deployment.name}\")\nprint(f\"Deployment status: {deployment_status.status}\")", + "language": "python", + "source_html": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# Create deployment config\ndeploy_suffix = uuid.uuid4().hex[:4]\nDEPLOYMENT_CONFIG_NAME = f"sft-model-deployment-cfg-{deploy_suffix}"\nDEPLOYMENT_NAME = f"sft-model-deployment-{deploy_suffix}"\n\ndeployment_config = client.inference.deployment_configs.create(\n workspace="default",\n name=DEPLOYMENT_CONFIG_NAME,\n nim_deployment=NIMDeploymentParam(\n image_name="nvcr.io/nim/nvidia/llm-nim",\n image_tag="1.15.5",\n gpu=1,\n model_name=job.spec.output.name, # ModelEntity name from training,\n model_namespace="default", # Workspace where ModelEntity lives\n additional_envs={"NIM_MODEL_PROFILE": "vllm"}\n ),\n)\n\n# Deploy model using deployment_config created above\ndeployment = client.inference.deployments.create(\n workspace="default",\n name=DEPLOYMENT_NAME,\n config=deployment_config.name\n)\n\n\n# Check deployment status\ndeployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace="default"\n)\n\nprint(f"Deployment name: {deployment.name}")\nprint(f"Deployment status: {deployment_status.status}")\n" + }, + { + "type": "markdown", + "source": "The deployment service automatically:\n- Downloads model weights from the Files service\n- Provisions storage (PVC) for the weights\n- Configures and starts the NIM container\n\n**Multi-GPU Deployment:**\n\nFor larger models requiring multiple GPUs, configure parallelism with environment variables:\n\n```python\ndeployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=\"sft-model-config-multigpu\",\n \n nim_deployment={\n \"image_name\": \"nvcr.io/nim/nvidia/llm-nim\",\n \"image_tag\": \"1.13.1\",\n \"gpu\": 2, # Total GPUs\n \"additional_envs\": {\n \"NIM_TENSOR_PARALLEL_SIZE\": \"2\", # Tensor parallelism\n \"NIM_PIPELINE_PARALLEL_SIZE\": \"1\" # Pipeline parallelism\n }\n }\n)\n```", + "source_html": "

The deployment service automatically:

\n
    \n
  • Downloads model weights from the Files service
    • \n
  • Provisions storage (PVC) for the weights
    • \n
  • Configures and starts the NIM container
    • \n
\n

Multi-GPU Deployment:

\n

For larger models requiring multiple GPUs, configure parallelism with environment variables:

\n
deployment_config = client.inference.deployment_configs.create(\n    workspace="default",\n    name="sft-model-config-multigpu",\n    \n    nim_deployment={\n        "image_name": "nvcr.io/nim/nvidia/llm-nim",\n        "image_tag": "1.13.1",\n        "gpu": 2,  # Total GPUs\n        "additional_envs": {\n            "NIM_TENSOR_PARALLEL_SIZE": "2",  # Tensor parallelism\n            "NIM_PIPELINE_PARALLEL_SIZE": "1"  # Pipeline parallelism\n        }\n    }\n)\n
\n" + }, + { + "type": "markdown", + "source": "**Single-Node Constraint:** Model deployments are limited to a single node. The maximum `gpu` value depends on the total GPUs available on a single node in your cluster. Multi-node deployments are not supported.\n\n---\n\n#### GPU Parallelism\n\nBy default, NIM uses all GPUs for tensor parallelism (TP). You can customize this behavior using the `NIM_TENSOR_PARALLEL_SIZE` and `NIM_PIPELINE_PARALLEL_SIZE` environment variables.\n\n| Strategy | Description | Best For |\n|----------|-------------|----------|\n| **Tensor Parallel (TP)** | Splits model layers across GPUs | Lowest latency |\n| **Pipeline Parallel (PP)** | Splits model depth across GPUs | Highest throughput |\n\n**Formula:** `gpu` = `NIM_TENSOR_PARALLEL_SIZE` × `NIM_PIPELINE_PARALLEL_SIZE`\n\n---\n\n#### Example Configurations\n\n**Default (TP=8, PP=1) — Lowest Latency**\n```\n\"gpu\": 8\n# NIM automatically sets NIM_TENSOR_PARALLEL_SIZE=8\n```\n\n**Balanced (TP=4, PP=2)**\n```\n\"gpu\": 8,\n\"additional_envs\": {\n \"NIM_TENSOR_PARALLEL_SIZE\": \"4\",\n \"NIM_PIPELINE_PARALLEL_SIZE\": \"2\"\n}\n```\n\n**Throughput Optimized (TP=2, PP=4)**\n```\n\"gpu\": 8,\n\"additional_envs\": {\n \"NIM_TENSOR_PARALLEL_SIZE\": \"2\",\n \"NIM_PIPELINE_PARALLEL_SIZE\": \"4\"\n}\n```", + "source_html": "

Single-Node Constraint: Model deployments are limited to a single node. The maximum gpu value depends on the total GPUs available on a single node in your cluster. Multi-node deployments are not supported.

\n
\n

GPU Parallelism

\n

By default, NIM uses all GPUs for tensor parallelism (TP). You can customize this behavior using the NIM_TENSOR_PARALLEL_SIZE and NIM_PIPELINE_PARALLEL_SIZE environment variables.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
StrategyDescriptionBest For
Tensor Parallel (TP)Splits model layers across GPUsLowest latency
Pipeline Parallel (PP)Splits model depth across GPUsHighest throughput
\n

Formula: gpu = NIM_TENSOR_PARALLEL_SIZE × NIM_PIPELINE_PARALLEL_SIZE

\n
\n

Example Configurations

\n

Default (TP=8, PP=1) — Lowest Latency

\n
"gpu": 8\n# NIM automatically sets NIM_TENSOR_PARALLEL_SIZE=8\n
\n

Balanced (TP=4, PP=2)

\n
"gpu": 8,\n"additional_envs": {\n    "NIM_TENSOR_PARALLEL_SIZE": "4",\n    "NIM_PIPELINE_PARALLEL_SIZE": "2"\n}\n
\n

Throughput Optimized (TP=2, PP=4)

\n
"gpu": 8,\n"additional_envs": {\n    "NIM_TENSOR_PARALLEL_SIZE": "2",\n    "NIM_PIPELINE_PARALLEL_SIZE": "4"\n}\n
\n" + }, + { + "type": "markdown", + "source": "### Track Deployment Status", + "source_html": "

Track Deployment Status

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Poll deployment status every 15 seconds until ready\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f\"Monitoring deployment '{deployment.name}'...\")\nprint(f\"Timeout: {TIMEOUT_MINUTES} minutes\\n\")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace=\"default\"\n )\n\n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n\n clear_output(wait=True)\n print(f\"Deployment: {deployment.name}\")\n print(f\"Status: {deployment_status.status}\")\n print(f\"Elapsed time: {elapsed_min}m {elapsed_sec}s\")\n\n # Check if deployment is ready\n if deployment_status.status == \"READY\":\n print(\"\\nDeployment is ready!\")\n if not client.models.wait_for_gateway(deployment.name, workspace=\"default\", timeout=60):\n raise RuntimeError(\"Inference gateway did not become ready\")\n break\n\n # Check for failure states\n if deployment_status.status in (\"FAILED\", \"ERROR\", \"TERMINATED\", \"LOST\"):\n raise RuntimeError(f\"Deployment failed with status: {deployment_status.status}\")\n\n # Check timeout\n if elapsed > timeout_seconds:\n raise TimeoutError(f\"Deployment timeout after {TIMEOUT_MINUTES} minutes\")\n\n time.sleep(15)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Poll deployment status every 15 seconds until ready\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f"Monitoring deployment '{deployment.name}'...")\nprint(f"Timeout: {TIMEOUT_MINUTES} minutes\\n")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace="default"\n )\n\n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n\n clear_output(wait=True)\n print(f"Deployment: {deployment.name}")\n print(f"Status: {deployment_status.status}")\n print(f"Elapsed time: {elapsed_min}m {elapsed_sec}s")\n\n # Check if deployment is ready\n if deployment_status.status == "READY":\n print("\\nDeployment is ready!")\n if not client.models.wait_for_gateway(deployment.name, workspace="default", timeout=60):\n raise RuntimeError("Inference gateway did not become ready")\n break\n\n # Check for failure states\n if deployment_status.status in ("FAILED", "ERROR", "TERMINATED", "LOST"):\n raise RuntimeError(f"Deployment failed with status: {deployment_status.status}")\n\n # Check timeout\n if elapsed > timeout_seconds:\n raise TimeoutError(f"Deployment timeout after {TIMEOUT_MINUTES} minutes")\n\n time.sleep(15)\n" + }, + { + "type": "markdown", + "source": "### 9. Evaluate Your Model\n\nAfter training, evaluate whether your model meets your requirements:\n\n#### Quick Manual Evaluation", + "source_html": "

9. Evaluate Your Model

\n

After training, evaluate whether your model meets your requirements:

\n

Quick Manual Evaluation

\n" + }, + { + "type": "code", + "source": "# Wait for deployment to be ready, then test\n# Test the fine-tuned model with a question answering prompt\ncontext = \"The Apollo 11 mission was the first manned mission to land on the Moon. It was launched on July 16, 1969, and Neil Armstrong became the first person to walk on the lunar surface on July 20, 1969. Buzz Aldrin joined him shortly after, while Michael Collins remained in lunar orbit.\"\nquestion = \"Who was the first person to walk on the Moon?\"\n\nmessages = [\n {\"role\": \"user\", \"content\": f\"Based on the following context, answer the question.\\n\\nContext: {context}\\n\\nQuestion: {question}\"}\n]\n\nresponse = client.inference.gateway.provider.post(\n \"v1/chat/completions\",\n name=deployment.name,\n workspace=\"default\",\n body={\n \"model\": f\"default/{job.spec.output.name}\",\n \"messages\": messages,\n \"temperature\": 0,\n \"max_tokens\": 128\n }\n)\n\nprint(\"=\" * 60)\nprint(\"MODEL EVALUATION\")\nprint(\"=\" * 60)\nprint(f\"Question: {question}\")\nprint(f\"Expected: Neil Armstrong\")\nprint(f\"Model output: {response['choices'][0]['message']['content']}\")", + "language": "python", + "source_html": "# Wait for deployment to be ready, then test\n# Test the fine-tuned model with a question answering prompt\ncontext = "The Apollo 11 mission was the first manned mission to land on the Moon. It was launched on July 16, 1969, and Neil Armstrong became the first person to walk on the lunar surface on July 20, 1969. Buzz Aldrin joined him shortly after, while Michael Collins remained in lunar orbit."\nquestion = "Who was the first person to walk on the Moon?"\n\nmessages = [\n {"role": "user", "content": f"Based on the following context, answer the question.\\n\\nContext: {context}\\n\\nQuestion: {question}"}\n]\n\nresponse = client.inference.gateway.provider.post(\n "v1/chat/completions",\n name=deployment.name,\n workspace="default",\n body={\n "model": f"default/{job.spec.output.name}",\n "messages": messages,\n "temperature": 0,\n "max_tokens": 128\n }\n)\n\nprint("=" * 60)\nprint("MODEL EVALUATION")\nprint("=" * 60)\nprint(f"Question: {question}")\nprint(f"Expected: Neil Armstrong")\nprint(f"Model output: {response['choices'][0]['message']['content']}")\n" + }, + { + "type": "markdown", + "source": "#### Evaluation Best Practices\n\n**Manual Evaluation** (Recommended)\n- Test with real-world examples from your use case\n- Compare responses to base model and expected outputs\n- Verify the model exhibits desired behavior changes\n- Check edge cases and error handling\n\n**What to look for:**\n- ✅ Model follows your desired output format\n- ✅ Applies domain knowledge correctly\n- ✅ Maintains general language capabilities\n- ✅ Avoids unwanted behaviors or biases\n- ❌ Doesn't hallucinate facts not in training data\n- ❌ Doesn't produce repetitive or nonsensical outputs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated HuggingFace models (Llama, Gemma), accept the license on the model page\n- Check the `model_uri` format is correct (`fileset://`)\n- Ensure you have accepted the model's terms of service on HuggingFace\n- Check job status and logs: `client.customization.jobs.retrieve(name=job.name, workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n1. **First try:** Reduce `micro_batch_size` from 2 to 1\n2. **Still OOM:** Reduce `batch_size` from 4 to 2\n3. **Still OOM:** Reduce `max_seq_length` from 2048 to 1024 or 512\n4. **Last resort:** Increase GPU count and use `tensor_parallel_size` for model sharding\n\n**Loss curves not decreasing (underfitting):**\n- Increase training duration: `epochs: 5-10` instead of 3\n- Adjust learning rate: Try `1e-5` to `1e-4`\n- Add warmup: Set `warmup_steps` to ~10% of total training steps\n- Check data quality: Verify formatting, remove duplicates, ensure diversity\n\n**Training loss decreases but validation loss increases (overfitting):**\n- Reduce epochs: Try `epochs: 1-2` instead of 5+\n- Lower learning rate: Use `2e-5` or `1e-5`\n- Increase dataset size and diversity\n- Verify train/validation split has no data leakage\n\n**Model output quality is poor despite good training metrics:**\n- Training metrics optimize for loss, not your actual task—evaluate on real use cases\n- Review data quality, format, and diversity—metrics can be misleading with poor data\n- Try a different base model size or architecture\n- Adjust learning rate and batch size\n- Compare to baseline: Test base model to ensure fine-tuning improved performance\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=job.spec.output.name, workspace=\"default\")`\n- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace=\"default\")`\n- Ensure sufficient GPU resources available for model size\n- Verify NIM image tag `1.13.1` is compatible with your model\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning", + "source_html": "

Evaluation Best Practices

\n

Manual Evaluation (Recommended)

\n
    \n
  • Test with real-world examples from your use case
    • \n
  • Compare responses to base model and expected outputs
    • \n
  • Verify the model exhibits desired behavior changes
    • \n
  • Check edge cases and error handling
    • \n
\n

What to look for:

\n
    \n
  • ✅ Model follows your desired output format
    • \n
  • ✅ Applies domain knowledge correctly
    • \n
  • ✅ Maintains general language capabilities
    • \n
  • ✅ Avoids unwanted behaviors or biases
    • \n
  • ❌ Doesn't hallucinate facts not in training data
    • \n
  • ❌ Doesn't produce repetitive or nonsensical outputs
    • \n
\n
\n

Hyperparameters

\n

For detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.

\n
\n

Troubleshooting

\n

Job fails during model download:

\n
    \n
  • Verify authentication secrets are configured (refer to Managing Secrets)
    • \n
  • For gated HuggingFace models (Llama, Gemma), accept the license on the model page
    • \n
  • Check the model_uri format is correct (fileset://)
    • \n
  • Ensure you have accepted the model's terms of service on HuggingFace
    • \n
  • Check job status and logs: client.customization.jobs.retrieve(name=job.name, workspace="default")
    • \n
\n

Job fails with OOM (Out of Memory) error:

\n
    \n
  1. First try: Reduce micro_batch_size from 2 to 1
    2. \n
  2. Still OOM: Reduce batch_size from 4 to 2
    3. \n
  3. Still OOM: Reduce max_seq_length from 2048 to 1024 or 512
    4. \n
  4. Last resort: Increase GPU count and use tensor_parallel_size for model sharding
    5. \n
\n

Loss curves not decreasing (underfitting):

\n
    \n
  • Increase training duration: epochs: 5-10 instead of 3
    • \n
  • Adjust learning rate: Try 1e-5 to 1e-4
    • \n
  • Add warmup: Set warmup_steps to ~10% of total training steps
    • \n
  • Check data quality: Verify formatting, remove duplicates, ensure diversity
    • \n
\n

Training loss decreases but validation loss increases (overfitting):

\n
    \n
  • Reduce epochs: Try epochs: 1-2 instead of 5+
    • \n
  • Lower learning rate: Use 2e-5 or 1e-5
    • \n
  • Increase dataset size and diversity
    • \n
  • Verify train/validation split has no data leakage
    • \n
\n

Model output quality is poor despite good training metrics:

\n
    \n
  • Training metrics optimize for loss, not your actual task—evaluate on real use cases
    • \n
  • Review data quality, format, and diversity—metrics can be misleading with poor data
    • \n
  • Try a different base model size or architecture
    • \n
  • Adjust learning rate and batch size
    • \n
  • Compare to baseline: Test base model to ensure fine-tuning improved performance
    • \n
\n

Deployment fails:

\n
    \n
  • Verify output model exists: client.models.retrieve(name=job.spec.output.name, workspace="default")
    • \n
  • Check deployment logs: client.inference.deployments.get_logs(name=deployment.name, workspace="default")
    • \n
  • Ensure sufficient GPU resources available for model size
    • \n
  • Verify NIM image tag 1.13.1 is compatible with your model
    • \n
\n

Next Steps

\n\n" + } + ] +} \ No newline at end of file diff --git a/docs/fern/components/notebooks/sft-customization-job.ts b/docs/fern/components/notebooks/sft-customization-job.ts new file mode 100644 index 00000000000..c318666d8ac --- /dev/null +++ b/docs/fern/components/notebooks/sft-customization-job.ts @@ -0,0 +1,194 @@ +/** Auto-generated by ipynb-to-fern-json.py - do not edit */ +export default { cells: [ + { + "type": "markdown", + "source": "\n\n\n# Full SFT Customization\n\nLearn how to fine-tune all model weights using supervised fine-tuning (SFT) to customize LLM behavior for your specific tasks.\n\n## About\n\nSupervised Fine-Tuning (SFT) customizes model behavior, injects new knowledge, and optimizes performance for specific domains and tasks. Full SFT modifies **all model weights** during training, providing maximum customization flexibility.\n\n**What you can achieve with SFT:**\n\n- 🎯 **Specialize for domains:** Fine-tune models on legal texts, medical records, or financial data\n- 💡 **Inject knowledge:** Add new information not present in the base model\n- 📈 **Improve accuracy:** Optimize for specific tasks like sentiment analysis, summarization, or code generation\n\n### SFT vs LoRA: Understanding the Trade-offs\n\n**Full SFT** trains all model parameters (for example, all 70 billion weights in Llama 70B):\n\n- ✅ Maximum model adaptation and knowledge injection\n- ✅ Can fundamentally change model behavior\n- ✅ Best for significant domain shifts or specialized tasks\n- ❌ Requires substantial GPU resources (4-8x more than LoRA)\n- ❌ Produces full model weights (~140GB for Llama 70B)\n- ❌ Longer training time\n\n**LoRA** trains only ~1% of weights by adding thin matrices to existing weights:\n\n- ✅ 75-95% less memory required\n- ✅ Faster training (2-4x speedup)\n- ✅ Produces small adapter files (~100-500MB)\n- ✅ Multiple adapters can share one base model\n- ❌ Limited adaptation capability compared to full fine-tuning\n\n**When to choose Full SFT:**\n\n- Training small models (1B-8B) where resource cost is manageable\n- Need fundamental behavior changes (for example, medical diagnosis, legal reasoning)\n- Injecting substantial new knowledge not in the base model\n\n**When to choose LoRA:** Refer to the [LoRA tutorial](./lora-customization-job) for most use cases, especially with large models (70B+) or limited GPU resources.", + "source_html": "\n\n

Full SFT Customization

\n

Learn how to fine-tune all model weights using supervised fine-tuning (SFT) to customize LLM behavior for your specific tasks.

\n

About

\n

Supervised Fine-Tuning (SFT) customizes model behavior, injects new knowledge, and optimizes performance for specific domains and tasks. Full SFT modifies all model weights during training, providing maximum customization flexibility.

\n

What you can achieve with SFT:

\n
    \n
  • 🎯 Specialize for domains: Fine-tune models on legal texts, medical records, or financial data
    • \n
  • 💡 Inject knowledge: Add new information not present in the base model
    • \n
  • 📈 Improve accuracy: Optimize for specific tasks like sentiment analysis, summarization, or code generation
    • \n
\n

SFT vs LoRA: Understanding the Trade-offs

\n

Full SFT trains all model parameters (for example, all 70 billion weights in Llama 70B):

\n
    \n
  • ✅ Maximum model adaptation and knowledge injection
    • \n
  • ✅ Can fundamentally change model behavior
    • \n
  • ✅ Best for significant domain shifts or specialized tasks
    • \n
  • ❌ Requires substantial GPU resources (4-8x more than LoRA)
    • \n
  • ❌ Produces full model weights (~140GB for Llama 70B)
    • \n
  • ❌ Longer training time
    • \n
\n

LoRA trains only ~1% of weights by adding thin matrices to existing weights:

\n
    \n
  • ✅ 75-95% less memory required
    • \n
  • ✅ Faster training (2-4x speedup)
    • \n
  • ✅ Produces small adapter files (~100-500MB)
    • \n
  • ✅ Multiple adapters can share one base model
    • \n
  • ❌ Limited adaptation capability compared to full fine-tuning
    • \n
\n

When to choose Full SFT:

\n
    \n
  • Training small models (1B-8B) where resource cost is manageable
    • \n
  • Need fundamental behavior changes (for example, medical diagnosis, legal reasoning)
    • \n
  • Injecting substantial new knowledge not in the base model
    • \n
\n

When to choose LoRA: Refer to the LoRA tutorial for most use cases, especially with large models (70B+) or limited GPU resources.

\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n2. **Installed the Python SDK** (included with `pip install nemo-platform`)", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Completed the Quickstart to install and deploy NeMo Platform locally
    2. \n
  2. Installed the Python SDK (included with pip install nemo-platform)
    3. \n
\n" + }, + { + "type": "markdown", + "source": "## Quick Start\n\n### 1. Initialize SDK\n\nThe SDK needs to know your NeMo Platform server URL. By default, `http://localhost:8080` is used in accordance with the [Quickstart](../../get-started/quickstart.md) guide. If NeMo Platform is running at a custom location, you can override the URL by setting the `NMP_BASE_URL` environment variable:\n\n```sh\nexport NMP_BASE_URL=\n```", + "source_html": "

Quick Start

\n

1. Initialize SDK

\n

The SDK needs to know your NeMo Platform server URL. By default, http://localhost:8080 is used in accordance with the Quickstart guide. If NeMo Platform is running at a custom location, you can override the URL by setting the NMP_BASE_URL environment variable:

\n
export NMP_BASE_URL=<YOUR_NMP_BASE_URL>\n
\n" + }, + { + "type": "code", + "source": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace=\"default\"\n)", + "language": "python", + "source_html": "import json\nimport os\nfrom nemo_platform import NeMoPlatform, ConflictError\n\nNMP_BASE_URL = os.environ.get("NMP_BASE_URL", "http://localhost:8080")\nclient = NeMoPlatform(\n base_url=NMP_BASE_URL,\n workspace="default"\n)\n" + }, + { + "type": "markdown", + "source": "### 2. Prepare Dataset\n\nCreate your data in JSONL format—one JSON object per line. The platform auto-detects your data format. Supported dataset formats are listed below.\n\n**Flexible Data Setup:**\n- **No validation file?** The platform automatically creates a 10% validation split\n- **Multiple files?** Upload to `training/` or `validation/` subdirectories—they will be automatically merged\n- **Format detection:** Your data format is auto-detected at training time\n\nIn this tutorial the following dataset directory structure will be used:\n```\nmy_dataset\n`-- training.jsonl\n`-- validation.jsonl\n```", + "source_html": "

2. Prepare Dataset

\n

Create your data in JSONL format—one JSON object per line. The platform auto-detects your data format. Supported dataset formats are listed below.

\n

Flexible Data Setup:

\n
    \n
  • No validation file? The platform automatically creates a 10% validation split
    • \n
  • Multiple files? Upload to training/ or validation/ subdirectories—they will be automatically merged
    • \n
  • Format detection: Your data format is auto-detected at training time
    • \n
\n

In this tutorial the following dataset directory structure will be used:

\n
my_dataset\n`-- training.jsonl\n`-- validation.jsonl\n
\n" + }, + { + "type": "markdown", + "source": "#### Simple Prompt/Completion Format\nThe simplest format with input prompt and expected completion:\n- **`prompt`**: The input prompt for the model\n- **`completion`**: The expected output response", + "source_html": "

Simple Prompt/Completion Format

\n

The simplest format with input prompt and expected completion:

\n
    \n
  • prompt: The input prompt for the model
    • \n
  • completion: The expected output response
    • \n
\n" + }, + { + "type": "markdown", + "source": "```json\n{\"prompt\": \"Write an email to confirm our hotel reservation.\", \"completion\": \"Dear Hotel Team, I am writing to confirm our reservation for two guests...\"}\n```", + "source_html": "
{"prompt": "Write an email to confirm our hotel reservation.", "completion": "Dear Hotel Team, I am writing to confirm our reservation for two guests..."}\n
\n" + }, + { + "type": "markdown", + "source": "#### Chat Format (for conversational models)\nFor multi-turn conversations, use the messages format:\n- **`messages`**: List of message objects with `role` and `content` fields\n- Roles: `system`, `user`, `assistant`", + "source_html": "

Chat Format (for conversational models)

\n

For multi-turn conversations, use the messages format:

\n
    \n
  • messages: List of message objects with role and content fields
    • \n
  • Roles: system, user, assistant
    • \n
\n" + }, + { + "type": "markdown", + "source": "```json\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}, {\"role\": \"user\", \"content\": \"What is AI?\"}, {\"role\": \"assistant\", \"content\": \"AI is...\"}]}\n```", + "source_html": "
{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "What is AI?"}, {"role": "assistant", "content": "AI is..."}]}\n
\n" + }, + { + "type": "markdown", + "source": "#### Custom Format (specify columns in job)\nYou can use custom field names and map them during job creation:\n- Define your own field names\n- Map them to prompt/completion in the job configuration", + "source_html": "

Custom Format (specify columns in job)

\n

You can use custom field names and map them during job creation:

\n
    \n
  • Define your own field names
    • \n
  • Map them to prompt/completion in the job configuration
    • \n
\n" + }, + { + "type": "markdown", + "source": "```json\n{\"question\": \"What is 2+2?\", \"answer\": \"4\"}\n```", + "source_html": "
{"question": "What is 2+2?", "answer": "4"}\n
\n" + }, + { + "type": "markdown", + "source": "### 3. Create Dataset FileSet and Upload Training Data", + "source_html": "

3. Create Dataset FileSet and Upload Training Data

\n" + }, + { + "type": "markdown", + "source": "Install huggingface datasets package to download public [rajpurkar/squad](https://huggingface.co/datasets/rajpurkar/squad) dataset if it is not installed in your Python environment:\n\n```sh\npip install datasets\n```", + "source_html": "

Install huggingface datasets package to download public rajpurkar/squad dataset if it is not installed in your Python environment:

\n
pip install datasets\n
\n" + }, + { + "type": "markdown", + "source": "#### Download rajpurkar/squad Dataset\n\nSQuAD (Stanford Question Answering Dataset) is a reading comprehension dataset consisting of questions posed on Wikipedia articles, where the answer is a segment of text from the corresponding passage.", + "source_html": "

Download rajpurkar/squad Dataset

\n

SQuAD (Stanford Question Answering Dataset) is a reading comprehension dataset consisting of questions posed on Wikipedia articles, where the answer is a segment of text from the corresponding passage.

\n" + }, + { + "type": "code", + "source": "from pathlib import Path\nfrom datasets import load_dataset, DatasetDict\nimport json\n\n# Load the SQuAD dataset from Hugging Face\nprint(\"Loading dataset rajpurkar/squad\")\nraw_dataset = load_dataset(\"rajpurkar/squad\")\nif not isinstance(raw_dataset, DatasetDict):\n raise ValueError(\"Dataset does not contain expected splits\")\n\nprint(\"Loaded dataset\")\n\n# Configuration\nVALIDATION_PROPORTION = 0.05\nSEED = 1234\n\n# For the purpose of this tutorial, we'll use a subset of the dataset\n# The larger the datasets, the better the model will perform but longer the training will take\ntraining_size = 3000\nvalidation_size = 300\nDATASET_PATH = Path(\"sft-dataset\").absolute()\n\n# Create directory if it doesn't exist\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Get the train split and create a validation split from it\ntrain_set = raw_dataset.get('train')\nsplit_dataset = train_set.train_test_split(test_size=VALIDATION_PROPORTION, seed=SEED)\n\n# Select subsets for the tutorial\ntrain_ds = split_dataset['train'].select(range(min(training_size, len(split_dataset['train']))))\nvalidation_ds = split_dataset['test'].select(range(min(validation_size, len(split_dataset['test']))))\n\n# Convert SQuAD format to prompt/completion format and save to JSONL\ndef convert_squad_to_sft_format(example):\n \"\"\"Convert SQuAD format to prompt/completion format for SFT training.\"\"\"\n prompt = f\"Context: {example['context']} Question: {example['question']} Answer:\"\n completion = example[\"answers\"][\"text\"][0] # Take the first answer\n return {\"prompt\": prompt, \"completion\": completion}\n\n# Save training data\nwith open(f\"{DATASET_PATH}/training.jsonl\", \"w\", encoding=\"utf-8\") as f:\n for example in train_ds:\n converted = convert_squad_to_sft_format(example)\n f.write(json.dumps(converted) + \"\\n\")\n\n# Save validation data\nwith open(f\"{DATASET_PATH}/validation.jsonl\", \"w\", encoding=\"utf-8\") as f:\n for example in validation_ds:\n converted = convert_squad_to_sft_format(example)\n f.write(json.dumps(converted) + \"\\n\")\n\nprint(f\"Saved training.jsonl with {len(train_ds)} rows\")\nprint(f\"Saved validation.jsonl with {len(validation_ds)} rows\")\n\n# Show a sample from the training data\nprint(\"\\nSample from training data:\")\nwith open(f\"{DATASET_PATH}/training.jsonl\", 'r') as f:\n first_line = f.readline()\n sample = json.loads(first_line)\n print(f\"Prompt: {sample['prompt'][:200]}...\")\n print(f\"Completion: {sample['completion']}\")", + "language": "python", + "source_html": "from pathlib import Path\nfrom datasets import load_dataset, DatasetDict\nimport json\n\n# Load the SQuAD dataset from Hugging Face\nprint("Loading dataset rajpurkar/squad")\nraw_dataset = load_dataset("rajpurkar/squad")\nif not isinstance(raw_dataset, DatasetDict):\n raise ValueError("Dataset does not contain expected splits")\n\nprint("Loaded dataset")\n\n# Configuration\nVALIDATION_PROPORTION = 0.05\nSEED = 1234\n\n# For the purpose of this tutorial, we'll use a subset of the dataset\n# The larger the datasets, the better the model will perform but longer the training will take\ntraining_size = 3000\nvalidation_size = 300\nDATASET_PATH = Path("sft-dataset").absolute()\n\n# Create directory if it doesn't exist\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n# Get the train split and create a validation split from it\ntrain_set = raw_dataset.get('train')\nsplit_dataset = train_set.train_test_split(test_size=VALIDATION_PROPORTION, seed=SEED)\n\n# Select subsets for the tutorial\ntrain_ds = split_dataset['train'].select(range(min(training_size, len(split_dataset['train']))))\nvalidation_ds = split_dataset['test'].select(range(min(validation_size, len(split_dataset['test']))))\n\n# Convert SQuAD format to prompt/completion format and save to JSONL\ndef convert_squad_to_sft_format(example):\n """Convert SQuAD format to prompt/completion format for SFT training."""\n prompt = f"Context: {example['context']} Question: {example['question']} Answer:"\n completion = example["answers"]["text"][0] # Take the first answer\n return {"prompt": prompt, "completion": completion}\n\n# Save training data\nwith open(f"{DATASET_PATH}/training.jsonl", "w", encoding="utf-8") as f:\n for example in train_ds:\n converted = convert_squad_to_sft_format(example)\n f.write(json.dumps(converted) + "\\n")\n\n# Save validation data\nwith open(f"{DATASET_PATH}/validation.jsonl", "w", encoding="utf-8") as f:\n for example in validation_ds:\n converted = convert_squad_to_sft_format(example)\n f.write(json.dumps(converted) + "\\n")\n\nprint(f"Saved training.jsonl with {len(train_ds)} rows")\nprint(f"Saved validation.jsonl with {len(validation_ds)} rows")\n\n# Show a sample from the training data\nprint("\\nSample from training data:")\nwith open(f"{DATASET_PATH}/training.jsonl", 'r') as f:\n first_line = f.readline()\n sample = json.loads(first_line)\n print(f"Prompt: {sample['prompt'][:200]}...")\n print(f"Completion: {sample['completion']}")\n" + }, + { + "type": "code", + "source": "# Create fileset to store SFT training data\nDATASET_NAME = \"sft-dataset\"\n\ntry:\n client.files.filesets.create(\n workspace=\"default\",\n name=DATASET_NAME,\n description=\"SFT training data\"\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\n# Upload training data files individually to ensure correct structure\nclient.files.upload(\n local_path=DATASET_PATH, # Local directory with your JSONL files\n remote_path=\"\",\n fileset=DATASET_NAME,\n workspace=\"default\"\n)\n\n# Validate training data is uploaded correctly\nprint(\"Training data:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace=\"default\").data], indent=2))", + "language": "python", + "source_html": "# Create fileset to store SFT training data\nDATASET_NAME = "sft-dataset"\n\ntry:\n client.files.filesets.create(\n workspace="default",\n name=DATASET_NAME,\n description="SFT training data"\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\n# Upload training data files individually to ensure correct structure\nclient.files.upload(\n local_path=DATASET_PATH, # Local directory with your JSONL files\n remote_path="",\n fileset=DATASET_NAME,\n workspace="default"\n)\n\n# Validate training data is uploaded correctly\nprint("Training data:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=DATASET_NAME, workspace="default").data], indent=2))\n" + }, + { + "type": "markdown", + "source": "### 4. Secrets Setup\n\nIf you plan to use NGC or HuggingFace models, you will need to configure authentication:\n\n- **NGC models** (`ngc://` URIs): Requires NGC API key\n- **HuggingFace models** (`hf://` URIs): Requires HF token for gated/private models\n\n\nConfigure these as secrets in your platform. Refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md) for detailed instructions.\n\nGet your credentials to access base models:\n- [NGC API Key](https://ngc.nvidia.com/) (Setup → Generate API Key)\n- [HuggingFace Token](https://huggingface.co/settings/tokens) (Create token with Read access)\n\n\n---\n\n#### Quick Setup Example\n\nIn this tutorial we are going to work with [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) model from HuggingFace. Ensure that you have sufficient permissions to download the model. If you cannot access the files on the [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) Hugging Face page, request access\n\n**HuggingFace Authentication:**\n- For gated models (Llama, Gemma), you must provide a HuggingFace token via the `token_secret` parameter\n- Get your token from [HuggingFace Settings](https://huggingface.co/settings/tokens) (requires Read access)\n- Accept the model's terms on the HuggingFace model page before using it. Example: [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main)\n- For public models, you can omit the `token_secret` parameter when creating a fileset for model in the next step", + "source_html": "

4. Secrets Setup

\n

If you plan to use NGC or HuggingFace models, you will need to configure authentication:

\n
    \n
  • NGC models (ngc:// URIs): Requires NGC API key
    • \n
  • HuggingFace models (hf:// URIs): Requires HF token for gated/private models
    • \n
\n

Configure these as secrets in your platform. Refer to Managing Secrets for detailed instructions.

\n

Get your credentials to access base models:

\n\n
\n

Quick Setup Example

\n

In this tutorial we are going to work with meta-llama/Llama-3.2-1B-Instruct model from HuggingFace. Ensure that you have sufficient permissions to download the model. If you cannot access the files on the meta-llama/Llama-3.2-1B-Instruct Hugging Face page, request access

\n

HuggingFace Authentication:

\n
    \n
  • For gated models (Llama, Gemma), you must provide a HuggingFace token via the token_secret parameter
    • \n
  • Get your token from HuggingFace Settings (requires Read access)
    • \n
  • Accept the model's terms on the HuggingFace model page before using it. Example: meta-llama/Llama-3.2-1B-Instruct
    • \n
  • For public models, you can omit the token_secret parameter when creating a fileset for model in the next step
    • \n
\n" + }, + { + "type": "code", + "source": "# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set\nHF_TOKEN = os.getenv(\"HF_TOKEN\")\nNGC_API_KEY = os.getenv(\"NGC_API_KEY\")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f\"{label} is not set\")\n try:\n secret = client.secrets.create(\n name=name,\n workspace=\"default\",\n value=value,\n )\n print(f\"Created secret: {name}\")\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n return client.secrets.retrieve(name=name, workspace=\"default\")\n\n\n# Create HuggingFace token secret\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")\nprint(\"HF_TOKEN secret:\")\nprint(hf_secret.model_dump_json(indent=2))\n\n# Create NGC API key secret\n# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n# ngc_api_key = create_or_get_secret(\"ngc-api-key\", NGC_API_KEY, \"NGC_API_KEY\")", + "language": "python", + "source_html": "# Export the HF_TOKEN and NGC_API_KEY environment variables if they are not already set\nHF_TOKEN = os.getenv("HF_TOKEN")\nNGC_API_KEY = os.getenv("NGC_API_KEY")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f"{label} is not set")\n try:\n secret = client.secrets.create(\n name=name,\n workspace="default",\n value=value,\n )\n print(f"Created secret: {name}")\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n return client.secrets.retrieve(name=name, workspace="default")\n\n\n# Create HuggingFace token secret\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\nprint("HF_TOKEN secret:")\nprint(hf_secret.model_dump_json(indent=2))\n\n# Create NGC API key secret\n# Uncomment the line below if you have NGC API Key and want to finetune NGC models\n# ngc_api_key = create_or_get_secret("ngc-api-key", NGC_API_KEY, "NGC_API_KEY")\n" + }, + { + "type": "markdown", + "source": "### 5. Create Base Model FileSet and Model Entity\n\nCreate a fileset pointing to [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct/tree/main) model in HuggingFace that we will train with SFT. Then create a Model Entity that references this fileset. Model downloading will take place at training time.\n\nNote: for public models, you can omit the `token_secret` parameter when creating a model fileset.", + "source_html": "

5. Create Base Model FileSet and Model Entity

\n

Create a fileset pointing to meta-llama/Llama-3.2-1B-Instruct model in HuggingFace that we will train with SFT. Then create a Model Entity that references this fileset. Model downloading will take place at training time.

\n

Note: for public models, you can omit the token_secret parameter when creating a model fileset.

\n" + }, + { + "type": "code", + "source": "import time\n\n# Create a fileset pointing to the desired HuggingFace model\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = \"meta-llama/Llama-3.2-1B-Instruct\"\nMODEL_NAME = \"llama-3-2-1b-base\"\n\n# Ensure you have a HuggingFace token secret created\ntry:\n base_model_fs = client.files.filesets.create(\n workspace=\"default\",\n name=MODEL_NAME,\n description=\"Llama 3.2 1B base model from HuggingFace\",\n storage=HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type=\"model\",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\n print(f\"Created base model fileset: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model fileset already exists. Skipping creation.\")\n base_model_fs = client.files.filesets.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\n# Create the Model Entity representation.\ntry:\n base_model = client.models.create(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n )\n print(f\"Created Model Entity: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model already exists. Updating fileset if different.\")\n base_model = client.models.update(\n workspace=\"default\",\n name=MODEL_NAME,\n fileset=f\"default/{MODEL_NAME}\",\n )\n\nprint(f\"\\nBase model fileset: fileset://default/{base_model.name}\")\nprint(\"Base model fileset files list:\")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace=\"default\").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint(\"\\nWaiting for ModelSpec to be populated...\")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f\"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds\")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace=\"default\",\n name=MODEL_NAME,\n )\n\nprint(f\"ModelSpec populated: {base_model.spec}\")", + "language": "python", + "source_html": "import time\n\n# Create a fileset pointing to the desired HuggingFace model\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = "meta-llama/Llama-3.2-1B-Instruct"\nMODEL_NAME = "llama-3-2-1b-base"\n\n# Ensure you have a HuggingFace token secret created\ntry:\n base_model_fs = client.files.filesets.create(\n workspace="default",\n name=MODEL_NAME,\n description="Llama 3.2 1B base model from HuggingFace",\n storage=HuggingfaceStorageConfigParam(\n type="huggingface",\n # repo_id is the full model name from Hugging Face\n repo_id=HF_REPO_ID,\n repo_type="model",\n # we use the secret created in the previous step\n token_secret=hf_secret.name\n )\n )\n print(f"Created base model fileset: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model fileset already exists. Skipping creation.")\n base_model_fs = client.files.filesets.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\n# Create the Model Entity representation.\ntry:\n base_model = client.models.create(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n )\n print(f"Created Model Entity: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model already exists. Updating fileset if different.")\n base_model = client.models.update(\n workspace="default",\n name=MODEL_NAME,\n fileset=f"default/{MODEL_NAME}",\n )\n\nprint(f"\\nBase model fileset: fileset://default/{base_model.name}")\nprint("Base model fileset files list:")\nprint(json.dumps([f.model_dump() for f in client.files.list(fileset=MODEL_NAME, workspace="default").data], indent=2))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint("\\nWaiting for ModelSpec to be populated...")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds")\n time.sleep(2)\n base_model = client.models.retrieve(\n workspace="default",\n name=MODEL_NAME,\n )\n\nprint(f"ModelSpec populated: {base_model.spec}")\n" + }, + { + "type": "markdown", + "source": "### 6. Create SFT Finetuning Job\nCreate a customization job with an inline target referencing the base model and dataset filesets created in previous steps.", + "source_html": "

6. Create SFT Finetuning Job

\n

Create a customization job with an inline target referencing the base model and dataset filesets created in previous steps.

\n" + }, + { + "type": "markdown", + "source": "\n**GPU Requirements:**\n- 1B models: 1 GPU (24GB+ VRAM)\n- 3B models: 1-2 GPUs \n- 8B models: 2-4 GPUs\n- 70B models: 8+ GPUs \n\nAdjust `num_gpus_per_node` based on your model size.", + "source_html": "

GPU Requirements:

\n
    \n
  • 1B models: 1 GPU (24GB+ VRAM)
    • \n
  • 3B models: 1-2 GPUs
    • \n
  • 8B models: 2-4 GPUs
    • \n
  • 70B models: 8+ GPUs
    • \n
\n

Adjust num_gpus_per_node based on your model size.

\n" + }, + { + "type": "code", + "source": "import uuid\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam,\n)\n\njob_suffix = uuid.uuid4().hex[:4]\n\nJOB_NAME = f\"my-sft-job-{job_suffix}\"\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace=\"default\",\n spec=CustomizationJobInputParam(\n model=f\"default/{base_model.name}\",\n dataset=f\"fileset://default/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=2,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n )\n)\n\nprint(f\"Job ID: {job.name}\")\nprint(f\"Output model: {job.spec.output.name}\")", + "language": "python", + "source_html": "import uuid\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam,\n)\n\njob_suffix = uuid.uuid4().hex[:4]\n\nJOB_NAME = f"my-sft-job-{job_suffix}"\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n workspace="default",\n spec=CustomizationJobInputParam(\n model=f"default/{base_model.name}",\n dataset=f"fileset://default/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=2,\n batch_size=64,\n learning_rate=0.00005,\n max_seq_length=2048,\n micro_batch_size=1,\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n )\n)\n\nprint(f"Job ID: {job.name}")\nprint(f"Output model: {job.spec.output.name}")\n" + }, + { + "type": "markdown", + "source": "### 7. Track Training Progress", + "source_html": "

7. Track Training Progress

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Poll job status every 10 seconds until completed\nwhile True:\n status = client.customization.jobs.get_status(\n name=job.name,\n workspace=\"default\"\n )\n\n clear_output(wait=True)\n print(f\"Job Status: {status.model_dump_json(indent=2)}\")\n\n # Extract training progress from nested steps structure\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n\n for job_step in status.steps or []:\n if job_step.name == \"customization-training-job\":\n for task in job_step.tasks or []:\n task_details = task.status_details or {}\n step = task_details.get(\"step\")\n max_steps = task_details.get(\"max_steps\")\n training_phase = task_details.get(\"phase\")\n break\n break\n\n if step is not None and max_steps is not None:\n progress_pct = (step / max_steps) * 100\n print(f\"Training Progress: Step {step}/{max_steps} ({progress_pct:.1f}%)\")\n if training_phase:\n print(f\"Training Phase: {training_phase}\")\n else:\n print(\"Training step not started yet or progress info not available\")\n\n # Exit loop when job is completed (or failed/cancelled)\n if status.status in (\"completed\", \"failed\", \"cancelled\", \"error\"):\n print(f\"\\nJob finished with status: {status.status}\")\n break\n\n time.sleep(10)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Poll job status every 10 seconds until completed\nwhile True:\n status = client.customization.jobs.get_status(\n name=job.name,\n workspace="default"\n )\n\n clear_output(wait=True)\n print(f"Job Status: {status.model_dump_json(indent=2)}")\n\n # Extract training progress from nested steps structure\n step: int | None = None\n max_steps: int | None = None\n training_phase: str | None = None\n\n for job_step in status.steps or []:\n if job_step.name == "customization-training-job":\n for task in job_step.tasks or []:\n task_details = task.status_details or {}\n step = task_details.get("step")\n max_steps = task_details.get("max_steps")\n training_phase = task_details.get("phase")\n break\n break\n\n if step is not None and max_steps is not None:\n progress_pct = (step / max_steps) * 100\n print(f"Training Progress: Step {step}/{max_steps} ({progress_pct:.1f}%)")\n if training_phase:\n print(f"Training Phase: {training_phase}")\n else:\n print("Training step not started yet or progress info not available")\n\n # Exit loop when job is completed (or failed/cancelled)\n if status.status in ("completed", "failed", "cancelled", "error"):\n print(f"\\nJob finished with status: {status.status}")\n break\n\n time.sleep(10)\n" + }, + { + "type": "markdown", + "source": "**Interpreting SFT Training Metrics:**\n\nMonitor the relationship between training and validation loss curves:\n\n| Scenario | Interpretation | Action |\n|----------|----------------|--------|\n| **Both decreasing together** | Model is learning well | Continue training |\n| **Training decreases, validation flat/increasing** | Overfitting | Reduce epochs, add data |\n| **Both flat/not decreasing** | Underfitting | Increase LR, check data |\n| **Sudden spikes** | Training instability | Lower learning rate |\n\n**Note:** Training metrics measure optimization progress, not final model quality. Always evaluate the deployed model on your specific use case.", + "source_html": "

Interpreting SFT Training Metrics:

\n

Monitor the relationship between training and validation loss curves:

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
ScenarioInterpretationAction
Both decreasing togetherModel is learning wellContinue training
Training decreases, validation flat/increasingOverfittingReduce epochs, add data
Both flat/not decreasingUnderfittingIncrease LR, check data
Sudden spikesTraining instabilityLower learning rate
\n

Note: Training metrics measure optimization progress, not final model quality. Always evaluate the deployed model on your specific use case.

\n" + }, + { + "type": "markdown", + "source": "(ft-deploy-full-weight-model)=\n\n### 8. Deploy Fine-Tuned Model\n\nAfter training completes, deploy using the Deployment Management Service:", + "source_html": "

(ft-deploy-full-weight-model)=

\n

8. Deploy Fine-Tuned Model

\n

After training completes, deploy using the Deployment Management Service:

\n" + }, + { + "type": "code", + "source": "# Validate model entity exists\nmodel_entity = client.models.retrieve(workspace='default', name=job.spec.output.name)\nprint(model_entity.model_dump_json(indent=2))", + "language": "python", + "source_html": "# Validate model entity exists\nmodel_entity = client.models.retrieve(workspace='default', name=job.spec.output.name)\nprint(model_entity.model_dump_json(indent=2))\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# Create deployment config\ndeploy_suffix = uuid.uuid4().hex[:4]\nDEPLOYMENT_CONFIG_NAME = f\"sft-model-deployment-cfg-{deploy_suffix}\"\nDEPLOYMENT_NAME = f\"sft-model-deployment-{deploy_suffix}\"\n\ndeployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=DEPLOYMENT_CONFIG_NAME,\n nim_deployment=NIMDeploymentParam(\n image_name=\"nvcr.io/nim/nvidia/llm-nim\",\n image_tag=\"1.15.5\",\n gpu=1,\n model_name=job.spec.output.name, # ModelEntity name from training,\n model_namespace=\"default\", # Workspace where ModelEntity lives\n additional_envs={\"NIM_MODEL_PROFILE\": \"vllm\"}\n ),\n)\n\n# Deploy model using deployment_config created above\ndeployment = client.inference.deployments.create(\n workspace=\"default\",\n name=DEPLOYMENT_NAME,\n config=deployment_config.name\n)\n\n\n# Check deployment status\ndeployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace=\"default\"\n)\n\nprint(f\"Deployment name: {deployment.name}\")\nprint(f\"Deployment status: {deployment_status.status}\")", + "language": "python", + "source_html": "from nemo_platform.types.inference import NIMDeploymentParam\n\n# Create deployment config\ndeploy_suffix = uuid.uuid4().hex[:4]\nDEPLOYMENT_CONFIG_NAME = f"sft-model-deployment-cfg-{deploy_suffix}"\nDEPLOYMENT_NAME = f"sft-model-deployment-{deploy_suffix}"\n\ndeployment_config = client.inference.deployment_configs.create(\n workspace="default",\n name=DEPLOYMENT_CONFIG_NAME,\n nim_deployment=NIMDeploymentParam(\n image_name="nvcr.io/nim/nvidia/llm-nim",\n image_tag="1.15.5",\n gpu=1,\n model_name=job.spec.output.name, # ModelEntity name from training,\n model_namespace="default", # Workspace where ModelEntity lives\n additional_envs={"NIM_MODEL_PROFILE": "vllm"}\n ),\n)\n\n# Deploy model using deployment_config created above\ndeployment = client.inference.deployments.create(\n workspace="default",\n name=DEPLOYMENT_NAME,\n config=deployment_config.name\n)\n\n\n# Check deployment status\ndeployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace="default"\n)\n\nprint(f"Deployment name: {deployment.name}")\nprint(f"Deployment status: {deployment_status.status}")\n" + }, + { + "type": "markdown", + "source": "The deployment service automatically:\n- Downloads model weights from the Files service\n- Provisions storage (PVC) for the weights\n- Configures and starts the NIM container\n\n**Multi-GPU Deployment:**\n\nFor larger models requiring multiple GPUs, configure parallelism with environment variables:\n\n```python\ndeployment_config = client.inference.deployment_configs.create(\n workspace=\"default\",\n name=\"sft-model-config-multigpu\",\n \n nim_deployment={\n \"image_name\": \"nvcr.io/nim/nvidia/llm-nim\",\n \"image_tag\": \"1.13.1\",\n \"gpu\": 2, # Total GPUs\n \"additional_envs\": {\n \"NIM_TENSOR_PARALLEL_SIZE\": \"2\", # Tensor parallelism\n \"NIM_PIPELINE_PARALLEL_SIZE\": \"1\" # Pipeline parallelism\n }\n }\n)\n```", + "source_html": "

The deployment service automatically:

\n
    \n
  • Downloads model weights from the Files service
    • \n
  • Provisions storage (PVC) for the weights
    • \n
  • Configures and starts the NIM container
    • \n
\n

Multi-GPU Deployment:

\n

For larger models requiring multiple GPUs, configure parallelism with environment variables:

\n
deployment_config = client.inference.deployment_configs.create(\n    workspace="default",\n    name="sft-model-config-multigpu",\n    \n    nim_deployment={\n        "image_name": "nvcr.io/nim/nvidia/llm-nim",\n        "image_tag": "1.13.1",\n        "gpu": 2,  # Total GPUs\n        "additional_envs": {\n            "NIM_TENSOR_PARALLEL_SIZE": "2",  # Tensor parallelism\n            "NIM_PIPELINE_PARALLEL_SIZE": "1"  # Pipeline parallelism\n        }\n    }\n)\n
\n" + }, + { + "type": "markdown", + "source": "**Single-Node Constraint:** Model deployments are limited to a single node. The maximum `gpu` value depends on the total GPUs available on a single node in your cluster. Multi-node deployments are not supported.\n\n---\n\n#### GPU Parallelism\n\nBy default, NIM uses all GPUs for tensor parallelism (TP). You can customize this behavior using the `NIM_TENSOR_PARALLEL_SIZE` and `NIM_PIPELINE_PARALLEL_SIZE` environment variables.\n\n| Strategy | Description | Best For |\n|----------|-------------|----------|\n| **Tensor Parallel (TP)** | Splits model layers across GPUs | Lowest latency |\n| **Pipeline Parallel (PP)** | Splits model depth across GPUs | Highest throughput |\n\n**Formula:** `gpu` = `NIM_TENSOR_PARALLEL_SIZE` × `NIM_PIPELINE_PARALLEL_SIZE`\n\n---\n\n#### Example Configurations\n\n**Default (TP=8, PP=1) — Lowest Latency**\n```\n\"gpu\": 8\n# NIM automatically sets NIM_TENSOR_PARALLEL_SIZE=8\n```\n\n**Balanced (TP=4, PP=2)**\n```\n\"gpu\": 8,\n\"additional_envs\": {\n \"NIM_TENSOR_PARALLEL_SIZE\": \"4\",\n \"NIM_PIPELINE_PARALLEL_SIZE\": \"2\"\n}\n```\n\n**Throughput Optimized (TP=2, PP=4)**\n```\n\"gpu\": 8,\n\"additional_envs\": {\n \"NIM_TENSOR_PARALLEL_SIZE\": \"2\",\n \"NIM_PIPELINE_PARALLEL_SIZE\": \"4\"\n}\n```", + "source_html": "

Single-Node Constraint: Model deployments are limited to a single node. The maximum gpu value depends on the total GPUs available on a single node in your cluster. Multi-node deployments are not supported.

\n
\n

GPU Parallelism

\n

By default, NIM uses all GPUs for tensor parallelism (TP). You can customize this behavior using the NIM_TENSOR_PARALLEL_SIZE and NIM_PIPELINE_PARALLEL_SIZE environment variables.

\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
StrategyDescriptionBest For
Tensor Parallel (TP)Splits model layers across GPUsLowest latency
Pipeline Parallel (PP)Splits model depth across GPUsHighest throughput
\n

Formula: gpu = NIM_TENSOR_PARALLEL_SIZE × NIM_PIPELINE_PARALLEL_SIZE

\n
\n

Example Configurations

\n

Default (TP=8, PP=1) — Lowest Latency

\n
"gpu": 8\n# NIM automatically sets NIM_TENSOR_PARALLEL_SIZE=8\n
\n

Balanced (TP=4, PP=2)

\n
"gpu": 8,\n"additional_envs": {\n    "NIM_TENSOR_PARALLEL_SIZE": "4",\n    "NIM_PIPELINE_PARALLEL_SIZE": "2"\n}\n
\n

Throughput Optimized (TP=2, PP=4)

\n
"gpu": 8,\n"additional_envs": {\n    "NIM_TENSOR_PARALLEL_SIZE": "2",\n    "NIM_PIPELINE_PARALLEL_SIZE": "4"\n}\n
\n" + }, + { + "type": "markdown", + "source": "### Track Deployment Status", + "source_html": "

Track Deployment Status

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\n# Poll deployment status every 15 seconds until ready\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f\"Monitoring deployment '{deployment.name}'...\")\nprint(f\"Timeout: {TIMEOUT_MINUTES} minutes\\n\")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace=\"default\"\n )\n\n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n\n clear_output(wait=True)\n print(f\"Deployment: {deployment.name}\")\n print(f\"Status: {deployment_status.status}\")\n print(f\"Elapsed time: {elapsed_min}m {elapsed_sec}s\")\n\n # Check if deployment is ready\n if deployment_status.status == \"READY\":\n print(\"\\nDeployment is ready!\")\n if not client.models.wait_for_gateway(deployment.name, workspace=\"default\", timeout=60):\n raise RuntimeError(\"Inference gateway did not become ready\")\n break\n\n # Check for failure states\n if deployment_status.status in (\"FAILED\", \"ERROR\", \"TERMINATED\", \"LOST\"):\n raise RuntimeError(f\"Deployment failed with status: {deployment_status.status}\")\n\n # Check timeout\n if elapsed > timeout_seconds:\n raise TimeoutError(f\"Deployment timeout after {TIMEOUT_MINUTES} minutes\")\n\n time.sleep(15)", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\n# Poll deployment status every 15 seconds until ready\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f"Monitoring deployment '{deployment.name}'...")\nprint(f"Timeout: {TIMEOUT_MINUTES} minutes\\n")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=deployment.name,\n workspace="default"\n )\n\n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n\n clear_output(wait=True)\n print(f"Deployment: {deployment.name}")\n print(f"Status: {deployment_status.status}")\n print(f"Elapsed time: {elapsed_min}m {elapsed_sec}s")\n\n # Check if deployment is ready\n if deployment_status.status == "READY":\n print("\\nDeployment is ready!")\n if not client.models.wait_for_gateway(deployment.name, workspace="default", timeout=60):\n raise RuntimeError("Inference gateway did not become ready")\n break\n\n # Check for failure states\n if deployment_status.status in ("FAILED", "ERROR", "TERMINATED", "LOST"):\n raise RuntimeError(f"Deployment failed with status: {deployment_status.status}")\n\n # Check timeout\n if elapsed > timeout_seconds:\n raise TimeoutError(f"Deployment timeout after {TIMEOUT_MINUTES} minutes")\n\n time.sleep(15)\n" + }, + { + "type": "markdown", + "source": "### 9. Evaluate Your Model\n\nAfter training, evaluate whether your model meets your requirements:\n\n#### Quick Manual Evaluation", + "source_html": "

9. Evaluate Your Model

\n

After training, evaluate whether your model meets your requirements:

\n

Quick Manual Evaluation

\n" + }, + { + "type": "code", + "source": "# Wait for deployment to be ready, then test\n# Test the fine-tuned model with a question answering prompt\ncontext = \"The Apollo 11 mission was the first manned mission to land on the Moon. It was launched on July 16, 1969, and Neil Armstrong became the first person to walk on the lunar surface on July 20, 1969. Buzz Aldrin joined him shortly after, while Michael Collins remained in lunar orbit.\"\nquestion = \"Who was the first person to walk on the Moon?\"\n\nmessages = [\n {\"role\": \"user\", \"content\": f\"Based on the following context, answer the question.\\n\\nContext: {context}\\n\\nQuestion: {question}\"}\n]\n\nresponse = client.inference.gateway.provider.post(\n \"v1/chat/completions\",\n name=deployment.name,\n workspace=\"default\",\n body={\n \"model\": f\"default/{job.spec.output.name}\",\n \"messages\": messages,\n \"temperature\": 0,\n \"max_tokens\": 128\n }\n)\n\nprint(\"=\" * 60)\nprint(\"MODEL EVALUATION\")\nprint(\"=\" * 60)\nprint(f\"Question: {question}\")\nprint(f\"Expected: Neil Armstrong\")\nprint(f\"Model output: {response['choices'][0]['message']['content']}\")", + "language": "python", + "source_html": "# Wait for deployment to be ready, then test\n# Test the fine-tuned model with a question answering prompt\ncontext = "The Apollo 11 mission was the first manned mission to land on the Moon. It was launched on July 16, 1969, and Neil Armstrong became the first person to walk on the lunar surface on July 20, 1969. Buzz Aldrin joined him shortly after, while Michael Collins remained in lunar orbit."\nquestion = "Who was the first person to walk on the Moon?"\n\nmessages = [\n {"role": "user", "content": f"Based on the following context, answer the question.\\n\\nContext: {context}\\n\\nQuestion: {question}"}\n]\n\nresponse = client.inference.gateway.provider.post(\n "v1/chat/completions",\n name=deployment.name,\n workspace="default",\n body={\n "model": f"default/{job.spec.output.name}",\n "messages": messages,\n "temperature": 0,\n "max_tokens": 128\n }\n)\n\nprint("=" * 60)\nprint("MODEL EVALUATION")\nprint("=" * 60)\nprint(f"Question: {question}")\nprint(f"Expected: Neil Armstrong")\nprint(f"Model output: {response['choices'][0]['message']['content']}")\n" + }, + { + "type": "markdown", + "source": "#### Evaluation Best Practices\n\n**Manual Evaluation** (Recommended)\n- Test with real-world examples from your use case\n- Compare responses to base model and expected outputs\n- Verify the model exhibits desired behavior changes\n- Check edge cases and error handling\n\n**What to look for:**\n- ✅ Model follows your desired output format\n- ✅ Applies domain knowledge correctly\n- ✅ Maintains general language capabilities\n- ✅ Avoids unwanted behaviors or biases\n- ❌ Doesn't hallucinate facts not in training data\n- ❌ Doesn't produce repetitive or nonsensical outputs\n\n---\n\n## Hyperparameters\n\nFor detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the [Hyperparameter Reference](../manage-customization-jobs/hyperparameters.md).\n\n---\n\n\n## Troubleshooting\n\n**Job fails during model download:**\n- Verify authentication secrets are configured (refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md))\n- For gated HuggingFace models (Llama, Gemma), accept the license on the model page\n- Check the `model_uri` format is correct (`fileset://`)\n- Ensure you have accepted the model's terms of service on HuggingFace\n- Check job status and logs: `client.customization.jobs.retrieve(name=job.name, workspace=\"default\")`\n\n**Job fails with OOM (Out of Memory) error:**\n1. **First try:** Reduce `micro_batch_size` from 2 to 1\n2. **Still OOM:** Reduce `batch_size` from 4 to 2\n3. **Still OOM:** Reduce `max_seq_length` from 2048 to 1024 or 512\n4. **Last resort:** Increase GPU count and use `tensor_parallel_size` for model sharding\n\n**Loss curves not decreasing (underfitting):**\n- Increase training duration: `epochs: 5-10` instead of 3\n- Adjust learning rate: Try `1e-5` to `1e-4`\n- Add warmup: Set `warmup_steps` to ~10% of total training steps\n- Check data quality: Verify formatting, remove duplicates, ensure diversity\n\n**Training loss decreases but validation loss increases (overfitting):**\n- Reduce epochs: Try `epochs: 1-2` instead of 5+\n- Lower learning rate: Use `2e-5` or `1e-5`\n- Increase dataset size and diversity\n- Verify train/validation split has no data leakage\n\n**Model output quality is poor despite good training metrics:**\n- Training metrics optimize for loss, not your actual task—evaluate on real use cases\n- Review data quality, format, and diversity—metrics can be misleading with poor data\n- Try a different base model size or architecture\n- Adjust learning rate and batch size\n- Compare to baseline: Test base model to ensure fine-tuning improved performance\n\n**Deployment fails:**\n- Verify output model exists: `client.models.retrieve(name=job.spec.output.name, workspace=\"default\")`\n- Check deployment logs: `client.inference.deployments.get_logs(name=deployment.name, workspace=\"default\")`\n- Ensure sufficient GPU resources available for model size\n- Verify NIM image tag `1.13.1` is compatible with your model\n\n\n## Next Steps\n\n- [Monitor training metrics](fine-tune-metrics) in detail\n- [Evaluate your fine-tuned model](../../evaluator/index) using the Evaluator service\n- Learn about [LoRA customization](./lora-customization-job) for resource-efficient fine-tuning", + "source_html": "

Evaluation Best Practices

\n

Manual Evaluation (Recommended)

\n
    \n
  • Test with real-world examples from your use case
    • \n
  • Compare responses to base model and expected outputs
    • \n
  • Verify the model exhibits desired behavior changes
    • \n
  • Check edge cases and error handling
    • \n
\n

What to look for:

\n
    \n
  • ✅ Model follows your desired output format
    • \n
  • ✅ Applies domain knowledge correctly
    • \n
  • ✅ Maintains general language capabilities
    • \n
  • ✅ Avoids unwanted behaviors or biases
    • \n
  • ❌ Doesn't hallucinate facts not in training data
    • \n
  • ❌ Doesn't produce repetitive or nonsensical outputs
    • \n
\n
\n

Hyperparameters

\n

For detailed information on all available hyperparameters, recommended values, and tuning guidance, refer to the Hyperparameter Reference.

\n
\n

Troubleshooting

\n

Job fails during model download:

\n
    \n
  • Verify authentication secrets are configured (refer to Managing Secrets)
    • \n
  • For gated HuggingFace models (Llama, Gemma), accept the license on the model page
    • \n
  • Check the model_uri format is correct (fileset://)
    • \n
  • Ensure you have accepted the model's terms of service on HuggingFace
    • \n
  • Check job status and logs: client.customization.jobs.retrieve(name=job.name, workspace="default")
    • \n
\n

Job fails with OOM (Out of Memory) error:

\n
    \n
  1. First try: Reduce micro_batch_size from 2 to 1
    2. \n
  2. Still OOM: Reduce batch_size from 4 to 2
    3. \n
  3. Still OOM: Reduce max_seq_length from 2048 to 1024 or 512
    4. \n
  4. Last resort: Increase GPU count and use tensor_parallel_size for model sharding
    5. \n
\n

Loss curves not decreasing (underfitting):

\n
    \n
  • Increase training duration: epochs: 5-10 instead of 3
    • \n
  • Adjust learning rate: Try 1e-5 to 1e-4
    • \n
  • Add warmup: Set warmup_steps to ~10% of total training steps
    • \n
  • Check data quality: Verify formatting, remove duplicates, ensure diversity
    • \n
\n

Training loss decreases but validation loss increases (overfitting):

\n
    \n
  • Reduce epochs: Try epochs: 1-2 instead of 5+
    • \n
  • Lower learning rate: Use 2e-5 or 1e-5
    • \n
  • Increase dataset size and diversity
    • \n
  • Verify train/validation split has no data leakage
    • \n
\n

Model output quality is poor despite good training metrics:

\n
    \n
  • Training metrics optimize for loss, not your actual task—evaluate on real use cases
    • \n
  • Review data quality, format, and diversity—metrics can be misleading with poor data
    • \n
  • Try a different base model size or architecture
    • \n
  • Adjust learning rate and batch size
    • \n
  • Compare to baseline: Test base model to ensure fine-tuning improved performance
    • \n
\n

Deployment fails:

\n
    \n
  • Verify output model exists: client.models.retrieve(name=job.spec.output.name, workspace="default")
    • \n
  • Check deployment logs: client.inference.deployments.get_logs(name=deployment.name, workspace="default")
    • \n
  • Ensure sufficient GPU resources available for model size
    • \n
  • Verify NIM image tag 1.13.1 is compatible with your model
    • \n
\n

Next Steps

\n\n" + } +] }; diff --git a/docs/fern/components/notebooks/tool-calling.json b/docs/fern/components/notebooks/tool-calling.json new file mode 100644 index 00000000000..d8af8b353c5 --- /dev/null +++ b/docs/fern/components/notebooks/tool-calling.json @@ -0,0 +1,289 @@ +{ + "cells": [ + { + "type": "markdown", + "source": "\n\n\n# Tool-Calling Fine-Tuning with Synthetic Data\n\nLearn how to fine-tune Llama 3.2 1B Instruct for tool calling using LoRA and synthetic data from Data Designer.\n\n## About\n\nTool calling enables LLMs to interact with external functions and APIs by generating structured function calls instead of plain text. Fine-tuning a model specifically for tool calling improves its ability to select the correct function and produce valid arguments.\n\nThis tutorial walks through the full workflow:\n\n1. **Data preparation** -- Generate a synthetic dataset with Data Designer, convert it to OpenAI chat format, and filter for compatibility with the Llama 3.2 chat template\n2. **LoRA fine-tuning** -- Train a LoRA adapter on the prepared dataset\n3. **Deployment and inference** -- Deploy the fine-tuned model and compare tool calling behavior against the base model\n4. **Evaluation** -- Measure tool calling accuracy using the platform's evaluation service", + "source_html": "\n\n

Tool-Calling Fine-Tuning with Synthetic Data

\n

Learn how to fine-tune Llama 3.2 1B Instruct for tool calling using LoRA and synthetic data from Data Designer.

\n

About

\n

Tool calling enables LLMs to interact with external functions and APIs by generating structured function calls instead of plain text. Fine-tuning a model specifically for tool calling improves its ability to select the correct function and produce valid arguments.

\n

This tutorial walks through the full workflow:

\n
    \n
  1. Data preparation -- Generate a synthetic dataset with Data Designer, convert it to OpenAI chat format, and filter for compatibility with the Llama 3.2 chat template
    2. \n
  2. LoRA fine-tuning -- Train a LoRA adapter on the prepared dataset
    3. \n
  3. Deployment and inference -- Deploy the fine-tuned model and compare tool calling behavior against the base model
    4. \n
  4. Evaluation -- Measure tool calling accuracy using the platform's evaluation service
    5. \n
\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Installed the Python SDK with the Data Designer extra** (`uv pip install nemo-platform[data-designer]`)\n1. **Completed the [Quickstart](../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n1. (Optional if running outside of Quickstart) **Authenticated with the platform** using the CLI:\n ```bash\n nemo auth login\n ```\n For non-default URLs: `nemo auth login --base-url `\n1. Set your **Hugging Face token** as an environment variable before running the notebook (`export HF_TOKEN=\"\"`)\n1. **Accepted the dataset license** for [Salesforce/xlam-function-calling-60k](https://huggingface.co/datasets/Salesforce/xlam-function-calling-60k) on Hugging Face (used for evaluation) ", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Installed the Python SDK with the Data Designer extra (uv pip install nemo-platform[data-designer])
    2. \n
  2. Completed the Quickstart to install and deploy NeMo Platform locally
    3. \n
  3. (Optional if running outside of Quickstart) Authenticated with the platform using the CLI:
    nemo auth login\n
    \nFor non-default URLs: nemo auth login --base-url <YOUR_NMP_BASE_URL>
    4. \n
  4. Set your Hugging Face token as an environment variable before running the notebook (export HF_TOKEN="<your-token>")
    5. \n
  5. Accepted the dataset license for Salesforce/xlam-function-calling-60k on Hugging Face (used for evaluation)
    6. \n
\n" + }, + { + "type": "markdown", + "source": "## 1. Initialize SDK\n\nThe SDK reads its cluster URL and authentication credentials from the config file managed by `nemo auth login`. For the default [Quickstart](../get-started/quickstart.md) deployment at `http://localhost:8080`, no extra configuration is needed.\n\nWe also define the provider and model we will be using for synthetic data generation. These must be available in your NeMo Platform deployment.\n\n> **Note:** This tutorial uses config-based initialization (`NeMoPlatform(workspace=...)`) instead of the explicit `base_url` pattern shown elsewhere in the docs to make it easier to switch between Quickstart and running on a cluster.", + "source_html": "

1. Initialize SDK

\n

The SDK reads its cluster URL and authentication credentials from the config file managed by nemo auth login. For the default Quickstart deployment at http://localhost:8080, no extra configuration is needed.

\n

We also define the provider and model we will be using for synthetic data generation. These must be available in your NeMo Platform deployment.

\n
\n

Note: This tutorial uses config-based initialization (NeMoPlatform(workspace=...)) instead of the explicit base_url pattern shown elsewhere in the docs to make it easier to switch between Quickstart and running on a cluster.

\n
\n" + }, + { + "type": "code", + "source": "import os\n\nfrom datasets import load_dataset as hf_load_dataset\nfrom nemo_platform import NeMoPlatform, ConflictError\nimport data_designer.config as dd\nfrom pydantic import BaseModel, Field\nfrom typing import List\n\nWORKSPACE = \"default\"\nclient = NeMoPlatform(workspace=WORKSPACE)\n\nMODEL_PROVIDER = f\"{WORKSPACE}/nvidia-build\"\nMODEL_ID = \"nvidia/nemotron-3-nano-30b-a3b\"\nMODEL_ALIAS = \"nemotron\"\nmodel_configs = [\n dd.ModelConfig(\n provider=MODEL_PROVIDER,\n model=MODEL_ID,\n alias=MODEL_ALIAS,\n inference_parameters=dd.ChatCompletionInferenceParams(\n temperature=1.0,\n top_p=1.0,\n max_tokens=2048,\n ),\n )\n]", + "language": "python", + "source_html": "import os\n\nfrom datasets import load_dataset as hf_load_dataset\nfrom nemo_platform import NeMoPlatform, ConflictError\nimport data_designer.config as dd\nfrom pydantic import BaseModel, Field\nfrom typing import List\n\nWORKSPACE = "default"\nclient = NeMoPlatform(workspace=WORKSPACE)\n\nMODEL_PROVIDER = f"{WORKSPACE}/nvidia-build"\nMODEL_ID = "nvidia/nemotron-3-nano-30b-a3b"\nMODEL_ALIAS = "nemotron"\nmodel_configs = [\n dd.ModelConfig(\n provider=MODEL_PROVIDER,\n model=MODEL_ID,\n alias=MODEL_ALIAS,\n inference_parameters=dd.ChatCompletionInferenceParams(\n temperature=1.0,\n top_p=1.0,\n max_tokens=2048,\n ),\n )\n]\n" + }, + { + "type": "markdown", + "source": "## 2. Generate synthetic data\n\n### Define tool calling schemas\n\n- We use Pydantic models to define the exact structure of our tool definitions and tool calls.\n- Data Designer enforces these schemas during generation.", + "source_html": "

2. Generate synthetic data

\n

Define tool calling schemas

\n
    \n
  • We use Pydantic models to define the exact structure of our tool definitions and tool calls.
    • \n
  • Data Designer enforces these schemas during generation.
    • \n
\n" + }, + { + "type": "code", + "source": "class ParameterDef(BaseModel):\n \"\"\"A single parameter in a function definition.\"\"\"\n name: str = Field(description=\"Parameter name\")\n type: str = Field(description=\"JSON schema type (string, integer, number, boolean, array, object)\")\n description: str = Field(description=\"What this parameter does\")\n\n\nclass FunctionDef(BaseModel):\n \"\"\"A tool/function definition.\"\"\"\n name: str = Field(description=\"Function name in snake_case\")\n description: str = Field(description=\"What this function does\")\n parameters: List[ParameterDef] = Field(description=\"List of function parameters\")\n\n\nclass ToolDefinitions(BaseModel):\n \"\"\"A set of available tools.\"\"\"\n tools: List[FunctionDef] = Field(description=\"2-4 tool/function definitions\")\n\n \nclass ArgumentDef(BaseModel):\n \"\"\"A single argument name/value pair in a tool call instance.\"\"\"\n name: str = Field(description=\"Parameter name\")\n value: str = Field(description=\"The value to pass, as a JSON-encoded string\")\n\n\nclass ToolCallArg(BaseModel):\n \"\"\"A single tool call with arguments.\"\"\"\n name: str = Field(description=\"Name of the function to call\")\n arguments: List[ArgumentDef] = Field(description=\"Arguments to pass to the function\")\n\nclass ExpectedToolCalls(BaseModel):\n \"\"\"The expected tool calls for a given query. The model we're fine-tuning has a chat template that only supports one tool call per query in its template.\"\"\"\n tool_calls: List[ToolCallArg] = Field(description=\"The tool call to be made.\")", + "language": "python", + "source_html": "class ParameterDef(BaseModel):\n """A single parameter in a function definition."""\n name: str = Field(description="Parameter name")\n type: str = Field(description="JSON schema type (string, integer, number, boolean, array, object)")\n description: str = Field(description="What this parameter does")\n\n\nclass FunctionDef(BaseModel):\n """A tool/function definition."""\n name: str = Field(description="Function name in snake_case")\n description: str = Field(description="What this function does")\n parameters: List[ParameterDef] = Field(description="List of function parameters")\n\n\nclass ToolDefinitions(BaseModel):\n """A set of available tools."""\n tools: List[FunctionDef] = Field(description="2-4 tool/function definitions")\n\n \nclass ArgumentDef(BaseModel):\n """A single argument name/value pair in a tool call instance."""\n name: str = Field(description="Parameter name")\n value: str = Field(description="The value to pass, as a JSON-encoded string")\n\n\nclass ToolCallArg(BaseModel):\n """A single tool call with arguments."""\n name: str = Field(description="Name of the function to call")\n arguments: List[ArgumentDef] = Field(description="Arguments to pass to the function")\n\nclass ExpectedToolCalls(BaseModel):\n """The expected tool calls for a given query. The model we're fine-tuning has a chat template that only supports one tool call per query in its template."""\n tool_calls: List[ToolCallArg] = Field(description="The tool call to be made.")\n" + }, + { + "type": "markdown", + "source": "### Build data columns\n\n- The config builder provides an intuitive interface for defining your dataset schema.", + "source_html": "

Build data columns

\n
    \n
  • The config builder provides an intuitive interface for defining your dataset schema.
    • \n
\n" + }, + { + "type": "code", + "source": "config_builder = dd.DataDesignerConfigBuilder(model_configs=model_configs)\n\nconfig_builder.add_column(\n dd.SamplerColumnConfig(\n name=\"domain\",\n sampler_type=\"category\",\n params={\"values\": [\n \"weather\",\n \"calendar\",\n \"web_search\",\n \"math\",\n \"file_management\",\n \"database\",\n \"email\",\n \"navigation\",\n \"e_commerce\",\n \"social_media\",\n ]},\n )\n)\n\nconfig_builder.add_column(\n dd.LLMStructuredColumnConfig(\n name=\"tools\",\n prompt=(\n \"Generate 2-4 realistic API function definitions for the '{% raw %}{{ domain }}{% endraw %}' domain. \"\n \"Each function should have a descriptive snake_case name, clear description, \"\n \"and well-typed parameters. Make them diverse — include functions with \"\n \"different parameter counts and types.\"\n ),\n output_format=ToolDefinitions,\n model_alias=MODEL_ALIAS,\n )\n)\n\nconfig_builder.add_column(\n dd.LLMTextColumnConfig(\n name=\"user_query\",\n prompt=(\n \"Given these available tools:\\n{% raw %}{{ tools }}{% endraw %}\\n\\n\"\n \"Generate a natural, realistic user query that would require calling \"\n \"one of these tools. The query should be something a real user \"\n \"would type. Vary complexity, but due to the limitations of our \"\n \"downstream model only require single tool calls. \"\n \"Respond with only the query, no other text.\"\n ),\n model_alias=MODEL_ALIAS,\n )\n)\n\nconfig_builder.add_column(\n dd.LLMStructuredColumnConfig(\n name=\"expected_tool_call\",\n prompt=(\n \"Given this user query: '{% raw %}{{ user_query }}{% endraw %}'\\n\"\n \"And these available tools: {% raw %}{{ tools }}{% endraw %}\\n\\n\"\n \"Determine exactly which tool should be called and with what arguments. \"\n \"Be precise with argument values — they should directly address the user's request.\"\n ),\n output_format=ExpectedToolCalls,\n model_alias=MODEL_ALIAS,\n )\n)", + "language": "python", + "source_html": "config_builder = dd.DataDesignerConfigBuilder(model_configs=model_configs)\n\nconfig_builder.add_column(\n dd.SamplerColumnConfig(\n name="domain",\n sampler_type="category",\n params={"values": [\n "weather",\n "calendar",\n "web_search",\n "math",\n "file_management",\n "database",\n "email",\n "navigation",\n "e_commerce",\n "social_media",\n ]},\n )\n)\n\nconfig_builder.add_column(\n dd.LLMStructuredColumnConfig(\n name="tools",\n prompt=(\n "Generate 2-4 realistic API function definitions for the '{% raw %}{{ domain }}{% endraw %}' domain. "\n "Each function should have a descriptive snake_case name, clear description, "\n "and well-typed parameters. Make them diverse — include functions with "\n "different parameter counts and types."\n ),\n output_format=ToolDefinitions,\n model_alias=MODEL_ALIAS,\n )\n)\n\nconfig_builder.add_column(\n dd.LLMTextColumnConfig(\n name="user_query",\n prompt=(\n "Given these available tools:\\n{% raw %}{{ tools }}{% endraw %}\\n\\n"\n "Generate a natural, realistic user query that would require calling "\n "one of these tools. The query should be something a real user "\n "would type. Vary complexity, but due to the limitations of our "\n "downstream model only require single tool calls. "\n "Respond with only the query, no other text."\n ),\n model_alias=MODEL_ALIAS,\n )\n)\n\nconfig_builder.add_column(\n dd.LLMStructuredColumnConfig(\n name="expected_tool_call",\n prompt=(\n "Given this user query: '{% raw %}{{ user_query }}{% endraw %}'\\n"\n "And these available tools: {% raw %}{{ tools }}{% endraw %}\\n\\n"\n "Determine exactly which tool should be called and with what arguments. "\n "Be precise with argument values — they should directly address the user's request."\n ),\n output_format=ExpectedToolCalls,\n model_alias=MODEL_ALIAS,\n )\n)\n" + }, + { + "type": "markdown", + "source": "### Preview the dataset\n\n1. Use the `preview` method to generate a small sample quickly.\n2. Inspect the results for quality and format issues.\n3. Adjust column configurations, prompts, or parameters as needed.\n4. Re-run the preview until satisfied.", + "source_html": "

Preview the dataset

\n
    \n
  1. Use the preview method to generate a small sample quickly.
    2. \n
  2. Inspect the results for quality and format issues.
    3. \n
  3. Adjust column configurations, prompts, or parameters as needed.
    4. \n
  4. Re-run the preview until satisfied.
    5. \n
\n" + }, + { + "type": "code", + "source": "preview = client.data_designer.preview(config_builder, num_records=4)", + "language": "python", + "source_html": "preview = client.data_designer.preview(config_builder, num_records=4)\n" + }, + { + "type": "markdown", + "source": "You can view one record at a time to inspect quality:", + "source_html": "

You can view one record at a time to inspect quality:

\n" + }, + { + "type": "code", + "source": "preview.display_sample_record()", + "language": "python", + "source_html": "preview.display_sample_record()\n" + }, + { + "type": "markdown", + "source": "Or view the entire preview dataset:", + "source_html": "

Or view the entire preview dataset:

\n" + }, + { + "type": "code", + "source": "preview.dataset", + "language": "python", + "source_html": "preview.dataset\n" + }, + { + "type": "markdown", + "source": "### Scale up\n\nThe preview functionality lets you iterate quickly on your data to ensure the right structure and quality. Once ready, the `create` method is used to generate the full training dataset.\n\nFor this tutorial we generate **60 samples** (~5 minutes) to keep iteration fast. For production use, generate **500+ samples** (~1 hour) for significantly better results — in our testing, 500 samples yielded **~93% function name accuracy**, compared to ~1.4% for the un-fine-tuned base model.", + "source_html": "

Scale up

\n

The preview functionality lets you iterate quickly on your data to ensure the right structure and quality. Once ready, the create method is used to generate the full training dataset.

\n

For this tutorial we generate 60 samples (~5 minutes) to keep iteration fast. For production use, generate 500+ samples (~1 hour) for significantly better results — in our testing, 500 samples yielded ~93% function name accuracy, compared to ~1.4% for the un-fine-tuned base model.

\n" + }, + { + "type": "code", + "source": "job = client.data_designer.create(config_builder, num_records=60)\njob.wait_until_done()", + "language": "python", + "source_html": "job = client.data_designer.create(config_builder, num_records=60)\njob.wait_until_done()\n" + }, + { + "type": "markdown", + "source": "Fetch the dataset from the job outputs and check out a preview.", + "source_html": "

Fetch the dataset from the job outputs and check out a preview.

\n" + }, + { + "type": "code", + "source": "results = job.download_artifacts()\ndataset = results.load_dataset()\ndataset.head()", + "language": "python", + "source_html": "results = job.download_artifacts()\ndataset = results.load_dataset()\ndataset.head()\n" + }, + { + "type": "markdown", + "source": "### Convert Dataset\n\n- Fine-tuning requires the [OpenAI messages+tools format](https://platform.openai.com/docs/guides/function-calling) with specific nesting.\n- Data Designer generated flat structures — now we wrap them in the OpenAI format.\n- Key transformations: wrap tools/calls in `{\"type\": \"function\", \"function\": {...}}`, convert parameter lists to properties dicts, add `\"content\": \"\"` to the assistant message.", + "source_html": "

Convert Dataset

\n
    \n
  • Fine-tuning requires the OpenAI messages+tools format with specific nesting.
    • \n
  • Data Designer generated flat structures — now we wrap them in the OpenAI format.
    • \n
  • Key transformations: wrap tools/calls in {"type": "function", "function": {...}}, convert parameter lists to properties dicts, add "content": "" to the assistant message.
    • \n
\n" + }, + { + "type": "code", + "source": "import json\n\ndef to_openai_format(row):\n \"\"\"Convert a DD-generated row to the OpenAI tool calling format expected by the Customizer.\"\"\"\n openai_tools = []\n for tool_def in row[\"tools\"][\"tools\"]:\n properties = {\n p[\"name\"]: {\"type\": p[\"type\"], \"description\": p[\"description\"]}\n for p in tool_def[\"parameters\"]\n }\n openai_tools.append({\n \"type\": \"function\",\n \"function\": {\n \"name\": tool_def[\"name\"],\n \"description\": tool_def[\"description\"],\n \"parameters\": {\"type\": \"object\", \"properties\": properties},\n },\n })\n\n openai_tool_calls = []\n for tc in row[\"expected_tool_call\"][\"tool_calls\"]:\n arguments = {}\n for arg in tc[\"arguments\"]:\n try:\n arguments[arg[\"name\"]] = json.loads(arg[\"value\"])\n except:\n arguments[arg[\"name\"]] = arg[\"value\"]\n openai_tool_calls.append({\n \"type\": \"function\",\n \"function\": {\n \"name\": tc[\"name\"],\n \"arguments\": arguments,\n },\n })\n\n return {\n \"messages\": [\n {\"role\": \"user\", \"content\": row[\"user_query\"]},\n {\"role\": \"assistant\", \"content\": \"\", \"tool_calls\": openai_tool_calls},\n ],\n \"tools\": openai_tools,\n }\n\n\ntraining_data = dataset.apply(to_openai_format, axis=1, result_type=\"expand\")\n\n# Llama 3.2 1B's chat template supports only single tool calls per turn — filter out any\n# generated samples that contain multiple tool calls in the assistant response.\npre_filter = len(training_data)\ntraining_data = training_data[\n training_data[\"messages\"].apply(lambda msgs: len(msgs[1].get(\"tool_calls\", [])) == 1)\n].reset_index(drop=True)\nprint(f\"Filtered {pre_filter} -> {len(training_data)} rows (kept single tool-call only)\")\n\ntraining_data.head()", + "language": "python", + "source_html": "import json\n\ndef to_openai_format(row):\n """Convert a DD-generated row to the OpenAI tool calling format expected by the Customizer."""\n openai_tools = []\n for tool_def in row["tools"]["tools"]:\n properties = {\n p["name"]: {"type": p["type"], "description": p["description"]}\n for p in tool_def["parameters"]\n }\n openai_tools.append({\n "type": "function",\n "function": {\n "name": tool_def["name"],\n "description": tool_def["description"],\n "parameters": {"type": "object", "properties": properties},\n },\n })\n\n openai_tool_calls = []\n for tc in row["expected_tool_call"]["tool_calls"]:\n arguments = {}\n for arg in tc["arguments"]:\n try:\n arguments[arg["name"]] = json.loads(arg["value"])\n except:\n arguments[arg["name"]] = arg["value"]\n openai_tool_calls.append({\n "type": "function",\n "function": {\n "name": tc["name"],\n "arguments": arguments,\n },\n })\n\n return {\n "messages": [\n {"role": "user", "content": row["user_query"]},\n {"role": "assistant", "content": "", "tool_calls": openai_tool_calls},\n ],\n "tools": openai_tools,\n }\n\n\ntraining_data = dataset.apply(to_openai_format, axis=1, result_type="expand")\n\n# Llama 3.2 1B's chat template supports only single tool calls per turn — filter out any\n# generated samples that contain multiple tool calls in the assistant response.\npre_filter = len(training_data)\ntraining_data = training_data[\n training_data["messages"].apply(lambda msgs: len(msgs[1].get("tool_calls", [])) == 1)\n].reset_index(drop=True)\nprint(f"Filtered {pre_filter} -> {len(training_data)} rows (kept single tool-call only)")\n\ntraining_data.head()\n" + }, + { + "type": "markdown", + "source": "### Sample and Split into Train / Validation\n\nWe use all available synthetic examples and split them 85/15 into training and validation sets. Evaluation uses a separate golden dataset (xLAM) prepared in the next section.", + "source_html": "

Sample and Split into Train / Validation

\n

We use all available synthetic examples and split them 85/15 into training and validation sets. Evaluation uses a separate golden dataset (xLAM) prepared in the next section.

\n" + }, + { + "type": "code", + "source": "import random\nfrom pathlib import Path\n\nSEED = 1234\nrandom.seed(SEED)\n\nSAMPLE_SIZE = 60\ntraining_records = training_data.to_dict(\"records\")\nsampled = random.sample(training_records, min(SAMPLE_SIZE, len(training_records)))\n\ntrain_size = max(1, int(0.85 * len(sampled)))\nval_size = max(1, len(sampled) - train_size)\n\ntrain_data = sampled[:train_size]\nval_data = sampled[train_size:]\n\nprint(f\"Training: {len(train_data)} examples\")\nprint(f\"Validation: {len(val_data)} examples\")\n\n# Save training and validation data\nDATASET_PATH = Path(\"tool-calling-dataset\").absolute()\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n\ndef save_jsonl(filepath: str, data: list) -> None:\n with open(filepath, \"w\") as f:\n for entry in data:\n f.write(json.dumps(entry) + \"\\n\")\n\n\nsave_jsonl(f\"{DATASET_PATH}/training.jsonl\", train_data)\nprint(f\"\\nSaved training.jsonl ({len(train_data)} rows)\")\n\nsave_jsonl(f\"{DATASET_PATH}/validation.jsonl\", val_data)\nprint(f\"Saved validation.jsonl ({len(val_data)} rows)\")", + "language": "python", + "source_html": "import random\nfrom pathlib import Path\n\nSEED = 1234\nrandom.seed(SEED)\n\nSAMPLE_SIZE = 60\ntraining_records = training_data.to_dict("records")\nsampled = random.sample(training_records, min(SAMPLE_SIZE, len(training_records)))\n\ntrain_size = max(1, int(0.85 * len(sampled)))\nval_size = max(1, len(sampled) - train_size)\n\ntrain_data = sampled[:train_size]\nval_data = sampled[train_size:]\n\nprint(f"Training: {len(train_data)} examples")\nprint(f"Validation: {len(val_data)} examples")\n\n# Save training and validation data\nDATASET_PATH = Path("tool-calling-dataset").absolute()\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n\ndef save_jsonl(filepath: str, data: list) -> None:\n with open(filepath, "w") as f:\n for entry in data:\n f.write(json.dumps(entry) + "\\n")\n\n\nsave_jsonl(f"{DATASET_PATH}/training.jsonl", train_data)\nprint(f"\\nSaved training.jsonl ({len(train_data)} rows)")\n\nsave_jsonl(f"{DATASET_PATH}/validation.jsonl", val_data)\nprint(f"Saved validation.jsonl ({len(val_data)} rows)")\n" + }, + { + "type": "markdown", + "source": "### Prepare Evaluation Dataset from xLAM\n\nRather than evaluating on the same synthetic data we trained on, we use the [Salesforce/xlam-function-calling-60k](https://huggingface.co/datasets/Salesforce/xlam-function-calling-60k) golden dataset as an independent test set. This gives a more meaningful measure of how well fine-tuning generalizes.\n\nWe download the dataset, convert it to OpenAI format, filter for single tool-call examples (matching our model's capability), and sample a test set sized at 15% of our training data.", + "source_html": "

Prepare Evaluation Dataset from xLAM

\n

Rather than evaluating on the same synthetic data we trained on, we use the Salesforce/xlam-function-calling-60k golden dataset as an independent test set. This gives a more meaningful measure of how well fine-tuning generalizes.

\n

We download the dataset, convert it to OpenAI format, filter for single tool-call examples (matching our model's capability), and sample a test set sized at 15% of our training data.

\n" + }, + { + "type": "code", + "source": "LIMIT_TOOL_PROPERTIES = 8\n\n\ndef normalize_type(param_type: str) -> str:\n \"\"\"Normalize Python type hints to JSON schema types.\"\"\"\n param_type = param_type.strip()\n if \",\" in param_type and \"default\" in param_type:\n param_type = param_type.split(\",\")[0].strip()\n if param_type.startswith(\"default=\"):\n return \"string\"\n param_type = param_type.replace(\", optional\", \"\").strip()\n if param_type.startswith(\"Callable\"):\n return \"string\"\n if param_type.startswith((\"Tuple\", \"List[\", \"Set\")) or param_type in (\"list\", \"set\"):\n return \"array\"\n type_mapping = {\n \"str\": \"string\", \"int\": \"integer\", \"float\": \"number\",\n \"bool\": \"boolean\", \"list\": \"array\", \"dict\": \"object\",\n \"List\": \"array\", \"Dict\": \"object\", \"Set\": \"array\",\n }\n return type_mapping.get(param_type, \"string\")\n\n\ndef convert_xlam_tools(tools):\n \"\"\"Convert xLAM tool definitions to OpenAI function spec format.\"\"\"\n if isinstance(tools, str):\n tools = json.loads(tools)\n if not isinstance(tools, list):\n return []\n openai_tools = []\n for tool in tools:\n if not isinstance(tool, dict) or not isinstance(tool.get(\"parameters\"), dict):\n continue\n properties = {}\n for name, info in tool[\"parameters\"].items():\n if not isinstance(info, dict):\n continue\n param = {\n \"description\": info.get(\"description\", \"\"),\n \"type\": normalize_type(info.get(\"type\", \"\")),\n }\n properties[name] = param\n if len(properties) > LIMIT_TOOL_PROPERTIES:\n return []\n openai_tools.append({\n \"type\": \"function\",\n \"function\": {\n \"name\": tool[\"name\"],\n \"description\": tool[\"description\"],\n \"parameters\": {\"type\": \"object\", \"properties\": properties},\n },\n })\n return openai_tools\n\n\ndef convert_xlam_example(example):\n \"\"\"Convert a single xLAM example to OpenAI eval format.\"\"\"\n tools = convert_xlam_tools(example.get(\"tools\", \"[]\"))\n if not tools:\n return None\n answers = json.loads(example.get(\"answers\", \"[]\"))\n tool_calls = [\n {\"type\": \"function\", \"function\": {\"name\": t[\"name\"], \"arguments\": t.get(\"arguments\", {})}}\n for t in answers\n ]\n if len(tool_calls) != 1:\n return None\n return {\n \"messages\": [{\"role\": \"user\", \"content\": example[\"query\"]}],\n \"tools\": tools,\n \"tool_calls\": tool_calls,\n }\n\n\nxlam_dataset = hf_load_dataset(\"Salesforce/xlam-function-calling-60k\", split=\"train\")\nprint(f\"xLAM dataset size: {len(xlam_dataset)}\")\n\nxlam_converted = [r for ex in xlam_dataset if (r := convert_xlam_example(ex)) is not None]\nprint(f\"After filtering (single tool-call, ≤{LIMIT_TOOL_PROPERTIES} params): {len(xlam_converted)}\")\n\neval_size = max(1, int(0.15 * len(train_data)))\neval_rows = random.sample(xlam_converted, min(eval_size, len(xlam_converted)))\nprint(f\"Sampled {len(eval_rows)} eval examples (15% of {len(train_data)} training examples)\")\n\nEVAL_DATASET_NAME = \"tool-calling-eval-dataset\"\nEVAL_DATASET_PATH = Path(\"tool-calling-eval-dataset\").absolute()\nos.makedirs(EVAL_DATASET_PATH, exist_ok=True)\nsave_jsonl(f\"{EVAL_DATASET_PATH}/evaluation.jsonl\", eval_rows)\n\nprint(f\"\\nSaved to {EVAL_DATASET_PATH}/evaluation.jsonl\")\nprint(\"\\nSample evaluation row:\")\nprint(json.dumps(eval_rows[0], indent=2))", + "language": "python", + "source_html": "LIMIT_TOOL_PROPERTIES = 8\n\n\ndef normalize_type(param_type: str) -> str:\n """Normalize Python type hints to JSON schema types."""\n param_type = param_type.strip()\n if "," in param_type and "default" in param_type:\n param_type = param_type.split(",")[0].strip()\n if param_type.startswith("default="):\n return "string"\n param_type = param_type.replace(", optional", "").strip()\n if param_type.startswith("Callable"):\n return "string"\n if param_type.startswith(("Tuple", "List[", "Set")) or param_type in ("list", "set"):\n return "array"\n type_mapping = {\n "str": "string", "int": "integer", "float": "number",\n "bool": "boolean", "list": "array", "dict": "object",\n "List": "array", "Dict": "object", "Set": "array",\n }\n return type_mapping.get(param_type, "string")\n\n\ndef convert_xlam_tools(tools):\n """Convert xLAM tool definitions to OpenAI function spec format."""\n if isinstance(tools, str):\n tools = json.loads(tools)\n if not isinstance(tools, list):\n return []\n openai_tools = []\n for tool in tools:\n if not isinstance(tool, dict) or not isinstance(tool.get("parameters"), dict):\n continue\n properties = {}\n for name, info in tool["parameters"].items():\n if not isinstance(info, dict):\n continue\n param = {\n "description": info.get("description", ""),\n "type": normalize_type(info.get("type", "")),\n }\n properties[name] = param\n if len(properties) > LIMIT_TOOL_PROPERTIES:\n return []\n openai_tools.append({\n "type": "function",\n "function": {\n "name": tool["name"],\n "description": tool["description"],\n "parameters": {"type": "object", "properties": properties},\n },\n })\n return openai_tools\n\n\ndef convert_xlam_example(example):\n """Convert a single xLAM example to OpenAI eval format."""\n tools = convert_xlam_tools(example.get("tools", "[]"))\n if not tools:\n return None\n answers = json.loads(example.get("answers", "[]"))\n tool_calls = [\n {"type": "function", "function": {"name": t["name"], "arguments": t.get("arguments", {})}}\n for t in answers\n ]\n if len(tool_calls) != 1:\n return None\n return {\n "messages": [{"role": "user", "content": example["query"]}],\n "tools": tools,\n "tool_calls": tool_calls,\n }\n\n\nxlam_dataset = hf_load_dataset("Salesforce/xlam-function-calling-60k", split="train")\nprint(f"xLAM dataset size: {len(xlam_dataset)}")\n\nxlam_converted = [r for ex in xlam_dataset if (r := convert_xlam_example(ex)) is not None]\nprint(f"After filtering (single tool-call, ≤{LIMIT_TOOL_PROPERTIES} params): {len(xlam_converted)}")\n\neval_size = max(1, int(0.15 * len(train_data)))\neval_rows = random.sample(xlam_converted, min(eval_size, len(xlam_converted)))\nprint(f"Sampled {len(eval_rows)} eval examples (15% of {len(train_data)} training examples)")\n\nEVAL_DATASET_NAME = "tool-calling-eval-dataset"\nEVAL_DATASET_PATH = Path("tool-calling-eval-dataset").absolute()\nos.makedirs(EVAL_DATASET_PATH, exist_ok=True)\nsave_jsonl(f"{EVAL_DATASET_PATH}/evaluation.jsonl", eval_rows)\n\nprint(f"\\nSaved to {EVAL_DATASET_PATH}/evaluation.jsonl")\nprint("\\nSample evaluation row:")\nprint(json.dumps(eval_rows[0], indent=2))\n" + }, + { + "type": "markdown", + "source": "### Upload Training Dataset\n\nCreate a fileset and upload the training/validation data to the platform:", + "source_html": "

Upload Training Dataset

\n

Create a fileset and upload the training/validation data to the platform:

\n" + }, + { + "type": "code", + "source": "DATASET_NAME = \"tool-calling-dataset\"\n\ntry:\n client.files.filesets.create(\n name=DATASET_NAME,\n description=\"synthetic tool-calling training and validation data in OpenAI chat format\",\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\nclient.files.upload(\n local_path=DATASET_PATH,\n remote_path=\"\",\n fileset=DATASET_NAME,\n)\n\nprint(\"\\nTraining data files:\")\nprint(json.dumps(\n [f.model_dump() for f in client.files.list(fileset=DATASET_NAME).data],\n indent=2,\n))", + "language": "python", + "source_html": "DATASET_NAME = "tool-calling-dataset"\n\ntry:\n client.files.filesets.create(\n name=DATASET_NAME,\n description="synthetic tool-calling training and validation data in OpenAI chat format",\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\nclient.files.upload(\n local_path=DATASET_PATH,\n remote_path="",\n fileset=DATASET_NAME,\n)\n\nprint("\\nTraining data files:")\nprint(json.dumps(\n [f.model_dump() for f in client.files.list(fileset=DATASET_NAME).data],\n indent=2,\n))\n" + }, + { + "type": "markdown", + "source": "### Upload Evaluation Dataset\n\nUpload the evaluation data as a separate fileset:", + "source_html": "

Upload Evaluation Dataset

\n

Upload the evaluation data as a separate fileset:

\n" + }, + { + "type": "code", + "source": "try:\n client.files.filesets.create(\n name=EVAL_DATASET_NAME,\n description=\"synthetic tool-calling evaluation data (messages, tools, ground-truth tool_calls)\",\n )\n print(f\"Created fileset: {EVAL_DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{EVAL_DATASET_NAME}' already exists, continuing...\")\n\nclient.files.upload(\n local_path=EVAL_DATASET_PATH,\n remote_path=\"\",\n fileset=EVAL_DATASET_NAME,\n)\n\nprint(\"\\nEvaluation data files:\")\nprint(json.dumps(\n [f.model_dump() for f in client.files.list(fileset=EVAL_DATASET_NAME).data],\n indent=2,\n))", + "language": "python", + "source_html": "try:\n client.files.filesets.create(\n name=EVAL_DATASET_NAME,\n description="synthetic tool-calling evaluation data (messages, tools, ground-truth tool_calls)",\n )\n print(f"Created fileset: {EVAL_DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{EVAL_DATASET_NAME}' already exists, continuing...")\n\nclient.files.upload(\n local_path=EVAL_DATASET_PATH,\n remote_path="",\n fileset=EVAL_DATASET_NAME,\n)\n\nprint("\\nEvaluation data files:")\nprint(json.dumps(\n [f.model_dump() for f in client.files.list(fileset=EVAL_DATASET_NAME).data],\n indent=2,\n))\n" + }, + { + "type": "markdown", + "source": "## 3. Model and Secrets Setup\n\n### Configure HuggingFace Authentication\n\nWe will fine-tune [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) from HuggingFace. Since this is a gated model, you need:\n\n1. A HuggingFace token with Read access from [HuggingFace Settings](https://huggingface.co/settings/tokens)\n2. Accept the model's license on the [model page](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct)", + "source_html": "

3. Model and Secrets Setup

\n

Configure HuggingFace Authentication

\n

We will fine-tune meta-llama/Llama-3.2-1B-Instruct from HuggingFace. Since this is a gated model, you need:

\n
    \n
  1. A HuggingFace token with Read access from HuggingFace Settings
    2. \n
  2. Accept the model's license on the model page
    3. \n
\n" + }, + { + "type": "code", + "source": "HF_TOKEN = os.environ.get(\"HF_TOKEN\")\nif HF_TOKEN is None:\n raise ValueError(\"HF_TOKEN is not set.\")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f\"{label} is not set\")\n try:\n secret = client.secrets.create(\n name=name,\n value=value,\n )\n print(f\"Created secret: {name}\")\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n return client.secrets.retrieve(name=name)\n\n\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")\nprint(hf_secret.model_dump_json(indent=2))", + "language": "python", + "source_html": "HF_TOKEN = os.environ.get("HF_TOKEN")\nif HF_TOKEN is None:\n raise ValueError("HF_TOKEN is not set.")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f"{label} is not set")\n try:\n secret = client.secrets.create(\n name=name,\n value=value,\n )\n print(f"Created secret: {name}")\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n return client.secrets.retrieve(name=name)\n\n\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\nprint(hf_secret.model_dump_json(indent=2))\n" + }, + { + "type": "markdown", + "source": "### Create Base Model FileSet and Model Entity\n\nCreate a fileset pointing to Llama 3.2 1B Instruct on HuggingFace, then create a Model Entity that references it. The model will be downloaded at training time.\n\nFor tool calling, set `tool_call_config` in the fileset's `metadata`. The NIM serving layer needs to know *how* to parse tool calls from the model's raw output, but that information isn't embedded in the model checkpoint itself -- it depends on the model family and chat template. By declaring it on the fileset, the platform's model-spec task merges it into the Model Entity's `spec.tool_call_config`, which the deployment backend then reads to configure the NIM automatically:\n- `tool_call_parser` -- tells the NIM how to parse tool calls from the model output (e.g., `\"llama3_json\"` for Llama 3.x models)\n- `auto_tool_choice` -- enables the model to decide when to call tools", + "source_html": "

Create Base Model FileSet and Model Entity

\n

Create a fileset pointing to Llama 3.2 1B Instruct on HuggingFace, then create a Model Entity that references it. The model will be downloaded at training time.

\n

For tool calling, set tool_call_config in the fileset's metadata. The NIM serving layer needs to know how to parse tool calls from the model's raw output, but that information isn't embedded in the model checkpoint itself -- it depends on the model family and chat template. By declaring it on the fileset, the platform's model-spec task merges it into the Model Entity's spec.tool_call_config, which the deployment backend then reads to configure the NIM automatically:

\n
    \n
  • tool_call_parser -- tells the NIM how to parse tool calls from the model output (e.g., "llama3_json" for Llama 3.x models)
    • \n
  • auto_tool_choice -- enables the model to decide when to call tools
    • \n
\n" + }, + { + "type": "code", + "source": "import time\n\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = \"meta-llama/Llama-3.2-1B-Instruct\"\nMODEL_NAME = \"llama-3-2-1b-base\"\n\ntry:\n base_model_fs = client.files.filesets.create(\n name=MODEL_NAME,\n description=\"Llama 3.2 1B Instruct base model from HuggingFace\",\n storage=HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n repo_id=HF_REPO_ID,\n repo_type=\"model\",\n token_secret=hf_secret.name,\n ),\n metadata={\n \"model\": {\n \"tool_calling\": {\n \"tool_call_parser\": \"llama3_json\",\n \"auto_tool_choice\": True,\n }\n }\n },\n )\n print(f\"Created base model fileset: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model fileset already exists. Skipping creation.\")\n base_model_fs = client.files.filesets.retrieve(\n name=MODEL_NAME,\n )\n\ntry:\n base_model = client.models.create(\n name=MODEL_NAME,\n fileset=f\"{WORKSPACE}/{MODEL_NAME}\",\n )\n print(f\"Created Model Entity: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model already exists. Updating fileset if different.\")\n base_model = client.models.update(\n name=MODEL_NAME,\n fileset=f\"{WORKSPACE}/{MODEL_NAME}\",\n )\n\nprint(f\"\\nBase model fileset: fileset://{WORKSPACE}/{base_model.name}\")\nprint(\"Base model fileset files list:\")\nprint(json.dumps(\n [f.model_dump() for f in client.files.list(fileset=MODEL_NAME).data],\n indent=2,\n))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint(\"\\nWaiting for ModelSpec to be populated...\")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f\"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds\")\n time.sleep(2)\n base_model = client.models.retrieve(\n name=MODEL_NAME,\n )\n\nprint(f\"ModelSpec populated: {base_model.spec.model_dump()}\")", + "language": "python", + "source_html": "import time\n\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = "meta-llama/Llama-3.2-1B-Instruct"\nMODEL_NAME = "llama-3-2-1b-base"\n\ntry:\n base_model_fs = client.files.filesets.create(\n name=MODEL_NAME,\n description="Llama 3.2 1B Instruct base model from HuggingFace",\n storage=HuggingfaceStorageConfigParam(\n type="huggingface",\n repo_id=HF_REPO_ID,\n repo_type="model",\n token_secret=hf_secret.name,\n ),\n metadata={\n "model": {\n "tool_calling": {\n "tool_call_parser": "llama3_json",\n "auto_tool_choice": True,\n }\n }\n },\n )\n print(f"Created base model fileset: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model fileset already exists. Skipping creation.")\n base_model_fs = client.files.filesets.retrieve(\n name=MODEL_NAME,\n )\n\ntry:\n base_model = client.models.create(\n name=MODEL_NAME,\n fileset=f"{WORKSPACE}/{MODEL_NAME}",\n )\n print(f"Created Model Entity: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model already exists. Updating fileset if different.")\n base_model = client.models.update(\n name=MODEL_NAME,\n fileset=f"{WORKSPACE}/{MODEL_NAME}",\n )\n\nprint(f"\\nBase model fileset: fileset://{WORKSPACE}/{base_model.name}")\nprint("Base model fileset files list:")\nprint(json.dumps(\n [f.model_dump() for f in client.files.list(fileset=MODEL_NAME).data],\n indent=2,\n))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint("\\nWaiting for ModelSpec to be populated...")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds")\n time.sleep(2)\n base_model = client.models.retrieve(\n name=MODEL_NAME,\n )\n\nprint(f"ModelSpec populated: {base_model.spec.model_dump()}")\n" + }, + { + "type": "markdown", + "source": "## 4. Create LoRA Fine-Tuning Job\n\nCreate a customization job using SFT training with LoRA PEFT. The `peft=LoRaParamsParam()` parameter enables LoRA instead of full-weight fine-tuning.\n\n**LoRA defaults** (can be overridden in `LoRaParamsParam()`):\n- `rank`: LoRA rank (dimensionality of the low-rank matrices)\n- `alpha`: Scaling factor for LoRA updates\n- `target_modules`: Which model layers to apply LoRA to", + "source_html": "

4. Create LoRA Fine-Tuning Job

\n

Create a customization job using SFT training with LoRA PEFT. The peft=LoRaParamsParam() parameter enables LoRA instead of full-weight fine-tuning.

\n

LoRA defaults (can be overridden in LoRaParamsParam()):

\n
    \n
  • rank: LoRA rank (dimensionality of the low-rank matrices)
    • \n
  • alpha: Scaling factor for LoRA updates
    • \n
  • target_modules: Which model layers to apply LoRA to
    • \n
\n" + }, + { + "type": "code", + "source": "import uuid\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam,\n LoRaParamsParam,\n DeploymentParamsParam,\n)\n\njob_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f\"tool-calling-lora-{job_suffix}\"\n\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n spec=CustomizationJobInputParam(\n model=f\"{WORKSPACE}/{base_model.name}\",\n dataset=f\"fileset://{WORKSPACE}/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=4,\n batch_size=4,\n learning_rate=0.0001,\n max_seq_length=2048,\n micro_batch_size=1,\n peft=LoRaParamsParam(),\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n deployment_config=DeploymentParamsParam(\n lora_enabled=True,\n ),\n ),\n)\n\nprint(job.model_dump_json(indent=2))", + "language": "python", + "source_html": "import uuid\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam,\n LoRaParamsParam,\n DeploymentParamsParam,\n)\n\njob_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f"tool-calling-lora-{job_suffix}"\n\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n spec=CustomizationJobInputParam(\n model=f"{WORKSPACE}/{base_model.name}",\n dataset=f"fileset://{WORKSPACE}/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=4,\n batch_size=4,\n learning_rate=0.0001,\n max_seq_length=2048,\n micro_batch_size=1,\n peft=LoRaParamsParam(),\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n deployment_config=DeploymentParamsParam(\n lora_enabled=True,\n ),\n ),\n)\n\nprint(job.model_dump_json(indent=2))\n" + }, + { + "type": "markdown", + "source": "### Track Training Progress", + "source_html": "

Track Training Progress

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\nTERMINAL_JOB_STATUSES = {\"completed\", \"cancelled\", \"error\"}\n\n\ndef wait_for_job(poll_fn, label, timeout_minutes=60, poll_interval=10, display_fn=None):\n \"\"\"Poll a platform job until it reaches a terminal state.\n\n Both Customizer and Evaluator use the Core Jobs service, so the same\n PlatformJobStatus values apply: created, pending, active, cancelled,\n cancelling, error, completed, paused, pausing, resuming.\n\n Args:\n poll_fn: Callable returning a status object with .name and .status attributes.\n label: Display label for progress output.\n timeout_minutes: Maximum time to wait before returning.\n poll_interval: Seconds between polls.\n display_fn: Optional callable(status) to print extra details after the header.\n\n Returns:\n The final status object.\n \"\"\"\n start = time.time()\n timeout = timeout_minutes * 60\n\n while True:\n status = poll_fn()\n elapsed = time.time() - start\n elapsed_min, elapsed_sec = divmod(int(elapsed), 60)\n\n clear_output(wait=True)\n print(f\"[{label}] Job: {status.name}\")\n print(f\"[{label}] Status: {status.status}\")\n print(f\"[{label}] Elapsed: {elapsed_min}m {elapsed_sec}s\")\n\n if display_fn:\n display_fn(status)\n\n if status.status in TERMINAL_JOB_STATUSES:\n print(f\"\\n[{label}] Job finished: {status.status}\")\n return status\n\n if elapsed > timeout:\n print(f\"\\n[{label}] Timeout after {timeout_minutes} minutes\")\n return status\n\n time.sleep(poll_interval)\n\n\ndef training_progress(status):\n \"\"\"Extract and display training step progress.\"\"\"\n for step in status.steps or []:\n if step.name == \"customization-training-job\":\n for task in step.tasks or []:\n details = task.status_details or {}\n s, mx = details.get(\"step\"), details.get(\"max_steps\")\n if s is not None and mx is not None:\n print(f\"Training: Step {s}/{mx} ({(s / mx) * 100:.1f}%)\")\n if phase := details.get(\"phase\"):\n print(f\"Phase: {phase}\")\n return\n print(\"Training step not started yet\")\n\nprint(\"Defined wait_for_job helper function\")", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\nTERMINAL_JOB_STATUSES = {"completed", "cancelled", "error"}\n\n\ndef wait_for_job(poll_fn, label, timeout_minutes=60, poll_interval=10, display_fn=None):\n """Poll a platform job until it reaches a terminal state.\n\n Both Customizer and Evaluator use the Core Jobs service, so the same\n PlatformJobStatus values apply: created, pending, active, cancelled,\n cancelling, error, completed, paused, pausing, resuming.\n\n Args:\n poll_fn: Callable returning a status object with .name and .status attributes.\n label: Display label for progress output.\n timeout_minutes: Maximum time to wait before returning.\n poll_interval: Seconds between polls.\n display_fn: Optional callable(status) to print extra details after the header.\n\n Returns:\n The final status object.\n """\n start = time.time()\n timeout = timeout_minutes * 60\n\n while True:\n status = poll_fn()\n elapsed = time.time() - start\n elapsed_min, elapsed_sec = divmod(int(elapsed), 60)\n\n clear_output(wait=True)\n print(f"[{label}] Job: {status.name}")\n print(f"[{label}] Status: {status.status}")\n print(f"[{label}] Elapsed: {elapsed_min}m {elapsed_sec}s")\n\n if display_fn:\n display_fn(status)\n\n if status.status in TERMINAL_JOB_STATUSES:\n print(f"\\n[{label}] Job finished: {status.status}")\n return status\n\n if elapsed > timeout:\n print(f"\\n[{label}] Timeout after {timeout_minutes} minutes")\n return status\n\n time.sleep(poll_interval)\n\n\ndef training_progress(status):\n """Extract and display training step progress."""\n for step in status.steps or []:\n if step.name == "customization-training-job":\n for task in step.tasks or []:\n details = task.status_details or {}\n s, mx = details.get("step"), details.get("max_steps")\n if s is not None and mx is not None:\n print(f"Training: Step {s}/{mx} ({(s / mx) * 100:.1f}%)")\n if phase := details.get("phase"):\n print(f"Phase: {phase}")\n return\n print("Training step not started yet")\n\nprint("Defined wait_for_job helper function")\n" + }, + { + "type": "code", + "source": "job_status = wait_for_job(\n poll_fn=lambda: client.customization.jobs.get_status(name=job.name),\n label=\"Training\",\n timeout_minutes=120,\n display_fn=training_progress,\n)", + "language": "python", + "source_html": "job_status = wait_for_job(\n poll_fn=lambda: client.customization.jobs.get_status(name=job.name),\n label="Training",\n timeout_minutes=120,\n display_fn=training_progress,\n)\n" + }, + { + "type": "markdown", + "source": "## 5. Verify Auto-Deployed Model\n\nSince we set `lora_enabled=True` in the customization job's `deployment_config`, the platform automatically creates a NIM deployment for the base model after training completes. The LoRA adapter is attached to the base model entity (enabled by default) and the deployment serves both the base weights and the adapter through a single NIM instance.", + "source_html": "

5. Verify Auto-Deployed Model

\n

Since we set lora_enabled=True in the customization job's deployment_config, the platform automatically creates a NIM deployment for the base model after training completes. The LoRA adapter is attached to the base model entity (enabled by default) and the deployment serves both the base weights and the adapter through a single NIM instance.

\n" + }, + { + "type": "code", + "source": "ADAPTER_NAME = job.spec.output.name\nprint(f\"Looking for adapter: {ADAPTER_NAME}\")\n\n# The adapter may not be attached to the model entity immediately after\n# training completes — poll until it appears.\nADAPTER_TIMEOUT = 120\nadapter_start = time.time()\nadapter = None\nwhile time.time() - adapter_start < ADAPTER_TIMEOUT:\n base_model = client.models.retrieve(name=MODEL_NAME)\n matches = [a for a in (base_model.adapters or []) if a.name == ADAPTER_NAME]\n if matches:\n adapter = matches[0]\n break\n print(f\"Adapter not yet attached, retrying... ({int(time.time() - adapter_start)}s)\")\n time.sleep(5)\n\nif adapter is None:\n raise TimeoutError(\n f\"Adapter '{ADAPTER_NAME}' not found on model '{MODEL_NAME}' within {ADAPTER_TIMEOUT}s\"\n )\n\nprint(f\"Base model: {base_model.name}\")\nprint(f\"Adapter:\\n{adapter.model_dump_json(indent=2)}\")", + "language": "python", + "source_html": "ADAPTER_NAME = job.spec.output.name\nprint(f"Looking for adapter: {ADAPTER_NAME}")\n\n# The adapter may not be attached to the model entity immediately after\n# training completes — poll until it appears.\nADAPTER_TIMEOUT = 120\nadapter_start = time.time()\nadapter = None\nwhile time.time() - adapter_start < ADAPTER_TIMEOUT:\n base_model = client.models.retrieve(name=MODEL_NAME)\n matches = [a for a in (base_model.adapters or []) if a.name == ADAPTER_NAME]\n if matches:\n adapter = matches[0]\n break\n print(f"Adapter not yet attached, retrying... ({int(time.time() - adapter_start)}s)")\n time.sleep(5)\n\nif adapter is None:\n raise TimeoutError(\n f"Adapter '{ADAPTER_NAME}' not found on model '{MODEL_NAME}' within {ADAPTER_TIMEOUT}s"\n )\n\nprint(f"Base model: {base_model.name}")\nprint(f"Adapter:\\n{adapter.model_dump_json(indent=2)}")\n" + }, + { + "type": "markdown", + "source": "### Track Deployment Status", + "source_html": "

Track Deployment Status

\n" + }, + { + "type": "code", + "source": "DEPLOYMENT_NAME = f\"sft-deploy-{MODEL_NAME}\"\n\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f\"Monitoring deployment '{DEPLOYMENT_NAME}'...\")\nprint(f\"Timeout: {TIMEOUT_MINUTES} minutes\\n\")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=DEPLOYMENT_NAME,\n )\n\n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n\n clear_output(wait=True)\n print(f\"Deployment: {DEPLOYMENT_NAME}\")\n print(f\"Status: {deployment_status.status}\")\n print(f\"Elapsed time: {elapsed_min}m {elapsed_sec}s\")\n\n if deployment_status.status == \"READY\":\n print(\"\\nDeployment is ready!\")\n if not client.models.wait_for_gateway(DEPLOYMENT_NAME, workspace=WORKSPACE, timeout=60):\n raise RuntimeError(\"Inference gateway did not become ready\")\n break\n\n if deployment_status.status in (\"FAILED\", \"ERROR\", \"TERMINATED\", \"LOST\", \"DELETED\"):\n raise RuntimeError(f\"Deployment failed with status: {deployment_status.status}\")\n\n if elapsed > timeout_seconds:\n raise TimeoutError(f\"Deployment timeout after {TIMEOUT_MINUTES} minutes\")\n\n time.sleep(15)", + "language": "python", + "source_html": "DEPLOYMENT_NAME = f"sft-deploy-{MODEL_NAME}"\n\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f"Monitoring deployment '{DEPLOYMENT_NAME}'...")\nprint(f"Timeout: {TIMEOUT_MINUTES} minutes\\n")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=DEPLOYMENT_NAME,\n )\n\n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n\n clear_output(wait=True)\n print(f"Deployment: {DEPLOYMENT_NAME}")\n print(f"Status: {deployment_status.status}")\n print(f"Elapsed time: {elapsed_min}m {elapsed_sec}s")\n\n if deployment_status.status == "READY":\n print("\\nDeployment is ready!")\n if not client.models.wait_for_gateway(DEPLOYMENT_NAME, workspace=WORKSPACE, timeout=60):\n raise RuntimeError("Inference gateway did not become ready")\n break\n\n if deployment_status.status in ("FAILED", "ERROR", "TERMINATED", "LOST", "DELETED"):\n raise RuntimeError(f"Deployment failed with status: {deployment_status.status}")\n\n if elapsed > timeout_seconds:\n raise TimeoutError(f"Deployment timeout after {TIMEOUT_MINUTES} minutes")\n\n time.sleep(15)\n" + }, + { + "type": "markdown", + "source": "## 6. Compare Base vs Fine-Tuned Inference\n\nTest the deployed model with a tool calling prompt. Since `lora_enabled=True`, the NIM can serve both the base model and the LoRA-adapted model through the same deployment.", + "source_html": "

6. Compare Base vs Fine-Tuned Inference

\n

Test the deployed model with a tool calling prompt. Since lora_enabled=True, the NIM can serve both the base model and the LoRA-adapted model through the same deployment.

\n" + }, + { + "type": "code", + "source": "test_messages = [\n {\"role\": \"user\", \"content\": \"Calculate the factorial of 12 using math functions.\"},\n]\n\ntest_tools = [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"math_factorial\",\n \"description\": \"Calculate the factorial of a given number.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"number\": {\n \"type\": \"integer\",\n \"description\": \"The number for which factorial needs to be calculated.\",\n }\n },\n \"required\": [\"number\"],\n },\n },\n }\n]\n\n\ndef test_tool_calling(model_name: str, label: str):\n \"\"\"Send a tool calling request and display the response.\"\"\"\n response = client.inference.gateway.model.post(\n \"v1/chat/completions\",\n name=model_name,\n body={\n \"messages\": test_messages,\n \"tools\": test_tools,\n \"tool_choice\": \"auto\",\n \"temperature\": 0,\n \"max_tokens\": 256,\n },\n )\n\n print(f\"{'=' * 60}\")\n print(f\" {label}\")\n print(f\"{'=' * 60}\")\n print(json.dumps(response, indent=2))\n\n\ntest_tool_calling(MODEL_NAME, \"BASE MODEL (before fine-tuning)\")\ntest_tool_calling(ADAPTER_NAME, \"FINE-TUNED MODEL (after LoRA)\")", + "language": "python", + "source_html": "test_messages = [\n {"role": "user", "content": "Calculate the factorial of 12 using math functions."},\n]\n\ntest_tools = [\n {\n "type": "function",\n "function": {\n "name": "math_factorial",\n "description": "Calculate the factorial of a given number.",\n "parameters": {\n "type": "object",\n "properties": {\n "number": {\n "type": "integer",\n "description": "The number for which factorial needs to be calculated.",\n }\n },\n "required": ["number"],\n },\n },\n }\n]\n\n\ndef test_tool_calling(model_name: str, label: str):\n """Send a tool calling request and display the response."""\n response = client.inference.gateway.model.post(\n "v1/chat/completions",\n name=model_name,\n body={\n "messages": test_messages,\n "tools": test_tools,\n "tool_choice": "auto",\n "temperature": 0,\n "max_tokens": 256,\n },\n )\n\n print(f"{'=' * 60}")\n print(f" {label}")\n print(f"{'=' * 60}")\n print(json.dumps(response, indent=2))\n\n\ntest_tool_calling(MODEL_NAME, "BASE MODEL (before fine-tuning)")\ntest_tool_calling(ADAPTER_NAME, "FINE-TUNED MODEL (after LoRA)")\n" + }, + { + "type": "markdown", + "source": "## 7. Evaluate with Tool Calling Metrics\n\nUse the platform's evaluation service to systematically measure tool calling accuracy on the test dataset. The [Tool Calling metric](../evaluator/metrics/agentic.md) produces two scores:\n\n- **`function_name_accuracy`** -- Did the model call the correct function?\n- **`function_name_and_args_accuracy`** -- Did it call the correct function with the correct arguments?\n\n### Create Metric Entity", + "source_html": "

7. Evaluate with Tool Calling Metrics

\n

Use the platform's evaluation service to systematically measure tool calling accuracy on the test dataset. The Tool Calling metric produces two scores:

\n
    \n
  • function_name_accuracy -- Did the model call the correct function?
    • \n
  • function_name_and_args_accuracy -- Did it call the correct function with the correct arguments?
    • \n
\n

Create Metric Entity

\n" + }, + { + "type": "code", + "source": "METRIC_NAME = \"tool-calling-accuracy\"\n\ntry:\n client.evaluation.metrics.create(\n name=METRIC_NAME,\n type=\"tool-calling\",\n reference=\"{% raw %}{{ tool_calls }}{% endraw %}\",\n )\n print(f\"Created metric: {METRIC_NAME}\")\nexcept ConflictError:\n print(f\"Metric '{METRIC_NAME}' already exists, continuing...\")", + "language": "python", + "source_html": "METRIC_NAME = "tool-calling-accuracy"\n\ntry:\n client.evaluation.metrics.create(\n name=METRIC_NAME,\n type="tool-calling",\n reference="{% raw %}{{ tool_calls }}{% endraw %}",\n )\n print(f"Created metric: {METRIC_NAME}")\nexcept ConflictError:\n print(f"Metric '{METRIC_NAME}' already exists, continuing...")\n" + }, + { + "type": "markdown", + "source": "### Launch Evaluation Jobs\n\nSubmit evaluation jobs for both the fine-tuned adapter and the base model. Both jobs start running on the platform in parallel.", + "source_html": "

Launch Evaluation Jobs

\n

Submit evaluation jobs for both the fine-tuned adapter and the base model. Both jobs start running on the platform in parallel.

\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.evaluation import (\n MetricOnlineJobParam,\n EvaluationParamsParam,\n)\n\nft_eval_job = client.evaluation.metric_jobs.create(\n spec=MetricOnlineJobParam(\n metric=f\"{WORKSPACE}/{METRIC_NAME}\",\n model=f\"{WORKSPACE}/{ADAPTER_NAME}\",\n prompt_template={\n \"messages\": \"{% raw %}{{ messages | tojson }}{% endraw %}\",\n \"tools\": \"{% raw %}{{ tools | tojson }}{% endraw %}\",\n \"tool_choice\": \"auto\",\n },\n dataset=f\"{WORKSPACE}/{EVAL_DATASET_NAME}\",\n params=EvaluationParamsParam(parallelism=8),\n ),\n)\nprint(f\"Fine-tuned model eval job: {ft_eval_job.name}\")\n\nbase_eval_job = client.evaluation.metric_jobs.create(\n spec=MetricOnlineJobParam(\n metric=f\"{WORKSPACE}/{METRIC_NAME}\",\n model=f\"{WORKSPACE}/{MODEL_NAME}\",\n prompt_template={\n \"messages\": \"{% raw %}{{ messages | tojson }}{% endraw %}\",\n \"tools\": \"{% raw %}{{ tools | tojson }}{% endraw %}\",\n \"tool_choice\": \"auto\",\n },\n dataset=f\"{WORKSPACE}/{EVAL_DATASET_NAME}\",\n params=EvaluationParamsParam(parallelism=8),\n ),\n)\nprint(f\"Base model eval job: {base_eval_job.name}\")", + "language": "python", + "source_html": "from nemo_platform.types.evaluation import (\n MetricOnlineJobParam,\n EvaluationParamsParam,\n)\n\nft_eval_job = client.evaluation.metric_jobs.create(\n spec=MetricOnlineJobParam(\n metric=f"{WORKSPACE}/{METRIC_NAME}",\n model=f"{WORKSPACE}/{ADAPTER_NAME}",\n prompt_template={\n "messages": "{% raw %}{{ messages | tojson }}{% endraw %}",\n "tools": "{% raw %}{{ tools | tojson }}{% endraw %}",\n "tool_choice": "auto",\n },\n dataset=f"{WORKSPACE}/{EVAL_DATASET_NAME}",\n params=EvaluationParamsParam(parallelism=8),\n ),\n)\nprint(f"Fine-tuned model eval job: {ft_eval_job.name}")\n\nbase_eval_job = client.evaluation.metric_jobs.create(\n spec=MetricOnlineJobParam(\n metric=f"{WORKSPACE}/{METRIC_NAME}",\n model=f"{WORKSPACE}/{MODEL_NAME}",\n prompt_template={\n "messages": "{% raw %}{{ messages | tojson }}{% endraw %}",\n "tools": "{% raw %}{{ tools | tojson }}{% endraw %}",\n "tool_choice": "auto",\n },\n dataset=f"{WORKSPACE}/{EVAL_DATASET_NAME}",\n params=EvaluationParamsParam(parallelism=8),\n ),\n)\nprint(f"Base model eval job: {base_eval_job.name}")\n" + }, + { + "type": "markdown", + "source": "### Wait for Model Evaluation Jobs", + "source_html": "

Wait for Model Evaluation Jobs

\n" + }, + { + "type": "code", + "source": "ft_result = wait_for_job(\n poll_fn=lambda: client.evaluation.metric_jobs.retrieve(name=ft_eval_job.name),\n label=\"Fine-tuned eval\",\n)\nprint(ft_result)\n\nbase_result = wait_for_job(\n poll_fn=lambda: client.evaluation.metric_jobs.retrieve(name=base_eval_job.name),\n label=\"Base eval\",\n)\nprint(base_result)", + "language": "python", + "source_html": "ft_result = wait_for_job(\n poll_fn=lambda: client.evaluation.metric_jobs.retrieve(name=ft_eval_job.name),\n label="Fine-tuned eval",\n)\nprint(ft_result)\n\nbase_result = wait_for_job(\n poll_fn=lambda: client.evaluation.metric_jobs.retrieve(name=base_eval_job.name),\n label="Base eval",\n)\nprint(base_result)\n" + }, + { + "type": "markdown", + "source": "### Compare Results", + "source_html": "

Compare Results

\n" + }, + { + "type": "code", + "source": "def get_scores(eval_job) -> dict:\n \"\"\"Download aggregate scores from a completed evaluation job.\"\"\"\n result = client.evaluation.metric_jobs.results.aggregate_scores.download(\n job=eval_job.name,\n )\n scores = {}\n for score in result.scores:\n scores[score.name] = score.mean\n return scores\n\n\nft_scores = get_scores(ft_eval_job)\nbase_scores = get_scores(base_eval_job)\n\nprint(\"=\" * 70)\nprint(f\"{'Metric':<40} {'Base Model':>12} {'Fine-Tuned':>12}\")\nprint(\"=\" * 70)\n\nfor metric_name in [\"function_name_accuracy\", \"function_name_and_args_accuracy\"]:\n base_val = base_scores.get(metric_name, \"N/A\")\n ft_val = ft_scores.get(metric_name, \"N/A\")\n\n base_str = f\"{base_val:.3f}\" if isinstance(base_val, (int, float)) else str(base_val)\n ft_str = f\"{ft_val:.3f}\" if isinstance(ft_val, (int, float)) else str(ft_val)\n\n print(f\"{metric_name:<40} {base_str:>12} {ft_str:>12}\")\n\nprint(\"=\" * 70)\n\nif isinstance(ft_scores.get(\"function_name_accuracy\"), (int, float)) and isinstance(\n base_scores.get(\"function_name_accuracy\"), (int, float)\n):\n improvement = ft_scores[\"function_name_accuracy\"] - base_scores[\"function_name_accuracy\"]\n print(f\"\\nFunction name accuracy improvement: {improvement:+.3f}\")", + "language": "python", + "source_html": "def get_scores(eval_job) -> dict:\n """Download aggregate scores from a completed evaluation job."""\n result = client.evaluation.metric_jobs.results.aggregate_scores.download(\n job=eval_job.name,\n )\n scores = {}\n for score in result.scores:\n scores[score.name] = score.mean\n return scores\n\n\nft_scores = get_scores(ft_eval_job)\nbase_scores = get_scores(base_eval_job)\n\nprint("=" * 70)\nprint(f"{'Metric':<40} {'Base Model':>12} {'Fine-Tuned':>12}")\nprint("=" * 70)\n\nfor metric_name in ["function_name_accuracy", "function_name_and_args_accuracy"]:\n base_val = base_scores.get(metric_name, "N/A")\n ft_val = ft_scores.get(metric_name, "N/A")\n\n base_str = f"{base_val:.3f}" if isinstance(base_val, (int, float)) else str(base_val)\n ft_str = f"{ft_val:.3f}" if isinstance(ft_val, (int, float)) else str(ft_val)\n\n print(f"{metric_name:<40} {base_str:>12} {ft_str:>12}")\n\nprint("=" * 70)\n\nif isinstance(ft_scores.get("function_name_accuracy"), (int, float)) and isinstance(\n base_scores.get("function_name_accuracy"), (int, float)\n):\n improvement = ft_scores["function_name_accuracy"] - base_scores["function_name_accuracy"]\n print(f"\\nFunction name accuracy improvement: {improvement:+.3f}")\n" + }, + { + "type": "markdown", + "source": "## Done!\n\nNext Steps:\n- **Hyperparameter tuning**: See the [Training Configuration Reference](../customizer/manage-customization-jobs/hyperparameters.md) for all available hyperparameters, valid ranges, and tuning guidance.\n- **Troubleshooting**:\n - If you run into issues with customization jobs see [Troubleshooting Customizer](../troubleshooting/customizer.md).\n - If you run into issues with evaluation jobs see [Troubleshooting Evaluator](../troubleshooting/evaluator.md).", + "source_html": "

Done!

\n

Next Steps:

\n\n" + } + ] +} \ No newline at end of file diff --git a/docs/fern/components/notebooks/tool-calling.ts b/docs/fern/components/notebooks/tool-calling.ts new file mode 100644 index 00000000000..d75520ea8cd --- /dev/null +++ b/docs/fern/components/notebooks/tool-calling.ts @@ -0,0 +1,288 @@ +/** Auto-generated by ipynb-to-fern-json.py - do not edit */ +export default { cells: [ + { + "type": "markdown", + "source": "\n\n\n# Tool-Calling Fine-Tuning with Synthetic Data\n\nLearn how to fine-tune Llama 3.2 1B Instruct for tool calling using LoRA and synthetic data from Data Designer.\n\n## About\n\nTool calling enables LLMs to interact with external functions and APIs by generating structured function calls instead of plain text. Fine-tuning a model specifically for tool calling improves its ability to select the correct function and produce valid arguments.\n\nThis tutorial walks through the full workflow:\n\n1. **Data preparation** -- Generate a synthetic dataset with Data Designer, convert it to OpenAI chat format, and filter for compatibility with the Llama 3.2 chat template\n2. **LoRA fine-tuning** -- Train a LoRA adapter on the prepared dataset\n3. **Deployment and inference** -- Deploy the fine-tuned model and compare tool calling behavior against the base model\n4. **Evaluation** -- Measure tool calling accuracy using the platform's evaluation service", + "source_html": "\n\n

Tool-Calling Fine-Tuning with Synthetic Data

\n

Learn how to fine-tune Llama 3.2 1B Instruct for tool calling using LoRA and synthetic data from Data Designer.

\n

About

\n

Tool calling enables LLMs to interact with external functions and APIs by generating structured function calls instead of plain text. Fine-tuning a model specifically for tool calling improves its ability to select the correct function and produce valid arguments.

\n

This tutorial walks through the full workflow:

\n
    \n
  1. Data preparation -- Generate a synthetic dataset with Data Designer, convert it to OpenAI chat format, and filter for compatibility with the Llama 3.2 chat template
    2. \n
  2. LoRA fine-tuning -- Train a LoRA adapter on the prepared dataset
    3. \n
  3. Deployment and inference -- Deploy the fine-tuned model and compare tool calling behavior against the base model
    4. \n
  4. Evaluation -- Measure tool calling accuracy using the platform's evaluation service
    5. \n
\n" + }, + { + "type": "markdown", + "source": "## Prerequisites\n\nBefore starting this tutorial, ensure you have:\n\n1. **Installed the Python SDK with the Data Designer extra** (`uv pip install nemo-platform[data-designer]`)\n1. **Completed the [Quickstart](../get-started/quickstart.md)** to install and deploy NeMo Platform locally\n1. (Optional if running outside of Quickstart) **Authenticated with the platform** using the CLI:\n ```bash\n nemo auth login\n ```\n For non-default URLs: `nemo auth login --base-url `\n1. Set your **Hugging Face token** as an environment variable before running the notebook (`export HF_TOKEN=\"\"`)\n1. **Accepted the dataset license** for [Salesforce/xlam-function-calling-60k](https://huggingface.co/datasets/Salesforce/xlam-function-calling-60k) on Hugging Face (used for evaluation) ", + "source_html": "

Prerequisites

\n

Before starting this tutorial, ensure you have:

\n
    \n
  1. Installed the Python SDK with the Data Designer extra (uv pip install nemo-platform[data-designer])
    2. \n
  2. Completed the Quickstart to install and deploy NeMo Platform locally
    3. \n
  3. (Optional if running outside of Quickstart) Authenticated with the platform using the CLI:
    nemo auth login\n
    \nFor non-default URLs: nemo auth login --base-url <YOUR_NMP_BASE_URL>
    4. \n
  4. Set your Hugging Face token as an environment variable before running the notebook (export HF_TOKEN="<your-token>")
    5. \n
  5. Accepted the dataset license for Salesforce/xlam-function-calling-60k on Hugging Face (used for evaluation)
    6. \n
\n" + }, + { + "type": "markdown", + "source": "## 1. Initialize SDK\n\nThe SDK reads its cluster URL and authentication credentials from the config file managed by `nemo auth login`. For the default [Quickstart](../get-started/quickstart.md) deployment at `http://localhost:8080`, no extra configuration is needed.\n\nWe also define the provider and model we will be using for synthetic data generation. These must be available in your NeMo Platform deployment.\n\n> **Note:** This tutorial uses config-based initialization (`NeMoPlatform(workspace=...)`) instead of the explicit `base_url` pattern shown elsewhere in the docs to make it easier to switch between Quickstart and running on a cluster.", + "source_html": "

1. Initialize SDK

\n

The SDK reads its cluster URL and authentication credentials from the config file managed by nemo auth login. For the default Quickstart deployment at http://localhost:8080, no extra configuration is needed.

\n

We also define the provider and model we will be using for synthetic data generation. These must be available in your NeMo Platform deployment.

\n
\n

Note: This tutorial uses config-based initialization (NeMoPlatform(workspace=...)) instead of the explicit base_url pattern shown elsewhere in the docs to make it easier to switch between Quickstart and running on a cluster.

\n
\n" + }, + { + "type": "code", + "source": "import os\n\nfrom datasets import load_dataset as hf_load_dataset\nfrom nemo_platform import NeMoPlatform, ConflictError\nimport data_designer.config as dd\nfrom pydantic import BaseModel, Field\nfrom typing import List\n\nWORKSPACE = \"default\"\nclient = NeMoPlatform(workspace=WORKSPACE)\n\nMODEL_PROVIDER = f\"{WORKSPACE}/nvidia-build\"\nMODEL_ID = \"nvidia/nemotron-3-nano-30b-a3b\"\nMODEL_ALIAS = \"nemotron\"\nmodel_configs = [\n dd.ModelConfig(\n provider=MODEL_PROVIDER,\n model=MODEL_ID,\n alias=MODEL_ALIAS,\n inference_parameters=dd.ChatCompletionInferenceParams(\n temperature=1.0,\n top_p=1.0,\n max_tokens=2048,\n ),\n )\n]", + "language": "python", + "source_html": "import os\n\nfrom datasets import load_dataset as hf_load_dataset\nfrom nemo_platform import NeMoPlatform, ConflictError\nimport data_designer.config as dd\nfrom pydantic import BaseModel, Field\nfrom typing import List\n\nWORKSPACE = "default"\nclient = NeMoPlatform(workspace=WORKSPACE)\n\nMODEL_PROVIDER = f"{WORKSPACE}/nvidia-build"\nMODEL_ID = "nvidia/nemotron-3-nano-30b-a3b"\nMODEL_ALIAS = "nemotron"\nmodel_configs = [\n dd.ModelConfig(\n provider=MODEL_PROVIDER,\n model=MODEL_ID,\n alias=MODEL_ALIAS,\n inference_parameters=dd.ChatCompletionInferenceParams(\n temperature=1.0,\n top_p=1.0,\n max_tokens=2048,\n ),\n )\n]\n" + }, + { + "type": "markdown", + "source": "## 2. Generate synthetic data\n\n### Define tool calling schemas\n\n- We use Pydantic models to define the exact structure of our tool definitions and tool calls.\n- Data Designer enforces these schemas during generation.", + "source_html": "

2. Generate synthetic data

\n

Define tool calling schemas

\n
    \n
  • We use Pydantic models to define the exact structure of our tool definitions and tool calls.
    • \n
  • Data Designer enforces these schemas during generation.
    • \n
\n" + }, + { + "type": "code", + "source": "class ParameterDef(BaseModel):\n \"\"\"A single parameter in a function definition.\"\"\"\n name: str = Field(description=\"Parameter name\")\n type: str = Field(description=\"JSON schema type (string, integer, number, boolean, array, object)\")\n description: str = Field(description=\"What this parameter does\")\n\n\nclass FunctionDef(BaseModel):\n \"\"\"A tool/function definition.\"\"\"\n name: str = Field(description=\"Function name in snake_case\")\n description: str = Field(description=\"What this function does\")\n parameters: List[ParameterDef] = Field(description=\"List of function parameters\")\n\n\nclass ToolDefinitions(BaseModel):\n \"\"\"A set of available tools.\"\"\"\n tools: List[FunctionDef] = Field(description=\"2-4 tool/function definitions\")\n\n \nclass ArgumentDef(BaseModel):\n \"\"\"A single argument name/value pair in a tool call instance.\"\"\"\n name: str = Field(description=\"Parameter name\")\n value: str = Field(description=\"The value to pass, as a JSON-encoded string\")\n\n\nclass ToolCallArg(BaseModel):\n \"\"\"A single tool call with arguments.\"\"\"\n name: str = Field(description=\"Name of the function to call\")\n arguments: List[ArgumentDef] = Field(description=\"Arguments to pass to the function\")\n\nclass ExpectedToolCalls(BaseModel):\n \"\"\"The expected tool calls for a given query. The model we're fine-tuning has a chat template that only supports one tool call per query in its template.\"\"\"\n tool_calls: List[ToolCallArg] = Field(description=\"The tool call to be made.\")", + "language": "python", + "source_html": "class ParameterDef(BaseModel):\n """A single parameter in a function definition."""\n name: str = Field(description="Parameter name")\n type: str = Field(description="JSON schema type (string, integer, number, boolean, array, object)")\n description: str = Field(description="What this parameter does")\n\n\nclass FunctionDef(BaseModel):\n """A tool/function definition."""\n name: str = Field(description="Function name in snake_case")\n description: str = Field(description="What this function does")\n parameters: List[ParameterDef] = Field(description="List of function parameters")\n\n\nclass ToolDefinitions(BaseModel):\n """A set of available tools."""\n tools: List[FunctionDef] = Field(description="2-4 tool/function definitions")\n\n \nclass ArgumentDef(BaseModel):\n """A single argument name/value pair in a tool call instance."""\n name: str = Field(description="Parameter name")\n value: str = Field(description="The value to pass, as a JSON-encoded string")\n\n\nclass ToolCallArg(BaseModel):\n """A single tool call with arguments."""\n name: str = Field(description="Name of the function to call")\n arguments: List[ArgumentDef] = Field(description="Arguments to pass to the function")\n\nclass ExpectedToolCalls(BaseModel):\n """The expected tool calls for a given query. The model we're fine-tuning has a chat template that only supports one tool call per query in its template."""\n tool_calls: List[ToolCallArg] = Field(description="The tool call to be made.")\n" + }, + { + "type": "markdown", + "source": "### Build data columns\n\n- The config builder provides an intuitive interface for defining your dataset schema.", + "source_html": "

Build data columns

\n
    \n
  • The config builder provides an intuitive interface for defining your dataset schema.
    • \n
\n" + }, + { + "type": "code", + "source": "config_builder = dd.DataDesignerConfigBuilder(model_configs=model_configs)\n\nconfig_builder.add_column(\n dd.SamplerColumnConfig(\n name=\"domain\",\n sampler_type=\"category\",\n params={\"values\": [\n \"weather\",\n \"calendar\",\n \"web_search\",\n \"math\",\n \"file_management\",\n \"database\",\n \"email\",\n \"navigation\",\n \"e_commerce\",\n \"social_media\",\n ]},\n )\n)\n\nconfig_builder.add_column(\n dd.LLMStructuredColumnConfig(\n name=\"tools\",\n prompt=(\n \"Generate 2-4 realistic API function definitions for the '{% raw %}{{ domain }}{% endraw %}' domain. \"\n \"Each function should have a descriptive snake_case name, clear description, \"\n \"and well-typed parameters. Make them diverse — include functions with \"\n \"different parameter counts and types.\"\n ),\n output_format=ToolDefinitions,\n model_alias=MODEL_ALIAS,\n )\n)\n\nconfig_builder.add_column(\n dd.LLMTextColumnConfig(\n name=\"user_query\",\n prompt=(\n \"Given these available tools:\\n{% raw %}{{ tools }}{% endraw %}\\n\\n\"\n \"Generate a natural, realistic user query that would require calling \"\n \"one of these tools. The query should be something a real user \"\n \"would type. Vary complexity, but due to the limitations of our \"\n \"downstream model only require single tool calls. \"\n \"Respond with only the query, no other text.\"\n ),\n model_alias=MODEL_ALIAS,\n )\n)\n\nconfig_builder.add_column(\n dd.LLMStructuredColumnConfig(\n name=\"expected_tool_call\",\n prompt=(\n \"Given this user query: '{% raw %}{{ user_query }}{% endraw %}'\\n\"\n \"And these available tools: {% raw %}{{ tools }}{% endraw %}\\n\\n\"\n \"Determine exactly which tool should be called and with what arguments. \"\n \"Be precise with argument values — they should directly address the user's request.\"\n ),\n output_format=ExpectedToolCalls,\n model_alias=MODEL_ALIAS,\n )\n)", + "language": "python", + "source_html": "config_builder = dd.DataDesignerConfigBuilder(model_configs=model_configs)\n\nconfig_builder.add_column(\n dd.SamplerColumnConfig(\n name="domain",\n sampler_type="category",\n params={"values": [\n "weather",\n "calendar",\n "web_search",\n "math",\n "file_management",\n "database",\n "email",\n "navigation",\n "e_commerce",\n "social_media",\n ]},\n )\n)\n\nconfig_builder.add_column(\n dd.LLMStructuredColumnConfig(\n name="tools",\n prompt=(\n "Generate 2-4 realistic API function definitions for the '{% raw %}{{ domain }}{% endraw %}' domain. "\n "Each function should have a descriptive snake_case name, clear description, "\n "and well-typed parameters. Make them diverse — include functions with "\n "different parameter counts and types."\n ),\n output_format=ToolDefinitions,\n model_alias=MODEL_ALIAS,\n )\n)\n\nconfig_builder.add_column(\n dd.LLMTextColumnConfig(\n name="user_query",\n prompt=(\n "Given these available tools:\\n{% raw %}{{ tools }}{% endraw %}\\n\\n"\n "Generate a natural, realistic user query that would require calling "\n "one of these tools. The query should be something a real user "\n "would type. Vary complexity, but due to the limitations of our "\n "downstream model only require single tool calls. "\n "Respond with only the query, no other text."\n ),\n model_alias=MODEL_ALIAS,\n )\n)\n\nconfig_builder.add_column(\n dd.LLMStructuredColumnConfig(\n name="expected_tool_call",\n prompt=(\n "Given this user query: '{% raw %}{{ user_query }}{% endraw %}'\\n"\n "And these available tools: {% raw %}{{ tools }}{% endraw %}\\n\\n"\n "Determine exactly which tool should be called and with what arguments. "\n "Be precise with argument values — they should directly address the user's request."\n ),\n output_format=ExpectedToolCalls,\n model_alias=MODEL_ALIAS,\n )\n)\n" + }, + { + "type": "markdown", + "source": "### Preview the dataset\n\n1. Use the `preview` method to generate a small sample quickly.\n2. Inspect the results for quality and format issues.\n3. Adjust column configurations, prompts, or parameters as needed.\n4. Re-run the preview until satisfied.", + "source_html": "

Preview the dataset

\n
    \n
  1. Use the preview method to generate a small sample quickly.
    2. \n
  2. Inspect the results for quality and format issues.
    3. \n
  3. Adjust column configurations, prompts, or parameters as needed.
    4. \n
  4. Re-run the preview until satisfied.
    5. \n
\n" + }, + { + "type": "code", + "source": "preview = client.data_designer.preview(config_builder, num_records=4)", + "language": "python", + "source_html": "preview = client.data_designer.preview(config_builder, num_records=4)\n" + }, + { + "type": "markdown", + "source": "You can view one record at a time to inspect quality:", + "source_html": "

You can view one record at a time to inspect quality:

\n" + }, + { + "type": "code", + "source": "preview.display_sample_record()", + "language": "python", + "source_html": "preview.display_sample_record()\n" + }, + { + "type": "markdown", + "source": "Or view the entire preview dataset:", + "source_html": "

Or view the entire preview dataset:

\n" + }, + { + "type": "code", + "source": "preview.dataset", + "language": "python", + "source_html": "preview.dataset\n" + }, + { + "type": "markdown", + "source": "### Scale up\n\nThe preview functionality lets you iterate quickly on your data to ensure the right structure and quality. Once ready, the `create` method is used to generate the full training dataset.\n\nFor this tutorial we generate **60 samples** (~5 minutes) to keep iteration fast. For production use, generate **500+ samples** (~1 hour) for significantly better results — in our testing, 500 samples yielded **~93% function name accuracy**, compared to ~1.4% for the un-fine-tuned base model.", + "source_html": "

Scale up

\n

The preview functionality lets you iterate quickly on your data to ensure the right structure and quality. Once ready, the create method is used to generate the full training dataset.

\n

For this tutorial we generate 60 samples (~5 minutes) to keep iteration fast. For production use, generate 500+ samples (~1 hour) for significantly better results — in our testing, 500 samples yielded ~93% function name accuracy, compared to ~1.4% for the un-fine-tuned base model.

\n" + }, + { + "type": "code", + "source": "job = client.data_designer.create(config_builder, num_records=60)\njob.wait_until_done()", + "language": "python", + "source_html": "job = client.data_designer.create(config_builder, num_records=60)\njob.wait_until_done()\n" + }, + { + "type": "markdown", + "source": "Fetch the dataset from the job outputs and check out a preview.", + "source_html": "

Fetch the dataset from the job outputs and check out a preview.

\n" + }, + { + "type": "code", + "source": "results = job.download_artifacts()\ndataset = results.load_dataset()\ndataset.head()", + "language": "python", + "source_html": "results = job.download_artifacts()\ndataset = results.load_dataset()\ndataset.head()\n" + }, + { + "type": "markdown", + "source": "### Convert Dataset\n\n- Fine-tuning requires the [OpenAI messages+tools format](https://platform.openai.com/docs/guides/function-calling) with specific nesting.\n- Data Designer generated flat structures — now we wrap them in the OpenAI format.\n- Key transformations: wrap tools/calls in `{\"type\": \"function\", \"function\": {...}}`, convert parameter lists to properties dicts, add `\"content\": \"\"` to the assistant message.", + "source_html": "

Convert Dataset

\n
    \n
  • Fine-tuning requires the OpenAI messages+tools format with specific nesting.
    • \n
  • Data Designer generated flat structures — now we wrap them in the OpenAI format.
    • \n
  • Key transformations: wrap tools/calls in {"type": "function", "function": {...}}, convert parameter lists to properties dicts, add "content": "" to the assistant message.
    • \n
\n" + }, + { + "type": "code", + "source": "import json\n\ndef to_openai_format(row):\n \"\"\"Convert a DD-generated row to the OpenAI tool calling format expected by the Customizer.\"\"\"\n openai_tools = []\n for tool_def in row[\"tools\"][\"tools\"]:\n properties = {\n p[\"name\"]: {\"type\": p[\"type\"], \"description\": p[\"description\"]}\n for p in tool_def[\"parameters\"]\n }\n openai_tools.append({\n \"type\": \"function\",\n \"function\": {\n \"name\": tool_def[\"name\"],\n \"description\": tool_def[\"description\"],\n \"parameters\": {\"type\": \"object\", \"properties\": properties},\n },\n })\n\n openai_tool_calls = []\n for tc in row[\"expected_tool_call\"][\"tool_calls\"]:\n arguments = {}\n for arg in tc[\"arguments\"]:\n try:\n arguments[arg[\"name\"]] = json.loads(arg[\"value\"])\n except:\n arguments[arg[\"name\"]] = arg[\"value\"]\n openai_tool_calls.append({\n \"type\": \"function\",\n \"function\": {\n \"name\": tc[\"name\"],\n \"arguments\": arguments,\n },\n })\n\n return {\n \"messages\": [\n {\"role\": \"user\", \"content\": row[\"user_query\"]},\n {\"role\": \"assistant\", \"content\": \"\", \"tool_calls\": openai_tool_calls},\n ],\n \"tools\": openai_tools,\n }\n\n\ntraining_data = dataset.apply(to_openai_format, axis=1, result_type=\"expand\")\n\n# Llama 3.2 1B's chat template supports only single tool calls per turn — filter out any\n# generated samples that contain multiple tool calls in the assistant response.\npre_filter = len(training_data)\ntraining_data = training_data[\n training_data[\"messages\"].apply(lambda msgs: len(msgs[1].get(\"tool_calls\", [])) == 1)\n].reset_index(drop=True)\nprint(f\"Filtered {pre_filter} -> {len(training_data)} rows (kept single tool-call only)\")\n\ntraining_data.head()", + "language": "python", + "source_html": "import json\n\ndef to_openai_format(row):\n """Convert a DD-generated row to the OpenAI tool calling format expected by the Customizer."""\n openai_tools = []\n for tool_def in row["tools"]["tools"]:\n properties = {\n p["name"]: {"type": p["type"], "description": p["description"]}\n for p in tool_def["parameters"]\n }\n openai_tools.append({\n "type": "function",\n "function": {\n "name": tool_def["name"],\n "description": tool_def["description"],\n "parameters": {"type": "object", "properties": properties},\n },\n })\n\n openai_tool_calls = []\n for tc in row["expected_tool_call"]["tool_calls"]:\n arguments = {}\n for arg in tc["arguments"]:\n try:\n arguments[arg["name"]] = json.loads(arg["value"])\n except:\n arguments[arg["name"]] = arg["value"]\n openai_tool_calls.append({\n "type": "function",\n "function": {\n "name": tc["name"],\n "arguments": arguments,\n },\n })\n\n return {\n "messages": [\n {"role": "user", "content": row["user_query"]},\n {"role": "assistant", "content": "", "tool_calls": openai_tool_calls},\n ],\n "tools": openai_tools,\n }\n\n\ntraining_data = dataset.apply(to_openai_format, axis=1, result_type="expand")\n\n# Llama 3.2 1B's chat template supports only single tool calls per turn — filter out any\n# generated samples that contain multiple tool calls in the assistant response.\npre_filter = len(training_data)\ntraining_data = training_data[\n training_data["messages"].apply(lambda msgs: len(msgs[1].get("tool_calls", [])) == 1)\n].reset_index(drop=True)\nprint(f"Filtered {pre_filter} -> {len(training_data)} rows (kept single tool-call only)")\n\ntraining_data.head()\n" + }, + { + "type": "markdown", + "source": "### Sample and Split into Train / Validation\n\nWe use all available synthetic examples and split them 85/15 into training and validation sets. Evaluation uses a separate golden dataset (xLAM) prepared in the next section.", + "source_html": "

Sample and Split into Train / Validation

\n

We use all available synthetic examples and split them 85/15 into training and validation sets. Evaluation uses a separate golden dataset (xLAM) prepared in the next section.

\n" + }, + { + "type": "code", + "source": "import random\nfrom pathlib import Path\n\nSEED = 1234\nrandom.seed(SEED)\n\nSAMPLE_SIZE = 60\ntraining_records = training_data.to_dict(\"records\")\nsampled = random.sample(training_records, min(SAMPLE_SIZE, len(training_records)))\n\ntrain_size = max(1, int(0.85 * len(sampled)))\nval_size = max(1, len(sampled) - train_size)\n\ntrain_data = sampled[:train_size]\nval_data = sampled[train_size:]\n\nprint(f\"Training: {len(train_data)} examples\")\nprint(f\"Validation: {len(val_data)} examples\")\n\n# Save training and validation data\nDATASET_PATH = Path(\"tool-calling-dataset\").absolute()\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n\ndef save_jsonl(filepath: str, data: list) -> None:\n with open(filepath, \"w\") as f:\n for entry in data:\n f.write(json.dumps(entry) + \"\\n\")\n\n\nsave_jsonl(f\"{DATASET_PATH}/training.jsonl\", train_data)\nprint(f\"\\nSaved training.jsonl ({len(train_data)} rows)\")\n\nsave_jsonl(f\"{DATASET_PATH}/validation.jsonl\", val_data)\nprint(f\"Saved validation.jsonl ({len(val_data)} rows)\")", + "language": "python", + "source_html": "import random\nfrom pathlib import Path\n\nSEED = 1234\nrandom.seed(SEED)\n\nSAMPLE_SIZE = 60\ntraining_records = training_data.to_dict("records")\nsampled = random.sample(training_records, min(SAMPLE_SIZE, len(training_records)))\n\ntrain_size = max(1, int(0.85 * len(sampled)))\nval_size = max(1, len(sampled) - train_size)\n\ntrain_data = sampled[:train_size]\nval_data = sampled[train_size:]\n\nprint(f"Training: {len(train_data)} examples")\nprint(f"Validation: {len(val_data)} examples")\n\n# Save training and validation data\nDATASET_PATH = Path("tool-calling-dataset").absolute()\nos.makedirs(DATASET_PATH, exist_ok=True)\n\n\ndef save_jsonl(filepath: str, data: list) -> None:\n with open(filepath, "w") as f:\n for entry in data:\n f.write(json.dumps(entry) + "\\n")\n\n\nsave_jsonl(f"{DATASET_PATH}/training.jsonl", train_data)\nprint(f"\\nSaved training.jsonl ({len(train_data)} rows)")\n\nsave_jsonl(f"{DATASET_PATH}/validation.jsonl", val_data)\nprint(f"Saved validation.jsonl ({len(val_data)} rows)")\n" + }, + { + "type": "markdown", + "source": "### Prepare Evaluation Dataset from xLAM\n\nRather than evaluating on the same synthetic data we trained on, we use the [Salesforce/xlam-function-calling-60k](https://huggingface.co/datasets/Salesforce/xlam-function-calling-60k) golden dataset as an independent test set. This gives a more meaningful measure of how well fine-tuning generalizes.\n\nWe download the dataset, convert it to OpenAI format, filter for single tool-call examples (matching our model's capability), and sample a test set sized at 15% of our training data.", + "source_html": "

Prepare Evaluation Dataset from xLAM

\n

Rather than evaluating on the same synthetic data we trained on, we use the Salesforce/xlam-function-calling-60k golden dataset as an independent test set. This gives a more meaningful measure of how well fine-tuning generalizes.

\n

We download the dataset, convert it to OpenAI format, filter for single tool-call examples (matching our model's capability), and sample a test set sized at 15% of our training data.

\n" + }, + { + "type": "code", + "source": "LIMIT_TOOL_PROPERTIES = 8\n\n\ndef normalize_type(param_type: str) -> str:\n \"\"\"Normalize Python type hints to JSON schema types.\"\"\"\n param_type = param_type.strip()\n if \",\" in param_type and \"default\" in param_type:\n param_type = param_type.split(\",\")[0].strip()\n if param_type.startswith(\"default=\"):\n return \"string\"\n param_type = param_type.replace(\", optional\", \"\").strip()\n if param_type.startswith(\"Callable\"):\n return \"string\"\n if param_type.startswith((\"Tuple\", \"List[\", \"Set\")) or param_type in (\"list\", \"set\"):\n return \"array\"\n type_mapping = {\n \"str\": \"string\", \"int\": \"integer\", \"float\": \"number\",\n \"bool\": \"boolean\", \"list\": \"array\", \"dict\": \"object\",\n \"List\": \"array\", \"Dict\": \"object\", \"Set\": \"array\",\n }\n return type_mapping.get(param_type, \"string\")\n\n\ndef convert_xlam_tools(tools):\n \"\"\"Convert xLAM tool definitions to OpenAI function spec format.\"\"\"\n if isinstance(tools, str):\n tools = json.loads(tools)\n if not isinstance(tools, list):\n return []\n openai_tools = []\n for tool in tools:\n if not isinstance(tool, dict) or not isinstance(tool.get(\"parameters\"), dict):\n continue\n properties = {}\n for name, info in tool[\"parameters\"].items():\n if not isinstance(info, dict):\n continue\n param = {\n \"description\": info.get(\"description\", \"\"),\n \"type\": normalize_type(info.get(\"type\", \"\")),\n }\n properties[name] = param\n if len(properties) > LIMIT_TOOL_PROPERTIES:\n return []\n openai_tools.append({\n \"type\": \"function\",\n \"function\": {\n \"name\": tool[\"name\"],\n \"description\": tool[\"description\"],\n \"parameters\": {\"type\": \"object\", \"properties\": properties},\n },\n })\n return openai_tools\n\n\ndef convert_xlam_example(example):\n \"\"\"Convert a single xLAM example to OpenAI eval format.\"\"\"\n tools = convert_xlam_tools(example.get(\"tools\", \"[]\"))\n if not tools:\n return None\n answers = json.loads(example.get(\"answers\", \"[]\"))\n tool_calls = [\n {\"type\": \"function\", \"function\": {\"name\": t[\"name\"], \"arguments\": t.get(\"arguments\", {})}}\n for t in answers\n ]\n if len(tool_calls) != 1:\n return None\n return {\n \"messages\": [{\"role\": \"user\", \"content\": example[\"query\"]}],\n \"tools\": tools,\n \"tool_calls\": tool_calls,\n }\n\n\nxlam_dataset = hf_load_dataset(\"Salesforce/xlam-function-calling-60k\", split=\"train\")\nprint(f\"xLAM dataset size: {len(xlam_dataset)}\")\n\nxlam_converted = [r for ex in xlam_dataset if (r := convert_xlam_example(ex)) is not None]\nprint(f\"After filtering (single tool-call, ≤{LIMIT_TOOL_PROPERTIES} params): {len(xlam_converted)}\")\n\neval_size = max(1, int(0.15 * len(train_data)))\neval_rows = random.sample(xlam_converted, min(eval_size, len(xlam_converted)))\nprint(f\"Sampled {len(eval_rows)} eval examples (15% of {len(train_data)} training examples)\")\n\nEVAL_DATASET_NAME = \"tool-calling-eval-dataset\"\nEVAL_DATASET_PATH = Path(\"tool-calling-eval-dataset\").absolute()\nos.makedirs(EVAL_DATASET_PATH, exist_ok=True)\nsave_jsonl(f\"{EVAL_DATASET_PATH}/evaluation.jsonl\", eval_rows)\n\nprint(f\"\\nSaved to {EVAL_DATASET_PATH}/evaluation.jsonl\")\nprint(\"\\nSample evaluation row:\")\nprint(json.dumps(eval_rows[0], indent=2))", + "language": "python", + "source_html": "LIMIT_TOOL_PROPERTIES = 8\n\n\ndef normalize_type(param_type: str) -> str:\n """Normalize Python type hints to JSON schema types."""\n param_type = param_type.strip()\n if "," in param_type and "default" in param_type:\n param_type = param_type.split(",")[0].strip()\n if param_type.startswith("default="):\n return "string"\n param_type = param_type.replace(", optional", "").strip()\n if param_type.startswith("Callable"):\n return "string"\n if param_type.startswith(("Tuple", "List[", "Set")) or param_type in ("list", "set"):\n return "array"\n type_mapping = {\n "str": "string", "int": "integer", "float": "number",\n "bool": "boolean", "list": "array", "dict": "object",\n "List": "array", "Dict": "object", "Set": "array",\n }\n return type_mapping.get(param_type, "string")\n\n\ndef convert_xlam_tools(tools):\n """Convert xLAM tool definitions to OpenAI function spec format."""\n if isinstance(tools, str):\n tools = json.loads(tools)\n if not isinstance(tools, list):\n return []\n openai_tools = []\n for tool in tools:\n if not isinstance(tool, dict) or not isinstance(tool.get("parameters"), dict):\n continue\n properties = {}\n for name, info in tool["parameters"].items():\n if not isinstance(info, dict):\n continue\n param = {\n "description": info.get("description", ""),\n "type": normalize_type(info.get("type", "")),\n }\n properties[name] = param\n if len(properties) > LIMIT_TOOL_PROPERTIES:\n return []\n openai_tools.append({\n "type": "function",\n "function": {\n "name": tool["name"],\n "description": tool["description"],\n "parameters": {"type": "object", "properties": properties},\n },\n })\n return openai_tools\n\n\ndef convert_xlam_example(example):\n """Convert a single xLAM example to OpenAI eval format."""\n tools = convert_xlam_tools(example.get("tools", "[]"))\n if not tools:\n return None\n answers = json.loads(example.get("answers", "[]"))\n tool_calls = [\n {"type": "function", "function": {"name": t["name"], "arguments": t.get("arguments", {})}}\n for t in answers\n ]\n if len(tool_calls) != 1:\n return None\n return {\n "messages": [{"role": "user", "content": example["query"]}],\n "tools": tools,\n "tool_calls": tool_calls,\n }\n\n\nxlam_dataset = hf_load_dataset("Salesforce/xlam-function-calling-60k", split="train")\nprint(f"xLAM dataset size: {len(xlam_dataset)}")\n\nxlam_converted = [r for ex in xlam_dataset if (r := convert_xlam_example(ex)) is not None]\nprint(f"After filtering (single tool-call, ≤{LIMIT_TOOL_PROPERTIES} params): {len(xlam_converted)}")\n\neval_size = max(1, int(0.15 * len(train_data)))\neval_rows = random.sample(xlam_converted, min(eval_size, len(xlam_converted)))\nprint(f"Sampled {len(eval_rows)} eval examples (15% of {len(train_data)} training examples)")\n\nEVAL_DATASET_NAME = "tool-calling-eval-dataset"\nEVAL_DATASET_PATH = Path("tool-calling-eval-dataset").absolute()\nos.makedirs(EVAL_DATASET_PATH, exist_ok=True)\nsave_jsonl(f"{EVAL_DATASET_PATH}/evaluation.jsonl", eval_rows)\n\nprint(f"\\nSaved to {EVAL_DATASET_PATH}/evaluation.jsonl")\nprint("\\nSample evaluation row:")\nprint(json.dumps(eval_rows[0], indent=2))\n" + }, + { + "type": "markdown", + "source": "### Upload Training Dataset\n\nCreate a fileset and upload the training/validation data to the platform:", + "source_html": "

Upload Training Dataset

\n

Create a fileset and upload the training/validation data to the platform:

\n" + }, + { + "type": "code", + "source": "DATASET_NAME = \"tool-calling-dataset\"\n\ntry:\n client.files.filesets.create(\n name=DATASET_NAME,\n description=\"synthetic tool-calling training and validation data in OpenAI chat format\",\n )\n print(f\"Created fileset: {DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n\nclient.files.upload(\n local_path=DATASET_PATH,\n remote_path=\"\",\n fileset=DATASET_NAME,\n)\n\nprint(\"\\nTraining data files:\")\nprint(json.dumps(\n [f.model_dump() for f in client.files.list(fileset=DATASET_NAME).data],\n indent=2,\n))", + "language": "python", + "source_html": "DATASET_NAME = "tool-calling-dataset"\n\ntry:\n client.files.filesets.create(\n name=DATASET_NAME,\n description="synthetic tool-calling training and validation data in OpenAI chat format",\n )\n print(f"Created fileset: {DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{DATASET_NAME}' already exists, continuing...")\n\nclient.files.upload(\n local_path=DATASET_PATH,\n remote_path="",\n fileset=DATASET_NAME,\n)\n\nprint("\\nTraining data files:")\nprint(json.dumps(\n [f.model_dump() for f in client.files.list(fileset=DATASET_NAME).data],\n indent=2,\n))\n" + }, + { + "type": "markdown", + "source": "### Upload Evaluation Dataset\n\nUpload the evaluation data as a separate fileset:", + "source_html": "

Upload Evaluation Dataset

\n

Upload the evaluation data as a separate fileset:

\n" + }, + { + "type": "code", + "source": "try:\n client.files.filesets.create(\n name=EVAL_DATASET_NAME,\n description=\"synthetic tool-calling evaluation data (messages, tools, ground-truth tool_calls)\",\n )\n print(f\"Created fileset: {EVAL_DATASET_NAME}\")\nexcept ConflictError:\n print(f\"Fileset '{EVAL_DATASET_NAME}' already exists, continuing...\")\n\nclient.files.upload(\n local_path=EVAL_DATASET_PATH,\n remote_path=\"\",\n fileset=EVAL_DATASET_NAME,\n)\n\nprint(\"\\nEvaluation data files:\")\nprint(json.dumps(\n [f.model_dump() for f in client.files.list(fileset=EVAL_DATASET_NAME).data],\n indent=2,\n))", + "language": "python", + "source_html": "try:\n client.files.filesets.create(\n name=EVAL_DATASET_NAME,\n description="synthetic tool-calling evaluation data (messages, tools, ground-truth tool_calls)",\n )\n print(f"Created fileset: {EVAL_DATASET_NAME}")\nexcept ConflictError:\n print(f"Fileset '{EVAL_DATASET_NAME}' already exists, continuing...")\n\nclient.files.upload(\n local_path=EVAL_DATASET_PATH,\n remote_path="",\n fileset=EVAL_DATASET_NAME,\n)\n\nprint("\\nEvaluation data files:")\nprint(json.dumps(\n [f.model_dump() for f in client.files.list(fileset=EVAL_DATASET_NAME).data],\n indent=2,\n))\n" + }, + { + "type": "markdown", + "source": "## 3. Model and Secrets Setup\n\n### Configure HuggingFace Authentication\n\nWe will fine-tune [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) from HuggingFace. Since this is a gated model, you need:\n\n1. A HuggingFace token with Read access from [HuggingFace Settings](https://huggingface.co/settings/tokens)\n2. Accept the model's license on the [model page](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct)", + "source_html": "

3. Model and Secrets Setup

\n

Configure HuggingFace Authentication

\n

We will fine-tune meta-llama/Llama-3.2-1B-Instruct from HuggingFace. Since this is a gated model, you need:

\n
    \n
  1. A HuggingFace token with Read access from HuggingFace Settings
    2. \n
  2. Accept the model's license on the model page
    3. \n
\n" + }, + { + "type": "code", + "source": "HF_TOKEN = os.environ.get(\"HF_TOKEN\")\nif HF_TOKEN is None:\n raise ValueError(\"HF_TOKEN is not set.\")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f\"{label} is not set\")\n try:\n secret = client.secrets.create(\n name=name,\n value=value,\n )\n print(f\"Created secret: {name}\")\n return secret\n except ConflictError:\n print(f\"Secret '{name}' already exists, continuing...\")\n return client.secrets.retrieve(name=name)\n\n\nhf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")\nprint(hf_secret.model_dump_json(indent=2))", + "language": "python", + "source_html": "HF_TOKEN = os.environ.get("HF_TOKEN")\nif HF_TOKEN is None:\n raise ValueError("HF_TOKEN is not set.")\n\n\ndef create_or_get_secret(name: str, value: str | None, label: str):\n if not value:\n raise ValueError(f"{label} is not set")\n try:\n secret = client.secrets.create(\n name=name,\n value=value,\n )\n print(f"Created secret: {name}")\n return secret\n except ConflictError:\n print(f"Secret '{name}' already exists, continuing...")\n return client.secrets.retrieve(name=name)\n\n\nhf_secret = create_or_get_secret("hf-token", HF_TOKEN, "HF_TOKEN")\nprint(hf_secret.model_dump_json(indent=2))\n" + }, + { + "type": "markdown", + "source": "### Create Base Model FileSet and Model Entity\n\nCreate a fileset pointing to Llama 3.2 1B Instruct on HuggingFace, then create a Model Entity that references it. The model will be downloaded at training time.\n\nFor tool calling, set `tool_call_config` in the fileset's `metadata`. The NIM serving layer needs to know *how* to parse tool calls from the model's raw output, but that information isn't embedded in the model checkpoint itself -- it depends on the model family and chat template. By declaring it on the fileset, the platform's model-spec task merges it into the Model Entity's `spec.tool_call_config`, which the deployment backend then reads to configure the NIM automatically:\n- `tool_call_parser` -- tells the NIM how to parse tool calls from the model output (e.g., `\"llama3_json\"` for Llama 3.x models)\n- `auto_tool_choice` -- enables the model to decide when to call tools", + "source_html": "

Create Base Model FileSet and Model Entity

\n

Create a fileset pointing to Llama 3.2 1B Instruct on HuggingFace, then create a Model Entity that references it. The model will be downloaded at training time.

\n

For tool calling, set tool_call_config in the fileset's metadata. The NIM serving layer needs to know how to parse tool calls from the model's raw output, but that information isn't embedded in the model checkpoint itself -- it depends on the model family and chat template. By declaring it on the fileset, the platform's model-spec task merges it into the Model Entity's spec.tool_call_config, which the deployment backend then reads to configure the NIM automatically:

\n
    \n
  • tool_call_parser -- tells the NIM how to parse tool calls from the model output (e.g., "llama3_json" for Llama 3.x models)
    • \n
  • auto_tool_choice -- enables the model to decide when to call tools
    • \n
\n" + }, + { + "type": "code", + "source": "import time\n\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = \"meta-llama/Llama-3.2-1B-Instruct\"\nMODEL_NAME = \"llama-3-2-1b-base\"\n\ntry:\n base_model_fs = client.files.filesets.create(\n name=MODEL_NAME,\n description=\"Llama 3.2 1B Instruct base model from HuggingFace\",\n storage=HuggingfaceStorageConfigParam(\n type=\"huggingface\",\n repo_id=HF_REPO_ID,\n repo_type=\"model\",\n token_secret=hf_secret.name,\n ),\n metadata={\n \"model\": {\n \"tool_calling\": {\n \"tool_call_parser\": \"llama3_json\",\n \"auto_tool_choice\": True,\n }\n }\n },\n )\n print(f\"Created base model fileset: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model fileset already exists. Skipping creation.\")\n base_model_fs = client.files.filesets.retrieve(\n name=MODEL_NAME,\n )\n\ntry:\n base_model = client.models.create(\n name=MODEL_NAME,\n fileset=f\"{WORKSPACE}/{MODEL_NAME}\",\n )\n print(f\"Created Model Entity: {MODEL_NAME}\")\nexcept ConflictError:\n print(f\"Base model already exists. Updating fileset if different.\")\n base_model = client.models.update(\n name=MODEL_NAME,\n fileset=f\"{WORKSPACE}/{MODEL_NAME}\",\n )\n\nprint(f\"\\nBase model fileset: fileset://{WORKSPACE}/{base_model.name}\")\nprint(\"Base model fileset files list:\")\nprint(json.dumps(\n [f.model_dump() for f in client.files.list(fileset=MODEL_NAME).data],\n indent=2,\n))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint(\"\\nWaiting for ModelSpec to be populated...\")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f\"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds\")\n time.sleep(2)\n base_model = client.models.retrieve(\n name=MODEL_NAME,\n )\n\nprint(f\"ModelSpec populated: {base_model.spec.model_dump()}\")", + "language": "python", + "source_html": "import time\n\nfrom nemo_platform.types.files import HuggingfaceStorageConfigParam\n\nHF_REPO_ID = "meta-llama/Llama-3.2-1B-Instruct"\nMODEL_NAME = "llama-3-2-1b-base"\n\ntry:\n base_model_fs = client.files.filesets.create(\n name=MODEL_NAME,\n description="Llama 3.2 1B Instruct base model from HuggingFace",\n storage=HuggingfaceStorageConfigParam(\n type="huggingface",\n repo_id=HF_REPO_ID,\n repo_type="model",\n token_secret=hf_secret.name,\n ),\n metadata={\n "model": {\n "tool_calling": {\n "tool_call_parser": "llama3_json",\n "auto_tool_choice": True,\n }\n }\n },\n )\n print(f"Created base model fileset: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model fileset already exists. Skipping creation.")\n base_model_fs = client.files.filesets.retrieve(\n name=MODEL_NAME,\n )\n\ntry:\n base_model = client.models.create(\n name=MODEL_NAME,\n fileset=f"{WORKSPACE}/{MODEL_NAME}",\n )\n print(f"Created Model Entity: {MODEL_NAME}")\nexcept ConflictError:\n print(f"Base model already exists. Updating fileset if different.")\n base_model = client.models.update(\n name=MODEL_NAME,\n fileset=f"{WORKSPACE}/{MODEL_NAME}",\n )\n\nprint(f"\\nBase model fileset: fileset://{WORKSPACE}/{base_model.name}")\nprint("Base model fileset files list:")\nprint(json.dumps(\n [f.model_dump() for f in client.files.list(fileset=MODEL_NAME).data],\n indent=2,\n))\n\n# Wait for ModelSpec to be populated from the checkpoint\nprint("\\nWaiting for ModelSpec to be populated...")\nSPEC_TIMEOUT_SECONDS = 120\nspec_start = time.time()\nwhile not base_model.spec:\n if time.time() - spec_start > SPEC_TIMEOUT_SECONDS:\n raise TimeoutError(f"ModelSpec not populated within {SPEC_TIMEOUT_SECONDS} seconds")\n time.sleep(2)\n base_model = client.models.retrieve(\n name=MODEL_NAME,\n )\n\nprint(f"ModelSpec populated: {base_model.spec.model_dump()}")\n" + }, + { + "type": "markdown", + "source": "## 4. Create LoRA Fine-Tuning Job\n\nCreate a customization job using SFT training with LoRA PEFT. The `peft=LoRaParamsParam()` parameter enables LoRA instead of full-weight fine-tuning.\n\n**LoRA defaults** (can be overridden in `LoRaParamsParam()`):\n- `rank`: LoRA rank (dimensionality of the low-rank matrices)\n- `alpha`: Scaling factor for LoRA updates\n- `target_modules`: Which model layers to apply LoRA to", + "source_html": "

4. Create LoRA Fine-Tuning Job

\n

Create a customization job using SFT training with LoRA PEFT. The peft=LoRaParamsParam() parameter enables LoRA instead of full-weight fine-tuning.

\n

LoRA defaults (can be overridden in LoRaParamsParam()):

\n
    \n
  • rank: LoRA rank (dimensionality of the low-rank matrices)
    • \n
  • alpha: Scaling factor for LoRA updates
    • \n
  • target_modules: Which model layers to apply LoRA to
    • \n
\n" + }, + { + "type": "code", + "source": "import uuid\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam,\n LoRaParamsParam,\n DeploymentParamsParam,\n)\n\njob_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f\"tool-calling-lora-{job_suffix}\"\n\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n spec=CustomizationJobInputParam(\n model=f\"{WORKSPACE}/{base_model.name}\",\n dataset=f\"fileset://{WORKSPACE}/{DATASET_NAME}\",\n training=SftTrainingParam(\n type=\"sft\",\n epochs=4,\n batch_size=4,\n learning_rate=0.0001,\n max_seq_length=2048,\n micro_batch_size=1,\n peft=LoRaParamsParam(),\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n deployment_config=DeploymentParamsParam(\n lora_enabled=True,\n ),\n ),\n)\n\nprint(job.model_dump_json(indent=2))", + "language": "python", + "source_html": "import uuid\nfrom nemo_platform.types.customization import (\n CustomizationJobInputParam,\n SftTrainingParam,\n ParallelismParamsParam,\n LoRaParamsParam,\n DeploymentParamsParam,\n)\n\njob_suffix = uuid.uuid4().hex[:4]\nJOB_NAME = f"tool-calling-lora-{job_suffix}"\n\n\njob = client.customization.jobs.create(\n name=JOB_NAME,\n spec=CustomizationJobInputParam(\n model=f"{WORKSPACE}/{base_model.name}",\n dataset=f"fileset://{WORKSPACE}/{DATASET_NAME}",\n training=SftTrainingParam(\n type="sft",\n epochs=4,\n batch_size=4,\n learning_rate=0.0001,\n max_seq_length=2048,\n micro_batch_size=1,\n peft=LoRaParamsParam(),\n parallelism=ParallelismParamsParam(\n num_gpus_per_node=1,\n num_nodes=1,\n tensor_parallel_size=1,\n pipeline_parallel_size=1,\n ),\n ),\n deployment_config=DeploymentParamsParam(\n lora_enabled=True,\n ),\n ),\n)\n\nprint(job.model_dump_json(indent=2))\n" + }, + { + "type": "markdown", + "source": "### Track Training Progress", + "source_html": "

Track Training Progress

\n" + }, + { + "type": "code", + "source": "import time\nfrom IPython.display import clear_output\n\nTERMINAL_JOB_STATUSES = {\"completed\", \"cancelled\", \"error\"}\n\n\ndef wait_for_job(poll_fn, label, timeout_minutes=60, poll_interval=10, display_fn=None):\n \"\"\"Poll a platform job until it reaches a terminal state.\n\n Both Customizer and Evaluator use the Core Jobs service, so the same\n PlatformJobStatus values apply: created, pending, active, cancelled,\n cancelling, error, completed, paused, pausing, resuming.\n\n Args:\n poll_fn: Callable returning a status object with .name and .status attributes.\n label: Display label for progress output.\n timeout_minutes: Maximum time to wait before returning.\n poll_interval: Seconds between polls.\n display_fn: Optional callable(status) to print extra details after the header.\n\n Returns:\n The final status object.\n \"\"\"\n start = time.time()\n timeout = timeout_minutes * 60\n\n while True:\n status = poll_fn()\n elapsed = time.time() - start\n elapsed_min, elapsed_sec = divmod(int(elapsed), 60)\n\n clear_output(wait=True)\n print(f\"[{label}] Job: {status.name}\")\n print(f\"[{label}] Status: {status.status}\")\n print(f\"[{label}] Elapsed: {elapsed_min}m {elapsed_sec}s\")\n\n if display_fn:\n display_fn(status)\n\n if status.status in TERMINAL_JOB_STATUSES:\n print(f\"\\n[{label}] Job finished: {status.status}\")\n return status\n\n if elapsed > timeout:\n print(f\"\\n[{label}] Timeout after {timeout_minutes} minutes\")\n return status\n\n time.sleep(poll_interval)\n\n\ndef training_progress(status):\n \"\"\"Extract and display training step progress.\"\"\"\n for step in status.steps or []:\n if step.name == \"customization-training-job\":\n for task in step.tasks or []:\n details = task.status_details or {}\n s, mx = details.get(\"step\"), details.get(\"max_steps\")\n if s is not None and mx is not None:\n print(f\"Training: Step {s}/{mx} ({(s / mx) * 100:.1f}%)\")\n if phase := details.get(\"phase\"):\n print(f\"Phase: {phase}\")\n return\n print(\"Training step not started yet\")\n\nprint(\"Defined wait_for_job helper function\")", + "language": "python", + "source_html": "import time\nfrom IPython.display import clear_output\n\nTERMINAL_JOB_STATUSES = {"completed", "cancelled", "error"}\n\n\ndef wait_for_job(poll_fn, label, timeout_minutes=60, poll_interval=10, display_fn=None):\n """Poll a platform job until it reaches a terminal state.\n\n Both Customizer and Evaluator use the Core Jobs service, so the same\n PlatformJobStatus values apply: created, pending, active, cancelled,\n cancelling, error, completed, paused, pausing, resuming.\n\n Args:\n poll_fn: Callable returning a status object with .name and .status attributes.\n label: Display label for progress output.\n timeout_minutes: Maximum time to wait before returning.\n poll_interval: Seconds between polls.\n display_fn: Optional callable(status) to print extra details after the header.\n\n Returns:\n The final status object.\n """\n start = time.time()\n timeout = timeout_minutes * 60\n\n while True:\n status = poll_fn()\n elapsed = time.time() - start\n elapsed_min, elapsed_sec = divmod(int(elapsed), 60)\n\n clear_output(wait=True)\n print(f"[{label}] Job: {status.name}")\n print(f"[{label}] Status: {status.status}")\n print(f"[{label}] Elapsed: {elapsed_min}m {elapsed_sec}s")\n\n if display_fn:\n display_fn(status)\n\n if status.status in TERMINAL_JOB_STATUSES:\n print(f"\\n[{label}] Job finished: {status.status}")\n return status\n\n if elapsed > timeout:\n print(f"\\n[{label}] Timeout after {timeout_minutes} minutes")\n return status\n\n time.sleep(poll_interval)\n\n\ndef training_progress(status):\n """Extract and display training step progress."""\n for step in status.steps or []:\n if step.name == "customization-training-job":\n for task in step.tasks or []:\n details = task.status_details or {}\n s, mx = details.get("step"), details.get("max_steps")\n if s is not None and mx is not None:\n print(f"Training: Step {s}/{mx} ({(s / mx) * 100:.1f}%)")\n if phase := details.get("phase"):\n print(f"Phase: {phase}")\n return\n print("Training step not started yet")\n\nprint("Defined wait_for_job helper function")\n" + }, + { + "type": "code", + "source": "job_status = wait_for_job(\n poll_fn=lambda: client.customization.jobs.get_status(name=job.name),\n label=\"Training\",\n timeout_minutes=120,\n display_fn=training_progress,\n)", + "language": "python", + "source_html": "job_status = wait_for_job(\n poll_fn=lambda: client.customization.jobs.get_status(name=job.name),\n label="Training",\n timeout_minutes=120,\n display_fn=training_progress,\n)\n" + }, + { + "type": "markdown", + "source": "## 5. Verify Auto-Deployed Model\n\nSince we set `lora_enabled=True` in the customization job's `deployment_config`, the platform automatically creates a NIM deployment for the base model after training completes. The LoRA adapter is attached to the base model entity (enabled by default) and the deployment serves both the base weights and the adapter through a single NIM instance.", + "source_html": "

5. Verify Auto-Deployed Model

\n

Since we set lora_enabled=True in the customization job's deployment_config, the platform automatically creates a NIM deployment for the base model after training completes. The LoRA adapter is attached to the base model entity (enabled by default) and the deployment serves both the base weights and the adapter through a single NIM instance.

\n" + }, + { + "type": "code", + "source": "ADAPTER_NAME = job.spec.output.name\nprint(f\"Looking for adapter: {ADAPTER_NAME}\")\n\n# The adapter may not be attached to the model entity immediately after\n# training completes — poll until it appears.\nADAPTER_TIMEOUT = 120\nadapter_start = time.time()\nadapter = None\nwhile time.time() - adapter_start < ADAPTER_TIMEOUT:\n base_model = client.models.retrieve(name=MODEL_NAME)\n matches = [a for a in (base_model.adapters or []) if a.name == ADAPTER_NAME]\n if matches:\n adapter = matches[0]\n break\n print(f\"Adapter not yet attached, retrying... ({int(time.time() - adapter_start)}s)\")\n time.sleep(5)\n\nif adapter is None:\n raise TimeoutError(\n f\"Adapter '{ADAPTER_NAME}' not found on model '{MODEL_NAME}' within {ADAPTER_TIMEOUT}s\"\n )\n\nprint(f\"Base model: {base_model.name}\")\nprint(f\"Adapter:\\n{adapter.model_dump_json(indent=2)}\")", + "language": "python", + "source_html": "ADAPTER_NAME = job.spec.output.name\nprint(f"Looking for adapter: {ADAPTER_NAME}")\n\n# The adapter may not be attached to the model entity immediately after\n# training completes — poll until it appears.\nADAPTER_TIMEOUT = 120\nadapter_start = time.time()\nadapter = None\nwhile time.time() - adapter_start < ADAPTER_TIMEOUT:\n base_model = client.models.retrieve(name=MODEL_NAME)\n matches = [a for a in (base_model.adapters or []) if a.name == ADAPTER_NAME]\n if matches:\n adapter = matches[0]\n break\n print(f"Adapter not yet attached, retrying... ({int(time.time() - adapter_start)}s)")\n time.sleep(5)\n\nif adapter is None:\n raise TimeoutError(\n f"Adapter '{ADAPTER_NAME}' not found on model '{MODEL_NAME}' within {ADAPTER_TIMEOUT}s"\n )\n\nprint(f"Base model: {base_model.name}")\nprint(f"Adapter:\\n{adapter.model_dump_json(indent=2)}")\n" + }, + { + "type": "markdown", + "source": "### Track Deployment Status", + "source_html": "

Track Deployment Status

\n" + }, + { + "type": "code", + "source": "DEPLOYMENT_NAME = f\"sft-deploy-{MODEL_NAME}\"\n\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f\"Monitoring deployment '{DEPLOYMENT_NAME}'...\")\nprint(f\"Timeout: {TIMEOUT_MINUTES} minutes\\n\")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=DEPLOYMENT_NAME,\n )\n\n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n\n clear_output(wait=True)\n print(f\"Deployment: {DEPLOYMENT_NAME}\")\n print(f\"Status: {deployment_status.status}\")\n print(f\"Elapsed time: {elapsed_min}m {elapsed_sec}s\")\n\n if deployment_status.status == \"READY\":\n print(\"\\nDeployment is ready!\")\n if not client.models.wait_for_gateway(DEPLOYMENT_NAME, workspace=WORKSPACE, timeout=60):\n raise RuntimeError(\"Inference gateway did not become ready\")\n break\n\n if deployment_status.status in (\"FAILED\", \"ERROR\", \"TERMINATED\", \"LOST\", \"DELETED\"):\n raise RuntimeError(f\"Deployment failed with status: {deployment_status.status}\")\n\n if elapsed > timeout_seconds:\n raise TimeoutError(f\"Deployment timeout after {TIMEOUT_MINUTES} minutes\")\n\n time.sleep(15)", + "language": "python", + "source_html": "DEPLOYMENT_NAME = f"sft-deploy-{MODEL_NAME}"\n\nTIMEOUT_MINUTES = 30\nstart_time = time.time()\ntimeout_seconds = TIMEOUT_MINUTES * 60\n\nprint(f"Monitoring deployment '{DEPLOYMENT_NAME}'...")\nprint(f"Timeout: {TIMEOUT_MINUTES} minutes\\n")\n\nwhile True:\n deployment_status = client.inference.deployments.retrieve(\n name=DEPLOYMENT_NAME,\n )\n\n elapsed = time.time() - start_time\n elapsed_min = int(elapsed // 60)\n elapsed_sec = int(elapsed % 60)\n\n clear_output(wait=True)\n print(f"Deployment: {DEPLOYMENT_NAME}")\n print(f"Status: {deployment_status.status}")\n print(f"Elapsed time: {elapsed_min}m {elapsed_sec}s")\n\n if deployment_status.status == "READY":\n print("\\nDeployment is ready!")\n if not client.models.wait_for_gateway(DEPLOYMENT_NAME, workspace=WORKSPACE, timeout=60):\n raise RuntimeError("Inference gateway did not become ready")\n break\n\n if deployment_status.status in ("FAILED", "ERROR", "TERMINATED", "LOST", "DELETED"):\n raise RuntimeError(f"Deployment failed with status: {deployment_status.status}")\n\n if elapsed > timeout_seconds:\n raise TimeoutError(f"Deployment timeout after {TIMEOUT_MINUTES} minutes")\n\n time.sleep(15)\n" + }, + { + "type": "markdown", + "source": "## 6. Compare Base vs Fine-Tuned Inference\n\nTest the deployed model with a tool calling prompt. Since `lora_enabled=True`, the NIM can serve both the base model and the LoRA-adapted model through the same deployment.", + "source_html": "

6. Compare Base vs Fine-Tuned Inference

\n

Test the deployed model with a tool calling prompt. Since lora_enabled=True, the NIM can serve both the base model and the LoRA-adapted model through the same deployment.

\n" + }, + { + "type": "code", + "source": "test_messages = [\n {\"role\": \"user\", \"content\": \"Calculate the factorial of 12 using math functions.\"},\n]\n\ntest_tools = [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"math_factorial\",\n \"description\": \"Calculate the factorial of a given number.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"number\": {\n \"type\": \"integer\",\n \"description\": \"The number for which factorial needs to be calculated.\",\n }\n },\n \"required\": [\"number\"],\n },\n },\n }\n]\n\n\ndef test_tool_calling(model_name: str, label: str):\n \"\"\"Send a tool calling request and display the response.\"\"\"\n response = client.inference.gateway.model.post(\n \"v1/chat/completions\",\n name=model_name,\n body={\n \"messages\": test_messages,\n \"tools\": test_tools,\n \"tool_choice\": \"auto\",\n \"temperature\": 0,\n \"max_tokens\": 256,\n },\n )\n\n print(f\"{'=' * 60}\")\n print(f\" {label}\")\n print(f\"{'=' * 60}\")\n print(json.dumps(response, indent=2))\n\n\ntest_tool_calling(MODEL_NAME, \"BASE MODEL (before fine-tuning)\")\ntest_tool_calling(ADAPTER_NAME, \"FINE-TUNED MODEL (after LoRA)\")", + "language": "python", + "source_html": "test_messages = [\n {"role": "user", "content": "Calculate the factorial of 12 using math functions."},\n]\n\ntest_tools = [\n {\n "type": "function",\n "function": {\n "name": "math_factorial",\n "description": "Calculate the factorial of a given number.",\n "parameters": {\n "type": "object",\n "properties": {\n "number": {\n "type": "integer",\n "description": "The number for which factorial needs to be calculated.",\n }\n },\n "required": ["number"],\n },\n },\n }\n]\n\n\ndef test_tool_calling(model_name: str, label: str):\n """Send a tool calling request and display the response."""\n response = client.inference.gateway.model.post(\n "v1/chat/completions",\n name=model_name,\n body={\n "messages": test_messages,\n "tools": test_tools,\n "tool_choice": "auto",\n "temperature": 0,\n "max_tokens": 256,\n },\n )\n\n print(f"{'=' * 60}")\n print(f" {label}")\n print(f"{'=' * 60}")\n print(json.dumps(response, indent=2))\n\n\ntest_tool_calling(MODEL_NAME, "BASE MODEL (before fine-tuning)")\ntest_tool_calling(ADAPTER_NAME, "FINE-TUNED MODEL (after LoRA)")\n" + }, + { + "type": "markdown", + "source": "## 7. Evaluate with Tool Calling Metrics\n\nUse the platform's evaluation service to systematically measure tool calling accuracy on the test dataset. The [Tool Calling metric](../evaluator/metrics/agentic.md) produces two scores:\n\n- **`function_name_accuracy`** -- Did the model call the correct function?\n- **`function_name_and_args_accuracy`** -- Did it call the correct function with the correct arguments?\n\n### Create Metric Entity", + "source_html": "

7. Evaluate with Tool Calling Metrics

\n

Use the platform's evaluation service to systematically measure tool calling accuracy on the test dataset. The Tool Calling metric produces two scores:

\n
    \n
  • function_name_accuracy -- Did the model call the correct function?
    • \n
  • function_name_and_args_accuracy -- Did it call the correct function with the correct arguments?
    • \n
\n

Create Metric Entity

\n" + }, + { + "type": "code", + "source": "METRIC_NAME = \"tool-calling-accuracy\"\n\ntry:\n client.evaluation.metrics.create(\n name=METRIC_NAME,\n type=\"tool-calling\",\n reference=\"{% raw %}{{ tool_calls }}{% endraw %}\",\n )\n print(f\"Created metric: {METRIC_NAME}\")\nexcept ConflictError:\n print(f\"Metric '{METRIC_NAME}' already exists, continuing...\")", + "language": "python", + "source_html": "METRIC_NAME = "tool-calling-accuracy"\n\ntry:\n client.evaluation.metrics.create(\n name=METRIC_NAME,\n type="tool-calling",\n reference="{% raw %}{{ tool_calls }}{% endraw %}",\n )\n print(f"Created metric: {METRIC_NAME}")\nexcept ConflictError:\n print(f"Metric '{METRIC_NAME}' already exists, continuing...")\n" + }, + { + "type": "markdown", + "source": "### Launch Evaluation Jobs\n\nSubmit evaluation jobs for both the fine-tuned adapter and the base model. Both jobs start running on the platform in parallel.", + "source_html": "

Launch Evaluation Jobs

\n

Submit evaluation jobs for both the fine-tuned adapter and the base model. Both jobs start running on the platform in parallel.

\n" + }, + { + "type": "code", + "source": "from nemo_platform.types.evaluation import (\n MetricOnlineJobParam,\n EvaluationParamsParam,\n)\n\nft_eval_job = client.evaluation.metric_jobs.create(\n spec=MetricOnlineJobParam(\n metric=f\"{WORKSPACE}/{METRIC_NAME}\",\n model=f\"{WORKSPACE}/{ADAPTER_NAME}\",\n prompt_template={\n \"messages\": \"{% raw %}{{ messages | tojson }}{% endraw %}\",\n \"tools\": \"{% raw %}{{ tools | tojson }}{% endraw %}\",\n \"tool_choice\": \"auto\",\n },\n dataset=f\"{WORKSPACE}/{EVAL_DATASET_NAME}\",\n params=EvaluationParamsParam(parallelism=8),\n ),\n)\nprint(f\"Fine-tuned model eval job: {ft_eval_job.name}\")\n\nbase_eval_job = client.evaluation.metric_jobs.create(\n spec=MetricOnlineJobParam(\n metric=f\"{WORKSPACE}/{METRIC_NAME}\",\n model=f\"{WORKSPACE}/{MODEL_NAME}\",\n prompt_template={\n \"messages\": \"{% raw %}{{ messages | tojson }}{% endraw %}\",\n \"tools\": \"{% raw %}{{ tools | tojson }}{% endraw %}\",\n \"tool_choice\": \"auto\",\n },\n dataset=f\"{WORKSPACE}/{EVAL_DATASET_NAME}\",\n params=EvaluationParamsParam(parallelism=8),\n ),\n)\nprint(f\"Base model eval job: {base_eval_job.name}\")", + "language": "python", + "source_html": "from nemo_platform.types.evaluation import (\n MetricOnlineJobParam,\n EvaluationParamsParam,\n)\n\nft_eval_job = client.evaluation.metric_jobs.create(\n spec=MetricOnlineJobParam(\n metric=f"{WORKSPACE}/{METRIC_NAME}",\n model=f"{WORKSPACE}/{ADAPTER_NAME}",\n prompt_template={\n "messages": "{% raw %}{{ messages | tojson }}{% endraw %}",\n "tools": "{% raw %}{{ tools | tojson }}{% endraw %}",\n "tool_choice": "auto",\n },\n dataset=f"{WORKSPACE}/{EVAL_DATASET_NAME}",\n params=EvaluationParamsParam(parallelism=8),\n ),\n)\nprint(f"Fine-tuned model eval job: {ft_eval_job.name}")\n\nbase_eval_job = client.evaluation.metric_jobs.create(\n spec=MetricOnlineJobParam(\n metric=f"{WORKSPACE}/{METRIC_NAME}",\n model=f"{WORKSPACE}/{MODEL_NAME}",\n prompt_template={\n "messages": "{% raw %}{{ messages | tojson }}{% endraw %}",\n "tools": "{% raw %}{{ tools | tojson }}{% endraw %}",\n "tool_choice": "auto",\n },\n dataset=f"{WORKSPACE}/{EVAL_DATASET_NAME}",\n params=EvaluationParamsParam(parallelism=8),\n ),\n)\nprint(f"Base model eval job: {base_eval_job.name}")\n" + }, + { + "type": "markdown", + "source": "### Wait for Model Evaluation Jobs", + "source_html": "

Wait for Model Evaluation Jobs

\n" + }, + { + "type": "code", + "source": "ft_result = wait_for_job(\n poll_fn=lambda: client.evaluation.metric_jobs.retrieve(name=ft_eval_job.name),\n label=\"Fine-tuned eval\",\n)\nprint(ft_result)\n\nbase_result = wait_for_job(\n poll_fn=lambda: client.evaluation.metric_jobs.retrieve(name=base_eval_job.name),\n label=\"Base eval\",\n)\nprint(base_result)", + "language": "python", + "source_html": "ft_result = wait_for_job(\n poll_fn=lambda: client.evaluation.metric_jobs.retrieve(name=ft_eval_job.name),\n label="Fine-tuned eval",\n)\nprint(ft_result)\n\nbase_result = wait_for_job(\n poll_fn=lambda: client.evaluation.metric_jobs.retrieve(name=base_eval_job.name),\n label="Base eval",\n)\nprint(base_result)\n" + }, + { + "type": "markdown", + "source": "### Compare Results", + "source_html": "

Compare Results

\n" + }, + { + "type": "code", + "source": "def get_scores(eval_job) -> dict:\n \"\"\"Download aggregate scores from a completed evaluation job.\"\"\"\n result = client.evaluation.metric_jobs.results.aggregate_scores.download(\n job=eval_job.name,\n )\n scores = {}\n for score in result.scores:\n scores[score.name] = score.mean\n return scores\n\n\nft_scores = get_scores(ft_eval_job)\nbase_scores = get_scores(base_eval_job)\n\nprint(\"=\" * 70)\nprint(f\"{'Metric':<40} {'Base Model':>12} {'Fine-Tuned':>12}\")\nprint(\"=\" * 70)\n\nfor metric_name in [\"function_name_accuracy\", \"function_name_and_args_accuracy\"]:\n base_val = base_scores.get(metric_name, \"N/A\")\n ft_val = ft_scores.get(metric_name, \"N/A\")\n\n base_str = f\"{base_val:.3f}\" if isinstance(base_val, (int, float)) else str(base_val)\n ft_str = f\"{ft_val:.3f}\" if isinstance(ft_val, (int, float)) else str(ft_val)\n\n print(f\"{metric_name:<40} {base_str:>12} {ft_str:>12}\")\n\nprint(\"=\" * 70)\n\nif isinstance(ft_scores.get(\"function_name_accuracy\"), (int, float)) and isinstance(\n base_scores.get(\"function_name_accuracy\"), (int, float)\n):\n improvement = ft_scores[\"function_name_accuracy\"] - base_scores[\"function_name_accuracy\"]\n print(f\"\\nFunction name accuracy improvement: {improvement:+.3f}\")", + "language": "python", + "source_html": "def get_scores(eval_job) -> dict:\n """Download aggregate scores from a completed evaluation job."""\n result = client.evaluation.metric_jobs.results.aggregate_scores.download(\n job=eval_job.name,\n )\n scores = {}\n for score in result.scores:\n scores[score.name] = score.mean\n return scores\n\n\nft_scores = get_scores(ft_eval_job)\nbase_scores = get_scores(base_eval_job)\n\nprint("=" * 70)\nprint(f"{'Metric':<40} {'Base Model':>12} {'Fine-Tuned':>12}")\nprint("=" * 70)\n\nfor metric_name in ["function_name_accuracy", "function_name_and_args_accuracy"]:\n base_val = base_scores.get(metric_name, "N/A")\n ft_val = ft_scores.get(metric_name, "N/A")\n\n base_str = f"{base_val:.3f}" if isinstance(base_val, (int, float)) else str(base_val)\n ft_str = f"{ft_val:.3f}" if isinstance(ft_val, (int, float)) else str(ft_val)\n\n print(f"{metric_name:<40} {base_str:>12} {ft_str:>12}")\n\nprint("=" * 70)\n\nif isinstance(ft_scores.get("function_name_accuracy"), (int, float)) and isinstance(\n base_scores.get("function_name_accuracy"), (int, float)\n):\n improvement = ft_scores["function_name_accuracy"] - base_scores["function_name_accuracy"]\n print(f"\\nFunction name accuracy improvement: {improvement:+.3f}")\n" + }, + { + "type": "markdown", + "source": "## Done!\n\nNext Steps:\n- **Hyperparameter tuning**: See the [Training Configuration Reference](../customizer/manage-customization-jobs/hyperparameters.md) for all available hyperparameters, valid ranges, and tuning guidance.\n- **Troubleshooting**:\n - If you run into issues with customization jobs see [Troubleshooting Customizer](../troubleshooting/customizer.md).\n - If you run into issues with evaluation jobs see [Troubleshooting Evaluator](../troubleshooting/evaluator.md).", + "source_html": "

Done!

\n

Next Steps:

\n\n" + } +] }; From 3b94676dad6ae2bc5a8667a82d80fa7a62bb3649 Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Fri, 22 May 2026 10:50:05 -0400 Subject: [PATCH 06/26] docs(fern-migrate): mount nav at docs/fern/versions/latest.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the Fern nav YAML so the converted pages actually show up in the sidebar. Every nav entry repoints from the legacy `./latest/pages/.mdx` shape to `../../.mdx`, matching the docs/
/ layout established in C2/C3. Also whitelists fern/versions/latest.yml in docs/.gitignore — the existing `*latest*` rule was meant to ignore mkdocs build artifacts and was hiding the fern nav file. Dropped one stale Studio > Optimizations entry whose underlying docs/studio/optimizations.mdx wasn't part of the squash on NVIDIA-NeMo/nemo-platform main yet. Validation: - `npx fern check`: 0 errors, 1 warning (pre-existing accent contrast). - `validate-mdx`: 193 MDX files parse cleanly. Signed-off-by: Lawrence Lane --- docs/.gitignore | 1 + docs/fern/versions/latest.yml | 290 ++++++++++++++++++++++++++++++++++ 2 files changed, 291 insertions(+) create mode 100644 docs/fern/versions/latest.yml diff --git a/docs/.gitignore b/docs/.gitignore index 464ca612883..b07258ad637 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -22,6 +22,7 @@ _test_serve/ *.7z *.tmp *latest* +!fern/versions/latest.yml _static/versions1.json helm/_values.yaml diff --git a/docs/fern/versions/latest.yml b/docs/fern/versions/latest.yml new file mode 100644 index 00000000000..d25e17d4464 --- /dev/null +++ b/docs/fern/versions/latest.yml @@ -0,0 +1,290 @@ +navigation: +- section: Documentation + contents: + - page: Home + path: ../../index.mdx + - section: Get Started + path: ../../get-started/setup.mdx + contents: + - section: Core Concepts + path: ../../get-started/concepts/index.mdx + contents: + - page: Entities + path: ../../get-started/concepts/entities.mdx + - page: Entity References + path: ../../get-started/concepts/entity-references.mdx + - page: Filtering + path: ../../get-started/concepts/filtering.mdx + - page: Manage Files + path: ../../get-started/concepts/manage-files.mdx + - page: Manage Secrets + path: ../../get-started/concepts/manage-secrets.mdx + - page: Projects + path: ../../get-started/concepts/projects.mdx + - page: Workspaces + path: ../../get-started/concepts/workspaces.mdx + - section: Studio + path: ../../studio/index.mdx + contents: + - page: Agents + path: ../../studio/agents.mdx + - page: Monitor + path: ../../studio/monitor.mdx + - page: Suggestions + path: ../../studio/suggestions.mdx + - section: Models and Inference + path: ../../run-inference/about.mdx + contents: + - section: Tutorials + path: ../../run-inference/tutorials/index.mdx + contents: + - page: Deploy Models + path: ../../run-inference/tutorials/deploy-models.mdx + - page: Run Inference + path: ../../run-inference/tutorials/run-inference.mdx + - section: Agents + path: ../../agents/index.mdx + contents: + - page: Optimize Agents + path: ../../agents/optimization.mdx + - page: Secure Agents + path: ../../agents/security.mdx + - page: Plugins and Skills + path: ../../agents/plugins.mdx + - section: Design Synthetic Data + path: ../../data-designer/index.mdx + contents: + - page: Execution Modes + path: ../../data-designer/execution-modes.mdx + - page: CLI + path: ../../data-designer/cli.mdx + - section: Tutorials + path: ../../data-designer/tutorials/index.mdx + contents: + - page: The Basics + path: ../../data-designer/tutorials/basics.mdx + - page: Seeding with External Datasets + path: ../../data-designer/tutorials/seeding.mdx + - page: SDK Resources + path: ../../data-designer/sdk-resources.mdx + - page: Migrating from Standalone Library + path: ../../data-designer/migration.mdx + - section: Anonymize Data + path: ../../anonymizer/index.mdx + contents: + - page: Quickstart + path: ../../anonymizer/quickstart.mdx + - section: Tutorials + path: ../../anonymizer/tutorials/index.mdx + contents: + - page: Preview a Config + path: ../../anonymizer/tutorials/preview.mdx + - page: Run an Anonymizer Job + path: ../../anonymizer/tutorials/run.mdx + - page: SDK Resources + path: ../../anonymizer/sdk-resources.mdx + - page: CLI Reference + path: ../../anonymizer/cli.mdx + - section: Guardrail Models + path: ../../guardrails/index.mdx + contents: + - section: Core Concepts + path: ../../guardrails/concepts/index.mdx + contents: + - page: Architecture + path: ../../guardrails/concepts/architecture.mdx + - section: Configurations + path: ../../guardrails/concepts/configurations/index.mdx + contents: + - page: Configuration Structure + path: ../../guardrails/concepts/configurations/configuration-structure.mdx + - page: Default Configurations + path: ../../guardrails/concepts/configurations/default-configs.mdx + - page: Manage Configurations + path: ../../guardrails/concepts/configurations/manage-configs.mdx + - page: Running Inference + path: ../../guardrails/concepts/inference.mdx + - page: Running Checks + path: ../../guardrails/concepts/checks.mdx + - section: Tutorials + path: ../../guardrails/tutorials/index.mdx + contents: + - page: Content Safety + path: ../../guardrails/tutorials/content-safety.mdx + - page: Deploy NemoGuard NIMs + path: ../../guardrails/tutorials/deploy-nemoguard-nims.mdx + - page: Injection Detection + path: ../../guardrails/tutorials/injection-detection.mdx + - page: Multimodal Data + path: ../../guardrails/tutorials/multimodal-data.mdx + - page: Parallel Rails + path: ../../guardrails/tutorials/parallel-rails.mdx + - page: Terminology + path: ../../guardrails/terminology.mdx + - page: Observability + path: ../../guardrails/observability.mdx + - section: Evaluate Models + path: ../../evaluator/index.mdx + contents: + - section: Tutorials + path: ../../evaluator/tutorials/index.mdx + contents: + - page: Run LLM-as-a-Judge Evaluation + path: ../../evaluator/tutorials/run-llm-judge-evaluation.mdx + - page: SDK Resources + path: ../../evaluator/sdk-resources.mdx + - section: Metrics + path: ../../evaluator/metrics/index.mdx + contents: + - page: Manage Metrics + path: ../../evaluator/metrics/manage-metrics.mdx + - page: LLM-as-a-Judge + path: ../../evaluator/metrics/llm-as-a-judge.mdx + - page: RAG Metrics + path: ../../evaluator/metrics/rag.mdx + - page: Similarity Metrics + path: ../../evaluator/metrics/similarity.mdx + - page: Agentic Metrics + path: ../../evaluator/metrics/agentic.mdx + - page: Bring Your Own Metric + path: ../../evaluator/metrics/remote.mdx + - page: Agent Configuration + path: ../../evaluator/metrics/agent-configuration.mdx + - page: Model Configuration + path: ../../evaluator/metrics/model-configuration.mdx + - page: Job Management + path: ../../evaluator/metrics/job-management.mdx + - page: Metric Results + path: ../../evaluator/metrics/results.mdx + - section: Benchmarks + contents: + - page: Industry Benchmarks + path: ../../evaluator/benchmarks/industry.mdx + - page: Agentic Benchmarks + path: ../../evaluator/benchmarks/agentic.mdx + - page: Custom Benchmarks + path: ../../evaluator/benchmarks/custom.mdx + - page: HuggingFace Secret + path: ../../evaluator/benchmarks/hf-secret.mdx + - section: Fine-tune Models + contents: + - section: Tutorials + path: ../../customizer/tutorials/index.mdx + contents: + - page: Format Training Dataset + path: ../../customizer/tutorials/format-training-dataset.mdx + - page: Full SFT Customization + path: ../../customizer/tutorials/sft-customization-job.mdx + - page: LoRA Model Customization + path: ../../customizer/tutorials/lora-customization-job.mdx + - page: DPO Customization + path: ../../customizer/tutorials/dpo-customization-job.mdx + - page: Knowledge Distillation Customization + path: ../../customizer/tutorials/distillation-customization-job.mdx + - page: Embedding Model Customization + path: ../../customizer/tutorials/embedding-customization-job.mdx + - page: Optimize for Tokens/GPU Throughput + path: ../../customizer/tutorials/optimize-throughput.mdx + - section: Models + contents: + - page: Embedding + path: ../../customizer/models/embedding.mdx + - section: Manage Customization Jobs + path: ../../customizer/manage-customization-jobs/index.mdx + contents: + - page: Create a Customization Job + path: ../../customizer/manage-customization-jobs/create-job.mdx + - section: Manage Model Entities + contents: + - page: Create a Model FileSet + path: ../../customizer/manage-model-entities/create-fileset.mdx + - section: Vulnerability Scanning + path: ../../auditor/index.mdx + contents: + - section: Tutorials + path: ../../auditor/tutorials/index.mdx + contents: + - page: Run an Audit Locally + path: ../../auditor/tutorials/run-audit-locally.mdx + - page: SDK Resources + path: ../../auditor/sdk-resources.mdx + - section: Configurations + path: ../../auditor/configs/index.mdx + contents: + - page: Selecting Probes + path: ../../auditor/configs/probes.mdx + - page: Schema + path: ../../auditor/configs/schema.mdx + - section: Targets + path: ../../auditor/targets/index.mdx + contents: + - page: Inference Gateway + path: ../../auditor/targets/inference-gateway.mdx + - page: Schema + path: ../../auditor/targets/schema.mdx + - section: Safe Synthesizer + contents: + - section: Tutorials + contents: + - page: Safe Synthesizer 101 + path: ../../safe-synthesizer/tutorials/safe-synthesizer-101.mdx + - section: Example Applications + contents: + - page: Tool Calling + path: ../../example-applications/tool-calling.mdx + - section: Reference + contents: + - section: Release Notes + path: ../../about/release-notes/index.mdx + contents: + - page: Current Release + path: ../../about/release-notes/current-release.mdx + - page: System Requirements + path: ../../requirements.mdx + - page: Support Matrix + path: ../../support-matrix.mdx + - page: API Reference + path: ../../api/index.mdx + - section: Python SDK + path: ../../pysdk/index.mdx + contents: + - page: Client APIs + path: ../../pysdk/client/index.mdx + - section: CLI Reference + path: ../../cli/index.mdx + contents: + - page: Configuration + path: ../../cli/configuration.mdx + - page: Working with Resources + path: ../../cli/working-with-resources.mdx + - page: Full CLI Reference + path: ../../cli/reference.mdx + - page: Troubleshooting + path: ../../cli/troubleshooting.mdx + - page: Configuration Reference + path: ../../set-up/config-reference.mdx + - section: Authorization + contents: + - page: Permissions Reference + path: ../../auth/authorization/permissions-reference.mdx + - section: Troubleshooting + path: ../../troubleshooting/index.mdx + contents: + - page: Customizer + path: ../../troubleshooting/customizer.mdx + - page: Data Designer + path: ../../troubleshooting/data-designer.mdx + - page: Evaluator + path: ../../troubleshooting/evaluator.mdx + - page: Guardrails + path: ../../troubleshooting/guardrails.mdx + - page: Studio + path: ../../troubleshooting/studio.mdx + - section: Contributing + contents: + - page: Skills Spec + path: ../../contributing/skills-spec.mdx + - page: EULA + path: ../../eula.mdx + - page: Acknowledgements + path: ../../acknowledgements/index.mdx From 543b466059bb5b19746fe4dc34a3caa653b68cc0 Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Fri, 22 May 2026 11:17:09 -0400 Subject: [PATCH 07/26] docs(fern): switch to NVIDIA global-theme; drop bespoke brand assets Use `global-theme: nvidia` from NVIDIA/fern-components for shared colors, footer, logos, layout, and Adobe DTM JS. Keep only a partial `logo: { right-text: NeMo Platform }` override and the functional notebook-viewer/button CSS. Set `multi-source: true` on the instance to match Curator's pattern. Repoint navbar GitHub link to the new NVIDIA-NeMo/nemo-platform repo. Co-Authored-By: Claude Opus 4.7 Signed-off-by: Lawrence Lane --- docs/fern/assets/NVIDIA_dark.svg | 37 -- docs/fern/assets/NVIDIA_light.svg | 36 - docs/fern/assets/NVIDIA_symbol.svg | 24 - docs/fern/assets/favicon.ico | 3 - docs/fern/assets/nvidia-logo-white.png | 3 - docs/fern/components/CustomFooter.tsx | 96 --- docs/fern/docs.yml | 31 +- docs/fern/main.css | 872 ------------------------- 8 files changed, 3 insertions(+), 1099 deletions(-) delete mode 100644 docs/fern/assets/NVIDIA_dark.svg delete mode 100644 docs/fern/assets/NVIDIA_light.svg delete mode 100644 docs/fern/assets/NVIDIA_symbol.svg delete mode 100644 docs/fern/assets/favicon.ico delete mode 100644 docs/fern/assets/nvidia-logo-white.png delete mode 100644 docs/fern/components/CustomFooter.tsx delete mode 100644 docs/fern/main.css diff --git a/docs/fern/assets/NVIDIA_dark.svg b/docs/fern/assets/NVIDIA_dark.svg deleted file mode 100644 index fe67b898bfd..00000000000 --- a/docs/fern/assets/NVIDIA_dark.svg +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/fern/assets/NVIDIA_light.svg b/docs/fern/assets/NVIDIA_light.svg deleted file mode 100644 index 568ee177ba0..00000000000 --- a/docs/fern/assets/NVIDIA_light.svg +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/fern/assets/NVIDIA_symbol.svg b/docs/fern/assets/NVIDIA_symbol.svg deleted file mode 100644 index fd57037f326..00000000000 --- a/docs/fern/assets/NVIDIA_symbol.svg +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/docs/fern/assets/favicon.ico b/docs/fern/assets/favicon.ico deleted file mode 100644 index cea91cdbfb4..00000000000 --- a/docs/fern/assets/favicon.ico +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bedcb14e0470ff8abe26405a630e9604ac4fa581b385a7cfb00420c22f541b6e -size 25214 diff --git a/docs/fern/assets/nvidia-logo-white.png b/docs/fern/assets/nvidia-logo-white.png deleted file mode 100644 index f6c73aed43f..00000000000 --- a/docs/fern/assets/nvidia-logo-white.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:00b499146c0d590c254ff8ab555298cfa88605938bf883a880a58067eccdb571 -size 47302 diff --git a/docs/fern/components/CustomFooter.tsx b/docs/fern/components/CustomFooter.tsx deleted file mode 100644 index db2d9da6b0b..00000000000 --- a/docs/fern/components/CustomFooter.tsx +++ /dev/null @@ -1,96 +0,0 @@ -/** - * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: LicenseRef-NvidiaProprietary - */ - -/** - * Custom footer for NVIDIA docs (Fern native header/footer). - * Markup and class names match the original custom-app footer 1:1 so that - * fern/main.css (footer + Built with Fern styles) applies correctly: - * dark mode logo, responsive layout, and Built with Fern tooltip. - */ -export default function CustomFooter() { - const currentYear = new Date().getFullYear(); - const logoUrl = - "https://fern-image-hosting.s3.us-east-1.amazonaws.com/nvidia/NVIDIA_Logo_0.svg"; - - return ( - - ); -} diff --git a/docs/fern/docs.yml b/docs/fern/docs.yml index 1b1fcb9978e..f6296223ce2 100644 --- a/docs/fern/docs.yml +++ b/docs/fern/docs.yml @@ -1,41 +1,16 @@ instances: - url: nemo-platform.docs.buildwithfern.com + multi-source: true title: NVIDIA NeMo Platform -footer: ./components/CustomFooter.tsx -layout: - searchbar-placement: header - page-width: 1376px - sidebar-width: 248px - content-width: 812px - tabs-placement: header - hide-feedback: true -colors: - accentPrimary: - dark: '#76B900' - light: '#76B900' - background: - light: '#FFFFFF' - dark: '#000000' -theme: - page-actions: toolbar - footer-nav: minimal +global-theme: nvidia logo: - dark: ./assets/NVIDIA_dark.svg - light: ./assets/NVIDIA_light.svg - height: 20 - href: / right-text: NeMo Platform -favicon: ./assets/NVIDIA_symbol.svg -js: -- url: https://assets.adobedtm.com/5d4962a43b79/c1061d2c5e7b/launch-191c2462b890.min.js - strategy: beforeInteractive css: -- ./main.css - ./styles/notebook-viewer.css - ./styles/button.css navbar-links: - type: github - value: https://github.com/NVIDIA-NeMo/Platform + value: https://github.com/NVIDIA-NeMo/nemo-platform experimental: mdx-components: - ./components diff --git a/docs/fern/main.css b/docs/fern/main.css deleted file mode 100644 index 936c895aaa3..00000000000 --- a/docs/fern/main.css +++ /dev/null @@ -1,872 +0,0 @@ -/*! - * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: LicenseRef-NvidiaProprietary - * - * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual - * property and proprietary rights in and to this material, related - * documentation and any modifications thereto. Any use, reproduction, - * disclosure or distribution of this material and related documentation - * without an express license agreement from NVIDIA CORPORATION or - * its affiliates is strictly prohibited. - */ - -/* Color themes for light and dark modes */ -:root { - /* Brand Colors */ - --nv-color-green: #74B900; - --nv-color-green-2: #004B31; - --nv-color-black: #000000; - --nv-color-white: #FFFFFF; - - /* Grey Scale - Light */ - --nv-light-grey-1: #f7f7f7; - --nv-light-grey-2: #EEEEEE; - --nv-light-grey-3: #DDDDDD; - --nv-light-grey-4: #CCCCCC; - --nv-light-grey-5: #999999; - - /* Grey Scale - Dark */ - --nv-dark-grey-1: #111111; - --nv-dark-grey-2: #1A1A1A; - --nv-dark-grey-3: #222222; - --nv-dark-grey-4: #333333; - --nv-dark-grey-5: #666666; - - /* Colors by Usage */ - --nv-color-text: #000000; - --nv-color-bg-default: #FFFFFF; - --nv-color-bg-alt: #f7f7f7; - --nv-color-success: #76B900; - --nv-color-error: #f44336; - - /* Theme-independent settings */ - --rounded: 999px; -} -main { - min-height: calc(100vh - 200px); - } -/* Typography - Headers */ -h1 { - font-size: 36px; - font-weight: 700; - line-height: 1.25em; /* 45px */ -} - -h2 { - font-size: 28px; - font-weight: 700; - line-height: 1.25em; /* 35px */ -} - -h3 { - font-size: 24px; - font-weight: 700; - line-height: 1.25em; /* 30px */ -} - -h4 { - font-size: 20px; - font-weight: 700; - line-height: 1.25em; /* 25px */ -} - -/* Typography - Paragraphs */ -.prose{ - color: var(--nv-dark-grey-2) !important; -} -.dark .prose{ - color: var(--nv-light-grey-2) !important; -} -p { - text-decoration-thickness: 3px; -} -.fern-mdx-link { - color: var(--tw-prose-body); - text-decoration-color: var(--accent); - font-weight: var(--font-weight-normal); -} - -/* Badge links: hide redundant external-link icon (badges already indicate links) */ -.badge-links .fern-mdx-link svg { - display: none; -} - -/* Light theme (default) */ -html:not([data-theme]),html[data-theme=light] { - --pst-color-background: #fff; - --pst-color-on-background: #fff; - --pst-color-shadow: #ccc; - --pst-color-heading: #000; - --pst-color-text-base: #1a1a1a; - --pst-color-text-muted: #666; - --pst-color-surface: #f7f7f7; - --pst-color-on-surface: #333; - --pst-color-primary: var(--nv-color-green-2); - --pst-color-table-row-hover-bg: var(--nv-color-green); - --pst-color-link: var(--pst-color-text-base); - --pst-color-link-hover: var(--pst-color-text-base); - --pst-color-inline-code: var(--pst-color-primary); - --pst-color-inline-code-links: var(--pst-color-primary); - --pst-color-secondary: var(--pst-color-primary); - --pst-color-secondary-bg: var(--nv-color-green); - --pst-color-accent: var(--nv-color-green); -} - -/* Dark theme */ -html[data-theme=dark] { - --pst-color-background: #111; - --pst-color-on-background: #000; - --pst-color-shadow: #000; - --pst-color-heading: #fff; - --pst-color-text-base: #eee; - --pst-color-text-muted: #999; - --pst-color-surface: #1a1a1a; - --pst-color-on-surface: #ddd; - --pst-color-primary: var(--nv-color-green); - --pst-color-table-row-hover-bg: var(--nv-color-green-2); - --pst-color-link: var(--pst-color-text-base); - --pst-color-link-hover: var(--pst-color-text-base); - --pst-color-inline-code: var(--pst-color-primary); - --pst-color-inline-code-links: var(--pst-color-primary); - --pst-color-secondary: var(--pst-color-primary); - --pst-color-secondary-bg: var(--nv-color-green-2); - --pst-color-accent: var(--nv-color-green); -} - -/* Product and verion selector styling */ - -.fern-product-selector { - border-radius: 8px; - pointer-events: none !important; - padding-right: 2px; -} - -.product-dropdown-trigger svg{ - display: none !important; -} - -.fern-product-selector .product-dropdown-trigger p{ - font-weight: bold !important; -} -.fern-product-selector-radio-group { - display: grid; - grid-template-columns: repeat(3, 1fr); - gap: 8px; - max-width: 1000px; -} - -@media (max-width: 768px) { - .fern-product-selector-radio-group { - grid-template-columns: repeat(2, 1fr); - } -} -.fern-version-selector { - transform: translateY(-1px); -} - -.fern-version-selector .version-dropdown-trigger{ - outline: 1px solid var(--border, var(--grayscale-a5)) !important; - border-radius: 5px; - transition: box-shadow 0.3s ease, outline 0.3s ease; -} -.product-dropdown-trigger{ - padding-left: 0px !important; -} - -.product-dropdown-trigger, .version-dropdown-trigger{ - background-color: transparent !important; -} -.product-dropdown-trigger svg:hover{ - stroke: var(--nv-color-green) !important; -} -.version-dropdown-trigger:hover{ - box-shadow: 0 0 0 1px var(--nv-color-green) !important; -} -.version-dropdown-trigger svg:hover{ - stroke: var(--nv-color-green) !important; -} -/* Sidebar styling */ -#fern-sidebar { - border-right: 1px solid var(--border, var(--grayscale-a5)) !important; - height: 100vh !important; -} -.fern-sidebar-link:not(:hover){ - background-color: transparent !important; -} -.fern-sidebar-link { - padding-left: 1rem !important; - padding-right: 1rem !important; - padding-top: 0.5rem !important; - padding-bottom: 0.5rem !important; - border-radius: 0px !important; - &.nested { - padding-left: 1rem !important; - } -} -/* Section-level sidebar links (pages that have children) should match sidebar heading padding */ -.fern-sidebar-group > li > .fern-sidebar-link:has(+ .fern-sidebar-group) { - padding-left: 0.25rem !important; -} -.fern-sidebar-group{ - padding: 0 !important -} -#fern-sidebar-scroll-area{ - padding-right: 0 !important -} - -/* header styling */ -.fern-header-content{ - padding-left: 18.5px; - margin-top: -5px; - margin-bottom: -5px; -} -#fern-header { - border-color: var(--border, var(--grayscale-a5)) !important; -} -@keyframes header-background-fade { - 0% { - background-color: transparent; - } - 100% { - background-color: var(--header-background); - } - } - -[data-theme=default]#fern-header { -animation: header-background-fade linear; -animation-timeline: scroll(); -animation-range: 0 50px; -} -.fern-header-navbar-links .fern-button{ - background-color: transparent !important; -} -.fern-header-navbar-links > button{ - background-color: transparent !important; -} -.fern-header-logo-container > div > div > a > img{ - padding-right: 0.5rem; -} -.fern-header-logo-container .font-heading{ - font-size: 16px !important; - font-weight: bold !important; - color: var(--grayscale-a12) !important; - border-inline: 1px solid var(--border, var(--grayscale-a5)); - padding: 15px 1rem; - margin: -20px 0.5rem; -} -@media (max-width: 1024px) { - .fern-header-logo-container .font-heading{ - display: none !important; - } -} -/* Search bar styling */ -#fern-search-button{ - background-color: transparent !important; - border-radius: var(--rounded); - transition: box-shadow 0.3s ease, outline 0.3s ease; -} -#fern-search-button:hover{ - box-shadow: 0 0 0 1px var(--nv-color-green) !important; -} -#fern-search-button .fern-kbd{ - display: none; -} - -.fern-layout-footer-toolbar button{ - background-color: transparent !important; - border-color: transparent !important; - padding-inline: 0px !important; -} - -/* ========== Custom footer (native React component) – 1:1 with original ========== */ -.bd-footer { - border-top: 1px solid var(--border, var(--grayscale-a5)) !important; - font-family: NVIDIA, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif !important; - font-size: 0.875rem; - padding: 2rem 0; - width: 100%; -} -.bd-footer * { - font-family: inherit; -} -.bd-footer__inner { - padding: 0 2rem; -} -.footer-items__start { - display: flex; - flex-direction: column; - gap: 1.5rem; -} -.footer-logos-container { - display: flex; - align-items: center; - justify-content: space-between; - width: 100%; - gap: 1rem; -} -.footer-brand { - display: inline-block; - text-decoration: none; -} -.footer-brand .logo__image { - height: 24px; - width: auto; - transition: opacity 0.2s ease; -} -.footer-brand:hover .logo__image { - opacity: 0.8; -} -.footer-brand-fern { - display: flex; - align-items: center; - margin-left: auto; -} -/* Logo theme visibility – .dark is on ancestor in Fern */ -.only-light { - display: block; - filter: invert(1); -} -.only-dark { - display: none; -} -.dark .only-light { - display: none; -} -.dark .only-dark { - display: block; - filter: none; -} -.footer-links { - display: flex; - flex-wrap: wrap; - gap: 0.25rem 0.5rem; - line-height: 1.65; - margin: 0; - padding: 0; -} -.footer-links a { - color: var(--grayscale-a11); - text-decoration: none; - transition: color 0.2s ease; - white-space: nowrap; -} -.pipe-separator { - color: var(--grayscale-a11); - white-space: nowrap; -} -.copyright { - color: var(--grayscale-a11); - font-size: 0.875rem; - line-height: 1.65; - margin: 0; -} -@media (max-width: 768px) { - .bd-footer { padding: 1.5rem 0; } - .bd-footer__inner { padding: 0 1.5rem; } - .footer-items__start { gap: 1rem; } - .footer-links { flex-direction: row; gap: 0.5rem 0.75rem; } - .footer-links a { white-space: normal; word-break: break-word; } -} -@media (max-width: 480px) { - .footer-links { gap: 0.5rem; } - .footer-links a { font-size: 0.8125rem; } - .copyright { font-size: 0.8125rem; } -} -/* Built with Fern link + tooltip */ -.built-with-fern-link { - display: flex; - align-items: baseline; - gap: 0.25rem; - text-decoration: none; - position: relative; -} -.built-with-fern-logo { - height: 1rem; - margin: 0; - transition: filter 150ms ease; -} -.built-with-fern-logo path { fill: var(--grayscale-a12); } -.built-with-fern-link:hover .built-with-fern-logo { filter: saturate(1) opacity(1); } -.built-with-fern-link:hover .built-with-fern-logo path:nth-child(2) { fill: #51C233; } -.built-with-fern-tooltip { - position: absolute; - top: 50%; - right: calc(100%); - bottom: auto; - left: auto; - transform: translateY(-50%); - margin: 0; - margin-right: 0.5rem; - padding: 0.5rem 0.75rem; - background-color: #FFFFFF; - color: #000000; - font-size: 0.85rem; - border-radius: 0.375rem; - border: 1px solid var(--grayscale-a5); - white-space: nowrap; - pointer-events: none; - opacity: 0; - transition: opacity 150ms ease; - transition-delay: 0s; - z-index: 50; - box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); - width: max-content; -} -.built-with-fern-link:hover .built-with-fern-tooltip { - opacity: 1; - transition-delay: 0.75s; -} -.dark .built-with-fern-tooltip { - background-color: #000000; - color: #FFFFFF; -} -.built-with-fern-logo-dark { display: none; } -.dark .built-with-fern-logo-light { display: none; } -.dark .built-with-fern-logo-dark { display: block; } -@media (prefers-color-scheme: dark) { - .built-with-fern-logo-light { display: none; } - .built-with-fern-logo-dark { display: block; } -} - -/* Footer styling */ -.fern-footer-nav{ - border-radius: var(--rounded); - background-color: transparent !important; - transition: box-shadow 0.3s ease, outline 0.3s ease; -} -/* Hide line numbers */ -.code-block-line-gutter { - display: none !important; -} -.fern-footer-prev h4, .fern-footer-next h4{ - font-size: inherit !important; -} -.fern-sidebar-link.nested[data-state="active"]:before { - left: -0px !important; - bottom: -0px !important; - top: -0px !important; - width: 2px !important; -} -.fern-sidebar-link[data-state="active"] { - color: unset !important; -} - -.fern-selection-item .fern-selection-item-icon{ - border-color: transparent !important; -} -/* Button styling */ -.fern-button{ - border-radius: var(--rounded); - font-weight: bold; -} -.fern-button.filled.primary{ - color: var(--nv-color-black); -} -.dark .fern-button.filled.primary{ - background-color: var(--nv-color-white); -} -.dark .fern-button.filled.primary:hover{ - background-color: var(--nv-light-grey-2); -} -.fern-button.outlined.normal{ - background-color: transparent; - --tw-ring-color: transparent; - color: var(--nv-color-black); -} -.fern-button.outlined.normal:hover{ - color: var(--nv-color-green) -} -.dark .fern-button.outlined.normal{ - color: var(--nv-color-white); -} -.dark .fern-button.outlined.normal:hover{ - color: var(--nv-color-green); -} -/* Card styling */ -.fern-card{ - transition: box-shadow 0.3s ease, outline 0.3s ease; -} -svg.card-icon{ - height: 24px !important; - width: 24px !important; -} -.card-icon{ - background-color: transparent !important; -} -.fern-card:hover{ - box-shadow: 0 0 0 1px var(--nv-color-green) !important; -} -.fern-docs-badge{ - border-radius: var(--rounded); -} -.fern-page-actions button:hover{ - background-color: transparent !important; -} -.fern-page-actions a:hover{ - background-color: transparent !important; -} -/* Moving logo to footer */ -#builtwithfern, #builtwithfern * { - display: none !important; -} - -/* Landing Page Gradients */ -/* Top: Simple radial gradient (no mask, responsive) */ -.landing-gradient-top { - position: absolute; - top: 0; - left: 0; - right: 0; - height: 800px; - background: radial-gradient(ellipse 100% 100% at 50% 10%, - rgba(191, 242, 48, 0.15) 0%, - rgba(158, 228, 179, 0.12) 30%, - rgba(124, 215, 254, 0.12) 50%, - rgba(124, 215, 254, 0.06) 75%, - transparent 100%); - pointer-events: none; - z-index: 0; -} - -/* Bottom: Masked gradient for organic transition */ -.landing-gradient-bottom { - position: absolute; - bottom: -282px; - left: 0; - right: 0; - height: 1232px; - background: linear-gradient(85deg, #BFF230 41.98%, #7CD7FE 99.52%); - opacity: 0.05; - pointer-events: none; - z-index: 5; - mask-image: url('https://www.figma.com/api/mcp/asset/27509afa-9c16-46bb-8415-4395e2e5a347'); - mask-repeat: no-repeat; - mask-position: 0% -17px; - mask-size: 100% auto; - -webkit-mask-image: url('https://www.figma.com/api/mcp/asset/27509afa-9c16-46bb-8415-4395e2e5a347'); - -webkit-mask-repeat: no-repeat; - -webkit-mask-position: 0% -17px; - -webkit-mask-size: 100% auto; -} - -/* Landing Page Gradients Wrapper */ -.landing-page-gradients { - position: relative; - width: 100%; - margin-top: -100px; - padding-top: 100px; - overflow: visible; - background: #181818; -} - -/* Hero Section (Landing page only) */ -.hero-section { - position: relative; - width: 100%; - padding: 3rem 6rem; - margin: 0 auto; - overflow: visible; - display: flex; - flex-direction: column; - align-items: center; - z-index: 10; -} - -/* Hero Section Content - constrain width */ -.hero-section > * { - position: relative; - z-index: 100; - max-width: 1440px; - width: 100%; -} - -/* Tablet and Mobile: fix spacing and layout */ -@media (max-width: 1024px) { - /* Extend dark background behind header */ - .landing-page body, .landing-page html, .landing-page main { - background: #181818 !important; - } - - .landing-page-gradients { - margin-top: -100px; - padding-top: 100px; - } - - .hero-section { - padding: 2rem 2rem; - } - - .hero-section > * { - max-width: none; - } - - .hero-content-grid { - grid-template-columns: 1fr; - gap: 2rem; - } - - .hero-heading { - font-size: 36px; - } - - .hero-subtitle { - font-size: 16px; - } - - .hero-title-section { - margin-bottom: 2rem; - } -} - -/* Small mobile only */ -@media (max-width: 600px) { - .hero-heading { - font-size: 28px; - } - - .hero-section { - padding: 1.5rem 1.5rem; - } -} - -.hero-section h1, -.hero-section h2, -.hero-section h3, -.hero-section h4, -.hero-section h5, -.hero-section h6 { - pointer-events: none !important; -} -/* Hero Title Section */ -.hero-title-section { - text-align: center; - margin-bottom: 4rem; - position: relative; - z-index: 100; -} - -.hero-heading { - font-size: 48px; - font-weight: 700; - line-height: 1.2; - margin: 0 0 1rem 0; - color: var(--nv-color-white); -} - -.hero-subtitle { - font-size: 18px; - line-height: 1.5; - margin: 0; - color: var(--nv-color-white); -} - -/* Hero Content Grid */ -.hero-content-grid { - display: grid; - grid-template-columns: repeat(2, 1fr); - gap: 3rem; - align-items: start; - position: relative; - z-index: 100; -} - -.hero-column { - display: flex; - flex-direction: column; - gap: 1rem; -} - -.hero-column-title { - font-size: 24px; - font-weight: 700; - margin: 0; - color: var(--nv-color-white); -} - -.hero-column-subtitle { - font-size: 16px; - margin: 0 0 1rem 0; - color: var(--nv-color-white); -} - -/* Hero Card Container (Left Column) */ -.hero-card-container { - display: flex; - flex-direction: column; - border-radius: 8px; - overflow: hidden; - border: 1px solid var(--border, var(--grayscale-a5)); - margin-top: 1.5rem !important; - background: rgba(26, 26, 26, 0.2); - backdrop-filter: blur(6px); -} - -.hero-card-image { - width: 100%; - height: auto; - display: block; -} - -.hero-card-content { - padding: 1.5rem; - display: flex; - flex-direction: row; - gap: 1rem; - align-items: center; - justify-content: space-between; - background: rgba(26, 26, 26, 0.2); - backdrop-filter: blur(6px); -} - -.hero-card-text-wrapper { - flex: 1; -} - -.hero-card-text { - margin: 0; - font-size: 14px; - line-height: 1.5; - color: var(--nv-color-white); -} - -.hero-card-button-wrapper { - flex-shrink: 0; -} -.hero-card-button-wrapper .fern-mdx-link{ - text-decoration: none !important; -} - -.hero-card-button { - white-space: nowrap; -} - -/* Hero Cards */ - -.hero-column .fern-card { - padding: 9px 17px; - background-color: rgba(26, 26, 26, 0.2) !important; - backdrop-filter: blur(6px); -} - -.hero-section .fern-card{ - color: white !important; -} - -.hero-column .card-icon { - font-size: 64px !important; - width: 64px !important; - height: 64px !important; -} - -.hero-column .card-icon svg, -.hero-column .card-icon i { - font-size: 64px !important; - width: 64px !important; - height: 64px !important; -} - -.hero-column .fern-card-title { - font-size: 16px; - font-weight: 500; - line-height: 24px; -} - -.hero-column .fern-card p { - font-size: 14px; - line-height: 20px; - color: white !important; -} - -/* Body Section */ -.body-section { - display: flex; - padding: 4rem 16rem; - flex-direction: column; - justify-content: center; - align-items: center; - gap: 4rem; - align-self: stretch; - position: relative; - z-index: 1; - background: #181818; -} - -/* Body Section Content - constrain width */ -.body-section > * { - max-width: 1440px; - width: 100%; - position: relative; - z-index: 10; -} - -.code-block .fern-code-link{ - text-decoration: underline !important; - text-decoration-color: var(--accent) !important; - text-underline-offset: 1px !important; - text-decoration-style: underline !important; -} - -/* Mobile Styles */ -@media (max-width: 768px) { - .hero-section { - padding: 2rem 1.5rem; - } - - .hero-title-section { - margin-bottom: 2rem; - } - - .hero-heading { - font-size: 32px; - } - - .hero-subtitle { - font-size: 16px; - } - - .hero-content-grid { - grid-template-columns: 1fr; - gap: 2rem; - } - - .hero-column-title { - font-size: 20px; - } - - .hero-column-subtitle { - font-size: 14px; - } - - .hero-card-content { - flex-direction: column; - align-items: flex-start; - } - - .hero-card-button-wrapper { - align-self: flex-start; - } - - .hero-column .card-icon, - .hero-column .card-icon svg, - .hero-column .card-icon i { - font-size: 40px !important; - width: 40px !important; - height: 40px !important; - } - - .hero-column .fern-card-title { - font-size: 14px; - } - - .hero-column .fern-card p { - font-size: 11px; - } - - .body-section { - padding: 2rem 1.5rem; - } - - .fern-selection-item-icon.use-icon { - display: none !important; - } -} \ No newline at end of file From 520ed0aec3321d614e8eeffa242ebbaac5989e89 Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Fri, 22 May 2026 11:53:04 -0400 Subject: [PATCH 08/26] docs(fern): replace @/ imports with Fern-native tags for out-of-fern pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pages mounted from outside docs/fern/ (via `path: ../../foo.mdx` in versions/latest.yml) couldn't resolve `@/snippets/...` or `@/components/...` imports — Fern's bundler refuses the `..`-escape back into the fern dir, surfacing as `Could not resolve` errors at build time. Three swaps, all using Fern's documented patterns: - Snippets: `import X from "@/snippets/foo.mdx"` + `` → `` (Fern-native JSX, bypasses esbuild). 18 pages. - Components: drop `import { Button } from "@/components/Button"` — bare ` - - - - - - - - -
- - diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml deleted file mode 120000 index e9738fedff9..00000000000 --- a/docs/api/openapi.yaml +++ /dev/null @@ -1 +0,0 @@ -../../openapi/openapi.yaml \ No newline at end of file diff --git a/docs/customizer/index.mdx b/docs/customizer/index.mdx index 93969f1d169..fca8b95d18b 100644 --- a/docs/customizer/index.mdx +++ b/docs/customizer/index.mdx @@ -152,7 +152,7 @@ Learn how to optimize the token-per-GPU throughput for a LoRA optimization job. View the available hyperparameters and their valid options that you can set when creating a customization job. - + View the OpenAPI specification for Customizer. diff --git a/docs/customizer/manage-customization-jobs/customization-job-reference.mdx b/docs/customizer/manage-customization-jobs/customization-job-reference.mdx index 38bd1f27a17..a2d5958ac75 100644 --- a/docs/customizer/manage-customization-jobs/customization-job-reference.mdx +++ b/docs/customizer/manage-customization-jobs/customization-job-reference.mdx @@ -27,7 +27,7 @@ All job configuration (model, dataset, training, and output) is specified in the ## Complete API Reference -For generated REST API details, see the [Customizer API Reference](../../api/index.md#tag-customizer) and +For generated REST API details, see the [Customizer API Reference](/documentation/reference/api-reference) and search for `CustomizationJobInput`. --- diff --git a/docs/customizer/manage-customization-jobs/hyperparameters.mdx b/docs/customizer/manage-customization-jobs/hyperparameters.mdx index 4e75bc7b257..a0bad160a73 100644 --- a/docs/customizer/manage-customization-jobs/hyperparameters.mdx +++ b/docs/customizer/manage-customization-jobs/hyperparameters.mdx @@ -10,7 +10,7 @@ Want to learn about training concepts at a high level? Check out the [Customizat ## Complete Schema Reference -For generated REST API details, see the [Customizer API Reference](../../api/index.md#tag-customizer) and +For generated REST API details, see the [Customizer API Reference](/documentation/reference/api-reference) and search for `CustomizationJobInput`. Training is configured in the job's `spec.training` object. @@ -26,7 +26,7 @@ The `training` field is a discriminated union on the `type` field. Each training | `training.type` | `sft`, `dpo`, `distillation` | Training method (discriminated union) | | `training.peft` | `{ type: "lora", rank: 8, ... }` or omit | PEFT adapter configuration. If set, trains an adapter; if omitted, performs full-weight training | -For generated SFT schema details, see the [Customizer API Reference](../../api/index.md#tag-customizer) +For generated SFT schema details, see the [Customizer API Reference](/documentation/reference/api-reference) and search for `SFTTrainingInput`. ### DPO Configuration @@ -41,7 +41,7 @@ When `training.type` is `"dpo"`, additional DPO-specific fields are available: | `preference_loss_weight` | Weight for preference loss | `1.0` | | `sft_loss_weight` | Weight for SFT loss | `0.0` | -For generated DPO schema details, see the [Customizer API Reference](../../api/index.md#tag-customizer) +For generated DPO schema details, see the [Customizer API Reference](/documentation/reference/api-reference) and search for `DPOTrainingInput`. @@ -114,7 +114,7 @@ When `training.type` is `"distillation"`, additional KD-specific fields are avai | `distillation_temperature` | Softmax temperature for KD. Higher = softer probability distributions | `1.0–5.0` (start with `2.0`) | For generated distillation schema details, see the -[Customizer API Reference](../../api/index.md#tag-customizer) and search for +[Customizer API Reference](/documentation/reference/api-reference) and search for `DistillationTrainingInput`. diff --git a/docs/evaluator/index.mdx b/docs/evaluator/index.mdx index 8d2bc8ae304..39d8c9f7480 100644 --- a/docs/evaluator/index.mdx +++ b/docs/evaluator/index.mdx @@ -147,7 +147,7 @@ Most teams get the best results by starting metric-first, then moving to benchma ## Where to Go Next - For metric workflows, see [Metric Jobs](metrics/index.md) and [Metric Results](metrics/results.md). -- For full endpoint details, see the [Evaluator API Reference](../api/index.md#tag-evaluator). +- For full endpoint details, see the [Evaluator API Reference](/documentation/reference/api-reference). --- diff --git a/docs/fern/README.md b/docs/fern/README.md index 1128d89c23e..7d65c9e8cfd 100644 --- a/docs/fern/README.md +++ b/docs/fern/README.md @@ -76,13 +76,14 @@ Avoid source-path links such as `/get-started/concepts/workspaces`, `/latest/get ## API Reference -The API reference page lives at `versions/latest/pages/api/index.mdx` and renders `versions/latest/pages/api/openapi.yaml` with: +The REST API reference is generated natively by Fern from the OpenAPI spec — no `` embed. Two pieces wire it up: -```mdx - -``` +- `generators.yml` declares the spec: `api.specs[].openapi: ./openapi/openapi.yaml`. +- `versions/latest.yml` surfaces it with an `- api: API Reference` navigation node (under the **Reference** section). + +`openapi/openapi.yaml` is a symlink to the repo-root `openapi/openapi.yaml` (the generated source of truth), so the reference tracks the Platform API automatically. Regenerate the spec with `make refresh-openapi` from the repo root, then run `npm run check`. -Regenerate or replace `openapi.yaml` when the Platform API changes, then run `npm run check`. +Fern groups endpoints by their OpenAPI tag in the sidebar (Customizer, Evaluator, Guardrails, …), which replaces the old per-service filter chips. Link to it from other pages with the nav URL `/documentation/reference/api-reference`. ## CI and Publishing diff --git a/docs/fern/generators.yml b/docs/fern/generators.yml new file mode 100644 index 00000000000..a2c12e1747a --- /dev/null +++ b/docs/fern/generators.yml @@ -0,0 +1,3 @@ +api: + specs: + - openapi: ./openapi/openapi.yaml diff --git a/docs/fern/openapi/openapi.yaml b/docs/fern/openapi/openapi.yaml new file mode 120000 index 00000000000..e08fc74b0a3 --- /dev/null +++ b/docs/fern/openapi/openapi.yaml @@ -0,0 +1 @@ +../../../openapi/openapi.yaml \ No newline at end of file diff --git a/docs/fern/versions/latest.yml b/docs/fern/versions/latest.yml index d25e17d4464..d790e9051df 100644 --- a/docs/fern/versions/latest.yml +++ b/docs/fern/versions/latest.yml @@ -243,8 +243,7 @@ navigation: path: ../../requirements.mdx - page: Support Matrix path: ../../support-matrix.mdx - - page: API Reference - path: ../../api/index.mdx + - api: API Reference - section: Python SDK path: ../../pysdk/index.mdx contents: diff --git a/docs/get-started/concepts/entities.mdx b/docs/get-started/concepts/entities.mdx index 52fb4980d0c..d46740cd941 100644 --- a/docs/get-started/concepts/entities.mdx +++ b/docs/get-started/concepts/entities.mdx @@ -31,4 +31,4 @@ In most cases, you interact with entities through service-specific APIs rather t - **Evaluator** — Create metrics, benchmarks, and evaluation jobs; retrieve results. See [About Evaluating](../../evaluator/index.md) and the evaluation tutorials. - **Models and inference** — Register model entities, create deployments, and run inference through the gateway. See [about](../../run-inference/about.md) and the inference tutorials. -These service APIs provide typed interfaces and validation specific to each resource type, built on top of the entity system. For raw HTTP access, see the [Entity Store API reference](../../api/index.md#tag-entity-store). +These service APIs provide typed interfaces and validation specific to each resource type, built on top of the entity system. For raw HTTP access, see the [Entity Store API reference](/documentation/reference/api-reference). diff --git a/docs/pysdk/client/index.mdx b/docs/pysdk/client/index.mdx index d2c99de6b52..4e4fae8658c 100644 --- a/docs/pysdk/client/index.mdx +++ b/docs/pysdk/client/index.mdx @@ -148,7 +148,7 @@ for SDK-level errors. ## Client Attributes The NeMo Platform clients provide access to API resources through the following attributes. -For endpoint details, see the [REST API Reference](../../api/index.md). +For endpoint details, see the [REST API Reference](/documentation/reference/api-reference). ### Organization diff --git a/docs/pysdk/index.mdx b/docs/pysdk/index.mdx index d09ddef4648..1e5083f9f09 100644 --- a/docs/pysdk/index.mdx +++ b/docs/pysdk/index.mdx @@ -28,4 +28,4 @@ This project will download and install additional third-party open source softwa ## Next Steps - Read more about connecting with the [client APIs](./client/index.md). -- Browse the [REST API Reference](../api/index.md). +- Browse the [REST API Reference](/documentation/reference/api-reference). diff --git a/docs/run-inference/about.mdx b/docs/run-inference/about.mdx index f0e074be375..5b6b665718e 100644 --- a/docs/run-inference/about.mdx +++ b/docs/run-inference/about.mdx @@ -294,4 +294,4 @@ working as designed — use `/health/ready` instead. ## API Reference -For complete API details, refer to the [Inference Gateway API Reference](../api/index.md#tag-inference-gateway) and [SDK Reference](../pysdk/index.md). +For complete API details, refer to the [Inference Gateway API Reference](/documentation/reference/api-reference) and [SDK Reference](../pysdk/index.md). diff --git a/docs/safe-synthesizer/about/reference.mdx b/docs/safe-synthesizer/about/reference.mdx index b57e38c0076..be325f5f1e5 100644 --- a/docs/safe-synthesizer/about/reference.mdx +++ b/docs/safe-synthesizer/about/reference.mdx @@ -8,7 +8,7 @@ description: "" This page summarizes the main configuration groups available when creating NeMo Safe Synthesizer jobs. For generated REST API schema details, see the -[Safe Synthesizer API Reference](../../api/index.md#tag-safe-synthesizer). +[Safe Synthesizer API Reference](/documentation/reference/api-reference). ## Job Spec (Plugin / REST) @@ -29,7 +29,7 @@ The `SafeSynthesizerParameters` schema defines the main configuration structure ### SafeSynthesizerParameters All fields are optional at the top level. For nested field constraints, see the -[Safe Synthesizer API Reference](../../api/index.md#tag-safe-synthesizer) and +[Safe Synthesizer API Reference](/documentation/reference/api-reference) and search for the schema name in the **Type** column. | Field | Type | Constraints / description | diff --git a/docs/troubleshooting/data-designer.mdx b/docs/troubleshooting/data-designer.mdx index 3d559f29078..73cbba9dd64 100644 --- a/docs/troubleshooting/data-designer.mdx +++ b/docs/troubleshooting/data-designer.mdx @@ -52,5 +52,5 @@ Look for startup errors, provider authentication failures, port conflicts, or Da - Tutorial: [Data Designer Tutorials](../data-designer/tutorials/index.md) covers common generation workflows. - How-to: [Set Up NeMo Platform](../get-started/setup.md) covers local setup and service startup. - Explanation: [Data Designer Overview](../data-designer/index.md) explains configuration, execution, and service behavior. -- Reference: [API Reference](../api/index.md#tag-data-designer) lists Data Designer endpoints. +- Reference: [API Reference](/documentation/reference/api-reference) lists Data Designer endpoints. - Troubleshooting: [Troubleshooting Overview](index.md) links to other service troubleshooting pages. diff --git a/docs/troubleshooting/studio.mdx b/docs/troubleshooting/studio.mdx index 1ef869a9a15..43d232edf39 100644 --- a/docs/troubleshooting/studio.mdx +++ b/docs/troubleshooting/studio.mdx @@ -40,4 +40,4 @@ curl -X POST http://localhost:8080/v1/datasets \ ``` All resource creation and update APIs support the `project` parameter. -Refer to the [Python SDK](../pysdk/index.md) and [API references](../api/index.md) for more details. +Refer to the [Python SDK](../pysdk/index.md) and [API references](/documentation/reference/api-reference) for more details. From 190698cc47c77f20bfa91c1e6509ffa6b7f93729 Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Thu, 4 Jun 2026 13:28:31 -0400 Subject: [PATCH 15/26] docs(fern): rewrite internal links to canonical Fern URLs The migration kept MkDocs-style relative .md links, which don't resolve in Fern's URL/slug space (Fern uses title-derived slugs and treats each page as a directory), producing ~810 broken links reported by `fern docs broken-links`. Rewrite every internal markdown and JSX-href link that targets a published page to its canonical absolute URL (/documentation/), derived from the nav in versions/latest.yml. 421 links across 114 files. This is the form docs/fern/README.md already recommends. Broken-link count drops from 810 to 98; the remainder point to pages that exist but are not yet in the nav (handled separately). Co-Authored-By: Claude Opus 4.8 Signed-off-by: Lawrence Lane --- docs/about/release-notes/current-release.mdx | 2 +- docs/about/release-notes/index.mdx | 2 +- docs/agents/index.mdx | 14 ++++---- docs/agents/optimization.mdx | 10 +++--- docs/agents/plugins.mdx | 6 ++-- docs/agents/security.mdx | 14 ++++---- docs/anonymizer/cli.mdx | 6 ++-- docs/anonymizer/index.mdx | 10 +++--- docs/anonymizer/quickstart.mdx | 16 +++++----- docs/anonymizer/tutorials/index.mdx | 6 ++-- docs/anonymizer/tutorials/preview.mdx | 8 ++--- docs/anonymizer/tutorials/run.mdx | 8 ++--- docs/auditor/configs/index.mdx | 4 +-- docs/auditor/configs/schema.mdx | 4 +-- docs/auditor/index.mdx | 32 +++++++++---------- docs/auditor/sdk-resources.mdx | 6 ++-- docs/auditor/targets/index.mdx | 6 ++-- docs/auditor/targets/schema.mdx | 2 +- docs/auditor/tutorials/index.mdx | 8 ++--- docs/auditor/tutorials/run-audit-locally.mdx | 16 +++++----- docs/auth/authorization/index.mdx | 2 +- docs/auth/authorization/managing-access.mdx | 4 +-- docs/auth/deployment/configuration.mdx | 2 +- docs/cli/index.mdx | 12 +++---- docs/cli/troubleshooting.mdx | 4 +-- docs/customizer/about.mdx | 4 +-- docs/customizer/index.mdx | 18 +++++------ .../manage-customization-jobs/create-job.mdx | 2 +- .../customization-job-reference.mdx | 10 +++--- .../manage-customization-jobs/index.mdx | 10 +++--- .../manage-model-entities/create-fileset.mdx | 2 +- .../create-model-entity.mdx | 6 ++-- .../manage-model-entities/index.mdx | 4 +-- docs/customizer/models/data-format.mdx | 2 +- docs/customizer/models/embedding.mdx | 2 +- docs/customizer/models/index.mdx | 6 ++-- .../tutorials/format-training-dataset.mdx | 4 +-- docs/customizer/tutorials/index.mdx | 4 +-- docs/customizer/tutorials/metrics.mdx | 6 ++-- .../understand-configurations-and-models.mdx | 4 +-- docs/data-designer/cli.mdx | 2 +- docs/data-designer/index.mdx | 12 +++---- docs/data-designer/migration.mdx | 8 ++--- docs/data-designer/tutorials/basics.mdx | 6 ++-- docs/data-designer/tutorials/index.mdx | 12 +++---- docs/data-designer/tutorials/seeding.mdx | 4 +-- docs/evaluator/benchmarks/agentic.mdx | 6 ++-- docs/evaluator/benchmarks/custom.mdx | 20 ++++++------ .../discover-industry-benchmarks.mdx | 2 +- docs/evaluator/benchmarks/index.mdx | 12 +++---- docs/evaluator/benchmarks/industry.mdx | 6 ++-- docs/evaluator/benchmarks/job-management.mdx | 2 +- .../benchmarks/manage-benchmarks.mdx | 2 +- docs/evaluator/index.mdx | 18 +++++------ .../evaluator/metrics/agent-configuration.mdx | 14 ++++---- docs/evaluator/metrics/agentic.mdx | 22 ++++++------- docs/evaluator/metrics/index.mdx | 22 ++++++------- docs/evaluator/metrics/job-management.mdx | 4 +-- docs/evaluator/metrics/llm-as-a-judge.mdx | 12 +++---- docs/evaluator/metrics/manage-metrics.mdx | 8 ++--- .../evaluator/metrics/model-configuration.mdx | 2 +- docs/evaluator/metrics/rag.mdx | 18 +++++------ docs/evaluator/metrics/remote.mdx | 12 +++---- docs/evaluator/metrics/similarity.mdx | 10 +++--- docs/evaluator/tutorials/index.mdx | 8 ++--- .../tutorials/run-llm-judge-evaluation.mdx | 12 +++---- docs/get-started/concepts/entities.mdx | 4 +-- docs/get-started/concepts/index.mdx | 14 ++++---- docs/get-started/concepts/manage-secrets.mdx | 2 +- docs/get-started/setup.mdx | 8 ++--- docs/guardrails/concepts/architecture.mdx | 10 +++--- docs/guardrails/concepts/checks.mdx | 4 +-- .../configuration-structure.mdx | 6 ++-- .../configurations/default-configs.mdx | 12 +++---- .../concepts/configurations/index.mdx | 6 ++-- .../configurations/manage-configs.mdx | 2 +- docs/guardrails/concepts/index.mdx | 8 ++--- docs/guardrails/concepts/inference.mdx | 12 +++---- docs/guardrails/index.mdx | 18 +++++------ docs/guardrails/observability.mdx | 2 +- docs/guardrails/tutorials/content-safety.mdx | 2 +- .../tutorials/deploy-nemoguard-nims.mdx | 6 ++-- docs/guardrails/tutorials/index.mdx | 14 ++++---- .../tutorials/injection-detection.mdx | 2 +- docs/guardrails/tutorials/multimodal-data.mdx | 2 +- docs/guardrails/tutorials/parallel-rails.mdx | 2 +- docs/index.mdx | 14 ++++---- docs/notebooks/ndd_evaluator.mdx | 6 ++-- docs/pysdk/index.mdx | 2 +- docs/requirements.mdx | 6 ++-- docs/run-inference/about.mdx | 4 +-- docs/run-inference/tutorials/index.mdx | 2 +- .../run-inference/tutorials/run-inference.mdx | 4 +-- docs/safe-synthesizer/about/evaluation.mdx | 2 +- docs/safe-synthesizer/about/index.mdx | 4 +-- docs/safe-synthesizer/about/jobs.mdx | 2 +- .../about/pii-replacement.mdx | 2 +- docs/safe-synthesizer/getting-started.mdx | 2 +- .../tutorials/differential-privacy.mdx | 2 +- docs/safe-synthesizer/tutorials/index.mdx | 2 +- .../tutorials/safe-synthesizer-101.mdx | 2 +- docs/set-up/helm/install.mdx | 2 +- docs/set-up/helm/persistent-volumes.mdx | 2 +- docs/set-up/helm/prerequisites.mdx | 4 +-- docs/set-up/manage-jobs.mdx | 6 ++-- docs/set-up/milvus.mdx | 6 ++-- docs/studio/agents.mdx | 6 ++-- docs/studio/index.mdx | 12 +++---- docs/studio/monitor.mdx | 4 +-- docs/studio/suggestions.mdx | 6 ++-- docs/troubleshooting/data-designer.mdx | 8 ++--- docs/troubleshooting/evaluator.mdx | 4 +-- docs/troubleshooting/index.mdx | 8 ++--- docs/troubleshooting/studio.mdx | 2 +- 114 files changed, 403 insertions(+), 403 deletions(-) diff --git a/docs/about/release-notes/current-release.mdx b/docs/about/release-notes/current-release.mdx index adbb7f68eb0..96f5a1ec7c7 100644 --- a/docs/about/release-notes/current-release.mdx +++ b/docs/about/release-notes/current-release.mdx @@ -88,7 +88,7 @@ make bootstrap nemo setup ``` -See [Setup](../../get-started/setup.md) for prerequisites and provider +See [Setup](/documentation/get-started) for prerequisites and provider configuration. ## Compatibility diff --git a/docs/about/release-notes/index.mdx b/docs/about/release-notes/index.mdx index bae0d9293f4..87e0e634647 100644 --- a/docs/about/release-notes/index.mdx +++ b/docs/about/release-notes/index.mdx @@ -4,4 +4,4 @@ description: "" --- Check out the latest release notes for NeMo Platform. -- [Release 0.1.0](current-release.md) +- [Release 0.1.0](/documentation/reference/release-notes/current-release) diff --git a/docs/agents/index.mdx b/docs/agents/index.mdx index 727b8a0ff94..49b5ea808c0 100644 --- a/docs/agents/index.mdx +++ b/docs/agents/index.mdx @@ -55,7 +55,7 @@ workflow: llm_name: llm ``` -The [Inference Gateway](../run-inference/about.md) is the local platform's +The [Inference Gateway](/documentation/models-and-inference) is the local platform's model proxy. Agents send model requests to it instead of to provider APIs directly, so the platform can resolve model names, attach credentials, and route through middleware on the agent's behalf. Two conventions apply when a @@ -92,7 +92,7 @@ Model traffic from inside the agent process routes back through the Inference Ga A virtual model is a platform-managed wrapper around one or more backend models. An agent can point at a virtual model entity while the virtual model -handles routing, format translation, and [guardrails](../guardrails/index.md) +handles routing, format translation, and [guardrails](/documentation/guardrail-models) behind the scenes — no changes to the agent's workflow YAML required. ### Applying Changes Through Candidate Agents @@ -110,11 +110,11 @@ undeploying the candidate. ## Common Tasks -- [Optimize Agents](optimization.md): analyze deployed agents for model routing, +- [Optimize Agents](/documentation/agents/optimize-agents): analyze deployed agents for model routing, skill, prompt, and new-model opportunities. -- [Secure Agents](security.md): check guardrail coverage and scan recent +- [Secure Agents](/documentation/agents/secure-agents): check guardrail coverage and scan recent telemetry for sensitive data. -- [Plugins and Skills](plugins.md): understand how agent, middleware, and +- [Plugins and Skills](/documentation/agents/plugins-and-skills): understand how agent, middleware, and coding-agent integrations extend the local platform. -- [Agentic Metrics](../evaluator/metrics/agentic.md): evaluate tool use, goal completion, topic adherence, answer accuracy, and trajectories. -- [Agent Configuration](../evaluator/metrics/agent-configuration.md): use agents as online evaluation targets. +- [Agentic Metrics](/documentation/evaluate-models/metrics/agentic-metrics): evaluate tool use, goal completion, topic adherence, answer accuracy, and trajectories. +- [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration): use agents as online evaluation targets. diff --git a/docs/agents/optimization.mdx b/docs/agents/optimization.mdx index d7a1f77f788..9586d8b3907 100644 --- a/docs/agents/optimization.mdx +++ b/docs/agents/optimization.mdx @@ -29,7 +29,7 @@ Optimizer state is stored in the `nemo-agent-optimizer` fileset: - `optimizer_snapshot.json`: model and agent names from the latest run. Security-oriented suggestions such as missing guardrails, PII exposure, or -leaked secrets are covered in [Secure Agents](security.md). +leaked secrets are covered in [Secure Agents](/documentation/agents/secure-agents). ## Prerequisites @@ -77,7 +77,7 @@ smaller compatible model. Switchyard is the inference middleware that lets a virtual model split traffic across multiple backend models. The common optimization pattern is to create a -[virtual model](../run-inference/about.md) with a strong model and a weaker, +[virtual model](/documentation/models-and-inference) with a strong model and a weaker, cheaper model, then evaluate whether the route split preserves application quality. @@ -357,6 +357,6 @@ do not change the remote agent behavior. ## Next steps -- [Agent overview](index.md): review how platform-managed agents are registered, deployed, invoked, evaluated, and optimized. -- [Agent evaluation](../evaluator/metrics/agent-configuration.md): configure agents as online evaluation targets and choose the right agent response mapping. -- [CLI reference](../cli/reference.md): look up complete command options and global CLI flags for scripted workflows. +- [Agent overview](/documentation/agents): review how platform-managed agents are registered, deployed, invoked, evaluated, and optimized. +- [Agent evaluation](/documentation/evaluate-models/metrics/agent-configuration): configure agents as online evaluation targets and choose the right agent response mapping. +- [CLI reference](/documentation/reference/cli-reference/full-cli-reference): look up complete command options and global CLI flags for scripted workflows. diff --git a/docs/agents/plugins.mdx b/docs/agents/plugins.mdx index 13e5fb70126..762530c2de5 100644 --- a/docs/agents/plugins.mdx +++ b/docs/agents/plugins.mdx @@ -67,8 +67,8 @@ The skills that drive the agent lifecycle are: | `nemo-try-agent` | Sends a query to a deployed agent. | | `nemo-status` | Read-only platform health dashboard. | | `nemo-teardown` | Guided shutdown with confirmation. | -| `agents-optimize` | Selects a deployed agent, establishes an evaluation baseline, and suggests Switchyard routing, model swaps, skill optimization, prompt tuning, and new-model evaluations. See [Optimize Agents](optimization.md). | -| `agents-secure` | Selects a deployed agent, checks guardrail coverage, and scans recent telemetry for sensitive data. See [Secure Agents](security.md). | +| `agents-optimize` | Selects a deployed agent, establishes an evaluation baseline, and suggests Switchyard routing, model swaps, skill optimization, prompt tuning, and new-model evaluations. See [Optimize Agents](/documentation/agents/optimize-agents). | +| `agents-secure` | Selects a deployed agent, checks guardrail coverage, and scans recent telemetry for sensitive data. See [Secure Agents](/documentation/agents/secure-agents). | Plugin-owned skills cover guardrails, evaluations, optimization, data designer, anonymizer, and auditor. They are installed with their plugin and @@ -95,7 +95,7 @@ Common middleware patterns: - **Guardrails middleware** for request and response checks. For command details, see the [Inference Gateway API Reference](/documentation/reference/api-reference) -and the [CLI reference](../cli/reference.md). +and the [CLI reference](/documentation/reference/cli-reference/full-cli-reference). ## Writing Plugins diff --git a/docs/agents/security.mdx b/docs/agents/security.mdx index 511b411ee44..c3bedd7a03a 100644 --- a/docs/agents/security.mdx +++ b/docs/agents/security.mdx @@ -49,7 +49,7 @@ with `nemo models list`): - `nvidia-llama-3-1-nemotron-safety-guard-8b-v3` For how to create the guardrail config that the virtual model references, see -the [Guardrails documentation](../guardrails/index.md). +the [Guardrails documentation](/documentation/guardrail-models). There are two steps: create the guarded VirtualModel, then update the agent's `llms` block to reference it. @@ -80,7 +80,7 @@ Wire the same `` on both `--request-middleware` (for input rails) and `--response-middleware` (for output rails). Omit a side if the config defines no flows for it. For the full middleware schema, entity-backed vs inline configs, and caching behavior, refer to -[Guardrails Architecture](../guardrails/concepts/architecture.md). +[Guardrails Architecture](/documentation/guardrail-models/core-concepts/architecture). @@ -143,7 +143,7 @@ client.inference.virtual_models.create( In the agent's workflow YAML, set `model_name` on the relevant `llms` entry to the guarded VirtualModel's entity reference, with slashes converted to hyphens -(per the [agent configuration conventions](index.md#agent-definition)): +(per the [agent configuration conventions](/documentation/agents#agent-definition)): ```yaml llms: @@ -158,7 +158,7 @@ unchanged and unaware of the rails. For the end-to-end request flow, streaming behavior, header forwarding, and the `guardrails` request options, refer to -[Running Inference with Guardrails](../guardrails/concepts/inference.md). +[Running Inference with Guardrails](/documentation/guardrail-models/core-concepts/running-inference). Redeploy the agent, re-run evaluation, and compare quality, cost, latency, and safety signals against the baseline before promoting. @@ -255,9 +255,9 @@ nemo files download nemo-agent-security \ ## Next Steps -- [Optimize Agents](optimization.md): reduce cost and improve quality after +- [Optimize Agents](/documentation/agents/optimize-agents): reduce cost and improve quality after security coverage is in place. -- [Guardrails](../guardrails/index.md): create and manage guardrail +- [Guardrails](/documentation/guardrail-models): create and manage guardrail configurations. -- [Models and Inference](../run-inference/about.md): manage model providers, +- [Models and Inference](/documentation/models-and-inference): manage model providers, model entities, and virtual models. diff --git a/docs/anonymizer/cli.mdx b/docs/anonymizer/cli.mdx index 0c0b54bc1f9..84da4cc3081 100644 --- a/docs/anonymizer/cli.mdx +++ b/docs/anonymizer/cli.mdx @@ -4,7 +4,7 @@ description: "" --- -This reference covers the `nemo anonymizer` commands exposed by the Anonymizer plugin. For end-to-end walkthroughs, see the [tutorials](tutorials/index.md). +This reference covers the `nemo anonymizer` commands exposed by the Anonymizer plugin. For end-to-end walkthroughs, see the [tutorials](/documentation/anonymize-data/tutorials). ## Command Surface @@ -73,7 +73,7 @@ nemo anonymizer preview run --spec-file /tmp/anonymizer-preview.yaml > /tmp/prev jq -R 'fromjson? | select(.kind == "preview_dataset") | .records' /tmp/preview.ndjson ``` -Frame kinds: `log`, `preview_dataset`, `trace_dataset`, `failed_records`, `heartbeat`, `done`, `error`. See the [preview tutorial](tutorials/preview.md) for details. +Frame kinds: `log`, `preview_dataset`, `trace_dataset`, `failed_records`, `heartbeat`, `done`, `error`. See the [preview tutorial](/documentation/anonymize-data/tutorials/preview-a-config) for details. ## `nemo anonymizer run` @@ -140,7 +140,7 @@ results = job.download_artifacts() dataset = results.load_dataset() ``` -See [SDK Resources](sdk-resources.md) for the full `AnonymizerJobResource` / `AnonymizerJobResults` surface. +See [SDK Resources](/documentation/anonymize-data/sdk-resources) for the full `AnonymizerJobResource` / `AnonymizerJobResults` surface. Compared to `run run`, `run submit` rejects local file paths in `data.source` (use a fileset reference or `http(s)` URL) and requires explicit `model_configs` because the job runs outside the CLI process. diff --git a/docs/anonymizer/index.mdx b/docs/anonymizer/index.mdx index f706d3fe28a..376bdaeb25b 100644 --- a/docs/anonymizer/index.mdx +++ b/docs/anonymizer/index.mdx @@ -16,7 +16,7 @@ The library defines **what** to anonymize and **how**. The platform decides **wh -The code snippets below are for conceptual demonstration purposes only. For runnable examples, see the [quickstart](quickstart.md) and [tutorials](tutorials/index.md). +The code snippets below are for conceptual demonstration purposes only. For runnable examples, see the [quickstart](/documentation/anonymize-data/quickstart) and [tutorials](/documentation/anonymize-data/tutorials). ### 1. Build a config with the library @@ -111,22 +111,22 @@ This package is a thin wrapper around the [NVIDIA NeMo Anonymizer library](https - + Install the plugin, configure inference, and run your first preview and job. - + Walk through preview (`anonymizer.preview`) and job execution (`anonymizer.run`) end to end. - + Reference for the `anonymizer` SDK accessor, preview result, and job result objects. - + Reference for `nemo anonymizer` commands and their spec files. diff --git a/docs/anonymizer/quickstart.mdx b/docs/anonymizer/quickstart.mdx index a2ed3dd98ff..c0d3fa58e94 100644 --- a/docs/anonymizer/quickstart.mdx +++ b/docs/anonymizer/quickstart.mdx @@ -13,7 +13,7 @@ This guide walks through previewing and running an Anonymizer job on NeMo Platfo ## Step 1: Install the Plugin -Follow the [Setup guide](../get-started/setup.md) to install NeMo Platform and complete `nemo setup`. From a repo checkout, run `uv sync` at the repo root; the root workspace includes the Anonymizer plugin, so no separate editable plugin install step is needed. `nemo services run` then picks up the plugin automatically and mounts `/apis/anonymizer/...` on the gateway. +Follow the [Setup guide](/documentation/get-started) to install NeMo Platform and complete `nemo setup`. From a repo checkout, run `uv sync` at the repo root; the root workspace includes the Anonymizer plugin, so no separate editable plugin install step is needed. `nemo services run` then picks up the plugin automatically and mounts `/apis/anonymizer/...` on the gateway. Verify the CLI is registered: @@ -37,9 +37,9 @@ anonymizer = sdk.anonymizer ## Step 3: Configure Inference -Anonymizer routes inference through the [Inference Gateway service](../run-inference/about.md). You need a model provider configured before running anything that uses `model_configs`. +Anonymizer routes inference through the [Inference Gateway service](/documentation/models-and-inference). You need a model provider configured before running anything that uses `model_configs`. -`nemo setup` walks you through creating a provider secret and registering an Inference Gateway provider as part of the install flow. If you skipped that step or want to add another provider, re-run `nemo setup` — see the [Setup guide](../get-started/setup.md) for details. +`nemo setup` walks you through creating a provider secret and registering an Inference Gateway provider as part of the install flow. If you skipped that step or want to add another provider, re-run `nemo setup` — see the [Setup guide](/documentation/get-started) for details. ## Step 4: Upload an Input Fileset @@ -158,7 +158,7 @@ nemo anonymizer preview submit \ --base-url "${NMP_BASE_URL:-http://localhost:8080}" ``` -The CLI streams newline-delimited JSON frames (`preview_dataset`, `trace_dataset`, `failed_records`, ...) to stdout. See the [preview tutorial](tutorials/preview.md) for the frame schema and `jq` recipes. +The CLI streams newline-delimited JSON frames (`preview_dataset`, `trace_dataset`, `failed_records`, ...) to stdout. See the [preview tutorial](/documentation/anonymize-data/tutorials/preview-a-config) for the frame schema and `jq` recipes. @@ -197,7 +197,7 @@ print(dataset.head()) print(f"records={len(dataset)} failures={len(results.load_failed_records())}") ``` -`sdk.anonymizer.run()` returns an `AnonymizerJobResource`. `wait_until_done=True` blocks until the job reaches a terminal state; `download_artifacts()` fetches the job artifacts and returns an `AnonymizerJobResults` for in-memory access. See [SDK Resources](sdk-resources.md) for the full surface. +`sdk.anonymizer.run()` returns an `AnonymizerJobResource`. `wait_until_done=True` blocks until the job reaches a terminal state; `download_artifacts()` fetches the job artifacts and returns an `AnonymizerJobResults` for in-memory access. See [SDK Resources](/documentation/anonymize-data/sdk-resources) for the full surface. The CLI equivalent submits the same spec. First write it to YAML: @@ -281,7 +281,7 @@ The trace dataset (and the dataset itself for `annotate` / `substitute` strategi ## Next Steps -- **Tutorials:** Walk through preview and run flows in detail in the [tutorials](tutorials/index.md). -- **SDK reference:** See [SDK Resources](sdk-resources.md) for the `anonymizer` accessor, preview result, and job result types. -- **CLI reference:** See [CLI Reference](cli.md) for spec-file fields and command flags. +- **Tutorials:** Walk through preview and run flows in detail in the [tutorials](/documentation/anonymize-data/tutorials). +- **SDK reference:** See [SDK Resources](/documentation/anonymize-data/sdk-resources) for the `anonymizer` accessor, preview result, and job result types. +- **CLI reference:** See [CLI Reference](/documentation/anonymize-data/cli-reference) for spec-file fields and command flags. - **Library docs:** Detection, replacement strategy parameters, and rewrite mode are documented in the [open-source library](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs). diff --git a/docs/anonymizer/tutorials/index.mdx b/docs/anonymizer/tutorials/index.mdx index 37b731f074a..da854636903 100644 --- a/docs/anonymizer/tutorials/index.mdx +++ b/docs/anonymizer/tutorials/index.mdx @@ -79,7 +79,7 @@ When using Anonymizer as a NeMo Platform service: ## Prerequisites -Before starting these tutorials, complete the [Quick Start](../quickstart.md) to: +Before starting these tutorials, complete the [Quick Start](/documentation/anonymize-data/quickstart) to: - Install the plugin and verify the `nemo anonymizer` CLI. - Configure an inference provider used in `model_configs`. @@ -89,14 +89,14 @@ Before starting these tutorials, complete the [Quick Start](../quickstart.md) to - + Stream a small anonymized sample to iterate on `AnonymizerConfig` and `model_configs`. Covers `sdk.anonymizer.preview`, `nemo anonymizer preview run` / `preview submit`, and the NDJSON frame stream. beginner anonymizer - + Run the full pipeline locally with `nemo anonymizer run run` or submit it to the Jobs worker with `nemo anonymizer run submit`. Load `dataset.parquet`, `trace.parquet`, and `failed_records.json` artifacts. diff --git a/docs/anonymizer/tutorials/preview.mdx b/docs/anonymizer/tutorials/preview.mdx index 447c3493af2..c98963a2db7 100644 --- a/docs/anonymizer/tutorials/preview.mdx +++ b/docs/anonymizer/tutorials/preview.mdx @@ -10,7 +10,7 @@ For detection and replacement strategy details, see the [open-source library doc ## Prerequisites -- The Anonymizer plugin installed and the `nemo anonymizer` CLI available. See the [Quick Start](../quickstart.md). +- The Anonymizer plugin installed and the `nemo anonymizer` CLI available. See the [Quick Start](/documentation/anonymize-data/quickstart). - An inference provider configured (default examples use `nvidia-build`). - A fileset named `anonymizer-inputs` with `anonymizer-input.csv` uploaded (created in the Quick Start). @@ -179,7 +179,7 @@ jq -R 'fromjson? | select(.kind == "preview_dataset") | .records' \ /tmp/anonymizer-preview.ndjson ``` -If `preview submit` returns 404 against the gateway, the plugin service isn't mounted. Confirm the plugin is installed and restart `nemo services run`; see [Quick Start — Step 1](../quickstart.md#step-1-install-the-plugin). +If `preview submit` returns 404 against the gateway, the plugin service isn't mounted. Confirm the plugin is installed and restart `nemo services run`; see [Quick Start — Step 1](/documentation/anonymize-data/quickstart#step-1-install-the-plugin). ## Input Source Forms @@ -222,6 +222,6 @@ nemo anonymizer validate --config /tmp/anonymizer-config.yaml ## Next Steps -- Run a full job and inspect parquet output in the [run tutorial](run.md). -- Refer to [SDK Resources](../sdk-resources.md) for `AnonymizerPreviewResult` and `AnonymizerResource.preview` details. +- Run a full job and inspect parquet output in the [run tutorial](/documentation/anonymize-data/tutorials/run-an-anonymizer-job). +- Refer to [SDK Resources](/documentation/anonymize-data/sdk-resources) for `AnonymizerPreviewResult` and `AnonymizerResource.preview` details. - Learn about rewrite and replacement strategy parameters in the [library docs](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs). diff --git a/docs/anonymizer/tutorials/run.mdx b/docs/anonymizer/tutorials/run.mdx index 4afbb2d2291..e5fbb2ba83c 100644 --- a/docs/anonymizer/tutorials/run.mdx +++ b/docs/anonymizer/tutorials/run.mdx @@ -10,7 +10,7 @@ For detection, rewrite, and replacement strategy details, see the [open-source l ## Prerequisites -- The Anonymizer plugin installed and the `nemo anonymizer` CLI available. See the [Quick Start](../quickstart.md). +- The Anonymizer plugin installed and the `nemo anonymizer` CLI available. See the [Quick Start](/documentation/anonymize-data/quickstart). - An inference provider configured (default examples use `nvidia-build`). - A fileset named `anonymizer-inputs` with `anonymizer-input.csv` uploaded (created in the Quick Start). @@ -234,7 +234,7 @@ trace = results.load_trace() failed = results.load_failed_records() ``` -`AnonymizerJobResults` exposes `load_dataset()`, `load_trace()`, `load_failed_records()`, and `display_record()` over the same underlying files. See [SDK Resources](../sdk-resources.md#anonymizerjobresults). +`AnonymizerJobResults` exposes `load_dataset()`, `load_trace()`, `load_failed_records()`, and `display_record()` over the same underlying files. See [SDK Resources](/documentation/anonymize-data/sdk-resources#anonymizerjobresults). ## Inspect the Schema Without Running @@ -262,6 +262,6 @@ For `run submit`, provider endpoints are re-resolved at runtime so the job uses ## Next Steps -- Iterate faster with [preview](preview.md) before scaling to a full job. -- Refer to [SDK Resources](../sdk-resources.md) for `AnonymizerJobResource` and `AnonymizerJobResults` details. +- Iterate faster with [preview](/documentation/anonymize-data/tutorials/preview-a-config) before scaling to a full job. +- Refer to [SDK Resources](/documentation/anonymize-data/sdk-resources) for `AnonymizerJobResource` and `AnonymizerJobResults` details. - Replacement strategy parameters and rewrite mode are documented in the [library docs](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs). diff --git a/docs/auditor/configs/index.mdx b/docs/auditor/configs/index.mdx index 5cc8f2f7ef9..a3534c73dd7 100644 --- a/docs/auditor/configs/index.mdx +++ b/docs/auditor/configs/index.mdx @@ -17,7 +17,7 @@ A configuration has four sub-blocks plus a free-form description. Each sub-block | `plugins` | The probes and detectors to run (`probe_spec`, `detector_spec`) and any plugin-specific options. | | `reporting` | Output file prefix and directory, optional taxonomy, and which modules are surfaced in summaries. | -The full field reference is in [Configuration Schema](schema.md). +The full field reference is in [Configuration Schema](/documentation/vulnerability-scanning/configurations/schema). ## The Seeded Default @@ -54,7 +54,7 @@ config = client.auditor.configs.create( ) ``` -The `probe_spec` field takes a comma-separated list of probe categories or fully-qualified probe classes. See [Selecting Probes](probes.md) for the syntax and worked examples. +The `probe_spec` field takes a comma-separated list of probe categories or fully-qualified probe classes. See [Selecting Probes](/documentation/vulnerability-scanning/configurations/selecting-probes) for the syntax and worked examples. ## List Configurations diff --git a/docs/auditor/configs/schema.mdx b/docs/auditor/configs/schema.mdx index 073f061d66f..4138ee57cb3 100644 --- a/docs/auditor/configs/schema.mdx +++ b/docs/auditor/configs/schema.mdx @@ -46,7 +46,7 @@ Controls how each probe is executed. | `deprefix` | `bool` | `True` | Strip prompt prefixes from responses before detection. | | `eval_threshold` | `float` (0–1) | `0.5` | Detector score threshold above which a response is flagged as a hit. | | `generations` | `int` (≥1) | `5` | Generations per probe. Lower for fast iteration; raise for statistical confidence. | -| `probe_tags` | `str \\| None` | `None` | Comma-separated tag filter (`"owasp:llm06,payload:hallucination"`). Intersects with `plugins.probe_spec`. See [Selecting Probes](probes.md). | +| `probe_tags` | `str \\| None` | `None` | Comma-separated tag filter (`"owasp:llm06,payload:hallucination"`). Intersects with `plugins.probe_spec`. See [Selecting Probes](/documentation/vulnerability-scanning/configurations/selecting-probes). | | `user_agent` | `str` | `"garak/{version} (LLM vulnerability scanner https://garak.ai)"` | HTTP `User-Agent` garak sends. The `{version}` placeholder is substituted at runtime. | ## `AuditPluginsData` @@ -57,7 +57,7 @@ Controls probe, detector, generator, buff, and harness selection. |-------|------|---------|-------------| | `model_type` | `str \\| None` | `None` | Garak generator type override. Normally set on the target instead. | | `model_name` | `str \\| None` | `None` | Garak model name override. Normally set on the target instead. | -| `probe_spec` | `str` | `"all"` | Comma-separated probe selection. See [Selecting Probes](probes.md). | +| `probe_spec` | `str` | `"all"` | Comma-separated probe selection. See [Selecting Probes](/documentation/vulnerability-scanning/configurations/selecting-probes). | | `detector_spec` | `str` | `"auto"` | Detector selection — `"auto"` lets each probe pick its detectors. | | `extended_detectors` | `bool` | `False` | Enable garak's extended detector set. | | `buff_spec` | `str \\| None` | `None` | Comma-separated buff selection (prompt mutations applied before sending). | diff --git a/docs/auditor/index.mdx b/docs/auditor/index.mdx index bbf6393019a..ea63b24b7b5 100644 --- a/docs/auditor/index.mdx +++ b/docs/auditor/index.mdx @@ -13,8 +13,8 @@ description: Scan and audit large language models for jailbreaks, prompt injecti \{\{__auditor_long_name}} audits LLMs by probing them with adversarial prompts and detecting failures such as jailbreaks, prompt injection, encoding bypasses, and unsafe output generation. It is powered by [garak](https://github.com/NVIDIA/garak), NVIDIA's open-source LLM vulnerability scanner, and integrates with NeMo Platform so audits can target any model reachable through the Inference Gateway. - - + + --- @@ -22,11 +22,11 @@ It is powered by [garak](https://github.com/NVIDIA/garak), NVIDIA's open-source A typical audit looks like the following: -1. Create an [audit target](targets/index.md) for the model you want to test. -1. Create an [audit configuration](configs/index.md) that selects which garak probes and detectors to run, along with reporting settings. -1. [Run the audit](tutorials/run-audit-locally.md) and inspect the resulting JSONL, HTML, and hitlog reports. +1. Create an [audit target](/documentation/vulnerability-scanning/targets) for the model you want to test. +1. Create an [audit configuration](/documentation/vulnerability-scanning/configurations) that selects which garak probes and detectors to run, along with reporting settings. +1. [Run the audit](/documentation/vulnerability-scanning/tutorials/run-an-audit-locally) and inspect the resulting JSONL, HTML, and hitlog reports. -The plugin exposes both [synchronous and asynchronous](sdk-resources.md) Python entry points for each step. +The plugin exposes both [synchronous and asynchronous](/documentation/vulnerability-scanning/sdk-resources) Python entry points for each step. --- @@ -34,9 +34,9 @@ The plugin exposes both [synchronous and asynchronous](sdk-resources.md) Python Before you can run audits, you need a working NeMo Platform install with the auditor plugin enabled and a garak interpreter on disk. -- Follow [Setup](../get-started/setup.md) to install the platform and start local services. +- Follow [Setup](/documentation/get-started) to install the platform and start local services. - Install garak in a Python virtual environment. By default the plugin invokes `~/.auditor/.venv/bin/python -m garak`; override the interpreter path with `NEMO_AUDITOR_GARAK_PYTHON` if you installed it elsewhere. -- Configure at least one [Inference Gateway provider](../run-inference/about.md) so audits can route requests to the model under test. +- Configure at least one [Inference Gateway provider](/documentation/models-and-inference) so audits can route requests to the model under test. --- @@ -44,22 +44,22 @@ Before you can run audits, you need a working NeMo Platform install with the aud - + Define the model under test — generator type, model identifier, and inference endpoint. - + Choose probes, detectors, and reporting settings for the audit. - + End-to-end walkthrough: create entities, run the audit in-process, read the report artifacts. - + Reference for the `client.auditor` SDK surface: `configs`, `targets`, and `run()`. @@ -71,22 +71,22 @@ Reference for the `client.auditor` SDK surface: `configs`, `targets`, and `run() - + Field reference for `AuditConfig` and its `system`, `run`, `plugins`, and `reporting` sub-models. - + Field reference for `AuditTarget` (`type`, `model`, `options`). - + `probe_spec`, `probe_tags`, and `detector_spec` syntax with worked examples. - + How `nmp_uri_spec` resolves a target's URI through a NeMo Platform provider. diff --git a/docs/auditor/sdk-resources.mdx b/docs/auditor/sdk-resources.mdx index 099271786e2..def3a1095d4 100644 --- a/docs/auditor/sdk-resources.mdx +++ b/docs/auditor/sdk-resources.mdx @@ -8,7 +8,7 @@ description: "" The \{\{__auditor_short_name}} plugin mounts a Python SDK surface on the `nemo_platform` client at `client.auditor`. This page documents that surface: how to manage audit configurations and targets in the entity store, and how to run an audit in-process using the local execution path. -The CRUD methods exposed on `client.auditor.configs` and `client.auditor.targets` are 1:1 mirrors of the [audit configuration](configs/index.md) and [audit target](targets/index.md) lifecycle and use the same `AuditConfig` and `AuditTarget` pydantic schemas the entity store persists. +The CRUD methods exposed on `client.auditor.configs` and `client.auditor.targets` are 1:1 mirrors of the [audit configuration](/documentation/vulnerability-scanning/configurations) and [audit target](/documentation/vulnerability-scanning/targets) lifecycle and use the same `AuditConfig` and `AuditTarget` pydantic schemas the entity store persists. ## AuditorPluginResource @@ -37,7 +37,7 @@ auditor = client.auditor # AuditorPluginResource ### `configs` sub-resource -Five CRUD methods for `AuditConfig` entities. The full field reference is in [Configuration Schema](configs/schema.md). +Five CRUD methods for `AuditConfig` entities. The full field reference is in [Configuration Schema](/documentation/vulnerability-scanning/configurations/schema). | Method | Description | Returns | |--------|-------------|---------| @@ -49,7 +49,7 @@ Five CRUD methods for `AuditConfig` entities. The full field reference is in [Co ### `targets` sub-resource -Five CRUD methods for `AuditTarget` entities. The full field reference is in [Target Schema](targets/schema.md). +Five CRUD methods for `AuditTarget` entities. The full field reference is in [Target Schema](/documentation/vulnerability-scanning/targets/schema). | Method | Description | Returns | |--------|-------------|---------| diff --git a/docs/auditor/targets/index.mdx b/docs/auditor/targets/index.mdx index 6c1aa63f0c6..654922561a6 100644 --- a/docs/auditor/targets/index.mdx +++ b/docs/auditor/targets/index.mdx @@ -12,10 +12,10 @@ An `AuditTarget` identifies the model under test. It pairs a garak generator cla |-------|-------------| | `type` | A fully-qualified [garak generator class](https://reference.garak.ai/en/latest/generators.html), such as `nim.NVOpenAIChat`, `openai.OpenAIGenerator`, `rest.RestGenerator`, or `test.Blank`. | | `model` | The provider's model identifier passed through to garak (for example, `meta/llama-3.1-8b-instruct`). | -| `options` | A nested dict whose top-level key is the generator namespace (`nim`, `openai`, `rest`, ...). Contents are passed through to garak unchanged, with one exception: the [`nmp_uri_spec`](inference-gateway.md) sentinel is resolved to a concrete `uri` at run time. | +| `options` | A nested dict whose top-level key is the generator namespace (`nim`, `openai`, `rest`, ...). Contents are passed through to garak unchanged, with one exception: the [`nmp_uri_spec`](/documentation/vulnerability-scanning/targets/inference-gateway) sentinel is resolved to a concrete `uri` at run time. | | `description` | Optional free-form description shown in listings. | -See [Target Schema](schema.md) for the full field reference. +See [Target Schema](/documentation/vulnerability-scanning/targets/schema) for the full field reference. ## Common Target Types @@ -43,7 +43,7 @@ target = client.auditor.targets.create( ) ``` -The `nmp_uri_spec` block resolves to a concrete `uri` at run time. See [Inference Gateway](inference-gateway.md) for details. +The `nmp_uri_spec` block resolves to a concrete `uri` at run time. See [Inference Gateway](/documentation/vulnerability-scanning/targets/inference-gateway) for details. ### OpenAI-compatible Endpoint diff --git a/docs/auditor/targets/schema.mdx b/docs/auditor/targets/schema.mdx index 3854d14bece..8c5d2d9478a 100644 --- a/docs/auditor/targets/schema.mdx +++ b/docs/auditor/targets/schema.mdx @@ -23,4 +23,4 @@ The entity store adds the standard `NemoEntity` fields on retrieval: `id`, `enti `options` is intentionally opaque to the plugin — its contents are passed through to garak as the generator's `--generator_option_file` payload. Refer to garak's [generator documentation](https://reference.garak.ai/en/latest/generators.html) for the options each generator class accepts. -The plugin recognizes one sentinel inside `options.`: an `nmp_uri_spec` block is resolved at run time to a concrete `uri` value via the NeMo Platform Inference Gateway. See [Inference Gateway](inference-gateway.md) for the resolution rules and conflict semantics. +The plugin recognizes one sentinel inside `options.`: an `nmp_uri_spec` block is resolved at run time to a concrete `uri` value via the NeMo Platform Inference Gateway. See [Inference Gateway](/documentation/vulnerability-scanning/targets/inference-gateway) for the resolution rules and conflict semantics. diff --git a/docs/auditor/tutorials/index.mdx b/docs/auditor/tutorials/index.mdx index 1b6238b3cab..d9d106ba41a 100644 --- a/docs/auditor/tutorials/index.mdx +++ b/docs/auditor/tutorials/index.mdx @@ -4,15 +4,15 @@ description: "" --- -Use these tutorials to get hands-on with [\{\{__auditor_short_name}}](../index.md). +Use these tutorials to get hands-on with [\{\{__auditor_short_name}}](/documentation/vulnerability-scanning). ## Before You Start -Set up [a local instance of the platform](../../get-started/setup.md), install garak in a Python virtual environment (the plugin invokes `~/.auditor/.venv/bin/python` by default), and configure at least one [Inference Gateway provider](../../run-inference/about.md) before running the tutorial below. +Set up [a local instance of the platform](/documentation/get-started), install garak in a Python virtual environment (the plugin invokes `~/.auditor/.venv/bin/python` by default), and configure at least one [Inference Gateway provider](/documentation/models-and-inference) before running the tutorial below. - + Persist a configuration and target, run a single audit in-process, and inspect the resulting JSONL, HTML, and hitlog report artifacts. @@ -24,4 +24,4 @@ Persist a configuration and target, run a single audit in-process, and inspect t ## How It Works -For the conceptual overview of the audit workflow, see [About Auditing Models](../index.md). For the full SDK surface, see [SDK Resources](../sdk-resources.md). +For the conceptual overview of the audit workflow, see [About Auditing Models](/documentation/vulnerability-scanning). For the full SDK surface, see [SDK Resources](/documentation/vulnerability-scanning/sdk-resources). diff --git a/docs/auditor/tutorials/run-audit-locally.mdx b/docs/auditor/tutorials/run-audit-locally.mdx index 1b377e2637b..80d0fc4c13a 100644 --- a/docs/auditor/tutorials/run-audit-locally.mdx +++ b/docs/auditor/tutorials/run-audit-locally.mdx @@ -21,9 +21,9 @@ This tutorial takes approximately **10 minutes** to complete, plus however long ## Prerequisites -- Install and start NeMo Platform using the [Setup guide](../../get-started/setup.md). +- Install and start NeMo Platform using the [Setup guide](/documentation/get-started). - Install garak in a Python virtual environment so the plugin can shell out to it. By default the plugin invokes `~/.auditor/.venv/bin/python -m garak`. Override the interpreter path with `NEMO_AUDITOR_GARAK_PYTHON` if your install lives elsewhere. -- Configure at least one Inference Gateway provider — this tutorial uses a `build` provider named `build` that routes to NVIDIA-build models, but any chat-completion-compatible provider works. See [Inference Gateway](../targets/inference-gateway.md) for details on the `nmp_uri_spec` block used below. +- Configure at least one Inference Gateway provider — this tutorial uses a `build` provider named `build` that routes to NVIDIA-build models, but any chat-completion-compatible provider works. See [Inference Gateway](/documentation/vulnerability-scanning/targets/inference-gateway) for details on the `nmp_uri_spec` block used below. --- @@ -55,7 +55,7 @@ print(auditor.plugin_status()) # {'plugin': 'auditor', 'status': 'ok', ...} ``` -If `plugin_status()` raises a connection error, the platform is not running or `NMP_BASE_URL` is misconfigured. Refer back to [Setup](../../get-started/setup.md). +If `plugin_status()` raises a connection error, the platform is not running or `NMP_BASE_URL` is misconfigured. Refer back to [Setup](/documentation/get-started). --- @@ -84,7 +84,7 @@ config = auditor.configs.create( print(config.model_dump_json(indent=2)) ``` -For the full set of options on each sub-block, see [Configuration Schema](../configs/schema.md). For probe selection syntax, see [Selecting Probes](../configs/probes.md). +For the full set of options on each sub-block, see [Configuration Schema](/documentation/vulnerability-scanning/configurations/schema). For probe selection syntax, see [Selecting Probes](/documentation/vulnerability-scanning/configurations/selecting-probes). --- @@ -113,7 +113,7 @@ target = auditor.targets.create( print(target.model_dump_json(indent=2)) ``` -The `nmp_uri_spec` sentinel inside `options.nim` tells the plugin to resolve a concrete URI from the Inference Gateway provider at run time. See [Inference Gateway](../targets/inference-gateway.md) for the full conflict rules and resolution behavior. +The `nmp_uri_spec` sentinel inside `options.nim` tells the plugin to resolve a concrete URI from the Inference Gateway provider at run time. See [Inference Gateway](/documentation/vulnerability-scanning/targets/inference-gateway) for the full conflict rules and resolution behavior. --- @@ -221,6 +221,6 @@ Report artifacts are left in place under the scheduler's temporary directory; th You created an `AuditConfig` and `AuditTarget`, ran a single audit locally with `client.auditor.run(...)`, and loaded the resulting reports. -- For the full SDK surface — including async variants — see [SDK Resources](../sdk-resources.md). -- For more probe selection options, see [Selecting Probes](../configs/probes.md). -- For other generator types (OpenAI-compatible endpoints, REST targets, test targets), see [Audit Targets](../targets/index.md). +- For the full SDK surface — including async variants — see [SDK Resources](/documentation/vulnerability-scanning/sdk-resources). +- For more probe selection options, see [Selecting Probes](/documentation/vulnerability-scanning/configurations/selecting-probes). +- For other generator types (OpenAI-compatible endpoints, REST targets, test targets), see [Audit Targets](/documentation/vulnerability-scanning/targets). diff --git a/docs/auth/authorization/index.mdx b/docs/auth/authorization/index.mdx index 667ecec9ee0..c72b903b0de 100644 --- a/docs/auth/authorization/index.mdx +++ b/docs/auth/authorization/index.mdx @@ -38,7 +38,7 @@ Add users to workspaces, assign roles, manage members. Token-level scope model and two-layer authorization. - + Complete list of all permissions with role assignments. diff --git a/docs/auth/authorization/managing-access.mdx b/docs/auth/authorization/managing-access.mdx index 28071d08932..19661bf045e 100644 --- a/docs/auth/authorization/managing-access.mdx +++ b/docs/auth/authorization/managing-access.mdx @@ -6,14 +6,14 @@ Add users to workspaces, assign roles, and control who can access your resources -The SDK examples on this page use `NeMoPlatform()` with no arguments so that the client reads your active CLI context (set by `nemo auth login`). That is the right pattern for authorization workflows: you act as your logged-in identity and pass the workspace explicitly in each API call. For the standard local initialization pattern, see [CLI and SDK initialization](../../get-started/setup.md#setup-init). +The SDK examples on this page use `NeMoPlatform()` with no arguments so that the client reads your active CLI context (set by `nemo auth login`). That is the right pattern for authorization workflows: you act as your logged-in identity and pass the workspace explicitly in each API call. For the standard local initialization pattern, see [CLI and SDK initialization](/documentation/get-started#setup-init). ## Creating Workspaces Workspaces are the primary authorization boundary — all resources belong to a workspace, and access is controlled at the workspace level. When you create a workspace, you automatically become its Admin. -Create separate workspaces to isolate teams (`ml-research`, `nlp-team`), environments (`dev`, `staging`, `prod`), or projects. For detailed workspace management, refer to [Workspaces](../../get-started/concepts/workspaces.md). +Create separate workspaces to isolate teams (`ml-research`, `nlp-team`), environments (`dev`, `staging`, `prod`), or projects. For detailed workspace management, refer to [Workspaces](/documentation/get-started/core-concepts/workspaces). diff --git a/docs/auth/deployment/configuration.mdx b/docs/auth/deployment/configuration.mdx index abf48f4de56..50d8a2dbb66 100644 --- a/docs/auth/deployment/configuration.mdx +++ b/docs/auth/deployment/configuration.mdx @@ -37,7 +37,7 @@ auth: admin_email: "your-admin@company.com" ``` -For a complete reference of all `auth` fields and their defaults, see the [platform configuration reference](../../set-up/config-reference.md). Auth-related values are found under `platformConfig.auth` in the values file. +For a complete reference of all `auth` fields and their defaults, see the [platform configuration reference](/documentation/reference/configuration-reference). Auth-related values are found under `platformConfig.auth` in the values file. For OIDC-specific fields (`auth.oidc`), see [OIDC Setup](../authentication/oidc.md). diff --git a/docs/cli/index.mdx b/docs/cli/index.mdx index a8d52fa6dfd..f417b388d81 100644 --- a/docs/cli/index.mdx +++ b/docs/cli/index.mdx @@ -48,7 +48,7 @@ When installed in a virtual environment, the `nemo` command is only available wh nemo --help ``` -If you see `Unknown command: nemo`, see the [troubleshooting](troubleshooting.md) page. +If you see `Unknown command: nemo`, see the [troubleshooting](/documentation/reference/cli-reference/troubleshooting) page. ## Getting Started @@ -67,7 +67,7 @@ nemo auth login nemo workspaces list ``` -This command outputs a list of available workspaces. If the connection fails, see [troubleshooting](troubleshooting.md) for common issues and solutions. +This command outputs a list of available workspaces. If the connection fails, see [troubleshooting](/documentation/reference/cli-reference/troubleshooting) for common issues and solutions. ## Command Structure @@ -133,7 +133,7 @@ nemo models list -f csv --all-pages --no-truncate --output-columns all > models. ``` ## Next Steps -- [configuration](configuration.md) - Contexts, authentication, environment variables, and shell completion -- [working-with-resources](working-with-resources.md) - Output formats, pagination, and input methods -- [troubleshooting](troubleshooting.md) - Common issues and solutions -- [reference](reference.md) - Full command reference +- [configuration](/documentation/reference/cli-reference/configuration) - Contexts, authentication, environment variables, and shell completion +- [working-with-resources](/documentation/reference/cli-reference/working-with-resources) - Output formats, pagination, and input methods +- [troubleshooting](/documentation/reference/cli-reference/troubleshooting) - Common issues and solutions +- [reference](/documentation/reference/cli-reference/full-cli-reference) - Full command reference diff --git a/docs/cli/troubleshooting.mdx b/docs/cli/troubleshooting.mdx index bb3904de93d..9c5550e029e 100644 --- a/docs/cli/troubleshooting.mdx +++ b/docs/cli/troubleshooting.mdx @@ -100,5 +100,5 @@ nemo config view ## Getting Help - Use `--help` on any command for usage information -- See the [reference](reference.md) for complete command documentation -- See [configuration](configuration.md) for detailed configuration options +- See the [reference](/documentation/reference/cli-reference/full-cli-reference) for complete command documentation +- See [configuration](/documentation/reference/cli-reference/configuration) for detailed configuration options diff --git a/docs/customizer/about.mdx b/docs/customizer/about.mdx index 45e895f87c1..be0a996313a 100644 --- a/docs/customizer/about.mdx +++ b/docs/customizer/about.mdx @@ -147,7 +147,7 @@ Below are some examples of how you might format your dataset to perform a handfu When testing models trained with prompt/completion datasets, use the `/v1/completions` endpoint instead of `/v1/chat/completions`. -For details, refer to the [Dataset Formatting tutorial](tutorials/format-training-dataset.md#format-a-prompt-completion-dataset). +For details, refer to the [Dataset Formatting tutorial](/documentation/fine-tune-models/tutorials/format-training-dataset#format-a-prompt-completion-dataset). #### Document Classification @@ -197,7 +197,7 @@ completion: "" Most of the models support Instruction Templates for training, the expected dataset conforms with the standard [OpenAI messages format](https://platform.openai.com/docs/guides/fine-tuning#multi-turn-chat-examples). Additionally, some models support tool calling which have additional optional parameters of `tools` at the top level of each entry and `tool_calls` per message. -For more information refer to our [in-depth instructions](tutorials/format-training-dataset.md#format-a-conversation-dataset). +For more information refer to our [in-depth instructions](/documentation/fine-tune-models/tutorials/format-training-dataset#format-a-conversation-dataset). ## Hyperparameters diff --git a/docs/customizer/index.mdx b/docs/customizer/index.mdx index fca8b95d18b..55207812b49 100644 --- a/docs/customizer/index.mdx +++ b/docs/customizer/index.mdx @@ -11,14 +11,14 @@ Learn how to fine-tune models by making requests to NVIDIA NeMo Customizer throu At a high level, the fine-tuning workflow consists of the following steps: 1. [Create a Model Entity](manage-model-entities/index.md) pointing to your base model checkpoint (stored as a FileSet). -1. Format a compatible [dataset](tutorials/format-training-dataset.md). -1. [Create a customization job](manage-customization-jobs/index.md) referencing the Model Entity. +1. Format a compatible [dataset](/documentation/fine-tune-models/tutorials/format-training-dataset). +1. [Create a customization job](/documentation/fine-tune-models/manage-customization-jobs) referencing the Model Entity. 1. Monitor the job until it completes. 1. The customization job automatically creates either: - **LoRA jobs**: An adapter attached to the original Model Entity - **Full fine-tuning jobs**: A new Model Entity with the customized weights -1. [Deploy the model](../run-inference/about.md) using the Deployment Management Service. -1. Move on to [Evaluate the output model](../evaluator/index.md). +1. [Deploy the model](/documentation/models-and-inference) using the Deployment Management Service. +1. Move on to [Evaluate the output model](/documentation/evaluate-models). --- @@ -49,7 +49,7 @@ View the available Phi models from Microsoft, designed for strong reasoning capa View the available GPT-OSS models supported for Full SFT customization. - + View the available embedding models for question-answering and retrieval tasks. @@ -63,7 +63,7 @@ Perform common fine-tuning tasks. - + Create, list, view, and cancel customization jobs. @@ -73,7 +73,7 @@ Create, list, view, and cancel customization jobs. Create FileSets and Model Entities to prepare base models for customization. - + Upload and manage datasets for training. @@ -89,7 +89,7 @@ Follow these tutorials to learn how to accomplish common fine-tuning tasks. - + Learn how to format datasets for different model types. @@ -157,7 +157,7 @@ View the available hyperparameters and their valid options that you can set when View the OpenAPI specification for Customizer. - + View troubleshooting tips for failed jobs. diff --git a/docs/customizer/manage-customization-jobs/create-job.mdx b/docs/customizer/manage-customization-jobs/create-job.mdx index b53a20a2f9a..7889da8e7a0 100644 --- a/docs/customizer/manage-customization-jobs/create-job.mdx +++ b/docs/customizer/manage-customization-jobs/create-job.mdx @@ -9,7 +9,7 @@ Before you can create a customization job, make sure that you have: - Obtained the base URL of your NeMo Platform. - Created a [FileSet and Model Entity](../manage-model-entities/index.md) for your base model. -- [Uploaded a dataset](../../get-started/concepts/manage-files.md) as a FileSet. +- [Uploaded a dataset](/documentation/get-started/core-concepts/manage-files) as a FileSet. - Determined the [training configuration](hyperparameters.md) you want to use for the customization job. - Verified that the platform has sufficient storage for the job. Full SFT jobs require approximately 3× the base model size in free disk space; LoRA jobs require approximately 1.5×. See [ft-tut-understand-models](../tutorials/understand-configurations-and-models.md) for details. If you are also deploying the model from a base checkpoint fileset, plan for ~2.5× model size overall for LoRA. - Set the `NMP_BASE_URL` environment variable to your NeMo Platform endpoint. diff --git a/docs/customizer/manage-customization-jobs/customization-job-reference.mdx b/docs/customizer/manage-customization-jobs/customization-job-reference.mdx index a2d5958ac75..fc930138a39 100644 --- a/docs/customizer/manage-customization-jobs/customization-job-reference.mdx +++ b/docs/customizer/manage-customization-jobs/customization-job-reference.mdx @@ -5,7 +5,7 @@ description: "" Use this page when you need field-level details for customization job specifications, the complete API schema, or integration options. -For concepts, see [Customization Job overview](index.md). +For concepts, see [Customization Job overview](/documentation/fine-tune-models/manage-customization-jobs). ## Key Fields @@ -86,7 +86,7 @@ nemo customization jobs create my-job \ The `api_key_secret` field references a stored secret containing your `WANDB_API_KEY`. Use the secret name (e.g., `"my-wandb-key"`) to resolve it from the request workspace. -To create the secret, see [Weights & Biases Keys](../../get-started/concepts/manage-secrets.md). +To create the secret, see [Weights & Biases Keys](/documentation/get-started/core-concepts/manage-secrets). | Field | Description | |-------|-------------| @@ -160,7 +160,7 @@ nemo customization jobs create my-job \ ## Next Steps -- [Create a customization job](create-job.md): Start a job with a model, dataset, training configuration, and optional integrations. +- [Create a customization job](/documentation/fine-tune-models/manage-customization-jobs/create-a-customization-job): Start a job with a model, dataset, training configuration, and optional integrations. - [Monitor training metrics](../tutorials/metrics.md): View logs and metrics through MLflow or W&B. -- [Manage secrets](../../get-started/concepts/manage-secrets.md): Store credentials such as W&B API keys and provider tokens. -- [Troubleshooting MLflow integrations](../../troubleshooting/customizer.md): Diagnose failed or misconfigured customization jobs. +- [Manage secrets](/documentation/get-started/core-concepts/manage-secrets): Store credentials such as W&B API keys and provider tokens. +- [Troubleshooting MLflow integrations](/documentation/reference/troubleshooting/customizer): Diagnose failed or misconfigured customization jobs. diff --git a/docs/customizer/manage-customization-jobs/index.mdx b/docs/customizer/manage-customization-jobs/index.mdx index 3b5458adeaf..c047f991c45 100644 --- a/docs/customizer/manage-customization-jobs/index.mdx +++ b/docs/customizer/manage-customization-jobs/index.mdx @@ -3,7 +3,7 @@ title: "Manage Customization Jobs" description: "" --- -Use customization jobs to fine-tune a [model](../models/index.md) using a [dataset](../../get-started/concepts/manage-files.md) and [hyperparameters](hyperparameters.md). +Use customization jobs to fine-tune a [model](../models/index.md) using a [dataset](/documentation/get-started/core-concepts/manage-files) and [hyperparameters](hyperparameters.md). ## How It Works @@ -16,7 +16,7 @@ This design keeps adapters organized with their parent models and simplifies dep ## Prerequisites -Before you can customize a model using a customization job, make sure that you have `prepared and uploaded a dataset <../tutorials/format-training-dataset>` to the dataset repository. See also [format-training-dataset](../tutorials/format-training-dataset.md) for dataset formatting requirements. +Before you can customize a model using a customization job, make sure that you have `prepared and uploaded a dataset <../tutorials/format-training-dataset>` to the dataset repository. See also [format-training-dataset](/documentation/fine-tune-models/tutorials/format-training-dataset) for dataset formatting requirements. --- @@ -26,12 +26,12 @@ Perform common customization job tasks. -The value for `NMP_BASE_URL` will depend on your deployment. After the standard [Setup](../../get-started/setup.md) flow, the default local URL is `http://localhost:8080`. Otherwise, consult with your cluster administrator. +The value for `NMP_BASE_URL` will depend on your deployment. After the standard [Setup](/documentation/get-started) flow, the default local URL is `http://localhost:8080`. Otherwise, consult with your cluster administrator. - + Create a customization job using SFT, DPO, or Knowledge Distillation. @@ -65,7 +65,7 @@ Refer to the following pages for more information on customization jobs. Review the hyperparameters that you can use to customize a model. - + View troubleshooting tips for failed jobs. diff --git a/docs/customizer/manage-model-entities/create-fileset.mdx b/docs/customizer/manage-model-entities/create-fileset.mdx index 6705342ecff..f5f996dfdb6 100644 --- a/docs/customizer/manage-model-entities/create-fileset.mdx +++ b/docs/customizer/manage-model-entities/create-fileset.mdx @@ -8,7 +8,7 @@ Create a FileSet containing your base model checkpoint before creating a Model E ## Prerequisites - Obtained the base URL of your NeMo Platform. -- For HuggingFace models: Created a secret with your HF token. Refer to [Manage Secrets](../../get-started/concepts/manage-secrets.md). +- For HuggingFace models: Created a secret with your HF token. Refer to [Manage Secrets](/documentation/get-started/core-concepts/manage-secrets). - Set the `NMP_BASE_URL` environment variable. ```bash diff --git a/docs/customizer/manage-model-entities/create-model-entity.mdx b/docs/customizer/manage-model-entities/create-model-entity.mdx index 1d1579ebf46..f695d7755cc 100644 --- a/docs/customizer/manage-model-entities/create-model-entity.mdx +++ b/docs/customizer/manage-model-entities/create-model-entity.mdx @@ -7,7 +7,7 @@ Create a Model Entity that references your FileSet to enable customization jobs. ## Prerequisites -- Created a FileSet containing your model checkpoint (refer to [Create a Model FileSet](create-fileset.md)). +- Created a FileSet containing your model checkpoint (refer to [Create a Model FileSet](/documentation/fine-tune-models/manage-model-entities/create-a-model-file-set)). - Set the `NMP_BASE_URL` environment variable. ```bash @@ -141,7 +141,7 @@ job = client.customization.jobs.create( ) ``` -Refer to [create-job](../manage-customization-jobs/create-job.md) for complete job creation details. +Refer to [create-job](/documentation/fine-tune-models/manage-customization-jobs/create-a-customization-job) for complete job creation details. --- @@ -169,5 +169,5 @@ else: ## Next Steps -- [Create a customization job](../manage-customization-jobs/create-job.md) +- [Create a customization job](/documentation/fine-tune-models/manage-customization-jobs/create-a-customization-job) - [Understand hyperparameters](../manage-customization-jobs/hyperparameters.md) diff --git a/docs/customizer/manage-model-entities/index.mdx b/docs/customizer/manage-model-entities/index.mdx index e8631426d57..6c0f15973d7 100644 --- a/docs/customizer/manage-model-entities/index.mdx +++ b/docs/customizer/manage-model-entities/index.mdx @@ -9,7 +9,7 @@ Before running a customization job, you need to set up a **Model Entity** that p - + Create a FileSet containing your base model checkpoint from HuggingFace, NGC, or local storage. @@ -56,7 +56,7 @@ Complete example of setting up a model for customization: -**HuggingFace Token**: If downloading from a gated HuggingFace repository (like Llama models), you will need to create a secret containing your HuggingFace API token first. Refer to [Manage Secrets](../../get-started/concepts/manage-secrets.md) for instructions. +**HuggingFace Token**: If downloading from a gated HuggingFace repository (like Llama models), you will need to create a secret containing your HuggingFace API token first. Refer to [Manage Secrets](/documentation/get-started/core-concepts/manage-secrets) for instructions. ```python diff --git a/docs/customizer/models/data-format.mdx b/docs/customizer/models/data-format.mdx index ac6f7e86ed6..a3aafa4be54 100644 --- a/docs/customizer/models/data-format.mdx +++ b/docs/customizer/models/data-format.mdx @@ -11,7 +11,7 @@ Use the following guidelines to prepare your training dataset for the supported - **File Format**: Save your training data as `.jsonl` files (one JSON object per line). - **Validation**: Each record is automatically validated against the appropriate schema when training begins. The required format depends on the `training_type` (SFT, DPO) specified in your job configuration. -For dataset creation tutorials, refer to [Format Training Dataset](../tutorials/format-training-dataset.md). +For dataset creation tutorials, refer to [Format Training Dataset](/documentation/fine-tune-models/tutorials/format-training-dataset). ## Dataset Formats diff --git a/docs/customizer/models/embedding.mdx b/docs/customizer/models/embedding.mdx index 42805cea321..28f8d109b6e 100644 --- a/docs/customizer/models/embedding.mdx +++ b/docs/customizer/models/embedding.mdx @@ -76,7 +76,7 @@ NVIDIA recommends evaluating fine-tuned embedding models against the baseline to This model supports inference deployment through NVIDIA Inference Microservices (NIM). After customization, access your model through the **Inference Gateway**: -1. **Deploy the model**: Create a ModelDeploymentConfig and ModelDeployment to deploy your fine-tuned model. See [about](../../run-inference/about.md) for details. +1. **Deploy the model**: Create a ModelDeploymentConfig and ModelDeployment to deploy your fine-tuned model. See [about](/documentation/models-and-inference) for details. 2. **Access through Inference Gateway**: The Inference Gateway provides unified access to all deployed models via three routing patterns: - **Model Entity routing**: `/v2/workspaces/{workspace}/inference/gateway/model/{name}/-/v1/embeddings` diff --git a/docs/customizer/models/index.mdx b/docs/customizer/models/index.mdx index d2d2214d7ab..f07f44c5772 100644 --- a/docs/customizer/models/index.mdx +++ b/docs/customizer/models/index.mdx @@ -9,7 +9,7 @@ Explore the model families and sizes supported by NVIDIA NeMo Customizer. For information on setting up model entities for customization, see the [Manage Model Entities](../manage-model-entities/index.md) guide. -For fine-tuning and deployment tutorials, see the [Tutorials](../tutorials/index.md) guide. +For fine-tuning and deployment tutorials, see the [Tutorials](/documentation/fine-tune-models/tutorials) guide. ## Before You Start @@ -38,7 +38,7 @@ View the available Llama Nemotron models from NVIDIA, including Nano and Super v View the available Phi models from Microsoft, designed for strong reasoning capabilities with efficient deployment. - + View the available embedding models optimized for retrieval and question-answering tasks. @@ -96,4 +96,4 @@ The following models support both chat and completion model training. |--|--|--| | [nvidia/llama-nemotron-embed-1b-v2](https://huggingface.co/nvidia/llama-nemotron-embed-1b-v2) | Full SFT, LoRA (merged) | Supported | -For detailed technical specifications and configuration information for embedding models, see the [Embedding Models](embedding.md) page. +For detailed technical specifications and configuration information for embedding models, see the [Embedding Models](/documentation/fine-tune-models/models/embedding) page. diff --git a/docs/customizer/tutorials/format-training-dataset.mdx b/docs/customizer/tutorials/format-training-dataset.mdx index 829ac042750..4bd02640dbb 100644 --- a/docs/customizer/tutorials/format-training-dataset.mdx +++ b/docs/customizer/tutorials/format-training-dataset.mdx @@ -228,7 +228,7 @@ To run inference with a chat model, you first need to deploy the model, then use -Before running inference, ensure your model is deployed. See [about](../../run-inference/about.md) for details on creating a ModelDeploymentConfig and ModelDeployment. +Before running inference, ensure your model is deployed. See [about](/documentation/models-and-inference) for details on creating a ModelDeploymentConfig and ModelDeployment. ```python @@ -285,7 +285,7 @@ To run inference with a completion model, use the Inference Gateway to send requ -Before running inference, ensure your model is deployed. See [about](../../run-inference/about.md) for details on creating a ModelDeploymentConfig and ModelDeployment. +Before running inference, ensure your model is deployed. See [about](/documentation/models-and-inference) for details on creating a ModelDeploymentConfig and ModelDeployment. ```python diff --git a/docs/customizer/tutorials/index.mdx b/docs/customizer/tutorials/index.mdx index 0919cd4a395..668f3780bdf 100644 --- a/docs/customizer/tutorials/index.mdx +++ b/docs/customizer/tutorials/index.mdx @@ -6,7 +6,7 @@ Use the tutorials in this section to gain a deeper understanding of how the NVID -Tutorials are organized by complexity and typically build on one another. The tutorials reference `NMP_BASE_URL`, which is the base URL of your NeMo Platform deployment. Refer to the [Setup guide](../../get-started/setup.md) for installation, setup, and platform URL guidance. +Tutorials are organized by complexity and typically build on one another. The tutorials reference `NMP_BASE_URL`, which is the base URL of your NeMo Platform deployment. Refer to the [Setup guide](/documentation/get-started) for installation, setup, and platform URL guidance. @@ -30,7 +30,7 @@ Learn the fundamentals of how NeMo Customizer works with Model Entities and Adap - + Learn how to format datasets for different model types. diff --git a/docs/customizer/tutorials/metrics.mdx b/docs/customizer/tutorials/metrics.mdx index e69d9fd42cd..1155bf516c2 100644 --- a/docs/customizer/tutorials/metrics.mdx +++ b/docs/customizer/tutorials/metrics.mdx @@ -88,7 +88,7 @@ MLflow integration is configured at the cluster level. Contact your administrato ### Using Weights & Biases -If your customization job was created with W&B integration enabled (see [Weights & Biases Integration](../manage-customization-jobs/create-job.md)): +If your customization job was created with W&B integration enabled (see [Weights & Biases Integration](/documentation/fine-tune-models/manage-customization-jobs/create-a-customization-job)): 1. Go to [wandb.ai](https://wandb.ai/home) and navigate to your project 2. Find the run corresponding to your customization job @@ -132,13 +132,13 @@ print(f"Status: {job.status}") The `api_key_secret` field references a stored secret containing your `WANDB_API_KEY`. Use the secret name (e.g., `"my-wandb-key"`) to resolve it from the request workspace. -To create the secret, see [Weights & Biases Keys](../../get-started/concepts/manage-secrets.md). +To create the secret, see [Weights & Biases Keys](/documentation/get-started/core-concepts/manage-secrets). Then view your results at [wandb.ai](https://wandb.ai/home) under your project. ![W&B charts example](../_images/wandb_charts_example.png) -The W&B integration is optional and must be configured when [creating the customization job](../manage-customization-jobs/create-job.md). When enabled, training metrics are sent to W&B using your API key. While we encrypt your API key and don't log it internally, please review W&B's terms of service before use. +The W&B integration is optional and must be configured when [creating the customization job](/documentation/fine-tune-models/manage-customization-jobs/create-a-customization-job). When enabled, training metrics are sent to W&B using your API key. While we encrypt your API key and don't log it internally, please review W&B's terms of service before use. diff --git a/docs/customizer/tutorials/understand-configurations-and-models.mdx b/docs/customizer/tutorials/understand-configurations-and-models.mdx index 65446912a01..1063ceb9d94 100644 --- a/docs/customizer/tutorials/understand-configurations-and-models.mdx +++ b/docs/customizer/tutorials/understand-configurations-and-models.mdx @@ -255,7 +255,7 @@ Ensure your platform's shared persistent volume has at least **3× the base mode of free space before starting a full SFT job, or **1.5×** for LoRA jobs. -For troubleshooting disk-related failures, see [customizer](../../troubleshooting/customizer.md). +For troubleshooting disk-related failures, see [customizer](/documentation/reference/troubleshooting/customizer). ### Parallelism Parameters Explained @@ -401,7 +401,7 @@ Now that you understand how Model Entities and Adapters work, you're ready to pr - + Learn how to prepare your data for fine-tuning. diff --git a/docs/data-designer/cli.mdx b/docs/data-designer/cli.mdx index 7f4a4f1f069..8ad1daf480e 100644 --- a/docs/data-designer/cli.mdx +++ b/docs/data-designer/cli.mdx @@ -28,7 +28,7 @@ def load_config_builder() -> dd.DataDesignerConfigBuilder: return config_builder ``` -The same configuration source can usually be used with `run` or `submit`. Resource choices determine whether it is compatible with NeMo Services execution; see [Execution Modes](execution-modes.md). +The same configuration source can usually be used with `run` or `submit`. Resource choices determine whether it is compatible with NeMo Services execution; see [Execution Modes](/documentation/design-synthetic-data/execution-modes). ## Run Versus Submit diff --git a/docs/data-designer/index.mdx b/docs/data-designer/index.mdx index 93235d84ceb..d5d89aaff99 100644 --- a/docs/data-designer/index.mdx +++ b/docs/data-designer/index.mdx @@ -19,7 +19,7 @@ Data Designer separates **configuration** from **execution**. The code snippets below are for conceptual demonstration purposes only. -For runnable examples, see the [tutorials](tutorials/index.md). +For runnable examples, see the [tutorials](/documentation/design-synthetic-data/tutorials). ### 1. Build Configurations @@ -60,7 +60,7 @@ The same configuration can run through different plugin surfaces: `run` versus `submit` primarily controls where the plugin workload execution happens. A local `run` can be fully local, but it is not an offline-only mode: it can still use the Files API, Secrets API, and Inference Gateway API from a running NeMo Services cluster when the configuration references the corresponding resources. -See [Execution Modes](execution-modes.md) for the full model. +See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for the full model. ## NeMo Services Integration @@ -79,22 +79,22 @@ These integrations are required for `submit` and SDK execution. They are optiona - + Understand local execution, NeMo Services execution, and NeMo resources. - + Run previews and create datasets with `nemo data-designer`. - + Learn through examples: basics, seeding, and more. - + Move configurations between local CLI and NeMo Services execution. diff --git a/docs/data-designer/migration.mdx b/docs/data-designer/migration.mdx index 07e277ec596..d40dc7c1758 100644 --- a/docs/data-designer/migration.mdx +++ b/docs/data-designer/migration.mdx @@ -6,7 +6,7 @@ description: "" Data Designer configurations are mostly portable across plugin execution modes. The main migration question is not "library versus service"; it is whether the workload executes locally in the CLI process or through NeMo Services, and which resources the configuration references. -See [Execution Modes](execution-modes.md) for the full two-axis model. +See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for the full two-axis model. ## What Usually Stays the Same @@ -80,7 +80,7 @@ Before switching execution modes, verify: ## Getting Help -- **Execution Modes:** See [Execution Modes](execution-modes.md) for the conceptual model. -- **CLI:** See [Data Designer CLI](cli.md) for `run`, `submit`, and persona commands. -- **Tutorials:** Follow the [tutorials](tutorials/index.md) for hands-on examples. +- **Execution Modes:** See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for the conceptual model. +- **CLI:** See [Data Designer CLI](/documentation/design-synthetic-data/cli) for `run`, `submit`, and persona commands. +- **Tutorials:** Follow the [tutorials](/documentation/design-synthetic-data/tutorials) for hands-on examples. - **Library Docs:** Refer to the [open-source library documentation](https://docs.nvidia.com/nemo/datadesigner/v0.6.0/getting-started/welcome) for configuration details. diff --git a/docs/data-designer/tutorials/basics.mdx b/docs/data-designer/tutorials/basics.mdx index be5a42123f2..61f72c08d47 100644 --- a/docs/data-designer/tutorials/basics.mdx +++ b/docs/data-designer/tutorials/basics.mdx @@ -10,7 +10,7 @@ For more detail about column behavior, see the [open-source library's version](h ## Prerequisites -Ensure you have completed the [tutorials prerequisites](index.md#prerequisites). This tutorial uses an Inference Gateway provider, so local CLI `run` and NeMo Services execution both need access to the Inference Gateway API in a running NeMo Services cluster. +Ensure you have completed the [tutorials prerequisites](/documentation/design-synthetic-data/tutorials#prerequisites). This tutorial uses an Inference Gateway provider, so local CLI `run` and NeMo Services execution both need access to the Inference Gateway API in a running NeMo Services cluster. ## Part 1: Build the Configuration @@ -303,7 +303,7 @@ When you use CLI `submit` or the SDK today: ## Next Steps -- **Seed data:** Learn how to use external datasets in the [seeding tutorial](seeding.md) -- **Execution modes:** Learn more about local and NeMo Services execution in [Execution Modes](../execution-modes.md) +- **Seed data:** Learn how to use external datasets in the [seeding tutorial](/documentation/design-synthetic-data/tutorials/seeding-with-external-datasets) +- **Execution modes:** Learn more about local and NeMo Services execution in [Execution Modes](/documentation/design-synthetic-data/execution-modes) - **Column types:** Explore all available column types in the [library documentation](https://docs.nvidia.com/nemo/datadesigner/v0.6.0/concepts/columns) - **Advanced features:** Learn about [processors](https://docs.nvidia.com/nemo/datadesigner/v0.6.0/concepts/processors) and [validation](https://docs.nvidia.com/nemo/datadesigner/v0.6.0/concepts/validators) diff --git a/docs/data-designer/tutorials/index.mdx b/docs/data-designer/tutorials/index.mdx index b4e85b1a4ec..3eb14227680 100644 --- a/docs/data-designer/tutorials/index.mdx +++ b/docs/data-designer/tutorials/index.mdx @@ -9,7 +9,7 @@ These tutorials demonstrate how to build Data Designer configurations and execut The code snippets on this page are for conceptual demonstration purposes only. -For runnable examples, jump ahead to the [Basics](basics.md) or [Seeding](seeding.md) tutorial. +For runnable examples, jump ahead to the [Basics](/documentation/design-synthetic-data/tutorials/the-basics) or [Seeding](/documentation/design-synthetic-data/tutorials/seeding-with-external-datasets) tutorial. ## Configuration and Execution @@ -54,7 +54,7 @@ job = data_designer.create(config_builder, num_records=1000) -`run` versus `submit` primarily controls where the workload executes. Local `run` can still use the Files API, Secrets API, and Inference Gateway API from a running NeMo Services cluster when the configuration references the corresponding resources. See [Execution Modes](../execution-modes.md) for details. +`run` versus `submit` primarily controls where the workload executes. Local `run` can still use the Files API, Secrets API, and Inference Gateway API from a running NeMo Services cluster when the configuration references the corresponding resources. See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for details. ## Execution-Specific Considerations @@ -70,22 +70,22 @@ When running through the plugin, supported resources depend on the execution mod ## Prerequisites -These tutorials use an [Inference Gateway](../../run-inference/about.md) provider for model calls, so a NeMo Services cluster must be running before you preview or create data — including with local CLI `run` (see [Execution Modes](../execution-modes.md#local-nemo-services-execution) for more about this distinction). -Complete [Setup](../../get-started/setup.md) to ensure you have the NeMo Services running locally and an inference provider available. +These tutorials use an [Inference Gateway](/documentation/models-and-inference) provider for model calls, so a NeMo Services cluster must be running before you preview or create data — including with local CLI `run` (see [Execution Modes](/documentation/design-synthetic-data/execution-modes#local-nemo-services-execution) for more about this distinction). +Complete [Setup](/documentation/get-started) to ensure you have the NeMo Services running locally and an inference provider available. These tutorials reference the default NVIDIA Build model provider, which is created as `default/nvidia-build` during setup. ## Tutorials - + Generate a product review dataset using samplers and LLM-generated text. Learn the fundamentals of building configurations and executing jobs. beginner data-designer - + Use external datasets to ground synthetic data generation. Generate realistic patient medical notes from symptom-to-diagnosis data. diff --git a/docs/data-designer/tutorials/seeding.mdx b/docs/data-designer/tutorials/seeding.mdx index ad481c593a8..ddfc98290ee 100644 --- a/docs/data-designer/tutorials/seeding.mdx +++ b/docs/data-designer/tutorials/seeding.mdx @@ -58,7 +58,7 @@ FilesetFileSeedSource( ## Prerequisites -Ensure you have completed the [tutorials prerequisites](index.md#prerequisites). This tutorial uses an Inference Gateway provider, so local CLI `run` and NeMo Services execution both need access to the Inference Gateway API in a running NeMo Services cluster. +Ensure you have completed the [tutorials prerequisites](/documentation/design-synthetic-data/tutorials#prerequisites). This tutorial uses an Inference Gateway provider, so local CLI `run` and NeMo Services execution both need access to the Inference Gateway API in a running NeMo Services cluster. ## Example: Medical Notes from Symptom Data @@ -343,6 +343,6 @@ When you configure a seed dataset: ## Next Steps -- **Execution modes:** Learn more about local and NeMo Services execution in [Execution Modes](../execution-modes.md) +- **Execution modes:** Learn more about local and NeMo Services execution in [Execution Modes](/documentation/design-synthetic-data/execution-modes) - **Column types:** Explore all available column types in the [library documentation](https://docs.nvidia.com/nemo/datadesigner/v0.6.0/concepts/columns) - **Processors:** Transform your data with processors in the [library documentation](https://docs.nvidia.com/nemo/datadesigner/v0.6.0/concepts/processors) diff --git a/docs/evaluator/benchmarks/agentic.mdx b/docs/evaluator/benchmarks/agentic.mdx index 9a0c07b4f2c..f564f0a8f85 100644 --- a/docs/evaluator/benchmarks/agentic.mdx +++ b/docs/evaluator/benchmarks/agentic.mdx @@ -116,9 +116,9 @@ After creating a job, navigate to [Benchmark Job Management](job-management.md) -- [Agent Configuration](../metrics/agent-configuration.md) - Use agents (generic or NAT) as targets in online evaluation and benchmark jobs -- [Agentic Evaluation Metrics](../metrics/agentic.md) - Detailed metric documentation for evaluating agentic workflows -- [Managing Secrets](../../get-started/concepts/manage-secrets.md) - Store API keys for external APIs +- [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration) - Use agents (generic or NAT) as targets in online evaluation and benchmark jobs +- [Agentic Evaluation Metrics](/documentation/evaluate-models/metrics/agentic-metrics) - Detailed metric documentation for evaluating agentic workflows +- [Managing Secrets](/documentation/get-started/core-concepts/manage-secrets) - Store API keys for external APIs - [Evaluation Results](results.md) - Understanding and downloading results diff --git a/docs/evaluator/benchmarks/custom.mdx b/docs/evaluator/benchmarks/custom.mdx index 09688621d05..adece802399 100644 --- a/docs/evaluator/benchmarks/custom.mdx +++ b/docs/evaluator/benchmarks/custom.mdx @@ -4,20 +4,20 @@ description: "" --- -Custom benchmarks allow you to create reusable evaluation suites tailored to your specific use case. A benchmark combines one or more [metrics](../metrics/index.md) with a dataset, enabling consistent evaluation across multiple models or pipeline versions. +Custom benchmarks allow you to create reusable evaluation suites tailored to your specific use case. A benchmark combines one or more [metrics](/documentation/evaluate-models/metrics) with a dataset, enabling consistent evaluation across multiple models or pipeline versions. -Custom benchmarks can only include custom metrics that you create in your workspace. System metrics (in the `system` workspace) cannot be included in custom benchmarks at this time. To use system metrics, refer to [Industry Benchmarks](industry.md). +Custom benchmarks can only include custom metrics that you create in your workspace. System metrics (in the `system` workspace) cannot be included in custom benchmarks at this time. To use system metrics, refer to [Industry Benchmarks](/documentation/evaluate-models/benchmarks/industry-benchmarks). ## Prerequisites Before creating a custom benchmark, ensure you have: -- A [workspace](../../get-started/concepts/workspaces.md) created for your project -- One or more [custom metrics](../metrics/index.md) defined in your workspace -- A dataset uploaded as a [fileset](../../get-started/concepts/manage-files.md) +- A [workspace](/documentation/get-started/core-concepts/workspaces) created for your project +- One or more [custom metrics](/documentation/evaluate-models/metrics) defined in your workspace +- A dataset uploaded as a [fileset](/documentation/get-started/core-concepts/manage-files) ```python import os @@ -31,7 +31,7 @@ client = NeMoPlatform( -Set `NMP_BASE_URL` to your NeMo Evaluator deployment endpoint. See [CLI and SDK initialization](../../get-started/setup.md#setup-init) for the full convention. +Set `NMP_BASE_URL` to your NeMo Evaluator deployment endpoint. See [CLI and SDK initialization](/documentation/get-started#setup-init) for the full convention. ## Dataset Requirements @@ -72,7 +72,7 @@ For **online evaluation**, your dataset contains inputs that will be sent to the -Upload your dataset to a fileset in any [supported format](../metrics/similarity.md). You can select specific files using a `#` fragment pattern (for example, `my-workspace/my-fileset#data.csv`). If no pattern is specified, all parsable files in the fileset are loaded. +Upload your dataset to a fileset in any [supported format](/documentation/evaluate-models/metrics/similarity-metrics). You can select specific files using a `#` fragment pattern (for example, `my-workspace/my-fileset#data.csv`). If no pattern is specified, all parsable files in the fileset are loaded. @@ -191,7 +191,7 @@ job = client.evaluation.benchmark_jobs.create( Online evaluation generates model responses at runtime, then evaluates them against your metrics. Use this to evaluate a model's live performance. -The `model` field accepts either an inline model definition or a model reference. Refer to [Model Configuration](../metrics/model-configuration.md) for details on both formats. +The `model` field accepts either an inline model definition or a model reference. Refer to [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) for details on both formats. #### With Inline Model @@ -246,7 +246,7 @@ Here is a complete workflow for creating a benchmark and running an evaluation. -This example assumes you have already created the metrics (`exact-match`, `f1-score`) and uploaded your dataset (`qa-test-data`). Refer to [Evaluation Metrics](../metrics/index.md) for how to create custom metrics. +This example assumes you have already created the metrics (`exact-match`, `f1-score`) and uploaded your dataset (`qa-test-data`). Refer to [Evaluation Metrics](/documentation/evaluate-models/metrics) for how to create custom metrics. ```python @@ -343,4 +343,4 @@ if status.status == "completed": 3. **Validate dataset**: Ensure your dataset files are valid and columns/keys are consistent across rows 4. **Test metrics first**: Run individual metric evaluations before combining into a benchmark -For additional help, refer to [Troubleshooting](../../troubleshooting/evaluator.md). +For additional help, refer to [Troubleshooting](/documentation/reference/troubleshooting/evaluator). diff --git a/docs/evaluator/benchmarks/discover-industry-benchmarks.mdx b/docs/evaluator/benchmarks/discover-industry-benchmarks.mdx index 48653ce3b76..6fd0dfb05c8 100644 --- a/docs/evaluator/benchmarks/discover-industry-benchmarks.mdx +++ b/docs/evaluator/benchmarks/discover-industry-benchmarks.mdx @@ -13,7 +13,7 @@ The `system` workspace is a reserved workspace for NeMo Platform that contains r -**Initialization:** This example uses `NeMoPlatform()` with no arguments so the SDK reads your active CLI context (set by `nemo auth login`). The `workspace="system"` is passed per-call to access the reserved system workspace. For the standard local initialization pattern, see [CLI and SDK initialization](../../get-started/setup.md#setup-init). +**Initialization:** This example uses `NeMoPlatform()` with no arguments so the SDK reads your active CLI context (set by `nemo auth login`). The `workspace="system"` is passed per-call to access the reserved system workspace. For the standard local initialization pattern, see [CLI and SDK initialization](/documentation/get-started#setup-init). ```python diff --git a/docs/evaluator/benchmarks/index.mdx b/docs/evaluator/benchmarks/index.mdx index 5d2340a8ee7..9a35e6038a4 100644 --- a/docs/evaluator/benchmarks/index.mdx +++ b/docs/evaluator/benchmarks/index.mdx @@ -24,13 +24,13 @@ Custom benchmarks are valuable for domain-specific evaluation where standard ben | Type | Use Case | Dataset | Metrics | |------|----------|---------|---------| -| [**Industry Benchmarks**](industry.md) | Compare against published baselines, regression testing, model selection | Canonical datasets (fixed) | Standardized metrics | -| [**Custom Benchmarks**](custom.md) | Domain-specific evaluation, production monitoring, task-specific assessment | Your evaluation data | Your choice of metrics | +| [**Industry Benchmarks**](/documentation/evaluate-models/benchmarks/industry-benchmarks) | Compare against published baselines, regression testing, model selection | Canonical datasets (fixed) | Standardized metrics | +| [**Custom Benchmarks**](/documentation/evaluate-models/benchmarks/custom-benchmarks) | Domain-specific evaluation, production monitoring, task-specific assessment | Your evaluation data | Your choice of metrics | ## Create Custom Benchmarks -Create a custom benchmark by combining metrics with your dataset. Before creating a benchmark, you will need to [create the metrics](../metrics/index.md) that define how to score your model's outputs. +Create a custom benchmark by combining metrics with your dataset. Before creating a benchmark, you will need to [create the metrics](/documentation/evaluate-models/metrics) that define how to score your model's outputs. ```python benchmark = client.evaluation.benchmarks.create( @@ -103,21 +103,21 @@ After successfully creating a job, refer to [Benchmark Job Management](job-manag - + Compose a custom benchmark with a collection of metrics to evaluate tasks bespoke to your needs. RAGAS BFCL - + Evaluate agent workflows including tool calling, goal accuracy, topic adherence, and trajectory evaluation. RAGAS BFCL - + Ready-to-use benchmarks for reasoning, code generation, safety, and language understanding with published datasets. diff --git a/docs/evaluator/benchmarks/industry.mdx b/docs/evaluator/benchmarks/industry.mdx index 8ca47a87639..026600cbc7a 100644 --- a/docs/evaluator/benchmarks/industry.mdx +++ b/docs/evaluator/benchmarks/industry.mdx @@ -10,7 +10,7 @@ NeMo Platform provides a streamlined API to evaluate large language models with Benchmarks provide standardized methods for comparing model performance across different capabilities. These benchmarks are widely used in the research community and provide reliable, reproducible metrics for model assessment. -Refer to the [Run an LLM Judge Evaluation](../tutorials/run-llm-judge-evaluation.md) tutorial for details on using evaluator jobs and result handling. +Refer to the [Run an LLM Judge Evaluation](/documentation/evaluate-models/tutorials/run-llm-as-a-judge-evaluation) tutorial for details on using evaluator jobs and result handling. - **Standard Datasets**: Most benchmarks include predefined datasets widely used in research. - **Reproducible Metrics**: Use established methodologies to calculate metrics. @@ -39,7 +39,7 @@ Many benchmarks offer multiple variants optimized for different model types: -The `model` field in all benchmark examples below accepts either an inline model definition or a model reference string (for example, `"my-workspace/my-model"`). Refer to [Model Configuration](../metrics/model-configuration.md) for details. +The `model` field in all benchmark examples below accepts either an inline model definition or a model reference string (for example, `"my-workspace/my-model"`). Refer to [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) for details. ## Advanced Reasoning @@ -857,7 +857,7 @@ For detailed results analysis, refer to [eval-benchmark-results](results.md). ### Troubleshooting Content Safety Benchmarks -Refer to [Evaluator Troubleshooting](../../troubleshooting/evaluator.md) for general troubleshooting steps for failed evaluation jobs. +Refer to [Evaluator Troubleshooting](/documentation/reference/troubleshooting/evaluator) for general troubleshooting steps for failed evaluation jobs.
This section covers common issues for the safety harness. diff --git a/docs/evaluator/benchmarks/job-management.mdx b/docs/evaluator/benchmarks/job-management.mdx index a44a1df260a..c39930101f0 100644 --- a/docs/evaluator/benchmarks/job-management.mdx +++ b/docs/evaluator/benchmarks/job-management.mdx @@ -24,7 +24,7 @@ while job_status.status in ("active", "pending", "created"): print(job_status) ``` -Refer to [Evaluator Troubleshooting](../../troubleshooting/evaluator.md) for help troubleshooting job failures. +Refer to [Evaluator Troubleshooting](/documentation/reference/troubleshooting/evaluator) for help troubleshooting job failures. ## Fetch Job Logs diff --git a/docs/evaluator/benchmarks/manage-benchmarks.mdx b/docs/evaluator/benchmarks/manage-benchmarks.mdx index 2530d46f3ce..75a2c99678f 100644 --- a/docs/evaluator/benchmarks/manage-benchmarks.mdx +++ b/docs/evaluator/benchmarks/manage-benchmarks.mdx @@ -145,6 +145,6 @@ client.evaluation.benchmarks.delete(name="my-custom-benchmark") ## Related Topics -- [Custom Benchmarks](custom.md) - Create custom benchmarks +- [Custom Benchmarks](/documentation/evaluate-models/benchmarks/custom-benchmarks) - Create custom benchmarks - [Benchmark Job Management](job-management.md) - Manage benchmark evaluation jobs - [Benchmark Results](results.md) - View and download benchmark evaluation results diff --git a/docs/evaluator/index.mdx b/docs/evaluator/index.mdx index 39d8c9f7480..0e0868491b8 100644 --- a/docs/evaluator/index.mdx +++ b/docs/evaluator/index.mdx @@ -9,7 +9,7 @@ Evaluation is powered by NeMo Platform, a cloud-native platform for evaluating l NeMo Platform enables real-time evaluations of your LLM application through APIs, guiding you in refining and optimizing LLMs for enhanced performance and real-world applicability. The NeMo Evaluator APIs can be seamlessly automated within development pipelines, enabling faster iterations without the need for live data. It is cost-effective and suitable for pre-deployment checks and regression testing. - + --- @@ -21,7 +21,7 @@ Evaluator separates **evaluation definition** from **execution**. The code snippets below are for conceptual demonstration purposes only. -For runnable examples see the [tutorials](tutorials/index.md) and [SDK resources](sdk-resources.md). +For runnable examples see the [tutorials](/documentation/evaluate-models/tutorials) and [SDK resources](/documentation/evaluate-models/sdk-resources). ### 1. Build RunConfig with the Library @@ -84,7 +84,7 @@ When using Evaluator as a NeMo Platform plugin : |---------|-------------------|-------------| | **Execution** | Local Python process | Local plugin runs for local experimentation and durable platform jobs for production | | **Inference** | Direct model or agent endpoint calls | The same as standalone and can also route through NeMo Platform Inference Gateway and platform-managed endpoints | -| **Datasets** | Inline rows and local files | Inline rows, local paths resolved at submission time, and NeMo Platform [Filesets](../get-started/concepts/manage-files.md) | +| **Datasets** | Inline rows and local files | Inline rows, local paths resolved at submission time, and NeMo Platform [Filesets](/documentation/get-started/core-concepts/manage-files) | | **Results Artifacts** | Results stored in memory | NeMo Platform artifact storage with typed result download | | **Authentication** | Local environment variables | Local environment variables for local runs and NeMo Platform Secrets service for remote jobs | @@ -104,17 +104,17 @@ There are two execution modes and two evaluation patterns: - **Offline evaluation**: Score existing dataset rows (for example, model outputs already generated). - **Online evaluation**: Generate outputs from a model as part of evaluation, then score them. -For deeper details, see [Evaluation Metrics](metrics/index.md). +For deeper details, see [Evaluation Metrics](/documentation/evaluate-models/metrics). --- ## Tutorials -After [setting up a local instance of the platform](../get-started/setup.md), use the following tutorials to learn how to accomplish common evaluation tasks. These step-by-step guides help you evaluate models using different benchmarks and metrics. +After [setting up a local instance of the platform](/documentation/get-started), use the following tutorials to learn how to accomplish common evaluation tasks. These step-by-step guides help you evaluate models using different benchmarks and metrics. - + Learn how to evaluate a fine-tuned model using the LLM Judge metric with a custom dataset. @@ -131,12 +131,12 @@ Learn how to evaluate a fine-tuned model using the LLM Judge metric with a custo Most teams get the best results by starting metric-first, then moving to benchmarks: 1. **Develop and validate your metrics first** - - Start with [Metrics](metrics/index.md) to define how quality should be scored for your use case. + - Start with [Metrics](/documentation/evaluate-models/metrics) to define how quality should be scored for your use case. - Use live evaluation (`POST /v2/workspaces/{workspace}/evaluation/metric-evaluate`) with small `DatasetRows` payloads to iterate quickly. 1. **Scale metric evaluation to jobs** - When metrics are validated, run async metric jobs (`/evaluation/metric-jobs`) on larger datasets. - - Use filesets for production-scale inputs. See [Manage Files](../get-started/concepts/manage-files.md). + - Use filesets for production-scale inputs. See [Manage Files](/documentation/get-started/core-concepts/manage-files). 1. **Monitor and analyze results** - Track job status and progress with job management APIs. @@ -146,7 +146,7 @@ Most teams get the best results by starting metric-first, then moving to benchma ## Where to Go Next -- For metric workflows, see [Metric Jobs](metrics/index.md) and [Metric Results](metrics/results.md). +- For metric workflows, see [Metric Jobs](/documentation/evaluate-models/metrics) and [Metric Results](/documentation/evaluate-models/metrics/metric-results). - For full endpoint details, see the [Evaluator API Reference](/documentation/reference/api-reference). --- diff --git a/docs/evaluator/metrics/agent-configuration.mdx b/docs/evaluator/metrics/agent-configuration.mdx index 4f793819d81..012bc52d59a 100644 --- a/docs/evaluator/metrics/agent-configuration.mdx +++ b/docs/evaluator/metrics/agent-configuration.mdx @@ -37,7 +37,7 @@ evaluator: Evaluator = client.evaluator # this object is an Evaluator resource If your agent endpoint requires authentication, configure `api_key_secret` on the `Agent`. -For local `evaluator.run(...)` calls, `api_key_secret` must name an environment variable available to the local Python process. For remote `evaluator.submit(...)` jobs, it must name a NeMo platform secret in the target workspace. See [Model API Authentication](model-configuration.md#model-api-authentication) for the local-versus-remote behavior. +For local `evaluator.run(...)` calls, `api_key_secret` must name an environment variable available to the local Python process. For remote `evaluator.submit(...)` jobs, it must name a NeMo platform secret in the target workspace. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication) for the local-versus-remote behavior. For remote `evaluator.submit(...)` jobs, create the secret in the platform workspace before submitting the job: @@ -66,7 +66,7 @@ You control the request shape with `body` and extract values from the response w | `url` | Yes | string | Base URL of the agent endpoint. | | `name` | Yes | string | Agent name or identifier. | | `format` | No | string | `generic` (default) or `nemo_agent_toolkit`. | -| `api_key_secret` | No | string | API key reference. See [Model API Authentication](model-configuration.md#model-api-authentication). | +| `api_key_secret` | No | string | API key reference. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). | | `body` | Yes | dict | Jinja template for the request payload. Use `{{ prompt }}`, `{{ messages }}`, or fields from the rendered prompt context. | | `response_path` | Yes | string | [JSONPath](https://datatracker.ietf.org/doc/html/rfc9535) expression to extract the response text. | | `trajectory_path` | No | string | JSONPath expression to extract the trajectory. | @@ -153,7 +153,7 @@ Use the `nemo_agent_toolkit` format when evaluating agents built with the [NeMo | `url` | Yes | string | Base URL of the agent endpoint. | | `name` | Yes | string | Agent name or identifier. | | `format` | Yes | string | Set to `nemo_agent_toolkit`. | -| `api_key_secret` | No | string | API key reference. See [Model API Authentication](model-configuration.md#model-api-authentication). | +| `api_key_secret` | No | string | API key reference. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). | ### Run a NAT Agent Evaluation @@ -208,7 +208,7 @@ Online evaluations accept **either** a model **or** an agent as the request targ ## Related -- [Model Configuration](model-configuration.md) - Inline model targets for LLM endpoints. -- [Agentic Evaluation Metrics](agentic.md) - Metrics for evaluating agent tool calling, goal accuracy, and trajectory. -- [LLM-as-a-Judge](llm-as-a-judge.md) - Custom judge-based evaluation with flexible scoring criteria. -- [Bring Your Own Metric](remote.md) - Integrate custom evaluation endpoints. +- [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) - Inline model targets for LLM endpoints. +- [Agentic Evaluation Metrics](/documentation/evaluate-models/metrics/agentic-metrics) - Metrics for evaluating agent tool calling, goal accuracy, and trajectory. +- [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge) - Custom judge-based evaluation with flexible scoring criteria. +- [Bring Your Own Metric](/documentation/evaluate-models/metrics/bring-your-own-metric) - Integrate custom evaluation endpoints. diff --git a/docs/evaluator/metrics/agentic.mdx b/docs/evaluator/metrics/agentic.mdx index 34280d188b6..4c28a8d5868 100644 --- a/docs/evaluator/metrics/agentic.mdx +++ b/docs/evaluator/metrics/agentic.mdx @@ -23,7 +23,7 @@ Evaluates the quality of the agent's final output using: - **Agent Goal Accuracy**: Measures whether the agent successfully completed the requested task. Refer to [Agent Goal Accuracy](#agent-goal-accuracy). - **Topic Adherence**: Assesses how well the agent maintained focus on the assigned topic throughout the conversation. Refer to [Topic Adherence](#topic-adherence). - **Answer Accuracy**: Evaluates the factual correctness of agent answers. Refer to [Answer Accuracy](#answer-accuracy). -- **Custom Metrics**: For domain-specific or custom evaluation criteria, use [LLM-as-a-Judge](llm-as-a-judge.md) with the `data` task type. +- **Custom Metrics**: For domain-specific or custom evaluation criteria, use [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge) with the `data` task type. **3. Trajectory Evaluation** Evaluates the agent's decision-making process by analyzing the entire sequence of actions taken to accomplish a goal. This includes assessing whether the agent chose appropriate tools in the correct order. Refer to [Trajectory Evaluation](#trajectory-evaluation) for the expected data format and current plugin SDK support. @@ -82,7 +82,7 @@ Before running agentic evaluations: 1. **Workspace**: Have a workspace created. All remote resources, including secrets and jobs, are scoped to a workspace. 2. **Judge LLM endpoint** *(for most metrics)*: Have access to an LLM that will serve as your judge. -3. **API key secret** *(if judge requires auth)*: If your judge endpoint requires authentication, [create a secret](../../get-started/concepts/manage-secrets.md) to store the API key. For local `run` versus remote `submit` behavior, see [Model API Authentication](model-configuration.md#model-api-authentication). +3. **API key secret** *(if judge requires auth)*: If your judge endpoint requires authentication, [create a secret](/documentation/get-started/core-concepts/manage-secrets) to store the API key. For local `run` versus remote `submit` behavior, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). 4. **Initialize the SDK**: ```python @@ -1236,9 +1236,9 @@ judge_model = Model( ) ``` -For more details on secret management, refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md). +For more details on secret management, refer to [Managing Secrets](/documentation/get-started/core-concepts/manage-secrets). -For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](model-configuration.md#model-api-authentication). +For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). --- @@ -1252,7 +1252,7 @@ result = job.get_result() print(result.aggregate_scores) ``` -Navigate to [Metrics Job Management](job-management.md) for more job lifecycle details. +Navigate to [Metrics Job Management](/documentation/evaluate-models/metrics/job-management) for more job lifecycle details. --- @@ -1277,14 +1277,14 @@ Use `RunConfig(limit_samples=...)` when you want to test a small slice of a larg 4. **RAGAS Dependency**: These metrics are powered by RAGAS and may have version-specific behavior. -5. **NaN troubleshooting for judge-based metrics**: If you see `nan_count > 0` with `mean = null`, check judge model authentication first (API key secret, endpoint access, and model permissions). See [Model API Authentication](model-configuration.md#model-api-authentication) for `api_key_secret` behavior. Some RAGAS metrics are known to convert auth failures into `NaN` scores instead of raising a hard error. +5. **NaN troubleshooting for judge-based metrics**: If you see `nan_count > 0` with `mean = null`, check judge model authentication first (API key secret, endpoint access, and model permissions). See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication) for `api_key_secret` behavior. Some RAGAS metrics are known to convert auth failures into `NaN` scores instead of raising a hard error. -- [Agent Configuration](agent-configuration.md) - Use agents (generic or NAT) as targets in online evaluation jobs -- [Agentic Benchmarks](../benchmarks/agentic.md) - BFCL benchmark for tool-calling evaluation -- [LLM-as-a-Judge](llm-as-a-judge.md) - Custom judge-based evaluation -- [Evaluation Results](results.md) - Understanding results -- [RAG Metrics](rag.md) - RAGAS metrics for RAG pipelines +- [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration) - Use agents (generic or NAT) as targets in online evaluation jobs +- [Agentic Benchmarks](/documentation/evaluate-models/benchmarks/agentic-benchmarks) - BFCL benchmark for tool-calling evaluation +- [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge) - Custom judge-based evaluation +- [Evaluation Results](/documentation/evaluate-models/metrics/metric-results) - Understanding results +- [RAG Metrics](/documentation/evaluate-models/metrics/rag-metrics) - RAGAS metrics for RAG pipelines diff --git a/docs/evaluator/metrics/index.mdx b/docs/evaluator/metrics/index.mdx index 9214b677bb4..f2110f754eb 100644 --- a/docs/evaluator/metrics/index.mdx +++ b/docs/evaluator/metrics/index.mdx @@ -87,15 +87,15 @@ Online evaluation jobs can target either a **model** (an OpenAI-compatible chat | **Model** | Standalone LLM endpoints using a standard chat completions API. | | **Agent** | Agentic systems, NeMo Agent Toolkit workflows, or custom HTTP endpoints with non-standard response formats. | -See [Model Configuration](model-configuration.md) and [Agent Configuration](agent-configuration.md) for setup details. +See [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) and [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration) for setup details. ## Built-in vs. Custom Metrics - **Built-in metrics**: Ready-to-use metrics provided by NeMo Platform (for example `exact-match`, `bleu`, `rouge`). - **Custom metrics**: Metrics you define for domain-specific evaluation needs. -To configure inline metric objects, see [Manage Metrics](manage-metrics.md). -For custom metric creation guides, start with [Similarity Metrics](similarity.md), [LLM-as-a-Judge](llm-as-a-judge.md), or [Bring Your Own Metric](remote.md). +To configure inline metric objects, see [Manage Metrics](/documentation/evaluate-models/metrics/manage-metrics). +For custom metric creation guides, start with [Similarity Metrics](/documentation/evaluate-models/metrics/similarity-metrics), [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge), or [Bring Your Own Metric](/documentation/evaluate-models/metrics/bring-your-own-metric). ## Datasets @@ -104,7 +104,7 @@ Evaluation jobs need dataset input. You can provide data in two ways: | Dataset Source | Description | Best For | |------|-------------|----------| | **DatasetRows** | Inline rows sent directly in the request | Quick testing and live evaluation | -| **FilesetRef** | Reference to a persisted [fileset](../../get-started/concepts/manage-files.md) (`workspace/fileset-name`) | Production jobs and reusable datasets | +| **FilesetRef** | Reference to a persisted [fileset](/documentation/get-started/core-concepts/manage-files) (`workspace/fileset-name`) | Production jobs and reusable datasets | Example of providing a `FilesetRef` to reference specific files or globs: @@ -131,42 +131,42 @@ Use the metric-type pages below to create and configure custom metrics. - + Use another LLM to evaluate outputs with flexible scoring criteria. Define custom rubrics or numerical ranges. custom-scoring rubrics - + Evaluate agent workflows including tool calling accuracy, goal completion, and topic adherence. RAGAS tool-calling - + Evaluate RAG pipelines for retrieval quality and answer generation using RAGAS metrics. faithfulness relevancy - + Create metrics for text similarity, exact matching, and standard NLP evaluations using Jinja2 templating. F1 ROUGE BLEU - + Integrate custom evaluation endpoints for domain-specific scoring. remote custom - + Configure agent endpoints (generic or NeMo Agent Toolkit) as targets for online evaluation jobs. @@ -187,4 +187,4 @@ Scores are the metric outputs produced during evaluation: ## Manage Metric Definitions -Create inline metric objects that can be reused from Python helpers or modules. See [Manage Metrics](manage-metrics.md) for SDK patterns. +Create inline metric objects that can be reused from Python helpers or modules. See [Manage Metrics](/documentation/evaluate-models/metrics/manage-metrics) for SDK patterns. diff --git a/docs/evaluator/metrics/job-management.mdx b/docs/evaluator/metrics/job-management.mdx index bfda31f5471..65fa3fa5763 100644 --- a/docs/evaluator/metrics/job-management.mdx +++ b/docs/evaluator/metrics/job-management.mdx @@ -59,7 +59,7 @@ if not job.check_if_complete(): `wait_until_done()` raises if the job reaches a terminal failure status or times out. -Refer to [Evaluator Troubleshooting](../../troubleshooting/evaluator.md) for help troubleshooting job failures. +Refer to [Evaluator Troubleshooting](/documentation/reference/troubleshooting/evaluator) for help troubleshooting job failures. ## Reconnect to a Job @@ -84,7 +84,7 @@ for row in result.row_scores: print(row.item, row.metrics) ``` -See [Evaluation Results](results.md) for details on aggregate and row-level result fields. +See [Evaluation Results](/documentation/evaluate-models/metrics/metric-results) for details on aggregate and row-level result fields. ## Download Job Artifacts diff --git a/docs/evaluator/metrics/llm-as-a-judge.mdx b/docs/evaluator/metrics/llm-as-a-judge.mdx index 101079294fb..50d19057ee0 100644 --- a/docs/evaluator/metrics/llm-as-a-judge.mdx +++ b/docs/evaluator/metrics/llm-as-a-judge.mdx @@ -50,7 +50,7 @@ evaluator: Evaluator = client.evaluator # this object is an Evaluator resource -The `model` field accepts both inline model definitions and model references (for example, `"my-workspace/my-model"`). Refer to [Model Configuration](model-configuration.md) for details. +The `model` field accepts both inline model definitions and model references (for example, `"my-workspace/my-model"`). Refer to [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) for details. Live evaluation is designed for rapid iteration when developing and refining your evaluation metrics. Use it to quickly test different judge prompts, scoring criteria, and data formats before committing to a full evaluation job. Results return immediately, making it easy to experiment and debug. @@ -609,7 +609,7 @@ metric = { If your judge model endpoint requires an API key, store it as a secret. The secret is automatically resolved from the same workspace as your evaluation. -For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](model-configuration.md#model-api-authentication). +For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). ### Create a Secret @@ -694,9 +694,9 @@ metric = { -- [Model Configuration](model-configuration.md) - Inline models vs model references -- [Evaluation Results](results.md) - Understanding and downloading results -- [Agentic Evaluation](agentic.md) - Evaluate agent workflows -- [RAG Evaluation](rag.md) - Evaluate retrieval-augmented generation +- [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) - Inline models vs model references +- [Evaluation Results](/documentation/evaluate-models/metrics/metric-results) - Understanding and downloading results +- [Agentic Evaluation](/documentation/evaluate-models/metrics/agentic-metrics) - Evaluate agent workflows +- [RAG Evaluation](/documentation/evaluate-models/metrics/rag-metrics) - Evaluate retrieval-augmented generation diff --git a/docs/evaluator/metrics/manage-metrics.mdx b/docs/evaluator/metrics/manage-metrics.mdx index 8e2f99a8dad..3e2706670b8 100644 --- a/docs/evaluator/metrics/manage-metrics.mdx +++ b/docs/evaluator/metrics/manage-metrics.mdx @@ -91,7 +91,7 @@ from nemo_evaluator_sdk import RunConfig config = RunConfig(parallelism=4, limit_samples=100) ``` -For online evaluations, provide a model or agent target and use the online parameter classes described in [Model Configuration](model-configuration.md) and [Agent Configuration](agent-configuration.md). +For online evaluations, provide a model or agent target and use the online parameter classes described in [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) and [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration). ## Submit a Durable Job @@ -117,6 +117,6 @@ result = job.get_result() ## Related Topics -- [Metric Results](results.md) - Work with `EvaluationResult`, aggregate scores, and row scores -- [Manage Metric Jobs](job-management.md) - Submit, monitor, reconnect to, and download job results -- [Similarity Metrics](similarity.md) - Configure exact match, F1, BLEU, ROUGE, and string/number checks +- [Metric Results](/documentation/evaluate-models/metrics/metric-results) - Work with `EvaluationResult`, aggregate scores, and row scores +- [Manage Metric Jobs](/documentation/evaluate-models/metrics/job-management) - Submit, monitor, reconnect to, and download job results +- [Similarity Metrics](/documentation/evaluate-models/metrics/similarity-metrics) - Configure exact match, F1, BLEU, ROUGE, and string/number checks diff --git a/docs/evaluator/metrics/model-configuration.mdx b/docs/evaluator/metrics/model-configuration.mdx index fde337b6865..96916985314 100644 --- a/docs/evaluator/metrics/model-configuration.mdx +++ b/docs/evaluator/metrics/model-configuration.mdx @@ -187,6 +187,6 @@ The plugin SDK examples on this page use inline `Model` objects. If your deploym -For evaluating agentic systems, use an `Agent` request target instead of a `Model`. See [Agent Configuration](agent-configuration.md). +For evaluating agentic systems, use an `Agent` request target instead of a `Model`. See [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration). diff --git a/docs/evaluator/metrics/rag.mdx b/docs/evaluator/metrics/rag.mdx index 9fd07c077eb..d559118574e 100644 --- a/docs/evaluator/metrics/rag.mdx +++ b/docs/evaluator/metrics/rag.mdx @@ -78,7 +78,7 @@ client.secrets.create( ) ``` -Reference secrets by name in your model configuration. For local `run` versus remote `submit` behavior, see [Model API Authentication](model-configuration.md#model-api-authentication). +Reference secrets by name in your model configuration. For local `run` versus remote `submit` behavior, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). ```python judge_model = Model( @@ -93,7 +93,7 @@ judge_model = Model( RAGAS metrics accept inline model definitions for `judge_model` and, where required, `embeddings_model`. - See [Model Configuration](model-configuration.md) for details. + See [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) for details. --- @@ -120,7 +120,7 @@ RAGAS metrics accept inline model definitions for `judge_model` and, where requi The metric examples below use these inline values: -For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](model-configuration.md#model-api-authentication). +For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). ```python judge_model = Model( @@ -860,7 +860,7 @@ client.secrets.create(name="judge-api-key", value="") client.secrets.create(name="embedding-api-key", value="") ``` -Reference secrets by name in your metric configuration. For local `run` versus remote `submit` behavior, see [Model API Authentication](model-configuration.md#model-api-authentication). +Reference secrets by name in your metric configuration. For local `run` versus remote `submit` behavior, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). ```python judge_model = Model( @@ -893,8 +893,8 @@ print(f"Saved artifacts under {artifacts_dir}") |-------|-------|----------| | `judge_model` is required | Missing judge LLM config for metric | Add `judge_model` to metric configuration | | `embeddings_model` is required | Using `response_relevancy` without embeddings | Add `embeddings_model` to metric configuration | -| Job stuck in "pending" | Model endpoint not accessible | Verify endpoint URLs and API key secrets. See [Model API Authentication](model-configuration.md#model-api-authentication) | -| Authentication failed | Invalid or missing API key | Check `api_key_secret` for the execution mode. See [Model API Authentication](model-configuration.md#model-api-authentication) | +| Job stuck in "pending" | Model endpoint not accessible | Verify endpoint URLs and API key secrets. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication) | +| Authentication failed | Invalid or missing API key | Check `api_key_secret` for the execution mode. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication) | | `nan_count > 0` and `mean = null` | Judge/model call failures, such as auth, endpoint, quota, or timeout. Some RAGAS metrics are known to return `NaN` instead of raising on these failures. | Inspect row-level `error`; verify API key, endpoint, and model access | | Low faithfulness scores | Context doesn't support the response | Improve retrieval or response generation | @@ -917,7 +917,7 @@ If you see `nan_count > 0` with `mean = null`, first validate judge model authen ## Important Notes -1. **Secret Management**: API keys should be referenced through `api_key_secret`, with different local `run` and remote `submit` behavior. See [Model API Authentication](model-configuration.md#model-api-authentication). Never pass API keys directly in the request. +1. **Secret Management**: API keys should be referenced through `api_key_secret`, with different local `run` and remote `submit` behavior. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). Never pass API keys directly in the request. 2. **Column Names**: RAGAS metrics use specific column names: - `user_input` (not `question`) - `response` (not `answer`) @@ -933,7 +933,7 @@ If you see `nan_count > 0` with `mean = null`, first validate judge model authen -- [LLM-as-a-Judge](llm-as-a-judge.md) - Custom judge-based evaluation -- [Agentic Metrics](agentic.md) - Evaluate agent workflows +- [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge) - Custom judge-based evaluation +- [Agentic Metrics](/documentation/evaluate-models/metrics/agentic-metrics) - Evaluate agent workflows diff --git a/docs/evaluator/metrics/remote.mdx b/docs/evaluator/metrics/remote.mdx index 83d5e66bb5c..e1e8c2c246e 100644 --- a/docs/evaluator/metrics/remote.mdx +++ b/docs/evaluator/metrics/remote.mdx @@ -4,7 +4,7 @@ description: "" --- -NeMo Platform offers [built-in metrics](index.md) that can be configured to evaluate on your custom data. Remote metrics let you bring your own metric logic into the NeMo Platform evaluation workflow by serving that logic from a REST API. +NeMo Platform offers [built-in metrics](/documentation/evaluate-models/metrics) that can be configured to evaluate on your custom data. Remote metrics let you bring your own metric logic into the NeMo Platform evaluation workflow by serving that logic from a REST API. A remote metric gives you control over the evaluation logic, request payload, and reported scores while the Evaluator plugin SDK handles dataset iteration, result aggregation, retries, and job execution. @@ -234,7 +234,7 @@ result = job.get_result() If your remote endpoint requires authentication, store the API key as a platform secret and reference it from your metric: -For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](model-configuration.md#model-api-authentication). +For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). ```python @@ -335,7 +335,7 @@ And must return: | `body` | dict | (Generic only) Jinja template for request payload | | `scores` | list | (Generic only) List of score configuration objects (refer to Score Configuration section) | | `evaluator_name` | string | (NAT only) Name of the NAT evaluator | -| `api_key_secret` | `SecretRef` | Optional API key reference. See [Model API Authentication](model-configuration.md#model-api-authentication) | +| `api_key_secret` | `SecretRef` | Optional API key reference. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication) | | `timeout_seconds` | float | Request timeout (default: 30.0) | | `max_retries` | int | Max retry attempts (default: 3) | @@ -384,8 +384,8 @@ RemoteScore( -- [Evaluation Results](results.md) - Understanding and downloading results -- [LLM-as-a-Judge](llm-as-a-judge.md) - Use an LLM to evaluate outputs -- [Agentic Evaluation](agentic.md) - Evaluate agent workflows +- [Evaluation Results](/documentation/evaluate-models/metrics/metric-results) - Understanding and downloading results +- [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge) - Use an LLM to evaluate outputs +- [Agentic Evaluation](/documentation/evaluate-models/metrics/agentic-metrics) - Evaluate agent workflows diff --git a/docs/evaluator/metrics/similarity.mdx b/docs/evaluator/metrics/similarity.mdx index 00b3a3053c2..27f016fda2b 100644 --- a/docs/evaluator/metrics/similarity.mdx +++ b/docs/evaluator/metrics/similarity.mdx @@ -4,11 +4,11 @@ description: "" --- -NeMo Platform offers [built-in metrics](index.md) that can be configured to evaluate on your custom data. Similarity metrics compare generated or precomputed text against references, labels, or numeric/string expectations. They support Jinja templates so you can map your dataset columns to the values each metric evaluates. +NeMo Platform offers [built-in metrics](/documentation/evaluate-models/metrics) that can be configured to evaluate on your custom data. Similarity metrics compare generated or precomputed text against references, labels, or numeric/string expectations. They support Jinja templates so you can map your dataset columns to the values each metric evaluates. Template functionality provides maximum flexibility for evaluating your models on proprietary, domain-specific, or novel tasks. You can bring your own datasets, define your own prompts and templates using [Jinja](https://github.com/pallets/jinja), and select the metrics that matter most for your use case. This approach is ideal when: -- You want to evaluate on tasks, data, or formats not covered by [industry benchmarks](../benchmarks/industry.md) or built-in metrics. +- You want to evaluate on tasks, data, or formats not covered by [industry benchmarks](/documentation/evaluate-models/benchmarks/industry-benchmarks) or built-in metrics. - You need to measure model performance using custom or business-specific criteria. - You want to experiment with new evaluation methodologies, metrics, or workflows. - You need to create custom prompts and templates for specific use cases. @@ -616,6 +616,6 @@ Keep field names consistent between the dataset rows and the templates you confi ## Related Topics -- [Metric Overview](index.md) -- [Model Configuration](model-configuration.md) -- [RAG Evaluation Metrics](rag.md) +- [Metric Overview](/documentation/evaluate-models/metrics) +- [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) +- [RAG Evaluation Metrics](/documentation/evaluate-models/metrics/rag-metrics) diff --git a/docs/evaluator/tutorials/index.mdx b/docs/evaluator/tutorials/index.mdx index a4f35c3c5d1..3b3eaaea032 100644 --- a/docs/evaluator/tutorials/index.mdx +++ b/docs/evaluator/tutorials/index.mdx @@ -4,15 +4,15 @@ description: "" --- -Use these tutorials to become familiar with [evaluation with NeMo Platform](../index.md). +Use these tutorials to become familiar with [evaluation with NeMo Platform](/documentation/evaluate-models). ## Before You Start -Set up [a local instance of the platform](../../get-started/setup.md) for the following tutorials. +Set up [a local instance of the platform](/documentation/get-started) for the following tutorials. - + Learn how to evaluate a fine-tuned model using the LLM Judge metric with a custom dataset. @@ -24,4 +24,4 @@ Learn how to evaluate a fine-tuned model using the LLM Judge metric with a custo ## How It Works -For the conceptual overview of how Evaluator separates definition (library) from execution (platform), see [About Evaluating → How It Works](../index.md#how-it-works-library-platform). For runnable SDK examples, see [SDK Resources](../sdk-resources.md). +For the conceptual overview of how Evaluator separates definition (library) from execution (platform), see [About Evaluating → How It Works](/documentation/evaluate-models#how-it-works-library-platform). For runnable SDK examples, see [SDK Resources](/documentation/evaluate-models/sdk-resources). diff --git a/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx b/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx index 2f3a8862add..27bb0113a17 100644 --- a/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx +++ b/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx @@ -38,7 +38,7 @@ This tutorial takes approximately **20 minutes** to complete. ## Prerequisites -1. Install and start NeMo Platform using the [Setup guide](../../get-started/setup.md). +1. Install and start NeMo Platform using the [Setup guide](/documentation/get-started). ```bash ! pip install nemo-platform[all] @@ -820,12 +820,12 @@ In this tutorial, you learned how to: ## Next Steps -- **Experiment with rubric scores**: Use [categorical rubrics](../metrics/llm-as-a-judge.md) instead of numeric ranges for more interpretable criteria +- **Experiment with rubric scores**: Use [categorical rubrics](/documentation/evaluate-models/metrics/llm-as-a-judge) instead of numeric ranges for more interpretable criteria - **Try different judge models**: Larger models often correlate better with human judgment -- **Explore other evaluation types**: [RAG evaluation](../metrics/rag.md) or [agentic evaluation](../metrics/agentic.md) +- **Explore other evaluation types**: [RAG evaluation](/documentation/evaluate-models/metrics/rag-metrics) or [agentic evaluation](/documentation/evaluate-models/metrics/agentic-metrics) ## Related -- [LLM-as-a-Judge Reference](../metrics/llm-as-a-judge.md) - Complete guide to judge configuration -- [SDK Resources](../sdk-resources.md) - Evaluator plugin SDK resource reference -- [Manage Metrics](../metrics/manage-metrics.md) - Using evaluator SDK metric objects +- [LLM-as-a-Judge Reference](/documentation/evaluate-models/metrics/llm-as-a-judge) - Complete guide to judge configuration +- [SDK Resources](/documentation/evaluate-models/sdk-resources) - Evaluator plugin SDK resource reference +- [Manage Metrics](/documentation/evaluate-models/metrics/manage-metrics) - Using evaluator SDK metric objects diff --git a/docs/get-started/concepts/entities.mdx b/docs/get-started/concepts/entities.mdx index d46740cd941..94c8ca98b52 100644 --- a/docs/get-started/concepts/entities.mdx +++ b/docs/get-started/concepts/entities.mdx @@ -28,7 +28,7 @@ Entities are always scoped to a **workspace**. When you create or reference an e In most cases, you interact with entities through service-specific APIs rather than a generic entity API. The platform CLI and Python SDK expose these through familiar commands and client methods: -- **Evaluator** — Create metrics, benchmarks, and evaluation jobs; retrieve results. See [About Evaluating](../../evaluator/index.md) and the evaluation tutorials. -- **Models and inference** — Register model entities, create deployments, and run inference through the gateway. See [about](../../run-inference/about.md) and the inference tutorials. +- **Evaluator** — Create metrics, benchmarks, and evaluation jobs; retrieve results. See [About Evaluating](/documentation/evaluate-models) and the evaluation tutorials. +- **Models and inference** — Register model entities, create deployments, and run inference through the gateway. See [about](/documentation/models-and-inference) and the inference tutorials. These service APIs provide typed interfaces and validation specific to each resource type, built on top of the entity system. For raw HTTP access, see the [Entity Store API reference](/documentation/reference/api-reference). diff --git a/docs/get-started/concepts/index.mdx b/docs/get-started/concepts/index.mdx index 5f292914c03..b9d4a5d20fc 100644 --- a/docs/get-started/concepts/index.mdx +++ b/docs/get-started/concepts/index.mdx @@ -6,37 +6,37 @@ Learn about the foundational concepts that underpin the NeMo Platform. - + Isolation boundaries for teams, users, or environments. - + Managing secrets required for your workloads. - + Managing files and datasets. - + Logical groupings for related work within a workspace. - + The underlying data objects that power platform services. - + How to refer to entities by string (e.g. workspace/entity_name) in the CLI and API. - + Query and filter list results using operators, bracket notation, or text syntax. diff --git a/docs/get-started/concepts/manage-secrets.mdx b/docs/get-started/concepts/manage-secrets.mdx index 7d82fb8a6ce..245f2299ace 100644 --- a/docs/get-started/concepts/manage-secrets.mdx +++ b/docs/get-started/concepts/manage-secrets.mdx @@ -235,7 +235,7 @@ client.secrets.create(name="nvidia-api-key", value=os.environ["NVIDIA_API_KEY"]) -After creating the secret, reference it by name when [registering the provider](../../run-inference/about.md#add-external-providers). +After creating the secret, reference it by name when [registering the provider](/documentation/models-and-inference#add-external-providers). ## Encryption Configuration diff --git a/docs/get-started/setup.mdx b/docs/get-started/setup.mdx index 93dd422d635..ebc95af7452 100644 --- a/docs/get-started/setup.mdx +++ b/docs/get-started/setup.mdx @@ -143,9 +143,9 @@ nemo setup --no-install-skills --no-deploy-agent | List available models | `nemo models list` | | Open the Studio UI | [http://localhost:8080](http://localhost:8080) | | Browse docs from the CLI | `nemo docs --list` | -| Evaluate a model | [Run LLM-as-a-Judge Evaluation](../evaluator/tutorials/run-llm-judge-evaluation.md) | -| Generate synthetic data | [Data Designer Tutorials](../data-designer/tutorials/index.md) | -| Optimize an agent | [Optimize Agents](../agents/optimization.md) | +| Evaluate a model | [Run LLM-as-a-Judge Evaluation](/documentation/evaluate-models/tutorials/run-llm-as-a-judge-evaluation) | +| Generate synthetic data | [Data Designer Tutorials](/documentation/design-synthetic-data/tutorials) | +| Optimize an agent | [Optimize Agents](/documentation/agents/optimize-agents) | ## Python SDK @@ -169,7 +169,7 @@ client = NeMoPlatform( ) ``` -An asynchronous client is also available. See the [SDK reference](../pysdk/index.md) for details. +An asynchronous client is also available. See the [SDK reference](/documentation/reference/python-sdk) for details. ## Troubleshooting diff --git a/docs/guardrails/concepts/architecture.mdx b/docs/guardrails/concepts/architecture.mdx index 33724e1fd02..3821ebcdf30 100644 --- a/docs/guardrails/concepts/architecture.mdx +++ b/docs/guardrails/concepts/architecture.mdx @@ -75,7 +75,7 @@ nemo inference virtual-models create my-guarded-model \ --response-middleware '[{"name":"nemo-guardrails","config_type":"guardrail_config","config_id":"default/my-config"}]' ``` -Wire the same config on both `request_middleware` (for input rails) and `response_middleware` (for output rails). If your config only defines input flows, you can omit `response_middleware` — the plugin skips output processing when no output flows are configured, and vice versa. See [Running Inference](inference.md) for the equivalent Python SDK call and a fuller end-to-end example. +Wire the same config on both `request_middleware` (for input rails) and `response_middleware` (for output rails). If your config only defines input flows, you can omit `response_middleware` — the plugin skips output processing when no output flows are configured, and vice versa. See [Running Inference](/documentation/guardrail-models/core-concepts/running-inference) for the equivalent Python SDK call and a fuller end-to-end example. --- @@ -99,7 +99,7 @@ NeMo Guardrails uses IGW's model routing for all model calls: - **Main model**: Owned by IGW. The VirtualModel's `default_model_entity` sets the model used for generation. Your guardrail configuration should omit the `type: "main"` model entry — IGW injects the per-request main model at runtime. - **Task models**: Declared in the guardrail configuration (e.g., `content_safety`, `topic_control`, `self_check_input`). The plugin resolves each task model's `base_url` through IGW's route table using the model's entity reference (`workspace/name` format). -For more details on model entities and providers, refer to [About Models and Inference](../../run-inference/about.md). +For more details on model entities and providers, refer to [About Models and Inference](/documentation/models-and-inference). --- @@ -115,6 +115,6 @@ The plugin caches `LLMRails` instances (the NeMo Guardrails runtime) by content ## Next Steps -- [Guardrail Configurations](configurations/index.md) — Define guardrail configurations (models, prompts, rails). -- [Running Inference](inference.md) — Make inference calls through a guarded VirtualModel. -- [Deploy NemoGuard NIMs](../tutorials/deploy-nemoguard-nims.md) — Deploy NemoGuard NIMs for content safety and topic control. +- [Guardrail Configurations](/documentation/guardrail-models/core-concepts/configurations) — Define guardrail configurations (models, prompts, rails). +- [Running Inference](/documentation/guardrail-models/core-concepts/running-inference) — Make inference calls through a guarded VirtualModel. +- [Deploy NemoGuard NIMs](/documentation/guardrail-models/tutorials/deploy-nemo-guard-ni-ms) — Deploy NemoGuard NIMs for content safety and topic control. diff --git a/docs/guardrails/concepts/checks.mdx b/docs/guardrails/concepts/checks.mdx index 7673dc41ca0..c8dfb906591 100644 --- a/docs/guardrails/concepts/checks.mdx +++ b/docs/guardrails/concepts/checks.mdx @@ -12,7 +12,7 @@ Use the checks endpoint to evaluate messages against input and output rails with -The checks endpoint is a standalone Guardrails API for evaluating messages against rails without running inference. For inference with guardrails applied automatically, use a [VirtualModel with guardrails middleware](inference.md). +The checks endpoint is a standalone Guardrails API for evaluating messages against rails without running inference. For inference with guardrails applied automatically, use a [VirtualModel with guardrails middleware](/documentation/guardrail-models/core-concepts/running-inference). @@ -139,7 +139,7 @@ print(check_result.model_dump_json(indent=2)) ## Debug Results The checks endpoint supports the same `guardrails.options.log` diagnostics as -guardrailed inference. Use these fields to debug scenarios where a check blocks or passes unexpectedly. For field descriptions, see [Log Options](inference.md#log-options). +guardrailed inference. Use these fields to debug scenarios where a check blocks or passes unexpectedly. For field descriptions, see [Log Options](/documentation/guardrail-models/core-concepts/running-inference#log-options). ```bash curl -s $NMP_BASE_URL/apis/guardrails/v2/workspaces/default/checks \ diff --git a/docs/guardrails/concepts/configurations/configuration-structure.mdx b/docs/guardrails/concepts/configurations/configuration-structure.mdx index e6a6ae6fe52..97f3908bc8d 100644 --- a/docs/guardrails/concepts/configurations/configuration-structure.mdx +++ b/docs/guardrails/concepts/configurations/configuration-structure.mdx @@ -26,7 +26,7 @@ Under the IGW plugin architecture, the main model is owned by IGW and specified Include a `type: "main"` entry only when you need to pin the engine or parameters used to talk to the main model — for example, a non-NIM `engine`, a custom `parameters.base_url`, or static `parameters.default_headers`. The `model` field on a main entry is treated as a template; the actual model name always comes from the inference request body. -You can configure task-specific models for any task that occurs during the guardrail process. `type` is a free-form label — rails reference it via `$model=` — so you can declare custom task types in addition to the common ones below (for example, a `vision_rails` model used in [Adding Safety Checks to Multimodal Data](../../tutorials/multimodal-data.md)): +You can configure task-specific models for any task that occurs during the guardrail process. `type` is a free-form label — rails reference it via `$model=` — so you can declare custom task types in addition to the common ones below (for example, a `vision_rails` model used in [Adding Safety Checks to Multimodal Data](/documentation/guardrail-models/tutorials/multimodal-data)): - **`content_safety`**: Content Safety check for detecting harmful content. - **`topic_control`**: Topic Control check for keeping conversations on-topic. @@ -54,7 +54,7 @@ models = [ Model Entities are automatically created when you: - - Create a Model Provider pointing to an external API (refer to [Add External Providers](../../../run-inference/about.md#add-external-providers)) + - Create a Model Provider pointing to an external API (refer to [Add External Providers](/documentation/models-and-inference#add-external-providers)) Use `client.models.list()` to retrieve available Model Entities in your workspace. @@ -269,4 +269,4 @@ Enabling `tracing.enable_content_capture` causes prompts and responses (includin ## Next Steps -- Refer to [Manage Configurations](manage-configs.md) to create, update, and manage your guardrail configurations. +- Refer to [Manage Configurations](/documentation/guardrail-models/core-concepts/configurations/manage-configurations) to create, update, and manage your guardrail configurations. diff --git a/docs/guardrails/concepts/configurations/default-configs.mdx b/docs/guardrails/concepts/configurations/default-configs.mdx index f504f084023..cb455a67ef7 100644 --- a/docs/guardrails/concepts/configurations/default-configs.mdx +++ b/docs/guardrails/concepts/configurations/default-configs.mdx @@ -27,9 +27,9 @@ Applies NemoGuard Content Safety checks to both user inputs and model outputs us The `content-safety` configuration requires the `system/nvidia-llama-3-1-nemotron-safety-guard-8b-v3` Model Entity to be deployed and accessible in the `system` workspace. -The model is available on [build.nvidia.com](https://build.nvidia.com/nvidia/llama-3_1-nemotron-safety-guard-8b-v3) if you do not have access to GPUs. See [Setup](../../../get-started/setup.md) for instructions on creating a `ModelProvider` that routes requests to that endpoint. +The model is available on [build.nvidia.com](https://build.nvidia.com/nvidia/llama-3_1-nemotron-safety-guard-8b-v3) if you do not have access to GPUs. See [Setup](/documentation/get-started) for instructions on creating a `ModelProvider` that routes requests to that endpoint. -See [About Models and Inference](../../../run-inference/about.md) for how models are made available through the platform. +See [About Models and Inference](/documentation/models-and-inference) for how models are made available through the platform. --- @@ -131,7 +131,7 @@ client.inference.virtual_models.delete(name="guarded-with-defaults") ## Related -- [Architecture](../architecture.md) — Understand the middleware pipeline and VirtualModel wiring -- [Configuration Structure](configuration-structure.md) — Configuration schema reference -- [Manage Configurations](manage-configs.md) — Create and manage your own configurations -- [Run Inference](../../../run-inference/tutorials/run-inference.md) — Run inference with a guardrail configuration +- [Architecture](/documentation/guardrail-models/core-concepts/architecture) — Understand the middleware pipeline and VirtualModel wiring +- [Configuration Structure](/documentation/guardrail-models/core-concepts/configurations/configuration-structure) — Configuration schema reference +- [Manage Configurations](/documentation/guardrail-models/core-concepts/configurations/manage-configurations) — Create and manage your own configurations +- [Run Inference](/documentation/models-and-inference/tutorials/run-inference) — Run inference with a guardrail configuration diff --git a/docs/guardrails/concepts/configurations/index.mdx b/docs/guardrails/concepts/configurations/index.mdx index 9303f53f8ad..36219bcd437 100644 --- a/docs/guardrails/concepts/configurations/index.mdx +++ b/docs/guardrails/concepts/configurations/index.mdx @@ -25,17 +25,17 @@ A guardrail configuration also supports general options such as instructions and - + Learn about models, prompts, and rails that make up a configuration. - + Create, list, retrieve, update, and delete guardrail configurations. - + Built-in configurations available in the `system` workspace: `default`, `content-safety`, and `self-check`. diff --git a/docs/guardrails/concepts/configurations/manage-configs.mdx b/docs/guardrails/concepts/configurations/manage-configs.mdx index 2f2f1451e6b..244c1a4a8ca 100644 --- a/docs/guardrails/concepts/configurations/manage-configs.mdx +++ b/docs/guardrails/concepts/configurations/manage-configs.mdx @@ -6,7 +6,7 @@ description: "" -Guardrail configuration management operations (create, update, retrieve, list, delete) are available through `client.guardrail.configs`. Refer to [Configuration Structure](configuration-structure.md) for details on the configuration data schema. +Guardrail configuration management operations (create, update, retrieve, list, delete) are available through `client.guardrail.configs`. Refer to [Configuration Structure](/documentation/guardrail-models/core-concepts/configurations/configuration-structure) for details on the configuration data schema. --- diff --git a/docs/guardrails/concepts/index.mdx b/docs/guardrails/concepts/index.mdx index 9ab5e4786ed..cbe8fe0cafc 100644 --- a/docs/guardrails/concepts/index.mdx +++ b/docs/guardrails/concepts/index.mdx @@ -8,7 +8,7 @@ NeMo Guardrails provides configurable safety checks that evaluate user inputs an --- -- **[Architecture](architecture.md)** — How guardrails runs as IGW middleware, VirtualModel wiring, and model routing. -- **[Configurations](configurations/index.md)** — Define rails, models, and prompts that make up a guardrail policy. -- **[Inference](inference.md)** — Run inference through a guarded VirtualModel and configure request options. -- **[Checks](checks.md)** — Evaluate messages against rails without running inference. +- **[Architecture](/documentation/guardrail-models/core-concepts/architecture)** — How guardrails runs as IGW middleware, VirtualModel wiring, and model routing. +- **[Configurations](/documentation/guardrail-models/core-concepts/configurations)** — Define rails, models, and prompts that make up a guardrail policy. +- **[Inference](/documentation/guardrail-models/core-concepts/running-inference)** — Run inference through a guarded VirtualModel and configure request options. +- **[Checks](/documentation/guardrail-models/core-concepts/running-checks)** — Evaluate messages against rails without running inference. diff --git a/docs/guardrails/concepts/inference.mdx b/docs/guardrails/concepts/inference.mdx index 410d5893f2e..fce29250754 100644 --- a/docs/guardrails/concepts/inference.mdx +++ b/docs/guardrails/concepts/inference.mdx @@ -9,16 +9,16 @@ description: "" NeMo Guardrails applies safety checks to inference requests through VirtualModels. When your application sends a request to a VirtualModel with guardrails middleware, the plugin runs input and output rails around the model call automatically. You use the standard IGW OpenAI-compatible endpoint — no separate guardrails endpoint is needed. -Platform-managed agents are the canonical consumer: pointing an agent's `llms` block at a guarded VirtualModel entity is how you secure the agent's model path. See [Secure Agents](../../agents/security.md) for the agent-side wiring. The rest of this page applies to any client calling a guarded VirtualModel, agent or not. +Platform-managed agents are the canonical consumer: pointing an agent's `llms` block at a guarded VirtualModel entity is how you secure the agent's model path. See [Secure Agents](/documentation/agents/secure-agents) for the agent-side wiring. The rest of this page applies to any client calling a guarded VirtualModel, agent or not. ## Prerequisites -- A guardrail configuration exists in your workspace. See [Manage Configurations](configurations/manage-configs.md) to create one. -- A VirtualModel is configured with `nemo-guardrails` middleware. See [Architecture](architecture.md) for wiring details. +- A guardrail configuration exists in your workspace. See [Manage Configurations](/documentation/guardrail-models/core-concepts/configurations/manage-configurations) to create one. +- A VirtualModel is configured with `nemo-guardrails` middleware. See [Architecture](/documentation/guardrail-models/core-concepts/architecture) for wiring details. -If you do not have access to GPUs, you can use NIMs hosted on `build.nvidia.com`. Refer to [Setup](../../get-started/setup.md) for instructions on configuring a `ModelProvider` that routes requests to an external endpoint. +If you do not have access to GPUs, you can use NIMs hosted on `build.nvidia.com`. Refer to [Setup](/documentation/get-started) for instructions on configuring a `ModelProvider` that routes requests to an external endpoint. For the examples below, create a guardrail configuration and VirtualModel: @@ -200,7 +200,7 @@ The `model` field in your request must reference a **VirtualModel** entity (`wor Task models in your guardrail configuration (content safety, topic control, etc.) must reference **Model Entities** using the same `workspace/name` format. The plugin resolves their endpoints through IGW's route table. -For VirtualModel wiring (`request_middleware`/`response_middleware`, entity-backed vs inline configs), refer to [Architecture](architecture.md). For an overview of how Model Entities and Model Providers fit together, refer to [About Models and Inference](../../run-inference/about.md). +For VirtualModel wiring (`request_middleware`/`response_middleware`, entity-backed vs inline configs), refer to [Architecture](/documentation/guardrail-models/core-concepts/architecture). For an overview of how Model Entities and Model Providers fit together, refer to [About Models and Inference](/documentation/models-and-inference). ## Inline Configuration @@ -255,7 +255,7 @@ client.inference.virtual_models.create( -The inline config shares the same `LLMRails` pool when its content hash matches an existing entry. The plugin warms inline configs on VirtualModel upsert too — it walks the VM's middleware entries, stabilizes each source, and dedups by content hash — so the first request after upsert finds a hot pool just like entity-backed configs do. See [Architecture](architecture.md#caching-and-performance) for the cache details. +The inline config shares the same `LLMRails` pool when its content hash matches an existing entry. The plugin warms inline configs on VirtualModel upsert too — it walks the VM's middleware entries, stabilizes each source, and dedups by content hash — so the first request after upsert finds a hot pool just like entity-backed configs do. See [Architecture](/documentation/guardrail-models/core-concepts/architecture#caching-and-performance) for the cache details. The optional `name` field sets the diagnostic label in logs (appears as ``). diff --git a/docs/guardrails/index.mdx b/docs/guardrails/index.mdx index 582df11b2b1..158271edf0a 100644 --- a/docs/guardrails/index.mdx +++ b/docs/guardrails/index.mdx @@ -12,10 +12,10 @@ Guardrail configurations define which checks run, which models perform the check ## Concepts -- **[Architecture](concepts/architecture.md)** — How guardrails fits into the inference pipeline as IGW middleware. -- **[Guardrail Configurations](concepts/configurations/index.md)** — Define rails, models, and prompts. -- **[Running Inference](concepts/inference.md)** — Make inference calls through a guarded VirtualModel. -- **[Running Checks](concepts/checks.md)** — Evaluate messages against rails without running inference. +- **[Architecture](/documentation/guardrail-models/core-concepts/architecture)** — How guardrails fits into the inference pipeline as IGW middleware. +- **[Guardrail Configurations](/documentation/guardrail-models/core-concepts/configurations)** — Define rails, models, and prompts. +- **[Running Inference](/documentation/guardrail-models/core-concepts/running-inference)** — Make inference calls through a guarded VirtualModel. +- **[Running Checks](/documentation/guardrail-models/core-concepts/running-checks)** — Evaluate messages against rails without running inference. --- @@ -23,8 +23,8 @@ Guardrail configurations define which checks run, which models perform the check These tutorials walk you through common guardrail tasks using NeMo Guardrails. -- **[Deploy NemoGuard NIMs](tutorials/deploy-nemoguard-nims.md)** — Deploy NemoGuard NIMs in your environment. -- **[Content Safety](tutorials/content-safety.md)** — Detect and block harmful content with NemoGuard NIMs. -- **[Parallel Rails](tutorials/parallel-rails.md)** — Run input and output rails in parallel. -- **[Multimodal Data](tutorials/multimodal-data.md)** — Safety checks for multimodal data. -- **[Injection Detection](tutorials/injection-detection.md)** — Detect SQL, XSS, and code injection. +- **[Deploy NemoGuard NIMs](/documentation/guardrail-models/tutorials/deploy-nemo-guard-ni-ms)** — Deploy NemoGuard NIMs in your environment. +- **[Content Safety](/documentation/guardrail-models/tutorials/content-safety)** — Detect and block harmful content with NemoGuard NIMs. +- **[Parallel Rails](/documentation/guardrail-models/tutorials/parallel-rails)** — Run input and output rails in parallel. +- **[Multimodal Data](/documentation/guardrail-models/tutorials/multimodal-data)** — Safety checks for multimodal data. +- **[Injection Detection](/documentation/guardrail-models/tutorials/injection-detection)** — Detect SQL, XSS, and code injection. diff --git a/docs/guardrails/observability.mdx b/docs/guardrails/observability.mdx index a3761d147be..fb1ef8484ed 100644 --- a/docs/guardrails/observability.mdx +++ b/docs/guardrails/observability.mdx @@ -12,7 +12,7 @@ NeMo Platform centrally manages OpenTelemetry across services. You can configure To export guardrail traces, OpenTelemetry must be enabled in your deployment. See [OpenTelemetry](../set-up/opentelemetry.md) for instructions on enabling OpenTelemetry. -You also need a VirtualModel configured with guardrails middleware. See [Architecture](concepts/architecture.md) for wiring details. +You also need a VirtualModel configured with guardrails middleware. See [Architecture](/documentation/guardrail-models/core-concepts/architecture) for wiring details. --- diff --git a/docs/guardrails/tutorials/content-safety.mdx b/docs/guardrails/tutorials/content-safety.mdx index 47725eaf013..e9c09710ac6 100644 --- a/docs/guardrails/tutorials/content-safety.mdx +++ b/docs/guardrails/tutorials/content-safety.mdx @@ -18,7 +18,7 @@ Before you begin: - You have access to a running NeMo Platform. - `NMP_BASE_URL` is set to the NeMo Platform base URL. -- A `ModelProvider` is configured with an LLM provider. Follow [Setup](../../get-started/setup.md) if you haven't done this yet. +- A `ModelProvider` is configured with an LLM provider. Follow [Setup](/documentation/get-started) if you haven't done this yet. This tutorial uses the following NIMs, available on `build.nvidia.com`: diff --git a/docs/guardrails/tutorials/deploy-nemoguard-nims.mdx b/docs/guardrails/tutorials/deploy-nemoguard-nims.mdx index 1711f25e599..22157887548 100644 --- a/docs/guardrails/tutorials/deploy-nemoguard-nims.mdx +++ b/docs/guardrails/tutorials/deploy-nemoguard-nims.mdx @@ -247,7 +247,7 @@ The jailbreak detection NIM exposes a `/v1/classify` endpoint rather than an Ope ### Content Safety and Topic Control -Reference the Model Entities in your guardrail configuration using the `workspace/name` format. For a complete example combining content safety and topic control rails, see [Executing Input and Output Rails in Parallel](parallel-rails.md). +Reference the Model Entities in your guardrail configuration using the `workspace/name` format. For a complete example combining content safety and topic control rails, see [Executing Input and Output Rails in Parallel](/documentation/guardrail-models/tutorials/parallel-rails). ### Jailbreak Detection @@ -334,5 +334,5 @@ print("Cleanup complete") ## Next Steps -- [Improving Content Safety with NemoGuard NIMs](content-safety.md) - Full content safety tutorial using `build.nvidia.com`-hosted NIMs -- [Executing Input and Output Rails in Parallel](parallel-rails.md) - Combine multiple rails for comprehensive safety coverage +- [Improving Content Safety with NemoGuard NIMs](/documentation/guardrail-models/tutorials/content-safety) - Full content safety tutorial using `build.nvidia.com`-hosted NIMs +- [Executing Input and Output Rails in Parallel](/documentation/guardrail-models/tutorials/parallel-rails) - Combine multiple rails for comprehensive safety coverage diff --git a/docs/guardrails/tutorials/index.mdx b/docs/guardrails/tutorials/index.mdx index dcd17216a74..8739613320f 100644 --- a/docs/guardrails/tutorials/index.mdx +++ b/docs/guardrails/tutorials/index.mdx @@ -8,11 +8,11 @@ Use the following tutorials to learn how to accomplish common guardrail tasks us -Before starting a tutorial, complete [Setup](../../get-started/setup.md) to configure a `ModelProvider` and discover models. The tutorials assume you have models available in the `default` workspace. +Before starting a tutorial, complete [Setup](/documentation/get-started) to configure a `ModelProvider` and discover models. The tutorials assume you have models available in the `default` workspace. -Once a `ModelProvider` is configured, use Model Entity references (`workspace/name` format) as the `model` in your guardrail configurations. The guardrails plugin resolves task model endpoints through IGW's route table. Refer to [Model Routing](../concepts/inference.md#guardrails-model-routing) for more details. +Once a `ModelProvider` is configured, use Model Entity references (`workspace/name` format) as the `model` in your guardrail configurations. The guardrails plugin resolves task model endpoints through IGW's route table. Refer to [Model Routing](/documentation/guardrail-models/core-concepts/running-inference#guardrails-model-routing) for more details. ```python guardrails_config = { @@ -29,35 +29,35 @@ guardrails_config = { - + Deploy NemoGuard NIMs in your environment nemo-guardrails nemoguard - + Use Content Safety checks to detect and block harmful content nemo-guardrails nemoguard - + Configure parallel rails for input and output guardrails. nemo-guardrails - + Safety checks for multimodal data with NeMo Guardrails API. nemo-guardrails - + Configure checks for SQL, XSS, template, and code injection. diff --git a/docs/guardrails/tutorials/injection-detection.mdx b/docs/guardrails/tutorials/injection-detection.mdx index 28122e8a5fc..d23c0715a31 100644 --- a/docs/guardrails/tutorials/injection-detection.mdx +++ b/docs/guardrails/tutorials/injection-detection.mdx @@ -36,7 +36,7 @@ Before you begin: - You have access to a running NeMo Platform. - `NMP_BASE_URL` is set to the NeMo Platform base URL. -- A `ModelProvider` is configured with an LLM provider. Follow [Setup](../../get-started/setup.md) if you haven't done this yet. +- A `ModelProvider` is configured with an LLM provider. Follow [Setup](/documentation/get-started) if you haven't done this yet. This tutorial uses the following NIM, available on `build.nvidia.com`: diff --git a/docs/guardrails/tutorials/multimodal-data.mdx b/docs/guardrails/tutorials/multimodal-data.mdx index 60ff54868b9..c76f7ad3f3a 100644 --- a/docs/guardrails/tutorials/multimodal-data.mdx +++ b/docs/guardrails/tutorials/multimodal-data.mdx @@ -27,7 +27,7 @@ Before you begin: - You have access to a running NeMo Platform. - `NMP_BASE_URL` is set to the NeMo Platform base URL. -- A `ModelProvider` is configured with an LLM provider. Follow [Setup](../../get-started/setup.md) if you haven't done this yet. +- A `ModelProvider` is configured with an LLM provider. Follow [Setup](/documentation/get-started) if you haven't done this yet. This tutorial uses the following NIM, available on `build.nvidia.com`: diff --git a/docs/guardrails/tutorials/parallel-rails.mdx b/docs/guardrails/tutorials/parallel-rails.mdx index 01df8ec09e1..e67fb1d725e 100644 --- a/docs/guardrails/tutorials/parallel-rails.mdx +++ b/docs/guardrails/tutorials/parallel-rails.mdx @@ -37,7 +37,7 @@ Before you begin: - You have access to a running NeMo Platform. - `NMP_BASE_URL` is set to the NeMo Platform base URL. -- A `ModelProvider` is configured with an LLM provider. Follow [Setup](../../get-started/setup.md) if you haven't done this yet. +- A `ModelProvider` is configured with an LLM provider. Follow [Setup](/documentation/get-started) if you haven't done this yet. This tutorial uses the following NIMs, available on `build.nvidia.com`: diff --git a/docs/index.mdx b/docs/index.mdx index 2944505d69d..cebc2f0af4a 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -19,18 +19,18 @@ NeMo Platform brings NVIDIA NeMo libraries together under one CLI, Python SDK, a The platform provides several shared capabilities to NeMo libraries. -- [Models and Inference](run-inference/about.md) for model providers, virtual +- [Models and Inference](/documentation/models-and-inference) for model providers, virtual models, and gateway calls. -- [Files](get-started/concepts/manage-files.md), [Secrets](get-started/concepts/manage-secrets.md), - [Entities](get-started/concepts/entities.md), and jobs for storing state and +- [Files](/documentation/get-started/core-concepts/manage-files), [Secrets](/documentation/get-started/core-concepts/manage-secrets), + [Entities](/documentation/get-started/core-concepts/entities), and jobs for storing state and running local work. ## Where to Go Next -- [Setup](get-started/setup.md) - install, configure providers, and run +- [Setup](/documentation/get-started) - install, configure providers, and run local services. -- [About Agents](agents/index.md) - learn the managed agent lifecycle. -- [Optimize Agents](agents/optimization.md) - improve cost, quality, and model +- [About Agents](/documentation/agents) - learn the managed agent lifecycle. +- [Optimize Agents](/documentation/agents/optimize-agents) - improve cost, quality, and model routing. -- [Secure Agents](agents/security.md) - harden agents with guardrails and data +- [Secure Agents](/documentation/agents/secure-agents) - harden agents with guardrails and data safety checks. diff --git a/docs/notebooks/ndd_evaluator.mdx b/docs/notebooks/ndd_evaluator.mdx index 4146e4bc2ba..48e61d3a0a0 100644 --- a/docs/notebooks/ndd_evaluator.mdx +++ b/docs/notebooks/ndd_evaluator.mdx @@ -652,6 +652,6 @@ display(df_results.style.format("{:.2f}")) ## Next Steps -- Explore more [Evaluator metrics](../evaluator/metrics/index.md) for additional evaluation scenarios. -- Learn more [Data Designer workflows](../data-designer/tutorials/index.md) for advanced synthetic generation pipelines. -- Review [Files service management](../get-started/concepts/manage-files.md) for organizing datasets and artifacts. +- Explore more [Evaluator metrics](/documentation/evaluate-models/metrics) for additional evaluation scenarios. +- Learn more [Data Designer workflows](/documentation/design-synthetic-data/tutorials) for advanced synthetic generation pipelines. +- Review [Files service management](/documentation/get-started/core-concepts/manage-files) for organizing datasets and artifacts. diff --git a/docs/pysdk/index.mdx b/docs/pysdk/index.mdx index 1e5083f9f09..af250136d5e 100644 --- a/docs/pysdk/index.mdx +++ b/docs/pysdk/index.mdx @@ -27,5 +27,5 @@ This project will download and install additional third-party open source softwa ## Next Steps -- Read more about connecting with the [client APIs](./client/index.md). +- Read more about connecting with the [client APIs](/documentation/reference/python-sdk/client-ap-is). - Browse the [REST API Reference](/documentation/reference/api-reference). diff --git a/docs/requirements.mdx b/docs/requirements.mdx index 8ab4222509a..9c87b7118e8 100644 --- a/docs/requirements.mdx +++ b/docs/requirements.mdx @@ -2,7 +2,7 @@ title: "Hardware and Software Requirements for NeMo Platform" description: "" --- -This page lists the requirements for the OSS local-install path for NeMo Platform 0.1.0. For the full compatibility table, see the [Support Matrix](support-matrix.md). +This page lists the requirements for the OSS local-install path for NeMo Platform 0.1.0. For the full compatibility table, see the [Support Matrix](/documentation/reference/support-matrix). The OSS 0.1.0 documentation is scoped to local setup with the Python package and `nemo setup`. Docker Compose, Helm, Kubernetes, and OpenShift deployment guides are not part of this release scope. @@ -12,7 +12,7 @@ The OSS 0.1.0 documentation is scoped to local setup with the Python package and |-----------|-------------|-------| | Python | 3.11, 3.12, or 3.13 (`>=3.11,<3.14`) | Use an isolated virtual environment. Python 3.14 and later are not part of the OSS local-install support range. | | Package installer | `uv` recommended; `pip` supported | The one-line installer sets up `uv` automatically when needed. | -| Operating system | Recent Linux or macOS release | See the [Support Matrix](support-matrix.md) for the supported OS list. | +| Operating system | Recent Linux or macOS release | See the [Support Matrix](/documentation/reference/support-matrix) for the supported OS list. | | Memory | 8 GB RAM minimum | 16 GB or more is recommended for larger local workflows. | | Disk space | 16 GB free disk space minimum | Additional space is needed for datasets, job outputs, and local model artifacts. | | Network | Outbound HTTPS access | Required for package installation and hosted model-provider APIs. | @@ -38,4 +38,4 @@ nemo --help curl -s http://localhost:8080/health/ready ``` -For setup instructions, see [Setup](get-started/setup.md). +For setup instructions, see [Setup](/documentation/get-started). diff --git a/docs/run-inference/about.mdx b/docs/run-inference/about.mdx index 5b6b665718e..1939a6bd056 100644 --- a/docs/run-inference/about.mdx +++ b/docs/run-inference/about.mdx @@ -31,7 +31,7 @@ Models service manages model entities and model providers. Model providers connect the platform to external inference APIs such as [NVIDIA Build](https://build.nvidia.com/) or OpenAI. The workflow is: -1. **Store the API key** as a [secret](../get-started/concepts/manage-secrets.md) in the platform +1. **Store the API key** as a [secret](/documentation/get-started/core-concepts/manage-secrets) in the platform 2. **Create a model provider** pointing to the external API with the secret reference 3. **Route inference** through the gateway using provider or model entity routing @@ -294,4 +294,4 @@ working as designed — use `/health/ready` instead. ## API Reference -For complete API details, refer to the [Inference Gateway API Reference](/documentation/reference/api-reference) and [SDK Reference](../pysdk/index.md). +For complete API details, refer to the [Inference Gateway API Reference](/documentation/reference/api-reference) and [SDK Reference](/documentation/reference/python-sdk). diff --git a/docs/run-inference/tutorials/index.mdx b/docs/run-inference/tutorials/index.mdx index a7d8d3bcb1d..30412310290 100644 --- a/docs/run-inference/tutorials/index.mdx +++ b/docs/run-inference/tutorials/index.mdx @@ -13,4 +13,4 @@ Learn how to run inference through the NeMo Platform. ## Guides -- [Run Inference](run-inference.md) — Route requests via model entity, provider, or OpenAI routing +- [Run Inference](/documentation/models-and-inference/tutorials/run-inference) — Route requests via model entity, provider, or OpenAI routing diff --git a/docs/run-inference/tutorials/run-inference.mdx b/docs/run-inference/tutorials/run-inference.mdx index 8baeeee415b..858c74a849a 100644 --- a/docs/run-inference/tutorials/run-inference.mdx +++ b/docs/run-inference/tutorials/run-inference.mdx @@ -8,7 +8,7 @@ Route inference requests through the gateway using model entity routing, provide -This tutorial assumes you have an external provider registered. The platform pre-configures a `system/nvidia-build` provider on startup, which most of the examples below use. To register your own provider, see [About Models and Inference](../about.md#model-providers). +This tutorial assumes you have an external provider registered. The platform pre-configures a `system/nvidia-build` provider on startup, which most of the examples below use. To register your own provider, see [About Models and Inference](/documentation/models-and-inference#model-providers). @@ -247,7 +247,7 @@ response = client.inference.gateway.openai.post( ### Using OpenAI Python SDK -Models service provides a convenient helper method `client.models.get_openai_client()` that can generate an OpenAI SDK client for the configured workspace. The SDK also offers helper methods for generating OpenAI-compatible URL strings for Inference Gateway. Refer to [SDK Helper Methods](../about.md#sdk-helper-methods) for more info. +Models service provides a convenient helper method `client.models.get_openai_client()` that can generate an OpenAI SDK client for the configured workspace. The SDK also offers helper methods for generating OpenAI-compatible URL strings for Inference Gateway. Refer to [SDK Helper Methods](/documentation/models-and-inference#sdk-helper-methods) for more info. ```python # Get pre-configured OpenAI client diff --git a/docs/safe-synthesizer/about/evaluation.mdx b/docs/safe-synthesizer/about/evaluation.mdx index 159f395c8ff..29dd3573f25 100644 --- a/docs/safe-synthesizer/about/evaluation.mdx +++ b/docs/safe-synthesizer/about/evaluation.mdx @@ -124,6 +124,6 @@ We enable evaluation by default. ## Related Topics -- [safe-synthesizer-101](../tutorials/safe-synthesizer-101.md): Get started with evaluation +- [safe-synthesizer-101](/documentation/safe-synthesizer/tutorials/safe-synthesizer-101): Get started with evaluation - [differential-privacy](../tutorials/differential-privacy.md): Learn about privacy metrics - [index](../tutorials/index.md): More tutorials diff --git a/docs/safe-synthesizer/about/index.mdx b/docs/safe-synthesizer/about/index.mdx index f371e73dd20..efd99b23cad 100644 --- a/docs/safe-synthesizer/about/index.mdx +++ b/docs/safe-synthesizer/about/index.mdx @@ -14,14 +14,14 @@ NVIDIA NeMo Safe Synthesizer enables you to create private versions of sensitive NeMo Safe Synthesizer allows you to generate synthetic data that maintains the statistical properties of your original dataset without exposing sensitive information about individual records. -NeMo Safe Synthesizer is best when you have the data you need, but because it is private or sensitive in nature you cannot use it as-is. NeMo Safe Synthesizer interpolates from existing data to generate a private, synthetic version, where new records have no one-to-one mapping to original records. If you do not have any data or want to extrapolate based on a very small set of examples, refer to [index](../../data-designer/index.md). NeMo Data Designer supports synthetic data creation from scratch or small seed for AI training and development use cases. +NeMo Safe Synthesizer is best when you have the data you need, but because it is private or sensitive in nature you cannot use it as-is. NeMo Safe Synthesizer interpolates from existing data to generate a private, synthetic version, where new records have no one-to-one mapping to original records. If you do not have any data or want to extrapolate based on a very small set of examples, refer to [index](/documentation/design-synthetic-data). NeMo Data Designer supports synthetic data creation from scratch or small seed for AI training and development use cases. ## NeMo Safe Synthesizer Job A complete NeMo Safe Synthesizer job consists of the following steps: -1. [Upload Data](../../get-started/concepts/manage-files.md): Add your tabular data to the Files API +1. [Upload Data](/documentation/get-started/core-concepts/manage-files): Add your tabular data to the Files API 2. Prepare Data: - [Configure PII Replacement](pii-replacement.md): Set up detection and replacement of sensitive information (recommended prior to the Synthesis step to ensure the model has no chance of learning the most sensitive information like names and addresses) - Configure training data organization and holdout splits diff --git a/docs/safe-synthesizer/about/jobs.mdx b/docs/safe-synthesizer/about/jobs.mdx index f653b0b9311..8ae0a5eef88 100644 --- a/docs/safe-synthesizer/about/jobs.mdx +++ b/docs/safe-synthesizer/about/jobs.mdx @@ -313,6 +313,6 @@ kubectl get events -n --sort-by='.lastTimestamp' ## Related Topics - [Host-Local Development and Testing](host-local-development.md): `run-local`, adapter reuse, and plugin tests -- [safe-synthesizer-101](../tutorials/safe-synthesizer-101.md): Get started with NeMo Safe Synthesizer jobs +- [safe-synthesizer-101](/documentation/safe-synthesizer/tutorials/safe-synthesizer-101): Get started with NeMo Safe Synthesizer jobs - [index](../tutorials/index.md): More hands-on tutorials - [reference](reference.md): Full parameter reference diff --git a/docs/safe-synthesizer/about/pii-replacement.mdx b/docs/safe-synthesizer/about/pii-replacement.mdx index 00a0d66c629..d708739554c 100644 --- a/docs/safe-synthesizer/about/pii-replacement.mdx +++ b/docs/safe-synthesizer/about/pii-replacement.mdx @@ -217,5 +217,5 @@ PII replacement is always recommended as a preprocessing step before synthesis. ## Related Topics -- [safe-synthesizer-101](../tutorials/safe-synthesizer-101.md): Getting started tutorial with PII replacement +- [safe-synthesizer-101](/documentation/safe-synthesizer/tutorials/safe-synthesizer-101): Getting started tutorial with PII replacement - [index](../tutorials/index.md): More tutorials diff --git a/docs/safe-synthesizer/getting-started.mdx b/docs/safe-synthesizer/getting-started.mdx index 409b1bb578b..f5ac9f0ade9 100644 --- a/docs/safe-synthesizer/getting-started.mdx +++ b/docs/safe-synthesizer/getting-started.mdx @@ -41,7 +41,7 @@ Platform job submission (Jobs API, Studio, tutorials) is documented separately i Run one of the [tutorials](tutorials/index.md) to create your first synthetic dataset: -- [Safe Synthesizer 101 Tutorial](tutorials/safe-synthesizer-101.md) - A beginner-friendly introduction +- [Safe Synthesizer 101 Tutorial](/documentation/safe-synthesizer/tutorials/safe-synthesizer-101) - A beginner-friendly introduction - [Differential Privacy Tutorial](tutorials/differential-privacy.md) - Generate differentially-private synthetic data --- diff --git a/docs/safe-synthesizer/tutorials/differential-privacy.mdx b/docs/safe-synthesizer/tutorials/differential-privacy.mdx index 0ea9d3c6d1d..2ac5ba62eb7 100644 --- a/docs/safe-synthesizer/tutorials/differential-privacy.mdx +++ b/docs/safe-synthesizer/tutorials/differential-privacy.mdx @@ -8,7 +8,7 @@ description: "" Learn how to apply differential privacy to achieve the maximum level of privacy with mathematical guarantees. This tutorial explores the privacy-utility tradeoff and demonstrates how to configure differential privacy parameters for optimal results. -If you have not yet completed the [Safe Synthesizer 101](safe-synthesizer-101.md) tutorial, consider starting there first. +If you have not yet completed the [Safe Synthesizer 101](/documentation/safe-synthesizer/tutorials/safe-synthesizer-101) tutorial, consider starting there first. ## Prerequisites diff --git a/docs/safe-synthesizer/tutorials/index.mdx b/docs/safe-synthesizer/tutorials/index.mdx index 1da7440c6a3..4b008c53eef 100644 --- a/docs/safe-synthesizer/tutorials/index.mdx +++ b/docs/safe-synthesizer/tutorials/index.mdx @@ -23,7 +23,7 @@ Before starting any tutorial, ensure you have: - + Learn the basics with your first Safe Synthesizer job, leveraging smart defaults. This tutorial covers uploading data, running a synthesis job with PII replacement, and reviewing evaluation reports. diff --git a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx index 3c53334fc15..f3eb2cd9937 100644 --- a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx +++ b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx @@ -124,7 +124,7 @@ print(f"✅ Using model provider: {provider_name}") -If you prefer not to send column data to `build.nvidia.com`, you can [deploy your own LLM](../../run-inference/tutorials/deploy-models.md) and create a custom model provider. Pass the fully-qualified provider name (`workspace/provider-name`) to `.with_classify_model_provider()` instead. +If you prefer not to send column data to `build.nvidia.com`, you can [deploy your own LLM](/documentation/models-and-inference/tutorials/deploy-models) and create a custom model provider. Pass the fully-qualified provider name (`workspace/provider-name`) to `.with_classify_model_provider()` instead. --- diff --git a/docs/set-up/helm/install.mdx b/docs/set-up/helm/install.mdx index 04ab7f7e0ed..68e3aaa8f5d 100644 --- a/docs/set-up/helm/install.mdx +++ b/docs/set-up/helm/install.mdx @@ -6,7 +6,7 @@ description: "" -Note: This setup is the full enterprise platform, meant for advanced use. If you're just getting started, check out [setting up a local instance of the platform](../../get-started/setup.md) instead — it's faster and easier to explore the basics. +Note: This setup is the full enterprise platform, meant for advanced use. If you're just getting started, check out [setting up a local instance of the platform](/documentation/get-started) instead — it's faster and easier to explore the basics. diff --git a/docs/set-up/helm/persistent-volumes.mdx b/docs/set-up/helm/persistent-volumes.mdx index 7263b5eb7a6..39839003e53 100644 --- a/docs/set-up/helm/persistent-volumes.mdx +++ b/docs/set-up/helm/persistent-volumes.mdx @@ -68,7 +68,7 @@ platformConfig: Replace `"nfs"` with your StorageClass name (e.g. `oci-nfs`, `gp3`). For NIM scaling and multi-node deployments, use a ReadWriteMany-capable StorageClass. -Refer to the [platform configuration documentation](../config-reference.md) for the full config reference. +Refer to the [platform configuration documentation](/documentation/reference/configuration-reference) for the full config reference. ## Persistent volume options diff --git a/docs/set-up/helm/prerequisites.mdx b/docs/set-up/helm/prerequisites.mdx index 4221fd4c50c..ee77f6ac529 100644 --- a/docs/set-up/helm/prerequisites.mdx +++ b/docs/set-up/helm/prerequisites.mdx @@ -10,7 +10,7 @@ Before installing the \{\{helm_chart_short_name}}, review requirements and creat ## Review Requirements -To check the hardware and software specifications required for installing the \{\{helm_chart_short_name}}, review [](../../requirements.md). +To check the hardware and software specifications required for installing the \{\{helm_chart_short_name}}, review [](/documentation/reference/system-requirements). For storage and persistent volume configuration (including ReadWriteMany and StorageClass configuration), see [](persistent-volumes.md). @@ -66,4 +66,4 @@ To learn more about using Helm charts in the NGC Catalog console in general, ref --namespace ``` -For more information about various secrets you might need to create and set up through the \{\{helm_chart_short_name}} depending on your use case, refer to [](../../get-started/concepts/manage-secrets.md). +For more information about various secrets you might need to create and set up through the \{\{helm_chart_short_name}} depending on your use case, refer to [](/documentation/get-started/core-concepts/manage-secrets). diff --git a/docs/set-up/manage-jobs.mdx b/docs/set-up/manage-jobs.mdx index 128d910915c..358c24ea496 100644 --- a/docs/set-up/manage-jobs.mdx +++ b/docs/set-up/manage-jobs.mdx @@ -48,7 +48,7 @@ jobs: node-pool-name: h200-pool ``` -For full configuration details, see the [platform configuration reference](config-reference.md). +For full configuration details, see the [platform configuration reference](/documentation/reference/configuration-reference). ### Default Execution Profiles @@ -57,7 +57,7 @@ The NeMo Platform defines a default execution profile for each execution backend - Default CPU Execution Profile (`cpu`): The default execution profile for CPU based jobs. - Default GPU Execution Profile (`gpu`): The default execution profile for GPU based jobs. -You may configure the default execution profiles by updating the `executor_defaults` section of the `jobs` section of the [platform configuration](config-reference.md). The structure of the `executor_defaults` matches the configuration of the execution backend configuration. +You may configure the default execution profiles by updating the `executor_defaults` section of the `jobs` section of the [platform configuration](/documentation/reference/configuration-reference). The structure of the `executor_defaults` matches the configuration of the execution backend configuration. ```yaml jobs: @@ -106,7 +106,7 @@ The NeMo Platform supports Docker as an execution backend for CPU and GPU based -If you are running GPU jobs with Docker, see [GPU Configuration](../get-started/setup.md) for information on configuring the shared GPU pool. This configuration is shared between the jobs and models services to prevent GPU resource conflicts. +If you are running GPU jobs with Docker, see [GPU Configuration](/documentation/get-started) for information on configuring the shared GPU pool. This configuration is shared between the jobs and models services to prevent GPU resource conflicts. diff --git a/docs/set-up/milvus.mdx b/docs/set-up/milvus.mdx index 3893c5100d0..77bc266e1f8 100644 --- a/docs/set-up/milvus.mdx +++ b/docs/set-up/milvus.mdx @@ -4,15 +4,15 @@ description: "" --- -NeMo Evaluator uses Milvus for vector database storage for [Retrieval evaluations](../evaluator/metrics/retriever.md) and [RAG evaluations](../evaluator/metrics/rag.md). +NeMo Evaluator uses Milvus for vector database storage for [Retrieval evaluations](../evaluator/metrics/retriever.md) and [RAG evaluations](/documentation/evaluate-models/metrics/rag-metrics). ## Configuration -To configure NeMo Evaluator to use Milvus, set the `milvus_url` in the [platform configuration](config-reference.md): +To configure NeMo Evaluator to use Milvus, set the `milvus_url` in the [platform configuration](/documentation/reference/configuration-reference): ```yaml evaluator: milvus_url: "milvus-standalone.default.svc.cluster.local:19530" ``` -See the [platform configuration reference](config-reference.md) for the complete NeMo Evaluator configuration reference. +See the [platform configuration reference](/documentation/reference/configuration-reference) for the complete NeMo Evaluator configuration reference. diff --git a/docs/studio/agents.mdx b/docs/studio/agents.mdx index f66c75545b4..74c9d0d9dab 100644 --- a/docs/studio/agents.mdx +++ b/docs/studio/agents.mdx @@ -27,6 +27,6 @@ Agents are created and updated through the `nemo agents` CLI or Agents API. NeMo ## Related Topics -- [About Agents](../agents/index.md) -- [Optimize Agents](../agents/optimization.md) -- [Secure Agents](../agents/security.md) +- [About Agents](/documentation/agents) +- [Optimize Agents](/documentation/agents/optimize-agents) +- [Secure Agents](/documentation/agents/secure-agents) diff --git a/docs/studio/index.mdx b/docs/studio/index.mdx index ff498af3e8c..4a984abd27d 100644 --- a/docs/studio/index.mdx +++ b/docs/studio/index.mdx @@ -15,7 +15,7 @@ NeMo Studio is the web app for AI development with NVIDIA NeMo Platform. It prov ## Getting Started -NeMo Studio is included with the platform. Follow the [Setup guide](../get-started/setup.md) to start NeMo Platform, then access Studio at `/studio` on your running server. +NeMo Studio is included with the platform. Follow the [Setup guide](/documentation/get-started) to start NeMo Platform, then access Studio at `/studio` on your running server. ## Features @@ -23,19 +23,19 @@ NeMo Studio is included with the platform. Follow the [Setup guide](../get-start Use the Studio Agents area to review platform-managed NeMo Agent Toolkit workflows, inspect agent details, deploy agents, open a chat session against a running deployment, and clean up deployments that are no longer needed. -For the full workflow, see [Studio Agents](agents.md). +For the full workflow, see [Studio Agents](/documentation/studio/agents). ### Suggestions Use **Agents > Suggestions** to review optimizer suggestions for deployed agents. Studio groups suggestions by workspace or agent and lets you filter by type, priority, scope, and agent. -For the full workflow, see [Studio Suggestions](suggestions.md). +For the full workflow, see [Studio Suggestions](/documentation/studio/suggestions). ### Monitor Use **Agents > Monitor** to inspect agent telemetry stored by the platform, including recent inference logs and token usage summaries. -For the full workflow, see [Studio Monitor](monitor.md). +For the full workflow, see [Studio Monitor](/documentation/studio/monitor). ### Workspaces @@ -58,7 +58,7 @@ You can upload any file type into a fileset. However, each service supports spec | Service | Supported File Types | | --------------------------------------------------- | ------------------------------------- | -| [Data Designer](../data-designer/index.md) | `.json`, `.jsonl`, `.csv`, `.parquet` | +| [Data Designer](/documentation/design-synthetic-data) | `.json`, `.jsonl`, `.csv`, `.parquet` | ### Jobs @@ -89,4 +89,4 @@ Navigate to **Jobs** in the workspace sidebar to see all jobs in the current wor ### Secrets -Store API keys and credentials to securely connect with external providers. See [manage-secrets](../get-started/concepts/manage-secrets.md) for details. +Store API keys and credentials to securely connect with external providers. See [manage-secrets](/documentation/get-started/core-concepts/manage-secrets) for details. diff --git a/docs/studio/monitor.mdx b/docs/studio/monitor.mdx index d43aa8d5ffc..e6f994f2878 100644 --- a/docs/studio/monitor.mdx +++ b/docs/studio/monitor.mdx @@ -22,5 +22,5 @@ The inference logs table shows recent agent requests from the loaded telemetry f ## Related Topics -- [Optimize Agents](../agents/optimization.md) -- [Secure Agents](../agents/security.md) +- [Optimize Agents](/documentation/agents/optimize-agents) +- [Secure Agents](/documentation/agents/secure-agents) diff --git a/docs/studio/suggestions.mdx b/docs/studio/suggestions.mdx index 80299f39211..8ebbbd02991 100644 --- a/docs/studio/suggestions.mdx +++ b/docs/studio/suggestions.mdx @@ -36,6 +36,6 @@ NeMo Studio can apply supported suggestions from the Suggestions page. Model opt ## Next Steps -- [NeMo Studio Agents](agents.md): review, deploy, chat with, and delete agents from NeMo Studio. -- [NeMo Studio Monitor](monitor.md): inspect recent agent telemetry, token usage, and inference logs. -- [Optimize Agents](../agents/optimization.md): run CLI-driven optimization and review the underlying checks. +- [NeMo Studio Agents](/documentation/studio/agents): review, deploy, chat with, and delete agents from NeMo Studio. +- [NeMo Studio Monitor](/documentation/studio/monitor): inspect recent agent telemetry, token usage, and inference logs. +- [Optimize Agents](/documentation/agents/optimize-agents): run CLI-driven optimization and review the underlying checks. diff --git a/docs/troubleshooting/data-designer.mdx b/docs/troubleshooting/data-designer.mdx index 73cbba9dd64..81b784751b1 100644 --- a/docs/troubleshooting/data-designer.mdx +++ b/docs/troubleshooting/data-designer.mdx @@ -49,8 +49,8 @@ Look for startup errors, provider authentication failures, port conflicts, or Da ## Next steps -- Tutorial: [Data Designer Tutorials](../data-designer/tutorials/index.md) covers common generation workflows. -- How-to: [Set Up NeMo Platform](../get-started/setup.md) covers local setup and service startup. -- Explanation: [Data Designer Overview](../data-designer/index.md) explains configuration, execution, and service behavior. +- Tutorial: [Data Designer Tutorials](/documentation/design-synthetic-data/tutorials) covers common generation workflows. +- How-to: [Set Up NeMo Platform](/documentation/get-started) covers local setup and service startup. +- Explanation: [Data Designer Overview](/documentation/design-synthetic-data) explains configuration, execution, and service behavior. - Reference: [API Reference](/documentation/reference/api-reference) lists Data Designer endpoints. -- Troubleshooting: [Troubleshooting Overview](index.md) links to other service troubleshooting pages. +- Troubleshooting: [Troubleshooting Overview](/documentation/reference/troubleshooting) links to other service troubleshooting pages. diff --git a/docs/troubleshooting/evaluator.mdx b/docs/troubleshooting/evaluator.mdx index 38b20bcac41..eb458eba6b4 100644 --- a/docs/troubleshooting/evaluator.mdx +++ b/docs/troubleshooting/evaluator.mdx @@ -4,11 +4,11 @@ description: "" --- -Use this documentation to troubleshoot issues that can arise when you work with [NVIDIA NeMo Evaluator](../evaluator/index.md). +Use this documentation to troubleshoot issues that can arise when you work with [NVIDIA NeMo Evaluator](/documentation/evaluate-models). -You can [get metric logs](../evaluator/metrics/job-management.md) or [get benchmark logs](../evaluator/benchmarks/job-management.md) for `COMPLETED` or `FAILED` jobs and use them to help troubleshoot. +You can [get metric logs](/documentation/evaluate-models/metrics/job-management) or [get benchmark logs](../evaluator/benchmarks/job-management.md) for `COMPLETED` or `FAILED` jobs and use them to help troubleshoot. --- diff --git a/docs/troubleshooting/index.mdx b/docs/troubleshooting/index.mdx index d603389b441..ac76d18af7c 100644 --- a/docs/troubleshooting/index.mdx +++ b/docs/troubleshooting/index.mdx @@ -5,22 +5,22 @@ description: "" This section provides troubleshooting information for the NeMo Platform. - + Learn how to troubleshoot common issues with NeMo Evaluator. - + Learn how to troubleshoot common issues with NeMo Guardrails. - + Learn how to troubleshoot common issues with NeMo Data Designer. - + Learn how to troubleshoot common issues with Studio. diff --git a/docs/troubleshooting/studio.mdx b/docs/troubleshooting/studio.mdx index 43d232edf39..02dc4f66d53 100644 --- a/docs/troubleshooting/studio.mdx +++ b/docs/troubleshooting/studio.mdx @@ -40,4 +40,4 @@ curl -X POST http://localhost:8080/v1/datasets \ ``` All resource creation and update APIs support the `project` parameter. -Refer to the [Python SDK](../pysdk/index.md) and [API references](/documentation/reference/api-reference) for more details. +Refer to the [Python SDK](/documentation/reference/python-sdk) and [API references](/documentation/reference/api-reference) for more details. From 3d27f0428051e4da113d028cccfd0e5976c7c3d4 Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Thu, 4 Jun 2026 13:40:40 -0400 Subject: [PATCH 16/26] docs(fern): configure hidden pages and resolve remaining broken links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the 61 pages that the original MkDocs build hid (auth/**, set-up/** + helm/**, customizer extras, safe-synthesizer/**, evaluator benchmark results) to the Fern nav as `hidden: true` sections — reachable by URL, absent from the sidebar (per Fern's hiding-content navigation feature). This both restores the original "hidden" intent and gives the ~98 links that targeted these pages a real destination. - versions/latest.yml: 5 hidden sections (Access Control, Self-Managed Deployment, Customizer Reference, Safe Synthesizer Reference, Evaluator Benchmark Reference), nested by folder, titles from the original nav. - Rewrite internal links to those now-published pages to canonical URLs. - Rewrite snippet (docs/fern/snippets) links to absolute URLs — snippets inline at varying depths, so relative links can't resolve. - Repoint 3 genuinely-dangling links whose targets no longer exist: get-started/quickstart -> get-started (Setup); evaluator retriever metrics -> evaluate-models metrics overview. `fern docs broken-links` now reports 0 (was 810); validate-mdx parses all 193 files cleanly; hidden pages verified reachable-by-URL and absent from the sidebar in a local fern dev build. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Lawrence Lane --- docs/auth/authentication/index.mdx | 14 +- docs/auth/authentication/oidc.mdx | 18 +- .../authentication/providers/azure-ad.mdx | 8 +- .../auth/authentication/providers/generic.mdx | 10 +- docs/auth/authentication/providers/index.mdx | 2 +- .../authentication/using-authentication.mdx | 12 +- docs/auth/authorization/api-scopes.mdx | 10 +- docs/auth/authorization/index.mdx | 10 +- docs/auth/authorization/managing-access.mdx | 4 +- .../authorization/permissions-reference.mdx | 12 +- docs/auth/authorization/policy-engine.mdx | 10 +- .../authorization/roles-and-permissions.mdx | 10 +- docs/auth/concepts.mdx | 20 +-- docs/auth/deployment/configuration.mdx | 18 +- .../deployment/credential-propagation.mdx | 6 +- docs/auth/deployment/gateway.mdx | 8 +- docs/auth/deployment/hardening.mdx | 22 +-- docs/auth/index.mdx | 20 +-- docs/auth/security-model.mdx | 20 +-- docs/auth/troubleshooting.mdx | 8 +- docs/customizer/about.mdx | 6 +- docs/customizer/index.mdx | 16 +- .../manage-customization-jobs/cancel-job.mdx | 2 +- .../manage-customization-jobs/create-job.mdx | 10 +- .../customization-job-reference.mdx | 6 +- .../get-job-status.mdx | 2 +- .../hyperparameters.mdx | 2 +- .../manage-customization-jobs/index.mdx | 10 +- .../list-active-jobs.mdx | 2 +- .../manage-model-entities/create-fileset.mdx | 2 +- .../create-model-entity.mdx | 2 +- .../manage-model-entities/index.mdx | 2 +- docs/customizer/models/embedding.mdx | 4 +- docs/customizer/models/gpt-oss.mdx | 4 +- docs/customizer/models/index.mdx | 16 +- docs/customizer/models/llama-nemotron.mdx | 2 +- docs/customizer/models/llama.mdx | 2 +- docs/customizer/models/mistral.mdx | 2 +- docs/customizer/models/phi.mdx | 2 +- docs/customizer/models/qwen.mdx | 2 +- .../tutorials/format-training-dataset.mdx | 2 +- docs/customizer/tutorials/import-hf-model.mdx | 4 +- docs/customizer/tutorials/index.mdx | 4 +- .../understand-configurations-and-models.mdx | 4 +- docs/evaluator/benchmarks/agentic.mdx | 4 +- docs/evaluator/benchmarks/custom.mdx | 6 +- docs/evaluator/benchmarks/index.mdx | 6 +- docs/evaluator/benchmarks/industry.mdx | 14 +- docs/evaluator/benchmarks/job-management.mdx | 2 +- .../benchmarks/manage-benchmarks.mdx | 4 +- .../snippets/_snippets/tutorials/prereqs.mdx | 4 +- .../_snippets/customizer-prereqs.mdx | 2 +- .../discover-industry-benchmarks.mdx | 2 +- docs/fern/versions/latest.yml | 165 ++++++++++++++++++ docs/guardrails/observability.mdx | 4 +- docs/helm/index.mdx | 2 +- .../safe-synthesizer/about/data-synthesis.mdx | 10 +- docs/safe-synthesizer/about/evaluation.mdx | 6 +- .../about/host-local-development.mdx | 8 +- docs/safe-synthesizer/about/index.mdx | 32 ++-- docs/safe-synthesizer/about/jobs.mdx | 8 +- .../about/pii-replacement.mdx | 4 +- docs/safe-synthesizer/about/reference.mdx | 10 +- docs/safe-synthesizer/getting-started.mdx | 12 +- .../tutorials/differential-privacy.mdx | 2 +- docs/safe-synthesizer/tutorials/index.mdx | 8 +- .../tutorials/safe-synthesizer-101.mdx | 10 +- docs/set-up/helm/file-storage.mdx | 4 +- docs/set-up/helm/index.mdx | 18 +- docs/set-up/helm/ingress.mdx | 12 +- docs/set-up/helm/install.mdx | 18 +- docs/set-up/helm/openshift.mdx | 8 +- docs/set-up/helm/persistent-volumes.mdx | 2 +- docs/set-up/helm/prerequisites.mdx | 2 +- docs/set-up/index.mdx | 6 +- docs/set-up/manage-jobs.mdx | 2 +- docs/set-up/milvus.mdx | 2 +- docs/set-up/opentelemetry.mdx | 2 +- docs/troubleshooting/customizer.mdx | 2 +- docs/troubleshooting/evaluator.mdx | 2 +- 80 files changed, 461 insertions(+), 296 deletions(-) diff --git a/docs/auth/authentication/index.mdx b/docs/auth/authentication/index.mdx index 27745d06431..e6c8354f936 100644 --- a/docs/auth/authentication/index.mdx +++ b/docs/auth/authentication/index.mdx @@ -4,19 +4,19 @@ description: "" --- NeMo Platform authenticates requests using **OpenID Connect (OIDC)**. You register an OAuth application in your identity provider, configure NeMo Platform with the issuer and client ID, and users sign in via the CLI, SDK, or browser. NeMo Platform validates the JWT on every request and extracts the user's identity for authorization. -For the quickstart (no IdP), see the [email-based shortcut](../index.md). For the authorization model, see [Authorization Concepts](../concepts.md). +For the quickstart (no IdP), see the [email-based shortcut](/documentation/access-control/overview). For the authorization model, see [Authorization Concepts](/documentation/access-control/concepts). ## Connect Your Identity Provider Start here — register an OAuth application in your IdP and configure NeMo Platform: -- [OIDC Setup](oidc.md) — Step-by-step: register an app, configure NeMo Platform, verify login. -- [Azure AD (Entra ID)](providers/azure-ad.md) — Azure-specific walkthrough (app registration, scopes, claim mapping). -- [Generic OIDC Provider](providers/generic.md) — Checklist for any OIDC-compliant IdP. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — Step-by-step: register an app, configure NeMo Platform, verify login. +- [Azure AD (Entra ID)](/documentation/access-control/authentication/providers/azure-ad-entra-id) — Azure-specific walkthrough (app registration, scopes, claim mapping). +- [Generic OIDC Provider](/documentation/access-control/authentication/providers/generic-oidc) — Checklist for any OIDC-compliant IdP. ## Log In and Make API Calls -Once your IdP is connected, see [Using Authentication](using-authentication.md) for the full walkthrough: device flow login, SDK and curl examples, token management, and config file reference. +Once your IdP is connected, see [Using Authentication](/documentation/access-control/authentication/using-authentication) for the full walkthrough: device flow login, SDK and curl examples, token management, and config file reference. | Method | Command / Action | Best For | |--------|-----------------|----------| @@ -54,5 +54,5 @@ The CLI and SDK call this endpoint automatically during `nemo auth login` or whe ## Related -- [Using Authentication](using-authentication.md) — Log in, make API calls, and manage tokens. -- [Security Model](../security-model.md) — Trust boundaries and the principal model. +- [Using Authentication](/documentation/access-control/authentication/using-authentication) — Log in, make API calls, and manage tokens. +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and the principal model. diff --git a/docs/auth/authentication/oidc.mdx b/docs/auth/authentication/oidc.mdx index c49197a8c80..5fe7d076102 100644 --- a/docs/auth/authentication/oidc.mdx +++ b/docs/auth/authentication/oidc.mdx @@ -6,7 +6,7 @@ Connect NeMo Platform to an external OIDC identity provider (IdP) so users sign **Prerequisites**: You need an OAuth application registered in your IdP with the client ID, issuer URL, and device flow grant enabled. See the [Minimum IdP Checklist](#minimum-idp-checklist) below. -For login after setup, see [Using Authentication](using-authentication.md). For the full config reference, see [Auth Configuration](../deployment/configuration.md). +For login after setup, see [Using Authentication](/documentation/access-control/authentication/using-authentication). For the full config reference, see [Auth Configuration](/documentation/access-control/deployment/configuration). ## How OIDC Fits in NeMo Platform @@ -38,7 +38,7 @@ In your IdP (Azure AD, Okta, Keycloak, etc.): 1. Create a new application registration 2. Note the **client ID** 3. Enable **device flow** (device authorization grant) -4. *(Optional)* Create custom API scopes (`platform:read`, `platform:write`) if you want token-level scope restrictions. See [API Scopes](../authorization/api-scopes.md). +4. *(Optional)* Create custom API scopes (`platform:read`, `platform:write`) if you want token-level scope restrictions. See [API Scopes](/documentation/access-control/authorization/api-scopes). 5. If you created custom scopes, grant admin consent for them (if required by your IdP) ### Step 2: Configure NeMo Platform @@ -63,7 +63,7 @@ auth: ### Step 3: Configure Scopes (Optional) -This step is only needed if you created custom API scopes in Step 1. If you skip scopes, NeMo Platform still enforces RBAC — scopes add an additional token-level restriction on top. See [API Scopes](../authorization/api-scopes.md). +This step is only needed if you created custom API scopes in Step 1. If you skip scopes, NeMo Platform still enforces RBAC — scopes add an additional token-level restriction on top. See [API Scopes](/documentation/access-control/authorization/api-scopes). If you are using scopes, configure the default scopes requested during login: @@ -126,7 +126,7 @@ Regardless of IdP, ensure: ## Claim mapping -NeMo Platform maps JWT claims to [trusted identity headers](../security-model.md#trusted-identity-headers) using `email_claim`, `subject_claim`, and `groups_claim` (defaults: `email`, `sub`, `groups`). Override them when your IdP uses different claim names — values must match what you use for workspace members and authorization. +NeMo Platform maps JWT claims to [trusted identity headers](/documentation/access-control/security-model#trusted-identity-headers) using `email_claim`, `subject_claim`, and `groups_claim` (defaults: `email`, `sub`, `groups`). Override them when your IdP uses different claim names — values must match what you use for workspace members and authorization. | Purpose | Config key | Default | Header | |---------|------------|---------|--------| @@ -137,17 +137,17 @@ NeMo Platform maps JWT claims to [trusted identity headers](../security-model.md -Group values are extracted from the JWT and included in the `X-NMP-Principal-Groups` header and OPA policy data, but **automatic group-to-role mapping is not yet implemented**. Role assignments must be made explicitly via the members API. See [Managing Access](../authorization/managing-access.md). +Group values are extracted from the JWT and included in the `X-NMP-Principal-Groups` header and OPA policy data, but **automatic group-to-role mapping is not yet implemented**. Role assignments must be made explicitly via the members API. See [Managing Access](/documentation/access-control/authorization/managing-access). ## Provider-Specific Notes -- **Azure AD**: See [Azure AD Setup](providers/azure-ad.md) (issuer, `additional_issuers`, device flow, claim overrides). +- **Azure AD**: See [Azure AD Setup](/documentation/access-control/authentication/providers/azure-ad-entra-id) (issuer, `additional_issuers`, device flow, claim overrides). - **Okta**: Choose the correct authorization server (custom vs org). Ensure device flow is enabled. - **Keycloak**: Configure realm, client, and role mappers. Enable device flow in client settings. -- **Other providers**: See [Generic OIDC Provider](providers/generic.md) for a provider-agnostic checklist. +- **Other providers**: See [Generic OIDC Provider](/documentation/access-control/authentication/providers/generic-oidc) for a provider-agnostic checklist. ## Related -- [Using Authentication](using-authentication.md) — Log in, make API calls, and manage tokens. -- [Auth Configuration](../deployment/configuration.md) — Full auth config reference. +- [Using Authentication](/documentation/access-control/authentication/using-authentication) — Log in, make API calls, and manage tokens. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Full auth config reference. diff --git a/docs/auth/authentication/providers/azure-ad.mdx b/docs/auth/authentication/providers/azure-ad.mdx index 88922268c2e..97066439625 100644 --- a/docs/auth/authentication/providers/azure-ad.mdx +++ b/docs/auth/authentication/providers/azure-ad.mdx @@ -4,7 +4,7 @@ description: "" --- Complete walkthrough for connecting NeMo Platform to Azure AD (Entra ID), from app registration to first successful login. -**Prerequisites**: Access to Azure Portal with permission to create app registrations. Familiarity with [OIDC Setup](../oidc.md). +**Prerequisites**: Access to Azure Portal with permission to create app registrations. Familiarity with [OIDC Setup](/documentation/access-control/authentication/oidc-setup). ## App Registration @@ -80,6 +80,6 @@ nemo auth status ## Related -- [OIDC Setup](../oidc.md) — Generic OIDC configuration. -- [OIDC Setup — Claim mapping](../oidc.md#claim-mapping) — JWT claims vs config defaults. -- [Auth Configuration](../../deployment/configuration.md) — Full config reference. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — Generic OIDC configuration. +- [OIDC Setup — Claim mapping](/documentation/access-control/authentication/oidc-setup#claim-mapping) — JWT claims vs config defaults. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Full config reference. diff --git a/docs/auth/authentication/providers/generic.mdx b/docs/auth/authentication/providers/generic.mdx index 0e8160b7b0f..d36094b56fa 100644 --- a/docs/auth/authentication/providers/generic.mdx +++ b/docs/auth/authentication/providers/generic.mdx @@ -2,9 +2,9 @@ title: "Generic OIDC Provider" description: "" --- -A checklist for connecting NeMo Platform to any OIDC-compliant identity provider not covered by the [Azure AD](azure-ad.md) page. +A checklist for connecting NeMo Platform to any OIDC-compliant identity provider not covered by the [Azure AD](/documentation/access-control/authentication/providers/azure-ad-entra-id) page. -**Prerequisites**: Familiarity with [OIDC Setup](../oidc.md). +**Prerequisites**: Familiarity with [OIDC Setup](/documentation/access-control/authentication/oidc-setup). ## Provider Checklist @@ -48,6 +48,6 @@ auth: ## Related -- [OIDC Setup](../oidc.md) — Full OIDC configuration guide. -- [OIDC Setup — Claim mapping](../oidc.md#claim-mapping) — JWT claims vs config defaults. -- [Auth Configuration](../../deployment/configuration.md) — Full config reference. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — Full OIDC configuration guide. +- [OIDC Setup — Claim mapping](/documentation/access-control/authentication/oidc-setup#claim-mapping) — JWT claims vs config defaults. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Full config reference. diff --git a/docs/auth/authentication/providers/index.mdx b/docs/auth/authentication/providers/index.mdx index 459da4cfb40..17a2f69f230 100644 --- a/docs/auth/authentication/providers/index.mdx +++ b/docs/auth/authentication/providers/index.mdx @@ -2,4 +2,4 @@ title: "Provider Guides" description: "" --- -Step-by-step setup instructions for specific identity providers. If your provider is not listed here, see the [Generic OIDC Provider](generic.md) checklist. +Step-by-step setup instructions for specific identity providers. If your provider is not listed here, see the [Generic OIDC Provider](/documentation/access-control/authentication/providers/generic-oidc) checklist. diff --git a/docs/auth/authentication/using-authentication.mdx b/docs/auth/authentication/using-authentication.mdx index 3bc4cd8ff9a..98db30298a2 100644 --- a/docs/auth/authentication/using-authentication.mdx +++ b/docs/auth/authentication/using-authentication.mdx @@ -4,7 +4,7 @@ description: "" --- How to log in, make authenticated API calls, and manage tokens with the CLI and SDK. -**Prerequisites**: OIDC must be configured on the platform. See [OIDC Setup](oidc.md). +**Prerequisites**: OIDC must be configured on the platform. See [OIDC Setup](/documentation/access-control/authentication/oidc-setup). ## Log In @@ -44,7 +44,7 @@ By default, the CLI requests the scopes configured in `auth.oidc.default_scopes` nemo auth login --scope "platform:read" ``` -See [API Scopes](../authorization/api-scopes.md) for the full list of available scopes. +See [API Scopes](/documentation/access-control/authorization/api-scopes) for the full list of available scopes. ### Non-Interactive Login (CI/CD) @@ -165,7 +165,7 @@ The OIDC token endpoint is **not** stored — it is discovered at runtime from y ## Related -- [OIDC Setup](oidc.md) — Configure your identity provider. -- [API Scopes](../authorization/api-scopes.md) — Scope model and available scopes. -- [Security Model](../security-model.md) — Trust boundaries and the principal model. -- [Troubleshooting](../troubleshooting.md) — Fix common 401/403 errors and login failures. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — Configure your identity provider. +- [API Scopes](/documentation/access-control/authorization/api-scopes) — Scope model and available scopes. +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and the principal model. +- [Troubleshooting](/documentation/access-control/troubleshooting) — Fix common 401/403 errors and login failures. diff --git a/docs/auth/authorization/api-scopes.mdx b/docs/auth/authorization/api-scopes.mdx index 1a6445ce2ab..9c00f521f83 100644 --- a/docs/auth/authorization/api-scopes.mdx +++ b/docs/auth/authorization/api-scopes.mdx @@ -4,7 +4,7 @@ description: "" --- API scopes are token-level access restrictions that sit on top of role-based permissions. They control which parts of the API a token can access, independent of the user's role. -For role-based permissions, see [Roles & Permissions](roles-and-permissions.md). For the RBAC model, see [Authorization Concepts](../concepts.md). +For role-based permissions, see [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions). For the RBAC model, see [Authorization Concepts](/documentation/access-control/concepts). ## How Scopes Work @@ -120,7 +120,7 @@ The PDP distinguishes between OIDC standard scopes and platform scopes: ## Related -- [Roles & Permissions](roles-and-permissions.md) — Role-based permission model. -- [Using Authentication](../authentication/using-authentication.md) — Log in, requesting scopes, and token management. -- [OIDC Setup](../authentication/oidc.md) — Configuring scope prefix and default scopes. -- [Security Model](../security-model.md) — Trust boundaries and two-layer authorization. +- [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions) — Role-based permission model. +- [Using Authentication](/documentation/access-control/authentication/using-authentication) — Log in, requesting scopes, and token management. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — Configuring scope prefix and default scopes. +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and two-layer authorization. diff --git a/docs/auth/authorization/index.mdx b/docs/auth/authorization/index.mdx index c72b903b0de..2ca4ead1016 100644 --- a/docs/auth/authorization/index.mdx +++ b/docs/auth/authorization/index.mdx @@ -17,23 +17,23 @@ Request → PDP → Scope check → Role binding check → Allow / Deny For a request to succeed, both the scope check (does the token allow it?) and the role check (does the user have permission?) must pass. -For the full conceptual background, see [Authorization Concepts](../concepts.md). For the security architecture, see [Security Model](../security-model.md). +For the full conceptual background, see [Authorization Concepts](/documentation/access-control/concepts). For the security architecture, see [Security Model](/documentation/access-control/security-model). ## Key Pages - + Complete permission matrix — what each role can do. - + Add users to workspaces, assign roles, manage members. - + Token-level scope model and two-layer authorization. @@ -43,7 +43,7 @@ Token-level scope model and two-layer authorization. Complete list of all permissions with role assignments. - + OPA / WASM policy engine internals and configuration. diff --git a/docs/auth/authorization/managing-access.mdx b/docs/auth/authorization/managing-access.mdx index 19661bf045e..1b97ffbf8ec 100644 --- a/docs/auth/authorization/managing-access.mdx +++ b/docs/auth/authorization/managing-access.mdx @@ -2,7 +2,7 @@ title: "Managing Access" description: "" --- -Add users to workspaces, assign roles, and control who can access your resources. For background on the authorization model, refer to [Authorization Concepts](../concepts.md). For what each role can do, refer to [Roles & Permissions](roles-and-permissions.md). +Add users to workspaces, assign roles, and control who can access your resources. For background on the authorization model, refer to [Authorization Concepts](/documentation/access-control/concepts). For what each role can do, refer to [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions). @@ -385,7 +385,7 @@ client.workspaces.members.delete(workspace="ml-team", principal_id="alice@exampl The **PlatformAdmin** role (set using `admin_email` in config) has full access to all workspaces and bypasses authorization checks. PlatformAdmin is typically used for initial platform setup, creating the first workspaces and granting Admin roles to team leads. After bootstrap, day-to-day access management should use workspace-level members (above). -For details on configuring the platform admin, refer to [Auth Configuration](../deployment/configuration.md). For the full security implications, refer to [Security Model](../security-model.md). +For details on configuring the platform admin, refer to [Auth Configuration](/documentation/access-control/deployment/configuration). For the full security implications, refer to [Security Model](/documentation/access-control/security-model). ## Deleting Workspaces diff --git a/docs/auth/authorization/permissions-reference.mdx b/docs/auth/authorization/permissions-reference.mdx index eb13618d77e..5b82494bca2 100644 --- a/docs/auth/authorization/permissions-reference.mdx +++ b/docs/auth/authorization/permissions-reference.mdx @@ -5,9 +5,9 @@ description: "" (permissions-reference)= -Complete reference of all permissions across the NeMo Platform APIs. Each permission controls access to a specific operation within an individual API. Permissions are assigned to users through [roles](roles-and-permissions.md). +Complete reference of all permissions across the NeMo Platform APIs. Each permission controls access to a specific operation within an individual API. Permissions are assigned to users through [roles](/documentation/access-control/authorization/roles-and-permissions). -For token-level access restrictions, see [API Scopes](api-scopes.md). For the RBAC model, see [Authorization Concepts](../concepts.md). +For token-level access restrictions, see [API Scopes](/documentation/access-control/authorization/api-scopes). For the RBAC model, see [Authorization Concepts](/documentation/access-control/concepts). @@ -142,7 +142,7 @@ PlatformAdmin is omitted — it bypasses permission checks entirely at the polic ## Related -- [Roles & Permissions](roles-and-permissions.md) — Role descriptions and hierarchy. -- [API Scopes](api-scopes.md) — Token-level scope restrictions. -- [Authorization Concepts](../concepts.md) — Workspaces, roles, bindings, and the RBAC model. -- [Security Model](../security-model.md) — Trust boundaries and authorization layers. +- [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions) — Role descriptions and hierarchy. +- [API Scopes](/documentation/access-control/authorization/api-scopes) — Token-level scope restrictions. +- [Authorization Concepts](/documentation/access-control/concepts) — Workspaces, roles, bindings, and the RBAC model. +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and authorization layers. diff --git a/docs/auth/authorization/policy-engine.mdx b/docs/auth/authorization/policy-engine.mdx index 5c0cf1a3685..3c8887a2881 100644 --- a/docs/auth/authorization/policy-engine.mdx +++ b/docs/auth/authorization/policy-engine.mdx @@ -4,7 +4,7 @@ description: "" --- The Policy Decision Point (PDP) evaluates every authorization request in NeMo Platform. It checks role bindings and scopes against the operation's requirements and returns allow or deny. This page covers the PDP internals, configuration, and operational details. -For the conceptual overview, see [Authorization Concepts](../concepts.md). For configuration, see [Auth Configuration](../deployment/configuration.md). +For the conceptual overview, see [Authorization Concepts](/documentation/access-control/concepts). For configuration, see [Auth Configuration](/documentation/access-control/deployment/configuration). ## How the PDP Works @@ -154,7 +154,7 @@ Loaded from the entity store: ## Related -- [Auth Configuration](../deployment/configuration.md) — PDP provider settings and environment variables. -- [Authorization Concepts](../concepts.md) — RBAC model and role propagation. -- [API Scopes](api-scopes.md) — Scope checking in the PDP. -- [Security Model](../security-model.md) — Architecture and trust boundaries. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — PDP provider settings and environment variables. +- [Authorization Concepts](/documentation/access-control/concepts) — RBAC model and role propagation. +- [API Scopes](/documentation/access-control/authorization/api-scopes) — Scope checking in the PDP. +- [Security Model](/documentation/access-control/security-model) — Architecture and trust boundaries. diff --git a/docs/auth/authorization/roles-and-permissions.mdx b/docs/auth/authorization/roles-and-permissions.mdx index 2e76a098fb2..5caab130b79 100644 --- a/docs/auth/authorization/roles-and-permissions.mdx +++ b/docs/auth/authorization/roles-and-permissions.mdx @@ -2,7 +2,7 @@ title: "Roles and Permissions" description: "" --- -The authoritative reference for NeMo Platform roles and their permissions. For background on how RBAC works, see [Authorization Concepts](../concepts.md). For managing workspace members, see [Managing Access](managing-access.md). +The authoritative reference for NeMo Platform roles and their permissions. For background on how RBAC works, see [Authorization Concepts](/documentation/access-control/concepts). For managing workspace members, see [Managing Access](/documentation/access-control/authorization/managing-access). ## Role Descriptions @@ -137,7 +137,7 @@ To leave a workspace where you are the only Admin, add another Admin first. ## Related -- [Authorization Concepts](../concepts.md) — Workspaces, roles, bindings, and the RBAC model. -- [Managing Access](managing-access.md) — Add users, assign roles, manage workspace members. -- [API Scopes](api-scopes.md) — Token-level scope restrictions. -- [Security Model](../security-model.md) — Trust boundaries and authorization layers. +- [Authorization Concepts](/documentation/access-control/concepts) — Workspaces, roles, bindings, and the RBAC model. +- [Managing Access](/documentation/access-control/authorization/managing-access) — Add users, assign roles, manage workspace members. +- [API Scopes](/documentation/access-control/authorization/api-scopes) — Token-level scope restrictions. +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and authorization layers. diff --git a/docs/auth/concepts.mdx b/docs/auth/concepts.mdx index 67ca92618e2..0930bc4b47a 100644 --- a/docs/auth/concepts.mdx +++ b/docs/auth/concepts.mdx @@ -4,7 +4,7 @@ description: "" --- NeMo Platform uses **workspace-scoped RBAC**: resources live in workspaces, users get roles (Viewer, Editor, Admin) per workspace, and a policy engine evaluates every request against role bindings and token scopes. -For the security architecture and trust boundaries, see [Security Model](security-model.md). For hands-on setup, see [Managing Access](authorization/managing-access.md). +For the security architecture and trust boundaries, see [Security Model](/documentation/access-control/security-model). For hands-on setup, see [Managing Access](/documentation/access-control/authorization/managing-access). ## Authorization Model Overview @@ -13,7 +13,7 @@ The model has four building blocks: 1. **Workspaces** — containers that own resources (models, datasets, jobs) 2. **Roles** — permission bundles (Viewer, Editor, Admin) granted to users per workspace 3. **Role bindings** — the link between a user, a role, and a workspace -4. **Scopes** — token-level restrictions on top of role permissions (see [API Scopes](authorization/api-scopes.md)) +4. **Scopes** — token-level restrictions on top of role permissions (see [API Scopes](/documentation/access-control/authorization/api-scopes)) ## Workspaces @@ -70,7 +70,7 @@ PlatformAdmin (all operations across all workspaces) Viewer (list, read, run inference) ``` -Custom roles can be defined at deployment time with arbitrary permission sets — they do not need to follow this hierarchy. For the complete permission matrix, see [Roles & Permissions](authorization/roles-and-permissions.md). +Custom roles can be defined at deployment time with arbitrary permission sets — they do not need to follow this hierarchy. For the complete permission matrix, see [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions). ### Predefined Roles @@ -106,7 +106,7 @@ Platform admins can: - Create and delete any workspace - Bypass all authorization checks -The PlatformAdmin role is granted via the `admin_email` config setting. See [Auth Configuration](deployment/configuration.md). +The PlatformAdmin role is granted via the `admin_email` config setting. See [Auth Configuration](/documentation/access-control/deployment/configuration). ## Role Bindings @@ -164,7 +164,7 @@ Every authorized request is evaluated by the PDP, which checks role bindings and - **Embedded** (default): A WASM-based policy engine built into the auth service. No external dependencies. - **External OPA**: An Open Policy Agent instance (sidecar or standalone service) fetches policy bundles from the auth service. -For technical details on the policy engine, see [Policy Engine](authorization/policy-engine.md). For configuration, see [Auth Configuration](deployment/configuration.md). +For technical details on the policy engine, see [Policy Engine](/documentation/access-control/authorization/policy-engine). For configuration, see [Auth Configuration](/documentation/access-control/deployment/configuration). ### Role Propagation Delay @@ -172,8 +172,8 @@ When you add or remove a member, the change takes effect after the next policy d ## Related -- [Security Model](security-model.md) — Architecture, trust boundaries, and authorization layers. -- [Roles & Permissions](authorization/roles-and-permissions.md) — Complete permission matrix. -- [API Scopes](authorization/api-scopes.md) — Token-level scope restrictions. -- [Managing Access](authorization/managing-access.md) — Add users to workspaces and assign roles. -- [Policy Engine](authorization/policy-engine.md) — OPA/WASM policy engine internals. +- [Security Model](/documentation/access-control/security-model) — Architecture, trust boundaries, and authorization layers. +- [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions) — Complete permission matrix. +- [API Scopes](/documentation/access-control/authorization/api-scopes) — Token-level scope restrictions. +- [Managing Access](/documentation/access-control/authorization/managing-access) — Add users to workspaces and assign roles. +- [Policy Engine](/documentation/access-control/authorization/policy-engine) — OPA/WASM policy engine internals. diff --git a/docs/auth/deployment/configuration.mdx b/docs/auth/deployment/configuration.mdx index 50d8a2dbb66..a3c9e881ed2 100644 --- a/docs/auth/deployment/configuration.mdx +++ b/docs/auth/deployment/configuration.mdx @@ -4,7 +4,7 @@ description: "" --- Complete reference for enabling and configuring platform authorization: the `auth` section in config, Helm values, environment variables, and the choice between embedded and external OPA. -For quickstart setup, see [Authentication and Authorization](../index.md). For OIDC settings, see [OIDC Setup](../authentication/oidc.md). +For quickstart setup, see [Authentication and Authorization](/documentation/access-control/overview). For OIDC settings, see [OIDC Setup](/documentation/access-control/authentication/oidc-setup). ## Enabling Authorization @@ -29,7 +29,7 @@ When `auth.enabled` is `false` (the default), all API requests are allowed witho ### Bootstrap Admin -When authorization is enabled, a platform administrator can be configured. Setting **`admin_email`** gives that identity the **PlatformAdmin** role at platform start. Use it to create the first workspaces and grant roles to other users. After bootstrap, manage access via workspaces and members as described in [Managing Access](../authorization/managing-access.md). +When authorization is enabled, a platform administrator can be configured. Setting **`admin_email`** gives that identity the **PlatformAdmin** role at platform start. Use it to create the first workspaces and grant roles to other users. After bootstrap, manage access via workspaces and members as described in [Managing Access](/documentation/access-control/authorization/managing-access). ```yaml auth: @@ -39,11 +39,11 @@ auth: For a complete reference of all `auth` fields and their defaults, see the [platform configuration reference](/documentation/reference/configuration-reference). Auth-related values are found under `platformConfig.auth` in the values file. -For OIDC-specific fields (`auth.oidc`), see [OIDC Setup](../authentication/oidc.md). +For OIDC-specific fields (`auth.oidc`), see [OIDC Setup](/documentation/access-control/authentication/oidc-setup). ## Authorization Engine: Embedded vs External OPA -The PDP can run in two modes. For technical details, see [Policy Engine](../authorization/policy-engine.md). +The PDP can run in two modes. For technical details, see [Policy Engine](/documentation/access-control/authorization/policy-engine). ### Embedded (default) @@ -123,8 +123,8 @@ auth: ## Related -- [Authentication and Authorization](../index.md) — Overview, auth methods, and getting started. -- [OIDC Setup](../authentication/oidc.md) — IdP configuration and CLI login. -- [Gateway Integration](gateway.md) — Using a gateway for authorization. -- [Managing Access](../authorization/managing-access.md) — Workspaces and member management. -- [Policy Engine](../authorization/policy-engine.md) — PDP internals and configuration. +- [Authentication and Authorization](/documentation/access-control/overview) — Overview, auth methods, and getting started. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — IdP configuration and CLI login. +- [Gateway Integration](/documentation/access-control/deployment/gateway-integration) — Using a gateway for authorization. +- [Managing Access](/documentation/access-control/authorization/managing-access) — Workspaces and member management. +- [Policy Engine](/documentation/access-control/authorization/policy-engine) — PDP internals and configuration. diff --git a/docs/auth/deployment/credential-propagation.mdx b/docs/auth/deployment/credential-propagation.mdx index 5c3f5232817..1b76d1c847d 100644 --- a/docs/auth/deployment/credential-propagation.mdx +++ b/docs/auth/deployment/credential-propagation.mdx @@ -15,7 +15,7 @@ The flow: 3. Secrets needed by the job are fetched on behalf of the user (the platform checks that the user has access before injecting them). 4. The job container uses the propagated principal to authenticate API calls back to NeMo Platform. -Job containers need to run inside the trust boundary so that their `X-NMP-Principal-*` headers are accepted by downstream services. Network policies and gateway configuration enforce this boundary. For the full architecture, see [Security Model](../security-model.md#job-credential-propagation). +Job containers need to run inside the trust boundary so that their `X-NMP-Principal-*` headers are accepted by downstream services. Network policies and gateway configuration enforce this boundary. For the full architecture, see [Security Model](/documentation/access-control/security-model#job-credential-propagation). ## Inference Auth Context @@ -32,5 +32,5 @@ Ensure Kubernetes RBAC prevents unauthorized access to job pods. Network policie ## Related -- [Security Model](../security-model.md) — Trust boundaries and job credential propagation. -- [Auth Configuration](configuration.md) — Platform auth configuration. +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and job credential propagation. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Platform auth configuration. diff --git a/docs/auth/deployment/gateway.mdx b/docs/auth/deployment/gateway.mdx index aba781795e1..ab5d85b0db6 100644 --- a/docs/auth/deployment/gateway.mdx +++ b/docs/auth/deployment/gateway.mdx @@ -4,7 +4,7 @@ description: "" --- In production, a gateway (reverse proxy, ingress controller, or service mesh) often sits in front of the NeMo Platform. This page explains how authorization works with and without gateway-level auth, what headers the gateway must set, and which paths skip authorization. -For the security architecture, see [Security Model](../security-model.md). +For the security architecture, see [Security Model](/documentation/access-control/security-model). ## Overview @@ -142,6 +142,6 @@ After configuring gateway-level auth, verify: ## Related -- [Auth Configuration](configuration.md) — Enabling auth and PDP provider (embedded vs OPA). -- [Security Model](../security-model.md) — Trust boundaries and gateway trust model. -- [Production Hardening](hardening.md) — Security checklist including gateway requirements. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Enabling auth and PDP provider (embedded vs OPA). +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and gateway trust model. +- [Production Hardening](/documentation/access-control/deployment/production-hardening) — Security checklist including gateway requirements. diff --git a/docs/auth/deployment/hardening.mdx b/docs/auth/deployment/hardening.mdx index 6dae532ebc2..60a3a498741 100644 --- a/docs/auth/deployment/hardening.mdx +++ b/docs/auth/deployment/hardening.mdx @@ -4,12 +4,12 @@ description: "" --- Security checklist for deploying NeMo Platform to production. Work through each section and verify your deployment meets these requirements. -For the security architecture, see [Security Model](../security-model.md). For configuration details, see [Auth Configuration](configuration.md). +For the security architecture, see [Security Model](/documentation/access-control/security-model). For configuration details, see [Auth Configuration](/documentation/access-control/deployment/configuration). ## Authentication - [ ] **Enable auth**: Set `auth.enabled: true` in platform config. -- [ ] **Configure OIDC**: Connect a production identity provider. See [OIDC Setup](../authentication/oidc.md). +- [ ] **Configure OIDC**: Connect a production identity provider. See [OIDC Setup](/documentation/access-control/authentication/oidc-setup). - [ ] **Disable password grant in production**: Password grant bypasses MFA. If your IdP supports it, disable the resource owner password grant for the NeMo Platform application registration. Restrict it to dedicated service accounts if CI/CD requires it. - [ ] **Set `admin_email` to a real platform admin**: Use a specific person's email, not a shared mailbox. The PlatformAdmin role bypasses all authorization checks. - [ ] **Verify token lifetime**: Check your IdP's access token and refresh token lifetimes. Shorter access token lifetimes (1 hour or less) reduce the impact of token theft. @@ -19,25 +19,25 @@ For the security architecture, see [Security Model](../security-model.md). For c - [ ] **Review default workspace bindings**: The `default` workspace grants Editor to `*` (all authenticated users). If your deployment requires tighter control, restrict this after bootstrap. - [ ] **Restrict PlatformAdmin**: Only one email should have PlatformAdmin. This role bypasses all authorization — treat it like a root account. -- [ ] **Use scoped tokens for CI/CD**: Request `platform:read` only for pipelines that don't need to modify resources. See [API Scopes](../authorization/api-scopes.md). +- [ ] **Use scoped tokens for CI/CD**: Request `platform:read` only for pipelines that don't need to modify resources. See [API Scopes](/documentation/access-control/authorization/api-scopes). - [ ] **Audit workspace access**: Periodically review workspace members (`nemo workspaces members list --workspace `) and remove stale access. - [ ] **Use wildcard bindings carefully**: Only grant `*` (all users) a role when you intentionally want shared access. Prefer Viewer over Editor for public workspaces. ## Gateway and Network -- [ ] **Strip auth headers from external requests**: Configure your ingress/gateway to remove `X-NMP-Principal-Id`, `X-NMP-Principal-Email`, `X-NMP-Principal-Groups`, `X-NMP-Principal-On-Behalf-Of`, `X-NMP-Authorized`, and `X-NMP-Scopes` from all incoming external traffic. See [Gateway Integration](gateway.md). +- [ ] **Strip auth headers from external requests**: Configure your ingress/gateway to remove `X-NMP-Principal-Id`, `X-NMP-Principal-Email`, `X-NMP-Principal-Groups`, `X-NMP-Principal-On-Behalf-Of`, `X-NMP-Authorized`, and `X-NMP-Scopes` from all incoming external traffic. See [Gateway Integration](/documentation/access-control/deployment/gateway-integration). - [ ] **Enable TLS termination**: Terminate TLS at the ingress or load balancer. Tokens in `Authorization` headers are sent in the clear without TLS. -- [ ] **Consider gateway-level auth**: For reduced latency and centralized authorization, configure Envoy `ext_authz` to call the PDP at the edge. See [Gateway Integration](gateway.md). +- [ ] **Consider gateway-level auth**: For reduced latency and centralized authorization, configure Envoy `ext_authz` to call the PDP at the edge. See [Gateway Integration](/documentation/access-control/deployment/gateway-integration). ## Policy Engine -- [ ] **Choose the right PDP provider**: Use embedded (default) for new deployments. Use external OPA if you already run OPA for other services. See [Policy Engine](../authorization/policy-engine.md). +- [ ] **Choose the right PDP provider**: Use embedded (default) for new deployments. Use external OPA if you already run OPA for other services. See [Policy Engine](/documentation/access-control/authorization/policy-engine). - [ ] **Set appropriate refresh interval**: `policy_data_refresh_interval` (embedded) or `bundle_cache_seconds` (external OPA) controls how quickly role changes take effect. Lower values = faster propagation but more load on the entity store. - [ ] **Monitor PDP health**: Ensure the auth service (embedded) or OPA sidecar (external) is healthy. If the PDP is unreachable, the middleware fails closed (returns 503). ## Secrets and Credentials -- [ ] **Verify CLI config file permissions**: Token storage at `~/.config/nmp/config.yaml` should have permissions `0600` (owner read/write only). Avoid storing this file in cloud-synced or shared directories. See [Using Authentication — Config File](../authentication/using-authentication.md#config-file) for full guidance. +- [ ] **Verify CLI config file permissions**: Token storage at `~/.config/nmp/config.yaml` should have permissions `0600` (owner read/write only). Avoid storing this file in cloud-synced or shared directories. See [Using Authentication — Config File](/documentation/access-control/authentication/using-authentication#config-file) for full guidance. - [ ] **Rotate IdP client secrets**: If your OIDC application uses a client secret, rotate it periodically per your organization's policy. - [ ] **Avoid storing tokens in source code or CI configs**: Use environment variables or secret managers for tokens in CI/CD pipelines. @@ -63,7 +63,7 @@ nemo workspaces list ## Related -- [Security Model](../security-model.md) — Architecture, trust boundaries, and authorization layers. -- [Auth Configuration](configuration.md) — Full configuration reference. -- [Gateway Integration](gateway.md) — Gateway auth and header stripping. -- [Roles & Permissions](../authorization/roles-and-permissions.md) — Permission matrix for role auditing. +- [Security Model](/documentation/access-control/security-model) — Architecture, trust boundaries, and authorization layers. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Full configuration reference. +- [Gateway Integration](/documentation/access-control/deployment/gateway-integration) — Gateway auth and header stripping. +- [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions) — Permission matrix for role auditing. diff --git a/docs/auth/index.mdx b/docs/auth/index.mdx index d4427671971..64a9ee7714b 100644 --- a/docs/auth/index.mdx +++ b/docs/auth/index.mdx @@ -13,12 +13,12 @@ Both layers are opt-in. When `auth.enabled` is `false` (the default), all reques ## How Authentication Works -NeMo Platform authenticates every request using a JWT from your OIDC identity provider. The token is sent in the `Authorization: Bearer ` header, and NeMo Platform validates the signature, issuer, audience, and expiry. Refer to [OIDC Setup](authentication/oidc.md) to connect your identity provider. +NeMo Platform authenticates every request using a JWT from your OIDC identity provider. The token is sent in the `Authorization: Bearer ` header, and NeMo Platform validates the signature, issuer, audience, and expiry. Refer to [OIDC Setup](/documentation/access-control/authentication/oidc-setup) to connect your identity provider. How you **obtain** the token depends on your context: -- **CLI** — Run `nemo auth login` to authenticate using the browser-based device flow. The CLI stores and auto-refreshes the token. Refer to [Using Authentication](authentication/using-authentication.md). -- **SDK** — After `nemo auth login`, the Python SDK automatically reads stored tokens from the CLI config and refreshes them transparently. Refer to [Using Authentication](authentication/using-authentication.md#python-sdk). +- **CLI** — Run `nemo auth login` to authenticate using the browser-based device flow. The CLI stores and auto-refreshes the token. Refer to [Using Authentication](/documentation/access-control/authentication/using-authentication). +- **SDK** — After `nemo auth login`, the Python SDK automatically reads stored tokens from the CLI config and refreshes them transparently. Refer to [Using Authentication](/documentation/access-control/authentication/using-authentication#python-sdk). - **HTTP** — For raw HTTP calls, fetch a token from your IdP (or from the CLI using `nemo auth token`) and pass it in the `Authorization: Bearer ` header. - **Studio** — When auth is enabled, Studio automatically redirects you to your IdP to sign in and uses the resulting token for all API calls. @@ -119,38 +119,38 @@ print(f"Found {len(workspaces.data)} workspaces") ### Production / Helm Deployment -For production or Helm-based deployments, enable auth by setting `platformConfig.auth.enabled: true` in your Helm values and configure the `auth:` section in platform config. Refer to [Auth Configuration](deployment/configuration.md) for the full reference and [OIDC Setup](authentication/oidc.md) to connect your identity provider. +For production or Helm-based deployments, enable auth by setting `platformConfig.auth.enabled: true` in your Helm values and configure the `auth:` section in platform config. Refer to [Auth Configuration](/documentation/access-control/deployment/configuration) for the full reference and [OIDC Setup](/documentation/access-control/authentication/oidc-setup) to connect your identity provider. ## Where to Go Next - + Understand how NeMo Platform authentication and authorization work together — trust boundaries, principal model, and authorization layers. - + Configure NeMo Platform to authenticate users using your OIDC identity provider. - + Add users to workspaces, assign roles, and control who can access your resources. - + Full configuration reference — enabling auth, PDP provider, OIDC settings, environment variables. - + Security checklist for production deployments — OIDC, gateway headers, scoped tokens, TLS. - + Fix common auth issues — 401/403 errors, login failures, role propagation delays. diff --git a/docs/auth/security-model.mdx b/docs/auth/security-model.mdx index de9a2ca87a5..a16aed29932 100644 --- a/docs/auth/security-model.mdx +++ b/docs/auth/security-model.mdx @@ -4,7 +4,7 @@ description: "" --- This page describes the security architecture of NeMo Platform authentication and authorization — how requests are authenticated, how access decisions are made, and where trust boundaries lie. It is intended for platform operators and security reviewers evaluating NeMo Platform for production deployment. -For hands-on setup, see [Auth Configuration](deployment/configuration.md). For the authorization model, see [Concepts](concepts.md). +For hands-on setup, see [Auth Configuration](/documentation/access-control/deployment/configuration). For the authorization model, see [Concepts](/documentation/access-control/concepts). ## Architecture Overview @@ -59,7 +59,7 @@ The request flow: -In quickstart deployments without an OIDC provider, the `X-NMP-Principal-*` headers are set directly by the client instead of being derived from a validated JWT. See [Getting Started](index.md). +In quickstart deployments without an OIDC provider, the `X-NMP-Principal-*` headers are set directly by the client instead of being derived from a validated JWT. See [Getting Started](/documentation/access-control/overview). ## Authentication Modes @@ -68,7 +68,7 @@ NeMo Platform supports two authentication modes: **service-level** (the service In both modes, identity arrives as a **JWT** from the client. The JWT is validated exactly once — either by the first NeMo Platform service or by the gateway. After validation, the authenticated identity (email, subject, groups) is propagated to downstream services via **trusted `X-NMP-Principal-*` headers**. Downstream services accept these headers without re-validating the JWT. -This "validate once, propagate via headers" design means that **network perimeter security is critical**: anything inside the trust boundary that receives `X-NMP-Principal-*` headers will trust them unconditionally. The gateway must strip these headers from all incoming external requests to prevent clients from forging an identity. See [Gateway Integration](deployment/gateway.md). +This "validate once, propagate via headers" design means that **network perimeter security is critical**: anything inside the trust boundary that receives `X-NMP-Principal-*` headers will trust them unconditionally. The gateway must strip these headers from all incoming external requests to prevent clients from forging an identity. See [Gateway Integration](/documentation/access-control/deployment/gateway-integration). ### Service-Level Authentication @@ -128,7 +128,7 @@ A service principal's ID has the form `service:` (e.g., `service:jobs`, `s -Service principals rely on perimeter security — the gateway strips `X-NMP-Principal-*` headers from incoming requests, so external callers cannot forge a `service:` identity. The gateway must also block external access to `/internal/*` paths. See [Gateway Integration](deployment/gateway.md). +Service principals rely on perimeter security — the gateway strips `X-NMP-Principal-*` headers from incoming requests, so external callers cannot forge a `service:` identity. The gateway must also block external access to `/internal/*` paths. See [Gateway Integration](/documentation/access-control/deployment/gateway-integration). #### On-Behalf-Of Delegation @@ -177,7 +177,7 @@ On top of RBAC, NeMo Platform supports **API scopes** as a second authorization Both must pass. This enables least-privilege token usage — for example, an Editor can create a read-only token (`platform:read` only) for monitoring scripts. -For details, see [Authorization Concepts](concepts.md), [Roles & Permissions](authorization/roles-and-permissions.md), and [API Scopes](authorization/api-scopes.md). +For details, see [Authorization Concepts](/documentation/access-control/concepts), [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions), and [API Scopes](/documentation/access-control/authorization/api-scopes). ## What NeMo Platform Does NOT Do @@ -185,12 +185,12 @@ The following are explicitly out of scope for the current implementation: - **Multi-tenancy with database isolation.** Workspaces provide logical isolation, not separate databases or schemas per tenant. - **Service mesh mTLS.** Service-to-service encryption and mutual authentication are delegated to the customer's infrastructure (e.g., Istio, Linkerd). -- **Built-in audit logging.** The embedded PDP does not produce a dedicated audit trail. If you need structured decision logs, use [External OPA](authorization/policy-engine.md#external-opa) — OPA's [decision logging](https://www.openpolicyagent.org/docs/latest/management-decision-logs/) can export every authorization decision to your log aggregator. +- **Built-in audit logging.** The embedded PDP does not produce a dedicated audit trail. If you need structured decision logs, use [External OPA](/documentation/access-control/authorization/policy-engine#external-opa) — OPA's [decision logging](https://www.openpolicyagent.org/docs/latest/management-decision-logs/) can export every authorization decision to your log aggregator. - **Custom RBAC at runtime.** Custom roles can be defined at deployment time via YAML configuration, but cannot be created or modified at runtime through the API. ## Related -- [Authorization Concepts](concepts.md) — Workspaces, roles, bindings, and the RBAC model. -- [Auth Configuration](deployment/configuration.md) — Enabling auth, PDP provider, OIDC settings. -- [Gateway Integration](deployment/gateway.md) — Gateway-level auth and header configuration. -- [Production Hardening](deployment/hardening.md) — Security checklist for production deployments. +- [Authorization Concepts](/documentation/access-control/concepts) — Workspaces, roles, bindings, and the RBAC model. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Enabling auth, PDP provider, OIDC settings. +- [Gateway Integration](/documentation/access-control/deployment/gateway-integration) — Gateway-level auth and header configuration. +- [Production Hardening](/documentation/access-control/deployment/production-hardening) — Security checklist for production deployments. diff --git a/docs/auth/troubleshooting.mdx b/docs/auth/troubleshooting.mdx index 29a3b219213..7de925a6137 100644 --- a/docs/auth/troubleshooting.mdx +++ b/docs/auth/troubleshooting.mdx @@ -162,7 +162,7 @@ If these steps don't resolve the issue: ## Related -- [Auth Configuration](deployment/configuration.md) — Configuration reference. -- [Security Model](security-model.md) — Architecture and trust boundaries. -- [OIDC Setup](authentication/oidc.md) — IdP configuration. -- [Gateway Integration](deployment/gateway.md) — Gateway auth and headers. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Configuration reference. +- [Security Model](/documentation/access-control/security-model) — Architecture and trust boundaries. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — IdP configuration. +- [Gateway Integration](/documentation/access-control/deployment/gateway-integration) — Gateway auth and headers. diff --git a/docs/customizer/about.mdx b/docs/customizer/about.mdx index be0a996313a..b5952b5e1b2 100644 --- a/docs/customizer/about.mdx +++ b/docs/customizer/about.mdx @@ -221,7 +221,7 @@ NeMo Platform Customizer supports various distributed training parallelization m [Tensor Parallelism](https://docs.nvidia.com/nemo-framework/user-guide/latest/nemotoolkit/features/parallelisms.html#tensor-parallelism) (TP) distributes the parameter tensor of an individual layer across GPUs. In addition to reducing model state memory usage, it also saves activation memory as the per-GPU tensor sizes shrink. The tradeoff is increased CPU overhead. -TP can be configured via `parallelism.tensor_parallel_size` in the [training configuration](manage-customization-jobs/hyperparameters.md). +TP can be configured via `parallelism.tensor_parallel_size` in the [training configuration](/documentation/customizer-reference/manage-jobs/training-configuration). @@ -232,7 +232,7 @@ As of release 25.10.0, AutoModel engines including Phi-4, Qwen, and Gemma suppor [Pipeline Parallelism](https://docs.nvidia.com/nemo-framework/user-guide/latest/nemotoolkit/features/parallelisms.html#pipeline-parallelism) (PP) distributes the layers of a neural network across GPUs. The GPUs then process the different layers sequentially. -PP can be configured via `parallelism.pipeline_parallel_size` in the [training configuration](manage-customization-jobs/hyperparameters.md). +PP can be configured via `parallelism.pipeline_parallel_size` in the [training configuration](/documentation/customizer-reference/manage-jobs/training-configuration). #### Configuration @@ -250,7 +250,7 @@ PP can be configured via `parallelism.pipeline_parallel_size` in the [training c [Sequence Parallelism](https://docs.nvidia.com/nemo-framework/user-guide/latest/nemotoolkit/features/parallelisms.html#sequence-parallelism) (SP) extends tensor-level model parallelism by distributing computing load and activation memory across multiple GPUs along the sequence dimension of transformer layers. This method is particularly useful when training on the datasets with longer sequences. It also benefits portions of the layer that have previously not been parallelized, enhancing overall model performance and efficiency. -Sequence Parallelism can be enabled/disabled using `parallelism.sequence_parallel` in the [training configuration](manage-customization-jobs/hyperparameters.md). +Sequence Parallelism can be enabled/disabled using `parallelism.sequence_parallel` in the [training configuration](/documentation/customizer-reference/manage-jobs/training-configuration). ## Sequence Packing diff --git a/docs/customizer/index.mdx b/docs/customizer/index.mdx index 55207812b49..b5ff1751fa6 100644 --- a/docs/customizer/index.mdx +++ b/docs/customizer/index.mdx @@ -10,7 +10,7 @@ Learn how to fine-tune models by making requests to NVIDIA NeMo Customizer throu At a high level, the fine-tuning workflow consists of the following steps: -1. [Create a Model Entity](manage-model-entities/index.md) pointing to your base model checkpoint (stored as a FileSet). +1. [Create a Model Entity](/documentation/customizer-reference/manage-model-entities/overview) pointing to your base model checkpoint (stored as a FileSet). 1. Format a compatible [dataset](/documentation/fine-tune-models/tutorials/format-training-dataset). 1. [Create a customization job](/documentation/fine-tune-models/manage-customization-jobs) referencing the Model Entity. 1. Monitor the job until it completes. @@ -29,22 +29,22 @@ Explore the model families and sizes supported by NVIDIA NeMo Customizer. - + View the available Llama models in the model catalog. - + View the available Llama Nemotron models from NVIDIA, including Nano and Super variants for efficient and advanced instruction tuning. - + View the available Phi models from Microsoft, designed for strong reasoning capabilities with efficient deployment. - + View the available GPT-OSS models supported for Full SFT customization. @@ -68,7 +68,7 @@ Perform common fine-tuning tasks. Create, list, view, and cancel customization jobs. - + Create FileSets and Model Entities to prepare base models for customization. @@ -124,7 +124,7 @@ Learn how to compress a larger teacher model into a smaller student model. nemo-customizer knowledge-distillation - + Learn how to check job metrics using MLFlow or Weights & Biases. @@ -147,7 +147,7 @@ Learn how to optimize the token-per-GPU throughput for a LoRA optimization job. - + View the available hyperparameters and their valid options that you can set when creating a customization job. diff --git a/docs/customizer/manage-customization-jobs/cancel-job.mdx b/docs/customizer/manage-customization-jobs/cancel-job.mdx index 20c2f3a1d06..472e45eb7f1 100644 --- a/docs/customizer/manage-customization-jobs/cancel-job.mdx +++ b/docs/customizer/manage-customization-jobs/cancel-job.mdx @@ -18,7 +18,7 @@ export NMP_BASE_URL="https://your-nmp-base-url" ## To Cancel a Customization Job -Running jobs may be cancelled. A cancelled job does not upload checkpoints. You need the job's name and workspace; you can get these from [List Active Jobs](list-active-jobs.md). +Running jobs may be cancelled. A cancelled job does not upload checkpoints. You need the job's name and workspace; you can get these from [List Active Jobs](/documentation/customizer-reference/manage-jobs/list-active-jobs). Use the SDK to cancel a customization job: diff --git a/docs/customizer/manage-customization-jobs/create-job.mdx b/docs/customizer/manage-customization-jobs/create-job.mdx index 7889da8e7a0..36c8d29354c 100644 --- a/docs/customizer/manage-customization-jobs/create-job.mdx +++ b/docs/customizer/manage-customization-jobs/create-job.mdx @@ -8,10 +8,10 @@ description: "" Before you can create a customization job, make sure that you have: - Obtained the base URL of your NeMo Platform. -- Created a [FileSet and Model Entity](../manage-model-entities/index.md) for your base model. +- Created a [FileSet and Model Entity](/documentation/customizer-reference/manage-model-entities/overview) for your base model. - [Uploaded a dataset](/documentation/get-started/core-concepts/manage-files) as a FileSet. -- Determined the [training configuration](hyperparameters.md) you want to use for the customization job. -- Verified that the platform has sufficient storage for the job. Full SFT jobs require approximately 3× the base model size in free disk space; LoRA jobs require approximately 1.5×. See [ft-tut-understand-models](../tutorials/understand-configurations-and-models.md) for details. If you are also deploying the model from a base checkpoint fileset, plan for ~2.5× model size overall for LoRA. +- Determined the [training configuration](/documentation/customizer-reference/manage-jobs/training-configuration) you want to use for the customization job. +- Verified that the platform has sufficient storage for the job. Full SFT jobs require approximately 3× the base model size in free disk space; LoRA jobs require approximately 1.5×. See [ft-tut-understand-models](/documentation/customizer-reference/tutorials/understanding-models-and-training) for details. If you are also deploying the model from a base checkpoint fileset, plan for ~2.5× model size overall for LoRA. - Set the `NMP_BASE_URL` environment variable to your NeMo Platform endpoint. ```bash @@ -131,14 +131,14 @@ job = client.customization.jobs.create( -See [Knowledge Distillation constraints](hyperparameters.md) for requirements on model compatibility, tokenizer, and GPU memory. +See [Knowledge Distillation constraints](/documentation/customizer-reference/manage-jobs/training-configuration) for requirements on model compatibility, tokenizer, and GPU memory. --- ## Job Configuration Reference -The job spec contains the model, dataset, training, output, deployment, and integration settings for the job. For key fields, the complete API schema, and W&B or MLflow integration options, see [Customization Job Reference](customization-job-reference.md). +The job spec contains the model, dataset, training, output, deployment, and integration settings for the job. For key fields, the complete API schema, and W&B or MLflow integration options, see [Customization Job Reference](/documentation/customizer-reference/manage-jobs/customization-job-reference). --- diff --git a/docs/customizer/manage-customization-jobs/customization-job-reference.mdx b/docs/customizer/manage-customization-jobs/customization-job-reference.mdx index fc930138a39..9cd62cc3df6 100644 --- a/docs/customizer/manage-customization-jobs/customization-job-reference.mdx +++ b/docs/customizer/manage-customization-jobs/customization-job-reference.mdx @@ -17,7 +17,7 @@ All job configuration (model, dataset, training, and output) is specified in the | `workspace` | Yes | Workspace where the job runs. Determines what datasets and models are authorized to be used in the job. | | `spec.model` | Yes | Reference to the Model Entity (`workspace/name` format) | | `spec.dataset` | Yes | Dataset URI (`fileset://workspace/name`) | -| `spec.training` | Yes | Training method and hyperparameters (see [Training Configuration](hyperparameters.md)) | +| `spec.training` | Yes | Training method and hyperparameters (see [Training Configuration](/documentation/customizer-reference/manage-jobs/training-configuration)) | | `spec.training.type` | Yes | Training method: `sft`, `distillation`, or `dpo` | | `spec.training.peft` | No | PEFT adapter configuration (e.g., `{"type": "lora", ...}`). Omit for full-weight training | | `spec.output` | No | Output artifact configuration (`{"name": "..."}`). Auto-generated if not provided | @@ -98,7 +98,7 @@ To create the secret, see [Weights & Biases Keys](/documentation/get-started/cor | `notes` | Notes or description for the run | | `base_url` | Base URL for self-hosted W&B servers (e.g., `https://wandb.mycompany.com`). Omit to use W&B cloud | -To view your training metrics in W&B after the job starts, see [ft-tut-metrics-wandb](../tutorials/metrics.md). +To view your training metrics in W&B after the job starts, see [ft-tut-metrics-wandb](/documentation/customizer-reference/tutorials/job-metrics). --- @@ -161,6 +161,6 @@ nemo customization jobs create my-job \ ## Next Steps - [Create a customization job](/documentation/fine-tune-models/manage-customization-jobs/create-a-customization-job): Start a job with a model, dataset, training configuration, and optional integrations. -- [Monitor training metrics](../tutorials/metrics.md): View logs and metrics through MLflow or W&B. +- [Monitor training metrics](/documentation/customizer-reference/tutorials/job-metrics): View logs and metrics through MLflow or W&B. - [Manage secrets](/documentation/get-started/core-concepts/manage-secrets): Store credentials such as W&B API keys and provider tokens. - [Troubleshooting MLflow integrations](/documentation/reference/troubleshooting/customizer): Diagnose failed or misconfigured customization jobs. diff --git a/docs/customizer/manage-customization-jobs/get-job-status.mdx b/docs/customizer/manage-customization-jobs/get-job-status.mdx index c86dbcdd86e..5013f63fb94 100644 --- a/docs/customizer/manage-customization-jobs/get-job-status.mdx +++ b/docs/customizer/manage-customization-jobs/get-job-status.mdx @@ -13,7 +13,7 @@ This endpoint provides granular execution details including: - **Training metrics**: `step`, `epoch`, `loss`, `lr` (learning rate), `grad_norm`, `val_loss` - **Progress tracking**: `downloaded_files`, `uploaded_bytes`, `progress_pct` -To list jobs or get job definitions (model entity, hyperparameters, spec), use [List Active Jobs](list-active-jobs.md) instead. +To list jobs or get job definitions (model entity, hyperparameters, spec), use [List Active Jobs](/documentation/customizer-reference/manage-jobs/list-active-jobs) instead. ## Prerequisites diff --git a/docs/customizer/manage-customization-jobs/hyperparameters.mdx b/docs/customizer/manage-customization-jobs/hyperparameters.mdx index a0bad160a73..20233e7ca7c 100644 --- a/docs/customizer/manage-customization-jobs/hyperparameters.mdx +++ b/docs/customizer/manage-customization-jobs/hyperparameters.mdx @@ -5,7 +5,7 @@ description: "" -Want to learn about training concepts at a high level? Check out the [Customization concepts](../about.md) page. +Want to learn about training concepts at a high level? Check out the [Customization concepts](/documentation/customizer-reference/customization-concepts) page. ## Complete Schema Reference diff --git a/docs/customizer/manage-customization-jobs/index.mdx b/docs/customizer/manage-customization-jobs/index.mdx index c047f991c45..55d9390b7c6 100644 --- a/docs/customizer/manage-customization-jobs/index.mdx +++ b/docs/customizer/manage-customization-jobs/index.mdx @@ -3,7 +3,7 @@ title: "Manage Customization Jobs" description: "" --- -Use customization jobs to fine-tune a [model](../models/index.md) using a [dataset](/documentation/get-started/core-concepts/manage-files) and [hyperparameters](hyperparameters.md). +Use customization jobs to fine-tune a [model](/documentation/customizer-reference/models/model-catalog) using a [dataset](/documentation/get-started/core-concepts/manage-files) and [hyperparameters](/documentation/customizer-reference/manage-jobs/training-configuration). ## How It Works @@ -36,17 +36,17 @@ The value for `NMP_BASE_URL` will depend on your deployment. After the standard Create a customization job using SFT, DPO, or Knowledge Distillation. - + Check the status of a customization job. - + List all active customization jobs to find a job name for use with Get Status or Cancel. - + Cancel a customization job using its name and workspace. @@ -60,7 +60,7 @@ Refer to the following pages for more information on customization jobs. - + Review the hyperparameters that you can use to customize a model. diff --git a/docs/customizer/manage-customization-jobs/list-active-jobs.mdx b/docs/customizer/manage-customization-jobs/list-active-jobs.mdx index 90531c8b522..7a6b644507a 100644 --- a/docs/customizer/manage-customization-jobs/list-active-jobs.mdx +++ b/docs/customizer/manage-customization-jobs/list-active-jobs.mdx @@ -7,7 +7,7 @@ List all customization jobs and their high-level status. This returns job defini -To get **detailed execution progress** (step-by-step status, training metrics like loss/epoch/step), use [Get Job Status](get-job-status.md) instead. +To get **detailed execution progress** (step-by-step status, training metrics like loss/epoch/step), use [Get Job Status](/documentation/customizer-reference/manage-jobs/get-job-status) instead. ## Prerequisites diff --git a/docs/customizer/manage-model-entities/create-fileset.mdx b/docs/customizer/manage-model-entities/create-fileset.mdx index f5f996dfdb6..94809e87a6d 100644 --- a/docs/customizer/manage-model-entities/create-fileset.mdx +++ b/docs/customizer/manage-model-entities/create-fileset.mdx @@ -279,4 +279,4 @@ for f in files[:10]: # Show first 10 ## Next Steps -Proceed to [create a Model Entity](create-model-entity.md). +Proceed to [create a Model Entity](/documentation/customizer-reference/manage-model-entities/create-a-model-entity). diff --git a/docs/customizer/manage-model-entities/create-model-entity.mdx b/docs/customizer/manage-model-entities/create-model-entity.mdx index f695d7755cc..7c540799ade 100644 --- a/docs/customizer/manage-model-entities/create-model-entity.mdx +++ b/docs/customizer/manage-model-entities/create-model-entity.mdx @@ -170,4 +170,4 @@ else: ## Next Steps - [Create a customization job](/documentation/fine-tune-models/manage-customization-jobs/create-a-customization-job) -- [Understand hyperparameters](../manage-customization-jobs/hyperparameters.md) +- [Understand hyperparameters](/documentation/customizer-reference/manage-jobs/training-configuration) diff --git a/docs/customizer/manage-model-entities/index.mdx b/docs/customizer/manage-model-entities/index.mdx index 6c0f15973d7..a9f3dd28763 100644 --- a/docs/customizer/manage-model-entities/index.mdx +++ b/docs/customizer/manage-model-entities/index.mdx @@ -14,7 +14,7 @@ Before running a customization job, you need to set up a **Model Entity** that p Create a FileSet containing your base model checkpoint from HuggingFace, NGC, or local storage. - + Create a Model Entity that references your FileSet and enables customization. diff --git a/docs/customizer/models/embedding.mdx b/docs/customizer/models/embedding.mdx index 28f8d109b6e..f22bb5e5a20 100644 --- a/docs/customizer/models/embedding.mdx +++ b/docs/customizer/models/embedding.mdx @@ -4,7 +4,7 @@ description: "" --- -This page provides detailed technical specifications for the embedding model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the embedding model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Llama Nemotron Embedding 1B v2 @@ -134,4 +134,4 @@ print(f"Embedding dimension: {len(embedding)}") For detailed fine-tuning instructions, refer to the [Embedding Customization tutorial](../tutorials/embedding-customization-job.ipynb). -For more information about formatting training datasets for the embedding model, refer to [Dataset Format Requirements](data-format.md). +For more information about formatting training datasets for the embedding model, refer to [Dataset Format Requirements](/documentation/customizer-reference/models/dataset-format). diff --git a/docs/customizer/models/gpt-oss.mdx b/docs/customizer/models/gpt-oss.mdx index db499fbb4c2..39f58491ca7 100644 --- a/docs/customizer/models/gpt-oss.mdx +++ b/docs/customizer/models/gpt-oss.mdx @@ -4,11 +4,11 @@ description: "" --- -This page provides detailed technical specifications for the OpenAI GPT-OSS model family supported by NeMo Customizer. For supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the OpenAI GPT-OSS model family supported by NeMo Customizer. For supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Before You Start -These models require a HuggingFace token to download. Create a secret with your HuggingFace API key, then create a FileSet and Model Entity referencing the model. See [index](../manage-model-entities/index.md) for setup instructions. +These models require a HuggingFace token to download. Create a secret with your HuggingFace API key, then create a FileSet and Model Entity referencing the model. See [index](/documentation/customizer-reference/manage-model-entities/overview) for setup instructions. --- diff --git a/docs/customizer/models/index.mdx b/docs/customizer/models/index.mdx index f07f44c5772..ec9acd03c9b 100644 --- a/docs/customizer/models/index.mdx +++ b/docs/customizer/models/index.mdx @@ -8,13 +8,13 @@ Explore the model families and sizes supported by NVIDIA NeMo Customizer. -For information on setting up model entities for customization, see the [Manage Model Entities](../manage-model-entities/index.md) guide. +For information on setting up model entities for customization, see the [Manage Model Entities](/documentation/customizer-reference/manage-model-entities/overview) guide. For fine-tuning and deployment tutorials, see the [Tutorials](/documentation/fine-tune-models/tutorials) guide. ## Before You Start -If downloading models hosted on Hugging Face, create a secret with your HuggingFace API key, then create a FileSet and Model Entity referencing the model. See [index](../manage-model-entities/index.md) for setup instructions. +If downloading models hosted on Hugging Face, create a secret with your HuggingFace API key, then create a FileSet and Model Entity referencing the model. See [index](/documentation/customizer-reference/manage-model-entities/overview) for setup instructions. --- @@ -23,17 +23,17 @@ If downloading models hosted on Hugging Face, create a secret with your HuggingF - + View the available Llama models from Meta, ranging from 8 billion to 70 billion parameters. - + View the available Llama Nemotron models from NVIDIA, including Nano and Super variants for efficient and advanced instruction tuning. - + View the available Phi models from Microsoft, designed for strong reasoning capabilities with efficient deployment. @@ -43,17 +43,17 @@ View the available Phi models from Microsoft, designed for strong reasoning capa View the available embedding models optimized for retrieval and question-answering tasks. - + View the available GPT-OSS models supported for customization. - + View the available Qwen models from Alibaba Cloud, including compact variants for efficient customization. - + View the available Mistral models, including Mistral and Ministral variants for instruction-following and reasoning tasks. diff --git a/docs/customizer/models/llama-nemotron.mdx b/docs/customizer/models/llama-nemotron.mdx index 3e8aeb1e14a..091b420d7c3 100644 --- a/docs/customizer/models/llama-nemotron.mdx +++ b/docs/customizer/models/llama-nemotron.mdx @@ -4,7 +4,7 @@ description: "" --- -This page provides detailed technical specifications for the Nemotron model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the Nemotron model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Llama 3.1 Nemotron Nano 8B v1 diff --git a/docs/customizer/models/llama.mdx b/docs/customizer/models/llama.mdx index f8587f022f3..31573aa778e 100644 --- a/docs/customizer/models/llama.mdx +++ b/docs/customizer/models/llama.mdx @@ -3,7 +3,7 @@ title: "Llama Models" description: "" --- -This page provides detailed technical specifications for the Llama model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the Llama model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Llama-3.2-3B Instruct diff --git a/docs/customizer/models/mistral.mdx b/docs/customizer/models/mistral.mdx index 445f794f750..dc3850f9c72 100644 --- a/docs/customizer/models/mistral.mdx +++ b/docs/customizer/models/mistral.mdx @@ -4,7 +4,7 @@ description: "" --- -This page provides detailed technical specifications for the Mistral model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the Mistral model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Mistral-7B-Instruct-v0.3 diff --git a/docs/customizer/models/phi.mdx b/docs/customizer/models/phi.mdx index 3f4ee2d6179..caeaf972a13 100644 --- a/docs/customizer/models/phi.mdx +++ b/docs/customizer/models/phi.mdx @@ -3,7 +3,7 @@ title: "Phi Models" description: "" --- -This page provides detailed technical specifications for the Phi model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the Phi model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Microsoft Phi-4 diff --git a/docs/customizer/models/qwen.mdx b/docs/customizer/models/qwen.mdx index 52b4f588142..3d414a3d3d4 100644 --- a/docs/customizer/models/qwen.mdx +++ b/docs/customizer/models/qwen.mdx @@ -4,7 +4,7 @@ description: "" --- -This page provides detailed technical specifications for the Qwen model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the Qwen model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Qwen2.5-1.5B-Instruct diff --git a/docs/customizer/tutorials/format-training-dataset.mdx b/docs/customizer/tutorials/format-training-dataset.mdx index 4bd02640dbb..7471f7b3df4 100644 --- a/docs/customizer/tutorials/format-training-dataset.mdx +++ b/docs/customizer/tutorials/format-training-dataset.mdx @@ -101,7 +101,7 @@ For illustrative purposes only, we show an example entry as multi-line JSON. ### Reasoning Considerations -Some models (such as [Llama Nemotron](../models/llama-nemotron.md)) support a **detailed thinking** mode, which you can toggle in the system message. This setting controls whether the model is encouraged to show step-by-step reasoning in its responses. +Some models (such as [Llama Nemotron](/documentation/customizer-reference/models/llama-nemotron)) support a **detailed thinking** mode, which you can toggle in the system message. This setting controls whether the model is encouraged to show step-by-step reasoning in its responses. - **Training data without reasoning:** Use `detailed thinking off` in the system message. - **Training data with reasoning:** Use `detailed thinking on` in the system message. diff --git a/docs/customizer/tutorials/import-hf-model.mdx b/docs/customizer/tutorials/import-hf-model.mdx index 8f6756ec07c..eb3212dfee1 100644 --- a/docs/customizer/tutorials/import-hf-model.mdx +++ b/docs/customizer/tutorials/import-hf-model.mdx @@ -47,7 +47,7 @@ Verify that all required services are running and accessible before proceeding. - ✅ **Recommended**: Phi models - ✅ **Alternative**: Gemma models (used in this tutorial) -For a complete list of tested models, see the [Model Catalog](../models/index.md). +For a complete list of tested models, see the [Model Catalog](/documentation/customizer-reference/models/model-catalog). --- @@ -765,4 +765,4 @@ curl -X POST "${NIM_PROXY_URL}/v1/chat/completions" \ ## Next Steps -Learn how to [check customization job metrics](metrics.md) to monitor the training progress and performance of your fine-tuned model. +Learn how to [check customization job metrics](/documentation/customizer-reference/tutorials/job-metrics) to monitor the training progress and performance of your fine-tuned model. diff --git a/docs/customizer/tutorials/index.mdx b/docs/customizer/tutorials/index.mdx index 668f3780bdf..5a6e0c5a937 100644 --- a/docs/customizer/tutorials/index.mdx +++ b/docs/customizer/tutorials/index.mdx @@ -16,7 +16,7 @@ Tutorials are organized by complexity and typically build on one another. The tu - + Learn the fundamentals of how NeMo Customizer works with Model Entities and Adapters, and how to choose the right approach for your project. @@ -86,7 +86,7 @@ Learn how to fine-tune embedding models using LoRA merged training for improved - + Learn how to check job metrics using MLflow or Weights & Biases. diff --git a/docs/customizer/tutorials/understand-configurations-and-models.mdx b/docs/customizer/tutorials/understand-configurations-and-models.mdx index 1063ceb9d94..910dcd9c512 100644 --- a/docs/customizer/tutorials/understand-configurations-and-models.mdx +++ b/docs/customizer/tutorials/understand-configurations-and-models.mdx @@ -391,7 +391,7 @@ model = client.models.create( ) ``` -For detailed guidance, see [Import HuggingFace Model](import-hf-model.md). +For detailed guidance, see [Import HuggingFace Model](/documentation/customizer-reference/tutorials/import-hugging-face-models). --- @@ -416,7 +416,7 @@ Create a parameter-efficient LoRA adapter. Use full supervised fine-tuning for maximum performance. - + Import and fine-tune private HuggingFace models. diff --git a/docs/evaluator/benchmarks/agentic.mdx b/docs/evaluator/benchmarks/agentic.mdx index f564f0a8f85..9df8d7fe577 100644 --- a/docs/evaluator/benchmarks/agentic.mdx +++ b/docs/evaluator/benchmarks/agentic.mdx @@ -110,7 +110,7 @@ Accuracy of tool call predictions ## Job Management -After creating a job, navigate to [Benchmark Job Management](job-management.md) to oversee its execution and monitor progress. +After creating a job, navigate to [Benchmark Job Management](/documentation/evaluator-benchmark-reference/benchmarks/job-management) to oversee its execution and monitor progress. --- @@ -119,6 +119,6 @@ After creating a job, navigate to [Benchmark Job Management](job-management.md) - [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration) - Use agents (generic or NAT) as targets in online evaluation and benchmark jobs - [Agentic Evaluation Metrics](/documentation/evaluate-models/metrics/agentic-metrics) - Detailed metric documentation for evaluating agentic workflows - [Managing Secrets](/documentation/get-started/core-concepts/manage-secrets) - Store API keys for external APIs -- [Evaluation Results](results.md) - Understanding and downloading results +- [Evaluation Results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results) - Understanding and downloading results diff --git a/docs/evaluator/benchmarks/custom.mdx b/docs/evaluator/benchmarks/custom.mdx index adece802399..0b1aa58d241 100644 --- a/docs/evaluator/benchmarks/custom.mdx +++ b/docs/evaluator/benchmarks/custom.mdx @@ -128,7 +128,7 @@ benchmark = client.evaluation.benchmarks.create( print(f"Created benchmark: {benchmark.name}") ``` -Refer to [Manage Benchmarks](manage-benchmarks.md) for listing and managing custom benchmarks. +Refer to [Manage Benchmarks](/documentation/evaluator-benchmark-reference/benchmarks/manage-benchmarks) for listing and managing custom benchmarks. ## Run Benchmark Evaluation Jobs @@ -234,11 +234,11 @@ job = client.evaluation.benchmark_jobs.create( ## Job Management -After successfully creating a job, navigate to [Benchmark Job Management](job-management.md) to oversee its execution and monitor progress. +After successfully creating a job, navigate to [Benchmark Job Management](/documentation/evaluator-benchmark-reference/benchmarks/job-management) to oversee its execution and monitor progress. ## Retrieve Results -After the job completes, retrieve and analyze results. Refer to [Benchmark Results](results.md) for detailed examples of downloading aggregate scores, row-level scores, and analyzing results with Pandas. +After the job completes, retrieve and analyze results. Refer to [Benchmark Results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results) for detailed examples of downloading aggregate scores, row-level scores, and analyzing results with Pandas. ## Complete Example diff --git a/docs/evaluator/benchmarks/index.mdx b/docs/evaluator/benchmarks/index.mdx index 9a35e6038a4..2042810fdf9 100644 --- a/docs/evaluator/benchmarks/index.mdx +++ b/docs/evaluator/benchmarks/index.mdx @@ -45,7 +45,7 @@ benchmark = client.evaluation.benchmarks.create( ) ``` -Refer to [Manage Benchmarks](manage-benchmarks.md) for listing and managing custom benchmarks. +Refer to [Manage Benchmarks](/documentation/evaluator-benchmark-reference/benchmarks/manage-benchmarks) for listing and managing custom benchmarks. ## Run Benchmark Jobs @@ -93,11 +93,11 @@ print(f"Job created: {job.name}") List, retrieve, and delete evaluation benchmarks using the Python SDK. You can discover industry benchmarks in the `system` workspace, list custom benchmarks in your workspace, retrieve detailed benchmark configurations, and delete custom benchmarks when no longer needed. -Refer to [Manage Benchmarks](manage-benchmarks.md) for complete SDK examples including pagination, sorting, filtering, and extended response options. +Refer to [Manage Benchmarks](/documentation/evaluator-benchmark-reference/benchmarks/manage-benchmarks) for complete SDK examples including pagination, sorting, filtering, and extended response options. ## Job Management -After successfully creating a job, refer to [Benchmark Job Management](job-management.md) to oversee its execution and monitor progress. +After successfully creating a job, refer to [Benchmark Job Management](/documentation/evaluator-benchmark-reference/benchmarks/job-management) to oversee its execution and monitor progress. ## Benchmark Categories diff --git a/docs/evaluator/benchmarks/industry.mdx b/docs/evaluator/benchmarks/industry.mdx index 026600cbc7a..247a512f041 100644 --- a/docs/evaluator/benchmarks/industry.mdx +++ b/docs/evaluator/benchmarks/industry.mdx @@ -287,7 +287,7 @@ for score in aggregate.scores: print(f"{score.name}: {score.mean:.1%}") ``` -For detailed results analysis, refer to [eval-benchmark-results](results.md). +For detailed results analysis, refer to [eval-benchmark-results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results). ## Instruction Following @@ -345,7 +345,7 @@ for score in aggregate.scores: print(f"{score.name}: {score.mean:.1%}") ``` -For detailed results analysis, refer to [eval-benchmark-results](results.md). +For detailed results analysis, refer to [eval-benchmark-results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results). ## Language Understanding @@ -541,7 +541,7 @@ for score in aggregate.scores: print(f"{score.name}: {score.mean:.1%}") ``` -For detailed results analysis, refer to [eval-benchmark-results](results.md). +For detailed results analysis, refer to [eval-benchmark-results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results). ## Math & Reasoning @@ -734,7 +734,7 @@ for score in aggregate.scores: print(f"{score.name}: {score.mean:.1%}") ``` -For detailed results analysis, refer to [eval-benchmark-results](results.md). +For detailed results analysis, refer to [eval-benchmark-results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results). ## Content Safety @@ -853,7 +853,7 @@ for score in aggregate.scores: print(f"{score.name}: {score.mean:.1%}") ``` -For detailed results analysis, refer to [eval-benchmark-results](results.md). +For detailed results analysis, refer to [eval-benchmark-results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results). ### Troubleshooting Content Safety Benchmarks @@ -1121,7 +1121,7 @@ for score in aggregate.scores: print(f"{score.name}: {score.mean:.1%}") ``` -For detailed results analysis, refer to [eval-benchmark-results](results.md). +For detailed results analysis, refer to [eval-benchmark-results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results). @@ -1169,4 +1169,4 @@ job = client.evaluation.benchmark_jobs.create( ## Job Management -After creating a job, navigate to [Benchmark Job Management](job-management.md) to oversee its execution and monitor progress. +After creating a job, navigate to [Benchmark Job Management](/documentation/evaluator-benchmark-reference/benchmarks/job-management) to oversee its execution and monitor progress. diff --git a/docs/evaluator/benchmarks/job-management.mdx b/docs/evaluator/benchmarks/job-management.mdx index c39930101f0..aa540dcd804 100644 --- a/docs/evaluator/benchmarks/job-management.mdx +++ b/docs/evaluator/benchmarks/job-management.mdx @@ -47,7 +47,7 @@ while logs_response.next_page: ## View Evaluation Results -Evaluation results are available after the evaluation job completes. Refer to [Benchmark Results](results.md) for details on fetching evaluation results to analyze the job output. +Evaluation results are available after the evaluation job completes. Refer to [Benchmark Results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results) for details on fetching evaluation results to analyze the job output. ### Aggregate Scores diff --git a/docs/evaluator/benchmarks/manage-benchmarks.mdx b/docs/evaluator/benchmarks/manage-benchmarks.mdx index 75a2c99678f..ce2c5225dc2 100644 --- a/docs/evaluator/benchmarks/manage-benchmarks.mdx +++ b/docs/evaluator/benchmarks/manage-benchmarks.mdx @@ -146,5 +146,5 @@ client.evaluation.benchmarks.delete(name="my-custom-benchmark") ## Related Topics - [Custom Benchmarks](/documentation/evaluate-models/benchmarks/custom-benchmarks) - Create custom benchmarks -- [Benchmark Job Management](job-management.md) - Manage benchmark evaluation jobs -- [Benchmark Results](results.md) - View and download benchmark evaluation results +- [Benchmark Job Management](/documentation/evaluator-benchmark-reference/benchmarks/job-management) - Manage benchmark evaluation jobs +- [Benchmark Results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results) - View and download benchmark evaluation results diff --git a/docs/fern/snippets/_snippets/tutorials/prereqs.mdx b/docs/fern/snippets/_snippets/tutorials/prereqs.mdx index dc251c763f4..c6be00bca9a 100644 --- a/docs/fern/snippets/_snippets/tutorials/prereqs.mdx +++ b/docs/fern/snippets/_snippets/tutorials/prereqs.mdx @@ -6,10 +6,10 @@ description: "" All platform resources—models, datasets, and more—must belong to a **workspace**. Workspaces provide organizational and authorization boundaries for your work. Within a workspace, you can optionally use **projects** to group related resources. -**If you're new to the platform**, start with the **[Setup guide](../../get-started/setup.md)** to learn how to deploy and evaluate models, and optimize agents using the platform end-to-end. +**If you're new to the platform**, start with the **[Setup guide](/documentation/get-started)** to learn how to deploy and evaluate models, and optimize agents using the platform end-to-end. **If you're already familiar** with workspaces and how to upload datasets to the platform, you can proceed directly with this tutorial. -For more information, see [Workspaces](../../get-started/concepts/workspaces.md) and [Projects](../../get-started/concepts/projects.md). +For more information, see [Workspaces](/documentation/get-started/core-concepts/workspaces) and [Projects](/documentation/get-started/core-concepts/projects). diff --git a/docs/fern/snippets/customizer/tutorials/_snippets/customizer-prereqs.mdx b/docs/fern/snippets/customizer/tutorials/_snippets/customizer-prereqs.mdx index 1412371f04e..ec094dad802 100644 --- a/docs/fern/snippets/customizer/tutorials/_snippets/customizer-prereqs.mdx +++ b/docs/fern/snippets/customizer/tutorials/_snippets/customizer-prereqs.mdx @@ -6,7 +6,7 @@ description: "" Before starting, make sure you have: -- NeMo Platform installed and deployed (see [Setup](../../get-started/setup.md)) +- NeMo Platform installed and deployed (see [Setup](/documentation/get-started)) - The `nemo-platform` Python SDK installed (`pip install nemo-platform[all]`) - (Optional) Weights & Biases account and API key for enhanced visualization diff --git a/docs/fern/snippets/evaluator/benchmarks/discover-industry-benchmarks.mdx b/docs/fern/snippets/evaluator/benchmarks/discover-industry-benchmarks.mdx index 9aae14e9c21..2d8f648a995 100644 --- a/docs/fern/snippets/evaluator/benchmarks/discover-industry-benchmarks.mdx +++ b/docs/fern/snippets/evaluator/benchmarks/discover-industry-benchmarks.mdx @@ -9,7 +9,7 @@ The `system` workspace is a reserved workspace for NeMo Platform that contains r -**Initialization:** This example uses `NeMoPlatform()` with no arguments so the SDK reads your active CLI context (set by `nemo auth login`). The `workspace="system"` is passed per-call to access the reserved system workspace. For the standard local initialization pattern, see [CLI and SDK initialization](../../get-started/setup.md#setup-init). +**Initialization:** This example uses `NeMoPlatform()` with no arguments so the SDK reads your active CLI context (set by `nemo auth login`). The `workspace="system"` is passed per-call to access the reserved system workspace. For the standard local initialization pattern, see [CLI and SDK initialization](/documentation/get-started#setup-init). ```python diff --git a/docs/fern/versions/latest.yml b/docs/fern/versions/latest.yml index d790e9051df..da97732d478 100644 --- a/docs/fern/versions/latest.yml +++ b/docs/fern/versions/latest.yml @@ -287,3 +287,168 @@ navigation: path: ../../eula.mdx - page: Acknowledgements path: ../../acknowledgements/index.mdx + - section: Access Control + hidden: true + contents: + - page: Concepts + path: ../../auth/concepts.mdx + - page: Overview + path: ../../auth/index.mdx + - page: Security Model + path: ../../auth/security-model.mdx + - page: Troubleshooting + path: ../../auth/troubleshooting.mdx + - section: Authentication + contents: + - page: OIDC Setup + path: ../../auth/authentication/oidc.mdx + - page: Using Authentication + path: ../../auth/authentication/using-authentication.mdx + - section: Providers + contents: + - page: Azure AD (Entra ID) + path: ../../auth/authentication/providers/azure-ad.mdx + - page: Generic OIDC + path: ../../auth/authentication/providers/generic.mdx + - section: Authorization + contents: + - page: API Scopes + path: ../../auth/authorization/api-scopes.mdx + - page: Managing Access + path: ../../auth/authorization/managing-access.mdx + - page: Policy Engine + path: ../../auth/authorization/policy-engine.mdx + - page: Roles and Permissions + path: ../../auth/authorization/roles-and-permissions.mdx + - section: Deployment + contents: + - page: Configuration + path: ../../auth/deployment/configuration.mdx + - page: Gateway Integration + path: ../../auth/deployment/gateway.mdx + - page: Production Hardening + path: ../../auth/deployment/hardening.mdx + - section: Self-Managed Deployment + hidden: true + contents: + - section: Helm + contents: + - page: Helm Reference + path: ../../helm/index.mdx + - section: Setup + contents: + - page: About + path: ../../set-up/index.mdx + - page: Milvus + path: ../../set-up/milvus.mdx + - page: Observability + path: ../../set-up/opentelemetry.mdx + - section: Helm + contents: + - page: Backup and Restore + path: ../../set-up/helm/backup-and-restore.mdx + - page: Database Setup + path: ../../set-up/helm/database-setup.mdx + - page: File Storage + path: ../../set-up/helm/file-storage.mdx + - page: Overview + path: ../../set-up/helm/index.mdx + - page: Ingress + path: ../../set-up/helm/ingress.mdx + - page: Install + path: ../../set-up/helm/install.mdx + - page: Multinode Networking + path: ../../set-up/helm/multinode-networking.mdx + - page: OpenShift + path: ../../set-up/helm/openshift.mdx + - page: Persistent Volumes + path: ../../set-up/helm/persistent-volumes.mdx + - page: Prerequisites + path: ../../set-up/helm/prerequisites.mdx + - section: Customizer Reference + hidden: true + contents: + - page: Customization Concepts + path: ../../customizer/about.mdx + - section: Manage Jobs + contents: + - page: Cancel Job + path: ../../customizer/manage-customization-jobs/cancel-job.mdx + - page: Customization Job Reference + path: ../../customizer/manage-customization-jobs/customization-job-reference.mdx + - page: Get Job Status + path: ../../customizer/manage-customization-jobs/get-job-status.mdx + - page: Training Configuration + path: ../../customizer/manage-customization-jobs/hyperparameters.mdx + - page: List Active Jobs + path: ../../customizer/manage-customization-jobs/list-active-jobs.mdx + - section: Manage Model Entities + contents: + - page: Create a Model Entity + path: ../../customizer/manage-model-entities/create-model-entity.mdx + - page: Overview + path: ../../customizer/manage-model-entities/index.mdx + - section: Models + contents: + - page: Dataset Format + path: ../../customizer/models/data-format.mdx + - page: GPT-OSS + path: ../../customizer/models/gpt-oss.mdx + - page: Model Catalog + path: ../../customizer/models/index.mdx + - page: Llama Nemotron + path: ../../customizer/models/llama-nemotron.mdx + - page: Llama + path: ../../customizer/models/llama.mdx + - page: Mistral + path: ../../customizer/models/mistral.mdx + - page: Phi + path: ../../customizer/models/phi.mdx + - page: Qwen + path: ../../customizer/models/qwen.mdx + - section: Tutorials + contents: + - page: Import HuggingFace Models + path: ../../customizer/tutorials/import-hf-model.mdx + - page: Job Metrics + path: ../../customizer/tutorials/metrics.mdx + - page: Understanding Models and Training + path: ../../customizer/tutorials/understand-configurations-and-models.mdx + - section: Safe Synthesizer Reference + hidden: true + contents: + - page: Getting Started + path: ../../safe-synthesizer/getting-started.mdx + - section: About + contents: + - page: Data Synthesis + path: ../../safe-synthesizer/about/data-synthesis.mdx + - page: Evaluation + path: ../../safe-synthesizer/about/evaluation.mdx + - page: Host-Local Development + path: ../../safe-synthesizer/about/host-local-development.mdx + - page: Overview + path: ../../safe-synthesizer/about/index.mdx + - page: Jobs + path: ../../safe-synthesizer/about/jobs.mdx + - page: PII Replacement + path: ../../safe-synthesizer/about/pii-replacement.mdx + - page: Parameters Reference + path: ../../safe-synthesizer/about/reference.mdx + - section: Tutorials + contents: + - page: Differential Privacy + path: ../../safe-synthesizer/tutorials/differential-privacy.mdx + - page: Overview + path: ../../safe-synthesizer/tutorials/index.mdx + - section: Evaluator Benchmark Reference + hidden: true + contents: + - section: Benchmarks + contents: + - page: Job Management + path: ../../evaluator/benchmarks/job-management.mdx + - page: Manage Benchmarks + path: ../../evaluator/benchmarks/manage-benchmarks.mdx + - page: Benchmark Results + path: ../../evaluator/benchmarks/results.mdx diff --git a/docs/guardrails/observability.mdx b/docs/guardrails/observability.mdx index fb1ef8484ed..a464288ee2e 100644 --- a/docs/guardrails/observability.mdx +++ b/docs/guardrails/observability.mdx @@ -10,7 +10,7 @@ NeMo Platform centrally manages OpenTelemetry across services. You can configure ## Prerequisites -To export guardrail traces, OpenTelemetry must be enabled in your deployment. See [OpenTelemetry](../set-up/opentelemetry.md) for instructions on enabling OpenTelemetry. +To export guardrail traces, OpenTelemetry must be enabled in your deployment. See [OpenTelemetry](/documentation/self-managed-deployment/setup/observability) for instructions on enabling OpenTelemetry. You also need a VirtualModel configured with guardrails middleware. See [Architecture](/documentation/guardrail-models/core-concepts/architecture) for wiring details. @@ -210,7 +210,7 @@ guardrails.request [server] action.has_llm_calls: false ``` -The `service.name` for all spans is determined by the platform's `OTEL_SERVICE_NAME` configuration. See [OpenTelemetry](../set-up/opentelemetry.md) for details. +The `service.name` for all spans is determined by the platform's `OTEL_SERVICE_NAME` configuration. See [OpenTelemetry](/documentation/self-managed-deployment/setup/observability) for details. --- diff --git a/docs/helm/index.mdx b/docs/helm/index.mdx index 9746a12f02a..750b08af200 100644 --- a/docs/helm/index.mdx +++ b/docs/helm/index.mdx @@ -17,7 +17,7 @@ description: "" ``` ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) -For deployment guide, see [Admin Setup](../set-up/index.md) in the NeMo Platform documentation. +For deployment guide, see [Admin Setup](/documentation/self-managed-deployment/setup/about) in the NeMo Platform documentation. ## Values diff --git a/docs/safe-synthesizer/about/data-synthesis.mdx b/docs/safe-synthesizer/about/data-synthesis.mdx index 403d0d3e5d1..a9bc08647ce 100644 --- a/docs/safe-synthesizer/about/data-synthesis.mdx +++ b/docs/safe-synthesizer/about/data-synthesis.mdx @@ -95,7 +95,7 @@ Enabling DP provides strong privacy guarantees but affects synthetic data qualit **Data size:** DP performs best with 10,000+ training records. Smaller datasets may experience significant quality degradation due to the noise required for privacy guarantees. -For hands-on guidance, refer to [differential-privacy](../tutorials/differential-privacy.md). For complete parameter documentation, refer to [reference](reference.md). +For hands-on guidance, refer to [differential-privacy](/documentation/safe-synthesizer-reference/tutorials/differential-privacy). For complete parameter documentation, refer to [reference](/documentation/safe-synthesizer-reference/about/parameters-reference). ## Configuration @@ -105,10 +105,10 @@ Synthesis behavior is controlled through configuration parameters: - **Generation**: Number of records, temperature, sampling strategies - **Privacy**: Differential privacy parameters (epsilon, delta, clipping) -For a complete list of all available parameters and their defaults, refer to [reference](reference.md). +For a complete list of all available parameters and their defaults, refer to [reference](/documentation/safe-synthesizer-reference/about/parameters-reference). ## Related Topics -- [reference](reference.md): Complete parameter reference -- [differential-privacy](../tutorials/differential-privacy.md): Learn about differential privacy in practice -- [index](../tutorials/index.md): More tutorials +- [reference](/documentation/safe-synthesizer-reference/about/parameters-reference): Complete parameter reference +- [differential-privacy](/documentation/safe-synthesizer-reference/tutorials/differential-privacy): Learn about differential privacy in practice +- [index](/documentation/safe-synthesizer-reference/tutorials/overview): More tutorials diff --git a/docs/safe-synthesizer/about/evaluation.mdx b/docs/safe-synthesizer/about/evaluation.mdx index 29dd3573f25..fa1ca766a24 100644 --- a/docs/safe-synthesizer/about/evaluation.mdx +++ b/docs/safe-synthesizer/about/evaluation.mdx @@ -88,7 +88,7 @@ You should expect some PII replay, and it is often not a cause for concern. We t - Lower epsilon (stronger privacy) generally yields higher DPS scores - Enabling DP can reduce **SQS** due to privacy-utility tradeoff (noise affects quality) -For differential privacy configuration and privacy-utility tradeoffs, refer to [data-synthesis](data-synthesis.md) and [differential-privacy](../tutorials/differential-privacy.md). +For differential privacy configuration and privacy-utility tradeoffs, refer to [data-synthesis](/documentation/safe-synthesizer-reference/about/data-synthesis) and [differential-privacy](/documentation/safe-synthesizer-reference/tutorials/differential-privacy). ## Evaluation Reports @@ -125,5 +125,5 @@ We enable evaluation by default. ## Related Topics - [safe-synthesizer-101](/documentation/safe-synthesizer/tutorials/safe-synthesizer-101): Get started with evaluation -- [differential-privacy](../tutorials/differential-privacy.md): Learn about privacy metrics -- [index](../tutorials/index.md): More tutorials +- [differential-privacy](/documentation/safe-synthesizer-reference/tutorials/differential-privacy): Learn about privacy metrics +- [index](/documentation/safe-synthesizer-reference/tutorials/overview): More tutorials diff --git a/docs/safe-synthesizer/about/host-local-development.mdx b/docs/safe-synthesizer/about/host-local-development.mdx index 2ba8ad393da..a683fc30c2a 100644 --- a/docs/safe-synthesizer/about/host-local-development.mdx +++ b/docs/safe-synthesizer/about/host-local-development.mdx @@ -10,7 +10,7 @@ Run NeMo Safe Synthesizer on your machine's GPU with `nemo safe-synthesizer run- ## Prerequisites -- CUDA-capable NVIDIA GPU on the host (80GB+ VRAM recommended; check with `nvidia-smi`). See [Getting Started](../getting-started.md). +- CUDA-capable NVIDIA GPU on the host (80GB+ VRAM recommended; check with `nvidia-smi`). See [Getting Started](/documentation/safe-synthesizer-reference/getting-started). - NeMo Platform repository checkout with the Safe Synthesizer plugin installed. - **No running platform required** for a typical local run when you pass `--data-source` — the NSS runtime can download base models from Hugging Face directly. @@ -164,7 +164,7 @@ Requires `RUN_NSS_LOCAL_E2E=1`, CUDA, and `nemo safe-synthesizer runtime setup`. ## Related topics -- [Parameters Reference](reference.md) — spec and `config` fields -- [Getting Started](../getting-started.md) — GPU and platform context -- [Jobs](jobs.md) — platform job lifecycle (separate from this run-local guide) +- [Parameters Reference](/documentation/safe-synthesizer-reference/about/parameters-reference) — spec and `config` fields +- [Getting Started](/documentation/safe-synthesizer-reference/getting-started) — GPU and platform context +- [Jobs](/documentation/safe-synthesizer-reference/about/jobs) — platform job lifecycle (separate from this run-local guide) - Plugin README: `plugins/nemo-safe-synthesizer/README.md` diff --git a/docs/safe-synthesizer/about/index.mdx b/docs/safe-synthesizer/about/index.mdx index efd99b23cad..714a00daf14 100644 --- a/docs/safe-synthesizer/about/index.mdx +++ b/docs/safe-synthesizer/about/index.mdx @@ -7,8 +7,8 @@ description: "" NVIDIA NeMo Safe Synthesizer enables you to create private versions of sensitive tabular datasets. The resulting data is entirely synthetic, with no one-to-one mapping to your original records. NeMo Safe Synthesizer is purpose-built for privacy compliance and data protection while preserving data utility for downstream AI tasks. - - + + --- @@ -23,17 +23,17 @@ A complete NeMo Safe Synthesizer job consists of the following steps: 1. [Upload Data](/documentation/get-started/core-concepts/manage-files): Add your tabular data to the Files API 2. Prepare Data: - - [Configure PII Replacement](pii-replacement.md): Set up detection and replacement of sensitive information (recommended prior to the Synthesis step to ensure the model has no chance of learning the most sensitive information like names and addresses) + - [Configure PII Replacement](/documentation/safe-synthesizer-reference/about/pii-replacement): Set up detection and replacement of sensitive information (recommended prior to the Synthesis step to ensure the model has no chance of learning the most sensitive information like names and addresses) - Configure training data organization and holdout splits 3. Configure Synthesis: - - [Training](data-synthesis.md): Set model selection and training parameters including differential privacy + - [Training](/documentation/safe-synthesizer-reference/about/data-synthesis): Set model selection and training parameters including differential privacy - Generate synthetic records - - [Evaluation](evaluation.md): Assess quality and privacy + - [Evaluation](/documentation/safe-synthesizer-reference/about/evaluation): Assess quality and privacy 4. Execute and Review: - - [Run and Monitor Job](jobs.md): Execute the job and track progress + - [Run and Monitor Job](/documentation/safe-synthesizer-reference/about/jobs): Execute the job and track progress - Download synthetic data and evaluation reports -Find all Safe Synthesizer configuration parameters in [Parameters Reference](reference.md). +Find all Safe Synthesizer configuration parameters in [Parameters Reference](/documentation/safe-synthesizer-reference/about/parameters-reference). --- @@ -43,14 +43,14 @@ Try out this early access API using Docker Compose or deploying the NeMo Platfor - + Get started with the NeMo Safe Synthesizer microservice locally using the NeMo Platform CLI. Easiest for local testing. standalone - + Deploy the NeMo Platform Helm Chart, which includes NeMo Safe Synthesizer. @@ -68,7 +68,7 @@ Get hands-on experience with Safe Synthesizer through step-by-step tutorials. - + Learn how to use NeMo Safe Synthesizer with hands-on tutorials covering basics to advanced topics. @@ -84,32 +84,32 @@ Learn how to use NeMo Safe Synthesizer with hands-on tutorials covering basics t - + Learn about LLM-based synthesis, differential privacy, and tabular fine-tuning for generating synthetic data. - + Understand how PII detection and replacement works to protect sensitive information before synthesis. - + Learn about quality and privacy metrics used to assess synthetic data including SQS and DPS scores. - + Understand the job lifecycle, configuration, and execution for Safe Synthesizer pipelines. - + Run on a host GPU with `nemo safe-synthesizer run-local` and `runtime` commands; reuse local adapters and run plugin tests. - + Reference all configuration parameters available when creating Safe Synthesizer jobs. diff --git a/docs/safe-synthesizer/about/jobs.mdx b/docs/safe-synthesizer/about/jobs.mdx index 8ae0a5eef88..d3610140d8b 100644 --- a/docs/safe-synthesizer/about/jobs.mdx +++ b/docs/safe-synthesizer/about/jobs.mdx @@ -112,7 +112,7 @@ When the job completes, access: For **platform jobs**, set `pretrained_model_job` in the job spec to a completed job that has an **`adapter`** result in Files. Reuse is generation-only (no retraining). Use either `pretrained_model_job` or `config.training.pretrained_model`, not both. -For **host-local** development (`nemo safe-synthesizer run-local`), set `config.training.pretrained_model` to a local adapter or work directory from an earlier run. See [Host-Local Development and Testing](host-local-development.md). +For **host-local** development (`nemo safe-synthesizer run-local`), set `config.training.pretrained_model` to a local adapter or work directory from an earlier run. See [Host-Local Development and Testing](/documentation/safe-synthesizer-reference/about/host-local-development). ## Job Builder API @@ -312,7 +312,7 @@ kubectl get events -n --sort-by='.lastTimestamp' ## Related Topics -- [Host-Local Development and Testing](host-local-development.md): `run-local`, adapter reuse, and plugin tests +- [Host-Local Development and Testing](/documentation/safe-synthesizer-reference/about/host-local-development): `run-local`, adapter reuse, and plugin tests - [safe-synthesizer-101](/documentation/safe-synthesizer/tutorials/safe-synthesizer-101): Get started with NeMo Safe Synthesizer jobs -- [index](../tutorials/index.md): More hands-on tutorials -- [reference](reference.md): Full parameter reference +- [index](/documentation/safe-synthesizer-reference/tutorials/overview): More hands-on tutorials +- [reference](/documentation/safe-synthesizer-reference/about/parameters-reference): Full parameter reference diff --git a/docs/safe-synthesizer/about/pii-replacement.mdx b/docs/safe-synthesizer/about/pii-replacement.mdx index d708739554c..c2170c061da 100644 --- a/docs/safe-synthesizer/about/pii-replacement.mdx +++ b/docs/safe-synthesizer/about/pii-replacement.mdx @@ -175,7 +175,7 @@ Beyond these built-in types, you can define custom entities using: ## Configuration -PII replacement is configured through the `replace_pii` section. For the full schema, refer to [reference](reference.md). +PII replacement is configured through the `replace_pii` section. For the full schema, refer to [reference](/documentation/safe-synthesizer-reference/about/parameters-reference). ```json { @@ -218,4 +218,4 @@ PII replacement is always recommended as a preprocessing step before synthesis. ## Related Topics - [safe-synthesizer-101](/documentation/safe-synthesizer/tutorials/safe-synthesizer-101): Getting started tutorial with PII replacement -- [index](../tutorials/index.md): More tutorials +- [index](/documentation/safe-synthesizer-reference/tutorials/overview): More tutorials diff --git a/docs/safe-synthesizer/about/reference.mdx b/docs/safe-synthesizer/about/reference.mdx index be325f5f1e5..520f1eba93f 100644 --- a/docs/safe-synthesizer/about/reference.mdx +++ b/docs/safe-synthesizer/about/reference.mdx @@ -20,7 +20,7 @@ Top-level fields on the Safe Synthesizer job spec (alongside `config`): | `pretrained_model_job` | Prior completed job whose **`adapter`** result in Files is reused for **generation-only** synthesis. Format: `` or `/`. Mutually exclusive with `config.training.pretrained_model`. | | `hf_token_secret` | Platform secret name for Hugging Face token during model initialization | -For host-local runs, see [Host-Local Development and Testing](host-local-development.md). Reuse a local adapter with `config.training.pretrained_model`, not `pretrained_model_job`. +For host-local runs, see [Host-Local Development and Testing](/documentation/safe-synthesizer-reference/about/host-local-development). Reuse a local adapter with `config.training.pretrained_model`, not `pretrained_model_job`. ## Top-Level Configuration @@ -79,7 +79,7 @@ Configuration for synthetic data quality and privacy assessment, including MIA, ## PII Replacement Configuration -Configuration for PII detection and replacement. See [pii-replacement](../about/pii-replacement.md) for conceptual documentation. +Configuration for PII detection and replacement. See [pii-replacement](/documentation/safe-synthesizer-reference/about/pii-replacement) for conceptual documentation. ### Column Classification Config (`replace_pii.globals.classify`) @@ -131,6 +131,6 @@ job = builder.create_job(name="my-job", project="my-project") ## Related Topics -- [data-synthesis](../about/data-synthesis.md) - Learn about synthesis concepts -- [evaluation](../about/evaluation.md) - Learn about evaluation metrics -- [index](../tutorials/index.md) - Hands-on tutorials +- [data-synthesis](/documentation/safe-synthesizer-reference/about/data-synthesis) - Learn about synthesis concepts +- [evaluation](/documentation/safe-synthesizer-reference/about/evaluation) - Learn about evaluation metrics +- [index](/documentation/safe-synthesizer-reference/tutorials/overview) - Hands-on tutorials diff --git a/docs/safe-synthesizer/getting-started.mdx b/docs/safe-synthesizer/getting-started.mdx index f5ac9f0ade9..91902a7071e 100644 --- a/docs/safe-synthesizer/getting-started.mdx +++ b/docs/safe-synthesizer/getting-started.mdx @@ -8,21 +8,21 @@ Get started with NeMo Safe Synthesizer for generating private synthetic versions ## Prerequisites -Before using NeMo Safe Synthesizer, complete the [NeMo Platform Quickstart](../get-started/quickstart.md) to install the CLI/SDK and deploy the platform. +Before using NeMo Safe Synthesizer, complete the [NeMo Platform Quickstart](/documentation/get-started) to install the CLI/SDK and deploy the platform. NeMo Safe Synthesizer has the following additional requirements: - An NVIDIA GPU **on the host machine** with 80GB+ VRAM (check with `nvidia-smi`). This is separate from any GPU inside a NIM container — Safe Synthesizer training runs directly on the host. - Sufficient disk space for generated datasets (50GB+ recommended) -For general platform troubleshooting (port conflicts, health checks, and so on), refer to the [main quickstart guide](../get-started/quickstart.md). +For general platform troubleshooting (port conflicts, health checks, and so on), refer to the [main quickstart guide](/documentation/get-started). --- ## Host-local CLI -For GPU development on your machine, install the Safe Synthesizer plugin from this repository and use `nemo safe-synthesizer run-local` (see [Host-Local Development and Testing](about/host-local-development.md)): +For GPU development on your machine, install the Safe Synthesizer plugin from this repository and use `nemo safe-synthesizer run-local` (see [Host-Local Development and Testing](/documentation/safe-synthesizer-reference/about/host-local-development)): ```shell BOOTSTRAP_LOCAL_PLUGIN_DIRS=plugins/nemo-safe-synthesizer make bootstrap-python @@ -33,15 +33,15 @@ uv run nemo safe-synthesizer run-local \ --output-dir ./nss-output ``` -Platform job submission (Jobs API, Studio, tutorials) is documented separately in [Jobs](about/jobs.md) and the [tutorials](tutorials/index.md). The `nemo safe-synthesizer` CLI today exposes **run-local** and **runtime** only. +Platform job submission (Jobs API, Studio, tutorials) is documented separately in [Jobs](/documentation/safe-synthesizer-reference/about/jobs) and the [tutorials](/documentation/safe-synthesizer-reference/tutorials/overview). The `nemo safe-synthesizer` CLI today exposes **run-local** and **runtime** only. --- ## Next Steps -Run one of the [tutorials](tutorials/index.md) to create your first synthetic dataset: +Run one of the [tutorials](/documentation/safe-synthesizer-reference/tutorials/overview) to create your first synthetic dataset: - [Safe Synthesizer 101 Tutorial](/documentation/safe-synthesizer/tutorials/safe-synthesizer-101) - A beginner-friendly introduction -- [Differential Privacy Tutorial](tutorials/differential-privacy.md) - Generate differentially-private synthetic data +- [Differential Privacy Tutorial](/documentation/safe-synthesizer-reference/tutorials/differential-privacy) - Generate differentially-private synthetic data --- diff --git a/docs/safe-synthesizer/tutorials/differential-privacy.mdx b/docs/safe-synthesizer/tutorials/differential-privacy.mdx index 2ac5ba62eb7..3872107f37d 100644 --- a/docs/safe-synthesizer/tutorials/differential-privacy.mdx +++ b/docs/safe-synthesizer/tutorials/differential-privacy.mdx @@ -12,7 +12,7 @@ If you have not yet completed the [Safe Synthesizer 101](/documentation/safe-syn ## Prerequisites -- Understanding of [differential privacy](../about/data-synthesis.md) +- Understanding of [differential privacy](/documentation/safe-synthesizer-reference/about/data-synthesis) - Safe Synthesizer deployment with GPU resources --- diff --git a/docs/safe-synthesizer/tutorials/index.mdx b/docs/safe-synthesizer/tutorials/index.mdx index 4b008c53eef..eb77aec049e 100644 --- a/docs/safe-synthesizer/tutorials/index.mdx +++ b/docs/safe-synthesizer/tutorials/index.mdx @@ -10,7 +10,7 @@ Learn how to run NeMo Safe Synthesizer jobs through hands-on tutorials to genera Before starting any tutorial, ensure you have: -- [NeMo Safe Synthesizer deployed](../getting-started.md) using Docker Compose or Helm +- [NeMo Safe Synthesizer deployed](/documentation/safe-synthesizer-reference/getting-started) using Docker Compose or Helm - Python environment with `nemo-platform` SDK installed: ```bash pip install nemo-platform[all] @@ -46,7 +46,7 @@ Learn the basics with your first Safe Synthesizer job, leveraging smart defaults - + Apply differential privacy to achieve the maximum level of privacy with mathematical guarantees. This tutorial explores the privacy-utility tradeoff and how to configure differential privacy parameters. @@ -69,11 +69,11 @@ Apply differential privacy to achieve the maximum level of privacy with mathemat After completing these tutorials, explore: -- **[index](../about/index.md)**: Understand core concepts and components +- **[index](/documentation/safe-synthesizer-reference/about/overview)**: Understand core concepts and components --- ## Need Help? - Check the [GitHub Issues](https://github.com/NVIDIA/GenerativeAIExamples/issues) for known issues -- Review the [jobs](../about/jobs.md) guide for job management +- Review the [jobs](/documentation/safe-synthesizer-reference/about/jobs) guide for job management diff --git a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx index f3eb2cd9937..844677399ae 100644 --- a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx +++ b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx @@ -10,7 +10,7 @@ Learn the fundamentals of NeMo Safe Synthesizer by creating your first Safe Synt Before you begin, make sure that you have: -- Access to a deployment of NeMo Safe Synthesizer (see [getting-started](../getting-started.md)) +- Access to a deployment of NeMo Safe Synthesizer (see [getting-started](/documentation/safe-synthesizer-reference/getting-started)) - **An NVIDIA GPU with 80 GB+ VRAM** — Safe Synthesizer requires GPU access for model training, even when using remote inference for other services. Verify with `nvidia-smi`. - Python environment with `nemo-platform` SDK installed - Basic understanding of Python and pandas @@ -309,7 +309,7 @@ job.display_report_in_notebook() ### Interpreting Scores -The evaluation report contains two high-level scores: Synthetic Quality Score (SQS) and Data Privacy Score (DPS). Both are measured out of 10, and higher is better. To learn more about how to interpret the scores, refer to the [evaluation guide](../about/evaluation.md). +The evaluation report contains two high-level scores: Synthetic Quality Score (SQS) and Data Privacy Score (DPS). Both are measured out of 10, and higher is better. To learn more about how to interpret the scores, refer to the [evaluation guide](/documentation/safe-synthesizer-reference/about/evaluation). --- @@ -319,11 +319,11 @@ Now that you've completed your first Safe Synthesizer job, explore more advanced ### Advanced Tutorials -- [Differential Privacy Tutorial](differential-privacy.md) - Apply mathematical privacy guarantees +- [Differential Privacy Tutorial](/documentation/safe-synthesizer-reference/tutorials/differential-privacy) - Apply mathematical privacy guarantees ### Documentation -- [index](../about/index.md) - Understand core concepts +- [index](/documentation/safe-synthesizer-reference/about/overview) - Understand core concepts ### Try These Next @@ -372,7 +372,7 @@ print(f"Total jobs: {len(all_jobs.data)}") - Use smaller model (adjust `training.pretrained_model`) - Check GPU availability -For more help, see [jobs](../about/jobs.md). +For more help, see [jobs](/documentation/safe-synthesizer-reference/about/jobs). **Error: "Dataset must have at least 200 records to use holdout."** diff --git a/docs/set-up/helm/file-storage.mdx b/docs/set-up/helm/file-storage.mdx index a9e60517315..dc155f1a820 100644 --- a/docs/set-up/helm/file-storage.mdx +++ b/docs/set-up/helm/file-storage.mdx @@ -19,7 +19,7 @@ If using S3 storage, you must provision and manage your own bucket and credentia ## Local Storage (Default) -By default, the Files service uses local filesystem storage. Files are stored on the shared PVC configured in [Persistent Volumes](./persistent-volumes.md). +By default, the Files service uses local filesystem storage. Files are stored on the shared PVC configured in [Persistent Volumes](/documentation/self-managed-deployment/setup/helm/persistent-volumes). The default configuration is equivalent to: @@ -31,7 +31,7 @@ platformConfig: path: /vol/files ``` -You do not need to add this to your `values.yaml`. Once the PVC is set up as described in [Persistent Volumes](./persistent-volumes.md), local storage works out of the box. +You do not need to add this to your `values.yaml`. Once the PVC is set up as described in [Persistent Volumes](/documentation/self-managed-deployment/setup/helm/persistent-volumes), local storage works out of the box. ## S3 Object Storage diff --git a/docs/set-up/helm/index.mdx b/docs/set-up/helm/index.mdx index 859ca343569..cf54bf85b32 100644 --- a/docs/set-up/helm/index.mdx +++ b/docs/set-up/helm/index.mdx @@ -8,63 +8,63 @@ The NeMo Platform is bundled in an all-in-one Helm chart that supports end-to-en - + Review the prerequisites for installing the \{\{helm_chart_short_name}}. cluster-admin - + Install the NeMo Platform using the Helm chart. cluster-admin on-prem cloud - + Set up an external database for the NeMo Platform. cluster-admin on-prem cloud - + Set up Ingress for the NeMo Platform. cluster-admin on-prem cloud - + Set up persistent volumes for the NeMo Platform. cluster-admin on-prem cloud - + Configure storage options for the Files service. cluster-admin on-prem cloud - + Configure high-performance east-west networking (EFA, InfiniBand, TCP-XO, SR-IOV) for multi-node training. cluster-admin cloud - + Install with OpenShift-compatible configuration. cluster-admin openshift - + Set up backup and restore configurations for the NeMo Platform. diff --git a/docs/set-up/helm/ingress.mdx b/docs/set-up/helm/ingress.mdx index b95c1702696..0fa0b46438f 100644 --- a/docs/set-up/helm/ingress.mdx +++ b/docs/set-up/helm/ingress.mdx @@ -6,7 +6,7 @@ description: "" The NeMo Platform Helm chart can expose the API service externally using **Kubernetes Ingress**, the **Gateway API HTTPRoute**, or on OpenShift an **OpenShift Route**. Choose one based on your cluster. -**Prerequisites:** Complete [Prerequisites](./prerequisites.md) and [Install](./install.md) (or [OpenShift](./openshift.md)) so the platform is installed. Ensure your cluster has an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) controller (e.g. Traefik, OpenShift IngressController) or a [Gateway API](https://gateway-api.sigs.k8s.io/) Gateway configured. +**Prerequisites:** Complete [Prerequisites](/documentation/self-managed-deployment/setup/helm/prerequisites) and [Install](/documentation/self-managed-deployment/setup/helm/install) (or [OpenShift](/documentation/self-managed-deployment/setup/helm/open-shift)) so the platform is installed. Ensure your cluster has an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) controller (e.g. Traefik, OpenShift IngressController) or a [Gateway API](https://gateway-api.sigs.k8s.io/) Gateway configured. ## Kubernetes Ingress @@ -49,7 +49,7 @@ To use standard Kubernetes [Ingress](https://kubernetes.io/docs/concepts/service Install or upgrade with your values file so the Ingress resource is created. After installation, use the URL shown in the Helm notes (`helm status nemo-platform`) or the host you configured. -**Advanced:** For multiple hostnames or different paths per host, leave `defaultHost` unset and use `ingress.hosts` (each entry has `name` and `paths`). See the [\{\{helm_chart_short_name}} reference](../../helm/index.md) for the full structure. +**Advanced:** For multiple hostnames or different paths per host, leave `defaultHost` unset and use `ingress.hosts` (each entry has `name` and `paths`). See the [\{\{helm_chart_short_name}} reference](/documentation/self-managed-deployment/helm/helm-reference) for the full structure. ## Gateway API HTTPRoute @@ -69,7 +69,7 @@ On clusters that use the [Gateway API](https://gateway-api.sigs.k8s.io/), you ca 2. Install or upgrade with this values file. The HTTPRoute will be created and bound to the specified Gateway(s). -For full option details (e.g. `filters`, `labels`, `annotations`), see the [\{\{helm_chart_short_name}} reference](../../helm/index.md). +For full option details (e.g. `filters`, `labels`, `annotations`), see the [\{\{helm_chart_short_name}} reference](/documentation/self-managed-deployment/helm/helm-reference). ## OpenShift Route @@ -89,12 +89,12 @@ On Red Hat OpenShift you can expose the API using an [OpenShift Route](https://d 3. Install or upgrade with your values file. After installation, use the URL from the Helm notes (`helm status nemo-platform`) or the host you configured. -For all options (`targetPort`, `annotations`, `labels`), see the [\{\{helm_chart_short_name}} reference](../../helm/index.md). +For all options (`targetPort`, `annotations`, `labels`), see the [\{\{helm_chart_short_name}} reference](/documentation/self-managed-deployment/helm/helm-reference). ## Related -- [Install](./install.md) — Install steps and upgrade commands -- [OpenShift](./openshift.md) — OpenShift: security context, and optional Route (`openshiftRoute.enabled`), Ingress, or HTTPRoute +- [Install](/documentation/self-managed-deployment/setup/helm/install) — Install steps and upgrade commands +- [OpenShift](/documentation/self-managed-deployment/setup/helm/open-shift) — OpenShift: security context, and optional Route (`openshiftRoute.enabled`), Ingress, or HTTPRoute ## Cloud Provider Specific Ingress diff --git a/docs/set-up/helm/install.mdx b/docs/set-up/helm/install.mdx index 68e3aaa8f5d..a13d34fadf2 100644 --- a/docs/set-up/helm/install.mdx +++ b/docs/set-up/helm/install.mdx @@ -10,7 +10,7 @@ Note: This setup is the full enterprise platform, meant for advanced use. If you -To deploy the NeMo Platform, follow these steps after completing the [Prerequisites](./prerequisites.md). +To deploy the NeMo Platform, follow these steps after completing the [Prerequisites](/documentation/self-managed-deployment/setup/helm/prerequisites). 1. Add the \{\{helm_chart_short_name}} to your local Helm repositories. @@ -23,14 +23,14 @@ To deploy the NeMo Platform, follow these steps after completing the [Prerequisi helm repo update ``` -2. Review the default values in the [\{\{helm_chart_short_name}} reference](../../helm/index.md). To override the default values, create a custom values file. Review the following while creating your custom values file. +2. Review the default values in the [\{\{helm_chart_short_name}} reference](/documentation/self-managed-deployment/helm/helm-reference). To override the default values, create a custom values file. Review the following while creating your custom values file. - - To configure an external database, see [Database Setup](./database-setup.md). - - To configure persistent volumes for jobs and files storage, see [Persistent Volumes](./persistent-volumes.md). - - To configure file storage options, see [File Storage](./file-storage.md). - - To configure ingress, see [Ingress](./ingress.md). - - To configure multi-node networking, see [Multi-Node Networking](./multinode-networking.md). - - To configure OpenShift-compatible security context overrides, see [OpenShift](./openshift.md). + - To configure an external database, see [Database Setup](/documentation/self-managed-deployment/setup/helm/database-setup). + - To configure persistent volumes for jobs and files storage, see [Persistent Volumes](/documentation/self-managed-deployment/setup/helm/persistent-volumes). + - To configure file storage options, see [File Storage](/documentation/self-managed-deployment/setup/helm/file-storage). + - To configure ingress, see [Ingress](/documentation/self-managed-deployment/setup/helm/ingress). + - To configure multi-node networking, see [Multi-Node Networking](/documentation/self-managed-deployment/setup/helm/multinode-networking). + - To configure OpenShift-compatible security context overrides, see [OpenShift](/documentation/self-managed-deployment/setup/helm/open-shift). 3. Install the Volcano scheduler before installing the chart. This is required for customization jobs that leverage multiple nodes. @@ -63,7 +63,7 @@ To deploy the NeMo Platform, follow these steps after completing the [Prerequisi kubectl get pods ``` -For Red Hat OpenShift, use OpenShift-compatible security context overrides so pods satisfy the restricted SCC. See [OpenShift](./openshift.md). +For Red Hat OpenShift, use OpenShift-compatible security context overrides so pods satisfy the restricted SCC. See [OpenShift](/documentation/self-managed-deployment/setup/helm/open-shift). To upgrade the deployment with new configurations, use the following command: diff --git a/docs/set-up/helm/openshift.mdx b/docs/set-up/helm/openshift.mdx index f0d6cd9d882..4452f29d502 100644 --- a/docs/set-up/helm/openshift.mdx +++ b/docs/set-up/helm/openshift.mdx @@ -37,7 +37,7 @@ You can override the default values for the NeMo Platform chart to make it compa fsGroup: 999 ``` -2. **Install** with your custom values and the OpenShift overrides (order matters; later files override earlier). Complete the [Prerequisites](./prerequisites.md) and follow [Install](./install.md), using the provided OpenShift values file. +2. **Install** with your custom values and the OpenShift overrides (order matters; later files override earlier). Complete the [Prerequisites](/documentation/self-managed-deployment/setup/helm/prerequisites) and follow [Install](/documentation/self-managed-deployment/setup/helm/install), using the provided OpenShift values file. ```sh helm upgrade --install --namespace \ @@ -50,9 +50,9 @@ You can override the default values for the NeMo Platform chart to make it compa You can expose the API using the following methods: -- **Kubernetes Ingress** — supported by OpenShift’s default IngressController; set `ingress.enabled: true` and `ingress.defaultHost` as in [Ingress](./ingress.md). -- **Gateway API HTTPRoute** — optional; configure `httpRoute` in values. See [Gateway API HTTPRoute](./ingress.md#gateway-api-httproute). -- **OpenShift Route** — enable the chart’s Route and set an optional hostname. See [OpenShift Route](./ingress.md#openshift-route) for steps. +- **Kubernetes Ingress** — supported by OpenShift’s default IngressController; set `ingress.enabled: true` and `ingress.defaultHost` as in [Ingress](/documentation/self-managed-deployment/setup/helm/ingress). +- **Gateway API HTTPRoute** — optional; configure `httpRoute` in values. See [Gateway API HTTPRoute](/documentation/self-managed-deployment/setup/helm/ingress#gateway-api-httproute). +- **OpenShift Route** — enable the chart’s Route and set an optional hostname. See [OpenShift Route](/documentation/self-managed-deployment/setup/helm/ingress#openshift-route) for steps. ## Troubleshooting diff --git a/docs/set-up/helm/persistent-volumes.mdx b/docs/set-up/helm/persistent-volumes.mdx index 39839003e53..4d6c1d183a7 100644 --- a/docs/set-up/helm/persistent-volumes.mdx +++ b/docs/set-up/helm/persistent-volumes.mdx @@ -16,7 +16,7 @@ The NeMo Platform chart creates a single shared PVC for jobs and files storage, -As an alternative to PVC-based file storage, you can configure the Files service to use S3 object storage. See [File Storage](./file-storage.md) for S3 configuration options. When using S3 for files, the shared PVC is still required for jobs storage. +As an alternative to PVC-based file storage, you can configure the Files service to use S3 object storage. See [File Storage](/documentation/self-managed-deployment/setup/helm/file-storage) for S3 configuration options. When using S3 for files, the shared PVC is still required for jobs storage. ### Option 1: Create a new PVC (default) diff --git a/docs/set-up/helm/prerequisites.mdx b/docs/set-up/helm/prerequisites.mdx index ee77f6ac529..4fa92ced39f 100644 --- a/docs/set-up/helm/prerequisites.mdx +++ b/docs/set-up/helm/prerequisites.mdx @@ -12,7 +12,7 @@ Before installing the \{\{helm_chart_short_name}}, review requirements and creat To check the hardware and software specifications required for installing the \{\{helm_chart_short_name}}, review [](/documentation/reference/system-requirements). -For storage and persistent volume configuration (including ReadWriteMany and StorageClass configuration), see [](persistent-volumes.md). +For storage and persistent volume configuration (including ReadWriteMany and StorageClass configuration), see [](/documentation/self-managed-deployment/setup/helm/persistent-volumes). --- diff --git a/docs/set-up/index.mdx b/docs/set-up/index.mdx index 957a4e19ff5..cf6e0d4e830 100644 --- a/docs/set-up/index.mdx +++ b/docs/set-up/index.mdx @@ -30,7 +30,7 @@ The following sections provide detailed instructions on how to deploy the NeMo P - + Install the NeMo Platform using the chart on your Kubernetes cluster. @@ -48,12 +48,12 @@ Review and manage other cluster settings. - + Review and configure how NeMo Platform use Open Telemetry for observability. - + Review and configure how NeMo Platform uses Milvus for vector database storage. diff --git a/docs/set-up/manage-jobs.mdx b/docs/set-up/manage-jobs.mdx index 358c24ea496..0e8c19d4f6a 100644 --- a/docs/set-up/manage-jobs.mdx +++ b/docs/set-up/manage-jobs.mdx @@ -219,7 +219,7 @@ Volcano Jobs are configured using the same configuration as Kubernetes Jobs, but - `scheduler_name`: The Volcano scheduler to use for the job. - `plugins`: The Volcano plugins to use for the job. - `max_retry`: The maximum number of retries for the job. -- `enable_multi_node_networking`: Enable multi-node networking injection. Sets annotations to trigger Kyverno policy mutations. This is only available if the platform is configured to use multi-node networking (see [Multi-Node Networking](helm/multinode-networking.md)). +- `enable_multi_node_networking`: Enable multi-node networking injection. Sets annotations to trigger Kyverno policy mutations. This is only available if the platform is configured to use multi-node networking (see [Multi-Node Networking](/documentation/self-managed-deployment/setup/helm/multinode-networking)). ```yaml jobs: diff --git a/docs/set-up/milvus.mdx b/docs/set-up/milvus.mdx index 77bc266e1f8..337da76d81e 100644 --- a/docs/set-up/milvus.mdx +++ b/docs/set-up/milvus.mdx @@ -4,7 +4,7 @@ description: "" --- -NeMo Evaluator uses Milvus for vector database storage for [Retrieval evaluations](../evaluator/metrics/retriever.md) and [RAG evaluations](/documentation/evaluate-models/metrics/rag-metrics). +NeMo Evaluator uses Milvus for vector database storage for [Retrieval evaluations](/documentation/evaluate-models/metrics) and [RAG evaluations](/documentation/evaluate-models/metrics/rag-metrics). ## Configuration diff --git a/docs/set-up/opentelemetry.mdx b/docs/set-up/opentelemetry.mdx index bc59232d785..6fdcf6bcefd 100644 --- a/docs/set-up/opentelemetry.mdx +++ b/docs/set-up/opentelemetry.mdx @@ -22,4 +22,4 @@ telemetry: OTEL_EXPORTER_OTLP_INSECURE: true ``` -For a complete list of the default values, refer to [Helm Configuration](../helm/index.md). +For a complete list of the default values, refer to [Helm Configuration](/documentation/self-managed-deployment/helm/helm-reference). diff --git a/docs/troubleshooting/customizer.mdx b/docs/troubleshooting/customizer.mdx index b21993c7649..3b90b8b408c 100644 --- a/docs/troubleshooting/customizer.mdx +++ b/docs/troubleshooting/customizer.mdx @@ -11,7 +11,7 @@ description: "" - The platform's shared persistent volume is likely full. Customization jobs require significant disk space: ~3× model size for full SFT, ~1.5× for LoRA. If you are also deploying the model from a base checkpoint fileset, plan for ~2.5× model size overall. - Clean up completed job artifacts or increase the PVC size (default: 200Gi at `/var/run/scratch/job`). - DPO/GRPO jobs also consume ephemeral node storage under `/tmp` via Ray workers — check node disk in addition to the PVC. -- See [ft-tut-understand-models](../customizer/tutorials/understand-configurations-and-models.md) for full storage requirement details. +- See [ft-tut-understand-models](/documentation/customizer-reference/tutorials/understanding-models-and-training) for full storage requirement details. **Job fails with OOM (Out of Memory):** 1. Reduce `micro_batch_size` to 1 diff --git a/docs/troubleshooting/evaluator.mdx b/docs/troubleshooting/evaluator.mdx index eb458eba6b4..0de8e70ab43 100644 --- a/docs/troubleshooting/evaluator.mdx +++ b/docs/troubleshooting/evaluator.mdx @@ -8,7 +8,7 @@ Use this documentation to troubleshoot issues that can arise when you work with -You can [get metric logs](/documentation/evaluate-models/metrics/job-management) or [get benchmark logs](../evaluator/benchmarks/job-management.md) for `COMPLETED` or `FAILED` jobs and use them to help troubleshoot. +You can [get metric logs](/documentation/evaluate-models/metrics/job-management) or [get benchmark logs](/documentation/evaluator-benchmark-reference/benchmarks/job-management) for `COMPLETED` or `FAILED` jobs and use them to help troubleshoot. --- From 021809a13a7654c48fc4a1bfa6ba1625faac0a3f Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Thu, 4 Jun 2026 14:02:37 -0400 Subject: [PATCH 17/26] docs: inline remaining substitution variables (escaped {{ }} forms) The earlier substitution-inlining pass only matched unescaped {{var}}. The migration had escaped many tokens as \{\{var}} for MDX safety, so those leaked through and rendered literally (e.g. {{__auditor_long_name}} on the auditor pages, {{helm_chart_short_name}} across the Helm pages). Inline every defined substitution variable (full set from the original mkdocs.yml `extra` block + substitutions.json, 44 vars) in both escaped and unescaped forms: - __auditor_long_name -> NVIDIA NeMo Auditor, __auditor_short_name -> NeMo Auditor - helm_chart_short_name -> NeMo Platform Helm Chart (x17) - volcano_version -> 1.9.0 Residual scan confirms 0 known-variable tokens remain. Genuine literal content is untouched: Data Designer column refs ({{ first_name }}, {{ product_name }}) and eval prompt-template placeholders ({{ input }}, {{ score_name }}, {{ clinical_scenario }}) are not substitution variables and are left as-is. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Lawrence Lane --- docs/auditor/index.mdx | 4 ++-- docs/auditor/sdk-resources.mdx | 4 ++-- docs/auditor/tutorials/index.mdx | 2 +- docs/auditor/tutorials/run-audit-locally.mdx | 2 +- docs/set-up/helm/index.mdx | 2 +- docs/set-up/helm/ingress.mdx | 6 +++--- docs/set-up/helm/install.mdx | 6 +++--- docs/set-up/helm/prerequisites.mdx | 8 ++++---- docs/set-up/index.mdx | 8 ++++---- docs/set-up/opentelemetry.mdx | 2 +- docs/set-up/security.mdx | 4 ++-- 11 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/auditor/index.mdx b/docs/auditor/index.mdx index ea63b24b7b5..5606c99f451 100644 --- a/docs/auditor/index.mdx +++ b/docs/auditor/index.mdx @@ -6,11 +6,11 @@ description: Scan and audit large language models for jailbreaks, prompt injecti -\{\{__auditor_long_name}} is released with _early access_ availability and is subject to limited support and potential API changes in future releases. +NVIDIA NeMo Auditor is released with _early access_ availability and is subject to limited support and potential API changes in future releases. -\{\{__auditor_long_name}} audits LLMs by probing them with adversarial prompts and detecting failures such as jailbreaks, prompt injection, encoding bypasses, and unsafe output generation. +NVIDIA NeMo Auditor audits LLMs by probing them with adversarial prompts and detecting failures such as jailbreaks, prompt injection, encoding bypasses, and unsafe output generation. It is powered by [garak](https://github.com/NVIDIA/garak), NVIDIA's open-source LLM vulnerability scanner, and integrates with NeMo Platform so audits can target any model reachable through the Inference Gateway. diff --git a/docs/auditor/sdk-resources.mdx b/docs/auditor/sdk-resources.mdx index def3a1095d4..35d2ca3b299 100644 --- a/docs/auditor/sdk-resources.mdx +++ b/docs/auditor/sdk-resources.mdx @@ -5,7 +5,7 @@ description: "" -The \{\{__auditor_short_name}} plugin mounts a Python SDK surface on the `nemo_platform` client at `client.auditor`. +The NeMo Auditor plugin mounts a Python SDK surface on the `nemo_platform` client at `client.auditor`. This page documents that surface: how to manage audit configurations and targets in the entity store, and how to run an audit in-process using the local execution path. The CRUD methods exposed on `client.auditor.configs` and `client.auditor.targets` are 1:1 mirrors of the [audit configuration](/documentation/vulnerability-scanning/configurations) and [audit target](/documentation/vulnerability-scanning/targets) lifecycle and use the same `AuditConfig` and `AuditTarget` pydantic schemas the entity store persists. @@ -13,7 +13,7 @@ The CRUD methods exposed on `client.auditor.configs` and `client.auditor.targets ## AuditorPluginResource -The `AuditorPluginResource` is the sync SDK object for working with the \{\{__auditor_short_name}} plugin. +The `AuditorPluginResource` is the sync SDK object for working with the NeMo Auditor plugin. It is accessed directly from a `NeMoPlatform` instance: ```python diff --git a/docs/auditor/tutorials/index.mdx b/docs/auditor/tutorials/index.mdx index d9d106ba41a..3017c1077a2 100644 --- a/docs/auditor/tutorials/index.mdx +++ b/docs/auditor/tutorials/index.mdx @@ -4,7 +4,7 @@ description: "" --- -Use these tutorials to get hands-on with [\{\{__auditor_short_name}}](/documentation/vulnerability-scanning). +Use these tutorials to get hands-on with [NeMo Auditor](/documentation/vulnerability-scanning). ## Before You Start diff --git a/docs/auditor/tutorials/run-audit-locally.mdx b/docs/auditor/tutorials/run-audit-locally.mdx index 80d0fc4c13a..4c4784eb6e3 100644 --- a/docs/auditor/tutorials/run-audit-locally.mdx +++ b/docs/auditor/tutorials/run-audit-locally.mdx @@ -4,7 +4,7 @@ description: "" --- -This tutorial walks through running a single audit end-to-end with the \{\{__auditor_short_name}} plugin SDK. You will persist an audit configuration and a target, run the audit in-process, and inspect the resulting report artifacts. +This tutorial walks through running a single audit end-to-end with the NeMo Auditor plugin SDK. You will persist an audit configuration and a target, run the audit in-process, and inspect the resulting report artifacts. **What you will learn:** diff --git a/docs/set-up/helm/index.mdx b/docs/set-up/helm/index.mdx index cf54bf85b32..df9042d3ec5 100644 --- a/docs/set-up/helm/index.mdx +++ b/docs/set-up/helm/index.mdx @@ -10,7 +10,7 @@ The NeMo Platform is bundled in an all-in-one Helm chart that supports end-to-en -Review the prerequisites for installing the \{\{helm_chart_short_name}}. +Review the prerequisites for installing the NeMo Platform Helm Chart. cluster-admin diff --git a/docs/set-up/helm/ingress.mdx b/docs/set-up/helm/ingress.mdx index 0fa0b46438f..6b90953f98a 100644 --- a/docs/set-up/helm/ingress.mdx +++ b/docs/set-up/helm/ingress.mdx @@ -49,7 +49,7 @@ To use standard Kubernetes [Ingress](https://kubernetes.io/docs/concepts/service Install or upgrade with your values file so the Ingress resource is created. After installation, use the URL shown in the Helm notes (`helm status nemo-platform`) or the host you configured. -**Advanced:** For multiple hostnames or different paths per host, leave `defaultHost` unset and use `ingress.hosts` (each entry has `name` and `paths`). See the [\{\{helm_chart_short_name}} reference](/documentation/self-managed-deployment/helm/helm-reference) for the full structure. +**Advanced:** For multiple hostnames or different paths per host, leave `defaultHost` unset and use `ingress.hosts` (each entry has `name` and `paths`). See the [NeMo Platform Helm Chart reference](/documentation/self-managed-deployment/helm/helm-reference) for the full structure. ## Gateway API HTTPRoute @@ -69,7 +69,7 @@ On clusters that use the [Gateway API](https://gateway-api.sigs.k8s.io/), you ca 2. Install or upgrade with this values file. The HTTPRoute will be created and bound to the specified Gateway(s). -For full option details (e.g. `filters`, `labels`, `annotations`), see the [\{\{helm_chart_short_name}} reference](/documentation/self-managed-deployment/helm/helm-reference). +For full option details (e.g. `filters`, `labels`, `annotations`), see the [NeMo Platform Helm Chart reference](/documentation/self-managed-deployment/helm/helm-reference). ## OpenShift Route @@ -89,7 +89,7 @@ On Red Hat OpenShift you can expose the API using an [OpenShift Route](https://d 3. Install or upgrade with your values file. After installation, use the URL from the Helm notes (`helm status nemo-platform`) or the host you configured. -For all options (`targetPort`, `annotations`, `labels`), see the [\{\{helm_chart_short_name}} reference](/documentation/self-managed-deployment/helm/helm-reference). +For all options (`targetPort`, `annotations`, `labels`), see the [NeMo Platform Helm Chart reference](/documentation/self-managed-deployment/helm/helm-reference). ## Related diff --git a/docs/set-up/helm/install.mdx b/docs/set-up/helm/install.mdx index a13d34fadf2..ee5f98a8252 100644 --- a/docs/set-up/helm/install.mdx +++ b/docs/set-up/helm/install.mdx @@ -12,7 +12,7 @@ Note: This setup is the full enterprise platform, meant for advanced use. If you To deploy the NeMo Platform, follow these steps after completing the [Prerequisites](/documentation/self-managed-deployment/setup/helm/prerequisites). -1. Add the \{\{helm_chart_short_name}} to your local Helm repositories. +1. Add the NeMo Platform Helm Chart to your local Helm repositories. ```sh helm repo add nmp https://helm.ngc.nvidia.com/nvidia/nemo-microservices \ @@ -23,7 +23,7 @@ To deploy the NeMo Platform, follow these steps after completing the [Prerequisi helm repo update ``` -2. Review the default values in the [\{\{helm_chart_short_name}} reference](/documentation/self-managed-deployment/helm/helm-reference). To override the default values, create a custom values file. Review the following while creating your custom values file. +2. Review the default values in the [NeMo Platform Helm Chart reference](/documentation/self-managed-deployment/helm/helm-reference). To override the default values, create a custom values file. Review the following while creating your custom values file. - To configure an external database, see [Database Setup](/documentation/self-managed-deployment/setup/helm/database-setup). - To configure persistent volumes for jobs and files storage, see [Persistent Volumes](/documentation/self-managed-deployment/setup/helm/persistent-volumes). @@ -36,7 +36,7 @@ To deploy the NeMo Platform, follow these steps after completing the [Prerequisi ```sh - kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v{{volcano_version}}/installer/volcano-development.yaml + kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.9.0/installer/volcano-development.yaml ``` diff --git a/docs/set-up/helm/prerequisites.mdx b/docs/set-up/helm/prerequisites.mdx index 4fa92ced39f..44a7a099321 100644 --- a/docs/set-up/helm/prerequisites.mdx +++ b/docs/set-up/helm/prerequisites.mdx @@ -4,13 +4,13 @@ description: "" --- -Before installing the \{\{helm_chart_short_name}}, review requirements and create necessary secrets using your NGC API key. +Before installing the NeMo Platform Helm Chart, review requirements and create necessary secrets using your NGC API key. --- ## Review Requirements -To check the hardware and software specifications required for installing the \{\{helm_chart_short_name}}, review [](/documentation/reference/system-requirements). +To check the hardware and software specifications required for installing the NeMo Platform Helm Chart, review [](/documentation/reference/system-requirements). For storage and persistent volume configuration (including ReadWriteMany and StorageClass configuration), see [](/documentation/self-managed-deployment/setup/helm/persistent-volumes). @@ -19,7 +19,7 @@ For storage and persistent volume configuration (including ReadWriteMany and Sto ## Create NGC API Key and Secrets -You need to have an NGC account, create an NGC API key, and create secrets using the key to access and download the \{\{helm_chart_short_name}} and Docker images. +You need to have an NGC account, create an NGC API key, and create secrets using the key to access and download the NeMo Platform Helm Chart and Docker images. To learn more about using Helm charts in the NGC Catalog console in general, refer to the [Helm Charts](https://docs.nvidia.com/ngc/gpu-cloud/ngc-catalog-user-guide/index.html#helm-charts) section in the _NGC Catalog User Guide_. @@ -66,4 +66,4 @@ To learn more about using Helm charts in the NGC Catalog console in general, ref --namespace ``` -For more information about various secrets you might need to create and set up through the \{\{helm_chart_short_name}} depending on your use case, refer to [](/documentation/get-started/core-concepts/manage-secrets). +For more information about various secrets you might need to create and set up through the NeMo Platform Helm Chart depending on your use case, refer to [](/documentation/get-started/core-concepts/manage-secrets). diff --git a/docs/set-up/index.mdx b/docs/set-up/index.mdx index cf6e0d4e830..4b23de05460 100644 --- a/docs/set-up/index.mdx +++ b/docs/set-up/index.mdx @@ -4,7 +4,7 @@ description: "" --- -This section describes how to set up the NeMo Platform on your Kubernetes cluster using the \{\{helm_chart_short_name}}. +This section describes how to set up the NeMo Platform on your Kubernetes cluster using the NeMo Platform Helm Chart. With this chart, you can deploy the NeMo Platform as a full deployment or a subset of the APIs as you need. This Platform Setup chapter is for the following personas. @@ -14,9 +14,9 @@ This Platform Setup chapter is for the following personas. --- -## \{\{helm_chart_short_name}} +## NeMo Platform Helm Chart -The [\{\{helm_chart_short_name}}](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo-microservices/helm-charts/nemo-platform) is an all-in-one Helm chart that bundles the complete NeMo Platform ecosystem and all required dependencies for full platform deployment. +The [NeMo Platform Helm Chart](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo-microservices/helm-charts/nemo-platform) is an all-in-one Helm chart that bundles the complete NeMo Platform ecosystem and all required dependencies for full platform deployment. You can also customize the configuration of your installation by updating the `values.yaml` file. You can also use the pre-configured tags to install only specific microservices that you need. @@ -26,7 +26,7 @@ For the chart assets and additional details, refer to the [NeMo Platform Collect ## Deploy the NeMo Platform with Helm -The following sections provide detailed instructions on how to deploy the NeMo Platform using the \{\{helm_chart_short_name}}. +The following sections provide detailed instructions on how to deploy the NeMo Platform using the NeMo Platform Helm Chart. diff --git a/docs/set-up/opentelemetry.mdx b/docs/set-up/opentelemetry.mdx index 6fdcf6bcefd..35984af0ec6 100644 --- a/docs/set-up/opentelemetry.mdx +++ b/docs/set-up/opentelemetry.mdx @@ -12,7 +12,7 @@ The NeMo Platform uses OpenTelemetry to collect telemetry data from the platform ## Helm Configuration -The \{\{helm_chart_short_name}} `values.yaml` exposes OpenTelemetry SDK options to configure the platform deployment. For example, to enable OpenTelemetry for the platform: +The NeMo Platform Helm Chart `values.yaml` exposes OpenTelemetry SDK options to configure the platform deployment. For example, to enable OpenTelemetry for the platform: ```yaml telemetry: diff --git a/docs/set-up/security.mdx b/docs/set-up/security.mdx index a76ef912bbf..9d9d554ca48 100644 --- a/docs/set-up/security.mdx +++ b/docs/set-up/security.mdx @@ -44,7 +44,7 @@ The following table lists the default network ports for each microservice or def | 8080/TCP | Volcano Scheduler metrics | | 9009/TCP | NeMo Customizer callback | -By default, the \{\{helm_chart_short_name}} configures databases with the following network ports. +By default, the NeMo Platform Helm Chart configures databases with the following network ports. Alternatively, you can configure each microservice to use an external database during installation. | Network Port | Database | @@ -56,7 +56,7 @@ Alternatively, you can configure each microservice to use an external database d | 9091/TCP | Milvus metrics | | 19530/TCP | Milvus API | -By default, the \{\{helm_chart_short_name}} installs an open telemetry collector, which uses the following network ports: +By default, the NeMo Platform Helm Chart installs an open telemetry collector, which uses the following network ports: - 4317/TCP - 4318/TCP From 32cdf8c370e97c6f248185df3ec10825c1e2308e Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Thu, 4 Jun 2026 14:28:41 -0400 Subject: [PATCH 18/26] docs(fern): gate unready features by excluding them from the build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per stakeholder: unready features must be fully unavailable (404, not indexable), not just hidden from the sidebar. Fern's `hidden: true` still builds and serves the page, so it is not sufficient. Fern only builds pages referenced in the nav, so the correct mechanism is to omit them from versions/latest.yml entirely — matching the old MkDocs hide_unready_docs hook, which dropped the same files from the build. - Remove all gated trees/pages from versions/latest.yml (auth/, customizer/, safe-synthesizer/, set-up/ + helm/, evaluator/benchmarks/, evaluator/metrics/{job-management,results}, run-inference deploy-models, example-applications/, troubleshooting/{cluster-setup,customizer}). This also supersedes the earlier `hidden: true` sections. - Keep all gated .mdx files in the repo so they stay maintained. - Delink the 13 inbound references from visible pages into gated pages to plain text (gated pages aren't built, so a rewritten URL would 404). - Add gated-nav.yml: ready-to-paste nav blocks to re-publish each feature when it ships (reference only; Fern does not read it). Document the approach in README.md. Verified: gated pages 404 on a local fern dev build, shipped pages serve, `fern docs broken-links` reports 0, validate-mdx parses 193 files cleanly. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Lawrence Lane --- docs/evaluator/index.mdx | 2 +- docs/evaluator/metrics/agentic.mdx | 6 +- docs/evaluator/metrics/llm-as-a-judge.mdx | 2 +- docs/evaluator/metrics/manage-metrics.mdx | 4 +- docs/evaluator/metrics/remote.mdx | 2 +- docs/evaluator/metrics/similarity.mdx | 2 +- docs/fern/README.md | 10 + docs/fern/gated-nav.yml | 219 ++++++++++++++++++++ docs/fern/versions/latest.yml | 231 ---------------------- docs/guardrails/observability.mdx | 4 +- docs/troubleshooting/evaluator.mdx | 2 +- 11 files changed, 241 insertions(+), 243 deletions(-) create mode 100644 docs/fern/gated-nav.yml diff --git a/docs/evaluator/index.mdx b/docs/evaluator/index.mdx index 0e0868491b8..77351eba74a 100644 --- a/docs/evaluator/index.mdx +++ b/docs/evaluator/index.mdx @@ -146,7 +146,7 @@ Most teams get the best results by starting metric-first, then moving to benchma ## Where to Go Next -- For metric workflows, see [Metric Jobs](/documentation/evaluate-models/metrics) and [Metric Results](/documentation/evaluate-models/metrics/metric-results). +- For metric workflows, see [Metric Jobs](/documentation/evaluate-models/metrics) and Metric Results. - For full endpoint details, see the [Evaluator API Reference](/documentation/reference/api-reference). --- diff --git a/docs/evaluator/metrics/agentic.mdx b/docs/evaluator/metrics/agentic.mdx index 4c28a8d5868..269c1caea6f 100644 --- a/docs/evaluator/metrics/agentic.mdx +++ b/docs/evaluator/metrics/agentic.mdx @@ -1252,7 +1252,7 @@ result = job.get_result() print(result.aggregate_scores) ``` -Navigate to [Metrics Job Management](/documentation/evaluate-models/metrics/job-management) for more job lifecycle details. +Navigate to Metrics Job Management for more job lifecycle details. --- @@ -1282,9 +1282,9 @@ Use `RunConfig(limit_samples=...)` when you want to test a small slice of a larg - [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration) - Use agents (generic or NAT) as targets in online evaluation jobs -- [Agentic Benchmarks](/documentation/evaluate-models/benchmarks/agentic-benchmarks) - BFCL benchmark for tool-calling evaluation +- Agentic Benchmarks - BFCL benchmark for tool-calling evaluation - [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge) - Custom judge-based evaluation -- [Evaluation Results](/documentation/evaluate-models/metrics/metric-results) - Understanding results +- Evaluation Results - Understanding results - [RAG Metrics](/documentation/evaluate-models/metrics/rag-metrics) - RAGAS metrics for RAG pipelines diff --git a/docs/evaluator/metrics/llm-as-a-judge.mdx b/docs/evaluator/metrics/llm-as-a-judge.mdx index 50d19057ee0..a22caec4b2d 100644 --- a/docs/evaluator/metrics/llm-as-a-judge.mdx +++ b/docs/evaluator/metrics/llm-as-a-judge.mdx @@ -695,7 +695,7 @@ metric = { - [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) - Inline models vs model references -- [Evaluation Results](/documentation/evaluate-models/metrics/metric-results) - Understanding and downloading results +- Evaluation Results - Understanding and downloading results - [Agentic Evaluation](/documentation/evaluate-models/metrics/agentic-metrics) - Evaluate agent workflows - [RAG Evaluation](/documentation/evaluate-models/metrics/rag-metrics) - Evaluate retrieval-augmented generation diff --git a/docs/evaluator/metrics/manage-metrics.mdx b/docs/evaluator/metrics/manage-metrics.mdx index 3e2706670b8..0784c869c51 100644 --- a/docs/evaluator/metrics/manage-metrics.mdx +++ b/docs/evaluator/metrics/manage-metrics.mdx @@ -117,6 +117,6 @@ result = job.get_result() ## Related Topics -- [Metric Results](/documentation/evaluate-models/metrics/metric-results) - Work with `EvaluationResult`, aggregate scores, and row scores -- [Manage Metric Jobs](/documentation/evaluate-models/metrics/job-management) - Submit, monitor, reconnect to, and download job results +- Metric Results - Work with `EvaluationResult`, aggregate scores, and row scores +- Manage Metric Jobs - Submit, monitor, reconnect to, and download job results - [Similarity Metrics](/documentation/evaluate-models/metrics/similarity-metrics) - Configure exact match, F1, BLEU, ROUGE, and string/number checks diff --git a/docs/evaluator/metrics/remote.mdx b/docs/evaluator/metrics/remote.mdx index e1e8c2c246e..8886217f4d3 100644 --- a/docs/evaluator/metrics/remote.mdx +++ b/docs/evaluator/metrics/remote.mdx @@ -384,7 +384,7 @@ RemoteScore( -- [Evaluation Results](/documentation/evaluate-models/metrics/metric-results) - Understanding and downloading results +- Evaluation Results - Understanding and downloading results - [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge) - Use an LLM to evaluate outputs - [Agentic Evaluation](/documentation/evaluate-models/metrics/agentic-metrics) - Evaluate agent workflows diff --git a/docs/evaluator/metrics/similarity.mdx b/docs/evaluator/metrics/similarity.mdx index 27f016fda2b..75138076e1f 100644 --- a/docs/evaluator/metrics/similarity.mdx +++ b/docs/evaluator/metrics/similarity.mdx @@ -8,7 +8,7 @@ NeMo Platform offers [built-in metrics](/documentation/evaluate-models/metrics) Template functionality provides maximum flexibility for evaluating your models on proprietary, domain-specific, or novel tasks. You can bring your own datasets, define your own prompts and templates using [Jinja](https://github.com/pallets/jinja), and select the metrics that matter most for your use case. This approach is ideal when: -- You want to evaluate on tasks, data, or formats not covered by [industry benchmarks](/documentation/evaluate-models/benchmarks/industry-benchmarks) or built-in metrics. +- You want to evaluate on tasks, data, or formats not covered by industry benchmarks or built-in metrics. - You need to measure model performance using custom or business-specific criteria. - You want to experiment with new evaluation methodologies, metrics, or workflows. - You need to create custom prompts and templates for specific use cases. diff --git a/docs/fern/README.md b/docs/fern/README.md index 7d65c9e8cfd..e55e37cee8e 100644 --- a/docs/fern/README.md +++ b/docs/fern/README.md @@ -85,6 +85,16 @@ The REST API reference is generated natively by Fern from the OpenAPI spec — n Fern groups endpoints by their OpenAPI tag in the sidebar (Customizer, Evaluator, Guardrails, …), which replaces the old per-service filter chips. Link to it from other pages with the nav URL `/documentation/reference/api-reference`. +## Gated (unready) features + +Some features are not shipped yet and must be **fully excluded from the build** — not just hidden from the sidebar. Fern's `hidden: true` still builds and serves the page (reachable by direct URL and indexable), so it is **not** used for this. Instead, the gated pages are simply **left out of `versions/latest.yml`**: Fern only builds pages referenced in the navigation, so an omitted page is never built (it 404s and is not indexed). This matches the old MkDocs `hide_unready_docs` hook, which dropped the same files from the build. + +The gated `.mdx` files stay in the repo so they remain maintained. The gated trees today are: `auth/`, `customizer/`, `safe-synthesizer/`, `set-up/` + `helm/`, `evaluator/benchmarks/`, plus individual pages (`evaluator/metrics/{job-management,results}`, `run-inference/tutorials/deploy-models`, `example-applications/`, `troubleshooting/{cluster-setup,customizer}`, `get-started/quickstart`). + +Inbound links from visible pages into gated pages are **delinked to plain text** (not rewritten URLs), since the target is not built — otherwise they would be broken links. + +`gated-nav.yml` holds the ready-to-paste navigation blocks for the gated features (it is a reference only; Fern does not read it). **To publish a feature when it ships:** move its block from `gated-nav.yml` into `versions/latest.yml`, re-link the inbound references that were delinked (search the docs for the feature name), then run `npm run check` and `npx fern-api docs broken-links`. + ## CI and Publishing | Workflow | Trigger | Purpose | diff --git a/docs/fern/gated-nav.yml b/docs/fern/gated-nav.yml new file mode 100644 index 00000000000..893479493a4 --- /dev/null +++ b/docs/fern/gated-nav.yml @@ -0,0 +1,219 @@ +# Gated (unready) feature navigation — REFERENCE ONLY, not built by Fern. +# +# These features are not shipped yet. Their .mdx files live in the repo (so they stay +# maintained), but they are intentionally omitted from versions/latest.yml so Fern does +# not build or serve them (they 404 and are not indexed) — matching the old mkdocs +# hide_unready_docs behavior. +# +# To publish a feature when it ships: move its block below into versions/latest.yml at the +# desired position, then re-link the inbound references that were delinked to plain text +# (search the docs for the feature name). Run `npm run check` and `fern docs broken-links`. + +- section: Access Control + contents: + - page: Overview + path: ../../auth/authentication/index.mdx + - page: OIDC Setup + path: ../../auth/authentication/oidc.mdx + - page: Azure AD (Entra ID) + path: ../../auth/authentication/providers/azure-ad.mdx + - page: Generic OIDC + path: ../../auth/authentication/providers/generic.mdx + - page: Overview + path: ../../auth/authentication/providers/index.mdx + - page: Using Authentication + path: ../../auth/authentication/using-authentication.mdx + - page: API Scopes + path: ../../auth/authorization/api-scopes.mdx + - page: Overview + path: ../../auth/authorization/index.mdx + - page: Managing Access + path: ../../auth/authorization/managing-access.mdx + - page: Permissions Reference + path: ../../auth/authorization/permissions-reference.mdx + - page: Policy Engine + path: ../../auth/authorization/policy-engine.mdx + - page: Roles and Permissions + path: ../../auth/authorization/roles-and-permissions.mdx + - page: Concepts + path: ../../auth/concepts.mdx + - page: Configuration + path: ../../auth/deployment/configuration.mdx + - page: Credential Propagation + path: ../../auth/deployment/credential-propagation.mdx + - page: Gateway Integration + path: ../../auth/deployment/gateway.mdx + - page: Production Hardening + path: ../../auth/deployment/hardening.mdx + - page: Overview + path: ../../auth/index.mdx + - page: Security Model + path: ../../auth/security-model.mdx + - page: Troubleshooting + path: ../../auth/troubleshooting.mdx +- section: Fine-tune Models + contents: + - page: Customization Concepts + path: ../../customizer/about.mdx + - page: About + path: ../../customizer/index.mdx + - page: Cancel Job + path: ../../customizer/manage-customization-jobs/cancel-job.mdx + - page: Create Job + path: ../../customizer/manage-customization-jobs/create-job.mdx + - page: Customization Job Reference + path: ../../customizer/manage-customization-jobs/customization-job-reference.mdx + - page: Get Job Status + path: ../../customizer/manage-customization-jobs/get-job-status.mdx + - page: Training Configuration + path: ../../customizer/manage-customization-jobs/hyperparameters.mdx + - page: Overview + path: ../../customizer/manage-customization-jobs/index.mdx + - page: List Active Jobs + path: ../../customizer/manage-customization-jobs/list-active-jobs.mdx + - page: Create a Model FileSet + path: ../../customizer/manage-model-entities/create-fileset.mdx + - page: Create a Model Entity + path: ../../customizer/manage-model-entities/create-model-entity.mdx + - page: Overview + path: ../../customizer/manage-model-entities/index.mdx + - page: Dataset Format + path: ../../customizer/models/data-format.mdx + - page: Embedding + path: ../../customizer/models/embedding.mdx + - page: GPT-OSS + path: ../../customizer/models/gpt-oss.mdx + - page: Model Catalog + path: ../../customizer/models/index.mdx + - page: Llama Nemotron + path: ../../customizer/models/llama-nemotron.mdx + - page: Llama + path: ../../customizer/models/llama.mdx + - page: Mistral + path: ../../customizer/models/mistral.mdx + - page: Phi + path: ../../customizer/models/phi.mdx + - page: Qwen + path: ../../customizer/models/qwen.mdx + - page: Distillation Customization Job + path: ../../customizer/tutorials/distillation-customization-job.mdx + - page: Dpo Customization Job + path: ../../customizer/tutorials/dpo-customization-job.mdx + - page: Embedding Customization Job + path: ../../customizer/tutorials/embedding-customization-job.mdx + - page: Format Training Dataset + path: ../../customizer/tutorials/format-training-dataset.mdx + - page: Import HuggingFace Models + path: ../../customizer/tutorials/import-hf-model.mdx + - page: Overview + path: ../../customizer/tutorials/index.mdx + - page: Lora Customization Job + path: ../../customizer/tutorials/lora-customization-job.mdx + - page: Job Metrics + path: ../../customizer/tutorials/metrics.mdx + - page: Optimize Throughput + path: ../../customizer/tutorials/optimize-throughput.mdx + - page: Sft Customization Job + path: ../../customizer/tutorials/sft-customization-job.mdx + - page: Understanding Models and Training + path: ../../customizer/tutorials/understand-configurations-and-models.mdx +- section: Safe Synthesizer + contents: + - page: Data Synthesis + path: ../../safe-synthesizer/about/data-synthesis.mdx + - page: Evaluation + path: ../../safe-synthesizer/about/evaluation.mdx + - page: Host-Local Development + path: ../../safe-synthesizer/about/host-local-development.mdx + - page: Overview + path: ../../safe-synthesizer/about/index.mdx + - page: Jobs + path: ../../safe-synthesizer/about/jobs.mdx + - page: PII Replacement + path: ../../safe-synthesizer/about/pii-replacement.mdx + - page: Parameters Reference + path: ../../safe-synthesizer/about/reference.mdx + - page: Getting Started + path: ../../safe-synthesizer/getting-started.mdx + - page: Differential Privacy + path: ../../safe-synthesizer/tutorials/differential-privacy.mdx + - page: Overview + path: ../../safe-synthesizer/tutorials/index.mdx + - page: Safe Synthesizer 101 + path: ../../safe-synthesizer/tutorials/safe-synthesizer-101.mdx +- section: Self-Managed Deployment + contents: + - page: Helm Reference + path: ../../helm/index.mdx + - page: Config Reference + path: ../../set-up/config-reference.mdx + - page: Backup and Restore + path: ../../set-up/helm/backup-and-restore.mdx + - page: Database Setup + path: ../../set-up/helm/database-setup.mdx + - page: File Storage + path: ../../set-up/helm/file-storage.mdx + - page: Overview + path: ../../set-up/helm/index.mdx + - page: Ingress + path: ../../set-up/helm/ingress.mdx + - page: Install + path: ../../set-up/helm/install.mdx + - page: Multinode Networking + path: ../../set-up/helm/multinode-networking.mdx + - page: OpenShift + path: ../../set-up/helm/openshift.mdx + - page: Persistent Volumes + path: ../../set-up/helm/persistent-volumes.mdx + - page: Prerequisites + path: ../../set-up/helm/prerequisites.mdx + - page: About + path: ../../set-up/index.mdx + - page: Jobs + path: ../../set-up/manage-jobs.mdx + - page: Milvus + path: ../../set-up/milvus.mdx + - page: Observability + path: ../../set-up/opentelemetry.mdx + - page: Security + path: ../../set-up/security.mdx +- section: evaluator + contents: + - page: Agentic Benchmarks + path: ../../evaluator/benchmarks/agentic.mdx + - page: Custom Benchmarks + path: ../../evaluator/benchmarks/custom.mdx + - page: Discover Industry Benchmarks + path: ../../evaluator/benchmarks/discover-industry-benchmarks.mdx + - page: HuggingFace Secret + path: ../../evaluator/benchmarks/hf-secret.mdx + - page: Overview + path: ../../evaluator/benchmarks/index.mdx + - page: Industry Benchmarks + path: ../../evaluator/benchmarks/industry.mdx + - page: Job Management + path: ../../evaluator/benchmarks/job-management.mdx + - page: Manage Benchmarks + path: ../../evaluator/benchmarks/manage-benchmarks.mdx + - page: Benchmark Results + path: ../../evaluator/benchmarks/results.mdx + - page: Job Management + path: ../../evaluator/metrics/job-management.mdx + - page: Metric Results + path: ../../evaluator/metrics/results.mdx +- section: example-applications + contents: + - page: About + path: ../../example-applications/about.mdx + - page: Tool Calling + path: ../../example-applications/tool-calling.mdx +- section: run-inference + contents: + - page: Deploy Models + path: ../../run-inference/tutorials/deploy-models.mdx +- section: troubleshooting + contents: + - page: Cluster Setup + path: ../../troubleshooting/cluster-setup.mdx + - page: Customizer + path: ../../troubleshooting/customizer.mdx diff --git a/docs/fern/versions/latest.yml b/docs/fern/versions/latest.yml index da97732d478..62e77947384 100644 --- a/docs/fern/versions/latest.yml +++ b/docs/fern/versions/latest.yml @@ -38,8 +38,6 @@ navigation: - section: Tutorials path: ../../run-inference/tutorials/index.mdx contents: - - page: Deploy Models - path: ../../run-inference/tutorials/deploy-models.mdx - page: Run Inference path: ../../run-inference/tutorials/run-inference.mdx - section: Agents @@ -152,52 +150,6 @@ navigation: path: ../../evaluator/metrics/agent-configuration.mdx - page: Model Configuration path: ../../evaluator/metrics/model-configuration.mdx - - page: Job Management - path: ../../evaluator/metrics/job-management.mdx - - page: Metric Results - path: ../../evaluator/metrics/results.mdx - - section: Benchmarks - contents: - - page: Industry Benchmarks - path: ../../evaluator/benchmarks/industry.mdx - - page: Agentic Benchmarks - path: ../../evaluator/benchmarks/agentic.mdx - - page: Custom Benchmarks - path: ../../evaluator/benchmarks/custom.mdx - - page: HuggingFace Secret - path: ../../evaluator/benchmarks/hf-secret.mdx - - section: Fine-tune Models - contents: - - section: Tutorials - path: ../../customizer/tutorials/index.mdx - contents: - - page: Format Training Dataset - path: ../../customizer/tutorials/format-training-dataset.mdx - - page: Full SFT Customization - path: ../../customizer/tutorials/sft-customization-job.mdx - - page: LoRA Model Customization - path: ../../customizer/tutorials/lora-customization-job.mdx - - page: DPO Customization - path: ../../customizer/tutorials/dpo-customization-job.mdx - - page: Knowledge Distillation Customization - path: ../../customizer/tutorials/distillation-customization-job.mdx - - page: Embedding Model Customization - path: ../../customizer/tutorials/embedding-customization-job.mdx - - page: Optimize for Tokens/GPU Throughput - path: ../../customizer/tutorials/optimize-throughput.mdx - - section: Models - contents: - - page: Embedding - path: ../../customizer/models/embedding.mdx - - section: Manage Customization Jobs - path: ../../customizer/manage-customization-jobs/index.mdx - contents: - - page: Create a Customization Job - path: ../../customizer/manage-customization-jobs/create-job.mdx - - section: Manage Model Entities - contents: - - page: Create a Model FileSet - path: ../../customizer/manage-model-entities/create-fileset.mdx - section: Vulnerability Scanning path: ../../auditor/index.mdx contents: @@ -222,16 +174,6 @@ navigation: path: ../../auditor/targets/inference-gateway.mdx - page: Schema path: ../../auditor/targets/schema.mdx - - section: Safe Synthesizer - contents: - - section: Tutorials - contents: - - page: Safe Synthesizer 101 - path: ../../safe-synthesizer/tutorials/safe-synthesizer-101.mdx - - section: Example Applications - contents: - - page: Tool Calling - path: ../../example-applications/tool-calling.mdx - section: Reference contents: - section: Release Notes @@ -260,17 +202,9 @@ navigation: path: ../../cli/reference.mdx - page: Troubleshooting path: ../../cli/troubleshooting.mdx - - page: Configuration Reference - path: ../../set-up/config-reference.mdx - - section: Authorization - contents: - - page: Permissions Reference - path: ../../auth/authorization/permissions-reference.mdx - section: Troubleshooting path: ../../troubleshooting/index.mdx contents: - - page: Customizer - path: ../../troubleshooting/customizer.mdx - page: Data Designer path: ../../troubleshooting/data-designer.mdx - page: Evaluator @@ -287,168 +221,3 @@ navigation: path: ../../eula.mdx - page: Acknowledgements path: ../../acknowledgements/index.mdx - - section: Access Control - hidden: true - contents: - - page: Concepts - path: ../../auth/concepts.mdx - - page: Overview - path: ../../auth/index.mdx - - page: Security Model - path: ../../auth/security-model.mdx - - page: Troubleshooting - path: ../../auth/troubleshooting.mdx - - section: Authentication - contents: - - page: OIDC Setup - path: ../../auth/authentication/oidc.mdx - - page: Using Authentication - path: ../../auth/authentication/using-authentication.mdx - - section: Providers - contents: - - page: Azure AD (Entra ID) - path: ../../auth/authentication/providers/azure-ad.mdx - - page: Generic OIDC - path: ../../auth/authentication/providers/generic.mdx - - section: Authorization - contents: - - page: API Scopes - path: ../../auth/authorization/api-scopes.mdx - - page: Managing Access - path: ../../auth/authorization/managing-access.mdx - - page: Policy Engine - path: ../../auth/authorization/policy-engine.mdx - - page: Roles and Permissions - path: ../../auth/authorization/roles-and-permissions.mdx - - section: Deployment - contents: - - page: Configuration - path: ../../auth/deployment/configuration.mdx - - page: Gateway Integration - path: ../../auth/deployment/gateway.mdx - - page: Production Hardening - path: ../../auth/deployment/hardening.mdx - - section: Self-Managed Deployment - hidden: true - contents: - - section: Helm - contents: - - page: Helm Reference - path: ../../helm/index.mdx - - section: Setup - contents: - - page: About - path: ../../set-up/index.mdx - - page: Milvus - path: ../../set-up/milvus.mdx - - page: Observability - path: ../../set-up/opentelemetry.mdx - - section: Helm - contents: - - page: Backup and Restore - path: ../../set-up/helm/backup-and-restore.mdx - - page: Database Setup - path: ../../set-up/helm/database-setup.mdx - - page: File Storage - path: ../../set-up/helm/file-storage.mdx - - page: Overview - path: ../../set-up/helm/index.mdx - - page: Ingress - path: ../../set-up/helm/ingress.mdx - - page: Install - path: ../../set-up/helm/install.mdx - - page: Multinode Networking - path: ../../set-up/helm/multinode-networking.mdx - - page: OpenShift - path: ../../set-up/helm/openshift.mdx - - page: Persistent Volumes - path: ../../set-up/helm/persistent-volumes.mdx - - page: Prerequisites - path: ../../set-up/helm/prerequisites.mdx - - section: Customizer Reference - hidden: true - contents: - - page: Customization Concepts - path: ../../customizer/about.mdx - - section: Manage Jobs - contents: - - page: Cancel Job - path: ../../customizer/manage-customization-jobs/cancel-job.mdx - - page: Customization Job Reference - path: ../../customizer/manage-customization-jobs/customization-job-reference.mdx - - page: Get Job Status - path: ../../customizer/manage-customization-jobs/get-job-status.mdx - - page: Training Configuration - path: ../../customizer/manage-customization-jobs/hyperparameters.mdx - - page: List Active Jobs - path: ../../customizer/manage-customization-jobs/list-active-jobs.mdx - - section: Manage Model Entities - contents: - - page: Create a Model Entity - path: ../../customizer/manage-model-entities/create-model-entity.mdx - - page: Overview - path: ../../customizer/manage-model-entities/index.mdx - - section: Models - contents: - - page: Dataset Format - path: ../../customizer/models/data-format.mdx - - page: GPT-OSS - path: ../../customizer/models/gpt-oss.mdx - - page: Model Catalog - path: ../../customizer/models/index.mdx - - page: Llama Nemotron - path: ../../customizer/models/llama-nemotron.mdx - - page: Llama - path: ../../customizer/models/llama.mdx - - page: Mistral - path: ../../customizer/models/mistral.mdx - - page: Phi - path: ../../customizer/models/phi.mdx - - page: Qwen - path: ../../customizer/models/qwen.mdx - - section: Tutorials - contents: - - page: Import HuggingFace Models - path: ../../customizer/tutorials/import-hf-model.mdx - - page: Job Metrics - path: ../../customizer/tutorials/metrics.mdx - - page: Understanding Models and Training - path: ../../customizer/tutorials/understand-configurations-and-models.mdx - - section: Safe Synthesizer Reference - hidden: true - contents: - - page: Getting Started - path: ../../safe-synthesizer/getting-started.mdx - - section: About - contents: - - page: Data Synthesis - path: ../../safe-synthesizer/about/data-synthesis.mdx - - page: Evaluation - path: ../../safe-synthesizer/about/evaluation.mdx - - page: Host-Local Development - path: ../../safe-synthesizer/about/host-local-development.mdx - - page: Overview - path: ../../safe-synthesizer/about/index.mdx - - page: Jobs - path: ../../safe-synthesizer/about/jobs.mdx - - page: PII Replacement - path: ../../safe-synthesizer/about/pii-replacement.mdx - - page: Parameters Reference - path: ../../safe-synthesizer/about/reference.mdx - - section: Tutorials - contents: - - page: Differential Privacy - path: ../../safe-synthesizer/tutorials/differential-privacy.mdx - - page: Overview - path: ../../safe-synthesizer/tutorials/index.mdx - - section: Evaluator Benchmark Reference - hidden: true - contents: - - section: Benchmarks - contents: - - page: Job Management - path: ../../evaluator/benchmarks/job-management.mdx - - page: Manage Benchmarks - path: ../../evaluator/benchmarks/manage-benchmarks.mdx - - page: Benchmark Results - path: ../../evaluator/benchmarks/results.mdx diff --git a/docs/guardrails/observability.mdx b/docs/guardrails/observability.mdx index a464288ee2e..2f3a2bae405 100644 --- a/docs/guardrails/observability.mdx +++ b/docs/guardrails/observability.mdx @@ -10,7 +10,7 @@ NeMo Platform centrally manages OpenTelemetry across services. You can configure ## Prerequisites -To export guardrail traces, OpenTelemetry must be enabled in your deployment. See [OpenTelemetry](/documentation/self-managed-deployment/setup/observability) for instructions on enabling OpenTelemetry. +To export guardrail traces, OpenTelemetry must be enabled in your deployment. See OpenTelemetry for instructions on enabling OpenTelemetry. You also need a VirtualModel configured with guardrails middleware. See [Architecture](/documentation/guardrail-models/core-concepts/architecture) for wiring details. @@ -210,7 +210,7 @@ guardrails.request [server] action.has_llm_calls: false ``` -The `service.name` for all spans is determined by the platform's `OTEL_SERVICE_NAME` configuration. See [OpenTelemetry](/documentation/self-managed-deployment/setup/observability) for details. +The `service.name` for all spans is determined by the platform's `OTEL_SERVICE_NAME` configuration. See OpenTelemetry for details. --- diff --git a/docs/troubleshooting/evaluator.mdx b/docs/troubleshooting/evaluator.mdx index 0de8e70ab43..4ef97907247 100644 --- a/docs/troubleshooting/evaluator.mdx +++ b/docs/troubleshooting/evaluator.mdx @@ -8,7 +8,7 @@ Use this documentation to troubleshoot issues that can arise when you work with -You can [get metric logs](/documentation/evaluate-models/metrics/job-management) or [get benchmark logs](/documentation/evaluator-benchmark-reference/benchmarks/job-management) for `COMPLETED` or `FAILED` jobs and use them to help troubleshoot. +You can get metric logs or get benchmark logs for `COMPLETED` or `FAILED` jobs and use them to help troubleshoot. --- From 7f8060c42087eb1564fbe34c83df62b6df4bb1cf Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Fri, 5 Jun 2026 10:32:43 -0400 Subject: [PATCH 19/26] docs(fern): automate delinking of references into gated pages The old MkDocs hide_unready_docs hook auto-delinked inbound references to gated pages on every build. Fern has no build hook, so reproduce it as a repo script + CI enforcement instead of a one-time manual pass. - scripts/delink-gated.mjs: flags (and with --fix, rewrites to plain text) any link from a published page into a gated page. "Gated" is derived from the nav (any .mdx not in versions/latest.yml), so there's no second list to maintain and it tracks the nav automatically. - package.json: check:gated-links (wired into `npm run check`, so CI fails on a new inbound link into a gated section), fix:gated-links (autofix), and broken-links. - fern-docs-ci.yaml: also run `npm run broken-links` as the broader dead-link backstop (it wasn't enforced in CI before). - README: document the automation and the one difference from the old hook (delinking lives in source, so re-publishing re-adds links). Co-Authored-By: Claude Opus 4.8 Signed-off-by: Lawrence Lane --- .github/workflows/fern-docs-ci.yaml | 6 ++ docs/fern/README.md | 13 ++- docs/fern/package.json | 5 +- docs/fern/scripts/delink-gated.mjs | 124 ++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 docs/fern/scripts/delink-gated.mjs diff --git a/.github/workflows/fern-docs-ci.yaml b/.github/workflows/fern-docs-ci.yaml index 7ab29435529..e39f0b44de6 100644 --- a/.github/workflows/fern-docs-ci.yaml +++ b/.github/workflows/fern-docs-ci.yaml @@ -38,3 +38,9 @@ jobs: env: FERN_TOKEN: ${{ secrets.DOCS_FERN_TOKEN }} run: npm run check + + - name: Check for broken links + working-directory: ./docs/fern + env: + FERN_TOKEN: ${{ secrets.DOCS_FERN_TOKEN }} + run: npm run broken-links diff --git a/docs/fern/README.md b/docs/fern/README.md index e55e37cee8e..7a20cb4224c 100644 --- a/docs/fern/README.md +++ b/docs/fern/README.md @@ -93,7 +93,18 @@ The gated `.mdx` files stay in the repo so they remain maintained. The gated tre Inbound links from visible pages into gated pages are **delinked to plain text** (not rewritten URLs), since the target is not built — otherwise they would be broken links. -`gated-nav.yml` holds the ready-to-paste navigation blocks for the gated features (it is a reference only; Fern does not read it). **To publish a feature when it ships:** move its block from `gated-nav.yml` into `versions/latest.yml`, re-link the inbound references that were delinked (search the docs for the feature name), then run `npm run check` and `npx fern-api docs broken-links`. +### Automated delinking + +The old MkDocs hook delinked those references automatically. Fern has no build hook, so `scripts/delink-gated.mjs` reproduces it. "Gated" is derived from the nav (any `.mdx` not in `versions/latest.yml`), so there is no second list to maintain. + +- `npm run check:gated-links` — fails if any **published** page links into a gated page (runs as part of `npm run check`, so CI enforces it). +- `npm run fix:gated-links` — rewrites those links to plain text in place. + +So if someone links into a gated section, CI flags it and one command delinks it — no hunting for dead links by hand. `npm run broken-links` (also run in CI) is the broader backstop for any other dead link. + +One difference from the old MkDocs hook: that hook ran at build time and kept the link in the source, so re-publishing a page reactivated its inbound links automatically. This script delinks in the **source**, so the references become plain text. When you publish a feature, re-add the links you want from those plain-text references as part of the same step that moves the nav block (`check:gated-links` won't block you — it only flags live links that still point at gated pages). This keeps local `fern docs dev` honest (no build-time source rewriting) at the cost of re-linking on publish. + +`gated-nav.yml` holds the ready-to-paste navigation blocks for the gated features (it is a reference only; Fern does not read it). **To publish a feature when it ships:** move its block from `gated-nav.yml` into `versions/latest.yml`, re-add any inbound links you want, then run `npm run check` and `npm run broken-links`. ## CI and Publishing diff --git a/docs/fern/package.json b/docs/fern/package.json index bb9a414c72b..4b5e4bbfeb8 100644 --- a/docs/fern/package.json +++ b/docs/fern/package.json @@ -1,9 +1,12 @@ { "private": true, "scripts": { - "check": "npx -y fern-api@latest check && npm run validate-mdx", + "check": "npx -y fern-api@latest check && npm run validate-mdx && npm run check:gated-links", "check:fern": "npx -y fern-api@latest check", "validate-mdx": "node scripts/validate-mdx.mjs", + "check:gated-links": "node scripts/delink-gated.mjs", + "fix:gated-links": "node scripts/delink-gated.mjs --fix", + "broken-links": "npx -y fern-api@latest docs broken-links", "dev": "npx -y fern-api@latest docs dev", "generate": "npx -y fern-api@latest generate --docs", "preview": "npx -y fern-api@latest generate --docs --preview" diff --git a/docs/fern/scripts/delink-gated.mjs b/docs/fern/scripts/delink-gated.mjs new file mode 100644 index 00000000000..a84e6cf266e --- /dev/null +++ b/docs/fern/scripts/delink-gated.mjs @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Delink references from published pages into gated (unready) pages. +// +// Gated pages are .mdx files that exist in the repo but are intentionally left +// out of versions/latest.yml so Fern never builds them (they 404 / aren't +// indexed). A link from a *published* page into a gated page is therefore a +// dead link. The old MkDocs hide_unready_docs hook rewrote such links to plain +// text on every build; Fern has no equivalent hook, so this script reproduces +// that behavior: +// +// node scripts/delink-gated.mjs # check: list inbound links into gated pages, exit 1 if any +// node scripts/delink-gated.mjs --fix # rewrite those links to plain text in place +// +// "Gated" is derived from the nav, not a hand-maintained list: any .mdx not +// referenced in versions/latest.yml is gated. Publishing a feature (adding its +// pages back to the nav) automatically makes its inbound links valid again, so +// `--check` stops flagging them. + +import { readFileSync, writeFileSync, readdirSync, existsSync } from "node:fs"; +import { join, dirname, resolve, relative } from "node:path"; +import { fileURLToPath } from "node:url"; + +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); +const FERN_DIR = resolve(SCRIPT_DIR, ".."); +const REPO_DOCS = resolve(FERN_DIR, ".."); // docs/ +const NAV = join(FERN_DIR, "versions", "latest.yml"); +const FIX = process.argv.includes("--fix"); + +// --- published pages: every `path:` referenced in the nav ------------------- +const navText = readFileSync(NAV, "utf8"); +const published = new Set(); +for (const m of navText.matchAll(/path:\s*(\S+\.mdx)\s*$/gm)) { + published.add(resolve(dirname(NAV), m[1])); +} + +// --- all doc pages (exclude the fern config/snippets tree) ------------------ +function walk(dir, out = []) { + for (const e of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, e.name); + if (e.isDirectory()) { + if (p === FERN_DIR) continue; // skip docs/fern (config, components, snippets) + walk(p, out); + } else if (e.name.endsWith(".mdx")) { + out.push(p); + } + } + return out; +} +const allPages = walk(REPO_DOCS); +const isGated = (absMdx) => existsSync(absMdx) && !published.has(absMdx); + +// resolve a link target (relative .md/.mdx, optional #anchor) from a source file +function resolveTarget(srcFile, target) { + const t = target.trim(); + if (/^(https?:|mailto:|tel:|#|\/)/.test(t)) return null; // external / absolute / anchor + const path = t.split("#")[0]; + if (!path) return null; + const base = path.replace(/\.(md|mdx)$/, "").replace(/\/$/, ""); + // not a doc link if it has some other extension (image, etc.) + const last = base.split("/").pop(); + if (last.includes(".")) return null; + const dir = dirname(srcFile); + for (const cand of [resolve(dir, base) + ".mdx", resolve(dir, base, "index.mdx")]) { + if (existsSync(cand)) return cand; + } + return null; +} + +// markdown links [text](target) and JSX href="target" +const MD = /\[([^\]]+)\]\(([^)\s]+)\)/g; +const HREF = /\bhref=("|')([^"']+)\1/g; + +let findings = []; +for (const file of allPages) { + if (!published.has(file)) continue; // only published pages can produce dead inbound links + let src = readFileSync(file, "utf8"); + let changed = false; + + src = src.replace(MD, (whole, text, target) => { + const tgt = resolveTarget(file, target); + if (tgt && isGated(tgt)) { + findings.push({ file, kind: "markdown", target, text }); + changed = true; + return text; // delink -> plain text + } + return whole; + }); + + // JSX hrefs (Cards/Buttons) can't be cleanly turned into plain text; report only. + for (const m of src.matchAll(HREF)) { + const tgt = resolveTarget(file, m[2]); + if (tgt && isGated(tgt)) { + findings.push({ file, kind: "jsx-href", target: m[2], text: null }); + } + } + + if (FIX && changed) writeFileSync(file, src); +} + +const md = findings.filter((f) => f.kind === "markdown"); +const jsx = findings.filter((f) => f.kind === "jsx-href"); + +if (findings.length === 0) { + console.log("delink-gated: no inbound links from published pages into gated pages."); + process.exit(0); +} + +const rel = (f) => relative(REPO_DOCS, f); +if (FIX) { + console.log(`delink-gated: delinked ${md.length} markdown link(s) into gated pages:`); + for (const f of md) console.log(` ${rel(f.file)}: "${f.text}" -> plain text (was ${f.target})`); +} else { + console.error(`delink-gated: ${md.length} markdown link(s) from published pages point into gated (unbuilt) pages.`); + for (const f of md) console.error(` ${rel(f.file)}: [${f.text}](${f.target})`); + console.error(`\nRun \`npm run fix:gated-links\` to delink them to plain text (or publish the target by adding it to versions/latest.yml).`); +} +if (jsx.length) { + console.error(`\n${jsx.length} component href(s) into gated pages need manual handling (Card/Button cannot auto-delink):`); + for (const f of jsx) console.error(` ${rel(f.file)}: href="${f.target}"`); +} +// In --fix mode, markdown links are resolved; only unfixable JSX hrefs should fail. +process.exit(FIX ? (jsx.length ? 1 : 0) : 1); From 23a64d672bce4838b2b7426f2b8c8c3247a63844 Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Fri, 5 Jun 2026 10:42:56 -0400 Subject: [PATCH 20/26] docs(fern): add make targets, docs/AGENTS.md, and fix stale MkDocs refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the Fern docs workflow discoverable and correct now that the site is Fern (not MkDocs): - Makefile: add docs-* convenience targets (docs, docs-check, docs-broken-links, docs-fix-links, docs-preview, docs-deps, docs-login, docs-publish) wrapping `cd docs/fern && npm run ...`, mirroring the Gym setup so contributors don't memorize fern-api invocations. - docs/AGENTS.md: short contributor/agent guide — build commands, the nav-is-the-build gating rule, gated-link delinking, generated pages, and the API reference wiring. Points to docs/fern/README.md for detail. - Fix stale references to the removed MkDocs Makefile: the `build-docs` and `start-docs-server` slash commands and the dev MCP tool's make_docs allowlist now use the Fern targets instead of `cd docs && make html/live`. - Correct docs/fern/README.md: it described Gym's layout (fern/versions/latest/pages, main.css, .yml workflows, "no install step"). Fix to our layout (pages under docs/, config under docs/fern/, styles/, .yaml workflows, npm ci required) and accurate CI triggers. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Lawrence Lane --- .claude/commands/build-docs.md | 16 +++--- .claude/commands/start-docs-server.md | 13 +++-- Makefile | 41 +++++++++++++++ docs/AGENTS.md | 39 +++++++++++++++ docs/fern/README.md | 72 +++++++++++++-------------- tools/mcp-dev-tools/nmp_dev_mcp.py | 41 +++++++-------- 6 files changed, 147 insertions(+), 75 deletions(-) create mode 100644 docs/AGENTS.md diff --git a/.claude/commands/build-docs.md b/.claude/commands/build-docs.md index eedd0585568..dcfb564facb 100644 --- a/.claude/commands/build-docs.md +++ b/.claude/commands/build-docs.md @@ -1,18 +1,18 @@ --- -description: Build documentation with various options +description: Build / validate the Fern documentation --- -Build the documentation using one of these modes: +The docs are built with Fern (not MkDocs). From the repo root: -1. **Fast build** (default) - Quick iteration, skips SDK docs: +1. **Validate** (what CI runs — `fern check` + MDX validation + gated-link check): ```bash - cd docs && make html-fast + make docs-check ``` -2. **Full build** - Complete build including SDK docs: +2. **Check for broken links:** ```bash - cd docs && make html + make docs-broken-links ``` -Ask the user which mode they prefer, then run the appropriate command. -After building, let them know they can view the docs at `docs/_build/html/index.html`. +First run on a machine needs `make docs-deps` (installs `docs/fern` tooling). +To preview locally with live reload, use `/start-docs-server` (`make docs`). The published site is built by the `Publish Fern Docs` workflow on push to `main`. See `docs/AGENTS.md` and `docs/fern/README.md`. diff --git a/.claude/commands/start-docs-server.md b/.claude/commands/start-docs-server.md index d504b2a1d4d..2afc18acf7b 100644 --- a/.claude/commands/start-docs-server.md +++ b/.claude/commands/start-docs-server.md @@ -1,14 +1,13 @@ --- -allowed-tools: Bash(cd docs && make live:*) -description: Start a live documentation server with auto-reload +allowed-tools: Bash(make docs:*), Bash(make docs-deps:*) +description: Start the Fern docs dev server with live reload --- -Start a live documentation server for development with full SDK documentation. +Start the Fern documentation dev server for local preview. From the repo root: -Run: ```bash -cd docs && make live +make docs ``` -The server will be available at `http://localhost:8000` after the initial build completes. -It will automatically rebuild when you change documentation files. +This runs `fern docs dev` and prints a local URL (e.g. http://localhost:3000); it reloads as you edit `.mdx` files. If this is the first run on the machine, run `make docs-deps` first to install the `docs/fern` tooling. +Note: gated (unready) pages are intentionally excluded from the build and will 404 in the preview — that is expected. See `docs/AGENTS.md`. diff --git a/Makefile b/Makefile index d104b9f0e3f..cf6921c3ece 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,47 @@ generate-cli-reference-docs: ## Generate the CLI reference documentation generate-config-reference-docs: ## Generate the platform config reference documentation uv run --frozen generate-config-docs +# ============================================================================ +# Fern documentation site (docs/fern) +# ============================================================================ +# Convenience wrappers around `cd docs/fern && npm run ...` so contributors +# don't have to remember the fern-api invocations. The CI workflows +# (.github/workflows/fern-docs-*.yaml) are the source of truth. See +# docs/fern/README.md and docs/AGENTS.md. First run on a machine: +# `make docs-deps` (and `make docs-login` once for the Fern org). + +.PHONY: docs-deps +docs-deps: ## Install the Fern docs tooling (docs/fern node deps) + cd docs/fern && npm ci + +.PHONY: docs +docs: ## Start the Fern docs dev server (local preview, prints a localhost URL) + cd docs/fern && npm run dev + +.PHONY: docs-check +docs-check: ## Validate the Fern docs (fern check + validate-mdx + gated-link check) + cd docs/fern && npm run check + +.PHONY: docs-broken-links +docs-broken-links: ## Report broken links across the built docs + cd docs/fern && npm run broken-links + +.PHONY: docs-fix-links +docs-fix-links: ## Delink references from published pages into gated (unready) pages + cd docs/fern && npm run fix:gated-links + +.PHONY: docs-preview +docs-preview: ## Build a shared Fern preview URL (needs DOCS_FERN_TOKEN) + cd docs/fern && npm run preview + +.PHONY: docs-login +docs-login: ## One-time Fern CLI auth (for the nvidia Fern org) + npx -y fern-api@latest login + +.PHONY: docs-publish +docs-publish: ## Trigger the Publish Fern Docs workflow (normally runs on push to main) + gh workflow run publish-fern-docs.yaml + .PHONY: update-cli update-cli: generate-cli-commands vendor-nemo-platform-ext generate-cli-reference-docs diff --git a/docs/AGENTS.md b/docs/AGENTS.md new file mode 100644 index 00000000000..96d710bc73c --- /dev/null +++ b/docs/AGENTS.md @@ -0,0 +1,39 @@ +# Documentation (Fern) + +The docs site is built with **[Fern](https://buildwithfern.com)**, not MkDocs/Sphinx. Pages are `.mdx` under `docs/`; the Fern config lives in `docs/fern/`. Read `docs/fern/README.md` for the full guide — this file is the short version for anyone (human or agent) editing docs. + +## Build / preview / check + +Run these from the repo root (they wrap `cd docs/fern && npm run …`): + +| Command | What it does | +|---|---| +| `make docs-deps` | Install docs tooling (first run on a machine) | +| `make docs` | Local dev server (live preview) | +| `make docs-check` | `fern check` + MDX validation + gated-link check (what CI runs) | +| `make docs-broken-links` | Report broken links | +| `make docs-fix-links` | Auto-delink references into gated pages | + +Local preview and the published site read the **same** `docs/fern/versions/latest.yml`, so what you see locally is what ships. + +## Rules that bite if you miss them + +- **Navigation is the build.** Fern only builds pages listed in `docs/fern/versions/latest.yml`. A `.mdx` not in the nav is **not built** (404, not indexed) — that is how unready features are gated. Do **not** use `hidden: true` for gating (it still builds/serves the page). +- **Gated (unready) features** stay in the repo but out of the nav: `auth/`, `customizer/`, `safe-synthesizer/`, `set-up/` + `helm/`, `evaluator/benchmarks/`, and a few individual pages. Ready-to-paste nav blocks for re-publishing are in `docs/fern/gated-nav.yml`. To publish one: move its block into `latest.yml`, re-add inbound links, run `make docs-check && make docs-broken-links`. +- **Don't link into gated pages.** A link from a published page into a gated page is a dead link. `make docs-check` fails on it; `make docs-fix-links` delinks it to plain text. (Replaces the old MkDocs `hide_unready_docs` auto-delinking.) +- **Internal links** use canonical nav URLs like `/documentation/get-started/core-concepts/workspaces`, not relative `.md`/source paths. `make docs-broken-links` is the check. +- **No `{{variable}}` substitutions.** Fern has no substitution step; product names are inlined as literal text. (Prompt-template tokens like `` `{{input}}` `` inside backticks are real content — leave them.) + +## Generated pages — do not hand-edit + +These are generated; edit the source and regenerate: + +| Page | Regenerate with | +|---|---| +| `docs/cli/reference.mdx`, `docs/fern/snippets/_snippets/cli-summary.mdx` | `make generate-cli-reference-docs` | +| `docs/set-up/config-reference.mdx` | `make generate-config-reference-docs` | +| `docs/auth/authorization/permissions-reference.mdx` | `uv run python services/core/auth/scripts/auth-tools.py generate-docs` | + +## API reference + +The REST API reference is rendered natively by Fern from the OpenAPI spec — `docs/fern/generators.yml` points at `docs/fern/openapi/openapi.yaml` (a symlink to the generated repo-root `openapi/openapi.yaml`), surfaced by the `- api: API Reference` nav node. No `` embed. diff --git a/docs/fern/README.md b/docs/fern/README.md index 7a20cb4224c..0a4499b3c24 100644 --- a/docs/fern/README.md +++ b/docs/fern/README.md @@ -1,55 +1,55 @@ # NeMo Platform Fern Docs -This directory holds the Fern MDX source for the NeMo Platform documentation site. - -All new published docs edits should land under `fern/`. Keep the legacy MkDocs tree under `../docs/` as migration source material and historical reference. +This directory holds the Fern **configuration** for the NeMo Platform documentation site. The page content (`.mdx`) lives in the parent `docs/` tree (e.g. `docs/get-started/setup.mdx`); this `docs/fern/` directory holds the navigation, theme, components, snippets, and OpenAPI wiring. The nav references pages with relative paths (`../..//.mdx`). ## Quick Links | What | Where | | --- | --- | | Fern dashboard | https://dashboard.buildwithfern.com (NVIDIA org) | -| Agent skill | [`../.claude/skills/nemo-platform-docs/SKILL.md`](../.claude/skills/nemo-platform-docs/SKILL.md) | -| CI workflows | [`../.github/workflows/fern-docs-*.yml`](../.github/workflows/) | -| Publish workflow | [`../.github/workflows/publish-fern-docs.yml`](../.github/workflows/publish-fern-docs.yml) | +| Contributor/agent guide | [`../AGENTS.md`](../AGENTS.md) | +| Make targets | `make docs`, `docs-check`, `docs-broken-links`, `docs-fix-links`, `docs-preview` (repo root) | +| CI workflows | [`../../.github/workflows/`](../../.github/workflows/) (`fern-docs-*.yaml`) | +| Publish workflow | [`../../.github/workflows/publish-fern-docs.yaml`](../../.github/workflows/publish-fern-docs.yaml) | ## Quickstart -First time on this machine: - -```bash -npx -y fern-api@latest login -``` - -Then run from `fern/`: +From the repo root (these wrap `cd docs/fern && npm run …`): ```bash -npm run check # Validate docs.yml, navigation, and MDX -npm run dev # Start local preview at http://localhost:3000 +make docs-deps # one-time: install docs/fern tooling (needed for MDX validation) +make docs-login # one-time per machine: Fern CLI auth for the nvidia org +make docs-check # validate: fern check + MDX validation + gated-link check (what CI runs) +make docs # start local preview (prints a localhost URL) ``` -`package.json` intentionally shells out to `npx -y fern-api@latest`, so there is no install step for local docs work. +`package.json` shells out to `npx -y fern-api@latest` for the Fern CLI itself, but `make docs-deps` (`npm ci`) is required once because the MDX validator (`@mdx-js/mdx`) is a local dependency. ## Layout ```text -fern/ -├── fern.config.json # Fern organization + CLI version -├── package.json # npm run check|dev|generate|preview -├── docs.yml # Site config, theme, assets, redirects, versions -├── main.css # NVIDIA theme overrides -├── assets/ # Logos, shared SVGs, and page images -├── components/ # Custom TSX components -└── versions/ - ├── latest.yml # Navigation tree - └── latest/pages/ # MDX content and page-local assets +docs/ # page content (.mdx), one tree per product area +├── get-started/ , agents/ , evaluator/ , ... # published pages +└── fern/ # <- this directory: Fern site config + ├── fern.config.json # Fern organization + CLI version + ├── package.json # npm run check|dev|generate|preview|broken-links|*-gated-links + ├── docs.yml # Site config, theme, css, redirects, versions + ├── generators.yml # OpenAPI spec wiring for the API reference + ├── openapi/openapi.yaml # symlink -> repo-root openapi/openapi.yaml (generated) + ├── styles/ # CSS (notebook-viewer.css, button.css) + ├── assets/ # logos, shared images + ├── components/ # custom TSX MDX components + ├── snippets/ # reusable fragments + ├── scripts/ # validate-mdx.mjs, delink-gated.mjs, ipynb-to-fern-json.py + ├── gated-nav.yml # reference nav blocks for gated (unready) features + └── versions/latest.yml # navigation tree (defines what gets built) ``` -The site uses a single `Latest` version. `versions/latest.yml` defines the sidebar and maps each page file to its canonical route. +The site uses a single `Latest` version. `versions/latest.yml` defines the sidebar and maps each page file to its canonical route — and, because Fern only builds pages listed there, it is also what gates unready content (see below). ## Authoring -Add pages under `versions/latest/pages/` and wire them into `versions/latest.yml`. +Add pages under the relevant `docs//` directory and wire them into `versions/latest.yml`. Use front matter for the rendered page title: @@ -110,14 +110,14 @@ One difference from the old MkDocs hook: that hook ran at build time and kept th | Workflow | Trigger | Purpose | | --- | --- | --- | -| `fern-docs-ci.yml` | trusted `pull-request/` mirror branch | Run `npm run check` with `DOCS_FERN_TOKEN` available | -| `fern-docs-preview-build.yml` | `pull_request` touching `fern/**` | Upload PR `fern/` sources with no secrets | -| `fern-docs-preview-comment.yml` | successful preview build workflow run | Generate a Fern preview with `DOCS_FERN_TOKEN` and post/update the PR comment | -| `publish-fern-docs.yml` | `main`, `docs/v*` tag, or manual dispatch | Publish the Fern docs site | +| `fern-docs-ci.yaml` | `pull_request` touching `docs/**` | `npm run check` (fern check + MDX + gated-link check) and `npm run broken-links` | +| `fern-docs-preview-build.yaml` | `pull_request` touching `docs/**` | Upload PR `docs/` sources as an artifact (no secrets — fork-safe) | +| `fern-docs-preview-comment.yaml` | successful preview build (`workflow_run`) | Build a Fern preview with `DOCS_FERN_TOKEN` and post/update the PR comment | +| `publish-fern-docs.yaml` | push to `main` touching `docs/**`, `docs/v*` tag, or manual dispatch | Publish the Fern docs site | -Required secret: `DOCS_FERN_TOKEN`, generated with `fern token` from an account that can publish to the NVIDIA Fern organization. +Required secret: `DOCS_FERN_TOKEN` (org-level), from `fern token` for an account that can publish to the NVIDIA Fern organization. -PRs that touch `fern/**` get a shared preview URL posted as a comment after the two-part preview workflow finishes. +PRs that touch `docs/**` get a shared preview URL posted as a comment after the two-part preview workflow finishes. Note: the `workflow_run`-triggered comment job only runs once these workflows are on the default branch (`main`), so the first preview appears after this PR merges. ## Troubleshooting @@ -125,6 +125,6 @@ PRs that touch `fern/**` get a shared preview URL posted as a comment after the | --- | --- | | `HTTP 403` or organization access error | Sign in at https://dashboard.buildwithfern.com, then run `npx -y fern-api@latest login` again | | `fern check` YAML error | Use 2-space indentation; make sure `path:` values are relative to `versions/latest.yml` | -| Page 404 in preview | Check that `versions/latest.yml` lists the page and links by canonical route | -| Broken internal link | Rewrite it to the nav-derived `/documentation/...` URL | +| Page 404 in preview | Check that `versions/latest.yml` lists the page (gated pages are intentionally omitted and *will* 404) | +| Broken internal link | Rewrite to the nav URL `/documentation/...`; if it targets a gated page, run `make docs-fix-links` to delink it. `make docs-broken-links` reports them all | | JSX or MDX parse error | Escape raw `{}`, `<`, or `>` in prose, and use Fern components instead of raw MkDocs syntax | diff --git a/tools/mcp-dev-tools/nmp_dev_mcp.py b/tools/mcp-dev-tools/nmp_dev_mcp.py index a6a91dd0ef3..71447d7102f 100644 --- a/tools/mcp-dev-tools/nmp_dev_mcp.py +++ b/tools/mcp-dev-tools/nmp_dev_mcp.py @@ -790,39 +790,32 @@ async def make_target(target: str) -> dict[str, Any]: # === DOCS MAKE TARGET OPERATIONS === - @server.tool(description="Run a docs Makefile target from the docs/ directory") + @server.tool(description="Run a Fern docs Makefile target (docs-*) from the repo root") async def make_docs(target: str) -> dict[str, Any]: """ - Run a make target from the docs/ subdirectory (equivalent to `cd docs && make `). + Run a Fern docs make target from the repo root (equivalent to `make `). + + The docs site is built with Fern; targets live in the root Makefile. Args: target: Make target name. Allowed targets: - Build: html, html-fast, publish, clean, env - Serve: live, live-fast, test-serve - Quality: linkcheck, lint-python - Utility: replace-substitutions, sync, help + Setup: docs-deps, docs-login + Serve: docs + Quality: docs-check, docs-broken-links, docs-fix-links + Publish: docs-preview, docs-publish Returns: Dictionary with make command results """ allowed_docs_targets = { - # Build - "html", - "html-fast", - "publish", - "clean", - "env", - # Serve - "live", - "live-fast", - "test-serve", - # Quality - "linkcheck", - "lint-python", - # Utility - "replace-substitutions", - "sync", - "help", + "docs-deps", + "docs-login", + "docs", + "docs-check", + "docs-broken-links", + "docs-fix-links", + "docs-preview", + "docs-publish", } if target not in allowed_docs_targets: @@ -831,7 +824,7 @@ async def make_docs(target: str) -> dict[str, Any]: "error": f"Target '{target}' not in allowlist. Allowed: {sorted(allowed_docs_targets)}", } - return run_command(["make", "-C", "docs", target], timeout=600) + return run_command(["make", target], timeout=600) return server From 03eb96c9a0878a02f8430c9efd9a92cc8ef806af Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Fri, 5 Jun 2026 12:17:33 -0400 Subject: [PATCH 21/26] fix(ci): ruff-format dev MCP tool and refresh permissions reference Two Lint-all failures: - lint-python-style: tools/mcp-dev-tools/nmp_dev_mcp.py was edited (Fern docs make targets) without `ruff format`. Reformat it. - lint-auth-config: docs/auth/authorization/permissions-reference.mdx was stale versus the merged static-authz.yaml (and still carried canonical /documentation links from the earlier hidden-section pass). Regenerate it with auth-tools.py so it matches the generator output. The page is gated (auth/** is out of the nav), so its relative .md links are not built and don't affect broken-links. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Lawrence Lane --- docs/auth/authorization/permissions-reference.mdx | 12 ++++++------ tools/mcp-dev-tools/nmp_dev_mcp.py | 4 +++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/auth/authorization/permissions-reference.mdx b/docs/auth/authorization/permissions-reference.mdx index 5b82494bca2..eb13618d77e 100644 --- a/docs/auth/authorization/permissions-reference.mdx +++ b/docs/auth/authorization/permissions-reference.mdx @@ -5,9 +5,9 @@ description: "" (permissions-reference)= -Complete reference of all permissions across the NeMo Platform APIs. Each permission controls access to a specific operation within an individual API. Permissions are assigned to users through [roles](/documentation/access-control/authorization/roles-and-permissions). +Complete reference of all permissions across the NeMo Platform APIs. Each permission controls access to a specific operation within an individual API. Permissions are assigned to users through [roles](roles-and-permissions.md). -For token-level access restrictions, see [API Scopes](/documentation/access-control/authorization/api-scopes). For the RBAC model, see [Authorization Concepts](/documentation/access-control/concepts). +For token-level access restrictions, see [API Scopes](api-scopes.md). For the RBAC model, see [Authorization Concepts](../concepts.md). @@ -142,7 +142,7 @@ PlatformAdmin is omitted — it bypasses permission checks entirely at the polic ## Related -- [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions) — Role descriptions and hierarchy. -- [API Scopes](/documentation/access-control/authorization/api-scopes) — Token-level scope restrictions. -- [Authorization Concepts](/documentation/access-control/concepts) — Workspaces, roles, bindings, and the RBAC model. -- [Security Model](/documentation/access-control/security-model) — Trust boundaries and authorization layers. +- [Roles & Permissions](roles-and-permissions.md) — Role descriptions and hierarchy. +- [API Scopes](api-scopes.md) — Token-level scope restrictions. +- [Authorization Concepts](../concepts.md) — Workspaces, roles, bindings, and the RBAC model. +- [Security Model](../security-model.md) — Trust boundaries and authorization layers. diff --git a/tools/mcp-dev-tools/nmp_dev_mcp.py b/tools/mcp-dev-tools/nmp_dev_mcp.py index 71447d7102f..00e3fbbd573 100644 --- a/tools/mcp-dev-tools/nmp_dev_mcp.py +++ b/tools/mcp-dev-tools/nmp_dev_mcp.py @@ -790,7 +790,9 @@ async def make_target(target: str) -> dict[str, Any]: # === DOCS MAKE TARGET OPERATIONS === - @server.tool(description="Run a Fern docs Makefile target (docs-*) from the repo root") + @server.tool( + description="Run a Fern docs Makefile target (docs-*) from the repo root" + ) async def make_docs(target: str) -> dict[str, Any]: """ Run a Fern docs make target from the repo root (equivalent to `make `). From 1a6362460159ab085e7d61d1d130061b7064d7e9 Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Tue, 9 Jun 2026 09:03:50 -0400 Subject: [PATCH 22/26] docs(fern): set production custom domain for docs site Point the Fern instance at docs.nvidia.com/nemo/platform via custom-domain and scope the buildwithfern source URL to the /nemo/platform path. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Lawrence Lane --- docs/fern/docs.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/fern/docs.yml b/docs/fern/docs.yml index f6296223ce2..471aa7b6b1a 100644 --- a/docs/fern/docs.yml +++ b/docs/fern/docs.yml @@ -1,5 +1,6 @@ instances: -- url: nemo-platform.docs.buildwithfern.com +- url: nemo-platform.docs.buildwithfern.com/nemo/platform + custom-domain: docs.nvidia.com/nemo/platform multi-source: true title: NVIDIA NeMo Platform global-theme: nvidia From 93cd2ed73cf4cf7d2c68e6719f3635d9a64bc5d4 Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Tue, 9 Jun 2026 09:06:56 -0400 Subject: [PATCH 23/26] docs(fern): gitignore auto-generated ai_examples_override.yml Fern writes docs/fern/openapi/ai_examples_override.yml as a cache of auto-generated API examples during a full docs build. It is not referenced by generators.yml and regenerates on every build, so it should not be tracked. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Lawrence Lane --- docs/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/.gitignore b/docs/.gitignore index b07258ad637..d5403b578fc 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -23,6 +23,10 @@ _test_serve/ *.tmp *latest* !fern/versions/latest.yml + +# Fern build artifact: auto-generated API examples cache (regenerated on every +# docs build; not referenced by generators.yml, so it must not be committed) +fern/openapi/ai_examples_override.yml _static/versions1.json helm/_values.yaml From 4561ef0324f87f00c138d1db3229da6655fbd49f Mon Sep 17 00:00:00 2001 From: Lawrence Lane Date: Tue, 9 Jun 2026 09:22:33 -0400 Subject: [PATCH 24/26] docs(fern): restore Safe Synthesizer section to the build The "Synthesize Safe Data" doc section was incorrectly gated out of the Fern nav. The old MkDocs site shipped these pages; its hide_unready_docs hook only listed "Safe Synthesizer" under api_tags (which filters the API reference), not under sections/paths (which hide doc pages). An audit of every gating decision against the old config confirms this was the only over-gated doc section (under-gated: 0). Restore the section and fix issues that were never caught while the pages were unbuilt: - Add the "Synthesize Safe Data" nav section (About, Tutorials, SDK Resources) to versions/latest.yml, in its original position. - Convert sdk-resources.md -> .mdx (Fern frontmatter, inline {{platform_name}} and {{nss_short_name}}). - Fix reference.mdx frontmatter title leak ("@nemo-nb: hide" -> "Parameters Reference"). - Correct stale canonical link slugs (safe-synthesizer-reference/* and safe-synthesizer/* -> synthesize-safe-data/*) across the section. - Delink references into still-gated pages (deploy-models, Helm chart) to match old-site behavior and the gating policy. Verified: validate-mdx, fern check, fern docs broken-links (0 errors), and delink-gated all pass. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Lawrence Lane --- docs/fern/versions/latest.yml | 27 ++++++++++++++++ .../safe-synthesizer/about/data-synthesis.mdx | 10 +++--- docs/safe-synthesizer/about/evaluation.mdx | 8 ++--- .../about/host-local-development.mdx | 4 +-- docs/safe-synthesizer/about/index.mdx | 32 +++++++++---------- docs/safe-synthesizer/about/jobs.mdx | 10 +++--- .../about/pii-replacement.mdx | 6 ++-- docs/safe-synthesizer/about/reference.mdx | 16 +++++----- docs/safe-synthesizer/getting-started.mdx | 8 ++--- .../{sdk-resources.md => sdk-resources.mdx} | 21 ++++++------ .../tutorials/differential-privacy.mdx | 4 +-- docs/safe-synthesizer/tutorials/index.mdx | 10 +++--- .../tutorials/safe-synthesizer-101.mdx | 10 +++--- 13 files changed, 98 insertions(+), 68 deletions(-) rename docs/safe-synthesizer/{sdk-resources.md => sdk-resources.mdx} (78%) diff --git a/docs/fern/versions/latest.yml b/docs/fern/versions/latest.yml index 1c824787f25..3259c244fc1 100644 --- a/docs/fern/versions/latest.yml +++ b/docs/fern/versions/latest.yml @@ -67,6 +67,33 @@ navigation: path: ../../data-designer/sdk-resources.mdx - page: Migrating from Standalone Library path: ../../data-designer/migration.mdx + - section: Synthesize Safe Data + path: ../../safe-synthesizer/getting-started.mdx + contents: + - section: About + path: ../../safe-synthesizer/about/index.mdx + contents: + - page: Data Synthesis + path: ../../safe-synthesizer/about/data-synthesis.mdx + - page: PII Replacement + path: ../../safe-synthesizer/about/pii-replacement.mdx + - page: Evaluation + path: ../../safe-synthesizer/about/evaluation.mdx + - page: Jobs + path: ../../safe-synthesizer/about/jobs.mdx + - page: Local and Subprocess Execution + path: ../../safe-synthesizer/about/host-local-development.mdx + - page: Parameters Reference + path: ../../safe-synthesizer/about/reference.mdx + - section: Tutorials + path: ../../safe-synthesizer/tutorials/index.mdx + contents: + - page: Safe Synthesizer 101 + path: ../../safe-synthesizer/tutorials/safe-synthesizer-101.mdx + - page: Differential Privacy + path: ../../safe-synthesizer/tutorials/differential-privacy.mdx + - page: SDK Resources + path: ../../safe-synthesizer/sdk-resources.mdx - section: Anonymize Data path: ../../anonymizer/index.mdx contents: diff --git a/docs/safe-synthesizer/about/data-synthesis.mdx b/docs/safe-synthesizer/about/data-synthesis.mdx index a9bc08647ce..9f40619f6f4 100644 --- a/docs/safe-synthesizer/about/data-synthesis.mdx +++ b/docs/safe-synthesizer/about/data-synthesis.mdx @@ -95,7 +95,7 @@ Enabling DP provides strong privacy guarantees but affects synthetic data qualit **Data size:** DP performs best with 10,000+ training records. Smaller datasets may experience significant quality degradation due to the noise required for privacy guarantees. -For hands-on guidance, refer to [differential-privacy](/documentation/safe-synthesizer-reference/tutorials/differential-privacy). For complete parameter documentation, refer to [reference](/documentation/safe-synthesizer-reference/about/parameters-reference). +For hands-on guidance, refer to [differential-privacy](/documentation/synthesize-safe-data/tutorials/differential-privacy). For complete parameter documentation, refer to [reference](/documentation/synthesize-safe-data/about/parameters-reference). ## Configuration @@ -105,10 +105,10 @@ Synthesis behavior is controlled through configuration parameters: - **Generation**: Number of records, temperature, sampling strategies - **Privacy**: Differential privacy parameters (epsilon, delta, clipping) -For a complete list of all available parameters and their defaults, refer to [reference](/documentation/safe-synthesizer-reference/about/parameters-reference). +For a complete list of all available parameters and their defaults, refer to [reference](/documentation/synthesize-safe-data/about/parameters-reference). ## Related Topics -- [reference](/documentation/safe-synthesizer-reference/about/parameters-reference): Complete parameter reference -- [differential-privacy](/documentation/safe-synthesizer-reference/tutorials/differential-privacy): Learn about differential privacy in practice -- [index](/documentation/safe-synthesizer-reference/tutorials/overview): More tutorials +- [reference](/documentation/synthesize-safe-data/about/parameters-reference): Complete parameter reference +- [differential-privacy](/documentation/synthesize-safe-data/tutorials/differential-privacy): Learn about differential privacy in practice +- [index](/documentation/synthesize-safe-data/tutorials): More tutorials diff --git a/docs/safe-synthesizer/about/evaluation.mdx b/docs/safe-synthesizer/about/evaluation.mdx index fa1ca766a24..f2ccf53b67f 100644 --- a/docs/safe-synthesizer/about/evaluation.mdx +++ b/docs/safe-synthesizer/about/evaluation.mdx @@ -88,7 +88,7 @@ You should expect some PII replay, and it is often not a cause for concern. We t - Lower epsilon (stronger privacy) generally yields higher DPS scores - Enabling DP can reduce **SQS** due to privacy-utility tradeoff (noise affects quality) -For differential privacy configuration and privacy-utility tradeoffs, refer to [data-synthesis](/documentation/safe-synthesizer-reference/about/data-synthesis) and [differential-privacy](/documentation/safe-synthesizer-reference/tutorials/differential-privacy). +For differential privacy configuration and privacy-utility tradeoffs, refer to [data-synthesis](/documentation/synthesize-safe-data/about/data-synthesis) and [differential-privacy](/documentation/synthesize-safe-data/tutorials/differential-privacy). ## Evaluation Reports @@ -124,6 +124,6 @@ We enable evaluation by default. ## Related Topics -- [safe-synthesizer-101](/documentation/safe-synthesizer/tutorials/safe-synthesizer-101): Get started with evaluation -- [differential-privacy](/documentation/safe-synthesizer-reference/tutorials/differential-privacy): Learn about privacy metrics -- [index](/documentation/safe-synthesizer-reference/tutorials/overview): More tutorials +- [safe-synthesizer-101](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101): Get started with evaluation +- [differential-privacy](/documentation/synthesize-safe-data/tutorials/differential-privacy): Learn about privacy metrics +- [index](/documentation/synthesize-safe-data/tutorials): More tutorials diff --git a/docs/safe-synthesizer/about/host-local-development.mdx b/docs/safe-synthesizer/about/host-local-development.mdx index b6f5fd67fd7..c5b5b2ee06d 100644 --- a/docs/safe-synthesizer/about/host-local-development.mdx +++ b/docs/safe-synthesizer/about/host-local-development.mdx @@ -12,7 +12,7 @@ This page covers local execution only. Platform job submission uses the Jobs API ## Prerequisites -- CUDA-capable NVIDIA GPU on the host (80GB+ VRAM recommended; check with `nvidia-smi`). See [Getting Started](/documentation/safe-synthesizer-reference/getting-started). +- CUDA-capable NVIDIA GPU on the host (80GB+ VRAM recommended; check with `nvidia-smi`). See [Getting Started](/documentation/synthesize-safe-data). - NeMo Platform repository checkout with the Safe Synthesizer plugin installed. - **No running platform required** for a typical local run when you pass `--data-source` — the NSS runtime can download base models from Hugging Face directly. @@ -210,5 +210,5 @@ Requires `RUN_NSS_LOCAL_E2E=1`, CUDA, and `nemo safe-synthesizer runtime setup`. ## Related topics -- [Getting Started](/documentation/safe-synthesizer-reference/getting-started) — GPU and local runtime prerequisites +- [Getting Started](/documentation/synthesize-safe-data) — GPU and local runtime prerequisites - Plugin README: `plugins/nemo-safe-synthesizer/README.md` \ No newline at end of file diff --git a/docs/safe-synthesizer/about/index.mdx b/docs/safe-synthesizer/about/index.mdx index e7e744965ab..46c08cad485 100644 --- a/docs/safe-synthesizer/about/index.mdx +++ b/docs/safe-synthesizer/about/index.mdx @@ -7,8 +7,8 @@ description: "" NVIDIA NeMo Safe Synthesizer enables you to create private versions of sensitive tabular datasets. The resulting data is entirely synthetic, with no one-to-one mapping to your original records. NeMo Safe Synthesizer is purpose-built for privacy compliance and data protection while preserving data utility for downstream AI tasks. - - + + --- @@ -23,17 +23,17 @@ A complete NeMo Safe Synthesizer job consists of the following steps: 1. [Upload Data](/documentation/get-started/core-concepts/manage-files): Add your tabular data to the Files API 2. Prepare Data: - - [Configure PII Replacement](/documentation/safe-synthesizer-reference/about/pii-replacement): Set up detection and replacement of sensitive information (recommended prior to the Synthesis step to ensure the model has no chance of learning the most sensitive information like names and addresses) + - [Configure PII Replacement](/documentation/synthesize-safe-data/about/pii-replacement): Set up detection and replacement of sensitive information (recommended prior to the Synthesis step to ensure the model has no chance of learning the most sensitive information like names and addresses) - Configure training data organization and holdout splits 3. Configure Synthesis: - - [Training](/documentation/safe-synthesizer-reference/about/data-synthesis): Set model selection and training parameters including differential privacy + - [Training](/documentation/synthesize-safe-data/about/data-synthesis): Set model selection and training parameters including differential privacy - Generate synthetic records - - [Evaluation](/documentation/safe-synthesizer-reference/about/evaluation): Assess quality and privacy + - [Evaluation](/documentation/synthesize-safe-data/about/evaluation): Assess quality and privacy 4. Execute and Review: - - [Run and Monitor Job](/documentation/safe-synthesizer-reference/about/jobs): Execute the job and track progress + - [Run and Monitor Job](/documentation/synthesize-safe-data/about/jobs): Execute the job and track progress - Download synthetic data and evaluation reports -Find all Safe Synthesizer configuration parameters in [Parameters Reference](/documentation/safe-synthesizer-reference/about/parameters-reference). +Find all Safe Synthesizer configuration parameters in [Parameters Reference](/documentation/synthesize-safe-data/about/parameters-reference). --- @@ -43,14 +43,14 @@ Try out this early access API using Docker Compose or deploying the NeMo Platfor - + Get started with the NeMo Safe Synthesizer microservice locally using the NeMo Platform CLI. Easiest for local testing. standalone - + Deploy the NeMo Platform Helm Chart, which includes NeMo Safe Synthesizer. @@ -68,7 +68,7 @@ Get hands-on experience with Safe Synthesizer through step-by-step tutorials. - + Learn how to use NeMo Safe Synthesizer with hands-on tutorials covering basics to advanced topics. @@ -84,32 +84,32 @@ Learn how to use NeMo Safe Synthesizer with hands-on tutorials covering basics t - + Learn about LLM-based synthesis, differential privacy, and tabular fine-tuning for generating synthetic data. - + Understand how PII detection and replacement works to protect sensitive information before synthesis. - + Learn about quality and privacy metrics used to assess synthetic data including SQS and DPS scores. - + Understand the job lifecycle, configuration, and execution for Safe Synthesizer pipelines. - + Run on a host GPU with `nemo safe-synthesizer run-local` and `runtime` commands; reuse local adapters and run plugin tests. - + Reference all configuration parameters available when creating Safe Synthesizer jobs. diff --git a/docs/safe-synthesizer/about/jobs.mdx b/docs/safe-synthesizer/about/jobs.mdx index 832d8abd7b9..8cc79f99df6 100644 --- a/docs/safe-synthesizer/about/jobs.mdx +++ b/docs/safe-synthesizer/about/jobs.mdx @@ -112,7 +112,7 @@ When the job completes, access: For **platform jobs**, set `pretrained_model_job` in the job spec to a completed job that has an **`adapter`** result in Files. Reuse is generation-only (no retraining). Use either `pretrained_model_job` or `config.training.pretrained_model`, not both. -For **host-local** development (`nemo safe-synthesizer run-local`), set `config.training.pretrained_model` to a local adapter or work directory from an earlier run. See [Local and Subprocess Execution](host-local-development.md). +For **host-local** development (`nemo safe-synthesizer run-local`), set `config.training.pretrained_model` to a local adapter or work directory from an earlier run. See [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution). ## Job Builder API @@ -312,7 +312,7 @@ kubectl get events -n --sort-by='.lastTimestamp' ## Related Topics -- [Local and Subprocess Execution](host-local-development.md): `run-local`, adapter reuse, and plugin tests -- [safe-synthesizer-101](../tutorials/safe-synthesizer-101.md): Get started with NeMo Safe Synthesizer jobs -- [index](../tutorials/index.md): More hands-on tutorials -- [reference](reference.md): Full parameter reference \ No newline at end of file +- [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution): `run-local`, adapter reuse, and plugin tests +- [safe-synthesizer-101](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101): Get started with NeMo Safe Synthesizer jobs +- [index](/documentation/synthesize-safe-data/tutorials): More hands-on tutorials +- [reference](/documentation/synthesize-safe-data/about/parameters-reference): Full parameter reference \ No newline at end of file diff --git a/docs/safe-synthesizer/about/pii-replacement.mdx b/docs/safe-synthesizer/about/pii-replacement.mdx index c2170c061da..20b3bc2b3f3 100644 --- a/docs/safe-synthesizer/about/pii-replacement.mdx +++ b/docs/safe-synthesizer/about/pii-replacement.mdx @@ -175,7 +175,7 @@ Beyond these built-in types, you can define custom entities using: ## Configuration -PII replacement is configured through the `replace_pii` section. For the full schema, refer to [reference](/documentation/safe-synthesizer-reference/about/parameters-reference). +PII replacement is configured through the `replace_pii` section. For the full schema, refer to [reference](/documentation/synthesize-safe-data/about/parameters-reference). ```json { @@ -217,5 +217,5 @@ PII replacement is always recommended as a preprocessing step before synthesis. ## Related Topics -- [safe-synthesizer-101](/documentation/safe-synthesizer/tutorials/safe-synthesizer-101): Getting started tutorial with PII replacement -- [index](/documentation/safe-synthesizer-reference/tutorials/overview): More tutorials +- [safe-synthesizer-101](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101): Getting started tutorial with PII replacement +- [index](/documentation/synthesize-safe-data/tutorials): More tutorials diff --git a/docs/safe-synthesizer/about/reference.mdx b/docs/safe-synthesizer/about/reference.mdx index 8ffa21381dd..6fb8454c0a6 100644 --- a/docs/safe-synthesizer/about/reference.mdx +++ b/docs/safe-synthesizer/about/reference.mdx @@ -1,5 +1,5 @@ --- -title: "@nemo-nb: hide" +title: "Parameters Reference" description: "" --- @@ -8,7 +8,7 @@ description: "" This page summarizes the main configuration groups available when creating NeMo Safe Synthesizer jobs. For generated REST API schema details, see the -[Safe Synthesizer API Reference](../../api/index.md#tag-safe-synthesizer). +[Safe Synthesizer API Reference](/documentation/reference/api-reference). ## Job Spec (Plugin / REST) @@ -20,7 +20,7 @@ Top-level fields on the Safe Synthesizer job spec (alongside `config`): | `pretrained_model_job` | Prior completed job whose **`adapter`** result in Files is reused for **generation-only** synthesis. Format: `` or `/`. Mutually exclusive with `config.training.pretrained_model`. | | `hf_token_secret` | Platform secret name for Hugging Face token during model initialization | -For host-local runs, see [Local and Subprocess Execution](host-local-development.md). Reuse a local adapter with `config.training.pretrained_model`, not `pretrained_model_job`. +For host-local runs, see [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution). Reuse a local adapter with `config.training.pretrained_model`, not `pretrained_model_job`. ## Top-Level Configuration @@ -29,7 +29,7 @@ The `SafeSynthesizerParameters` schema defines the main configuration structure ### SafeSynthesizerParameters All fields are optional at the top level. For nested field constraints, see the -[Safe Synthesizer API Reference](../../api/index.md#tag-safe-synthesizer) and +[Safe Synthesizer API Reference](/documentation/reference/api-reference) and search for the schema name in the **Type** column. | Field | Type | Constraints / description | @@ -79,7 +79,7 @@ Configuration for synthetic data quality and privacy assessment, including MIA, ## PII Replacement Configuration -Configuration for PII detection and replacement. See [pii-replacement](../about/pii-replacement.md) for conceptual documentation. +Configuration for PII detection and replacement. See [pii-replacement](/documentation/synthesize-safe-data/about/pii-replacement) for conceptual documentation. ### Column Classification Config (`replace_pii.globals.classify`) @@ -131,6 +131,6 @@ job = builder.create_job(name="my-job", project="my-project") ## Related Topics -- [data-synthesis](../about/data-synthesis.md) - Learn about synthesis concepts -- [evaluation](../about/evaluation.md) - Learn about evaluation metrics -- [index](../tutorials/index.md) - Hands-on tutorials +- [data-synthesis](/documentation/synthesize-safe-data/about/data-synthesis) - Learn about synthesis concepts +- [evaluation](/documentation/synthesize-safe-data/about/evaluation) - Learn about evaluation metrics +- [index](/documentation/synthesize-safe-data/tutorials) - Hands-on tutorials diff --git a/docs/safe-synthesizer/getting-started.mdx b/docs/safe-synthesizer/getting-started.mdx index 12f4f051a31..95ffdd1f5d8 100644 --- a/docs/safe-synthesizer/getting-started.mdx +++ b/docs/safe-synthesizer/getting-started.mdx @@ -22,7 +22,7 @@ For general platform troubleshooting (port conflicts, health checks, and so on), ## Host-local CLI -For GPU development on your machine, install the Safe Synthesizer plugin from this repository and use `nemo safe-synthesizer run-local` (see [Local and Subprocess Execution](/documentation/safe-synthesizer-reference/about/host-local-development)): +For GPU development on your machine, install the Safe Synthesizer plugin from this repository and use `nemo safe-synthesizer run-local` (see [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution)): ```shell BOOTSTRAP_LOCAL_PLUGIN_DIRS=plugins/nemo-safe-synthesizer make bootstrap-python @@ -41,8 +41,8 @@ The `run-local` command launches the Safe Synthesizer task in a separate runtime Create your first synthetic dataset: -- [Safe Synthesizer 101 Tutorial](/documentation/safe-synthesizer-reference/tutorials/safe-synthesizer-101) - a beginner-friendly introduction -- [Local and Subprocess Execution](/documentation/safe-synthesizer-reference/about/host-local-development) - local CLI and runtime task details -- [SDK Resources](/documentation/safe-synthesizer-reference/sdk-resources) - Python SDK methods for jobs, builders, logs, and results +- [Safe Synthesizer 101 Tutorial](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101) - a beginner-friendly introduction +- [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution) - local CLI and runtime task details +- [SDK Resources](/documentation/synthesize-safe-data/sdk-resources) - Python SDK methods for jobs, builders, logs, and results --- \ No newline at end of file diff --git a/docs/safe-synthesizer/sdk-resources.md b/docs/safe-synthesizer/sdk-resources.mdx similarity index 78% rename from docs/safe-synthesizer/sdk-resources.md rename to docs/safe-synthesizer/sdk-resources.mdx index 736587227b1..1f3c5ddaf9a 100644 --- a/docs/safe-synthesizer/sdk-resources.md +++ b/docs/safe-synthesizer/sdk-resources.mdx @@ -1,7 +1,10 @@ +--- +title: "Safe Synthesizer NeMo Platform SDK Resources" +description: "" +--- -# Safe Synthesizer {{platform_name}} SDK Resources -The `nemo_safe_synthesizer_plugin.sdk` module provides {{platform_name}}-specific helpers for creating and monitoring {{nss_short_name}} jobs. Use these objects when you want to submit jobs through the platform Jobs service and retrieve platform-managed results. +The `nemo_safe_synthesizer_plugin.sdk` module provides NeMo Platform-specific helpers for creating and monitoring NeMo Safe Synthesizer jobs. Use these objects when you want to submit jobs through the platform Jobs service and retrieve platform-managed results. ## SafeSynthesizerResource @@ -23,13 +26,13 @@ An async variant with the same namespace is available as `AsyncNeMoPlatform.safe ## SafeSynthesizerJobsResource -`client.safe_synthesizer.jobs` calls the plugin API for {{nss_short_name}} jobs. +`client.safe_synthesizer.jobs` calls the plugin API for NeMo Safe Synthesizer jobs. | Method | Description | |--------|-------------| -| `create(*, spec, name=None, workspace=None, project=None, timeout=None, **params)` | Creates a {{nss_short_name}} platform job from a job spec. | -| `list(*, workspace=None, **params)` | Lists {{nss_short_name}} jobs in a workspace. | -| `retrieve(name, *, workspace=None)` | Retrieves one {{nss_short_name}} job by name. | +| `create(*, spec, name=None, workspace=None, project=None, timeout=None, **params)` | Creates a NeMo Safe Synthesizer platform job from a job spec. | +| `list(*, workspace=None, **params)` | Lists NeMo Safe Synthesizer jobs in a workspace. | +| `retrieve(name, *, workspace=None)` | Retrieves one NeMo Safe Synthesizer job by name. | | `get_status(name, *, workspace=None)` | Returns the job status from the platform Jobs service. | | `get_logs(name, *, workspace=None, **kwargs)` | Returns paginated job logs from the platform Jobs service. | @@ -92,6 +95,6 @@ job = ( ## Related Topics -- [Safe Synthesizer 101](tutorials/safe-synthesizer-101.md) - submit and monitor a first job -- [Safe Synthesizer Jobs](about/jobs.md) - understand job lifecycle and troubleshooting -- [Parameters Reference](about/reference.md) - review job spec and configuration fields +- [Safe Synthesizer 101](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101) - submit and monitor a first job +- [Safe Synthesizer Jobs](/documentation/synthesize-safe-data/about/jobs) - understand job lifecycle and troubleshooting +- [Parameters Reference](/documentation/synthesize-safe-data/about/parameters-reference) - review job spec and configuration fields diff --git a/docs/safe-synthesizer/tutorials/differential-privacy.mdx b/docs/safe-synthesizer/tutorials/differential-privacy.mdx index 66b3f3b6b2f..3a47167dedb 100644 --- a/docs/safe-synthesizer/tutorials/differential-privacy.mdx +++ b/docs/safe-synthesizer/tutorials/differential-privacy.mdx @@ -8,11 +8,11 @@ description: "" Learn how to apply differential privacy to achieve the maximum level of privacy with mathematical guarantees. This tutorial explores the privacy-utility tradeoff and demonstrates how to configure differential privacy parameters for optimal results. -If you have not yet completed the [Safe Synthesizer 101](/documentation/safe-synthesizer/tutorials/safe-synthesizer-101) tutorial, consider starting there first. +If you have not yet completed the [Safe Synthesizer 101](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101) tutorial, consider starting there first. ## Prerequisites -- Understanding of [differential privacy](/documentation/safe-synthesizer-reference/about/data-synthesis) +- Understanding of [differential privacy](/documentation/synthesize-safe-data/about/data-synthesis) - Safe Synthesizer deployment with GPU resources --- diff --git a/docs/safe-synthesizer/tutorials/index.mdx b/docs/safe-synthesizer/tutorials/index.mdx index eb77aec049e..2f87ddfe8d3 100644 --- a/docs/safe-synthesizer/tutorials/index.mdx +++ b/docs/safe-synthesizer/tutorials/index.mdx @@ -10,7 +10,7 @@ Learn how to run NeMo Safe Synthesizer jobs through hands-on tutorials to genera Before starting any tutorial, ensure you have: -- [NeMo Safe Synthesizer deployed](/documentation/safe-synthesizer-reference/getting-started) using Docker Compose or Helm +- [NeMo Safe Synthesizer deployed](/documentation/synthesize-safe-data) using Docker Compose or Helm - Python environment with `nemo-platform` SDK installed: ```bash pip install nemo-platform[all] @@ -23,7 +23,7 @@ Before starting any tutorial, ensure you have: - + Learn the basics with your first Safe Synthesizer job, leveraging smart defaults. This tutorial covers uploading data, running a synthesis job with PII replacement, and reviewing evaluation reports. @@ -46,7 +46,7 @@ Learn the basics with your first Safe Synthesizer job, leveraging smart defaults - + Apply differential privacy to achieve the maximum level of privacy with mathematical guarantees. This tutorial explores the privacy-utility tradeoff and how to configure differential privacy parameters. @@ -69,11 +69,11 @@ Apply differential privacy to achieve the maximum level of privacy with mathemat After completing these tutorials, explore: -- **[index](/documentation/safe-synthesizer-reference/about/overview)**: Understand core concepts and components +- **[index](/documentation/synthesize-safe-data/about)**: Understand core concepts and components --- ## Need Help? - Check the [GitHub Issues](https://github.com/NVIDIA/GenerativeAIExamples/issues) for known issues -- Review the [jobs](/documentation/safe-synthesizer-reference/about/jobs) guide for job management +- Review the [jobs](/documentation/synthesize-safe-data/about/jobs) guide for job management diff --git a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx index 254a4108bd7..010fedb346b 100644 --- a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx +++ b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx @@ -10,7 +10,7 @@ Learn the fundamentals of NeMo Safe Synthesizer by creating your first Safe Synt Before you begin, make sure that you have: -- Access to a deployment of NeMo Safe Synthesizer (see [getting-started](/documentation/safe-synthesizer-reference/getting-started)) +- Access to a deployment of NeMo Safe Synthesizer (see [getting-started](/documentation/synthesize-safe-data)) - **An NVIDIA GPU with 80 GB+ VRAM** — Safe Synthesizer requires GPU access for model training, even when using remote inference for other services. Verify with `nvidia-smi`. - Python environment with `nemo-platform` SDK installed - Basic understanding of Python and pandas @@ -130,7 +130,7 @@ print(f"✅ Using model provider: {provider_name}") -If you prefer not to send column data to `build.nvidia.com`, you can [deploy your own LLM](/documentation/models-and-inference/tutorials/deploy-models) and create a custom model provider. Pass the fully-qualified provider name (`workspace/provider-name`) to `.with_classify_model_provider()` instead. +If you prefer not to send column data to `build.nvidia.com`, you can deploy your own LLM and create a custom model provider. Pass the fully-qualified provider name (`workspace/provider-name`) to `.with_classify_model_provider()` instead. @@ -324,8 +324,8 @@ The evaluation report contains two high-level scores: Synthetic Quality Score (S Now that you've completed your first Safe Synthesizer job, try the local CLI path: -- [Local and Subprocess Execution](../about/host-local-development.md) - run Safe Synthesizer directly on a host GPU -- [Getting Started](../getting-started.md) - review local runtime prerequisites +- [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution) - run Safe Synthesizer directly on a host GPU +- [Getting Started](/documentation/synthesize-safe-data) - review local runtime prerequisites ### Try These Next @@ -374,7 +374,7 @@ print(f"Total jobs: {len(all_jobs.data)}") - Use smaller model (adjust `training.pretrained_model`) - Check GPU availability -For local CLI failures, see [Local and Subprocess Execution](../about/host-local-development.md). +For local CLI failures, see [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution). **Error: "Dataset must have at least 200 records to use holdout."** From 8ae5b5816a8b224dc0ee7abb614d1fb1734301f9 Mon Sep 17 00:00:00 2001 From: Yamini Date: Tue, 9 Jun 2026 17:51:35 -0400 Subject: [PATCH 25/26] docs: resolve CodeRabbit conversion findings (MDX content fixes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address still-valid CodeRabbit review comments on the Fern docs port. The .md-link findings were already fixed upstream; these cover the conversion content/cosmetic bugs. Restored real example content dropped during MkDocs→Fern conversion (recovered from the original _snippets on main): - format-training-dataset: inline the conversational, detailed-thinking on/off, and tool-calling JSON examples into the Accordion/Tabs - import-hf-model: remove the empty "Python SDK" tabs (snippet source no longer in repo), keeping the working cURL/CLI tabs; fix invalid Python indentation in the deployed-model test snippet Other conversion fixes: - agents/security: detect guardrails via model_name, not base_url - agents/index: drop api_key from sample to match guidance - auditor/tutorials: remove MkDocs anchor + md-tag chips - cli/index: fix indented-as-code text, bash fence, "open-source" - auth/deployment/gateway: valid YAML indentation in Envoy example - customizer/get-job-status: nest example inside Accordion - customizer/models/data-format: unescape braces in JSON samples - customizer/tutorials/index: extensionless tutorial card hrefs - customizer/tutorials/metrics: single contiguous python fence - customizer/index: correct metrics tutorial card href - customizer/manage-customization-jobs/index: fix malformed link - evaluator/benchmarks/hf-secret: set page title CI-security findings (CR-1..5) intentionally left for docs/CI owner. Co-Authored-By: Claude Opus 4.8 --- docs/agents/index.mdx | 1 - docs/agents/security.mdx | 6 +- docs/auditor/tutorials/index.mdx | 3 +- docs/auth/deployment/gateway.mdx | 26 +-- docs/cli/index.mdx | 10 +- docs/customizer/index.mdx | 2 +- .../get-job-status.mdx | 3 +- .../manage-customization-jobs/index.mdx | 2 +- docs/customizer/models/data-format.mdx | 12 +- .../tutorials/format-training-dataset.mdx | 71 ++++++++ docs/customizer/tutorials/import-hf-model.mdx | 162 +++++------------- docs/customizer/tutorials/index.mdx | 12 +- docs/customizer/tutorials/metrics.mdx | 3 +- docs/evaluator/benchmarks/hf-secret.mdx | 2 +- 14 files changed, 154 insertions(+), 161 deletions(-) diff --git a/docs/agents/index.mdx b/docs/agents/index.mdx index 49b5ea808c0..1d535d101a0 100644 --- a/docs/agents/index.mdx +++ b/docs/agents/index.mdx @@ -46,7 +46,6 @@ functions: llms: llm: _type: openai - api_key: not-used # injected at deploy time model_name: nvidia-nemotron-3-nano-30b-a3b workflow: diff --git a/docs/agents/security.mdx b/docs/agents/security.mdx index c3bedd7a03a..404ebd071b7 100644 --- a/docs/agents/security.mdx +++ b/docs/agents/security.mdx @@ -102,9 +102,9 @@ nemo skills show agents-secure What it does under the hood: - Lists deployed agents and prompts you to choose one. -- Inspects each LLM's `base_url`. If it does not route through a - guardrails virtual model, suggests creating one with a content-safety, - topic-control, or safety-guard backend. +- Inspects each LLM's `model_name`. If it does not reference a guarded + virtual model (one with a content-safety, topic-control, or + safety-guard backend), suggests creating one. - Names the recommended guardrails catalog model and walks you through creating the guarded virtual model. - Persists suggestions to the `nemo-agent-security` fileset. diff --git a/docs/auditor/tutorials/index.mdx b/docs/auditor/tutorials/index.mdx index 3017c1077a2..7357530b1c8 100644 --- a/docs/auditor/tutorials/index.mdx +++ b/docs/auditor/tutorials/index.mdx @@ -2,7 +2,6 @@ title: "NeMo Auditor Tutorials" description: "" --- - Use these tutorials to get hands-on with [NeMo Auditor](/documentation/vulnerability-scanning). @@ -16,7 +15,7 @@ Set up [a local instance of the platform](/documentation/get-started), install g Persist a configuration and target, run a single audit in-process, and inspect the resulting JSONL, HTML, and hitlog report artifacts. -getting started local execution nemo-auditor +getting started · local execution · nemo-auditor diff --git a/docs/auth/deployment/gateway.mdx b/docs/auth/deployment/gateway.mdx index ab5d85b0db6..70ab56e0d64 100644 --- a/docs/auth/deployment/gateway.mdx +++ b/docs/auth/deployment/gateway.mdx @@ -117,19 +117,19 @@ Add `request_headers_to_remove` to your route configuration: ```yaml route_config: -virtual_hosts: -- name: nmp_service -domains: ["*"] -request_headers_to_remove: -- "x-nmp-principal-id" -- "x-nmp-principal-email" -- "x-nmp-principal-groups" -- "x-nmp-principal-on-behalf-of" -- "x-nmp-scopes" -- "x-nmp-authorized" -routes: -- match: { prefix: "/" } -route: { cluster: nmp_backend } + virtual_hosts: + - name: nmp_service + domains: ["*"] + request_headers_to_remove: + - "x-nmp-principal-id" + - "x-nmp-principal-email" + - "x-nmp-principal-groups" + - "x-nmp-principal-on-behalf-of" + - "x-nmp-scopes" + - "x-nmp-authorized" + routes: + - match: { prefix: "/" } + route: { cluster: nmp_backend } ``` diff --git a/docs/cli/index.mdx b/docs/cli/index.mdx index f417b388d81..b48fc037ffd 100644 --- a/docs/cli/index.mdx +++ b/docs/cli/index.mdx @@ -18,12 +18,16 @@ The NeMo Platform CLI (`nemo`) is a command-line tool for interacting with NeMo -This package downloads and installs additional third-party open source software projects. Review the license terms of these open source projects before use. +This package downloads and installs additional third-party open-source software projects. Review the license terms of these open-source projects before use. - If you previously installed the `nemo-microservices` package, uninstall it first to avoid conflicts: - pip uninstall nemo-microservices +If you previously installed the `nemo-microservices` package, uninstall it first to avoid conflicts: + +```bash +pip uninstall nemo-microservices +``` + ### Install in a Virtual Environment ```bash diff --git a/docs/customizer/index.mdx b/docs/customizer/index.mdx index b5ff1751fa6..b5a587b6175 100644 --- a/docs/customizer/index.mdx +++ b/docs/customizer/index.mdx @@ -124,7 +124,7 @@ Learn how to compress a larger teacher model into a smaller student model. nemo-customizer knowledge-distillation - + Learn how to check job metrics using MLFlow or Weights & Biases. diff --git a/docs/customizer/manage-customization-jobs/get-job-status.mdx b/docs/customizer/manage-customization-jobs/get-job-status.mdx index 5013f63fb94..fcc700fa17f 100644 --- a/docs/customizer/manage-customization-jobs/get-job-status.mdx +++ b/docs/customizer/manage-customization-jobs/get-job-status.mdx @@ -64,7 +64,6 @@ for step in status.steps or []: - **Active Job (Training in Progress)** ```json @@ -271,3 +270,5 @@ for step in status.steps or []: ] } ``` + + diff --git a/docs/customizer/manage-customization-jobs/index.mdx b/docs/customizer/manage-customization-jobs/index.mdx index 55d9390b7c6..631b40efad7 100644 --- a/docs/customizer/manage-customization-jobs/index.mdx +++ b/docs/customizer/manage-customization-jobs/index.mdx @@ -16,7 +16,7 @@ This design keeps adapters organized with their parent models and simplifies dep ## Prerequisites -Before you can customize a model using a customization job, make sure that you have `prepared and uploaded a dataset <../tutorials/format-training-dataset>` to the dataset repository. See also [format-training-dataset](/documentation/fine-tune-models/tutorials/format-training-dataset) for dataset formatting requirements. +Before you can customize a model using a customization job, make sure that you have [prepared and uploaded a dataset](/documentation/fine-tune-models/tutorials/format-training-dataset) to the dataset repository. --- diff --git a/docs/customizer/models/data-format.mdx b/docs/customizer/models/data-format.mdx index a3aafa4be54..2e1b2f2d419 100644 --- a/docs/customizer/models/data-format.mdx +++ b/docs/customizer/models/data-format.mdx @@ -180,9 +180,9 @@ Each line in your JSONL file must contain a JSON object with these required fiel #### Example Dataset Entry ``` -\{ +{ "prompt": [ - \{ + { "role": "user", "content": "Can you write some code in python to fetch folders based on path from a Site in Sharepoint?" } @@ -204,7 +204,7 @@ Each line in your JSONL file must contain a JSON object with these required fiel #### Example Dataset Entry ``` -\{ +{ "prompt": "your string", "completion": "your expected response" } @@ -224,11 +224,11 @@ Each line in your JSONL file must contain a JSON object with these required fiel #### Example Dataset Entry ``` -\{ +{ "system": "you are a robot", "conversations": [ - \{"from": "User", "value": "Choose a number that is greater than 0 and less than 2\n"}, - \{"from": "Assistant", "value": "1"} + {"from": "User", "value": "Choose a number that is greater than 0 and less than 2\n"}, + {"from": "Assistant", "value": "1"} ] } ``` diff --git a/docs/customizer/tutorials/format-training-dataset.mdx b/docs/customizer/tutorials/format-training-dataset.mdx index 7471f7b3df4..dc675e400a2 100644 --- a/docs/customizer/tutorials/format-training-dataset.mdx +++ b/docs/customizer/tutorials/format-training-dataset.mdx @@ -97,6 +97,23 @@ Example entry formatted for JSONL dataset file: For illustrative purposes only, we show an example entry as multi-line JSON. +```json +{ + "messages": [ + { + "role": "system", + "content": "" + }, { + "role": "user", + "content": "" + }, { + "role": "assistant", + "content": "" + } + ] +} +``` + ### Reasoning Considerations @@ -117,9 +134,25 @@ For example, if your original system message is `You are a helpful assistant.`, +```json +{"messages": [ + {"role": "system", "content": "detailed thinking off"}, + {"role": "user", "content": "What is 2 + 2?"}, + {"role": "assistant", "content": "4"} +]} +``` + +```json +{"messages": [ + {"role": "system", "content": "detailed thinking on"}, + {"role": "user", "content": "What is 2 + 2?"}, + {"role": "assistant", "content": "To solve 2 + 2, add 2 and 2 together. The answer is 4."} +]} +``` + @@ -144,6 +177,44 @@ Every sample must be a single line as in this example below. For illustrative purposes only, we show an example entry as multi-line JSON. +```json +{ + "messages": [ + { + "role": "user", + "content": "" + }, + { + "role": "assistant", + "content": "", + "tool_calls": [{ + "type": "function", + "function": { + "name": "fibonacci", + "arguments": {"n": 20} + } + }] + } + ], + "tools": [{ + "type": "function", + "function": { + "name": "fibonacci", + "description": "Calculates the nth Fibonacci number.", + "parameters": { + "type": "object", + "properties": { + "n": { + "description": "The position of the Fibonacci number.", + "type": "integer" + } + } + } + } + }] +} +``` + ##### Shared Tools diff --git a/docs/customizer/tutorials/import-hf-model.mdx b/docs/customizer/tutorials/import-hf-model.mdx index eb3212dfee1..a4a43ba4f8c 100644 --- a/docs/customizer/tutorials/import-hf-model.mdx +++ b/docs/customizer/tutorials/import-hf-model.mdx @@ -65,13 +65,6 @@ Next, create a model repository in the NeMo Data Store and upload the downloaded ### Create Namespace and Model Repository - - - - - - - ```bash # Set environment variables - Update these to match your deployment export NAMESPACE="my-org" @@ -97,9 +90,6 @@ curl -X POST "${DATASTORE_URL}/v1/hf/api/repos/create" \ }' ``` - - - ### Upload Model Files to Data Store Upload the downloaded model files to the Data Store repository: @@ -108,13 +98,6 @@ Upload the downloaded model files to the Data Store repository: After uploading the model files to the Data Store, create a model entity in the Entity Store to register the model with its metadata and specifications for use in customization jobs. - - - - - - - ```bash # Create model entity in Entity Store curl -X POST "${NEMO_BASE_URL}/v1/models" \ @@ -140,20 +123,10 @@ curl -X POST "${NEMO_BASE_URL}/v1/models" \ }' | jq . ``` - - - ## Deploy the Base Model Deploy the base model for inference with LoRA adapter support enabled, allowing it to load fine-tuned adapters from customization jobs. - - - - - - - ```bash curl "${NEMO_BASE_URL}/v1/deployment/configs" \ -X POST \ @@ -192,20 +165,10 @@ curl "${NEMO_BASE_URL}/v1/deployment/model-deployments" \ ``` - - - ## Create Customization Target Create a customization target that references the uploaded model in the Data Store. - - - - - - - ```bash # Create customization target @@ -224,18 +187,8 @@ curl -X POST \ }' | jq . ``` - - - Wait for the model to be downloaded and ready: - - - - - - - ```bash # Check target status with comprehensive handling while true; do @@ -268,20 +221,10 @@ done ``` - - - ## Create Customization Configuration Create a configuration for LoRA fine-tuning: - - - - - - - ```bash # Create customization configuration curl -X POST \ @@ -310,9 +253,6 @@ curl -X POST \ }' | jq . ``` - - - --- ## Prepare Training and Validation Datasets @@ -531,13 +471,6 @@ print(f"Created dataset entity: {dataset.namespace}/{dataset.name}") Start the LoRA fine-tuning job. The job will create an output artifact with the name specified in `output`, which you'll use later to access your fine-tuned model for inference. - - - - - - - ```bash # Create job and capture job ID RESPONSE=$(curl -s -X POST \ @@ -570,9 +503,6 @@ echo "Started job with ID: $JOB_ID" echo "Output name: $OUTPUT_NAME" ``` - - - Copy the following values from the response: - `id` (Job ID) @@ -582,13 +512,6 @@ We'll need them later to monitor the job's status and access the fine-tuned mode Check job progress: - - - - - - - ```bash # Monitor job status with comprehensive handling while true; do @@ -630,9 +553,6 @@ sleep 60 # Wait 1 minute before checking again done ``` - - - --- ## Test the Deployed Model @@ -665,50 +585,50 @@ lora_model_id = f"{NAMESPACE}/{MODEL_NAME}-lora@v{MODEL_VERSION}" # First, check if the models are available for inference # Note: The base model deployment must be running and registered with the NIM proxy try: -# List models from the entity store (shows all registered models) -models_response = client.models.list() -available_models = models_response.data - -print("Registered models:") -for model in available_models: -print(f" - {model.id}") - -# Test base model -print(f"\nTesting base model: {base_model_id}") -base_response = client.chat.completions.create( -model=base_model_id, -messages=[ -{ -"role": "user", -"content": "Hello, can you help me?" -} -], -max_tokens=100, -temperature=0.7 -) - -print("Base model response:") -print(base_response.choices[0].message.content) - -# Test LoRA-adapted model (if available) -print(f"\nTesting LoRA-adapted model: {lora_model_id}") -lora_response = client.chat.completions.create( -model=lora_model_id, -messages=[ -{ -"role": "user", -"content": "Hello, can you help me?" -} -], -max_tokens=100, -temperature=0.7 -) + # List models from the entity store (shows all registered models) + models_response = client.models.list() + available_models = models_response.data + + print("Registered models:") + for model in available_models: + print(f" - {model.id}") + + # Test base model + print(f"\nTesting base model: {base_model_id}") + base_response = client.chat.completions.create( + model=base_model_id, + messages=[ + { + "role": "user", + "content": "Hello, can you help me?" + } + ], + max_tokens=100, + temperature=0.7 + ) + + print("Base model response:") + print(base_response.choices[0].message.content) + + # Test LoRA-adapted model (if available) + print(f"\nTesting LoRA-adapted model: {lora_model_id}") + lora_response = client.chat.completions.create( + model=lora_model_id, + messages=[ + { + "role": "user", + "content": "Hello, can you help me?" + } + ], + max_tokens=100, + temperature=0.7 + ) -print("LoRA-adapted model response:") -print(lora_response.choices[0].message.content) + print("LoRA-adapted model response:") + print(lora_response.choices[0].message.content) except Exception as e: -print(f"Error testing model: {e}") + print(f"Error testing model: {e}") ``` diff --git a/docs/customizer/tutorials/index.mdx b/docs/customizer/tutorials/index.mdx index 5a6e0c5a937..f24ff9c0748 100644 --- a/docs/customizer/tutorials/index.mdx +++ b/docs/customizer/tutorials/index.mdx @@ -44,35 +44,35 @@ Learn how to format datasets for different model types. - + Learn how to perform supervised fine-tuning with LoRA adapters using custom data. nemo-customizer - + Learn how to perform supervised fine-tuning using custom data by modifying the all training parameters. nemo-customizer - + Learn how to align a model with DPO (Direct Preference Optimization) to prefer certain kinds of responses over others. nemo-customizer dpo - + Learn how to compress a larger teacher model into a smaller student model using knowledge distillation. nemo-customizer knowledge-distillation - + Learn how to fine-tune embedding models using LoRA merged training for improved question-answering and retrieval tasks. @@ -93,7 +93,7 @@ Learn how to check job metrics using MLflow or Weights & Biases. nemo-customizer mlflow wandb - + Learn how to optimize the token-per-GPU throughput for a LoRA optimization job. diff --git a/docs/customizer/tutorials/metrics.mdx b/docs/customizer/tutorials/metrics.mdx index 1155bf516c2..ddf9f2bef6c 100644 --- a/docs/customizer/tutorials/metrics.mdx +++ b/docs/customizer/tutorials/metrics.mdx @@ -94,14 +94,13 @@ If your customization job was created with W&B integration enabled (see [Weights 2. Find the run corresponding to your customization job 3. View training and validation loss curves, learning rate schedules, and other metrics under the run's dashboard +```python client = NeMoPlatform( base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), workspace="default", ) # Create a customization job with W&B integration - -```python job = client.customization.jobs.create( name="my-wandb-job", workspace="default", diff --git a/docs/evaluator/benchmarks/hf-secret.mdx b/docs/evaluator/benchmarks/hf-secret.mdx index 5eab395c89f..ac87458131d 100644 --- a/docs/evaluator/benchmarks/hf-secret.mdx +++ b/docs/evaluator/benchmarks/hf-secret.mdx @@ -1,5 +1,5 @@ --- -title: "" +title: "Create a Hugging Face Secret" description: "" --- Most benchmarks require a Hugging Face token (`hf_token`) to access gated datasets. Create this secret before running evaluations: From 8d2b15e72e8eff4847d62753a8780ed329163ba4 Mon Sep 17 00:00:00 2001 From: Yamini Date: Tue, 9 Jun 2026 18:23:29 -0400 Subject: [PATCH 26/26] ci(fern-docs): harden workflows per review (pin actions, least-priv) Resolve the CodeRabbit CI-security findings on the Fern docs workflows: - Pin actions/checkout and actions/setup-node to commit SHAs (v6.0.3 / v6.4.0), matching the existing SHA-pinned upload/download-artifact. - Add persist-credentials: false to all checkout steps (none need the git token after checkout). - Preview-comment job: take head_ref from the trusted github.event.workflow_run.head_branch instead of the PR-build artifact, and validate the artifact-supplied pr_number is a plain integer before using it in any privileged context. Keeps the fork-safe two-workflow design (workflow_run.pull_requests is empty for forks) while removing the untrusted-metadata injection vector. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/fern-docs-ci.yaml | 6 ++++-- .github/workflows/fern-docs-preview-build.yaml | 3 ++- .github/workflows/fern-docs-preview-comment.yaml | 16 +++++++++++++--- .github/workflows/publish-fern-docs.yaml | 6 ++++-- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/workflows/fern-docs-ci.yaml b/.github/workflows/fern-docs-ci.yaml index e39f0b44de6..ac1637bbd09 100644 --- a/.github/workflows/fern-docs-ci.yaml +++ b/.github/workflows/fern-docs-ci.yaml @@ -20,10 +20,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: '20' cache: 'npm' diff --git a/.github/workflows/fern-docs-preview-build.yaml b/.github/workflows/fern-docs-preview-build.yaml index aa40c0531ad..c4573400bc3 100644 --- a/.github/workflows/fern-docs-preview-build.yaml +++ b/.github/workflows/fern-docs-preview-build.yaml @@ -26,9 +26,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout PR - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 + persist-credentials: false - name: Save PR metadata env: diff --git a/.github/workflows/fern-docs-preview-comment.yaml b/.github/workflows/fern-docs-preview-comment.yaml index 3de5b29f1fe..4477b20c2df 100644 --- a/.github/workflows/fern-docs-preview-comment.yaml +++ b/.github/workflows/fern-docs-preview-comment.yaml @@ -36,12 +36,22 @@ jobs: - name: Read PR metadata id: metadata + env: + # head_ref comes from the trusted workflow_run event, not the artifact. + WORKFLOW_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} run: | - echo "pr_number=$(cat preview-metadata/pr_number)" >> "$GITHUB_OUTPUT" - echo "head_ref=$(cat preview-metadata/head_ref)" >> "$GITHUB_OUTPUT" + echo "head_ref=$WORKFLOW_HEAD_BRANCH" >> "$GITHUB_OUTPUT" + # pr_number originates from the (untrusted) PR-build artifact; validate + # it is a plain integer before using it in any privileged context. + PR_NUMBER=$(cat preview-metadata/pr_number) + if ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then + echo "::error::Invalid PR number in artifact metadata: '$PR_NUMBER'" + exit 1 + fi + echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT" - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: '20' diff --git a/.github/workflows/publish-fern-docs.yaml b/.github/workflows/publish-fern-docs.yaml index b7b55a79da6..66cfec036a7 100644 --- a/.github/workflows/publish-fern-docs.yaml +++ b/.github/workflows/publish-fern-docs.yaml @@ -33,10 +33,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: '20'