diff --git a/.claude/commands/build-docs.md b/.claude/commands/build-docs.md index eedd058556..dcfb564fac 100644 --- a/.claude/commands/build-docs.md +++ b/.claude/commands/build-docs.md @@ -1,18 +1,18 @@ --- -description: Build documentation with various options +description: Build / validate the Fern documentation --- -Build the documentation using one of these modes: +The docs are built with Fern (not MkDocs). From the repo root: -1. **Fast build** (default) - Quick iteration, skips SDK docs: +1. **Validate** (what CI runs — `fern check` + MDX validation + gated-link check): ```bash - cd docs && make html-fast + make docs-check ``` -2. **Full build** - Complete build including SDK docs: +2. **Check for broken links:** ```bash - cd docs && make html + make docs-broken-links ``` -Ask the user which mode they prefer, then run the appropriate command. -After building, let them know they can view the docs at `docs/_build/html/index.html`. +First run on a machine needs `make docs-deps` (installs `docs/fern` tooling). +To preview locally with live reload, use `/start-docs-server` (`make docs`). The published site is built by the `Publish Fern Docs` workflow on push to `main`. See `docs/AGENTS.md` and `docs/fern/README.md`. diff --git a/.claude/commands/start-docs-server.md b/.claude/commands/start-docs-server.md index d504b2a1d4..2afc18acf7 100644 --- a/.claude/commands/start-docs-server.md +++ b/.claude/commands/start-docs-server.md @@ -1,14 +1,13 @@ --- -allowed-tools: Bash(cd docs && make live:*) -description: Start a live documentation server with auto-reload +allowed-tools: Bash(make docs:*), Bash(make docs-deps:*) +description: Start the Fern docs dev server with live reload --- -Start a live documentation server for development with full SDK documentation. +Start the Fern documentation dev server for local preview. From the repo root: -Run: ```bash -cd docs && make live +make docs ``` -The server will be available at `http://localhost:8000` after the initial build completes. -It will automatically rebuild when you change documentation files. +This runs `fern docs dev` and prints a local URL (e.g. http://localhost:3000); it reloads as you edit `.mdx` files. If this is the first run on the machine, run `make docs-deps` first to install the `docs/fern` tooling. +Note: gated (unready) pages are intentionally excluded from the build and will 404 in the preview — that is expected. See `docs/AGENTS.md`. diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml deleted file mode 100644 index eb09e54d0c..0000000000 --- a/.github/workflows/docs.yaml +++ /dev/null @@ -1,324 +0,0 @@ -name: Documentation - -on: - pull_request: - types: [opened, synchronize, reopened, closed] - paths: - - "docs/**" - - "mkdocs.yml" - - "openapi/openapi.yaml" - - ".github/workflows/docs.yaml" - push: - branches: [main] - tags: ["**"] - paths: - - "docs/**" - - "mkdocs.yml" - - "openapi/openapi.yaml" - workflow_dispatch: - -permissions: - contents: write - issues: write - pages: read - pull-requests: write - -env: - PAGES_BRANCH: gh-pages - -jobs: - build: - name: Build docs - if: github.event_name != 'pull_request' || github.event.action != 'closed' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Fetch docs image assets - shell: bash - run: | - git lfs install --local - git lfs pull --include="docs/images/**" --exclude="" - for asset in docs/assets/favicon.ico docs/assets/nvidia-logo-white.png docs/images/main_nv_logo_square.png docs/images/favicon.ico docs/images/nvidia-logo-white.png; do - if head -n 1 "$asset" | grep -q '^version https://git-lfs.github.com/spec/v1$'; then - echo "::error file=${asset}::Docs asset is a Git LFS pointer; GitHub Pages would publish a broken image." - exit 1 - fi - done - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Build documentation - shell: bash - run: | - set -o pipefail - make -C docs publish 2>&1 | tee build.log - - - name: Check for errors - if: always() - run: | - if grep -qE "^(ERROR|CRITICAL)" build.log; then - echo "::error::MkDocs build produced errors" - grep -E "^(ERROR|CRITICAL)" build.log - exit 1 - fi - echo "Build completed successfully" - - - name: Upload site artifact - if: success() - uses: actions/upload-artifact@v4 - with: - name: docs-site - path: site/ - retention-days: 14 - - - name: Add PR comment with review instructions - if: > - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name != github.repository && - success() - uses: actions/github-script@v7 - with: - script: | - const marker = ''; - const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const body = [ - marker, - '### Documentation build succeeded', - '', - '**To review the docs locally:**', - `1. Download the \`docs-site\` artifact from the [Actions run](${runUrl})`, - '2. Unzip and open `index.html` in your browser', - '', - '**Or build from source:**', - '```bash', - 'make -C docs live', - '```', - 'Then open http://127.0.0.1:8000', - '', - 'To include temporarily hidden docs locally:', - '```bash', - 'make -C docs live-with-unready', - '```', - ].join('\n'); - - // Check if we already posted a comment - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - const existing = comments.find(c => c.body.includes(marker) || c.body.includes('Documentation build succeeded')); - if (existing) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body: body, - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: body, - }); - } - - deploy-pages: - name: Deploy docs to GitHub Pages - if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))) - needs: build - runs-on: ubuntu-latest - concurrency: - group: gh-pages-${{ github.repository }}-gh-pages - cancel-in-progress: false - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Fetch docs image assets - shell: bash - run: | - git lfs install --local - git lfs pull --include="docs/images/**" --exclude="" - for asset in docs/assets/favicon.ico docs/assets/nvidia-logo-white.png docs/images/main_nv_logo_square.png docs/images/favicon.ico docs/images/nvidia-logo-white.png; do - if head -n 1 "$asset" | grep -q '^version https://git-lfs.github.com/spec/v1$'; then - echo "::error file=${asset}::Docs asset is a Git LFS pointer; GitHub Pages would publish a broken image." - exit 1 - fi - done - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Configure git for mike - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - - name: Resolve GitHub Pages URL - id: pages - env: - GH_TOKEN: ${{ github.token }} - run: | - PAGES_URL=$(gh api "repos/${GITHUB_REPOSITORY}/pages" --jq .html_url) - echo "url=${PAGES_URL%/}/" >> "$GITHUB_OUTPUT" - - - name: Deploy docs - shell: bash - env: - DOCS_SITE_URL: ${{ steps.pages.outputs.url }} - run: | - if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then - VERSION="${GITHUB_REF_NAME#docs/}" - VERSION="${VERSION#v}" - make -C docs deploy-pages DOCS_SITE_URL="${DOCS_SITE_URL}" DOCS_VERSION="${VERSION}" DOCS_ALIAS=latest - else - make -C docs deploy-pages DOCS_SITE_URL="${DOCS_SITE_URL}" - fi - - deploy-pr-preview: - name: Deploy PR docs preview - if: > - github.event_name == 'pull_request' && - github.event.action != 'closed' && - github.event.pull_request.head.repo.full_name == github.repository - needs: build - runs-on: ubuntu-latest - concurrency: - group: gh-pages-${{ github.repository }}-gh-pages - cancel-in-progress: false - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Fetch docs image assets - shell: bash - run: | - git lfs install --local - git lfs pull --include="docs/images/**" --exclude="" - for asset in docs/assets/favicon.ico docs/assets/nvidia-logo-white.png docs/images/main_nv_logo_square.png docs/images/favicon.ico docs/images/nvidia-logo-white.png; do - if head -n 1 "$asset" | grep -q '^version https://git-lfs.github.com/spec/v1$'; then - echo "::error file=${asset}::Docs asset is a Git LFS pointer; GitHub Pages would publish a broken image." - exit 1 - fi - done - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Configure git for mike - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - - name: Resolve GitHub Pages URL - id: pages - env: - GH_TOKEN: ${{ github.token }} - run: | - PAGES_URL=$(gh api "repos/${GITHUB_REPOSITORY}/pages" --jq .html_url) - echo "url=${PAGES_URL%/}/" >> "$GITHUB_OUTPUT" - - - name: Deploy PR preview - env: - PAGES_URL: ${{ steps.pages.outputs.url }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: make -C docs deploy-pr-preview PR_NUMBER="${PR_NUMBER}" PR_PREVIEW_SITE_URL="${PAGES_URL}pr-preview/pr-${PR_NUMBER}/" - - - name: Add PR preview comment - uses: actions/github-script@v7 - env: - PREVIEW_URL: ${{ steps.pages.outputs.url }}pr-preview/pr-${{ github.event.pull_request.number }}/pr-${{ github.event.pull_request.number }}/ - with: - script: | - const marker = ''; - const previewUrl = process.env.PREVIEW_URL; - const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const shortSha = context.payload.pull_request.head.sha.slice(0, 7); - const body = [ - marker, - '### Documentation preview is ready', - '', - `Preview: [${previewUrl}](${previewUrl})`, - '', - `Built from \`${shortSha}\` in [workflow run](${runUrl}).`, - '', - 'This preview is deployed from this PR branch, updates when docs changes are pushed, and will be removed when the PR closes.', - ].join('\n'); - - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - const existing = comments.find(c => c.body.includes(marker)); - if (existing) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body, - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body, - }); - } - - cleanup-pr-preview: - name: Cleanup PR docs preview - if: > - github.event_name == 'pull_request' && - github.event.action == 'closed' && - github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - concurrency: - group: gh-pages-${{ github.repository }}-gh-pages - cancel-in-progress: false - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true - - - name: Configure git for mike - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - - name: Delete PR preview - run: make -C docs delete-pr-preview PR_NUMBER="${{ github.event.pull_request.number }}" diff --git a/.github/workflows/fern-docs-ci.yaml b/.github/workflows/fern-docs-ci.yaml new file mode 100644 index 0000000000..ac1637bbd0 --- /dev/null +++ b/.github/workflows/fern-docs-ci.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Validates Fern docs configuration on pull requests. +# See docs/fern/README.md and docs/fern/docs.yml `libraries`. + +name: Fern docs (check) + +on: + pull_request: + paths: + - 'docs/**' + - '.github/workflows/fern-docs-ci.yaml' + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: docs/fern/package-lock.json + + - name: Install docs tooling + working-directory: ./docs/fern + run: npm ci + + - name: Validate Fern configuration + working-directory: ./docs/fern + env: + FERN_TOKEN: ${{ secrets.DOCS_FERN_TOKEN }} + run: npm run check + + - name: Check for broken links + working-directory: ./docs/fern + env: + FERN_TOKEN: ${{ secrets.DOCS_FERN_TOKEN }} + run: npm run broken-links diff --git a/.github/workflows/fern-docs-preview-build.yaml b/.github/workflows/fern-docs-preview-build.yaml new file mode 100644 index 0000000000..c4573400bc --- /dev/null +++ b/.github/workflows/fern-docs-preview-build.yaml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Workflow 1 of 2 for Fern doc previews. +# +# Collects the docs/fern/ sources and PR metadata from the (possibly untrusted) +# PR branch and uploads them as an artifact. No secrets are used here, so this +# is safe to run on fork PRs via the regular pull_request trigger. +# +# The companion workflow (fern-docs-preview-comment.yaml) picks up the artifact, +# builds the preview with DOCS_FERN_TOKEN, and posts the PR comment. + +name: "Preview Fern Docs: Build" + +on: + pull_request: + paths: + - 'docs/**' + - '.github/workflows/fern-docs-preview-build.yaml' + +permissions: + contents: read + +jobs: + collect: + runs-on: ubuntu-latest + steps: + - name: Checkout PR + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Save PR metadata + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_REF: ${{ github.head_ref }} + BASE_REF: ${{ github.base_ref }} + run: | + mkdir -p preview-metadata + echo "$PR_NUMBER" > preview-metadata/pr_number + echo "$HEAD_REF" > preview-metadata/head_ref + git diff --name-only "origin/${BASE_REF}...HEAD" -- '*.mdx' > preview-metadata/changed_mdx_files 2>/dev/null || true + + - name: Upload fern sources and metadata + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: fern-preview + path: | + docs/ + preview-metadata/ + retention-days: 1 diff --git a/.github/workflows/fern-docs-preview-comment.yaml b/.github/workflows/fern-docs-preview-comment.yaml new file mode 100644 index 0000000000..4477b20c2d --- /dev/null +++ b/.github/workflows/fern-docs-preview-comment.yaml @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Workflow 2 of 2 for Fern doc previews. +# +# Triggered by workflow_run after "Preview Fern Docs: Build" completes. +# Downloads the docs/ artifact, builds a preview with DOCS_FERN_TOKEN, and +# posts a stable :herb: comment on the PR. This workflow never checks out the +# PR branch directly, keeping secrets isolated from untrusted code. +# +# Required configuration: +# - Organization secret: DOCS_FERN_TOKEN (from `fern token` for the nvidia Fern org) + +name: "Preview Fern Docs: Comment" + +on: + workflow_run: + workflows: ["Preview Fern Docs: Build"] + types: [completed] + +permissions: + pull-requests: write + actions: read + +jobs: + preview: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + steps: + - name: Download fern sources and metadata + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: fern-preview + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Read PR metadata + id: metadata + env: + # head_ref comes from the trusted workflow_run event, not the artifact. + WORKFLOW_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + run: | + echo "head_ref=$WORKFLOW_HEAD_BRANCH" >> "$GITHUB_OUTPUT" + # pr_number originates from the (untrusted) PR-build artifact; validate + # it is a plain integer before using it in any privileged context. + PR_NUMBER=$(cat preview-metadata/pr_number) + if ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then + echo "::error::Invalid PR number in artifact metadata: '$PR_NUMBER'" + exit 1 + fi + echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT" + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '20' + + - name: Generate preview URL + id: generate-docs + env: + FERN_TOKEN: ${{ secrets.DOCS_FERN_TOKEN }} + HEAD_REF: ${{ steps.metadata.outputs.head_ref }} + working-directory: ./docs/fern + run: | + OUTPUT=$(npx -y fern-api@latest generate --docs --preview --id "$HEAD_REF" 2>&1) + echo "$OUTPUT" + URL=$(echo "$OUTPUT" | grep -oP 'Published docs to \K.*(?= \()') + if [ -z "$URL" ]; then + echo "::error::Failed to generate preview URL. See fern output above." + exit 1 + fi + echo "preview_url=$URL" >> "$GITHUB_OUTPUT" + + - name: Build page links for changed MDX files + id: page-links + env: + FERN_TOKEN: ${{ secrets.DOCS_FERN_TOKEN }} + PREVIEW_URL: ${{ steps.generate-docs.outputs.preview_url }} + run: | + CHANGED_FILES="" + if [ -f preview-metadata/changed_mdx_files ]; then + CHANGED_FILES=$(cat preview-metadata/changed_mdx_files) + fi + + if [ -z "$CHANGED_FILES" ] || [ -z "$PREVIEW_URL" ]; then + echo "page_links=" >> "$GITHUB_OUTPUT"; exit 0 + fi + + BASE_URL=$(echo "$PREVIEW_URL" | grep -oP 'https?://[^/]+') + FILES_PARAM=$(echo "$CHANGED_FILES" | tr '\n' ',' | sed 's/,$//' \ + | python3 -c "import sys, urllib.parse; print(urllib.parse.quote(sys.stdin.read().strip(), safe=',/'))") + RESPONSE=$(curl -sf -H "FERN_TOKEN: $FERN_TOKEN" "${PREVIEW_URL}/api/fern-docs/get-slug-for-file?files=${FILES_PARAM}" 2>/dev/null) || { + echo "page_links=" >> "$GITHUB_OUTPUT"; exit 0 + } + + PAGE_LINKS=$(echo "$RESPONSE" | jq -r --arg url "$BASE_URL" \ + '.mappings[] | select(.slug != null) | "- [\(.slug)](\($url)/\(.slug))"') + + if [ -n "$PAGE_LINKS" ]; then + { echo "page_links<> "$GITHUB_OUTPUT" + else + echo "page_links=" >> "$GITHUB_OUTPUT" + fi + + - name: Post or update PR comment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ steps.metadata.outputs.pr_number }} + PREVIEW_URL: ${{ steps.generate-docs.outputs.preview_url }} + PAGE_LINKS: ${{ steps.page-links.outputs.page_links }} + run: | + # Build comment body + BODY=":herb: **Preview your docs:** <${PREVIEW_URL}>" + if [ -n "${PAGE_LINKS}" ]; then + BODY="${BODY} + + Here are the markdown pages you've updated: + ${PAGE_LINKS}" + fi + + # Hidden marker for upsert + MARKER="" + BODY="${BODY} + + ${MARKER}" + + # Find existing comment with marker + COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \ + --jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" | tr -d '\r' | head -1) + + if [ -n "$COMMENT_ID" ]; then + gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \ + -X PATCH -f body="$BODY" + else + gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \ + -f body="$BODY" + fi diff --git a/.github/workflows/publish-fern-docs.yaml b/.github/workflows/publish-fern-docs.yaml new file mode 100644 index 0000000000..66cfec036a --- /dev/null +++ b/.github/workflows/publish-fern-docs.yaml @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Publishes the Fern documentation site when a docs tag is pushed or manually triggered. +# +# To publish: git tag docs/v1.2.0 && git push origin docs/v1.2.0 +# Or use the "Run workflow" button in the Actions tab. +# +# Required configuration: +# - Organization secret: DOCS_FERN_TOKEN (from `fern token` for the nvidia Fern org) + +name: Publish Fern Docs + +on: + push: + branches: + - main + paths: + - 'docs/**' + tags: + - 'docs/v*' + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: fern-publish + cancel-in-progress: true + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '20' + + - name: Publish Docs + env: + FERN_TOKEN: ${{ secrets.DOCS_FERN_TOKEN }} + working-directory: ./docs/fern + run: npm run generate diff --git a/Makefile b/Makefile index 7e4a4930d6..cf6921c3ec 100644 --- a/Makefile +++ b/Makefile @@ -61,13 +61,54 @@ generate-cli-commands: ## Run generation of the CLI commands .PHONY: generate-cli-reference-docs generate-cli-reference-docs: ## Generate the CLI reference documentation - uv run --frozen packages/nemo_platform_ext/scripts/docs_generator.py reference > docs/cli/reference.md - uv run --frozen packages/nemo_platform_ext/scripts/docs_generator.py summary > docs/_snippets/cli-summary.md + uv run --frozen packages/nemo_platform_ext/scripts/docs_generator.py reference > docs/cli/reference.mdx + uv run --frozen packages/nemo_platform_ext/scripts/docs_generator.py summary > docs/fern/snippets/_snippets/cli-summary.mdx .PHONY: generate-config-reference-docs generate-config-reference-docs: ## Generate the platform config reference documentation uv run --frozen generate-config-docs +# ============================================================================ +# Fern documentation site (docs/fern) +# ============================================================================ +# Convenience wrappers around `cd docs/fern && npm run ...` so contributors +# don't have to remember the fern-api invocations. The CI workflows +# (.github/workflows/fern-docs-*.yaml) are the source of truth. See +# docs/fern/README.md and docs/AGENTS.md. First run on a machine: +# `make docs-deps` (and `make docs-login` once for the Fern org). + +.PHONY: docs-deps +docs-deps: ## Install the Fern docs tooling (docs/fern node deps) + cd docs/fern && npm ci + +.PHONY: docs +docs: ## Start the Fern docs dev server (local preview, prints a localhost URL) + cd docs/fern && npm run dev + +.PHONY: docs-check +docs-check: ## Validate the Fern docs (fern check + validate-mdx + gated-link check) + cd docs/fern && npm run check + +.PHONY: docs-broken-links +docs-broken-links: ## Report broken links across the built docs + cd docs/fern && npm run broken-links + +.PHONY: docs-fix-links +docs-fix-links: ## Delink references from published pages into gated (unready) pages + cd docs/fern && npm run fix:gated-links + +.PHONY: docs-preview +docs-preview: ## Build a shared Fern preview URL (needs DOCS_FERN_TOKEN) + cd docs/fern && npm run preview + +.PHONY: docs-login +docs-login: ## One-time Fern CLI auth (for the nvidia Fern org) + npx -y fern-api@latest login + +.PHONY: docs-publish +docs-publish: ## Trigger the Publish Fern Docs workflow (normally runs on push to main) + gh workflow run publish-fern-docs.yaml + .PHONY: update-cli update-cli: generate-cli-commands vendor-nemo-platform-ext generate-cli-reference-docs diff --git a/docs/.gitignore b/docs/.gitignore index 464ca61288..d5403b578f 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -22,6 +22,11 @@ _test_serve/ *.7z *.tmp *latest* +!fern/versions/latest.yml + +# Fern build artifact: auto-generated API examples cache (regenerated on every +# docs build; not referenced by generators.yml, so it must not be committed) +fern/openapi/ai_examples_override.yml _static/versions1.json helm/_values.yaml diff --git a/docs/AGENTS.md b/docs/AGENTS.md new file mode 100644 index 0000000000..96d710bc73 --- /dev/null +++ b/docs/AGENTS.md @@ -0,0 +1,39 @@ +# Documentation (Fern) + +The docs site is built with **[Fern](https://buildwithfern.com)**, not MkDocs/Sphinx. Pages are `.mdx` under `docs/`; the Fern config lives in `docs/fern/`. Read `docs/fern/README.md` for the full guide — this file is the short version for anyone (human or agent) editing docs. + +## Build / preview / check + +Run these from the repo root (they wrap `cd docs/fern && npm run …`): + +| Command | What it does | +|---|---| +| `make docs-deps` | Install docs tooling (first run on a machine) | +| `make docs` | Local dev server (live preview) | +| `make docs-check` | `fern check` + MDX validation + gated-link check (what CI runs) | +| `make docs-broken-links` | Report broken links | +| `make docs-fix-links` | Auto-delink references into gated pages | + +Local preview and the published site read the **same** `docs/fern/versions/latest.yml`, so what you see locally is what ships. + +## Rules that bite if you miss them + +- **Navigation is the build.** Fern only builds pages listed in `docs/fern/versions/latest.yml`. A `.mdx` not in the nav is **not built** (404, not indexed) — that is how unready features are gated. Do **not** use `hidden: true` for gating (it still builds/serves the page). +- **Gated (unready) features** stay in the repo but out of the nav: `auth/`, `customizer/`, `safe-synthesizer/`, `set-up/` + `helm/`, `evaluator/benchmarks/`, and a few individual pages. Ready-to-paste nav blocks for re-publishing are in `docs/fern/gated-nav.yml`. To publish one: move its block into `latest.yml`, re-add inbound links, run `make docs-check && make docs-broken-links`. +- **Don't link into gated pages.** A link from a published page into a gated page is a dead link. `make docs-check` fails on it; `make docs-fix-links` delinks it to plain text. (Replaces the old MkDocs `hide_unready_docs` auto-delinking.) +- **Internal links** use canonical nav URLs like `/documentation/get-started/core-concepts/workspaces`, not relative `.md`/source paths. `make docs-broken-links` is the check. +- **No `{{variable}}` substitutions.** Fern has no substitution step; product names are inlined as literal text. (Prompt-template tokens like `` `{{input}}` `` inside backticks are real content — leave them.) + +## Generated pages — do not hand-edit + +These are generated; edit the source and regenerate: + +| Page | Regenerate with | +|---|---| +| `docs/cli/reference.mdx`, `docs/fern/snippets/_snippets/cli-summary.mdx` | `make generate-cli-reference-docs` | +| `docs/set-up/config-reference.mdx` | `make generate-config-reference-docs` | +| `docs/auth/authorization/permissions-reference.mdx` | `uv run python services/core/auth/scripts/auth-tools.py generate-docs` | + +## API reference + +The REST API reference is rendered natively by Fern from the OpenAPI spec — `docs/fern/generators.yml` points at `docs/fern/openapi/openapi.yaml` (a symlink to the generated repo-root `openapi/openapi.yaml`), surfaced by the `- api: API Reference` nav node. No `` embed. diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md deleted file mode 100644 index 1f89376285..0000000000 --- a/docs/CONTRIBUTING.md +++ /dev/null @@ -1,225 +0,0 @@ -# Documentation Contributions - -This repository builds the NeMo Platform documentation with MkDocs. The current -source of truth is the root [`mkdocs.yml`](../mkdocs.yml), the documentation -source tree under [`docs/`](.), and the targets in [`docs/Makefile`](Makefile). - -Do not recreate the old Sphinx release flow as part of normal documentation -work. In particular, `docs/conf.py` and `docs/versions1.json` are gone, and -`docs/requirements-docs.txt` has been removed. - -## Current Stack - -- **Source:** Markdown, notebooks, snippets, images, and assets under `docs/`. -- **Config:** `mkdocs.yml` at the repository root. -- **Theme:** Material for MkDocs with local overrides in `docs/_overrides/`. -- **Environment:** `docs/.venv-mkdocs`, created by `make -C docs env`. -- **Dependencies:** `docs/requirements-mkdocs.txt`. -- **Build output:** `site/`. -- **Versioning and deploys:** `mike` publishes versioned docs to the `gh-pages` - branch through `docs/Makefile` targets. -- **CI:** `.github/workflows/docs.yaml` builds docs when `docs/**`, - `mkdocs.yml`, or `openapi/openapi.yaml` changes. - -The docs environment is intentionally separate from the main Python workspace. -The setup script uses `uv --no-config` to create and populate `.venv-mkdocs`. -Use the Make targets rather than installing MkDocs packages into the repo -environment by hand. - -## Directory Layout - -- `api/`: REST API landing page. -- `assets/`, `images/`, `stylesheets/`, `javascripts/`: MkDocs assets. -- `_hooks/`: MkDocs hook modules. -- `_overrides/`: Material for MkDocs theme overrides. -- `_scripts/`: docs helper scripts. -- `_snippets/`: reusable Markdown fragments that are included by pages. -- Feature directories such as `get-started/`, `guardrails/`, `evaluator/`, - `customizer/`, and `safe-synthesizer/`: published documentation content. - -## Local Commands - -Run docs commands from the repository root unless otherwise noted. - -```bash -make -C docs env -make -C docs live -make -C docs html -``` - -Useful variants: - -```bash -# Use another port if 8000 is busy. -LIVE_DOCS_PORT=8001 make -C docs live - -# Build or serve pages that are currently hidden from normal output. -make -C docs html-with-unready -make -C docs live-with-unready - -# Run the same strict MkDocs build used by CI. -make -C docs publish - -# Remove generated docs output and the MkDocs virtualenv. -make -C docs clean -``` - -Before opening or updating a PR, run the strict build: - -```bash -make -C docs publish -``` - -For changes that include JSON or Python fenced code blocks, also run: - -```bash -make -C docs check-code-blocks -``` - -To apply the supported code-block formatter: - -```bash -make -C docs format-code-blocks -``` - -For changes that touch notebooks or Python examples in the linted docs areas, -run: - -```bash -make -C docs lint-python -``` - -## Authoring Guidelines - -Add pages under the relevant `docs/` section and update the explicit `nav:` -block in `mkdocs.yml`. Pages that are not listed in `nav:` can still build, but -they will not appear in the published navigation unless linked from another -page. - -Use `index.md` for section landing pages. Keep section names and file names -stable when possible because redirects and external links may depend on them. -If you rename or move a published page, add a redirect in the `redirects` plugin -configuration in `mkdocs.yml`. - -Use MkDocs-supported Markdown, not Sphinx-only directives. The active extensions -include admonitions, details blocks, tabbed content, Mermaid fences, tables, -footnotes, definition lists, task lists, snippets, and syntax highlighting. - -Use substitutions from the `extra:` block in `mkdocs.yml`, such as -`{{ platform_name }}` or `{{ release }}`. Add new shared product names and -versions there instead of hard-coding them across many pages. - -Place reusable Markdown fragments under `_snippets/`. Snippet files are excluded -as standalone pages and are intended to be included from real pages with the -configured snippets extension. - -Put static assets under the existing docs asset directories. Image assets under -`docs/images/**` are fetched with Git LFS in CI. If local images render as tiny -text pointer files, run: - -```bash -git lfs pull --include="docs/images/**" --exclude="" -``` - -Notebooks are rendered by `mkdocs-jupyter` with execution disabled. Keep checked -in notebook output deliberate, small, and reviewable. - -## Generated Material - -Some docs are generated from code or OpenAPI output. Regenerate them from the -repo root instead of hand-editing generated files. - -```bash -make generate-cli-reference-docs -make generate-config-reference-docs -``` - -The API reference page uses `docs/api/openapi.yaml`, a tracked symlink to -`openapi/openapi.yaml`. When API routes, schemas, or service models change, -follow the repository SDK/OpenAPI workflow and regenerate the OpenAPI spec before -building the docs. - -The Python SDK reference is rendered through `mkdocstrings` from -`sdk/python/nemo-platform/src`. If SDK-facing API changes affect the docs, follow -the repository SDK generation process rather than editing generated SDK output -by hand. - -## Hidden Docs - -Some pages are temporarily gated by `extra.hidden_docs` in `mkdocs.yml` and the -`docs/_hooks/hide_unready_docs.py` hook. In normal builds, the hook removes -matching nav entries, source files, inline links, and API filter chips. - -Use the `with-unready` targets to inspect hidden docs locally: - -```bash -make -C docs live-with-unready -make -C docs html-with-unready -``` - -When a hidden area becomes ready, update `extra.hidden_docs` in `mkdocs.yml` -rather than deleting files or working around the hook from individual pages. - -## CI And Publishing - -The Documentation workflow runs `make -C docs publish` and uploads the `site/` -directory as a `docs-site` artifact. - -For pull requests from branches in the same repository, the workflow also -deploys a PR preview under GitHub Pages by using `make -C docs deploy-pr-preview`. -Pull requests from forks receive the build artifact but do not deploy a preview. - -Pushes to `main` deploy the `main` docs version with the `latest` alias. Tag -builds derive the docs version from the tag name, stripping an optional `docs/` -prefix and an optional leading `v`, then deploy that version with the `latest` -alias. - -Manual publishing uses these targets: - -```bash -make -C docs deploy-pages -PR_NUMBER= make -C docs deploy-pr-preview -PR_NUMBER= make -C docs delete-pr-preview -``` - -Deployment targets push to `gh-pages`; do not run them casually from a local -branch. - -## PR Expectations - -Keep docs updates in the same PR as the user-facing code change when that is the -fastest way to keep behavior and documentation in sync. For larger features, -draft the initial technical content near the code change and ask the docs owners -for review. - -For navigation moves, new top-level sections, broad terminology changes, or -anything that changes the information architecture, coordinate with the docs -owners before reshaping the tree. - -For small fixes, update the relevant Markdown directly and include the local -`make -C docs publish` result in the PR description. - -## Troubleshooting - -If `make -C docs live` fails because the port is already in use, set -`LIVE_DOCS_PORT`: - -```bash -LIVE_DOCS_PORT=8001 make -C docs live -``` - -If the docs virtualenv appears stale, rebuild it: - -```bash -make -C docs clean -make -C docs env -``` - -If a strict build reports missing files, check the `nav:` entries, redirects, -snippet includes, and hidden-doc patterns in `mkdocs.yml`. - -If API docs look stale, regenerate `openapi/openapi.yaml` through the repository -OpenAPI workflow and confirm that `docs/api/openapi.yaml` still points to it. - -If CI reports that a docs image is a Git LFS pointer, fetch the image content -with Git LFS and recommit the real asset state. diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 65ab373ae3..0000000000 --- a/docs/Makefile +++ /dev/null @@ -1,126 +0,0 @@ -# Makefile targets for MkDocs documentation - -.PHONY: help env mkdocs-env html html-fast html-with-unready mkdocs-html \ - publish mkdocs-publish live live-fast live-with-unready mkdocs-live \ - clean mkdocs-clean format-code-blocks check-code-blocks sync lint-python deploy-pages \ - deploy-pr-preview delete-pr-preview mkdocs-version require-pr-number - -# Fix locale if current setting is unsupported (common in containers). -ifeq ($(shell locale 2>&1 | grep -q "Cannot set" && echo broken),broken) -export LC_ALL=C -endif - -# Usage: -# make html # Build documentation with MkDocs -# make html-with-unready # Build with temporarily hidden docs included -# make live # Live server with auto-reload -# make live-with-unready # Live server with temporarily hidden docs included -# make publish # Production build (strict) - -MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -MKDOCS_ROOT = $(MAKEFILE_DIR).. -SITE_DIR = $(MKDOCS_ROOT)/site -VENV_MKDOCS = $(MAKEFILE_DIR).venv-mkdocs -VENV_MKDOCS_BIN = $(VENV_MKDOCS)/bin -VENV_PYTHON = $(VENV_MKDOCS_BIN)/python -MKDOCS = $(VENV_MKDOCS_BIN)/mkdocs -MIKE = $(VENV_MKDOCS_BIN)/mike -LIVE_DOCS_HOST ?= 127.0.0.1 -LIVE_DOCS_PORT ?= 8000 -PAGES_BRANCH ?= gh-pages -DOCS_SITE_URL ?= https://nvidia-nemo.github.io/nemo-platform/latest/ -DOCS_CANONICAL_VERSION ?= latest -DOCS_VERSION ?= main -DOCS_ALIAS ?= latest -PR_PREVIEW_VERSION = pr-$(PR_NUMBER) -PR_PREVIEW_PREFIX = pr-preview/$(PR_PREVIEW_VERSION) -PR_PREVIEW_SITE_URL ?= $(DOCS_SITE_URL)$(PR_PREVIEW_PREFIX)/ -PR_PREVIEW_CANONICAL_VERSION ?= $(PR_PREVIEW_VERSION) -export PATH := $(VENV_MKDOCS_BIN):$(HOME)/.local/bin:$(PATH) - -help: - @echo "Makefile commands:" - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' - -env: mkdocs-env ## Set up the documentation virtualenv - -mkdocs-env: ## Set up the MkDocs virtualenv - cd $(MAKEFILE_DIR) && bash _scripts/setup_mkdocs_env.sh - -html-fast: html ## Build documentation with MkDocs - -html: mkdocs-html ## Build documentation with MkDocs - -html-with-unready: mkdocs-env ## Build documentation with temporarily hidden docs included - @echo "Building with MkDocs, including temporarily hidden docs..." - cd $(MKDOCS_ROOT) && NMP_HIDE_UNREADY_DOCS=false $(MKDOCS) build --strict --config-file mkdocs.yml - -mkdocs-html: mkdocs-env ## Build documentation with MkDocs - @echo "Building with MkDocs..." - cd $(MKDOCS_ROOT) && $(MKDOCS) build --strict --config-file mkdocs.yml - -publish: mkdocs-publish ## Build production documentation with MkDocs - -mkdocs-publish: mkdocs-env ## Build production documentation with MkDocs in strict mode - @echo "Publishing with MkDocs..." - cd $(MKDOCS_ROOT) && $(MKDOCS) build --config-file mkdocs.yml --strict - -live-fast: live ## Start the MkDocs live-reload server - -live: mkdocs-live ## Start the MkDocs live-reload server - -live-with-unready: mkdocs-env ## Start the live server with temporarily hidden docs included - @echo "Starting MkDocs live server, including temporarily hidden docs..." - cd $(MKDOCS_ROOT) && NMP_HIDE_UNREADY_DOCS=false $(MKDOCS) serve --config-file mkdocs.yml --dev-addr $(LIVE_DOCS_HOST):$(LIVE_DOCS_PORT) - -mkdocs-live: mkdocs-env ## Start the MkDocs live-reload server - @echo "Starting MkDocs live server..." - cd $(MKDOCS_ROOT) && $(MKDOCS) serve --config-file mkdocs.yml --dev-addr $(LIVE_DOCS_HOST):$(LIVE_DOCS_PORT) - -clean: mkdocs-clean ## Remove documentation build artifacts and virtualenv - -mkdocs-clean: ## Remove MkDocs build artifacts and virtualenv - rm -rf $(SITE_DIR) $(VENV_MKDOCS) - -format-code-blocks: mkdocs-env ## Autoformat supported fenced code blocks - @echo "Formatting supported code blocks in markdown files..." - cd $(MAKEFILE_DIR) && $(VENV_PYTHON) _scripts/format_code_blocks.py --docs-dir $(MAKEFILE_DIR) - -check-code-blocks: mkdocs-env ## Check supported fenced code blocks are formatted - @echo "Checking supported code block formatting..." - cd $(MAKEFILE_DIR) && $(VENV_PYTHON) _scripts/format_code_blocks.py --docs-dir $(MAKEFILE_DIR) --check - -sync: ## Sync files from documentation/docs/ into docs/ - @echo "Syncing files from documentation/docs/ to docs/..." - @rsync -av $(MAKEFILE_DIR)/../documentation/docs/ $(MAKEFILE_DIR) - @rm -rf $(MAKEFILE_DIR)/../documentation/docs/ - @echo "Sync complete" - -lint-python: mkdocs-env ## Lint selected notebooks and run type checking - @echo "Linting Python code in notebooks..." - cd $(MAKEFILE_DIR) && $(VENV_PYTHON) _scripts/lint_notebooks.py run-inference/ safe-synthesizer/ --type-check - -deploy-pages: mkdocs-env ## Deploy main docs to GitHub Pages with mike - @echo "Deploying $(DOCS_VERSION) docs to $(PAGES_BRANCH) with alias $(DOCS_ALIAS)..." - cd $(MKDOCS_ROOT) && NMP_DOCS_SITE_URL="$(DOCS_SITE_URL)" NMP_DOCS_CANONICAL_VERSION="$(DOCS_CANONICAL_VERSION)" $(MIKE) deploy --push --update-aliases --alias-type redirect \ - --config-file mkdocs.yml --branch $(PAGES_BRANCH) $(DOCS_VERSION) $(DOCS_ALIAS) - cd $(MKDOCS_ROOT) && NMP_DOCS_SITE_URL="$(DOCS_SITE_URL)" NMP_DOCS_CANONICAL_VERSION="$(DOCS_CANONICAL_VERSION)" $(MIKE) set-default --push --config-file mkdocs.yml \ - --branch $(PAGES_BRANCH) $(DOCS_ALIAS) - -deploy-pr-preview: require-pr-number mkdocs-env ## Deploy a PR preview to GitHub Pages with mike - @echo "Deploying PR preview $(PR_PREVIEW_VERSION) under $(PR_PREVIEW_PREFIX)..." - cd $(MKDOCS_ROOT) && NMP_DOCS_SITE_URL="$(PR_PREVIEW_SITE_URL)" NMP_DOCS_CANONICAL_VERSION="$(PR_PREVIEW_CANONICAL_VERSION)" $(MIKE) deploy --push --update-aliases --alias-type redirect \ - --config-file mkdocs.yml --branch $(PAGES_BRANCH) --deploy-prefix $(PR_PREVIEW_PREFIX) \ - --title "PR #$(PR_NUMBER)" $(PR_PREVIEW_VERSION) - cd $(MKDOCS_ROOT) && NMP_DOCS_SITE_URL="$(PR_PREVIEW_SITE_URL)" NMP_DOCS_CANONICAL_VERSION="$(PR_PREVIEW_CANONICAL_VERSION)" $(MIKE) set-default --push --config-file mkdocs.yml \ - --branch $(PAGES_BRANCH) --deploy-prefix $(PR_PREVIEW_PREFIX) $(PR_PREVIEW_VERSION) - -delete-pr-preview: require-pr-number mkdocs-env ## Delete a PR preview from GitHub Pages - @echo "Deleting PR preview $(PR_PREVIEW_VERSION) from $(PR_PREVIEW_PREFIX)..." - cd $(MKDOCS_ROOT) && $(MIKE) delete --push --config-file mkdocs.yml \ - --branch $(PAGES_BRANCH) --deploy-prefix $(PR_PREVIEW_PREFIX) --all || true - -mkdocs-version: deploy-pages ## Deploy a versioned release with mike - -require-pr-number: - @test -n "$(PR_NUMBER)" || (echo "PR_NUMBER is required" && exit 1) diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 7607b58aba..0000000000 --- a/docs/README.md +++ /dev/null @@ -1,100 +0,0 @@ -# NeMo Platform Docs - -The documentation site is built with MkDocs from the repository-root -`mkdocs.yml` and the source files in this `docs/` directory. The generated site -is written to `../site/`. - -For contribution process details, see [`CONTRIBUTING.md`](CONTRIBUTING.md). - -## Prerequisites - -- Python 3.11 or later. Verify with `python3 --version`. See the - [Python downloads](https://www.python.org/downloads/) page for installation - options. -- [uv](https://docs.astral.sh/uv/). Verify with `uv --version`. -- GNU Make. Verify with `make --version`. -- Bash and a POSIX-like shell environment for the helper scripts. - -MkDocs and the required plugins are installed into `docs/.venv-mkdocs` from -`docs/requirements-mkdocs.txt`: - -```bash -make -C docs env -``` - -Node.js and npm are not required for the current MkDocs workflow. - -## Local Development - -Set up the docs-local virtual environment: - -```bash -make -C docs env -``` - -Serve the site with live reload: - -```bash -make -C docs live -``` - -Build the site with the same strict MkDocs mode used by CI: - -```bash -make -C docs publish -``` - -Use another local port if `8000` is already in use: - -```bash -LIVE_DOCS_PORT=8001 make -C docs live -``` - -To include pages that are temporarily hidden by `extra.hidden_docs` in -`mkdocs.yml`, use: - -```bash -make -C docs live-with-unready -make -C docs html-with-unready -``` - -## Useful Checks - -Check supported JSON and Python fenced code blocks: - -```bash -make -C docs check-code-blocks -``` - -Apply the formatter for supported code blocks: - -```bash -make -C docs format-code-blocks -``` - -Lint selected notebook-style docs: - -```bash -make -C docs lint-python -``` - -## Generated References - -Regenerate CLI and config reference docs from the repository root: - -```bash -make generate-cli-reference-docs -make generate-config-reference-docs -``` - -The REST API page at `docs/api/index.md` renders -`docs/api/openapi.yaml`, which is a tracked symlink to -`../openapi/openapi.yaml`. - -## Directory Layout - -For the docs tree reference, see -[`CONTRIBUTING.md#directory-layout`](CONTRIBUTING.md#directory-layout). - -MkDocs excludes this README and other docs-maintenance files from the published -site through `exclude_docs` in `mkdocs.yml`. diff --git a/docs/_hooks/hide_unready_docs.py b/docs/_hooks/hide_unready_docs.py deleted file mode 100644 index b32813204c..0000000000 --- a/docs/_hooks/hide_unready_docs.py +++ /dev/null @@ -1,214 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Hide temporarily unavailable docs from MkDocs builds. - -Configured by the `extra.hidden_docs` block in mkdocs.yml. -""" - -from __future__ import annotations - -import posixpath -import re -from collections.abc import Sequence -from fnmatch import fnmatchcase -from html import escape -from typing import Any - - -INLINE_LINK_RE = re.compile(r"(?]*\bclass="api-chip\b[^"]*")(?=[^>]*\bdata-tag="(?P[^"]*)")[^>]*>[^<]+[ \t]*\n?', - re.MULTILINE, -) - - -def on_config(config: Any, **_: Any) -> Any: - hidden_docs = _hidden_docs_config(config) - if not hidden_docs: - return config - - sections, path_patterns, _ = hidden_docs - nav = config.get("nav") - if nav: - config["nav"] = _filter_nav(nav, sections, path_patterns) - - return config - - -def on_files(files: Any, config: Any, **_: Any) -> Any: - hidden_docs = _hidden_docs_config(config) - if not hidden_docs: - return files - - _, path_patterns, _ = hidden_docs - for file in list(files): - src_path = _file_src_path(file) - if src_path and _matches_hidden_path(src_path, path_patterns): - files.remove(file) - - return files - - -def on_page_markdown(markdown: str, page: Any, config: Any, **_: Any) -> str: - hidden_docs = _hidden_docs_config(config) - if not hidden_docs: - return markdown - - _, path_patterns, api_tags = hidden_docs - page_src_path = _file_src_path(page.file) - - def replace_link(match: re.Match[str]) -> str: - label = match.group(1) - target = match.group(2) - target_path = _resolve_link_target(target, page_src_path) - if target_path and _matches_hidden_path(target_path, path_patterns): - return label - return match.group(0) - - markdown = INLINE_LINK_RE.sub(replace_link, markdown) - if page_src_path == "api/index.md": - markdown = _filter_api_reference(markdown, api_tags) - - return markdown - - -def _hidden_docs_config(config: Any) -> tuple[frozenset[str], tuple[str, ...], frozenset[str]] | None: - extra = config.get("extra", {}) - hidden_docs = extra.get("hidden_docs", {}) - if not isinstance(hidden_docs, dict) or not _is_enabled(hidden_docs.get("enabled")): - return None - - sections_val = _hidden_docs_sequence(hidden_docs, "sections") - paths_val = _hidden_docs_sequence(hidden_docs, "paths") - api_tags_val = _hidden_docs_sequence(hidden_docs, "api_tags") - - sections = frozenset(str(section) for section in sections_val) - path_patterns = tuple(_normalize_pattern(pattern) for pattern in paths_val if str(pattern).strip()) - api_tags = frozenset(str(tag) for tag in api_tags_val) - if not sections and not path_patterns and not api_tags: - return None - - return sections, path_patterns, api_tags - - -def _hidden_docs_sequence(hidden_docs: dict[str, Any], key: str) -> Sequence[Any]: - value = hidden_docs.get(key, ()) - if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): - raise ValueError(f"extra.hidden_docs.{key} must be a sequence, not {type(value).__name__}") - return value - - -def _is_enabled(value: Any) -> bool: - if isinstance(value, str): - return value.strip().lower() not in {"", "0", "false", "no", "off"} - return bool(value) - - -def _filter_nav(nav: list[Any], sections: frozenset[str], path_patterns: tuple[str, ...]) -> list[Any]: - filtered_items: list[Any] = [] - for item in nav: - filtered_item = _filter_nav_item(item, sections, path_patterns) - if filtered_item is not None: - filtered_items.append(filtered_item) - return filtered_items - - -def _filter_nav_item(item: Any, sections: frozenset[str], path_patterns: tuple[str, ...]) -> Any | None: - if isinstance(item, str): - return None if _matches_hidden_path(item, path_patterns) else item - - if not isinstance(item, dict): - return item - - filtered_item: dict[str, Any] = {} - for title, value in item.items(): - if title in sections: - continue - - if isinstance(value, str): - if not _matches_hidden_path(value, path_patterns): - filtered_item[title] = value - elif isinstance(value, list): - filtered_children = _filter_nav(value, sections, path_patterns) - if filtered_children: - filtered_item[title] = filtered_children - else: - filtered_item[title] = value - - return filtered_item or None - - -def _matches_hidden_path(path: str, patterns: tuple[str, ...]) -> bool: - normalized_path = _normalize_path(path) - for pattern in patterns: - if pattern.endswith("/**"): - directory = pattern[:-3] - if normalized_path == directory or normalized_path.startswith(f"{directory}/"): - return True - - if fnmatchcase(normalized_path, pattern): - return True - - return False - - -def _filter_api_reference(markdown: str, api_tags: frozenset[str]) -> str: - if not api_tags: - return markdown - - markdown = API_CHIP_RE.sub( - lambda match: "" if match.group("tag") in api_tags else match.group(0), - markdown, - ) - - if "data-hidden-tags=" in markdown: - return markdown - - hidden_tags = ",".join(escape(tag, quote=True) for tag in sorted(api_tags)) - return markdown.replace( - '
', - f'
', - 1, - ) - - -def _resolve_link_target(target: str, page_src_path: str) -> str | None: - target_path = target.strip() - if not target_path: - return None - - if target_path.startswith("<") and ">" in target_path: - target_path = target_path[1 : target_path.index(">")] - else: - target_path = target_path.split(maxsplit=1)[0] - - if ( - not target_path - or target_path.startswith("#") - or target_path.startswith("//") - or re.match(r"^[a-zA-Z][a-zA-Z0-9+.-]*:", target_path) - ): - return None - - target_path = target_path.split("#", 1)[0].split("?", 1)[0] - if not target_path: - return None - - if target_path.startswith("/"): - return _normalize_path(target_path) - - page_directory = posixpath.dirname(_normalize_path(page_src_path)) - return _normalize_path(posixpath.join(page_directory, target_path)) - - -def _file_src_path(file: Any) -> str: - return _normalize_path(getattr(file, "src_uri", "") or getattr(file, "src_path", "")) - - -def _normalize_pattern(pattern: Any) -> str: - return _normalize_path(str(pattern).strip()) - - -def _normalize_path(path: str) -> str: - return posixpath.normpath(path.strip().replace("\\", "/")).lstrip("./") diff --git a/docs/_images/nemo-platform-architecture.svg b/docs/_images/nemo-platform-architecture.svg deleted file mode 100644 index 4d1bea95ac..0000000000 --- a/docs/_images/nemo-platform-architecture.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/_overrides/.gitkeep b/docs/_overrides/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/docs/_overrides/main.html b/docs/_overrides/main.html deleted file mode 100644 index edf4a86bf3..0000000000 --- a/docs/_overrides/main.html +++ /dev/null @@ -1,10 +0,0 @@ -{% extends "base.html" %} - -{% block content %} - {{ super() }} - -{% endblock %} diff --git a/docs/_scripts/README.md b/docs/_scripts/README.md deleted file mode 100644 index 9bdaffc2af..0000000000 --- a/docs/_scripts/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# Documentation Scripts - -This directory contains helper scripts used by the active MkDocs documentation -workflow. Prefer calling these through `docs/Makefile` unless you are debugging a -script directly. - -## Active Scripts - -### `setup_mkdocs_env.sh` - -Creates or updates the docs-local `.venv-mkdocs` virtual environment with `uv` -and installs `docs/requirements-mkdocs.txt`. - -```bash -make -C docs env -``` - -### `format_code_blocks.py` - -Formats supported fenced code blocks in Markdown files. JSON blocks are parsed -and reserialized with stable indentation. Python blocks are formatted with Ruff -from the MkDocs virtual environment. - -```bash -make -C docs check-code-blocks -make -C docs format-code-blocks -``` - -### `lint_notebooks.py` - -Lints selected Markdown notebook sources and performs type checks for supported -Python code blocks. - -```bash -make -C docs lint-python -``` - -### `run_notebooks.py` - -Runs docs notebooks or Markdown notebook sources that use the `@nemo-nb` -markers. This is a direct utility rather than a Makefile target. - -```bash -uv run python docs/_scripts/run_notebooks.py docs/ -``` - -## Tests - -The `test_*.py` files validate the active docs helper scripts and can be run -with `uv run pytest` from the repository root. - -## Adding Scripts - -When adding a script: - -1. Put it in `docs/_scripts/`. -2. Wire it into `docs/Makefile` if it is part of the normal docs workflow. -3. Add or update focused tests when the behavior is nontrivial. -4. Document the Make target or direct command in this README. diff --git a/docs/_scripts/format_code_blocks.py b/docs/_scripts/format_code_blocks.py deleted file mode 100644 index 79cd7ba786..0000000000 --- a/docs/_scripts/format_code_blocks.py +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Format fenced code blocks in Markdown documentation.""" - -from __future__ import annotations - -import argparse -import importlib.util -import json -import os -import re -import subprocess -import sys -from dataclasses import dataclass -from functools import cache -from pathlib import Path -from typing import Iterable - -FENCE_RE = re.compile(r"^([ \t]*)(`{3,}|~{3,})(.*)$") -JSON_LANGUAGES = {"json", "output"} -PYTHON_LANGUAGES = {"python", "py"} -RAW_START = "{% raw %}" -RAW_END = "{% endraw %}" -RUFF_FORMAT_TIMEOUT_SECONDS = 30 -SKIP_DIRS = { - ".git", - ".venv-docs", - ".venv-mkdocs", - "_build", - "_test_serve", - "site", -} - - -@dataclass -class FileResult: - path: Path - changed: bool - blocks_changed: int - - -def find_markdown_files(paths: Iterable[Path], docs_dir: Path) -> list[Path]: - markdown_files: list[Path] = [] - for path in paths: - if not path.exists(): - raise FileNotFoundError(f"{path} does not exist") - if path.is_file(): - if path.suffix == ".md": - markdown_files.append(path) - continue - for root, dirs, files in os.walk(path): - dirs[:] = [directory for directory in dirs if directory not in SKIP_DIRS] - for filename in files: - file_path = Path(root) / filename - if file_path.suffix == ".md" and file_path.is_relative_to(docs_dir): - markdown_files.append(file_path) - return sorted(set(markdown_files)) - - -def fence_closes(line: str, fence_marker: str) -> bool: - stripped = line.lstrip(" \t") - marker_char = fence_marker[0] - marker_len = len(fence_marker) - if not stripped.startswith(marker_char * marker_len): - return False - return stripped.strip(marker_char).strip() == "" - - -def get_language(info_string: str) -> str: - stripped = info_string.strip() - if not stripped: - return "" - return stripped.split(maxsplit=1)[0].lower() - - -def remove_markdown_indent(lines: list[str], indent: str) -> str: - if not indent: - return "\n".join(lines) - return "\n".join(line[len(indent) :] if line.startswith(indent) else line for line in lines) - - -def format_json_lines(content_lines: list[str], indent: str, language: str) -> tuple[list[str], bool]: - if language not in JSON_LANGUAGES: - return content_lines, False - if any("{% raw %}" in line or "{% endraw %}" in line for line in content_lines): - return content_lines, False - - raw_content = remove_markdown_indent(content_lines, indent) - candidate = raw_content.strip() - if not candidate or candidate[0] not in "[{": - return content_lines, False - - try: - parsed = json.loads(candidate) - except json.JSONDecodeError: - return content_lines, False - - if not isinstance(parsed, (dict, list)): - return content_lines, False - - formatted = json.dumps(parsed, ensure_ascii=False, indent=2) - formatted_lines = [add_markdown_indent(line, indent) for line in formatted.splitlines()] - return formatted_lines, formatted_lines != content_lines - - -def format_python_lines(content_lines: list[str], indent: str, language: str) -> tuple[list[str], bool]: - if language not in PYTHON_LANGUAGES: - return content_lines, False - if not ruff_available(): - raise RuntimeError("Ruff is required to format Python code blocks. Run `make -C docs mkdocs-env` first.") - - raw_lines = [line[len(indent) :] if indent and line.startswith(indent) else line for line in content_lines] - raw_content = "\n".join(raw_lines).rstrip("\n") - if not raw_content.strip(): - return content_lines, False - - try: - result = subprocess.run( - [sys.executable, "-m", "ruff", "format", "--stdin-filename", "docs-code-block.py", "-"], - input=f"{raw_content}\n", - text=True, - capture_output=True, - check=False, - timeout=RUFF_FORMAT_TIMEOUT_SECONDS, - ) - except (OSError, subprocess.TimeoutExpired): - return content_lines, False - - if result.returncode != 0: - return content_lines, False - - formatted_lines = [add_markdown_indent(line, indent) for line in result.stdout.rstrip("\n").splitlines()] - return formatted_lines, formatted_lines != content_lines - - -def add_markdown_indent(line: str, indent: str) -> str: - if not line: - return line - return f"{indent}{line}" - - -@cache -def ruff_available() -> bool: - return importlib.util.find_spec("ruff") is not None - - -def format_code_lines( - content_lines: list[str], - indent: str, - language: str, - inside_raw_block: bool, -) -> tuple[list[str], bool]: - python_lines, python_changed = format_python_lines(content_lines, indent, language) - if python_changed: - return python_lines, True - if inside_raw_block: - return content_lines, False - return format_json_lines(content_lines, indent, language) - - -def format_markdown(text: str) -> tuple[str, int]: - trailing_newline = text.endswith("\n") - lines = text.splitlines() - formatted_lines: list[str] = [] - blocks_changed = 0 - index = 0 - inside_raw_block = False - - while index < len(lines): - line = lines[index] - fence_match = FENCE_RE.match(line) - if not fence_match: - formatted_lines.append(line) - inside_raw_block = update_raw_block_state(line, inside_raw_block) - index += 1 - continue - - indent, fence_marker, info_string = fence_match.groups() - language = get_language(info_string) - formatted_lines.append(line) - index += 1 - - content_lines: list[str] = [] - while index < len(lines) and not fence_closes(lines[index], fence_marker): - content_lines.append(lines[index]) - index += 1 - - new_content_lines, changed = format_code_lines(content_lines, indent, language, inside_raw_block) - if changed: - blocks_changed += 1 - formatted_lines.extend(new_content_lines) - - if index < len(lines): - formatted_lines.append(lines[index]) - index += 1 - - formatted = "\n".join(formatted_lines) - if trailing_newline: - formatted += "\n" - return formatted, blocks_changed - - -def update_raw_block_state(line: str, inside_raw_block: bool) -> bool: - raw_start = RAW_START in line - raw_end = RAW_END in line - if raw_start and not raw_end: - return True - if raw_end and not raw_start: - return False - return inside_raw_block - - -def process_file(path: Path, check: bool) -> FileResult: - original = path.read_text(encoding="utf-8") - formatted, blocks_changed = format_markdown(original) - changed = formatted != original - if changed and not check: - path.write_text(formatted, encoding="utf-8") - return FileResult(path=path, changed=changed, blocks_changed=blocks_changed) - - -def display_path(path: Path, docs_dir: Path) -> Path: - if path.is_relative_to(docs_dir): - return path.relative_to(docs_dir) - return path - - -def main() -> int: - parser = argparse.ArgumentParser(description="Format fenced code blocks in Markdown docs.") - parser.add_argument("paths", nargs="*", type=Path, help="Markdown files or directories to process.") - parser.add_argument( - "--docs-dir", type=Path, default=Path("."), help="Docs directory. Defaults to current directory." - ) - parser.add_argument("--check", action="store_true", help="Exit non-zero if any files would be changed.") - args = parser.parse_args() - - docs_dir = args.docs_dir.resolve() - paths = [path.resolve() for path in args.paths] if args.paths else [docs_dir] - - try: - markdown_files = find_markdown_files(paths, docs_dir) - except FileNotFoundError as exc: - print(f"ERROR: {exc}", file=sys.stderr) - return 2 - - changed_results: list[FileResult] = [] - total_blocks = 0 - try: - for path in markdown_files: - result = process_file(path, args.check) - if result.changed: - changed_results.append(result) - total_blocks += result.blocks_changed - except RuntimeError as exc: - print(f"ERROR: {exc}", file=sys.stderr) - return 2 - - if changed_results: - action = "Would format" if args.check else "Formatted" - for result in changed_results: - print(f"{action} {display_path(result.path, docs_dir)} ({result.blocks_changed} code block(s))") - print(f"{action} {len(changed_results)} file(s), {total_blocks} code block(s)") - return 1 if args.check else 0 - - print("All supported code blocks are already formatted.") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/docs/_scripts/lint_notebooks.py b/docs/_scripts/lint_notebooks.py deleted file mode 100644 index 1b62f8aaef..0000000000 --- a/docs/_scripts/lint_notebooks.py +++ /dev/null @@ -1,568 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Lint notebook documentation files for syntax errors. - -This script validates markdown notebooks without executing them, making it -suitable for CI/CD pipelines and pre-commit hooks. - -Usage: - uv run python docs/_scripts/lint_notebooks.py [ ...] - uv run python docs/_scripts/lint_notebooks.py docs/run-inference/ - uv run python docs/_scripts/lint_notebooks.py docs/run-inference/ docs/safe-synthesizer/ - uv run python docs/_scripts/lint_notebooks.py docs/run-inference/ --type-check - uv run python docs/_scripts/lint_notebooks.py docs/run-inference/ --markers-only # Only files with marker -""" - -import argparse -import ast -import re -import subprocess -import sys -import tempfile -from pathlib import Path -from typing import List, Tuple - -from nemo_nb import ( - MarkdownToNotebookConverter, - expand_includes, - find_processable_notebooks, - is_excluded_file, - is_in_excluded_dir, - print_conflicts_error, -) - -# Pattern to detect Python code blocks in markdown -RE_PYTHON_CODE_BLOCK = re.compile(r"^```python\s*$", re.MULTILINE) -SKIP_TYPE_CHECK_MARKER = "" - -# Pattern to match code fence opening (```python) with optional leading whitespace -RE_CODE_FENCE_OPEN = re.compile(r"^(\s*)(`{3,})python\s*$") -# Pattern to match code fence closing -RE_CODE_FENCE_CLOSE = re.compile(r"^(\s*)`{3,}\s*$") - - -def has_python_code_blocks(md_path: Path) -> bool: - """Check if a markdown file has Python code blocks. - - Args: - md_path: Path to the markdown file - - Returns: - True if the file contains at least one ```python code block - """ - try: - content = md_path.read_text(encoding="utf-8") - return bool(RE_PYTHON_CODE_BLOCK.search(content)) - except Exception: - return False - - -def extract_python_blocks_with_line_numbers(md_path: Path) -> List[Tuple[int, str]]: - """Extract Python code blocks from markdown with their starting line numbers. - - Args: - md_path: Path to the markdown file - - Returns: - List of (start_line_in_markdown, code_content) tuples. - start_line_in_markdown is 1-indexed and points to the first line of code - (the line after the opening ```python fence). - """ - try: - content = md_path.read_text(encoding="utf-8") - # Expand include directives before extracting code blocks - content = expand_includes(content, md_path.parent) - except Exception: - return [] - - lines = content.split("\n") - blocks: List[Tuple[int, str]] = [] - skip_next_python_block = False - - i = 0 - while i < len(lines): - line = lines[i] - if line.strip() == SKIP_TYPE_CHECK_MARKER: - skip_next_python_block = True - i += 1 - continue - - fence_match = RE_CODE_FENCE_OPEN.match(line) - - if fence_match: - fence_delimiter = fence_match.group(2) - fence_len = len(fence_delimiter) - indentation = fence_match.group(1) - indent_len = len(indentation) - - # Code starts on the next line (1-indexed for display) - code_start_line = i + 2 # +1 for 0-indexed to 1-indexed, +1 to skip fence line - - # Find closing fence - code_lines = [] - j = i + 1 - while j < len(lines): - close_match = RE_CODE_FENCE_CLOSE.match(lines[j]) - if close_match: - # Check if this fence closes our block (same or more backticks) - close_fence = lines[j].lstrip() - if len(close_fence.rstrip()) >= fence_len: - break - # Remove indentation from code line if present - code_line = lines[j] - if indent_len > 0 and code_line.startswith(indentation): - code_line = code_line[indent_len:] - code_lines.append(code_line) - j += 1 - - if code_lines and not skip_next_python_block: - blocks.append((code_start_line, "\n".join(code_lines))) - - skip_next_python_block = False - - i = j + 1 - else: - i += 1 - - return blocks - - -def prepare_notebook_for_type_check(notebook_path: Path) -> Tuple[str, List[int]]: - """ - Extract and combine Python code blocks from a markdown file for type checking. - - Args: - notebook_path: Path to the markdown file - - Returns: - Tuple of (combined_source, line_mapping) where line_mapping maps - 1-indexed line numbers in the combined source to 1-indexed line numbers - in the original markdown file. Returns ("", []) if extraction fails. - """ - blocks = extract_python_blocks_with_line_numbers(notebook_path) - - if not blocks: - return "", [] - - combined_lines: List[str] = [] - # line_mapping[i] = markdown line number for combined source line i+1 (1-indexed) - line_mapping: List[int] = [] - - for md_start_line, source in blocks: - source_lines = source.splitlines() - for offset, line in enumerate(source_lines): - combined_lines.append(line) - line_mapping.append(md_start_line + offset) - # Add blank line between blocks (maps to last line of previous block) - combined_lines.append("") - if source_lines: - line_mapping.append(md_start_line + len(source_lines) - 1) - else: - line_mapping.append(md_start_line) - - return "\n".join(combined_lines), line_mapping - - -def translate_line_number(line_in_combined: int, line_mapping: List[int]) -> int: - """Translate a line number from combined source to original markdown. - - Args: - line_in_combined: 1-indexed line number in the combined Python source - line_mapping: Mapping from combined line numbers to markdown line numbers - - Returns: - 1-indexed line number in the original markdown file - """ - if not line_mapping: - return line_in_combined - - # line_mapping is 0-indexed (line_mapping[0] = markdown line for combined line 1) - idx = line_in_combined - 1 - if 0 <= idx < len(line_mapping): - return line_mapping[idx] - - # Fallback: return original line number if out of range - return line_in_combined - - -def batch_type_check(notebook_paths: List[Path]) -> dict[Path, Tuple[int, str]]: - """ - Run ty type checker on multiple notebooks in a single batch. - - This is much faster than running ty separately for each notebook, - as it eliminates the repeated uv startup overhead. - - Args: - notebook_paths: List of notebook paths to type check - - Returns: - Dictionary mapping notebook path to (exit_code, output) tuple - """ - if not notebook_paths: - return {} - - # Prepare all notebooks and create temp files - temp_files: dict[Path, str] = {} # notebook_path -> temp_file_path - temp_to_notebook: dict[str, Path] = {} # temp_file_path -> notebook_path - line_mappings: dict[Path, List[int]] = {} # notebook_path -> line mapping - - try: - for nb_path in notebook_paths: - combined_source, line_mapping = prepare_notebook_for_type_check(nb_path) - if not combined_source: - continue - - f = tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) - f.write(combined_source) - f.close() - - temp_files[nb_path] = f.name - temp_to_notebook[f.name] = nb_path - line_mappings[nb_path] = line_mapping - - if not temp_files: - return {nb: (0, "") for nb in notebook_paths} - - # Run ty check once on all files - temp_paths = list(temp_files.values()) - result = subprocess.run( - ["uv", "run", "ty", "check", "--output-format", "concise"] + temp_paths, - capture_output=True, - text=True, - timeout=120, - ) - - # Parse output to map errors back to notebooks - output_by_notebook: dict[Path, List[str]] = {nb: [] for nb in notebook_paths} - - # Warnings to filter out (common SDK false positives) - ignored_warnings = ["possibly-unbound-attribute"] - - # Pattern to parse ty output: "path:line:col: message" - ty_output_pattern = re.compile(r"^(.+?):(\d+):(\d+): (.+)$") - - for line in result.stdout.splitlines(): - # Skip ignored warning types - if any(f"[{warning}]" in line for warning in ignored_warnings): - continue - - # ty output format: "path/to/file.py:line:col: error message" - for temp_path, nb_path in temp_to_notebook.items(): - if line.startswith(temp_path): - # Parse and translate line number - match = ty_output_pattern.match(line) - if match: - combined_line = int(match.group(2)) - col = match.group(3) - message = match.group(4) - md_line = translate_line_number(combined_line, line_mappings.get(nb_path, [])) - cleaned_line = f"{nb_path.name}:{md_line}:{col}: {message}" - else: - # Fallback: just replace path - cleaned_line = line.replace(temp_path, str(nb_path.name)) - output_by_notebook[nb_path].append(cleaned_line) - break - - # Build result dictionary - results: dict[Path, Tuple[int, str]] = {} - - # Check if ty produced any per-file output at all - has_any_output = any(output_by_notebook[nb] for nb in notebook_paths if nb in output_by_notebook) - - for nb_path in notebook_paths: - if nb_path in temp_files: - # Had Python code and was checked - nb_output = "\n".join(output_by_notebook[nb_path]) - # If this notebook had errors, return non-zero exit code - if output_by_notebook[nb_path]: - exit_code = 1 - elif result.returncode != 0 and not has_any_output: - # ty failed but produced no per-file output - ty itself failed - exit_code = 1 - nb_output = "Type checking failed: ty exited with non-zero status but produced no output" - else: - exit_code = 0 - results[nb_path] = (exit_code, nb_output) - else: - # No Python code or conversion failed - results[nb_path] = (0, "") - - return results - - except subprocess.TimeoutExpired: - return {nb: (1, "Type checking timed out") for nb in notebook_paths} - except FileNotFoundError: - return {nb: (0, "") for nb in notebook_paths} - except Exception as e: - return {nb: (1, f"Type checking failed: {e}") for nb in notebook_paths} - finally: - # Clean up temp files - for temp_path in temp_files.values(): - Path(temp_path).unlink(missing_ok=True) - - -def lint_notebook_syntax_only(notebook_path: Path) -> List[Tuple[str, str]]: - """ - Lint a notebook file for syntax errors only (fast). - - Args: - notebook_path: Path to the notebook file - - Returns: - List of (error_type, message) tuples - """ - errors = [] - - try: - # Read and expand includes before conversion - content = notebook_path.read_text(encoding="utf-8") - expanded_content = expand_includes(content, notebook_path.parent) - - # Write expanded content to temporary file for conversion - with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False, encoding="utf-8") as temp_file: - temp_file.write(expanded_content) - temp_path = Path(temp_file.name) - - try: - converter = MarkdownToNotebookConverter() - notebook = converter.convert(temp_path) - finally: - temp_path.unlink(missing_ok=True) - except Exception as e: - return [("conversion", f"Failed to convert notebook: {e}")] - - # Check Python cells for syntax errors - python_cells = [ - (i, c) - for i, c in enumerate(notebook["cells"]) - if c.get("cell_type") == "code" and c.get("metadata", {}).get("language", "python") == "python" - ] - - for cell_idx, cell in python_cells: - source = cell.get("source", []) - if isinstance(source, list): - source = "".join(source) - - if not source.strip(): - continue - - # Syntax check (fast, always enabled) - try: - ast.parse(source) - except SyntaxError as e: - errors.append(("syntax", f"Cell {cell_idx + 1}: {e.msg} at line {e.lineno}")) - - return errors - - -def lint_notebooks( - notebooks_dir: str, - enable_type_check: bool = False, - markers_only: bool = False, -) -> int: - """ - Lint all notebooks in a directory. - - Args: - notebooks_dir: Directory to search for notebooks - enable_type_check: Enable ty type checking - markers_only: If True, only lint files with @nemo-nb: process marker. - If False (default), lint any markdown file with Python code blocks. - - Returns: - Exit code: 0 if all valid, 1 if errors found - """ - path_obj = Path(notebooks_dir) - - # Handle single file case - if path_obj.is_file() and path_obj.suffix == ".md": - if is_excluded_file(path_obj): - print(f"Skipping {path_obj}: excluded filename") - return 0 - - content = path_obj.read_text(encoding="utf-8") - has_marker = "@nemo-nb: process" in content - has_python = has_python_code_blocks(path_obj) - - if markers_only and not has_marker: - print(f"Skipping {path_obj}: no @nemo-nb: process marker (use without --markers-only to lint anyway)") - return 0 - - if not has_marker and not has_python: - print(f"Skipping {path_obj}: no @nemo-nb: process marker and no Python code blocks") - return 0 - - return _lint_single_file(path_obj, enable_type_check) - - # Directory case - if markers_only: - # Only use marked files - result = find_processable_notebooks(notebooks_dir) - - if result.conflicts: - print_conflicts_error(result.conflicts) - return 1 - - md_files = list(result.md_files) - ipynb_files = list(result.ipynb_files) - else: - # Find all markdown files with Python code blocks - ipynb_files: List[Path] = [] - md_files: List[Path] = [] - - if path_obj.is_dir(): - all_md = list(path_obj.rglob("*.md")) - # Filter out excluded directories (_generated, _build), excluded filenames, and check for Python code - md_files = [ - md - for md in all_md - if not is_in_excluded_dir(md) and not is_excluded_file(md) and has_python_code_blocks(md) - ] - - # Also include .ipynb files with marker (for completeness) - result = find_processable_notebooks(notebooks_dir) - if result.conflicts: - print_conflicts_error(result.conflicts) - return 1 - ipynb_files = list(result.ipynb_files) - - total_count = len(ipynb_files) + len(md_files) - - if total_count == 0: - print(f"No markdown files with Python code blocks found in {notebooks_dir}") - if path_obj.is_dir(): - all_md = list(path_obj.rglob("*.md")) - if all_md: - print(f"(Found {len(all_md)} .md files total, but none had Python code blocks)") - return 0 - - print(f"Linting {total_count} file(s) with Python code...\n") - - failed = [] - - # Lint .ipynb files - for nb in ipynb_files: - # For .ipynb, we need to check if it's already a valid notebook - # This is a placeholder - you might want to add actual ipynb linting - print(f"ℹ {nb} (skipping .ipynb linting for now)") - - # Step 1: Syntax check all files (fast, always enabled) - print("Running syntax checks...") - syntax_results: dict[Path, List[Tuple[str, str]]] = {} - for md in md_files: - syntax_results[md] = lint_notebook_syntax_only(md) - - # Step 2: Batch type check all files (if enabled) - type_results: dict[Path, Tuple[int, str]] = {} - if enable_type_check: - print("Running batch type checking...") - type_results = batch_type_check(md_files) - else: - type_results = {md: (0, "") for md in md_files} - - # Step 3: Print results - print("\nResults:\n") - for md in md_files: - syntax_errors = syntax_results[md] - ty_exit_code, ty_output = type_results.get(md, (0, "")) - - has_errors = bool(syntax_errors) or ty_exit_code != 0 - - if not has_errors: - print(f"✓ {md}") - continue - - print(f"✗ {md}") - - for error_type, message in syntax_errors: - print(f" SYNTAX ERROR: {message}") - - if ty_exit_code != 0 and ty_output: - print(f" TYPE CHECK FAILED (exit code {ty_exit_code}):") - for line in ty_output.strip().splitlines(): - print(f" {line}") - - failed.append(md) - - print("\n" + "=" * 60) - if failed: - print(f"✗ FAILED: {len(failed)} page(s) have errors") - for nb in failed: - print(f" - {nb}") - return 1 - else: - print("✓ All pages passed") - return 0 - - -def _lint_single_file(md_path: Path, enable_type_check: bool) -> int: - """Lint a single markdown file. - - Args: - md_path: Path to the markdown file - enable_type_check: Enable ty type checking - - Returns: - Exit code: 0 if valid, 1 if errors found - """ - print(f"Linting {md_path}...\n") - - # Syntax check - syntax_errors = lint_notebook_syntax_only(md_path) - - # Type check - ty_exit_code, ty_output = 0, "" - if enable_type_check: - type_results = batch_type_check([md_path]) - ty_exit_code, ty_output = type_results.get(md_path, (0, "")) - - has_errors = bool(syntax_errors) or ty_exit_code != 0 - - if not has_errors: - print(f"✓ {md_path}") - print("\n" + "=" * 60) - print("✓ All notebooks passed") - return 0 - - print(f"✗ {md_path}") - - for error_type, message in syntax_errors: - print(f" SYNTAX ERROR: {message}") - - if ty_exit_code != 0 and ty_output: - print(f" TYPE CHECK FAILED (exit code {ty_exit_code}):") - for line in ty_output.strip().splitlines(): - print(f" {line}") - - print("\n" + "=" * 60) - print("✗ FAILED: 1 page has errors") - print(f" - {md_path}") - return 1 - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Lint notebook files for syntax and type errors without executing them" - ) - parser.add_argument("paths", nargs="+", help="Directories or files to lint") - parser.add_argument( - "--type-check", - action="store_true", - help="Enable ty type checking (slower but catches parameter errors, type mismatches, etc.)", - ) - parser.add_argument( - "--markers-only", - action="store_true", - help="Only lint files with @nemo-nb: process marker. " - "By default, any markdown file with Python code blocks is linted.", - ) - args = parser.parse_args() - - overall_exit = 0 - for path in args.paths: - result = lint_notebooks(path, args.type_check, args.markers_only) - if result != 0: - overall_exit = result - sys.exit(overall_exit) diff --git a/docs/_scripts/run_notebooks.py b/docs/_scripts/run_notebooks.py deleted file mode 100644 index 473923dad9..0000000000 --- a/docs/_scripts/run_notebooks.py +++ /dev/null @@ -1,66 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# run via root directory: -# uv run python docs/_scripts/run_notebooks.py docs/ -# -# Options: -# --language python Run only Python cells (default, skip shell/curl) -# --language shell Run only shell cells (skip Python) -# --use-temporary-venv Create and use a temporary virtualenv for running notebooks -# --requirements FILE Install packages from requirements file using uv (requires --use-temporary-venv) -# -# This script finds and runs notebooks with the @nemo-nb: process marker. -# It supports both: -# - .ipynb files (standard Jupyter notebooks) -# - .md files (markdown notebooks using nemo-nb format) -# -# For .md files, the script converts them to .ipynb first, runs them, -# then cleans up the generated notebook. -# -# Core logic lives in nmp.testing.notebooks and can be imported directly. - -import argparse -import sys - -from dotenv import load_dotenv -from nmp.testing.notebooks import run_notebooks - -# Load environment variables from .env file if it exists -load_dotenv() - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Run notebooks in a directory.") - parser.add_argument("dir", help="Directory containing notebooks to run") - parser.add_argument( - "--language", - choices=["all", "python", "shell"], - default="python", - help="Which cells to execute: python only (default), all of them, or shell only", - ) - parser.add_argument( - "--keep-temp-files", - action="store_true", - help="Keep the temporary files generated by the notebooks", - ) - parser.add_argument( - "--use-temporary-venv", - action="store_true", - help="Create a temporary virtualenv for running notebooks. " - "This allows testing library installations from scratch in a clean environment.", - ) - parser.add_argument( - "--requirements", - type=str, - default=None, - metavar="FILE", - help="Path to a requirements.txt file to install in the temporary virtualenv. " - "Requires --use-temporary-venv. Packages are installed using 'uv pip install -r'.", - ) - args = parser.parse_args() - - sys.exit( - run_notebooks( - args.dir, args.language, args.keep_temp_files, args.use_temporary_venv, args.requirements - ) - ) diff --git a/docs/_scripts/setup_mkdocs_env.sh b/docs/_scripts/setup_mkdocs_env.sh deleted file mode 100755 index 96b0d6009b..0000000000 --- a/docs/_scripts/setup_mkdocs_env.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# Sets up the MkDocs virtualenv using uv -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -DOCS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" -VENV_DIR="$DOCS_DIR/.venv-mkdocs" -PYTHON_BIN="${PYTHON:-python3}" - -if [ ! -d "$VENV_DIR" ]; then - echo "Creating MkDocs virtualenv at $VENV_DIR ..." - UV_PYTHON_DOWNLOADS=never uv --no-config venv --python "$PYTHON_BIN" "$VENV_DIR" -fi - -echo "Installing MkDocs dependencies..." -UV_PYTHON_DOWNLOADS=never uv --no-config pip install --python "$VENV_DIR" -r "$DOCS_DIR/requirements-mkdocs.txt" - -echo "✓ MkDocs environment ready: $VENV_DIR" -echo " Activate with: source $VENV_DIR/bin/activate" -echo " Or use directly: $VENV_DIR/bin/mkdocs" diff --git a/docs/_scripts/test_format_code_blocks.py b/docs/_scripts/test_format_code_blocks.py deleted file mode 100644 index 92ab8a4f7b..0000000000 --- a/docs/_scripts/test_format_code_blocks.py +++ /dev/null @@ -1,240 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from pathlib import Path - -from docs._scripts.format_code_blocks import display_path, format_markdown - - -def test_format_markdown_formats_json_fence() -> None: - markdown = """Before -```json -{"b": [1,2], "a": true} -``` -""" - - formatted, blocks_changed = format_markdown(markdown) - - assert blocks_changed == 1 - assert ( - formatted - == """Before -```json -{ - "b": [ - 1, - 2 - ], - "a": true -} -``` -""" - ) - - -def test_format_markdown_preserves_markdown_indent() -> None: - markdown = """??? "Output" - ```json - {"a":1} - ``` -""" - - formatted, blocks_changed = format_markdown(markdown) - - assert blocks_changed == 1 - assert ( - formatted - == """??? "Output" - ```json - { - "a": 1 - } - ``` -""" - ) - - -def test_format_markdown_skips_invalid_json() -> None: - markdown = """```json -{"a": 1,} -``` -""" - - formatted, blocks_changed = format_markdown(markdown) - - assert blocks_changed == 0 - assert formatted == markdown - - -def test_format_markdown_skips_raw_blocks() -> None: - markdown = """{% raw %} -```json -{"a":1} -``` -{% endraw %} -""" - - formatted, blocks_changed = format_markdown(markdown) - - assert blocks_changed == 0 - assert formatted == markdown - - -def test_format_markdown_formats_compact_python_indent() -> None: - markdown = """```python -model_configs = [ - dd.ModelConfig( - provider="system/nvidia-build", - ) -] -``` -""" - - formatted, blocks_changed = format_markdown(markdown) - - assert blocks_changed == 1 - assert ( - formatted - == """```python -model_configs = [ - dd.ModelConfig( - provider="system/nvidia-build", - ) -] -``` -""" - ) - - -def test_format_markdown_formats_compact_python_indent_in_raw_block() -> None: - markdown = """{% raw %} -```python -config_builder.add_column( - dd.ExpressionColumnConfig( - expr="{{ item.value }}", - ) -) -``` -{% endraw %} -""" - - formatted, blocks_changed = format_markdown(markdown) - - assert blocks_changed == 1 - assert ( - formatted - == """{% raw %} -```python -config_builder.add_column( - dd.ExpressionColumnConfig( - expr="{{ item.value }}", - ) -) -``` -{% endraw %} -""" - ) - - -def test_format_markdown_preserves_markdown_indent_for_python() -> None: - markdown = """=== "Python" - - ```python - if enabled: - print("enabled") - ``` -""" - - formatted, blocks_changed = format_markdown(markdown) - - assert blocks_changed == 1 - assert ( - formatted - == """=== "Python" - - ```python - if enabled: - print("enabled") - ``` -""" - ) - - -def test_format_markdown_does_not_indent_python_blank_lines() -> None: - markdown = ( - """!!! note - - ```python - import os -""" - " \n" - """ value=1 - ``` -""" - ) - - formatted, blocks_changed = format_markdown(markdown) - - assert blocks_changed == 1 - assert ( - formatted - == """!!! note - - ```python - import os - - value = 1 - ``` -""" - ) - - -def test_format_markdown_does_not_indent_json_blank_lines() -> None: - markdown = ( - """!!! note - - ```json - { -""" - " \n" - """ "a": 1 - } - ``` -""" - ) - - formatted, blocks_changed = format_markdown(markdown) - - assert blocks_changed == 1 - assert ( - formatted - == """!!! note - - ```json - { - "a": 1 - } - ``` -""" - ) - - -def test_format_markdown_skips_python_with_triple_quoted_strings() -> None: - markdown = '''```python -prompt = """ - keep this leading space -""" -``` -''' - - formatted, blocks_changed = format_markdown(markdown) - - assert blocks_changed == 0 - assert formatted == markdown - - -def test_display_path_handles_paths_outside_docs_dir() -> None: - docs_dir = Path("/repo/docs") - - assert display_path(Path("/repo/docs/page.md"), docs_dir) == Path("page.md") - assert display_path(Path("/tmp/page.md"), docs_dir) == Path("/tmp/page.md") diff --git a/docs/_scripts/test_lint_notebooks.py b/docs/_scripts/test_lint_notebooks.py deleted file mode 100644 index 8273345853..0000000000 --- a/docs/_scripts/test_lint_notebooks.py +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from pathlib import Path - -from docs._scripts.lint_notebooks import extract_python_blocks_with_line_numbers - - -def test_extract_python_blocks_skips_next_block_marked_for_type_check(tmp_path: Path) -> None: - markdown = """ - - -```python -from litellm import completion -completion(model="demo", messages=[]) -``` - -```python -print("kept") -``` -""" - notebook = tmp_path / "notebook.md" - notebook.write_text(markdown, encoding="utf-8") - - blocks = extract_python_blocks_with_line_numbers(notebook) - - assert blocks == [(10, 'print("kept")')] diff --git a/docs/_snippets/nvidia-build-model-provider.md b/docs/_snippets/nvidia-build-model-provider.md deleted file mode 100644 index e81f7d92e5..0000000000 --- a/docs/_snippets/nvidia-build-model-provider.md +++ /dev/null @@ -1,8 +0,0 @@ -!!! note - `nemo setup` pre-configures a `default/nvidia-build` model provider during local startup. - This provider routes inference requests to models hosted on `build.nvidia.com` using the API base URL `https://integrate.api.nvidia.com` - and the NGC API key with `Public API Endpoints` permissions provided during deployment. - - You can verify this provider exists by running `nemo inference providers list --workspace default`. - - The tutorials in these docs use this provider for inference, but you can alternatively create your own and use it instead. diff --git a/docs/_snippets/tutorials/cli-sdk-setup.md b/docs/_snippets/tutorials/cli-sdk-setup.md deleted file mode 100644 index f59a1bedd4..0000000000 --- a/docs/_snippets/tutorials/cli-sdk-setup.md +++ /dev/null @@ -1,19 +0,0 @@ - -=== "CLI" - - ```bash - # Configure CLI (if not already done) - nemo config set --base-url "$NMP_BASE_URL" --workspace default - ``` - -=== "Python SDK" - - ```python - import os - from nemo_platform import NeMoPlatform - - client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="default", - ) - ``` diff --git a/docs/_snippets/tutorials/prereqs.md b/docs/_snippets/tutorials/prereqs.md deleted file mode 100644 index 2d8ee45194..0000000000 --- a/docs/_snippets/tutorials/prereqs.md +++ /dev/null @@ -1,12 +0,0 @@ -# Platform Prerequisites - -??? "New to using {{platform_name}}?" - :icon: info - - All platform resources—models, datasets, and more—must belong to a **workspace**. Workspaces provide organizational and authorization boundaries for your work. Within a workspace, you can optionally use **projects** to group related resources. - - **If you're new to the platform**, start with the **[Setup guide](../../get-started/setup.md)** to learn how to deploy and evaluate models, and optimize agents using the platform end-to-end. - - **If you're already familiar** with workspaces and how to upload datasets to the platform, you can proceed directly with this tutorial. - - For more information, see [Workspaces](../../get-started/concepts/workspaces.md) and [Projects](../../get-started/concepts/projects.md). diff --git a/docs/about/release-notes/current-release.md b/docs/about/release-notes/current-release.mdx similarity index 94% rename from docs/about/release-notes/current-release.md rename to docs/about/release-notes/current-release.mdx index 592a60694d..96f5a1ec7c 100644 --- a/docs/about/release-notes/current-release.md +++ b/docs/about/release-notes/current-release.mdx @@ -1,6 +1,8 @@ -# v{{ release }} — 2026-05-20 - -First public release of {{platform_name}}. This release establishes the +--- +title: "v0.1.0 — 2026-05-20" +description: "" +--- +First public release of NeMo Platform. This release establishes the local-first OSS distribution and the core agent build, optimize, and secure flow. @@ -61,7 +63,7 @@ flow. - Workspace browser, agent list, and chat playground for deployed agents and models. The Studio agent build, optimize, and secure flow is in progress - and not part of {{ release }}. + and not part of 0.1.0. ### Coding-agent skills @@ -86,7 +88,7 @@ make bootstrap nemo setup ``` -See [Setup](../../get-started/setup.md) for prerequisites and provider +See [Setup](/documentation/get-started) for prerequisites and provider configuration. ## Compatibility @@ -99,7 +101,7 @@ configuration. ## Current constraints - **Local-only.** Cluster and scale-out deployment are on the roadmap for - enterprise use; {{ release }} supports a single local install. + enterprise use; 0.1.0 supports a single local install. - **Studio agent flow.** Studio ships with chat and workspace browsing today. The agent build, optimize, and security flows in Studio are in progress. diff --git a/docs/about/release-notes/index.md b/docs/about/release-notes/index.md deleted file mode 100644 index 8dcc771684..0000000000 --- a/docs/about/release-notes/index.md +++ /dev/null @@ -1,5 +0,0 @@ -# Release Notes for {{platform_name}} - -Check out the latest release notes for {{platform_name}}. - -- [Release {{ release }}](current-release.md) diff --git a/docs/about/release-notes/index.mdx b/docs/about/release-notes/index.mdx new file mode 100644 index 0000000000..87e0e63464 --- /dev/null +++ b/docs/about/release-notes/index.mdx @@ -0,0 +1,7 @@ +--- +title: "Release Notes for NeMo Platform" +description: "" +--- +Check out the latest release notes for NeMo Platform. + +- [Release 0.1.0](/documentation/reference/release-notes/current-release) diff --git a/docs/acknowledgements/index.md b/docs/acknowledgements/index.mdx similarity index 55% rename from docs/acknowledgements/index.md rename to docs/acknowledgements/index.mdx index b66eaa2474..2cdf051113 100644 --- a/docs/acknowledgements/index.md +++ b/docs/acknowledgements/index.mdx @@ -1,5 +1,7 @@ -# OSS License Acknowledgements - -The NVIDIA {{platform_name}} source repository includes open source license information in the [`LICENSES/`](https://github.com/NVIDIA-NeMo/nemo-platform/tree/main/LICENSES) directory. +--- +title: "OSS License Acknowledgements" +description: "" +--- +The NVIDIA NeMo Platform source repository includes open source license information in the [`LICENSES/`](https://github.com/NVIDIA-NeMo/nemo-platform/tree/main/LICENSES) directory. Review the license files in that directory, together with any license or acknowledgement files included in NVIDIA-distributed containers, model artifacts, and other separately distributed materials. Those artifacts may include additional components or terms that are not part of the source repository. diff --git a/docs/agents/index.md b/docs/agents/index.mdx similarity index 82% rename from docs/agents/index.md rename to docs/agents/index.mdx index 7108a240c4..1d535d101a 100644 --- a/docs/agents/index.md +++ b/docs/agents/index.mdx @@ -1,8 +1,10 @@ -# About Agents - +--- +title: "About Agents" +description: "" +--- -An agent on {{platform_name}} is a workflow that calls tools, talks to models +An agent on NeMo Platform is a workflow that calls tools, talks to models through the local platform, and runs as a managed service you can deploy, invoke, evaluate, and optimize as a unit. Agents are defined as NeMo Agent Toolkit (NAT) workflows and are managed through the `nemo agents` command @@ -14,7 +16,7 @@ NVIDIA NeMo Agent Toolkit is a flexible, lightweight, and unifying library that allows you to easily connect existing enterprise agents to data sources and tools across any framework. -{{platform_name}} uses NAT as the runtime wrapper around your agent so the +NeMo Platform uses NAT as the runtime wrapper around your agent so the platform can deploy it, evaluate it, optimize it, and route its model traffic through shared infrastructure. For the toolkit itself, see the [NeMo Agent Toolkit documentation](https://docs.nvidia.com/nemo/agent-toolkit/latest/). @@ -44,7 +46,6 @@ functions: llms: llm: _type: openai - api_key: not-used # injected at deploy time model_name: nvidia-nemotron-3-nano-30b-a3b workflow: @@ -53,14 +54,14 @@ workflow: llm_name: llm ``` -The [Inference Gateway](../run-inference/about.md) is the local platform's +The [Inference Gateway](/documentation/models-and-inference) is the local platform's model proxy. Agents send model requests to it instead of to provider APIs directly, so the platform can resolve model names, attach credentials, and route through middleware on the agent's behalf. Two conventions apply when a config targets a deployed agent: - **Model names use the Inference Gateway entity form**, with slashes and dots converted to hyphens (`nvidia/nemotron-3-nano-30b-a3b` becomes `nvidia-nemotron-3-nano-30b-a3b`). The Inference Gateway resolves the entity to the upstream provider that owns it. -- **Leave `base_url` and `api_key` unset on `openai` and `nim` LLMs.** {{platform_name}} injects an Inference Gateway URL when it deploys the agent, and the gateway retrieves upstream credentials from the secrets service. Setting `base_url` explicitly bypasses both. +- **Leave `base_url` and `api_key` unset on `openai` and `nim` LLMs.** NeMo Platform injects an Inference Gateway URL when it deploys the agent, and the gateway retrieves upstream credentials from the secrets service. Setting `base_url` explicitly bypasses both. ## Agent Lifecycle @@ -84,13 +85,13 @@ A platform-managed agent consists of three components: 1. **The agent entity.** `nemo agents create` stores the workflow YAML as an entity in the workspace. The same configuration can be redeployed, evaluated, or optimized without re-registering it. 1. **The deployment controller.** `nemo agents deploy` passes the stored config to the Agents service controller, which launches a `nat start fastapi` process for it, assigns a port, watches its health, and tears it down on `nemo agents undeploy`. -1. **The Agents gateway.** Client requests reach the agent at `/apis/agents/v2/workspaces//agents//-/`. The gateway resolves the agent to its current running deployment and proxies the request, including streaming responses. From a client's perspective, the agent is an OpenAI-compatible endpoint owned by {{platform_name}}. +1. **The Agents gateway.** Client requests reach the agent at `/apis/agents/v2/workspaces//agents//-/`. The gateway resolves the agent to its current running deployment and proxies the request, including streaming responses. From a client's perspective, the agent is an OpenAI-compatible endpoint owned by NeMo Platform. Model traffic from inside the agent process routes back through the Inference Gateway, which resolves model entity names to upstream providers and supplies their credentials. This is why agent configs do not carry `base_url` or `api_key` values — the deployment injects the gateway URL automatically, and the gateway looks up the rest. A virtual model is a platform-managed wrapper around one or more backend models. An agent can point at a virtual model entity while the virtual model -handles routing, format translation, and [guardrails](../guardrails/index.md) +handles routing, format translation, and [guardrails](/documentation/guardrail-models) behind the scenes — no changes to the agent's workflow YAML required. ### Applying Changes Through Candidate Agents @@ -108,11 +109,11 @@ undeploying the candidate. ## Common Tasks -- [Optimize Agents](optimization.md): analyze deployed agents for model routing, +- [Optimize Agents](/documentation/agents/optimize-agents): analyze deployed agents for model routing, skill, prompt, and new-model opportunities. -- [Secure Agents](security.md): check guardrail coverage and scan recent +- [Secure Agents](/documentation/agents/secure-agents): check guardrail coverage and scan recent telemetry for sensitive data. -- [Plugins and Skills](plugins.md): understand how agent, middleware, and +- [Plugins and Skills](/documentation/agents/plugins-and-skills): understand how agent, middleware, and coding-agent integrations extend the local platform. -- [Agentic Metrics](../evaluator/metrics/agentic.md): evaluate tool use, goal completion, topic adherence, answer accuracy, and trajectories. -- [Agent Configuration](../evaluator/metrics/agent-configuration.md): use agents as online evaluation targets. +- [Agentic Metrics](/documentation/evaluate-models/metrics/agentic-metrics): evaluate tool use, goal completion, topic adherence, answer accuracy, and trajectories. +- [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration): use agents as online evaluation targets. diff --git a/docs/agents/optimization.md b/docs/agents/optimization.md deleted file mode 100644 index a08f90ae6c..0000000000 --- a/docs/agents/optimization.md +++ /dev/null @@ -1,339 +0,0 @@ -# Optimize Agents - - - -Use the Agent Optimizer to analyze a deployed agent and act on improvement -suggestions. The optimizer inspects the agent's config, the workspace model -catalog, any prior optimizer snapshots, and optional evaluation baselines, -then writes suggestions you can review from the CLI or hand off to a coding -agent. - -This page covers the main path: establish a baseline, generate optimization -suggestions, apply a candidate change to a sibling agent, and review the -evaluation result before promotion. - -## What the Optimizer Checks - -| Suggestion type | Signal | Result | -|-----------------|--------|--------| -| Model optimization | An agent uses a single frontier model where a smaller model or route split may preserve quality at lower cost | Suggests a model swap or Switchyard random-routing virtual model | -| Skill optimization | The agent uses skills and has an evaluation suite | Suggests running `nemo agents optimize-skills` to improve skill files and keep changes that pass evaluation | -| Prompt optimization | The agent has an optimization config and baseline dataset | Suggests `nemo agents optimize run` for NAT prompt or parameter tuning | -| New model scan | Difference between the current model list and the previous optimizer snapshot | Suggests evaluating or auditing newly available models | - -Optimizer state is stored in the `nemo-agent-optimizer` fileset: - -- `optimizer_suggestions.jsonl`: one suggestion per line, including applied state. -- `optimizer_snapshot.json`: model and agent names from the latest run. - -Security-oriented suggestions such as missing guardrails, PII exposure, or -leaked secrets are covered in [Secure Agents](security.md). - -## Prerequisites - -Before running the optimizer, make sure you have: - -1. Local services running (`nemo services run`). -1. The agents plugin installed. For local development from this repository: - ```bash - uv pip install -e packages/nemo_platform_plugin -e plugins/nemo-agents - ``` -1. A workspace with at least one model provider and discovered model entities. -1. At least one deployed platform-managed agent. -1. An evaluation baseline before promoting a candidate agent. - -If you need a demo agent, start the platform and create the ReAct example: - -```bash -nemo services run -``` - -In another terminal: - -```bash -export NMP_BASE_URL=http://127.0.0.1:8080 -cd plugins/nemo-agents - -printf '%s' "$NVIDIA_API_KEY" | nemo secrets create ngc-api-key --from-file - -nemo inference providers create nvidia-build \ - --host-url https://integrate.api.nvidia.com \ - --api-key-secret-name ngc-api-key -nemo wait inference provider nvidia-build - -nemo agents create \ - --name react-agent \ - --agent-config examples/react-agent/react-agent.yml -nemo agents deploy --agent react-agent -nemo agents deployments wait --agent react-agent -``` - -The example agent uses `nvidia-nemotron-3-nano-30b-a3b`, so it can produce a -model optimization suggestion when the workspace model catalog contains a -smaller compatible model. - -## Optimize with Switchyard Routing - -Switchyard is the inference middleware that lets a virtual model split traffic -across multiple backend models. The common optimization pattern is to create a -[virtual model](../run-inference/about.md) with a strong model and a weaker, -cheaper model, then evaluate whether the route split preserves application -quality. - -Run `nemo models list` first and replace the placeholders below with model -entity names from your workspace that use the `OPENAI_CHAT` backend format. - -=== "CLI" - - The command below creates a virtual model that sends 80% of traffic to the - strong model and 20% to the weak one. - - ```bash - nemo inference virtual-models create routed-agent-model \ - --workspace default \ - --models '[ - {"model":"default/","backend_format":"OPENAI_CHAT"}, - {"model":"default/","backend_format":"OPENAI_CHAT"} - ]' \ - --request-middleware '[{ - "name":"nemo-switchyard", - "config_type":"random_routing", - "config":{ - "strong":{"model":"default/"}, - "weak":{"model":"default/"}, - "strong_probability":0.8, - "enable_stats":false - } - }]' - ``` - - Before wiring the virtual model to an agent, smoke-test the route by - making several minimal chat-completions calls and checking the returned - model name. The observed split should roughly match `strong_probability`. - -=== "Skill" - - Ask your coding agent: - - > Optimize my deployed agent. - - The `agents-optimize` skill picks a deployed agent, establishes an - evaluation baseline, runs the analysis steps below, and surfaces - suggestions for you to apply. - - Verify the skill is installed: - - ```bash - nemo skills show agents-optimize - ``` - - What it does under the hood: - - - Lists deployed agents and prompts you to choose one. - - Inspects the agent's `llms[*].model_name` and looks for cheaper compatible - models in the workspace catalog. - - Creates a Switchyard `random_routing` virtual model with an 80% strong / - 20% weak split and smoke-tests the route before wiring it to a sibling - agent. - - Suggests skill optimization, prompt tuning, and new-model evaluations - where the agent qualifies. - - Persists suggestions to the `nemo-agent-optimizer` fileset. - -=== "Python SDK" - - ```python - import os - from nemo_platform import NeMoPlatform - - client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="default", - ) - - client.inference.virtual_models.create( - name="routed-agent-model", - workspace="default", - models=[ - {"model": "default/", "backend_format": "OPENAI_CHAT"}, - {"model": "default/", "backend_format": "OPENAI_CHAT"}, - ], - request_middleware=[{ - "name": "nemo-switchyard", - "config_type": "random_routing", - "config": { - "strong": {"model": "default/"}, - "weak": {"model": "default/"}, - "strong_probability": 0.8, - "enable_stats": False, - }, - }], - ) - ``` - -## Optimize Skills - -Skill optimization applies when the agent depends on local skill files and has -an evaluation suite. The loop runs evaluations, analyzes failures, lets the -coding agent edit only the configured skills directory, reruns verification, -and keeps the change only when the evaluation result improves. - -=== "CLI" - - ```bash - nemo agents optimize-skills run --spec-file .agent-improver.yml - ``` - - Set `open_pr: true` in the YAML when you want the loop to prepare a - reviewable branch. - - A sample `.agent-improver.yml` is in - `plugins/nemo-agents/examples/agent-improver.example.yml`. - -=== "Skill" - - Ask your coding agent: - - > Optimize the skills used by my agent and keep the changes that improve evaluation scores. - - The `agents-optimize` skill drives the skill-optimization loop when the - selected agent has skills and an evaluation suite. Verify it is installed: - - ```bash - nemo skills show agents-optimize - ``` - - What it does under the hood: - - - Confirms the agent uses skills (a `--skills-path`, a `.agent-improver.yml`, - or skill files referenced from the config). - - Runs `nemo agents optimize-skills` against the configured skills directory. - - Re-runs evaluation and keeps the change only when scores improve. - - Persists outcomes to the `nemo-agent-optimizer` fileset. - -=== "Python SDK" - - ```python - from nemo_agents_plugin.jobs.optimize_skills import OptimizeSkillsJob - from nemo_platform_plugin.scheduler import NemoJobScheduler - - import yaml - from pathlib import Path - - spec = yaml.safe_load(Path(".agent-improver.yml").read_text()) - NemoJobScheduler().run_local( - OptimizeSkillsJob, - spec, - workspace="default", - ) - ``` - -## Inspect Saved Results - -Use the Files service to inspect what the optimizer saved: - -```bash -nemo files list nemo-agent-optimizer - -nemo files download nemo-agent-optimizer \ - --remote-path optimizer_suggestions.jsonl \ - -o optimizer_suggestions.jsonl - -nemo files download nemo-agent-optimizer \ - --remote-path optimizer_snapshot.json \ - -o optimizer_snapshot.json -``` - -Telemetry is optional. If agents use the `nemo_files` telemetry exporter, trace -files are written to `nemo-agent-telemetry`, and the optimizer samples the -largest JSONL file: - -```bash -nemo files list nemo-agent-telemetry -``` - -## Run Prompt and Parameter Tuning - -The `nemo agents optimize run` command runs the NAT optimizer path for -parameter or prompt tuning. Use it when you already have a NAT optimization -YAML and want to run `nat optimize` through the Agents plugin. - -For the ReAct example: - -=== "CLI" - - ```bash - nemo agents optimize run \ - --optimize-config plugins/nemo-agents/examples/react-agent/react-optimize.yml \ - --agent react-agent - ``` - -=== "Skill" - - Ask your coding agent: - - > Run prompt tuning on my deployed agent against this optimization config. - - The `agents-optimize` skill suggests `nemo agents optimize run` when the - agent has an optimization config and a baseline dataset. Verify it is - installed: - - ```bash - nemo skills show agents-optimize - ``` - - What it does under the hood: - - - Confirms the agent has a NAT optimization YAML. - - Runs `nemo agents optimize run` (or `submit` for platform jobs). - - Compares results against the evaluation baseline and surfaces deltas - for review. - -=== "Python SDK" - - ```python - import os - from pathlib import Path - - from nemo_agents_plugin.jobs.optimize_agent import OptimizeAgentJob - from nemo_platform import NeMoPlatform - from nemo_platform_plugin.scheduler import NemoJobScheduler - - WORKSPACE = "default" - optimize_config = Path("plugins/nemo-agents/examples/react-agent/react-optimize.yml") - - client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace=WORKSPACE, - ) - - result = NemoJobScheduler().run_local( - OptimizeAgentJob, - { - "optimize_config": str(optimize_config), - "agent": "react-agent", - "workspace": WORKSPACE, - }, - workspace=WORKSPACE, - sdk=client, - ) - print(result) - ``` - -When `--agent` is a platform-managed agent name, the job fetches the stored -agent config, merges it with the optimization config, injects the Inference -Gateway URL, and runs trials locally. When `--agent` is a raw HTTP endpoint, -the endpoint is treated as an opaque remote service, so local parameter sweeps -do not change the remote agent behavior. - -## Troubleshooting - -**No suggestions appear.** Confirm the workspace has agents, model entities, and a model catalog entry smaller than the agent's current model. New-model suggestions require a previous optimizer snapshot, so they do not appear on the first run. - -**The model evaluation fails.** Confirm the judge model in the eval config is available through the workspace Inference Gateway. You can replace the eval files in `-eval` with your own evaluation config and dataset. - -**Data safety suggestions do not appear.** Telemetry is optional. The optimizer only scans `nemo-agent-telemetry` when that fileset exists and contains JSONL trace files. - -## Next steps - -- [Agent overview](index.md): review how platform-managed agents are registered, deployed, invoked, evaluated, and optimized. -- [Agent evaluation](../evaluator/metrics/agent-configuration.md): configure agents as online evaluation targets and choose the right agent response mapping. -- [CLI reference](../cli/reference.md): look up complete command options and global CLI flags for scripted workflows. diff --git a/docs/agents/optimization.mdx b/docs/agents/optimization.mdx new file mode 100644 index 0000000000..9586d8b390 --- /dev/null +++ b/docs/agents/optimization.mdx @@ -0,0 +1,362 @@ +--- +title: "Optimize Agents" +description: "" +--- + + +Use the Agent Optimizer to analyze a deployed agent and act on improvement +suggestions. The optimizer inspects the agent's config, the workspace model +catalog, any prior optimizer snapshots, and optional evaluation baselines, +then writes suggestions you can review from the CLI or hand off to a coding +agent. + +This page covers the main path: establish a baseline, generate optimization +suggestions, apply a candidate change to a sibling agent, and review the +evaluation result before promotion. + +## What the Optimizer Checks + +| Suggestion type | Signal | Result | +|-----------------|--------|--------| +| Model optimization | An agent uses a single frontier model where a smaller model or route split may preserve quality at lower cost | Suggests a model swap or Switchyard random-routing virtual model | +| Skill optimization | The agent uses skills and has an evaluation suite | Suggests running `nemo agents optimize-skills` to improve skill files and keep changes that pass evaluation | +| Prompt optimization | The agent has an optimization config and baseline dataset | Suggests `nemo agents optimize run` for NAT prompt or parameter tuning | +| New model scan | Difference between the current model list and the previous optimizer snapshot | Suggests evaluating or auditing newly available models | + +Optimizer state is stored in the `nemo-agent-optimizer` fileset: + +- `optimizer_suggestions.jsonl`: one suggestion per line, including applied state. +- `optimizer_snapshot.json`: model and agent names from the latest run. + +Security-oriented suggestions such as missing guardrails, PII exposure, or +leaked secrets are covered in [Secure Agents](/documentation/agents/secure-agents). + +## Prerequisites + +Before running the optimizer, make sure you have: + +1. Local services running (`nemo services run`). +1. The agents plugin installed. For local development from this repository: + ```bash + uv pip install -e packages/nemo_platform_plugin -e plugins/nemo-agents + ``` +1. A workspace with at least one model provider and discovered model entities. +1. At least one deployed platform-managed agent. +1. An evaluation baseline before promoting a candidate agent. + +If you need a demo agent, start the platform and create the ReAct example: + +```bash +nemo services run +``` + +In another terminal: + +```bash +export NMP_BASE_URL=http://127.0.0.1:8080 +cd plugins/nemo-agents + +printf '%s' "$NVIDIA_API_KEY" | nemo secrets create ngc-api-key --from-file - +nemo inference providers create nvidia-build \ + --host-url https://integrate.api.nvidia.com \ + --api-key-secret-name ngc-api-key +nemo wait inference provider nvidia-build + +nemo agents create \ + --name react-agent \ + --agent-config examples/react-agent/react-agent.yml +nemo agents deploy --agent react-agent +nemo agents deployments wait --agent react-agent +``` + +The example agent uses `nvidia-nemotron-3-nano-30b-a3b`, so it can produce a +model optimization suggestion when the workspace model catalog contains a +smaller compatible model. + +## Optimize with Switchyard Routing + +Switchyard is the inference middleware that lets a virtual model split traffic +across multiple backend models. The common optimization pattern is to create a +[virtual model](/documentation/models-and-inference) with a strong model and a weaker, +cheaper model, then evaluate whether the route split preserves application +quality. + +Run `nemo models list` first and replace the placeholders below with model +entity names from your workspace that use the `OPENAI_CHAT` backend format. + + + + + +The command below creates a virtual model that sends 80% of traffic to the +strong model and 20% to the weak one. + +```bash +nemo inference virtual-models create routed-agent-model \ + --workspace default \ + --models '[ + {"model":"default/","backend_format":"OPENAI_CHAT"}, + {"model":"default/","backend_format":"OPENAI_CHAT"} + ]' \ + --request-middleware '[{ + "name":"nemo-switchyard", + "config_type":"random_routing", + "config":{ + "strong":{"model":"default/"}, + "weak":{"model":"default/"}, + "strong_probability":0.8, + "enable_stats":false + } + }]' +``` + +Before wiring the virtual model to an agent, smoke-test the route by +making several minimal chat-completions calls and checking the returned +model name. The observed split should roughly match `strong_probability`. + + + + +Ask your coding agent: + +> Optimize my deployed agent. + +The `agents-optimize` skill picks a deployed agent, establishes an +evaluation baseline, runs the analysis steps below, and surfaces +suggestions for you to apply. + +Verify the skill is installed: + +```bash +nemo skills show agents-optimize +``` + +What it does under the hood: + +- Lists deployed agents and prompts you to choose one. +- Inspects the agent's `llms[*].model_name` and looks for cheaper compatible + models in the workspace catalog. +- Creates a Switchyard `random_routing` virtual model with an 80% strong / + 20% weak split and smoke-tests the route before wiring it to a sibling + agent. +- Suggests skill optimization, prompt tuning, and new-model evaluations + where the agent qualifies. +- Persists suggestions to the `nemo-agent-optimizer` fileset. + + + + +```python +import os +from nemo_platform import NeMoPlatform + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) + +client.inference.virtual_models.create( + name="routed-agent-model", + workspace="default", + models=[ + {"model": "default/", "backend_format": "OPENAI_CHAT"}, + {"model": "default/", "backend_format": "OPENAI_CHAT"}, + ], + request_middleware=[{ + "name": "nemo-switchyard", + "config_type": "random_routing", + "config": { + "strong": {"model": "default/"}, + "weak": {"model": "default/"}, + "strong_probability": 0.8, + "enable_stats": False, + }, + }], +) +``` + + + + +## Optimize Skills + +Skill optimization applies when the agent depends on local skill files and has +an evaluation suite. The loop runs evaluations, analyzes failures, lets the +coding agent edit only the configured skills directory, reruns verification, +and keeps the change only when the evaluation result improves. + + + + + +```bash +nemo agents optimize-skills run --spec-file .agent-improver.yml +``` + +Set `open_pr: true` in the YAML when you want the loop to prepare a +reviewable branch. + +A sample `.agent-improver.yml` is in +`plugins/nemo-agents/examples/agent-improver.example.yml`. + + + + +Ask your coding agent: + +> Optimize the skills used by my agent and keep the changes that improve evaluation scores. + +The `agents-optimize` skill drives the skill-optimization loop when the +selected agent has skills and an evaluation suite. Verify it is installed: + +```bash +nemo skills show agents-optimize +``` + +What it does under the hood: + +- Confirms the agent uses skills (a `--skills-path`, a `.agent-improver.yml`, + or skill files referenced from the config). +- Runs `nemo agents optimize-skills` against the configured skills directory. +- Re-runs evaluation and keeps the change only when scores improve. +- Persists outcomes to the `nemo-agent-optimizer` fileset. + + + + +```python +import yaml +from pathlib import Path + +from nemo_agents_plugin.jobs.optimize_skills import OptimizeSkillsJob +from nemo_platform_plugin.scheduler import NemoJobScheduler + +spec = yaml.safe_load(Path(".agent-improver.yml").read_text()) +NemoJobScheduler().run_local( + OptimizeSkillsJob, + spec, + workspace="default", +) +``` + + + + +## Inspect Saved Results + +Use the Files service to inspect what the optimizer saved: + +```bash +nemo files list nemo-agent-optimizer + +nemo files download nemo-agent-optimizer \ + --remote-path optimizer_suggestions.jsonl \ + -o optimizer_suggestions.jsonl + +nemo files download nemo-agent-optimizer \ + --remote-path optimizer_snapshot.json \ + -o optimizer_snapshot.json +``` + +Telemetry is optional. If agents use the `nemo_files` telemetry exporter, trace +files are written to `nemo-agent-telemetry`, and the optimizer samples the +largest JSONL file: + +```bash +nemo files list nemo-agent-telemetry +``` + +## Run Prompt and Parameter Tuning + +The `nemo agents optimize run` command runs the NAT optimizer path for +parameter or prompt tuning. Use it when you already have a NAT optimization +YAML and want to run `nat optimize` through the Agents plugin. + +For the ReAct example: + + + + + +```bash +nemo agents optimize run \ + --optimize-config plugins/nemo-agents/examples/react-agent/react-optimize.yml \ + --agent react-agent +``` + + + + +Ask your coding agent: + +> Run prompt tuning on my deployed agent against this optimization config. + +The `agents-optimize` skill suggests `nemo agents optimize run` when the +agent has an optimization config and a baseline dataset. Verify it is +installed: + +```bash +nemo skills show agents-optimize +``` + +What it does under the hood: + +- Confirms the agent has a NAT optimization YAML. +- Runs `nemo agents optimize run` (or `submit` for platform jobs). +- Compares results against the evaluation baseline and surfaces deltas + for review. + + + + +```python +import os +from pathlib import Path + +from nemo_agents_plugin.jobs.optimize_agent import OptimizeAgentJob +from nemo_platform import NeMoPlatform +from nemo_platform_plugin.scheduler import NemoJobScheduler + +WORKSPACE = "default" +optimize_config = Path("plugins/nemo-agents/examples/react-agent/react-optimize.yml") + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace=WORKSPACE, +) + +result = NemoJobScheduler().run_local( + OptimizeAgentJob, + { + "optimize_config": str(optimize_config), + "agent": "react-agent", + "workspace": WORKSPACE, + }, + workspace=WORKSPACE, + sdk=client, +) +print(result) +``` + + + + +When `--agent` is a platform-managed agent name, the job fetches the stored +agent config, merges it with the optimization config, injects the Inference +Gateway URL, and runs trials locally. When `--agent` is a raw HTTP endpoint, +the endpoint is treated as an opaque remote service, so local parameter sweeps +do not change the remote agent behavior. + +## Troubleshooting + +**No suggestions appear.** Confirm the workspace has agents, model entities, and a model catalog entry smaller than the agent's current model. New-model suggestions require a previous optimizer snapshot, so they do not appear on the first run. + +**The model evaluation fails.** Confirm the judge model in the eval config is available through the workspace Inference Gateway. You can replace the eval files in `-eval` with your own evaluation config and dataset. + +**Data safety suggestions do not appear.** Telemetry is optional. The optimizer only scans `nemo-agent-telemetry` when that fileset exists and contains JSONL trace files. + +## Next steps + +- [Agent overview](/documentation/agents): review how platform-managed agents are registered, deployed, invoked, evaluated, and optimized. +- [Agent evaluation](/documentation/evaluate-models/metrics/agent-configuration): configure agents as online evaluation targets and choose the right agent response mapping. +- [CLI reference](/documentation/reference/cli-reference/full-cli-reference): look up complete command options and global CLI flags for scripted workflows. diff --git a/docs/agents/plugins.md b/docs/agents/plugins.mdx similarity index 89% rename from docs/agents/plugins.md rename to docs/agents/plugins.mdx index 373a2206bf..c9f1046b4f 100644 --- a/docs/agents/plugins.md +++ b/docs/agents/plugins.mdx @@ -1,8 +1,10 @@ -# Plugins and Skills - +--- +title: "Plugins and Skills" +description: "" +--- -{{platform_name}} {{ release }} is extended through Python plugins and +NeMo Platform 0.1.0 is extended through Python plugins and coding-agent skills. Plugins add runtime capabilities to the local platform. Skills teach your coding agent how to operate those capabilities on your behalf. @@ -65,8 +67,8 @@ The skills that drive the agent lifecycle are: | `nemo-try-agent` | Sends a query to a deployed agent. | | `nemo-status` | Read-only platform health dashboard. | | `nemo-teardown` | Guided shutdown with confirmation. | -| `agents-optimize` | Selects a deployed agent, establishes an evaluation baseline, and suggests Switchyard routing, model swaps, skill optimization, prompt tuning, and new-model evaluations. See [Optimize Agents](optimization.md). | -| `agents-secure` | Selects a deployed agent, checks guardrail coverage, and scans recent telemetry for sensitive data. See [Secure Agents](security.md). | +| `agents-optimize` | Selects a deployed agent, establishes an evaluation baseline, and suggests Switchyard routing, model swaps, skill optimization, prompt tuning, and new-model evaluations. See [Optimize Agents](/documentation/agents/optimize-agents). | +| `agents-secure` | Selects a deployed agent, checks guardrail coverage, and scans recent telemetry for sensitive data. See [Secure Agents](/documentation/agents/secure-agents). | Plugin-owned skills cover guardrails, evaluations, optimization, data designer, anonymizer, and auditor. They are installed with their plugin and @@ -92,8 +94,8 @@ Common middleware patterns: - **Switchyard translation** for cross-format model providers. - **Guardrails middleware** for request and response checks. -For command details, see the [Inference Gateway API Reference](../api/index.md#tag-inference-gateway) -and the [CLI reference](../cli/reference.md). +For command details, see the [Inference Gateway API Reference](/documentation/reference/api-reference) +and the [CLI reference](/documentation/reference/cli-reference/full-cli-reference). ## Writing Plugins @@ -101,5 +103,5 @@ Plugin authors use the `nemo-platform-plugin` package. A minimal plugin declares points in `pyproject.toml`, implements the matching class, and restarts `nemo services run` so the local platform discovers it. -Keep plugins local-first for OSS {{ release }}: avoid requiring a cluster, +Keep plugins local-first for OSS 0.1.0: avoid requiring a cluster, container runtime, or external control plane for the documented launch path. diff --git a/docs/agents/security.md b/docs/agents/security.mdx similarity index 54% rename from docs/agents/security.md rename to docs/agents/security.mdx index c6a59ebeba..404ebd071b 100644 --- a/docs/agents/security.md +++ b/docs/agents/security.mdx @@ -1,5 +1,7 @@ -# Secure Agents - +--- +title: "Secure Agents" +description: "" +--- Use the agent security workflow to check a deployed agent for guardrail @@ -47,94 +49,101 @@ with `nemo models list`): - `nvidia-llama-3-1-nemotron-safety-guard-8b-v3` For how to create the guardrail config that the virtual model references, see -the [Guardrails documentation](../guardrails/index.md). +the [Guardrails documentation](/documentation/guardrail-models). There are two steps: create the guarded VirtualModel, then update the agent's `llms` block to reference it. ### 1. Create a Guarded VirtualModel -=== "CLI" - - ```bash - nemo inference virtual-models create guarded-agent-model \ - --workspace default \ - --models '[{"model":"default/","backend_format":"OPENAI_CHAT"}]' \ - --request-middleware '[{ - "name":"nemo-guardrails", - "config_type":"guardrail_config", - "config_id":"default/" - }]' \ - --response-middleware '[{ - "name":"nemo-guardrails", - "config_type":"guardrail_config", - "config_id":"default/" - }]' - ``` - - Wire the same `` on both `--request-middleware` (for - input rails) and `--response-middleware` (for output rails). Omit a side - if the config defines no flows for it. For the full middleware schema, - entity-backed vs inline configs, and caching behavior, refer to - [Guardrails Architecture](../guardrails/concepts/architecture.md). - -=== "Skill" - - Ask your coding agent: - - > Secure my deployed agent. - - The `agents-secure` skill picks a deployed agent, checks guardrail - coverage, samples recent telemetry, and writes suggestions to the - `nemo-agent-security` fileset. - - Verify the skill is installed: - - ```bash - nemo skills show agents-secure - ``` - - What it does under the hood: - - - Lists deployed agents and prompts you to choose one. - - Inspects each LLM's `base_url`. If it does not route through a - guardrails virtual model, suggests creating one with a content-safety, - topic-control, or safety-guard backend. - - Names the recommended guardrails catalog model and walks you through - creating the guarded virtual model. - - Persists suggestions to the `nemo-agent-security` fileset. - -=== "Python SDK" - - ```python - import os - from nemo_platform import NeMoPlatform - - client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="default", - ) - - guardrail_mw = { - "name": "nemo-guardrails", - "config_type": "guardrail_config", - "config_id": "default/", - } - - client.inference.virtual_models.create( - name="guarded-agent-model", - workspace="default", - models=[{"model": "default/", "backend_format": "OPENAI_CHAT"}], - request_middleware=[guardrail_mw], - response_middleware=[guardrail_mw], - ) - ``` + + + + +```bash +nemo inference virtual-models create guarded-agent-model \ + --workspace default \ + --models '[{"model":"default/","backend_format":"OPENAI_CHAT"}]' \ + --request-middleware '[{ + "name":"nemo-guardrails", + "config_type":"guardrail_config", + "config_id":"default/" + }]' \ + --response-middleware '[{ + "name":"nemo-guardrails", + "config_type":"guardrail_config", + "config_id":"default/" + }]' +``` + +Wire the same `` on both `--request-middleware` (for +input rails) and `--response-middleware` (for output rails). Omit a side +if the config defines no flows for it. For the full middleware schema, +entity-backed vs inline configs, and caching behavior, refer to +[Guardrails Architecture](/documentation/guardrail-models/core-concepts/architecture). + + + + +Ask your coding agent: + +> Secure my deployed agent. + +The `agents-secure` skill picks a deployed agent, checks guardrail +coverage, samples recent telemetry, and writes suggestions to the +`nemo-agent-security` fileset. + +Verify the skill is installed: + +```bash +nemo skills show agents-secure +``` + +What it does under the hood: + +- Lists deployed agents and prompts you to choose one. +- Inspects each LLM's `model_name`. If it does not reference a guarded + virtual model (one with a content-safety, topic-control, or + safety-guard backend), suggests creating one. +- Names the recommended guardrails catalog model and walks you through + creating the guarded virtual model. +- Persists suggestions to the `nemo-agent-security` fileset. + + + + +```python +import os +from nemo_platform import NeMoPlatform + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) + +guardrail_mw = { + "name": "nemo-guardrails", + "config_type": "guardrail_config", + "config_id": "default/", +} + +client.inference.virtual_models.create( + name="guarded-agent-model", + workspace="default", + models=[{"model": "default/", "backend_format": "OPENAI_CHAT"}], + request_middleware=[guardrail_mw], + response_middleware=[guardrail_mw], +) +``` + + + ### 2. Point the Agent at the Guarded VirtualModel In the agent's workflow YAML, set `model_name` on the relevant `llms` entry to the guarded VirtualModel's entity reference, with slashes converted to hyphens -(per the [agent configuration conventions](index.md#agent-definition)): +(per the [agent configuration conventions](/documentation/agents#agent-definition)): ```yaml llms: @@ -149,7 +158,7 @@ unchanged and unaware of the rails. For the end-to-end request flow, streaming behavior, header forwarding, and the `guardrails` request options, refer to -[Running Inference with Guardrails](../guardrails/concepts/inference.md). +[Running Inference with Guardrails](/documentation/guardrail-models/core-concepts/running-inference). Redeploy the agent, re-run evaluation, and compare quality, cost, latency, and safety signals against the baseline before promoting. @@ -166,55 +175,62 @@ For leaked secrets, rotate the credential before doing any other cleanup. For PII in trace data, redact or regenerate affected traces before using them for evaluation or optimization. -=== "CLI" + + + - ```bash - nemo files list nemo-agent-telemetry - ``` +```bash +nemo files list nemo-agent-telemetry +``` + +Inspect the most recent trace files manually, or use the security skill +to do this for you. Cap the data you download — telemetry can be large. - Inspect the most recent trace files manually, or use the security skill - to do this for you. Cap the data you download — telemetry can be large. + + -=== "Skill" +Ask your coding agent: - Ask your coding agent: +> Scan recent telemetry for my agent for PII and leaked secrets. - > Scan recent telemetry for my agent for PII and leaked secrets. +The `agents-secure` skill caps downloaded telemetry at 1 GB and walks the +most recent traces first. Verify it is installed: - The `agents-secure` skill caps downloaded telemetry at 1 GB and walks the - most recent traces first. Verify it is installed: +```bash +nemo skills show agents-secure +``` - ```bash - nemo skills show agents-secure - ``` +What it does under the hood: - What it does under the hood: +- Samples recent files from `nemo-agent-telemetry` until the 1 GB cap. +- Runs a high-confidence regex pass for PII and leaked credentials. +- Writes findings with masked previews and follow-up actions to + `nemo-agent-security/security_suggestions.jsonl`. +- Optionally suggests a higher-recall GLiNER or NemoGuard scan on a + subset of traces when regex coverage is not enough. - - Samples recent files from `nemo-agent-telemetry` until the 1 GB cap. - - Runs a high-confidence regex pass for PII and leaked credentials. - - Writes findings with masked previews and follow-up actions to - `nemo-agent-security/security_suggestions.jsonl`. - - Optionally suggests a higher-recall GLiNER or NemoGuard scan on a - subset of traces when regex coverage is not enough. + + -=== "Python SDK" +The skill drives this workflow directly. To inspect what was written, +list and download files from the security fileset: - The skill drives this workflow directly. To inspect what was written, - list and download files from the security fileset: +```python +import os +from nemo_platform import NeMoPlatform - ```python - import os - from nemo_platform import NeMoPlatform +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) - client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="default", - ) +for f in client.files.list(fileset="nemo-agent-security"): + print(f.remote_path) +``` - for f in client.files.list(fileset="nemo-agent-security"): - print(f.remote_path) - ``` + + ## Review Results Use the Files service to inspect saved suggestions: @@ -239,9 +255,9 @@ nemo files download nemo-agent-security \ ## Next Steps -- [Optimize Agents](optimization.md): reduce cost and improve quality after +- [Optimize Agents](/documentation/agents/optimize-agents): reduce cost and improve quality after security coverage is in place. -- [Guardrails](../guardrails/index.md): create and manage guardrail +- [Guardrails](/documentation/guardrail-models): create and manage guardrail configurations. -- [Models and Inference](../run-inference/about.md): manage model providers, +- [Models and Inference](/documentation/models-and-inference): manage model providers, model entities, and virtual models. diff --git a/docs/anonymizer/cli.md b/docs/anonymizer/cli.mdx similarity index 94% rename from docs/anonymizer/cli.md rename to docs/anonymizer/cli.mdx index 73855f1e3a..84da4cc308 100644 --- a/docs/anonymizer/cli.md +++ b/docs/anonymizer/cli.mdx @@ -1,7 +1,10 @@ +--- +title: "CLI Reference" +description: "" +--- -# CLI Reference -This reference covers the `nemo anonymizer` commands exposed by the Anonymizer plugin. For end-to-end walkthroughs, see the [tutorials](tutorials/index.md). +This reference covers the `nemo anonymizer` commands exposed by the Anonymizer plugin. For end-to-end walkthroughs, see the [tutorials](/documentation/anonymize-data/tutorials). ## Command Surface @@ -11,7 +14,7 @@ This reference covers the `nemo anonymizer` commands exposed by the Anonymizer p | `nemo anonymizer preview run` | Generated from `NemoFunction` | Local streaming preview. | | `nemo anonymizer preview submit` | Generated from `NemoFunction` | Remote streaming preview against the plugin service. | | `nemo anonymizer run run` | Generated from `NemoJob` | Local job execution in the CLI process. | -| `nemo anonymizer run submit` | Generated from `NemoJob` | Submit an `anonymizer.run` job to the {{platform_name}} Jobs worker. | +| `nemo anonymizer run submit` | Generated from `NemoJob` | Submit an `anonymizer.run` job to the NeMo Platform Jobs worker. | | `nemo anonymizer run explain` | Generated from `NemoJob` | Print the job key, submit endpoint, and JSON schemas. | ## `nemo anonymizer validate` @@ -49,7 +52,7 @@ nemo anonymizer preview submit \ | Flag | Description | |------------------|---------------------------------------------------------------------------------------------------| | `--spec-file` | Path to the `PreviewRequest` YAML. | -| `--workspace` | {{platform_name}} workspace. Used to resolve fileset references and Inference Gateway providers. | +| `--workspace` | NeMo Platform workspace. Used to resolve fileset references and Inference Gateway providers. | | `--base-url` | Override the platform base URL (typically auto-populated from the CLI config). | ### Preview source kinds @@ -70,7 +73,7 @@ nemo anonymizer preview run --spec-file /tmp/anonymizer-preview.yaml > /tmp/prev jq -R 'fromjson? | select(.kind == "preview_dataset") | .records' /tmp/preview.ndjson ``` -Frame kinds: `log`, `preview_dataset`, `trace_dataset`, `failed_records`, `heartbeat`, `done`, `error`. See the [preview tutorial](tutorials/preview.md) for details. +Frame kinds: `log`, `preview_dataset`, `trace_dataset`, `failed_records`, `heartbeat`, `done`, `error`. See the [preview tutorial](/documentation/anonymize-data/tutorials/preview-a-config) for details. ## `nemo anonymizer run` @@ -137,7 +140,7 @@ results = job.download_artifacts() dataset = results.load_dataset() ``` -See [SDK Resources](sdk-resources.md) for the full `AnonymizerJobResource` / `AnonymizerJobResults` surface. +See [SDK Resources](/documentation/anonymize-data/sdk-resources) for the full `AnonymizerJobResource` / `AnonymizerJobResults` surface. Compared to `run run`, `run submit` rejects local file paths in `data.source` (use a fileset reference or `http(s)` URL) and requires explicit `model_configs` because the job runs outside the CLI process. diff --git a/docs/anonymizer/index.md b/docs/anonymizer/index.mdx similarity index 72% rename from docs/anonymizer/index.md rename to docs/anonymizer/index.mdx index f61bc01e61..d9c669c6d6 100644 --- a/docs/anonymizer/index.md +++ b/docs/anonymizer/index.mdx @@ -1,19 +1,24 @@ +--- +title: "Anonymizer Service" +description: "" +--- -# Anonymizer Service -The Anonymizer service detects personally identifiable information (PII) in text data on the {{platform_name}} and replaces or rewrites it. +The Anonymizer service detects personally identifiable information (PII) in text data on the NeMo Platform and replaces or rewrites it. ## Overview -The service wraps the open-source [NVIDIA NeMo Anonymizer library](https://github.com/NVIDIA-NeMo/Anonymizer) and exposes it through the {{platform_name}}'s Python SDK and CLI. The library still owns PII detection, replacement, rewrite, and config validation. The platform adds inference routing through the Inference Gateway, fileset-backed inputs, plugin-service execution for streaming preview, and a Jobs-worker path for full anonymization runs. +The service wraps the open-source [NVIDIA NeMo Anonymizer library](https://github.com/NVIDIA-NeMo/Anonymizer) and exposes it through the NeMo Platform's Python SDK and CLI. The library still owns PII detection, replacement, rewrite, and config validation. The platform adds inference routing through the Inference Gateway, fileset-backed inputs, plugin-service execution for streaming preview, and a Jobs-worker path for full anonymization runs. ## How It Works: Library + Platform The library defines **what** to anonymize and **how**. The platform decides **where the work runs** and **how models are reached**. -!!! note - The code snippets below are for conceptual demonstration purposes only. For runnable examples, see the [tutorials](tutorials/index.md). + +The code snippets below are for conceptual demonstration purposes only. For runnable examples, see the [tutorials](/documentation/anonymize-data/tutorials). + + ### 1. Build a config with the library Use `anonymizer.config` (installed automatically with the `nemo-anonymizer-plugin`) to define the replacement strategy: @@ -33,7 +38,7 @@ config = AnonymizerConfig( ### 2. Execute on the platform -Submit the config to the Anonymizer service with the {{platform_name}} SDK: +Submit the config to the Anonymizer service with the NeMo Platform SDK: ```python from nemo_anonymizer_plugin.app.task_config import PreviewRequest @@ -67,15 +72,15 @@ The SDK equivalent of `run submit` is `sdk.anonymizer.run(request)`, which retur ## Key Differences from Standalone Library -When using Anonymizer as a {{platform_name}} service: +When using Anonymizer as a NeMo Platform service: -| Feature | Standalone Library | {{platform_name}} Service | +| Feature | Standalone Library | NeMo Platform Service | |-------------------|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------------------| | **Inference** | Direct calls to NVIDIA Build defaults | Routes through the Inference Gateway via `model_configs` | | **Execution** | Local Python process | Streaming preview runs in the plugin service; full runs execute either in the local CLI (`run run`) or on the Jobs worker (`run submit`) | -| **Input sources** | Local file, `http(s)` URL | Local file (`run run` only), `http(s)` URL, or {{platform_name}} Fileset | -| **Artifacts** | Local filesystem | Local artifact directory (`persistent/results/artifacts`) for `run run`; {{platform_name}} job artifact storage for `run submit` | -| **Authentication**| Direct API keys | {{platform_name}} Secrets service | +| **Input sources** | Local file, `http(s)` URL | Local file (`run run` only), `http(s)` URL, or NeMo Platform Fileset | +| **Artifacts** | Local filesystem | Local artifact directory (`persistent/results/artifacts`) for `run run`; NeMo Platform job artifact storage for `run submit` | +| **Authentication**| Direct API keys | NeMo Platform Secrets service | ## Replacement Strategies @@ -98,36 +103,33 @@ This package is a thin wrapper around the [NVIDIA NeMo Anonymizer library](https - A `nemo anonymizer` CLI with `validate`, `preview`, and `run` command groups. - An `sdk.anonymizer` SDK accessor (`AnonymizerResource`, `AsyncAnonymizerResource`). - A streaming `anonymizer.preview` function that emits `preview_dataset`, `trace_dataset`, and `failed_records` frames from the plugin service. -- An `anonymizer.run` job that writes `dataset.parquet`, `trace.parquet`, `metadata.json`, and optional `failed_records.json`. The job can execute in the local CLI process (`nemo anonymizer run run`) or on the {{platform_name}} Jobs worker (`nemo anonymizer run submit` / `sdk.anonymizer.run`). +- An `anonymizer.run` job that writes `dataset.parquet`, `trace.parquet`, `metadata.json`, and optional `failed_records.json`. The job can execute in the local CLI process (`nemo anonymizer run run`) or on the NeMo Platform Jobs worker (`nemo anonymizer run submit` / `sdk.anonymizer.run`). - Fileset input handling (`fileset:///#`). - Inference Gateway routing for model providers referenced from `model_configs`. ## Next Steps -
- -- **[Tutorials](tutorials/index.md)** - - --- - - Walk through preview (`anonymizer.preview`) and job execution (`anonymizer.run`) end to end. + -- **[SDK Resources](sdk-resources.md)** + - --- +Walk through preview (`anonymizer.preview`) and job execution (`anonymizer.run`) end to end. - Reference for the `anonymizer` SDK accessor, preview result, and job result objects. + + -- **[CLI Reference](cli.md)** +Reference for the `anonymizer` SDK accessor, preview result, and job result objects. - --- + + - Reference for `nemo anonymizer` commands and their spec files. +Reference for `nemo anonymizer` commands and their spec files. -- **[Library Documentation](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs)** + + - --- +Detection, replacement strategies, rewrite mode, and other library internals. - Detection, replacement strategies, rewrite mode, and other library internals. + -
+ diff --git a/docs/anonymizer/sdk-resources.md b/docs/anonymizer/sdk-resources.mdx similarity index 78% rename from docs/anonymizer/sdk-resources.md rename to docs/anonymizer/sdk-resources.mdx index d6882a817e..d98954bc82 100644 --- a/docs/anonymizer/sdk-resources.md +++ b/docs/anonymizer/sdk-resources.mdx @@ -1,11 +1,14 @@ +--- +title: "Anonymizer NeMo Platform SDK Resources" +description: "" +--- -# Anonymizer {{platform_name}} SDK Resources -The `anonymizer.config` module (from the [NVIDIA NeMo Anonymizer library](https://github.com/NVIDIA-NeMo/Anonymizer)) builds `AnonymizerConfig` objects in a context-agnostic way. Once you are ready to execute that config against the {{platform_name}} Anonymizer service, you use objects from the `nemo_platform` SDK. This page describes the {{platform_name}}-specific objects. +The `anonymizer.config` module (from the [NVIDIA NeMo Anonymizer library](https://github.com/NVIDIA-NeMo/Anonymizer)) builds `AnonymizerConfig` objects in a context-agnostic way. Once you are ready to execute that config against the NeMo Platform Anonymizer service, you use objects from the `nemo_platform` SDK. This page describes the NeMo Platform-specific objects. ## AnonymizerResource -The `AnonymizerResource` is the entry point for working with Anonymizer on {{platform_name}}. It wraps the streaming preview endpoint and job submission for the plugin service. +The `AnonymizerResource` is the entry point for working with Anonymizer on NeMo Platform. It wraps the streaming preview endpoint and job submission for the plugin service. A `AnonymizerResource` is accessed directly from a `NeMoPlatform` instance: @@ -25,7 +28,7 @@ An `AsyncAnonymizerResource` with the same surface is available via `AsyncNeMoPl | Method | Description | |----------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| | `preview(request, *, workspace=None)` | Runs a streaming preview against the plugin service and returns an `AnonymizerPreviewResult` after the stream completes. | -| `run(request, *, workspace=None, wait_until_done=False)` | Submits an `anonymizer.run` job to the {{platform_name}} Jobs worker. Returns an `AnonymizerJobResource`. When `wait_until_done=True`, blocks until the job reaches a terminal state. | +| `run(request, *, workspace=None, wait_until_done=False)` | Submits an `anonymizer.run` job to the NeMo Platform Jobs worker. Returns an `AnonymizerJobResource`. When `wait_until_done=True`, blocks until the job reaches a terminal state. | | `get_job_resource(job_name, workspace=None)` | Returns an `AnonymizerJobResource` for an existing job (by job name). | `request` is a `PreviewRequest` or `AnonymizerRequest` instance from `nemo_anonymizer_plugin.app.task_config`. Both accept the same `config`, `data`, `model_configs`, and `selected_models` fields; `PreviewRequest` adds `num_records`. @@ -43,9 +46,11 @@ Both `preview` and `run` call the plugin service, so they require `model_configs | `failed_records` | `list[dict]` of per-record failures with reasons. Empty when nothing failed. | | `display_record(index=None)` | Renders a single trace record as HTML in a notebook. When `index` is omitted, cycles through records. | -??? "More about preview results" - `AnonymizerPreviewResult` holds everything in memory; nothing is persisted to disk by default. The `dataset` and `trace_dataset` fields are regular pandas DataFrames and can be saved with `to_csv` / `to_parquet`. + +`AnonymizerPreviewResult` holds everything in memory; nothing is persisted to disk by default. The `dataset` and `trace_dataset` fields are regular pandas DataFrames and can be saved with `to_csv` / `to_parquet`. + + ## AnonymizerJobResource `AnonymizerResource.run` returns an `AnonymizerJobResource`. You can also use `AnonymizerResource.get_job_resource` to get one for an existing job. @@ -87,19 +92,21 @@ dataset = results.load_dataset() | `load_failed_records()` | Returns `failed_records.json` as `list[dict]`. Returns `[]` when the file isn't present. | | `display_record(index=None)` | Renders a single trace record as HTML in a notebook. When `index` is omitted, cycles through records. | -??? "More about job results" - `AnonymizerJobResults` reads files lazily — methods load the corresponding parquet or JSON only when called. The underlying directory layout is: + + +`AnonymizerJobResults` reads files lazily — methods load the corresponding parquet or JSON only when called. The underlying directory layout is: - ```text - / - dataset.parquet - trace.parquet - metadata.json - failed_records.json # only when there were failures - ``` +```text +/ + dataset.parquet + trace.parquet + metadata.json + failed_records.json # only when there were failures +``` - By default, `download_artifacts` saves the tarball contents to a local directory named after the job; pass `path=` to override. +By default, `download_artifacts` saves the tarball contents to a local directory named after the job; pass `path=` to override. + ## Request Models Both request models live in `nemo_anonymizer_plugin.app.task_config`. @@ -112,8 +119,8 @@ Both request models live in `nemo_anonymizer_plugin.app.task_config`. |-------------------|-------------------------------------------------|-----------------------------------------------------------------------------| | `config` | `AnonymizerConfig` | Upstream library config (replace strategy or rewrite, detection params). | | `data` | `AnonymizerInputSpec` | Input source plus column metadata. See below. | -| `model_configs` | `list[data_designer.config.ModelConfig] \| None` | Model pool. `provider` references an Inference Gateway provider name. | -| `selected_models` | `SelectedModelsOverrides \| None` | Optional role overrides on top of bundled defaults. Requires `model_configs`. | +| `model_configs` | `list[data_designer.config.ModelConfig] \\| None` | Model pool. `provider` references an Inference Gateway provider name. | +| `selected_models` | `SelectedModelsOverrides \\| None` | Optional role overrides on top of bundled defaults. Requires `model_configs`. | `PreviewRequest` extends `AnonymizerRequest` with `num_records` @@ -121,8 +128,8 @@ Both request models live in `nemo_anonymizer_plugin.app.task_config`. |-------------------|-------------------------------------------------|-----------------------------------------------------------------------------| | `config` | `AnonymizerConfig` | Upstream library config (replace strategy or rewrite, detection params). | | `data` | `AnonymizerInputSpec` | Input source plus column metadata. See below. | -| `model_configs` | `list[data_designer.config.ModelConfig] \| None` | Model pool. `provider` references an Inference Gateway provider name. | -| `selected_models` | `SelectedModelsOverrides \| None` | Optional role overrides on top of bundled defaults. Requires `model_configs`. | +| `model_configs` | `list[data_designer.config.ModelConfig] \\| None` | Model pool. `provider` references an Inference Gateway provider name. | +| `selected_models` | `SelectedModelsOverrides \\| None` | Optional role overrides on top of bundled defaults. Requires `model_configs`. | | `num_records` | `int` (≥ 1, default `10`) | Preview-only. Number of records to preview. Capped by the service's `preview_num_records.max`. | ### AnonymizerInputSpec @@ -133,8 +140,8 @@ The plugin-owned API-boundary input spec: |----------------|----------------|------------------------------------------------------------------------------------------| | `source` | `str` | Local path, `http(s)` URL, or fileset reference for a CSV / Parquet file. | | `text_column` | `str` (default `"text"`) | Column containing text to anonymize. | -| `id_column` | `str \| None` | Optional record identifier column. | -| `data_summary` | `str \| None` | Optional short description of the data passed to Anonymizer library prompts. | +| `id_column` | `str \\| None` | Optional record identifier column. | +| `data_summary` | `str \\| None` | Optional short description of the data passed to Anonymizer library prompts. | Fileset references can take any of the three forms `fileset:///#`, `/#`, or `#`, and must resolve to a single `.csv` or `.parquet` file. @@ -144,8 +151,8 @@ Partial role → alias overrides for the three workflows. Each section is option | Field | Type | Description | |-------------|---------------------------------------|------------------------------------------------------------| -| `detection` | `dict[str, str \| list[str]] \| None` | Role → alias or alias pool for detection. | -| `replace` | `dict[str, str] \| None` | Role → alias for replacement (for example `replacement_generator`). | -| `rewrite` | `dict[str, str] \| None` | Role → alias for rewrite mode. | +| `detection` | `dict[str, str \\| list[str]] \\| None` | Role → alias or alias pool for detection. | +| `replace` | `dict[str, str] \\| None` | Role → alias for replacement (for example `replacement_generator`). | +| `rewrite` | `dict[str, str] \\| None` | Role → alias for rewrite mode. | Supplying overrides without `model_configs` raises a config validation error. diff --git a/docs/anonymizer/tutorials/index.md b/docs/anonymizer/tutorials/index.mdx similarity index 73% rename from docs/anonymizer/tutorials/index.md rename to docs/anonymizer/tutorials/index.mdx index 18a7057c06..cb2c45e600 100644 --- a/docs/anonymizer/tutorials/index.md +++ b/docs/anonymizer/tutorials/index.mdx @@ -1,5 +1,8 @@ +--- +title: "Tutorials" +description: "" +--- -# Tutorials These tutorials cover the two user-facing surfaces of the Anonymizer plugin: the streaming `preview` workflow for iteration, and the `run` job for full datasets. @@ -9,7 +12,7 @@ Anonymizer separates **configuration** (what to detect and how to replace it) fr **Part 1: Build the config (library)** -Use [`anonymizer.config`](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs) to define the rewrite or replacement strategy and detection options. This code is identical whether you run Anonymizer standalone or through the {{platform_name}} service. +Use [`anonymizer.config`](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs) to define the rewrite or replacement strategy and detection options. This code is identical whether you run Anonymizer standalone or through the NeMo Platform service. ```python from anonymizer.config.anonymizer_config import AnonymizerConfig @@ -66,17 +69,17 @@ preview = anonymizer.preview(PreviewRequest( ## Service-Specific Considerations -When using Anonymizer as a {{platform_name}} service: +When using Anonymizer as a NeMo Platform service: | Feature | Difference | Details | |----------------|---------------------------------------------------------|----------------------------------------------------------------------------------------| | **Inference** | Routes through the Inference Gateway | Configure providers once and reference them by name from `model_configs`. | | **Input data** | Filesets and HTTP(S) URLs (local paths only in local CLI execution) | Use `sdk.files.filesets.create` / `sdk.files.upload`, then reference with `#`. | -| **Artifacts** | Local or platform-managed | `run run` writes to `persistent/results/artifacts` locally; `run submit` stores artifacts in {{platform_name}} job storage. | +| **Artifacts** | Local or platform-managed | `run run` writes to `persistent/results/artifacts` locally; `run submit` stores artifacts in NeMo Platform job storage. | ## Prerequisites -Complete [Setup](../../get-started/setup.md) to install {{platform_name}}, run `nemo services run`, and configure an inference provider. The root workspace includes the Anonymizer plugin, so `nemo services run` discovers it automatically and mounts `/apis/anonymizer/...` on the gateway — no separate plugin install step is needed. Verify the CLI is registered: +Complete [Setup](/documentation/get-started) to install NeMo Platform, run `nemo services run`, and configure an inference provider. The root workspace includes the Anonymizer plugin, so `nemo services run` discovers it automatically and mounts `/apis/anonymizer/...` on the gateway — no separate plugin install step is needed. Verify the CLI is registered: ```bash nemo anonymizer --help @@ -84,9 +87,9 @@ nemo anonymizer --help You should see `validate`, `preview`, and `run` command groups. -These tutorials route inference through an [Inference Gateway](../../run-inference/about.md) provider, so a {{platform_name}} cluster must be running before you preview or run a job. The examples reference the default NVIDIA Build provider created during setup. +These tutorials route inference through an [Inference Gateway](/documentation/models-and-inference) provider, so a NeMo Platform cluster must be running before you preview or run a job. The examples reference the default NVIDIA Build provider created during setup. ---8<-- "_snippets/nvidia-build-model-provider.md" + ### Upload an Input Fileset @@ -138,22 +141,21 @@ The tutorials reference this file with `fileset://{WORKSPACE}/anonymizer-inputs# ## Tutorials -
+ -- **[Preview a Config](preview.md)** + - --- +Stream a small anonymized sample to iterate on `AnonymizerConfig` and `model_configs`. Covers `sdk.anonymizer.preview`, `nemo anonymizer preview run` / `preview submit`, and the NDJSON frame stream. - Stream a small anonymized sample to iterate on `AnonymizerConfig` and `model_configs`. Covers `sdk.anonymizer.preview`, `nemo anonymizer preview run` / `preview submit`, and the NDJSON frame stream. +beginner anonymizer - beginner anonymizer + + -- **[Run an Anonymizer Job](run.md)** +Run the full pipeline locally with `nemo anonymizer run run` or submit it to the Jobs worker with `nemo anonymizer run submit`. Load `dataset.parquet`, `trace.parquet`, and `failed_records.json` artifacts. - --- +intermediate anonymizer - Run the full pipeline locally with `nemo anonymizer run run` or submit it to the Jobs worker with `nemo anonymizer run submit`. Load `dataset.parquet`, `trace.parquet`, and `failed_records.json` artifacts. + - intermediate anonymizer - -
+ diff --git a/docs/anonymizer/tutorials/preview.md b/docs/anonymizer/tutorials/preview.mdx similarity index 89% rename from docs/anonymizer/tutorials/preview.md rename to docs/anonymizer/tutorials/preview.mdx index 6814c9728e..717cd934db 100644 --- a/docs/anonymizer/tutorials/preview.md +++ b/docs/anonymizer/tutorials/preview.mdx @@ -1,5 +1,8 @@ +--- +title: "Preview a Config" +description: "" +--- -# Preview a Config This tutorial walks through the `anonymizer.preview` flow: defining a request, streaming a small anonymized sample, and inspecting the resulting frames. @@ -7,9 +10,9 @@ For detection and replacement strategy details, see the [open-source library doc ## Prerequisites -Complete the [tutorials prerequisites](index.md#prerequisites), which cover: +Complete the [tutorials prerequisites](/documentation/anonymize-data/tutorials#prerequisites), which cover: -- A running {{platform_name}} cluster with the `nemo anonymizer` CLI available (see [Setup](../../get-started/setup.md)). +- A running NeMo Platform cluster with the `nemo anonymizer` CLI available (see [Setup](/documentation/get-started)). - An inference provider configured (default examples use `nvidia-build`). - A fileset named `anonymizer-inputs` with `anonymizer-input.csv` uploaded. @@ -105,20 +108,22 @@ preview.display_record(0) # render record 0 with entity highlights in a noteboo `AnonymizerResource.preview` calls the plugin service, so the same constraints as `preview submit` apply: use a fileset or `http(s)` source, and include `model_configs`. -??? "Async client" - `AsyncAnonymizerResource.preview` has the same signature and return type: + - ```python - import os - from nemo_platform import AsyncNeMoPlatform +`AsyncAnonymizerResource.preview` has the same signature and return type: - async_sdk = AsyncNeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace=WORKSPACE, - ) - preview = await async_sdk.anonymizer.preview(request) - ``` +```python +import os +from nemo_platform import AsyncNeMoPlatform + +async_sdk = AsyncNeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace=WORKSPACE, +) +preview = await async_sdk.anonymizer.preview(request) +``` + ## Step 3: Persist Preview Records `AnonymizerPreviewResult.dataset` and `trace_dataset` are pandas DataFrames, so save them with standard pandas methods: @@ -128,9 +133,11 @@ preview.dataset.to_csv("anonymized-preview.csv", index=False) preview.dataset.to_parquet("anonymized-preview.parquet", index=False) ``` -??? "More about preview results" - `AnonymizerPreviewResult` stores everything in memory; nothing is persisted to disk by default. The `dataset` field is a regular pandas DataFrame and can be saved with `to_csv` or `to_parquet`. + + +`AnonymizerPreviewResult` stores everything in memory; nothing is persisted to disk by default. The `dataset` field is a regular pandas DataFrame and can be saved with `to_csv` or `to_parquet`. + ## Step 4: Run from the CLI (Alternative) The CLI accepts the same request shape as a YAML spec file. Use `preview run` for local execution (allows local paths, model configs optional) or `preview submit` for the plugin service path (same as `sdk.anonymizer.preview`). @@ -174,7 +181,7 @@ jq -R 'fromjson? | select(.kind == "preview_dataset") | .records' \ /tmp/anonymizer-preview.ndjson ``` -If `preview submit` returns 404 against the gateway, the plugin service isn't mounted. Restart `nemo services run` so the plugin is discovered and remounts `/apis/anonymizer/...`; see [Setup](../../get-started/setup.md). +If `preview submit` returns 404 against the gateway, the plugin service isn't mounted. Restart `nemo services run` so the plugin is discovered and remounts `/apis/anonymizer/...`; see [Setup](/documentation/get-started). ## Input Source Forms @@ -217,6 +224,6 @@ nemo anonymizer validate --config /tmp/anonymizer-config.yaml ## Next Steps -- Run a full job and inspect parquet output in the [run tutorial](run.md). -- Refer to [SDK Resources](../sdk-resources.md) for `AnonymizerPreviewResult` and `AnonymizerResource.preview` details. +- Run a full job and inspect parquet output in the [run tutorial](/documentation/anonymize-data/tutorials/run-an-anonymizer-job). +- Refer to [SDK Resources](/documentation/anonymize-data/sdk-resources) for `AnonymizerPreviewResult` and `AnonymizerResource.preview` details. - Learn about rewrite and replacement strategy parameters in the [library docs](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs). diff --git a/docs/anonymizer/tutorials/run.md b/docs/anonymizer/tutorials/run.mdx similarity index 89% rename from docs/anonymizer/tutorials/run.md rename to docs/anonymizer/tutorials/run.mdx index 0663ffa615..2440f53649 100644 --- a/docs/anonymizer/tutorials/run.md +++ b/docs/anonymizer/tutorials/run.mdx @@ -1,15 +1,18 @@ +--- +title: "Run an Anonymizer Job" +description: "" +--- -# Run an Anonymizer Job -This tutorial walks through the `anonymizer.run` job: defining a run spec, executing it locally or on the {{platform_name}} Jobs worker, and loading the parquet artifacts it produces. +This tutorial walks through the `anonymizer.run` job: defining a run spec, executing it locally or on the NeMo Platform Jobs worker, and loading the parquet artifacts it produces. For detection, rewrite, and replacement strategy details, see the [open-source library documentation](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs). ## Prerequisites -Complete the [tutorials prerequisites](index.md#prerequisites), which cover: +Complete the [tutorials prerequisites](/documentation/anonymize-data/tutorials#prerequisites), which cover: -- A running {{platform_name}} cluster with the `nemo anonymizer` CLI available (see [Setup](../../get-started/setup.md)). +- A running NeMo Platform cluster with the `nemo anonymizer` CLI available (see [Setup](/documentation/get-started)). - An inference provider configured (default examples use `nvidia-build`). - A fileset named `anonymizer-inputs` with `anonymizer-input.csv` uploaded. @@ -22,7 +25,7 @@ There are three run commands: | Command | Where it runs | Local paths | `model_configs` required | Artifacts | |-------------------------------|----------------------------------------------|-------------|--------------------------|--------------------------------------------------------| | `nemo anonymizer run run` | Local CLI process via generated `is_local` path | Allowed | Optional | Written under `persistent/results/artifacts` locally | -| `nemo anonymizer run submit` | {{platform_name}} Jobs worker | Rejected | Required | Stored in {{platform_name}} job artifact storage; pull with `download_artifacts()` | +| `nemo anonymizer run submit` | NeMo Platform Jobs worker | Rejected | Required | Stored in NeMo Platform job artifact storage; pull with `download_artifacts()` | | `nemo anonymizer run explain` | Local schema introspection | n/a | n/a | Prints job key, submit endpoint, and input/spec schemas | Job artifacts (under the `artifacts/` directory): @@ -84,7 +87,7 @@ spec_path.write_text(yaml.safe_dump(request.model_dump(mode="json", exclude_none ## Step 3: Run the Job -Choose one execution path. Option A runs in the local CLI process. Option B submits the same request to the {{platform_name}} Jobs worker. +Choose one execution path. Option A runs in the local CLI process. Option B submits the same request to the NeMo Platform Jobs worker. ### Option A: Run Locally @@ -110,7 +113,7 @@ Use that path in the next step. ### Option B: Submit to the Jobs Worker -To execute the same spec on the {{platform_name}} Jobs worker instead of in the CLI process, use `run submit`: +To execute the same spec on the NeMo Platform Jobs worker instead of in the CLI process, use `run submit`: ```bash nemo anonymizer run submit \ @@ -233,7 +236,7 @@ trace = results.load_trace() failed = results.load_failed_records() ``` -`AnonymizerJobResults` exposes `load_dataset()`, `load_trace()`, `load_failed_records()`, and `display_record()` over the same underlying files. See [SDK Resources](../sdk-resources.md#anonymizerjobresults). +`AnonymizerJobResults` exposes `load_dataset()`, `load_trace()`, `load_failed_records()`, and `display_record()` over the same underlying files. See [SDK Resources](/documentation/anonymize-data/sdk-resources#anonymizerjobresults). ## Inspect the Schema Without Running @@ -261,6 +264,6 @@ For `run submit`, provider endpoints are re-resolved at runtime so the job uses ## Next Steps -- Iterate faster with [preview](preview.md) before scaling to a full job. -- Refer to [SDK Resources](../sdk-resources.md) for `AnonymizerJobResource` and `AnonymizerJobResults` details. +- Iterate faster with [preview](/documentation/anonymize-data/tutorials/preview-a-config) before scaling to a full job. +- Refer to [SDK Resources](/documentation/anonymize-data/sdk-resources) for `AnonymizerJobResource` and `AnonymizerJobResults` details. - Replacement strategy parameters and rewrite mode are documented in the [library docs](https://github.com/NVIDIA-NeMo/Anonymizer/tree/main/docs). diff --git a/docs/api/index.md b/docs/api/index.md deleted file mode 100644 index ae3f0034ca..0000000000 --- a/docs/api/index.md +++ /dev/null @@ -1,17 +0,0 @@ -# {{platform_name}} API Reference - -Explore the full {{platform_name}} API. Use the service buttons to filter by microservice. - -
- - - - - - - - - -
- - diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml deleted file mode 120000 index e9738fedff..0000000000 --- a/docs/api/openapi.yaml +++ /dev/null @@ -1 +0,0 @@ -../../openapi/openapi.yaml \ No newline at end of file diff --git a/docs/auditor/configs/index.md b/docs/auditor/configs/index.mdx similarity index 88% rename from docs/auditor/configs/index.md rename to docs/auditor/configs/index.mdx index 824a11d933..a3534c73dd 100644 --- a/docs/auditor/configs/index.md +++ b/docs/auditor/configs/index.mdx @@ -1,7 +1,10 @@ +--- +title: "Manage Audit Configurations" +description: "" +--- -# Manage Audit Configurations -An `AuditConfig` selects which garak probes and detectors run during an audit, how many generations to make per probe, and where the reports land. Configurations are persisted in the {{platform_name}} entity store and referenced by name from `client.auditor.run(...)`. +An `AuditConfig` selects which garak probes and detectors run during an audit, how many generations to make per probe, and where the reports land. Configurations are persisted in the NeMo Platform entity store and referenced by name from `client.auditor.run(...)`. ## What an AuditConfig Holds @@ -14,7 +17,7 @@ A configuration has four sub-blocks plus a free-form description. Each sub-block | `plugins` | The probes and detectors to run (`probe_spec`, `detector_spec`) and any plugin-specific options. | | `reporting` | Output file prefix and directory, optional taxonomy, and which modules are surfaced in summaries. | -The full field reference is in [Configuration Schema](schema.md). +The full field reference is in [Configuration Schema](/documentation/vulnerability-scanning/configurations/schema). ## The Seeded Default @@ -51,7 +54,7 @@ config = client.auditor.configs.create( ) ``` -The `probe_spec` field takes a comma-separated list of probe categories or fully-qualified probe classes. See [Selecting Probes](probes.md) for the syntax and worked examples. +The `probe_spec` field takes a comma-separated list of probe categories or fully-qualified probe classes. See [Selecting Probes](/documentation/vulnerability-scanning/configurations/selecting-probes) for the syntax and worked examples. ## List Configurations diff --git a/docs/auditor/configs/probes.md b/docs/auditor/configs/probes.mdx similarity index 98% rename from docs/auditor/configs/probes.md rename to docs/auditor/configs/probes.mdx index ae742b238b..974cd530b9 100644 --- a/docs/auditor/configs/probes.md +++ b/docs/auditor/configs/probes.mdx @@ -1,5 +1,8 @@ +--- +title: "Selecting Probes" +description: "" +--- -# Selecting Probes An `AuditConfig`'s `plugins.probe_spec` field controls which garak probes run during an audit. Two related fields refine the selection: `plugins.detector_spec` chooses how outputs are scored, and `run.probe_tags` filters the selected probes by hazard tag. diff --git a/docs/auditor/configs/schema.md b/docs/auditor/configs/schema.mdx similarity index 75% rename from docs/auditor/configs/schema.md rename to docs/auditor/configs/schema.mdx index bfa4d5399d..4138ee57cb 100644 --- a/docs/auditor/configs/schema.md +++ b/docs/auditor/configs/schema.mdx @@ -1,7 +1,10 @@ +--- +title: "Configuration Schema" +description: "" +--- -# Configuration Schema -This page lists every field on `AuditConfig` and its four sub-models. Defaults match the pydantic definitions in `nemo_auditor.entities`; the {{platform_name}} entity store validates writes against these schemas. +This page lists every field on `AuditConfig` and its four sub-models. Defaults match the pydantic definitions in `nemo_auditor.entities`; the NeMo Platform entity store validates writes against these schemas. ## `AuditConfig` @@ -9,7 +12,7 @@ This page lists every field on `AuditConfig` and its four sub-models. Defaults m |-------|------|---------|-------------| | `name` | `str` | _required_ | Unique configuration name within the workspace. | | `workspace` | `str` | _required_ | Workspace the configuration is persisted in. | -| `description` | `str \| None` | `None` | Free-form description shown in listings. | +| `description` | `str \\| None` | `None` | Free-form description shown in listings. | | `system` | `AuditSystemData` | _defaults_ | Garak runtime knobs — see below. | | `run` | `AuditRunData` | _defaults_ | Per-probe generation settings — see below. | | `plugins` | `AuditPluginsData` | _defaults_ | Probe and detector selection — see below. | @@ -26,7 +29,7 @@ Controls garak's process-level behavior. | `verbose` | `int` (0–1) | `0` | Garak verbosity level. `0` for normal output, `1` for verbose. | | `narrow_output` | `bool` | `False` | When `True`, garak narrows its terminal output. Useful for log scraping. | | `parallel_requests` | `bool` | `False` | Enable parallel request dispatch to the target. | -| `parallel_attempts` | `bool \| int` | `False` | Enable parallel attempts, or set an explicit attempt count. Higher values speed up audits but increase target load. | +| `parallel_attempts` | `bool \\| int` | `False` | Enable parallel attempts, or set an explicit attempt count. Higher values speed up audits but increase target load. | | `lite` | `bool` | `True` | Run only fast probe variants. Recommended for iterative development. Set to `False` to include slower, more thorough probes. | | `show_z` | `bool` | `False` | Show garak's z-score column in reports. | | `enable_experimental` | `bool` | `False` | Allow probes marked experimental to run. | @@ -39,11 +42,11 @@ Controls how each probe is executed. | Field | Type | Default | Description | |-------|------|---------|-------------| -| `seed` | `int \| None` | `None` | Random seed for reproducible runs. | +| `seed` | `int \\| None` | `None` | Random seed for reproducible runs. | | `deprefix` | `bool` | `True` | Strip prompt prefixes from responses before detection. | | `eval_threshold` | `float` (0–1) | `0.5` | Detector score threshold above which a response is flagged as a hit. | | `generations` | `int` (≥1) | `5` | Generations per probe. Lower for fast iteration; raise for statistical confidence. | -| `probe_tags` | `str \| None` | `None` | Comma-separated tag filter (`"owasp:llm06,payload:hallucination"`). Intersects with `plugins.probe_spec`. See [Selecting Probes](probes.md). | +| `probe_tags` | `str \\| None` | `None` | Comma-separated tag filter (`"owasp:llm06,payload:hallucination"`). Intersects with `plugins.probe_spec`. See [Selecting Probes](/documentation/vulnerability-scanning/configurations/selecting-probes). | | `user_agent` | `str` | `"garak/{version} (LLM vulnerability scanner https://garak.ai)"` | HTTP `User-Agent` garak sends. The `{version}` placeholder is substituted at runtime. | ## `AuditPluginsData` @@ -52,14 +55,14 @@ Controls probe, detector, generator, buff, and harness selection. | Field | Type | Default | Description | |-------|------|---------|-------------| -| `model_type` | `str \| None` | `None` | Garak generator type override. Normally set on the target instead. | -| `model_name` | `str \| None` | `None` | Garak model name override. Normally set on the target instead. | -| `probe_spec` | `str` | `"all"` | Comma-separated probe selection. See [Selecting Probes](probes.md). | +| `model_type` | `str \\| None` | `None` | Garak generator type override. Normally set on the target instead. | +| `model_name` | `str \\| None` | `None` | Garak model name override. Normally set on the target instead. | +| `probe_spec` | `str` | `"all"` | Comma-separated probe selection. See [Selecting Probes](/documentation/vulnerability-scanning/configurations/selecting-probes). | | `detector_spec` | `str` | `"auto"` | Detector selection — `"auto"` lets each probe pick its detectors. | | `extended_detectors` | `bool` | `False` | Enable garak's extended detector set. | -| `buff_spec` | `str \| None` | `None` | Comma-separated buff selection (prompt mutations applied before sending). | +| `buff_spec` | `str \\| None` | `None` | Comma-separated buff selection (prompt mutations applied before sending). | | `buffs_include_original_prompt` | `bool` | `False` | When buffs are active, also send the original prompt. | -| `buff_max` | `str \| None` | `None` | Maximum number of buffed variants per prompt. | +| `buff_max` | `str \\| None` | `None` | Maximum number of buffed variants per prompt. | | `detectors` | `dict[str, ModuleConfig]` | `{}` | Per-detector configuration overrides. | | `generators` | `dict[str, ModuleConfig]` | `{}` | Per-generator configuration overrides. Normally set on the target instead. | | `buffs` | `dict[str, ModuleConfig]` | `{}` | Per-buff configuration overrides. | @@ -73,6 +76,6 @@ Controls where reports land and how they are summarized. | Field | Type | Default | Description | |-------|------|---------|-------------| | `report_prefix` | `str` | `"run1"` | Filename prefix for the produced reports. `report_prefix="quick-scan"` yields `quick-scan.report.jsonl`, `quick-scan.report.html`, and `quick-scan.hitlog.jsonl`. | -| `taxonomy` | `str \| None` | `None` | Optional taxonomy label included in the report. | +| `taxonomy` | `str \\| None` | `None` | Optional taxonomy label included in the report. | | `report_dir` | `str` | `"garak_runs"` | Sub-directory of `$XDG_DATA_HOME/garak/` where garak writes its raw output. The plugin then copies the produced files into the local results sink. | | `show_100_pass_modules` | `bool` | `True` | Include 100%-pass modules in the report summary. Set to `False` to hide them for a tighter "failures only" view. | diff --git a/docs/auditor/index.md b/docs/auditor/index.md deleted file mode 100644 index 20b3e1a332..0000000000 --- a/docs/auditor/index.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -description: Scan and audit large language models for jailbreaks, prompt injection, encoding bypasses, and other safety failures using {{__auditor_short_name}}, powered by garak. ---- - - -# About Auditing Models - -!!! important "{{__auditor_long_name}} is released with _early access_ availability and is subject to limited support and potential API changes in future releases." - -{{__auditor_long_name}} audits LLMs by probing them with adversarial prompts and detecting failures such as jailbreaks, prompt injection, encoding bypasses, and unsafe output generation. -It is powered by [garak](https://github.com/NVIDIA/garak), NVIDIA's open-source LLM vulnerability scanner, and integrates with {{platform_name}} so audits can target any model reachable through the Inference Gateway. - -[**Tutorials**](tutorials/index.md){ .md-button } -[**SDK Resources**](sdk-resources.md){ .md-button } - ---- - -## Typical Workflow - -A typical audit looks like the following: - -1. Create an [audit target](targets/index.md) for the model you want to test. -1. Create an [audit configuration](configs/index.md) that selects which garak probes and detectors to run, along with reporting settings. -1. [Run the audit](tutorials/run-audit-locally.md) and inspect the resulting JSONL, HTML, and hitlog reports. - -The plugin exposes both [synchronous and asynchronous](sdk-resources.md) Python entry points for each step. - ---- - -## Setup - -Before you can run audits, you need a working {{platform_name}} install with the auditor plugin enabled and a garak interpreter on disk. - -- Follow [Setup](../get-started/setup.md) to install the platform and start local services. -- Install garak in a Python virtual environment. By default the plugin invokes `~/.auditor/.venv/bin/python -m garak`; override the interpreter path with `NEMO_AUDITOR_GARAK_PYTHON` if you installed it elsewhere. -- Configure at least one [Inference Gateway provider](../run-inference/about.md) so audits can route requests to the model under test. - ---- - -## Task Guides - -
- -- **[Audit Targets](targets/index.md)** - - --- - - Define the model under test — generator type, model identifier, and inference endpoint. - -- **[Audit Configurations](configs/index.md)** - - --- - - Choose probes, detectors, and reporting settings for the audit. - -- **[Run an Audit Locally](tutorials/run-audit-locally.md)** - - --- - - End-to-end walkthrough: create entities, run the audit in-process, read the report artifacts. - -- **[SDK Resources](sdk-resources.md)** - - --- - - Reference for the `client.auditor` SDK surface: `configs`, `targets`, and `run()`. - -
- -## References - -
- -- **[Configuration Schema](configs/schema.md)** - - --- - - Field reference for `AuditConfig` and its `system`, `run`, `plugins`, and `reporting` sub-models. - -- **[Target Schema](targets/schema.md)** - - --- - - Field reference for `AuditTarget` (`type`, `model`, `options`). - -- **[Selecting Probes](configs/probes.md)** - - --- - - `probe_spec`, `probe_tags`, and `detector_spec` syntax with worked examples. - -- **[Inference Gateway](targets/inference-gateway.md)** - - --- - - How `nmp_uri_spec` resolves a target's URI through a {{platform_name}} provider. - -
diff --git a/docs/auditor/index.mdx b/docs/auditor/index.mdx new file mode 100644 index 0000000000..5606c99f45 --- /dev/null +++ b/docs/auditor/index.mdx @@ -0,0 +1,95 @@ +--- +description: Scan and audit large language models for jailbreaks, prompt injection, encoding bypasses, and other safety failures using NeMo Auditor, powered by garak. +--- + + + + + +NVIDIA NeMo Auditor is released with _early access_ availability and is subject to limited support and potential API changes in future releases. + + + +NVIDIA NeMo Auditor audits LLMs by probing them with adversarial prompts and detecting failures such as jailbreaks, prompt injection, encoding bypasses, and unsafe output generation. +It is powered by [garak](https://github.com/NVIDIA/garak), NVIDIA's open-source LLM vulnerability scanner, and integrates with NeMo Platform so audits can target any model reachable through the Inference Gateway. + + + + +--- + +## Typical Workflow + +A typical audit looks like the following: + +1. Create an [audit target](/documentation/vulnerability-scanning/targets) for the model you want to test. +1. Create an [audit configuration](/documentation/vulnerability-scanning/configurations) that selects which garak probes and detectors to run, along with reporting settings. +1. [Run the audit](/documentation/vulnerability-scanning/tutorials/run-an-audit-locally) and inspect the resulting JSONL, HTML, and hitlog reports. + +The plugin exposes both [synchronous and asynchronous](/documentation/vulnerability-scanning/sdk-resources) Python entry points for each step. + +--- + +## Setup + +Before you can run audits, you need a working NeMo Platform install with the auditor plugin enabled and a garak interpreter on disk. + +- Follow [Setup](/documentation/get-started) to install the platform and start local services. +- Install garak in a Python virtual environment. By default the plugin invokes `~/.auditor/.venv/bin/python -m garak`; override the interpreter path with `NEMO_AUDITOR_GARAK_PYTHON` if you installed it elsewhere. +- Configure at least one [Inference Gateway provider](/documentation/models-and-inference) so audits can route requests to the model under test. + +--- + +## Task Guides + + + + + +Define the model under test — generator type, model identifier, and inference endpoint. + + + + +Choose probes, detectors, and reporting settings for the audit. + + + + +End-to-end walkthrough: create entities, run the audit in-process, read the report artifacts. + + + + +Reference for the `client.auditor` SDK surface: `configs`, `targets`, and `run()`. + + + + + +## References + + + + + +Field reference for `AuditConfig` and its `system`, `run`, `plugins`, and `reporting` sub-models. + + + + +Field reference for `AuditTarget` (`type`, `model`, `options`). + + + + +`probe_spec`, `probe_tags`, and `detector_spec` syntax with worked examples. + + + + +How `nmp_uri_spec` resolves a target's URI through a NeMo Platform provider. + + + + diff --git a/docs/auditor/sdk-resources.md b/docs/auditor/sdk-resources.mdx similarity index 85% rename from docs/auditor/sdk-resources.md rename to docs/auditor/sdk-resources.mdx index e356f2c50c..35d2ca3b29 100644 --- a/docs/auditor/sdk-resources.md +++ b/docs/auditor/sdk-resources.mdx @@ -1,16 +1,19 @@ +--- +title: "NeMo Auditor NeMo Platform SDK Resources" +description: "" +--- -# {{__auditor_short_name}} {{platform_name}} SDK Resources -The {{__auditor_short_name}} plugin mounts a Python SDK surface on the `nemo_platform` client at `client.auditor`. +The NeMo Auditor plugin mounts a Python SDK surface on the `nemo_platform` client at `client.auditor`. This page documents that surface: how to manage audit configurations and targets in the entity store, and how to run an audit in-process using the local execution path. -The CRUD methods exposed on `client.auditor.configs` and `client.auditor.targets` are 1:1 mirrors of the [audit configuration](configs/index.md) and [audit target](targets/index.md) lifecycle and use the same `AuditConfig` and `AuditTarget` pydantic schemas the entity store persists. +The CRUD methods exposed on `client.auditor.configs` and `client.auditor.targets` are 1:1 mirrors of the [audit configuration](/documentation/vulnerability-scanning/configurations) and [audit target](/documentation/vulnerability-scanning/targets) lifecycle and use the same `AuditConfig` and `AuditTarget` pydantic schemas the entity store persists. ## AuditorPluginResource -The `AuditorPluginResource` is the sync SDK object for working with the {{__auditor_short_name}} plugin. +The `AuditorPluginResource` is the sync SDK object for working with the NeMo Auditor plugin. It is accessed directly from a `NeMoPlatform` instance: ```python @@ -34,7 +37,7 @@ auditor = client.auditor # AuditorPluginResource ### `configs` sub-resource -Five CRUD methods for `AuditConfig` entities. The full field reference is in [Configuration Schema](configs/schema.md). +Five CRUD methods for `AuditConfig` entities. The full field reference is in [Configuration Schema](/documentation/vulnerability-scanning/configurations/schema). | Method | Description | Returns | |--------|-------------|---------| @@ -46,7 +49,7 @@ Five CRUD methods for `AuditConfig` entities. The full field reference is in [Co ### `targets` sub-resource -Five CRUD methods for `AuditTarget` entities. The full field reference is in [Target Schema](targets/schema.md). +Five CRUD methods for `AuditTarget` entities. The full field reference is in [Target Schema](/documentation/vulnerability-scanning/targets/schema). | Method | Description | Returns | |--------|-------------|---------| @@ -63,9 +66,9 @@ The work happens entirely on the host running the SDK call — there is no remot | Argument | Type | Required | Description | |----------|------|----------|-------------| -| `config` | `AuditConfig \| str` | Yes | An inline `AuditConfig` instance or a name string referencing one in the entity store. A bare name such as `"quick-scan"` resolves against the `workspace` argument; a qualified name such as `"prod/quick-scan"` always uses the workspace prefix. | -| `target` | `AuditTarget \| str` | Yes | An inline `AuditTarget` instance or a name string, with the same resolution rules as `config`. | -| `workspace` | `str \| None` | No | Workspace used both as the entity-lookup fallback and as the scope for the local `JobContext`. Defaults to `"default"`. | +| `config` | `AuditConfig \\| str` | Yes | An inline `AuditConfig` instance or a name string referencing one in the entity store. A bare name such as `"quick-scan"` resolves against the `workspace` argument; a qualified name such as `"prod/quick-scan"` always uses the workspace prefix. | +| `target` | `AuditTarget \\| str` | Yes | An inline `AuditTarget` instance or a name string, with the same resolution rules as `config`. | +| `workspace` | `str \\| None` | No | Workspace used both as the entity-lookup fallback and as the scope for the local `JobContext`. Defaults to `"default"`. | ### `run()` return value diff --git a/docs/auditor/targets/index.md b/docs/auditor/targets/index.mdx similarity index 87% rename from docs/auditor/targets/index.md rename to docs/auditor/targets/index.mdx index 434457a9e8..654922561a 100644 --- a/docs/auditor/targets/index.md +++ b/docs/auditor/targets/index.mdx @@ -1,7 +1,10 @@ +--- +title: "Manage Audit Targets" +description: "" +--- -# Manage Audit Targets -An `AuditTarget` identifies the model under test. It pairs a garak generator class (`type`) with a model identifier (`model`) and a generator-specific options dict (`options`). Targets are persisted in the {{platform_name}} entity store and referenced by name from `client.auditor.run(...)`. +An `AuditTarget` identifies the model under test. It pairs a garak generator class (`type`) with a model identifier (`model`) and a generator-specific options dict (`options`). Targets are persisted in the NeMo Platform entity store and referenced by name from `client.auditor.run(...)`. ## What an AuditTarget Holds @@ -9,16 +12,16 @@ An `AuditTarget` identifies the model under test. It pairs a garak generator cla |-------|-------------| | `type` | A fully-qualified [garak generator class](https://reference.garak.ai/en/latest/generators.html), such as `nim.NVOpenAIChat`, `openai.OpenAIGenerator`, `rest.RestGenerator`, or `test.Blank`. | | `model` | The provider's model identifier passed through to garak (for example, `meta/llama-3.1-8b-instruct`). | -| `options` | A nested dict whose top-level key is the generator namespace (`nim`, `openai`, `rest`, ...). Contents are passed through to garak unchanged, with one exception: the [`nmp_uri_spec`](inference-gateway.md) sentinel is resolved to a concrete `uri` at run time. | +| `options` | A nested dict whose top-level key is the generator namespace (`nim`, `openai`, `rest`, ...). Contents are passed through to garak unchanged, with one exception: the [`nmp_uri_spec`](/documentation/vulnerability-scanning/targets/inference-gateway) sentinel is resolved to a concrete `uri` at run time. | | `description` | Optional free-form description shown in listings. | -See [Target Schema](schema.md) for the full field reference. +See [Target Schema](/documentation/vulnerability-scanning/targets/schema) for the full field reference. ## Common Target Types ### NIM via Inference Gateway -Audit a NIM (or any OpenAI-compatible chat endpoint) routed through a {{platform_name}} provider: +Audit a NIM (or any OpenAI-compatible chat endpoint) routed through a NeMo Platform provider: ```python target = client.auditor.targets.create( @@ -40,7 +43,7 @@ target = client.auditor.targets.create( ) ``` -The `nmp_uri_spec` block resolves to a concrete `uri` at run time. See [Inference Gateway](inference-gateway.md) for details. +The `nmp_uri_spec` block resolves to a concrete `uri` at run time. See [Inference Gateway](/documentation/vulnerability-scanning/targets/inference-gateway) for details. ### OpenAI-compatible Endpoint diff --git a/docs/auditor/targets/inference-gateway.md b/docs/auditor/targets/inference-gateway.mdx similarity index 92% rename from docs/auditor/targets/inference-gateway.md rename to docs/auditor/targets/inference-gateway.mdx index fe113ccf64..b4e7820abd 100644 --- a/docs/auditor/targets/inference-gateway.md +++ b/docs/auditor/targets/inference-gateway.mdx @@ -1,7 +1,10 @@ +--- +title: "Routing Targets Through the Inference Gateway" +description: "" +--- -# Routing Targets Through the Inference Gateway -`AuditTarget.options` is passed through to garak unchanged, with one exception: the plugin recognizes a `nmp_uri_spec` sentinel and resolves it to a concrete URI at run time using the {{platform_name}} Inference Gateway. This keeps target definitions portable — you store provider references instead of host-specific URLs. +`AuditTarget.options` is passed through to garak unchanged, with one exception: the plugin recognizes a `nmp_uri_spec` sentinel and resolves it to a concrete URI at run time using the NeMo Platform Inference Gateway. This keeps target definitions portable — you store provider references instead of host-specific URLs. ## What `nmp_uri_spec` Is diff --git a/docs/auditor/targets/schema.md b/docs/auditor/targets/schema.mdx similarity index 76% rename from docs/auditor/targets/schema.md rename to docs/auditor/targets/schema.mdx index 243587980b..8c5d2d9478 100644 --- a/docs/auditor/targets/schema.md +++ b/docs/auditor/targets/schema.mdx @@ -1,7 +1,10 @@ +--- +title: "Target Schema" +description: "" +--- -# Target Schema -This page lists every field on `AuditTarget`. Defaults match the pydantic definition in `nemo_auditor.entities`; the {{platform_name}} entity store validates writes against this schema. +This page lists every field on `AuditTarget`. Defaults match the pydantic definition in `nemo_auditor.entities`; the NeMo Platform entity store validates writes against this schema. ## `AuditTarget` @@ -12,7 +15,7 @@ This page lists every field on `AuditTarget`. Defaults match the pydantic defini | `type` | `str` | Yes | A fully-qualified [garak generator class](https://reference.garak.ai/en/latest/generators.html), such as `nim.NVOpenAIChat`, `openai.OpenAIGenerator`, `rest.RestGenerator`, or `test.Blank`. | | `model` | `str` | Yes | Provider model identifier (for example, `meta/llama-3.1-8b-instruct`). Some generator types (`test.Blank`) ignore this field but the schema still requires it. | | `options` | `dict[str, Any]` | No | Nested generator-specific options. The top-level key matches the generator namespace (`nim`, `openai`, `rest`, ...). Defaults to `{}`. | -| `description` | `str \| None` | No | Free-form description shown in listings. Defaults to `None`. | +| `description` | `str \\| None` | No | Free-form description shown in listings. Defaults to `None`. | The entity store adds the standard `NemoEntity` fields on retrieval: `id`, `entity_type` (`"auditor_audit_target"`), `created_at`, `updated_at`, and `project`. @@ -20,4 +23,4 @@ The entity store adds the standard `NemoEntity` fields on retrieval: `id`, `enti `options` is intentionally opaque to the plugin — its contents are passed through to garak as the generator's `--generator_option_file` payload. Refer to garak's [generator documentation](https://reference.garak.ai/en/latest/generators.html) for the options each generator class accepts. -The plugin recognizes one sentinel inside `options.`: an `nmp_uri_spec` block is resolved at run time to a concrete `uri` value via the {{platform_name}} Inference Gateway. See [Inference Gateway](inference-gateway.md) for the resolution rules and conflict semantics. +The plugin recognizes one sentinel inside `options.`: an `nmp_uri_spec` block is resolved at run time to a concrete `uri` value via the NeMo Platform Inference Gateway. See [Inference Gateway](/documentation/vulnerability-scanning/targets/inference-gateway) for the resolution rules and conflict semantics. diff --git a/docs/auditor/tutorials/index.md b/docs/auditor/tutorials/index.md deleted file mode 100644 index 43fe5e1b1e..0000000000 --- a/docs/auditor/tutorials/index.md +++ /dev/null @@ -1,24 +0,0 @@ - -# {{__auditor_short_name}} Tutorials - -Use these tutorials to get hands-on with [{{__auditor_short_name}}](../index.md). - -## Before You Start - -Set up [a local instance of the platform](../../get-started/setup.md), install garak in a Python virtual environment (the plugin invokes `~/.auditor/.venv/bin/python` by default), and configure at least one [Inference Gateway provider](../../run-inference/about.md) before running the tutorial below. - -
- -- **[Run an Audit Locally](run-audit-locally.md)** - - --- - - Persist a configuration and target, run a single audit in-process, and inspect the resulting JSONL, HTML, and hitlog report artifacts. - - getting started local execution nemo-auditor - -
- -## How It Works - -For the conceptual overview of the audit workflow, see [About Auditing Models](../index.md). For the full SDK surface, see [SDK Resources](../sdk-resources.md). diff --git a/docs/auditor/tutorials/index.mdx b/docs/auditor/tutorials/index.mdx new file mode 100644 index 0000000000..7357530b1c --- /dev/null +++ b/docs/auditor/tutorials/index.mdx @@ -0,0 +1,26 @@ +--- +title: "NeMo Auditor Tutorials" +description: "" +--- + +Use these tutorials to get hands-on with [NeMo Auditor](/documentation/vulnerability-scanning). + +## Before You Start + +Set up [a local instance of the platform](/documentation/get-started), install garak in a Python virtual environment (the plugin invokes `~/.auditor/.venv/bin/python` by default), and configure at least one [Inference Gateway provider](/documentation/models-and-inference) before running the tutorial below. + + + + + +Persist a configuration and target, run a single audit in-process, and inspect the resulting JSONL, HTML, and hitlog report artifacts. + +getting started · local execution · nemo-auditor + + + + + +## How It Works + +For the conceptual overview of the audit workflow, see [About Auditing Models](/documentation/vulnerability-scanning). For the full SDK surface, see [SDK Resources](/documentation/vulnerability-scanning/sdk-resources). diff --git a/docs/auditor/tutorials/run-audit-locally.md b/docs/auditor/tutorials/run-audit-locally.mdx similarity index 84% rename from docs/auditor/tutorials/run-audit-locally.md rename to docs/auditor/tutorials/run-audit-locally.mdx index 326671f408..4c4784eb6e 100644 --- a/docs/auditor/tutorials/run-audit-locally.md +++ b/docs/auditor/tutorials/run-audit-locally.mdx @@ -1,24 +1,29 @@ +--- +title: "Run an Audit Locally" +description: "" +--- -# Run an Audit Locally -This tutorial walks through running a single audit end-to-end with the {{__auditor_short_name}} plugin SDK. You will persist an audit configuration and a target, run the audit in-process, and inspect the resulting report artifacts. +This tutorial walks through running a single audit end-to-end with the NeMo Auditor plugin SDK. You will persist an audit configuration and a target, run the audit in-process, and inspect the resulting report artifacts. **What you will learn:** -- Initialize the {{platform_name}} SDK and reach the `client.auditor` resource. +- Initialize the NeMo Platform SDK and reach the `client.auditor` resource. - Create an `AuditConfig` selecting which garak probes to run. - Create an `AuditTarget` pointing at a model through the Inference Gateway. - Execute the audit locally with `client.auditor.run(...)`. - Read the JSONL, HTML, and hitlog report artifacts the run produces. -!!! tip - This tutorial takes approximately **10 minutes** to complete, plus however long garak takes to run the selected probes against your target. + + +This tutorial takes approximately **10 minutes** to complete, plus however long garak takes to run the selected probes against your target. + ## Prerequisites -- Install and start {{platform_name}} using the [Setup guide](../../get-started/setup.md). +- Install and start NeMo Platform using the [Setup guide](/documentation/get-started). - Install garak in a Python virtual environment so the plugin can shell out to it. By default the plugin invokes `~/.auditor/.venv/bin/python -m garak`. Override the interpreter path with `NEMO_AUDITOR_GARAK_PYTHON` if your install lives elsewhere. -- Configure at least one Inference Gateway provider — this tutorial uses a `build` provider named `build` that routes to NVIDIA-build models, but any chat-completion-compatible provider works. See [Inference Gateway](../targets/inference-gateway.md) for details on the `nmp_uri_spec` block used below. +- Configure at least one Inference Gateway provider — this tutorial uses a `build` provider named `build` that routes to NVIDIA-build models, but any chat-completion-compatible provider works. See [Inference Gateway](/documentation/vulnerability-scanning/targets/inference-gateway) for details on the `nmp_uri_spec` block used below. --- @@ -50,7 +55,7 @@ print(auditor.plugin_status()) # {'plugin': 'auditor', 'status': 'ok', ...} ``` -If `plugin_status()` raises a connection error, the platform is not running or `NMP_BASE_URL` is misconfigured. Refer back to [Setup](../../get-started/setup.md). +If `plugin_status()` raises a connection error, the platform is not running or `NMP_BASE_URL` is misconfigured. Refer back to [Setup](/documentation/get-started). --- @@ -79,7 +84,7 @@ config = auditor.configs.create( print(config.model_dump_json(indent=2)) ``` -For the full set of options on each sub-block, see [Configuration Schema](../configs/schema.md). For probe selection syntax, see [Selecting Probes](../configs/probes.md). +For the full set of options on each sub-block, see [Configuration Schema](/documentation/vulnerability-scanning/configurations/schema). For probe selection syntax, see [Selecting Probes](/documentation/vulnerability-scanning/configurations/selecting-probes). --- @@ -108,7 +113,7 @@ target = auditor.targets.create( print(target.model_dump_json(indent=2)) ``` -The `nmp_uri_spec` sentinel inside `options.nim` tells the plugin to resolve a concrete URI from the Inference Gateway provider at run time. See [Inference Gateway](../targets/inference-gateway.md) for the full conflict rules and resolution behavior. +The `nmp_uri_spec` sentinel inside `options.nim` tells the plugin to resolve a concrete URI from the Inference Gateway provider at run time. See [Inference Gateway](/documentation/vulnerability-scanning/targets/inference-gateway) for the full conflict rules and resolution behavior. --- @@ -216,6 +221,6 @@ Report artifacts are left in place under the scheduler's temporary directory; th You created an `AuditConfig` and `AuditTarget`, ran a single audit locally with `client.auditor.run(...)`, and loaded the resulting reports. -- For the full SDK surface — including async variants — see [SDK Resources](../sdk-resources.md). -- For more probe selection options, see [Selecting Probes](../configs/probes.md). -- For other generator types (OpenAI-compatible endpoints, REST targets, test targets), see [Audit Targets](../targets/index.md). +- For the full SDK surface — including async variants — see [SDK Resources](/documentation/vulnerability-scanning/sdk-resources). +- For more probe selection options, see [Selecting Probes](/documentation/vulnerability-scanning/configurations/selecting-probes). +- For other generator types (OpenAI-compatible endpoints, REST targets, test targets), see [Audit Targets](/documentation/vulnerability-scanning/targets). diff --git a/docs/auth/authentication/index.md b/docs/auth/authentication/index.md deleted file mode 100644 index c0ef413948..0000000000 --- a/docs/auth/authentication/index.md +++ /dev/null @@ -1,56 +0,0 @@ -# Authentication - -{{platform_name}} authenticates requests using **OpenID Connect (OIDC)**. You register an OAuth application in your identity provider, configure {{platform_name}} with the issuer and client ID, and users sign in via the CLI, SDK, or browser. {{platform_name}} validates the JWT on every request and extracts the user's identity for authorization. - -For the quickstart (no IdP), see the [email-based shortcut](../index.md). For the authorization model, see [Authorization Concepts](../concepts.md). - -## Connect Your Identity Provider - -Start here — register an OAuth application in your IdP and configure {{platform_name}}: - -- [OIDC Setup](oidc.md) — Step-by-step: register an app, configure {{platform_name}}, verify login. -- [Azure AD (Entra ID)](providers/azure-ad.md) — Azure-specific walkthrough (app registration, scopes, claim mapping). -- [Generic OIDC Provider](providers/generic.md) — Checklist for any OIDC-compliant IdP. - -## Log In and Make API Calls - -Once your IdP is connected, see [Using Authentication](using-authentication.md) for the full walkthrough: device flow login, SDK and curl examples, token management, and config file reference. - -| Method | Command / Action | Best For | -|--------|-----------------|----------| -| **Device flow** (browser) | `nemo auth login` | Interactive use — opens browser to sign in with your IdP | -| **Password grant** | `nemo auth login --username --password ` | CI/CD pipelines — non-interactive | -| **Direct from IdP** | Use your IdP's token endpoint or workload identity | Custom integrations, service accounts | - -The CLI stores the token and auto-refreshes it before expiry. The SDK reads the stored token from the CLI config automatically — after `nemo auth login`, `NeMoPlatform()` works with no arguments. - -## Discovery Endpoint - -{{platform_name}} exposes an unauthenticated endpoint that clients and the SDK use to discover OIDC settings: - -```text -GET {BASE_URL}/apis/auth/discovery -``` - -Response: - -```json -{ - "auth_enabled": true, - "oidc": { - "issuer": "https://login.microsoftonline.com/{tenant}/v2.0", - "token_endpoint": "https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token", - "device_authorization_endpoint": "https://login.microsoftonline.com/{tenant}/oauth2/v2.0/devicecode", - "client_id": "...", - "default_scopes": "platform:read platform:write openid profile email offline_access", - "scope_prefix": "api://nmp/" - } -} -``` - -The CLI and SDK call this endpoint automatically during `nemo auth login` or when initializing the client. - -## Related - -- [Using Authentication](using-authentication.md) — Log in, make API calls, and manage tokens. -- [Security Model](../security-model.md) — Trust boundaries and the principal model. diff --git a/docs/auth/authentication/index.mdx b/docs/auth/authentication/index.mdx new file mode 100644 index 0000000000..e6c8354f93 --- /dev/null +++ b/docs/auth/authentication/index.mdx @@ -0,0 +1,58 @@ +--- +title: "Authentication" +description: "" +--- +NeMo Platform authenticates requests using **OpenID Connect (OIDC)**. You register an OAuth application in your identity provider, configure NeMo Platform with the issuer and client ID, and users sign in via the CLI, SDK, or browser. NeMo Platform validates the JWT on every request and extracts the user's identity for authorization. + +For the quickstart (no IdP), see the [email-based shortcut](/documentation/access-control/overview). For the authorization model, see [Authorization Concepts](/documentation/access-control/concepts). + +## Connect Your Identity Provider + +Start here — register an OAuth application in your IdP and configure NeMo Platform: + +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — Step-by-step: register an app, configure NeMo Platform, verify login. +- [Azure AD (Entra ID)](/documentation/access-control/authentication/providers/azure-ad-entra-id) — Azure-specific walkthrough (app registration, scopes, claim mapping). +- [Generic OIDC Provider](/documentation/access-control/authentication/providers/generic-oidc) — Checklist for any OIDC-compliant IdP. + +## Log In and Make API Calls + +Once your IdP is connected, see [Using Authentication](/documentation/access-control/authentication/using-authentication) for the full walkthrough: device flow login, SDK and curl examples, token management, and config file reference. + +| Method | Command / Action | Best For | +|--------|-----------------|----------| +| **Device flow** (browser) | `nemo auth login` | Interactive use — opens browser to sign in with your IdP | +| **Password grant** | `nemo auth login --username --password ` | CI/CD pipelines — non-interactive | +| **Direct from IdP** | Use your IdP's token endpoint or workload identity | Custom integrations, service accounts | + +The CLI stores the token and auto-refreshes it before expiry. The SDK reads the stored token from the CLI config automatically — after `nemo auth login`, `NeMoPlatform()` works with no arguments. + +## Discovery Endpoint + +NeMo Platform exposes an unauthenticated endpoint that clients and the SDK use to discover OIDC settings: + +```text +GET {BASE_URL}/apis/auth/discovery +``` + +Response: + +```json +{ + "auth_enabled": true, + "oidc": { + "issuer": "https://login.microsoftonline.com/{tenant}/v2.0", + "token_endpoint": "https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token", + "device_authorization_endpoint": "https://login.microsoftonline.com/{tenant}/oauth2/v2.0/devicecode", + "client_id": "...", + "default_scopes": "platform:read platform:write openid profile email offline_access", + "scope_prefix": "api://nmp/" + } +} +``` + +The CLI and SDK call this endpoint automatically during `nemo auth login` or when initializing the client. + +## Related + +- [Using Authentication](/documentation/access-control/authentication/using-authentication) — Log in, make API calls, and manage tokens. +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and the principal model. diff --git a/docs/auth/authentication/oidc.md b/docs/auth/authentication/oidc.mdx similarity index 65% rename from docs/auth/authentication/oidc.md rename to docs/auth/authentication/oidc.mdx index 886836b61a..5fe7d07610 100644 --- a/docs/auth/authentication/oidc.md +++ b/docs/auth/authentication/oidc.mdx @@ -1,19 +1,21 @@ -# OIDC Setup - -Connect {{platform_name}} to an external OIDC identity provider (IdP) so users sign in with your organization's identity and receive OAuth2 access tokens for API access. +--- +title: "OIDC Setup" +description: "" +--- +Connect NeMo Platform to an external OIDC identity provider (IdP) so users sign in with your organization's identity and receive OAuth2 access tokens for API access. **Prerequisites**: You need an OAuth application registered in your IdP with the client ID, issuer URL, and device flow grant enabled. See the [Minimum IdP Checklist](#minimum-idp-checklist) below. -For login after setup, see [Using Authentication](using-authentication.md). For the full config reference, see [Auth Configuration](../deployment/configuration.md). +For login after setup, see [Using Authentication](/documentation/access-control/authentication/using-authentication). For the full config reference, see [Auth Configuration](/documentation/access-control/deployment/configuration). -## How OIDC Fits in {{platform_name}} +## How OIDC Fits in NeMo Platform When OIDC is enabled: 1. **Platform configuration** points to your IdP (issuer URL, client ID, optional scope prefix and claim names). -2. **Discovery**: Clients and the platform discover token and device-auth endpoints from the IdP's `.well-known/openid-configuration` (or from {{platform_name}}'s aggregated discovery at `{BASE_URL}/apis/auth/discovery`). +2. **Discovery**: Clients and the platform discover token and device-auth endpoints from the IdP's `.well-known/openid-configuration` (or from NeMo Platform's aggregated discovery at `{BASE_URL}/apis/auth/discovery`). 3. **Login**: Users obtain access tokens via **device flow** (browser) or **password grant** (CI) using the CLI. -4. **API calls**: Requests send the access token in the `Authorization: Bearer ` header. {{platform_name}} validates the JWT (signature, issuer, audience, expiry) and extracts the principal and scopes for authorization. +4. **API calls**: Requests send the access token in the `Authorization: Bearer ` header. NeMo Platform validates the JWT (signature, issuer, audience, expiry) and extracts the principal and scopes for authorization. OIDC provides the identity; authorization (workspace roles, scopes) works the same as with any other auth method. @@ -22,9 +24,11 @@ OIDC provides the identity; authorization (workspace roles, scopes) works the sa - **Device authorization flow**: User runs `nemo auth login`; the CLI shows a code and opens the IdP page; after the user signs in and consents, the CLI receives an access token (and optionally a refresh token). Best for interactive use. - **Resource owner password grant**: For non-interactive environments (CI), `nemo auth login --username --password ` exchanges credentials for a token. Your IdP must support this grant type. -!!! warning - Password grant sends user credentials directly to the IdP. It bypasses MFA and is disabled by many production IdPs. Use it only for CI/testing. Prefer device flow for interactive users. + + +Password grant sends user credentials directly to the IdP. It bypasses MFA and is disabled by many production IdPs. Use it only for CI/testing. Prefer device flow for interactive users. + ## Step-by-Step Configuration ### Step 1: Register an OAuth Application @@ -34,10 +38,10 @@ In your IdP (Azure AD, Okta, Keycloak, etc.): 1. Create a new application registration 2. Note the **client ID** 3. Enable **device flow** (device authorization grant) -4. *(Optional)* Create custom API scopes (`platform:read`, `platform:write`) if you want token-level scope restrictions. See [API Scopes](../authorization/api-scopes.md). +4. *(Optional)* Create custom API scopes (`platform:read`, `platform:write`) if you want token-level scope restrictions. See [API Scopes](/documentation/access-control/authorization/api-scopes). 5. If you created custom scopes, grant admin consent for them (if required by your IdP) -### Step 2: Configure {{platform_name}} +### Step 2: Configure NeMo Platform Set the OIDC settings in your platform config under `auth.oidc`: @@ -59,7 +63,7 @@ auth: ### Step 3: Configure Scopes (Optional) -This step is only needed if you created custom API scopes in Step 1. If you skip scopes, {{platform_name}} still enforces RBAC — scopes add an additional token-level restriction on top. See [API Scopes](../authorization/api-scopes.md). +This step is only needed if you created custom API scopes in Step 1. If you skip scopes, NeMo Platform still enforces RBAC — scopes add an additional token-level restriction on top. See [API Scopes](/documentation/access-control/authorization/api-scopes). If you are using scopes, configure the default scopes requested during login: @@ -69,7 +73,7 @@ auth: default_scopes: "openid profile email offline_access platform:read platform:write" ``` -If your IdP prefixes scopes (e.g., Azure AD uses `api://client-id/platform:read`), set `scope_prefix` so {{platform_name}} strips it before authorization: +If your IdP prefixes scopes (e.g., Azure AD uses `api://client-id/platform:read`), set `scope_prefix` so NeMo Platform strips it before authorization: ```yaml auth: @@ -99,7 +103,7 @@ nemo workspaces list |-------|------|-------------| | `enabled` | bool | Enable OIDC token validation. | | `issuer` | string | IdP issuer URL (e.g., `https://login.microsoftonline.com//v2.0`). | -| `client_id` | string | OAuth client ID for {{platform_name}}. | +| `client_id` | string | OAuth client ID for NeMo Platform. | | `additional_issuers` | list | Extra issuer URLs to accept (e.g., Azure AD v1 format). | | `audience` | string | Expected token audience (defaults to `client_id`). | | `email_claim` | string | JWT claim for email (default: `email`). | @@ -122,7 +126,7 @@ Regardless of IdP, ensure: ## Claim mapping -{{platform_name}} maps JWT claims to [trusted identity headers](../security-model.md#trusted-identity-headers) using `email_claim`, `subject_claim`, and `groups_claim` (defaults: `email`, `sub`, `groups`). Override them when your IdP uses different claim names — values must match what you use for workspace members and authorization. +NeMo Platform maps JWT claims to [trusted identity headers](/documentation/access-control/security-model#trusted-identity-headers) using `email_claim`, `subject_claim`, and `groups_claim` (defaults: `email`, `sub`, `groups`). Override them when your IdP uses different claim names — values must match what you use for workspace members and authorization. | Purpose | Config key | Default | Header | |---------|------------|---------|--------| @@ -131,17 +135,19 @@ Regardless of IdP, ensure: | Groups | `groups_claim` | `groups` | `X-NMP-Principal-Groups` | -!!! note - Group values are extracted from the JWT and included in the `X-NMP-Principal-Groups` header and OPA policy data, but **automatic group-to-role mapping is not yet implemented**. Role assignments must be made explicitly via the members API. See [Managing Access](../authorization/managing-access.md). + + +Group values are extracted from the JWT and included in the `X-NMP-Principal-Groups` header and OPA policy data, but **automatic group-to-role mapping is not yet implemented**. Role assignments must be made explicitly via the members API. See [Managing Access](/documentation/access-control/authorization/managing-access). + ## Provider-Specific Notes -- **Azure AD**: See [Azure AD Setup](providers/azure-ad.md) (issuer, `additional_issuers`, device flow, claim overrides). +- **Azure AD**: See [Azure AD Setup](/documentation/access-control/authentication/providers/azure-ad-entra-id) (issuer, `additional_issuers`, device flow, claim overrides). - **Okta**: Choose the correct authorization server (custom vs org). Ensure device flow is enabled. - **Keycloak**: Configure realm, client, and role mappers. Enable device flow in client settings. -- **Other providers**: See [Generic OIDC Provider](providers/generic.md) for a provider-agnostic checklist. +- **Other providers**: See [Generic OIDC Provider](/documentation/access-control/authentication/providers/generic-oidc) for a provider-agnostic checklist. ## Related -- [Using Authentication](using-authentication.md) — Log in, make API calls, and manage tokens. -- [Auth Configuration](../deployment/configuration.md) — Full auth config reference. +- [Using Authentication](/documentation/access-control/authentication/using-authentication) — Log in, make API calls, and manage tokens. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Full auth config reference. diff --git a/docs/auth/authentication/providers/azure-ad.md b/docs/auth/authentication/providers/azure-ad.mdx similarity index 73% rename from docs/auth/authentication/providers/azure-ad.md rename to docs/auth/authentication/providers/azure-ad.mdx index 35df44ebf2..9706643962 100644 --- a/docs/auth/authentication/providers/azure-ad.md +++ b/docs/auth/authentication/providers/azure-ad.mdx @@ -1,13 +1,15 @@ -# Azure AD (Entra ID) Setup +--- +title: "Azure AD (Entra ID) Setup" +description: "" +--- +Complete walkthrough for connecting NeMo Platform to Azure AD (Entra ID), from app registration to first successful login. -Complete walkthrough for connecting {{platform_name}} to Azure AD (Entra ID), from app registration to first successful login. - -**Prerequisites**: Access to Azure Portal with permission to create app registrations. Familiarity with [OIDC Setup](../oidc.md). +**Prerequisites**: Access to Azure Portal with permission to create app registrations. Familiarity with [OIDC Setup](/documentation/access-control/authentication/oidc-setup). ## App Registration 1. In Azure Portal, go to **Azure Active Directory** → **App registrations** → **New registration**. -2. Name the application (e.g., "{{platform_name}}"). +2. Name the application (e.g., "NeMo Platform"). 3. Set **Supported account types** to your tenant configuration. 4. No redirect URI is needed for device flow, but setting one is good practice. 5. Note the **Application (client) ID** and **Directory (tenant) ID**. @@ -23,8 +25,8 @@ Complete walkthrough for connecting {{platform_name}} to Azure AD (Entra ID), fr 1. Go to **Expose an API**. 2. Set the **Application ID URI** (e.g., `api://`). 3. Add scopes: - - `platform:read` — "Read access to {{platform_name}} platform resources" - - `platform:write` — "Write access to {{platform_name}} platform resources" + - `platform:read` — "Read access to NeMo Platform platform resources" + - `platform:write` — "Write access to NeMo Platform platform resources" 4. Go to **API permissions** → **Add a permission** → **My APIs** → select your app → add the scopes. 5. Click **Grant admin consent** for the scopes. @@ -34,7 +36,7 @@ Complete walkthrough for connecting {{platform_name}} to Azure AD (Entra ID), fr 2. Select **Security groups** (or the group types your organization uses). 3. For the **Access token**, select **Group ID**. -## {{platform_name}} Configuration +## NeMo Platform Configuration ```yaml auth: @@ -78,6 +80,6 @@ nemo auth status ## Related -- [OIDC Setup](../oidc.md) — Generic OIDC configuration. -- [OIDC Setup — Claim mapping](../oidc.md#claim-mapping) — JWT claims vs config defaults. -- [Auth Configuration](../../deployment/configuration.md) — Full config reference. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — Generic OIDC configuration. +- [OIDC Setup — Claim mapping](/documentation/access-control/authentication/oidc-setup#claim-mapping) — JWT claims vs config defaults. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Full config reference. diff --git a/docs/auth/authentication/providers/generic.md b/docs/auth/authentication/providers/generic.mdx similarity index 65% rename from docs/auth/authentication/providers/generic.md rename to docs/auth/authentication/providers/generic.mdx index 5a98c34422..d36094b56f 100644 --- a/docs/auth/authentication/providers/generic.md +++ b/docs/auth/authentication/providers/generic.mdx @@ -1,8 +1,10 @@ -# Generic OIDC Provider +--- +title: "Generic OIDC Provider" +description: "" +--- +A checklist for connecting NeMo Platform to any OIDC-compliant identity provider not covered by the [Azure AD](/documentation/access-control/authentication/providers/azure-ad-entra-id) page. -A checklist for connecting {{platform_name}} to any OIDC-compliant identity provider not covered by the [Azure AD](azure-ad.md) page. - -**Prerequisites**: Familiarity with [OIDC Setup](../oidc.md). +**Prerequisites**: Familiarity with [OIDC Setup](/documentation/access-control/authentication/oidc-setup). ## Provider Checklist @@ -46,6 +48,6 @@ auth: ## Related -- [OIDC Setup](../oidc.md) — Full OIDC configuration guide. -- [OIDC Setup — Claim mapping](../oidc.md#claim-mapping) — JWT claims vs config defaults. -- [Auth Configuration](../../deployment/configuration.md) — Full config reference. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — Full OIDC configuration guide. +- [OIDC Setup — Claim mapping](/documentation/access-control/authentication/oidc-setup#claim-mapping) — JWT claims vs config defaults. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Full config reference. diff --git a/docs/auth/authentication/providers/index.md b/docs/auth/authentication/providers/index.md deleted file mode 100644 index 809ed1aa3c..0000000000 --- a/docs/auth/authentication/providers/index.md +++ /dev/null @@ -1,3 +0,0 @@ -# Provider Guides - -Step-by-step setup instructions for specific identity providers. If your provider is not listed here, see the [Generic OIDC Provider](generic.md) checklist. diff --git a/docs/auth/authentication/providers/index.mdx b/docs/auth/authentication/providers/index.mdx new file mode 100644 index 0000000000..17a2f69f23 --- /dev/null +++ b/docs/auth/authentication/providers/index.mdx @@ -0,0 +1,5 @@ +--- +title: "Provider Guides" +description: "" +--- +Step-by-step setup instructions for specific identity providers. If your provider is not listed here, see the [Generic OIDC Provider](/documentation/access-control/authentication/providers/generic-oidc) checklist. diff --git a/docs/auth/authentication/using-authentication.md b/docs/auth/authentication/using-authentication.mdx similarity index 74% rename from docs/auth/authentication/using-authentication.md rename to docs/auth/authentication/using-authentication.mdx index bece0ef611..98db30298a 100644 --- a/docs/auth/authentication/using-authentication.md +++ b/docs/auth/authentication/using-authentication.mdx @@ -1,8 +1,10 @@ -# Using Authentication - +--- +title: "Using Authentication" +description: "" +--- How to log in, make authenticated API calls, and manage tokens with the CLI and SDK. -**Prerequisites**: OIDC must be configured on the platform. See [OIDC Setup](oidc.md). +**Prerequisites**: OIDC must be configured on the platform. See [OIDC Setup](/documentation/access-control/authentication/oidc-setup). ## Log In @@ -42,15 +44,17 @@ By default, the CLI requests the scopes configured in `auth.oidc.default_scopes` nemo auth login --scope "platform:read" ``` -See [API Scopes](../authorization/api-scopes.md) for the full list of available scopes. +See [API Scopes](/documentation/access-control/authorization/api-scopes) for the full list of available scopes. ### Non-Interactive Login (CI/CD) For CI pipelines, use the password grant to obtain a token without a browser: `nemo auth login --username --password ` (or set `NMP_OIDC_USERNAME` / `NMP_OIDC_PASSWORD` environment variables). If your CI system can obtain tokens directly (e.g., workload identity federation), pass the token via `access_token` as shown in [Make API Calls](#make-api-calls) below. -!!! warning - Password grant sends credentials directly to the IdP and **bypasses MFA**. Many production IdPs disable it. Use a dedicated service account with minimal scopes where possible. + + +Password grant sends credentials directly to the IdP and **bypasses MFA**. Many production IdPs disable it. Use a dedicated service account with minimal scopes where possible. + ## Make API Calls ### Python SDK @@ -149,17 +153,19 @@ users: The OIDC token endpoint is **not** stored — it is discovered at runtime from your cluster's `/apis/auth/discovery` endpoint. This keeps the config portable across environments. -!!! warning - **Token storage security** — Access and refresh tokens are stored in plaintext. Protect this file: + + +**Token storage security** — Access and refresh tokens are stored in plaintext. Protect this file: - - **File permissions**: Ensure `0600` (owner read/write only). The CLI sets this by default — verify after manual edits: `chmod 600 ~/.config/nmp/config.yaml`. - - **Shared directories**: Do not store in cloud-synced folders (Dropbox, OneDrive, Google Drive) or shared home directories. - - **Refresh token rotation**: Configure your IdP to rotate refresh tokens on each use. A stolen refresh token becomes invalid after the legitimate client uses it once. - - **Logout when done**: Run `nemo auth logout` on shared or temporary machines. +- **File permissions**: Ensure `0600` (owner read/write only). The CLI sets this by default — verify after manual edits: `chmod 600 ~/.config/nmp/config.yaml`. +- **Shared directories**: Do not store in cloud-synced folders (Dropbox, OneDrive, Google Drive) or shared home directories. +- **Refresh token rotation**: Configure your IdP to rotate refresh tokens on each use. A stolen refresh token becomes invalid after the legitimate client uses it once. +- **Logout when done**: Run `nemo auth logout` on shared or temporary machines. + ## Related -- [OIDC Setup](oidc.md) — Configure your identity provider. -- [API Scopes](../authorization/api-scopes.md) — Scope model and available scopes. -- [Security Model](../security-model.md) — Trust boundaries and the principal model. -- [Troubleshooting](../troubleshooting.md) — Fix common 401/403 errors and login failures. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — Configure your identity provider. +- [API Scopes](/documentation/access-control/authorization/api-scopes) — Scope model and available scopes. +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and the principal model. +- [Troubleshooting](/documentation/access-control/troubleshooting) — Fix common 401/403 errors and login failures. diff --git a/docs/auth/authorization/api-scopes.md b/docs/auth/authorization/api-scopes.mdx similarity index 78% rename from docs/auth/authorization/api-scopes.md rename to docs/auth/authorization/api-scopes.mdx index fd59d99169..d8dfd12272 100644 --- a/docs/auth/authorization/api-scopes.md +++ b/docs/auth/authorization/api-scopes.mdx @@ -1,8 +1,10 @@ -# API Scopes - +--- +title: "API Scopes" +description: "" +--- API scopes are token-level access restrictions that sit on top of role-based permissions. They control which parts of the API a token can access, independent of the user's role. -For role-based permissions, see [Roles & Permissions](roles-and-permissions.md). For the RBAC model, see [Authorization Concepts](../concepts.md). +For role-based permissions, see [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions). For the RBAC model, see [Authorization Concepts](/documentation/access-control/concepts). ## How Scopes Work @@ -35,9 +37,11 @@ Each API has a read and write scope. A token with an API-specific scope can only | Safe Synthesizer | `safe-synthesizer:read` | `safe-synthesizer:write` | `/apis/safe-synthesizer/` | | Secrets | `secrets:read` | `secrets:write` | `/apis/secrets/` | -!!! note - The `entities:read` and `entities:write` scopes are for internal service-to-service usage. The generic entities API is not accessible to regular users — only PlatformAdmin and service principals can access it. Users interact with entities through feature-specific APIs (Models, Evaluation, etc.) which use their own scopes. + + +The `entities:read` and `entities:write` scopes are for internal service-to-service usage. The generic entities API is not accessible to regular users — only PlatformAdmin and service principals can access it. Users interact with entities through feature-specific APIs (Models, Evaluation, etc.) which use their own scopes. + ### Platform Scopes The `platform:*` scopes act as catch-alls that grant access to **all** APIs: @@ -86,7 +90,7 @@ This enables least-privilege tokens: an Editor can create a read-only token (`pl Scopes must be registered in your IdP as custom API scopes. The CLI requests them during the OAuth flow, and the IdP includes granted scopes in the access token. -If your IdP prefixes scopes (e.g., Azure AD uses `api://client-id/platform:read`), configure `scope_prefix` in the {{platform_name}} OIDC settings so the platform strips the prefix before authorization: +If your IdP prefixes scopes (e.g., Azure AD uses `api://client-id/platform:read`), configure `scope_prefix` in the NeMo Platform OIDC settings so the platform strips the prefix before authorization: ```yaml auth: @@ -115,7 +119,7 @@ The PDP distinguishes between OIDC standard scopes and platform scopes: ## Related -- [Roles & Permissions](roles-and-permissions.md) — Role-based permission model. -- [Using Authentication](../authentication/using-authentication.md) — Log in, requesting scopes, and token management. -- [OIDC Setup](../authentication/oidc.md) — Configuring scope prefix and default scopes. -- [Security Model](../security-model.md) — Trust boundaries and two-layer authorization. +- [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions) — Role-based permission model. +- [Using Authentication](/documentation/access-control/authentication/using-authentication) — Log in, requesting scopes, and token management. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — Configuring scope prefix and default scopes. +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and two-layer authorization. diff --git a/docs/auth/authorization/index.md b/docs/auth/authorization/index.md deleted file mode 100644 index 10e43e3444..0000000000 --- a/docs/auth/authorization/index.md +++ /dev/null @@ -1,54 +0,0 @@ -# Authorization - -{{platform_name}} authorization controls what authenticated users can do. Every API request is evaluated against the user's token scopes and role bindings before it is allowed. - -The authorization model has four building blocks: - -1. **Workspaces** — the authorization boundary. All resources belong to a workspace. -2. **Roles** — permission bundles (Viewer, Editor, Admin) granted per workspace. -3. **Role bindings** — the link between a user, a role, and a workspace. -4. **Scopes** — token-level restrictions that limit what the token can do, independent of the user's role. - -```text -Request → PDP → Scope check → Role binding check → Allow / Deny -``` - -For a request to succeed, both the scope check (does the token allow it?) and the role check (does the user have permission?) must pass. - -For the full conceptual background, see [Authorization Concepts](../concepts.md). For the security architecture, see [Security Model](../security-model.md). - -## Key Pages - -
- -- **[Roles & Permissions](roles-and-permissions.md)** - - --- - - Complete permission matrix — what each role can do. - -- **[Managing Access](managing-access.md)** - - --- - - Add users to workspaces, assign roles, manage members. - -- **[API Scopes](api-scopes.md)** - - --- - - Token-level scope model and two-layer authorization. - -- **[Permissions Reference](permissions-reference.md)** - - --- - - Complete list of all permissions with role assignments. - -- **[Policy Engine](policy-engine.md)** - - --- - - OPA / WASM policy engine internals and configuration. - -
\ No newline at end of file diff --git a/docs/auth/authorization/index.mdx b/docs/auth/authorization/index.mdx new file mode 100644 index 0000000000..2ca4ead101 --- /dev/null +++ b/docs/auth/authorization/index.mdx @@ -0,0 +1,52 @@ +--- +title: "Authorization" +description: "" +--- +NeMo Platform authorization controls what authenticated users can do. Every API request is evaluated against the user's token scopes and role bindings before it is allowed. + +The authorization model has four building blocks: + +1. **Workspaces** — the authorization boundary. All resources belong to a workspace. +2. **Roles** — permission bundles (Viewer, Editor, Admin) granted per workspace. +3. **Role bindings** — the link between a user, a role, and a workspace. +4. **Scopes** — token-level restrictions that limit what the token can do, independent of the user's role. + +```text +Request → PDP → Scope check → Role binding check → Allow / Deny +``` + +For a request to succeed, both the scope check (does the token allow it?) and the role check (does the user have permission?) must pass. + +For the full conceptual background, see [Authorization Concepts](/documentation/access-control/concepts). For the security architecture, see [Security Model](/documentation/access-control/security-model). + +## Key Pages + + + + + +Complete permission matrix — what each role can do. + + + + +Add users to workspaces, assign roles, manage members. + + + + +Token-level scope model and two-layer authorization. + + + + +Complete list of all permissions with role assignments. + + + + +OPA / WASM policy engine internals and configuration. + + + + diff --git a/docs/auth/authorization/managing-access.md b/docs/auth/authorization/managing-access.md deleted file mode 100644 index 9c373c7110..0000000000 --- a/docs/auth/authorization/managing-access.md +++ /dev/null @@ -1,369 +0,0 @@ -# Managing Access - -Add users to workspaces, assign roles, and control who can access your resources. For background on the authorization model, refer to [Authorization Concepts](../concepts.md). For what each role can do, refer to [Roles & Permissions](roles-and-permissions.md). - -!!! note - The SDK examples on this page use `NeMoPlatform()` with no arguments so that the client reads your active CLI context (set by `nemo auth login`). That is the right pattern for authorization workflows: you act as your logged-in identity and pass the workspace explicitly in each API call. For the standard local initialization pattern, see [CLI and SDK initialization](../../get-started/setup.md#setup-init). - -## Creating Workspaces - -Workspaces are the primary authorization boundary — all resources belong to a workspace, and access is controlled at the workspace level. When you create a workspace, you automatically become its Admin. - -Create separate workspaces to isolate teams (`ml-research`, `nlp-team`), environments (`dev`, `staging`, `prod`), or projects. For detailed workspace management, refer to [Workspaces](../../get-started/concepts/workspaces.md). - - -=== "CLI" - - ```bash - nemo workspaces create ml-team - - # Set the workspace as your default for subsequent commands - nemo config set --workspace ml-team - ``` - -=== "Python SDK" - - ```python - from nemo_platform import NeMoPlatform - - client = NeMoPlatform() - - workspace = client.workspaces.create( - name="ml-team", description="Machine learning team workspace" - ) - ``` - -## Managing Workspace Members - -Members are users who have been granted access to a workspace. Each member has one of three roles: - -- **Viewer** — Read-only access to all resources -- **Editor** — Can create, modify, and delete resources -- **Admin** — Full control, including managing members - -!!! note - **Role Propagation** - - When you add or change a member, the CLI and SDK wait for the change to propagate to the authorization engine before returning (up to 30 seconds). The member can use their new permissions immediately after the command completes. - -### Add a Member - -Grant someone access to a workspace by adding them as a member with a specific role. The principal is typically an email address that identifies the user in your identity provider. - - -=== "CLI" - - ```bash - nemo workspaces members create --principal alice@example.com --roles Editor --workspace ml-team - ``` - - ```json - { - "principal": "alice@example.com", - "roles": [ - "Editor" - ], - "granted_at": "2026-01-20T10:00:00Z", - "granted_by": "admin@example.com" - } - ``` - -=== "Python SDK" - - ```python - from nemo_platform import NeMoPlatform - - client = NeMoPlatform() - - # Add a member with Editor role - client.workspaces.members.create( - workspace="ml-team", principal="alice@example.com", roles=["Editor"] - ) - - # Add a member with Viewer role (read-only) - client.workspaces.members.create( - workspace="ml-team", principal="bob@example.com", roles=["Viewer"] - ) - - # Add a member with Admin role (full control) - client.workspaces.members.create( - workspace="ml-team", principal="charlie@example.com", roles=["Admin"] - ) - ``` - -### List Members - -View all members of a workspace to audit access or verify permissions. The response includes each member's principal, roles, and when access was granted. - - -=== "CLI" - - ```bash - nemo workspaces members list --workspace ml-team - ``` - - ```json - [ - { - "principal": "alice@example.com", - "roles": [ - "Editor" - ], - "granted_at": "2026-01-20T10:00:00Z", - "granted_by": "admin@example.com" - }, - { - "principal": "bob@example.com", - "roles": [ - "Viewer" - ], - "granted_at": "2026-01-20T10:01:00Z", - "granted_by": "admin@example.com" - }, - { - "principal": "charlie@example.com", - "roles": [ - "Admin" - ], - "granted_at": "2026-01-20T10:02:00Z", - "granted_by": "admin@example.com" - } - ] - ``` - -=== "Python SDK" - - ```python - from nemo_platform import NeMoPlatform - - client = NeMoPlatform() - - members = client.workspaces.members.list(workspace="ml-team") - - for member in members.data: - print(f"{member.principal}: {member.roles}") - ``` - -### Update Member Roles - -Change a member role to adjust their permissions, for example, promoting a Viewer to Editor when they need to create resources. - - -=== "CLI" - - ```bash - nemo workspaces members update bob@example.com --roles Editor --workspace ml-team - ``` - -=== "Python SDK" - - ```python - from nemo_platform import NeMoPlatform - - client = NeMoPlatform() - - # Promote a Viewer to Editor - client.workspaces.members.update( - workspace="ml-team", principal_id="bob@example.com", roles=["Editor"] - ) - ``` - -### Remove a Member - -Revoke a member's access by removing them from the workspace. This removes all their role bindings in the workspace — they will no longer be able to access any resources unless re-added. - - -=== "CLI" - - ```bash - nemo workspaces members delete alice@example.com --workspace ml-team - ``` - -=== "Python SDK" - - ```python - from nemo_platform import NeMoPlatform - - client = NeMoPlatform() - - client.workspaces.members.delete(workspace="ml-team", principal_id="alice@example.com") - ``` - -## Granting Access to All Users - -Use the wildcard principal `*` to grant a role to all authenticated users. This is useful for shared workspaces where you want broad access without adding each user individually. - -Common use cases: - -- **Shared datasets** — Grant Viewer to `*` so everyone can use common training data -- **Team shared space** — Grant Editor to `*` for a workspace where anyone can experiment -- **Published models** — Grant Viewer to `*` for production models that everyone should access - -### Make a Workspace Readable by Everyone - -Grant the Viewer role to `*` so all authenticated users can view resources. - - -=== "CLI" - - ```bash - nemo workspaces members create --principal "*" --roles Viewer --workspace shared-models - ``` - - ```json - { - "principal": "*", - "roles": [ - "Viewer" - ], - "granted_at": "2026-01-20T10:05:00Z", - "granted_by": "admin@example.com" - } - ``` - -=== "Python SDK" - - ```python - from nemo_platform import NeMoPlatform - - client = NeMoPlatform() - - client.workspaces.members.create(workspace="shared-models", principal="*", roles=["Viewer"]) - ``` - -### Make a Workspace Editable by Everyone - -Grant the Editor role to `*` so all authenticated users can create and modify resources. - - -=== "CLI" - - ```bash - nemo workspaces members create --principal "*" --roles Editor --workspace shared-datasets - ``` - -=== "Python SDK" - - ```python - from nemo_platform import NeMoPlatform - - client = NeMoPlatform() - - client.workspaces.members.create(workspace="shared-datasets", principal="*", roles=["Editor"]) - ``` - -### Remove Public Access - -Remove the wildcard binding to restrict the workspace to explicit members only. - - -=== "CLI" - - ```bash - nemo workspaces members delete "*" --workspace ml-team - ``` - -=== "Python SDK" - - ```python - from nemo_platform import NeMoPlatform - - client = NeMoPlatform() - - client.workspaces.members.delete(workspace="ml-team", principal_id="*") - ``` - -!!! note - **Default Workspace Access** - - The platform automatically grants wildcard access to built-in workspaces: - - - `default` workspace: All users have **Editor** access - - `system` workspace: All users have **Viewer** access (read-only) - - This allows users to start working immediately without explicit role assignment. - -## Admin Protection - -Every workspace must have at least one Admin to prevent orphaned workspaces. The platform enforces this rule: - -- You cannot remove the last Admin from a workspace -- You cannot change the last Admin's role to Viewer or Editor - -If you need to leave a workspace where you are the only Admin, add another Admin first: - - -=== "CLI" - - ```bash - # Add another admin first - nemo workspaces members create --principal charlie@example.com --roles Admin --workspace ml-team - - # Now you can remove yourself - nemo workspaces members delete alice@example.com --workspace ml-team - ``` - -=== "Python SDK" - - ```python - from nemo_platform import NeMoPlatform - - client = NeMoPlatform() - - # Add another admin first - client.workspaces.members.create( - workspace="ml-team", principal="charlie@example.com", roles=["Admin"] - ) - - # Now you can remove yourself - client.workspaces.members.delete(workspace="ml-team", principal_id="alice@example.com") - ``` - -## Platform Admin Access - -The **PlatformAdmin** role (set using `admin_email` in config) has full access to all workspaces and bypasses authorization checks. PlatformAdmin is typically used for initial platform setup, creating the first workspaces and granting Admin roles to team leads. After bootstrap, day-to-day access management should use workspace-level members (above). - -For details on configuring the platform admin, refer to [Auth Configuration](../deployment/configuration.md). For the full security implications, refer to [Security Model](../security-model.md). - -## Deleting Workspaces - -Admins can delete workspaces they manage. However, a workspace cannot be deleted if it contains resources (projects, datasets, models, and so on). The API returns a `409 Conflict` error listing which entity types exist: - -```json -{ - "detail": "Cannot delete workspace 'ml-team': workspace contains entities that must be deleted first: project (3), dataset (5)" -} -``` - -Delete all resources in the workspace before deleting the workspace itself: - - -=== "CLI" - - ```bash - # List and delete projects first - nemo projects list --workspace ml-team - nemo projects delete my-project --workspace ml-team - - # Then delete the workspace - nemo workspaces delete ml-team - ``` - -=== "Python SDK" - - ```python - from nemo_platform import NeMoPlatform, ConflictError - - client = NeMoPlatform() - - try: - client.workspaces.delete("ml-team") - except ConflictError as e: - print(f"Cannot delete workspace: {e}") - # Delete resources first, then retry - projects = client.projects.list(workspace="ml-team") - for project in projects.data: - client.projects.delete(project.name, workspace="ml-team") - # Now delete the workspace - client.workspaces.delete("ml-team") - ``` diff --git a/docs/auth/authorization/managing-access.mdx b/docs/auth/authorization/managing-access.mdx new file mode 100644 index 0000000000..1b97ffbf8e --- /dev/null +++ b/docs/auth/authorization/managing-access.mdx @@ -0,0 +1,438 @@ +--- +title: "Managing Access" +description: "" +--- +Add users to workspaces, assign roles, and control who can access your resources. For background on the authorization model, refer to [Authorization Concepts](/documentation/access-control/concepts). For what each role can do, refer to [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions). + + + +The SDK examples on this page use `NeMoPlatform()` with no arguments so that the client reads your active CLI context (set by `nemo auth login`). That is the right pattern for authorization workflows: you act as your logged-in identity and pass the workspace explicitly in each API call. For the standard local initialization pattern, see [CLI and SDK initialization](/documentation/get-started#setup-init). + + +## Creating Workspaces + +Workspaces are the primary authorization boundary — all resources belong to a workspace, and access is controlled at the workspace level. When you create a workspace, you automatically become its Admin. + +Create separate workspaces to isolate teams (`ml-research`, `nlp-team`), environments (`dev`, `staging`, `prod`), or projects. For detailed workspace management, refer to [Workspaces](/documentation/get-started/core-concepts/workspaces). + + + + + + +```bash +nemo workspaces create ml-team + +# Set the workspace as your default for subsequent commands +nemo config set --workspace ml-team +``` + + + + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform() + +workspace = client.workspaces.create( + name="ml-team", description="Machine learning team workspace" +) +``` + + + + +## Managing Workspace Members + +Members are users who have been granted access to a workspace. Each member has one of three roles: + +- **Viewer** — Read-only access to all resources +- **Editor** — Can create, modify, and delete resources +- **Admin** — Full control, including managing members + + + +**Role Propagation** + +When you add or change a member, the CLI and SDK wait for the change to propagate to the authorization engine before returning (up to 30 seconds). The member can use their new permissions immediately after the command completes. + + +### Add a Member + +Grant someone access to a workspace by adding them as a member with a specific role. The principal is typically an email address that identifies the user in your identity provider. + + + + + + +```bash +nemo workspaces members create --principal alice@example.com --roles Editor --workspace ml-team +``` + +```json +{ + "principal": "alice@example.com", + "roles": [ + "Editor" + ], + "granted_at": "2026-01-20T10:00:00Z", + "granted_by": "admin@example.com" +} +``` + + + + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform() + +# Add a member with Editor role +client.workspaces.members.create( + workspace="ml-team", principal="alice@example.com", roles=["Editor"] +) + +# Add a member with Viewer role (read-only) +client.workspaces.members.create( + workspace="ml-team", principal="bob@example.com", roles=["Viewer"] +) + +# Add a member with Admin role (full control) +client.workspaces.members.create( + workspace="ml-team", principal="charlie@example.com", roles=["Admin"] +) +``` + + + + +### List Members + +View all members of a workspace to audit access or verify permissions. The response includes each member's principal, roles, and when access was granted. + + + + + + +```bash +nemo workspaces members list --workspace ml-team +``` + +```json +[ + { + "principal": "alice@example.com", + "roles": [ + "Editor" + ], + "granted_at": "2026-01-20T10:00:00Z", + "granted_by": "admin@example.com" + }, + { + "principal": "bob@example.com", + "roles": [ + "Viewer" + ], + "granted_at": "2026-01-20T10:01:00Z", + "granted_by": "admin@example.com" + }, + { + "principal": "charlie@example.com", + "roles": [ + "Admin" + ], + "granted_at": "2026-01-20T10:02:00Z", + "granted_by": "admin@example.com" + } +] +``` + + + + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform() + +members = client.workspaces.members.list(workspace="ml-team") + +for member in members.data: + print(f"{member.principal}: {member.roles}") +``` + + + + +### Update Member Roles + +Change a member role to adjust their permissions, for example, promoting a Viewer to Editor when they need to create resources. + + + + + + +```bash +nemo workspaces members update bob@example.com --roles Editor --workspace ml-team +``` + + + + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform() + +# Promote a Viewer to Editor +client.workspaces.members.update( + workspace="ml-team", principal_id="bob@example.com", roles=["Editor"] +) +``` + + + + +### Remove a Member + +Revoke a member's access by removing them from the workspace. This removes all their role bindings in the workspace — they will no longer be able to access any resources unless re-added. + + + + + + +```bash +nemo workspaces members delete alice@example.com --workspace ml-team +``` + + + + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform() + +client.workspaces.members.delete(workspace="ml-team", principal_id="alice@example.com") +``` + + + + +## Granting Access to All Users + +Use the wildcard principal `*` to grant a role to all authenticated users. This is useful for shared workspaces where you want broad access without adding each user individually. + +Common use cases: + +- **Shared datasets** — Grant Viewer to `*` so everyone can use common training data +- **Team shared space** — Grant Editor to `*` for a workspace where anyone can experiment +- **Published models** — Grant Viewer to `*` for production models that everyone should access + +### Make a Workspace Readable by Everyone + +Grant the Viewer role to `*` so all authenticated users can view resources. + + + + + + +```bash +nemo workspaces members create --principal "*" --roles Viewer --workspace shared-models +``` + +```json +{ + "principal": "*", + "roles": [ + "Viewer" + ], + "granted_at": "2026-01-20T10:05:00Z", + "granted_by": "admin@example.com" +} +``` + + + + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform() + +client.workspaces.members.create(workspace="shared-models", principal="*", roles=["Viewer"]) +``` + + + + +### Make a Workspace Editable by Everyone + +Grant the Editor role to `*` so all authenticated users can create and modify resources. + + + + + + +```bash +nemo workspaces members create --principal "*" --roles Editor --workspace shared-datasets +``` + + + + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform() + +client.workspaces.members.create(workspace="shared-datasets", principal="*", roles=["Editor"]) +``` + + + + +### Remove Public Access + +Remove the wildcard binding to restrict the workspace to explicit members only. + + + + + + +```bash +nemo workspaces members delete "*" --workspace ml-team +``` + + + + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform() + +client.workspaces.members.delete(workspace="ml-team", principal_id="*") +``` + + + + + + +**Default Workspace Access** + +The platform automatically grants wildcard access to built-in workspaces: + +- `default` workspace: All users have **Editor** access +- `system` workspace: All users have **Viewer** access (read-only) + +This allows users to start working immediately without explicit role assignment. + + +## Admin Protection + +Every workspace must have at least one Admin to prevent orphaned workspaces. The platform enforces this rule: + +- You cannot remove the last Admin from a workspace +- You cannot change the last Admin's role to Viewer or Editor + +If you need to leave a workspace where you are the only Admin, add another Admin first: + + + + + + +```bash +# Add another admin first +nemo workspaces members create --principal charlie@example.com --roles Admin --workspace ml-team + +# Now you can remove yourself +nemo workspaces members delete alice@example.com --workspace ml-team +``` + + + + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform() + +# Add another admin first +client.workspaces.members.create( + workspace="ml-team", principal="charlie@example.com", roles=["Admin"] +) + +# Now you can remove yourself +client.workspaces.members.delete(workspace="ml-team", principal_id="alice@example.com") +``` + + + + +## Platform Admin Access + +The **PlatformAdmin** role (set using `admin_email` in config) has full access to all workspaces and bypasses authorization checks. PlatformAdmin is typically used for initial platform setup, creating the first workspaces and granting Admin roles to team leads. After bootstrap, day-to-day access management should use workspace-level members (above). + +For details on configuring the platform admin, refer to [Auth Configuration](/documentation/access-control/deployment/configuration). For the full security implications, refer to [Security Model](/documentation/access-control/security-model). + +## Deleting Workspaces + +Admins can delete workspaces they manage. However, a workspace cannot be deleted if it contains resources (projects, datasets, models, and so on). The API returns a `409 Conflict` error listing which entity types exist: + +```json +{ + "detail": "Cannot delete workspace 'ml-team': workspace contains entities that must be deleted first: project (3), dataset (5)" +} +``` + +Delete all resources in the workspace before deleting the workspace itself: + + + + + + +```bash +# List and delete projects first +nemo projects list --workspace ml-team +nemo projects delete my-project --workspace ml-team + +# Then delete the workspace +nemo workspaces delete ml-team +``` + + + + +```python +from nemo_platform import NeMoPlatform, ConflictError + +client = NeMoPlatform() + +try: + client.workspaces.delete("ml-team") +except ConflictError as e: + print(f"Cannot delete workspace: {e}") + # Delete resources first, then retry + projects = client.projects.list(workspace="ml-team") + for project in projects.data: + client.projects.delete(project.name, workspace="ml-team") + # Now delete the workspace + client.workspaces.delete("ml-team") +``` + + + + diff --git a/docs/auth/authorization/permissions-reference.md b/docs/auth/authorization/permissions-reference.mdx similarity index 50% rename from docs/auth/authorization/permissions-reference.md rename to docs/auth/authorization/permissions-reference.mdx index 9d28997c2b..e46b8452e4 100644 --- a/docs/auth/authorization/permissions-reference.md +++ b/docs/auth/authorization/permissions-reference.mdx @@ -1,89 +1,94 @@ +--- +title: "Permissions Reference" +description: "" +--- (permissions-reference)= -# Permissions Reference Complete reference of all permissions across the NeMo Platform APIs. Each permission controls access to a specific operation within an individual API. Permissions are assigned to users through [roles](roles-and-permissions.md). For token-level access restrictions, see [API Scopes](api-scopes.md). For the RBAC model, see [Authorization Concepts](../concepts.md). -!!! note - PlatformAdmin is omitted — it bypasses permission checks entirely at the policy level. + +PlatformAdmin is omitted — it bypasses permission checks entirely at the policy level. + + ## Entities API | Permission | Description | Viewer | Editor | Admin | |------------|-------------|:------:|:------:|:-----:| -| `entities.(read \| create \| update \| delete)` | Read, create, update, delete entities | | | | +| `entities.(read \\| create \\| update \\| delete)` | Read, create, update, delete entities | | | | ## Files API | Permission | Description | Viewer | Editor | Admin | |------------|-------------|:------:|:------:|:-----:| -| `filesets.(read \| list)` | Read, list files | ✓ | ✓ | ✓ | -| `filesets.(create \| update \| delete)` | Create, update, delete files | | ✓ | ✓ | +| `filesets.(read \\| list)` | Read, list files | ✓ | ✓ | ✓ | +| `filesets.(create \\| update \\| delete)` | Create, update, delete files | | ✓ | ✓ | ## Guardrails API | Permission | Description | Viewer | Editor | Admin | |------------|-------------|:------:|:------:|:-----:| | `guardrails.checks.exec` | Execute guardrail checks | | ✓ | ✓ | -| `guardrails.configs.(read \| list)` | Read, list guardrails configs | ✓ | ✓ | ✓ | -| `guardrails.configs.(create \| update \| delete)` | Create, update, delete guardrails configs | | ✓ | ✓ | +| `guardrails.configs.(read \\| list)` | Read, list guardrails configs | ✓ | ✓ | ✓ | +| `guardrails.configs.(create \\| update \\| delete)` | Create, update, delete guardrails configs | | ✓ | ✓ | ## IAM API | Permission | Description | Viewer | Editor | Admin | |------------|-------------|:------:|:------:|:-----:| -| `iam.(read \| list \| create \| delete)` | Read, list, create, delete iam | | | ✓ | +| `iam.(read \\| list \\| create \\| delete)` | Read, list, create, delete iam | | | ✓ | | `iam.bundle.read` | Download OPA authorization bundle (external OPA / advanced ops) | | | | ## Inference API | Permission | Description | Viewer | Editor | Admin | |------------|-------------|:------:|:------:|:-----:| -| `inference.deployment-configs.(read \| list)` | Read, list inference deployment-configs | ✓ | ✓ | ✓ | -| `inference.deployment-configs.(create \| delete)` | Create, delete inference deployment-configs | | ✓ | ✓ | -| `inference.deployments.(read \| list)` | Read, list inference deployments | ✓ | ✓ | ✓ | -| `inference.deployments.(create \| update \| delete)` | Create, update, delete inference deployments | | ✓ | ✓ | +| `inference.deployment-configs.(read \\| list)` | Read, list inference deployment-configs | ✓ | ✓ | ✓ | +| `inference.deployment-configs.(create \\| delete)` | Create, delete inference deployment-configs | | ✓ | ✓ | +| `inference.deployments.(read \\| list)` | Read, list inference deployments | ✓ | ✓ | ✓ | +| `inference.deployments.(create \\| update \\| delete)` | Create, update, delete inference deployments | | ✓ | ✓ | | `inference.gateway.model.exec` | Execute model gateway inference | ✓ | ✓ | ✓ | | `inference.gateway.openai.exec` | Execute OpenAI-compatible gateway inference | ✓ | ✓ | ✓ | | `inference.gateway.provider.exec` | Execute provider gateway inference | ✓ | ✓ | ✓ | -| `inference.providers.(read \| list)` | Read, list inference providers | ✓ | ✓ | ✓ | -| `inference.providers.(create \| update \| delete)` | Create, update, delete inference providers | | ✓ | ✓ | -| `inference.virtual-models.(read \| list)` | Read, list inference virtual-models | ✓ | ✓ | ✓ | -| `inference.virtual-models.(create \| update \| delete)` | Create, update, delete inference virtual-models | | ✓ | ✓ | +| `inference.providers.(read \\| list)` | Read, list inference providers | ✓ | ✓ | ✓ | +| `inference.providers.(create \\| update \\| delete)` | Create, update, delete inference providers | | ✓ | ✓ | +| `inference.virtual-models.(read \\| list)` | Read, list inference virtual-models | ✓ | ✓ | ✓ | +| `inference.virtual-models.(create \\| update \\| delete)` | Create, update, delete inference virtual-models | | ✓ | ✓ | ## Intake API | Permission | Description | Viewer | Editor | Admin | |------------|-------------|:------:|:------:|:-----:| -| `intake.annotations.(read \| list)` | Read, list intake annotations | ✓ | ✓ | ✓ | -| `intake.annotations.(create \| delete)` | Create, delete intake annotations | | ✓ | ✓ | -| `intake.evaluator-results.(read \| list)` | Read, list intake evaluator-results | ✓ | ✓ | ✓ | +| `intake.annotations.(read \\| list)` | Read, list intake annotations | ✓ | ✓ | ✓ | +| `intake.annotations.(create \\| delete)` | Create, delete intake annotations | | ✓ | ✓ | +| `intake.evaluator-results.(read \\| list)` | Read, list intake evaluator-results | ✓ | ✓ | ✓ | | `intake.evaluator-results.create` | Create intake evaluator results | | ✓ | ✓ | | `intake.experiment-groups.read` | Read intake experiment groups | ✓ | ✓ | ✓ | -| `intake.experiment-groups.(create \| update \| delete)` | Create, update, delete intake experiment-groups | | ✓ | ✓ | +| `intake.experiment-groups.(create \\| update \\| delete)` | Create, update, delete intake experiment-groups | | ✓ | ✓ | | `intake.experiments.read` | Read intake experiments | ✓ | ✓ | ✓ | -| `intake.experiments.(create \| update \| delete)` | Create, update, delete intake experiments | | ✓ | ✓ | +| `intake.experiments.(create \\| update \\| delete)` | Create, update, delete intake experiments | | ✓ | ✓ | | `intake.ingest.create` | Ingest traces into intake | | ✓ | ✓ | -| `intake.spans.(read \| list)` | Read, list intake spans | ✓ | ✓ | ✓ | +| `intake.spans.(read \\| list)` | Read, list intake spans | ✓ | ✓ | ✓ | | `intake.traces.read` | Read intake traces | ✓ | ✓ | ✓ | ## Jobs API | Permission | Description | Viewer | Editor | Admin | |------------|-------------|:------:|:------:|:-----:| -| `jobs.(read \| list)` | Read, list jobs | ✓ | ✓ | ✓ | -| `jobs.(create \| update \| delete \| cancel)` | Create, update, delete, cancel jobs | | ✓ | ✓ | +| `jobs.(read \\| list)` | Read, list jobs | ✓ | ✓ | ✓ | +| `jobs.(create \\| update \\| delete \\| cancel)` | Create, update, delete, cancel jobs | | ✓ | ✓ | ## Models API | Permission | Description | Viewer | Editor | Admin | |------------|-------------|:------:|:------:|:-----:| -| `models.(read \| list)` | Read, list models | ✓ | ✓ | ✓ | -| `models.(create \| update \| delete)` | Create, update, delete models | | ✓ | ✓ | -| `models.adapters.(read \| list)` | Read, list models adapters | ✓ | ✓ | ✓ | -| `models.adapters.(create \| update \| delete)` | Create, update, delete models adapters | | ✓ | ✓ | +| `models.(read \\| list)` | Read, list models | ✓ | ✓ | ✓ | +| `models.(create \\| update \\| delete)` | Create, update, delete models | | ✓ | ✓ | +| `models.adapters.(read \\| list)` | Read, list models adapters | ✓ | ✓ | ✓ | +| `models.adapters.(create \\| update \\| delete)` | Create, update, delete models adapters | | ✓ | ✓ | | `models.tool-call-plugin.set` | Whether this user can set tool_call_plugin on Models or Deployment Configs *(policy-enforced)* | | | ✓ | | `models.trust-remote-code.set` | Whether this user can set trust_remote_code on Models *(policy-enforced)* | | | ✓ | @@ -97,30 +102,30 @@ For token-level access restrictions, see [API Scopes](api-scopes.md). For the RB | Permission | Description | Viewer | Editor | Admin | |------------|-------------|:------:|:------:|:-----:| -| `projects.(read \| list)` | Read, list projects | ✓ | ✓ | ✓ | -| `projects.(create \| update \| delete)` | Create, update, delete projects | | ✓ | ✓ | +| `projects.(read \\| list)` | Read, list projects | ✓ | ✓ | ✓ | +| `projects.(create \\| update \\| delete)` | Create, update, delete projects | | ✓ | ✓ | ## Safe Synthesizer API | Permission | Description | Viewer | Editor | Admin | |------------|-------------|:------:|:------:|:-----:| -| `safe-synthesizer.jobs.(read \| list \| create \| delete \| cancel)` | Read, list, create, delete, cancel safe synthesizer jobs | | | | +| `safe-synthesizer.jobs.(read \\| list \\| create \\| delete \\| cancel)` | Read, list, create, delete, cancel safe synthesizer jobs | | | | ## Secrets API | Permission | Description | Viewer | Editor | Admin | |------------|-------------|:------:|:------:|:-----:| -| `secrets.(read \| list)` | Read, list secrets | ✓ | ✓ | ✓ | -| `secrets.(create \| update \| delete)` | Create, update, delete secrets | | ✓ | ✓ | -| `secrets.(access \| rotate)` | Access, rotate secrets | | | | +| `secrets.(read \\| list)` | Read, list secrets | ✓ | ✓ | ✓ | +| `secrets.(create \\| update \\| delete)` | Create, update, delete secrets | | ✓ | ✓ | +| `secrets.(access \\| rotate)` | Access, rotate secrets | | | | ## Workspaces API | Permission | Description | Viewer | Editor | Admin | |------------|-------------|:------:|:------:|:-----:| -| `workspaces.(read \| list)` | Read, list workspaces | ✓ | ✓ | ✓ | -| `workspaces.(update \| delete)` | Update, delete workspaces | | ✓ | ✓ | -| `workspaces.members.(list \| create \| update \| delete)` | List, create, update, delete workspaces members | | | ✓ | +| `workspaces.(read \\| list)` | Read, list workspaces | ✓ | ✓ | ✓ | +| `workspaces.(update \\| delete)` | Update, delete workspaces | | ✓ | ✓ | +| `workspaces.members.(list \\| create \\| update \\| delete)` | List, create, update, delete workspaces members | | | ✓ | | `workspaces.members.read` | Read workspace member details | | | | ## Related diff --git a/docs/auth/authorization/policy-engine.md b/docs/auth/authorization/policy-engine.mdx similarity index 82% rename from docs/auth/authorization/policy-engine.md rename to docs/auth/authorization/policy-engine.mdx index a78ab2f4ad..3c8887a288 100644 --- a/docs/auth/authorization/policy-engine.md +++ b/docs/auth/authorization/policy-engine.mdx @@ -1,8 +1,10 @@ -# Policy Engine +--- +title: "Policy Engine" +description: "" +--- +The Policy Decision Point (PDP) evaluates every authorization request in NeMo Platform. It checks role bindings and scopes against the operation's requirements and returns allow or deny. This page covers the PDP internals, configuration, and operational details. -The Policy Decision Point (PDP) evaluates every authorization request in {{platform_name}}. It checks role bindings and scopes against the operation's requirements and returns allow or deny. This page covers the PDP internals, configuration, and operational details. - -For the conceptual overview, see [Authorization Concepts](../concepts.md). For configuration, see [Auth Configuration](../deployment/configuration.md). +For the conceptual overview, see [Authorization Concepts](/documentation/access-control/concepts). For configuration, see [Auth Configuration](/documentation/access-control/deployment/configuration). ## How the PDP Works @@ -84,7 +86,7 @@ Use external OPA when you: - Already run OPA for other services and want a single policy engine - Need gateway-level auth via Envoy `ext_authz` with gRPC -- Want to add custom policy rules alongside {{platform_name}} authorization +- Want to add custom policy rules alongside NeMo Platform authorization - Prefer to manage OPA's lifecycle separately ### Bundle Caching and Propagation Delay @@ -152,7 +154,7 @@ Loaded from the entity store: ## Related -- [Auth Configuration](../deployment/configuration.md) — PDP provider settings and environment variables. -- [Authorization Concepts](../concepts.md) — RBAC model and role propagation. -- [API Scopes](api-scopes.md) — Scope checking in the PDP. -- [Security Model](../security-model.md) — Architecture and trust boundaries. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — PDP provider settings and environment variables. +- [Authorization Concepts](/documentation/access-control/concepts) — RBAC model and role propagation. +- [API Scopes](/documentation/access-control/authorization/api-scopes) — Scope checking in the PDP. +- [Security Model](/documentation/access-control/security-model) — Architecture and trust boundaries. diff --git a/docs/auth/authorization/roles-and-permissions.md b/docs/auth/authorization/roles-and-permissions.mdx similarity index 77% rename from docs/auth/authorization/roles-and-permissions.md rename to docs/auth/authorization/roles-and-permissions.mdx index 5a63a74ce6..5caab130b7 100644 --- a/docs/auth/authorization/roles-and-permissions.md +++ b/docs/auth/authorization/roles-and-permissions.mdx @@ -1,10 +1,12 @@ -# Roles and Permissions - -The authoritative reference for {{platform_name}} roles and their permissions. For background on how RBAC works, see [Authorization Concepts](../concepts.md). For managing workspace members, see [Managing Access](managing-access.md). +--- +title: "Roles and Permissions" +description: "" +--- +The authoritative reference for NeMo Platform roles and their permissions. For background on how RBAC works, see [Authorization Concepts](/documentation/access-control/concepts). For managing workspace members, see [Managing Access](/documentation/access-control/authorization/managing-access). ## Role Descriptions -{{platform_name}} provides four predefined roles, each designed for a specific user persona: +NeMo Platform provides four predefined roles, each designed for a specific user persona: **Viewer** — For stakeholders who need visibility into resources but should not modify them. @@ -28,7 +30,7 @@ The authoritative reference for {{platform_name}} roles and their permissions. F - Grant wildcard access (`*`) to the workspace - Change workspace visibility -**PlatformAdmin** — For platform operators who manage the entire {{platform_name}} deployment. This role bypasses all workspace-level authorization. +**PlatformAdmin** — For platform operators who manage the entire NeMo Platform deployment. This role bypasses all workspace-level authorization. - All Admin permissions across every workspace - Access all workspaces regardless of role bindings @@ -64,9 +66,11 @@ Rows are operations; columns are roles. A checkmark indicates the role has permi | Add / remove members | | | ✓ | ✓ | | Change workspace visibility | | | ✓ | ✓ | -!!! note - All authenticated users can create workspaces. The creator automatically becomes Admin. + + +All authenticated users can create workspaces. The creator automatically becomes Admin. + ### Resource Operations (Models, Datasets, Projects) | Operation | Viewer | Editor | Admin | PlatformAdmin | @@ -115,7 +119,7 @@ Example: ## Default Workspace Bindings -{{platform_name}} automatically provisions wildcard bindings on built-in workspaces: +NeMo Platform automatically provisions wildcard bindings on built-in workspaces: | Workspace | Wildcard Role | Effect | |-----------|--------------|--------| @@ -133,7 +137,7 @@ To leave a workspace where you are the only Admin, add another Admin first. ## Related -- [Authorization Concepts](../concepts.md) — Workspaces, roles, bindings, and the RBAC model. -- [Managing Access](managing-access.md) — Add users, assign roles, manage workspace members. -- [API Scopes](api-scopes.md) — Token-level scope restrictions. -- [Security Model](../security-model.md) — Trust boundaries and authorization layers. +- [Authorization Concepts](/documentation/access-control/concepts) — Workspaces, roles, bindings, and the RBAC model. +- [Managing Access](/documentation/access-control/authorization/managing-access) — Add users, assign roles, manage workspace members. +- [API Scopes](/documentation/access-control/authorization/api-scopes) — Token-level scope restrictions. +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and authorization layers. diff --git a/docs/auth/concepts.md b/docs/auth/concepts.mdx similarity index 80% rename from docs/auth/concepts.md rename to docs/auth/concepts.mdx index 6922e1b2ba..0930bc4b47 100644 --- a/docs/auth/concepts.md +++ b/docs/auth/concepts.mdx @@ -1,8 +1,10 @@ -# Authorization Concepts +--- +title: "Authorization Concepts" +description: "" +--- +NeMo Platform uses **workspace-scoped RBAC**: resources live in workspaces, users get roles (Viewer, Editor, Admin) per workspace, and a policy engine evaluates every request against role bindings and token scopes. -{{platform_name}} uses **workspace-scoped RBAC**: resources live in workspaces, users get roles (Viewer, Editor, Admin) per workspace, and a policy engine evaluates every request against role bindings and token scopes. - -For the security architecture and trust boundaries, see [Security Model](security-model.md). For hands-on setup, see [Managing Access](authorization/managing-access.md). +For the security architecture and trust boundaries, see [Security Model](/documentation/access-control/security-model). For hands-on setup, see [Managing Access](/documentation/access-control/authorization/managing-access). ## Authorization Model Overview @@ -11,7 +13,7 @@ The model has four building blocks: 1. **Workspaces** — containers that own resources (models, datasets, jobs) 2. **Roles** — permission bundles (Viewer, Editor, Admin) granted to users per workspace 3. **Role bindings** — the link between a user, a role, and a workspace -4. **Scopes** — token-level restrictions on top of role permissions (see [API Scopes](authorization/api-scopes.md)) +4. **Scopes** — token-level restrictions on top of role permissions (see [API Scopes](/documentation/access-control/authorization/api-scopes)) ## Workspaces @@ -68,7 +70,7 @@ PlatformAdmin (all operations across all workspaces) Viewer (list, read, run inference) ``` -Custom roles can be defined at deployment time with arbitrary permission sets — they do not need to follow this hierarchy. For the complete permission matrix, see [Roles & Permissions](authorization/roles-and-permissions.md). +Custom roles can be defined at deployment time with arbitrary permission sets — they do not need to follow this hierarchy. For the complete permission matrix, see [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions). ### Predefined Roles @@ -104,7 +106,7 @@ Platform admins can: - Create and delete any workspace - Bypass all authorization checks -The PlatformAdmin role is granted via the `admin_email` config setting. See [Auth Configuration](deployment/configuration.md). +The PlatformAdmin role is granted via the `admin_email` config setting. See [Auth Configuration](/documentation/access-control/deployment/configuration). ## Role Bindings @@ -144,7 +146,7 @@ Creating a workspace is a special operation: ### Default Workspaces -{{platform_name}} automatically provisions role bindings for the wildcard principal `*` on certain workspaces: +NeMo Platform automatically provisions role bindings for the wildcard principal `*` on certain workspaces: - **`default`**: All authenticated users have **Editor** role, allowing everyone to create and manage resources immediately - **`system`**: All authenticated users have **Viewer** role, providing read-only access to system-level resources @@ -162,7 +164,7 @@ Every authorized request is evaluated by the PDP, which checks role bindings and - **Embedded** (default): A WASM-based policy engine built into the auth service. No external dependencies. - **External OPA**: An Open Policy Agent instance (sidecar or standalone service) fetches policy bundles from the auth service. -For technical details on the policy engine, see [Policy Engine](authorization/policy-engine.md). For configuration, see [Auth Configuration](deployment/configuration.md). +For technical details on the policy engine, see [Policy Engine](/documentation/access-control/authorization/policy-engine). For configuration, see [Auth Configuration](/documentation/access-control/deployment/configuration). ### Role Propagation Delay @@ -170,8 +172,8 @@ When you add or remove a member, the change takes effect after the next policy d ## Related -- [Security Model](security-model.md) — Architecture, trust boundaries, and authorization layers. -- [Roles & Permissions](authorization/roles-and-permissions.md) — Complete permission matrix. -- [API Scopes](authorization/api-scopes.md) — Token-level scope restrictions. -- [Managing Access](authorization/managing-access.md) — Add users to workspaces and assign roles. -- [Policy Engine](authorization/policy-engine.md) — OPA/WASM policy engine internals. +- [Security Model](/documentation/access-control/security-model) — Architecture, trust boundaries, and authorization layers. +- [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions) — Complete permission matrix. +- [API Scopes](/documentation/access-control/authorization/api-scopes) — Token-level scope restrictions. +- [Managing Access](/documentation/access-control/authorization/managing-access) — Add users to workspaces and assign roles. +- [Policy Engine](/documentation/access-control/authorization/policy-engine) — OPA/WASM policy engine internals. diff --git a/docs/auth/deployment/configuration.md b/docs/auth/deployment/configuration.mdx similarity index 73% rename from docs/auth/deployment/configuration.md rename to docs/auth/deployment/configuration.mdx index 2cba801685..a3c9e881ed 100644 --- a/docs/auth/deployment/configuration.md +++ b/docs/auth/deployment/configuration.mdx @@ -1,8 +1,10 @@ -# Configuration Reference - +--- +title: "Configuration Reference" +description: "" +--- Complete reference for enabling and configuring platform authorization: the `auth` section in config, Helm values, environment variables, and the choice between embedded and external OPA. -For quickstart setup, see [Authentication and Authorization](../index.md). For OIDC settings, see [OIDC Setup](../authentication/oidc.md). +For quickstart setup, see [Authentication and Authorization](/documentation/access-control/overview). For OIDC settings, see [OIDC Setup](/documentation/access-control/authentication/oidc-setup). ## Enabling Authorization @@ -27,7 +29,7 @@ When `auth.enabled` is `false` (the default), all API requests are allowed witho ### Bootstrap Admin -When authorization is enabled, a platform administrator can be configured. Setting **`admin_email`** gives that identity the **PlatformAdmin** role at platform start. Use it to create the first workspaces and grant roles to other users. After bootstrap, manage access via workspaces and members as described in [Managing Access](../authorization/managing-access.md). +When authorization is enabled, a platform administrator can be configured. Setting **`admin_email`** gives that identity the **PlatformAdmin** role at platform start. Use it to create the first workspaces and grant roles to other users. After bootstrap, manage access via workspaces and members as described in [Managing Access](/documentation/access-control/authorization/managing-access). ```yaml auth: @@ -35,13 +37,13 @@ auth: admin_email: "your-admin@company.com" ``` -For a complete reference of all `auth` fields and their defaults, see the [platform configuration reference](../../set-up/config-reference.md). Auth-related values are found under `platformConfig.auth` in the values file. +For a complete reference of all `auth` fields and their defaults, see the [platform configuration reference](/documentation/reference/configuration-reference). Auth-related values are found under `platformConfig.auth` in the values file. -For OIDC-specific fields (`auth.oidc`), see [OIDC Setup](../authentication/oidc.md). +For OIDC-specific fields (`auth.oidc`), see [OIDC Setup](/documentation/access-control/authentication/oidc-setup). ## Authorization Engine: Embedded vs External OPA -The PDP can run in two modes. For technical details, see [Policy Engine](../authorization/policy-engine.md). +The PDP can run in two modes. For technical details, see [Policy Engine](/documentation/access-control/authorization/policy-engine). ### Embedded (default) @@ -121,8 +123,8 @@ auth: ## Related -- [Authentication and Authorization](../index.md) — Overview, auth methods, and getting started. -- [OIDC Setup](../authentication/oidc.md) — IdP configuration and CLI login. -- [Gateway Integration](gateway.md) — Using a gateway for authorization. -- [Managing Access](../authorization/managing-access.md) — Workspaces and member management. -- [Policy Engine](../authorization/policy-engine.md) — PDP internals and configuration. +- [Authentication and Authorization](/documentation/access-control/overview) — Overview, auth methods, and getting started. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — IdP configuration and CLI login. +- [Gateway Integration](/documentation/access-control/deployment/gateway-integration) — Using a gateway for authorization. +- [Managing Access](/documentation/access-control/authorization/managing-access) — Workspaces and member management. +- [Policy Engine](/documentation/access-control/authorization/policy-engine) — PDP internals and configuration. diff --git a/docs/auth/deployment/credential-propagation.md b/docs/auth/deployment/credential-propagation.md deleted file mode 100644 index 00f289a28a..0000000000 --- a/docs/auth/deployment/credential-propagation.md +++ /dev/null @@ -1,32 +0,0 @@ -# Credential Propagation - -How credentials flow through the system when {{platform_name}} runs jobs or serves inference. - -## Job Credential Propagation - -When a user submits a job (customization, evaluation, data generation), the job runs in a Kubernetes pod that needs to call {{platform_name}} APIs — to download datasets, upload results, and read secrets. The platform propagates the submitting user's identity into the job container so it operates with the user's permissions, not elevated service credentials. - -The flow: - -1. The user submits a job via the API. The platform records the user's principal identity. -2. The platform creates a Kubernetes job with the `NMP_PRINCIPAL` environment variable set to the submitting user's identity. -3. Secrets needed by the job are fetched on behalf of the user (the platform checks that the user has access before injecting them). -4. The job container uses the propagated principal to authenticate API calls back to {{platform_name}}. - -Job containers need to run inside the trust boundary so that their `X-NMP-Principal-*` headers are accepted by downstream services. Network policies and gateway configuration enforce this boundary. For the full architecture, see [Security Model](../security-model.md#job-credential-propagation). - -## Inference Auth Context - -When a model is deployed as an inference endpoint, incoming requests are authenticated at the gateway or service level before reaching the model. The user's auth context is evaluated before the request reaches the model container. - -## Trust Implications - -The `NMP_PRINCIPAL` environment variable is trusted by {{platform_name}} services. If a user can exec into a job pod and read this variable, they have the submitting user's identity for the duration of the job. - -!!! warning - Ensure Kubernetes RBAC prevents unauthorized access to job pods. Network policies should restrict which pods can reach {{platform_name}} internal endpoints. - -## Related - -- [Security Model](../security-model.md) — Trust boundaries and job credential propagation. -- [Auth Configuration](configuration.md) — Platform auth configuration. diff --git a/docs/auth/deployment/credential-propagation.mdx b/docs/auth/deployment/credential-propagation.mdx new file mode 100644 index 0000000000..1b76d1c847 --- /dev/null +++ b/docs/auth/deployment/credential-propagation.mdx @@ -0,0 +1,36 @@ +--- +title: "Credential Propagation" +description: "" +--- +How credentials flow through the system when NeMo Platform runs jobs or serves inference. + +## Job Credential Propagation + +When a user submits a job (customization, evaluation, data generation), the job runs in a Kubernetes pod that needs to call NeMo Platform APIs — to download datasets, upload results, and read secrets. The platform propagates the submitting user's identity into the job container so it operates with the user's permissions, not elevated service credentials. + +The flow: + +1. The user submits a job via the API. The platform records the user's principal identity. +2. The platform creates a Kubernetes job with the `NMP_PRINCIPAL` environment variable set to the submitting user's identity. +3. Secrets needed by the job are fetched on behalf of the user (the platform checks that the user has access before injecting them). +4. The job container uses the propagated principal to authenticate API calls back to NeMo Platform. + +Job containers need to run inside the trust boundary so that their `X-NMP-Principal-*` headers are accepted by downstream services. Network policies and gateway configuration enforce this boundary. For the full architecture, see [Security Model](/documentation/access-control/security-model#job-credential-propagation). + +## Inference Auth Context + +When a model is deployed as an inference endpoint, incoming requests are authenticated at the gateway or service level before reaching the model. The user's auth context is evaluated before the request reaches the model container. + +## Trust Implications + +The `NMP_PRINCIPAL` environment variable is trusted by NeMo Platform services. If a user can exec into a job pod and read this variable, they have the submitting user's identity for the duration of the job. + + + +Ensure Kubernetes RBAC prevents unauthorized access to job pods. Network policies should restrict which pods can reach NeMo Platform internal endpoints. + + +## Related + +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and job credential propagation. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Platform auth configuration. diff --git a/docs/auth/deployment/gateway.md b/docs/auth/deployment/gateway.md deleted file mode 100644 index dc3a2b2ed1..0000000000 --- a/docs/auth/deployment/gateway.md +++ /dev/null @@ -1,143 +0,0 @@ -# Gateway Integration - -In production, a gateway (reverse proxy, ingress controller, or service mesh) often sits in front of the {{platform_name}}. This page explains how authorization works with and without gateway-level auth, what headers the gateway must set, and which paths skip authorization. - -For the security architecture, see [Security Model](../security-model.md). - -## Overview - -{{platform_name}}'s authorization middleware runs **inside each service**. Every request is evaluated there unless auth is disabled or the request matches a bypass path. Optionally, the **gateway** can perform the authorization check (e.g., via Envoy `ext_authz`) and forward the request with a special header so that services **trust the gateway's decision** and do not call the PDP again. That reduces latency and centralizes auth at the edge. - -## Two Authorization Models - -### Service-Level Auth (Default) - -- The gateway forwards requests unchanged (aside from routing/TLS). -- Each service's middleware validates the token (or principal headers) and calls the PDP. -- **No gateway auth configuration required.** Easiest to set up. - -### Gateway-Level Auth - -- The gateway calls the PDP (e.g., via Envoy `ext_authz`) before forwarding. -- If the PDP allows the request, the gateway adds headers and forwards; otherwise it returns 403. -- Services see `x-nmp-authorized: true` and the principal headers, and **skip** their own PDP call. -- **Benefit**: One auth check per request at the edge; lower latency and fewer PDP calls. - -To use gateway-level auth you must configure your gateway to call the {{platform_name}} PDP and set the headers described below on allowed requests. - -!!! warning - **Security Requirement**: Your ingress/gateway **must** strip the following headers from all incoming external requests before forwarding to {{platform_name}}: - - - `X-NMP-Principal-Id`, `X-NMP-Principal-Email`, `X-NMP-Principal-Groups`, `X-NMP-Principal-On-Behalf-Of` - - `X-NMP-Authorized`, `X-NMP-Scopes` - - If external clients can set these headers, they can forge any identity or bypass authorization entirely. The gateway should also block external access to `/internal/*` paths (used for service-to-service communication). - -## Required Headers (Gateway-Level Auth) - -When the gateway has already authorized the request, it must set: - -| Header | Description | -|--------|-------------| -| `X-NMP-Authorized` | Must be `true` so services trust the gateway's decision and skip PDP. | -| `X-NMP-Principal-Id` | Principal identifier (e.g., user ID or email). Required. | -| `X-NMP-Principal-Email` | User email (optional but recommended). | -| `X-NMP-Principal-Groups` | Comma-separated group names (optional). | - -Header names are case-insensitive; services normalize them. - -## Bypass Paths - -The following are **not** subject to authorization checks; they are always allowed: - -- **Health and readiness**: `/health`, `/healthz`, `/ready`, `/readyz`, `/health/live`, `/health/ready`, `/metrics` -- **Discovery**: `/apis/auth/discovery` (for CLI/SDK OIDC discovery) -- **PDP endpoints**: Paths under `/apis/auth/v2/authz/` are restricted to service principals only. The middleware rejects external and regular-user requests automatically. -- **Studio**: Paths under `/studio` (the Studio UI handles its own OIDC login) - -Configure the gateway so these paths are not sent to the PDP (or are always allowed) when using gateway-level auth. - -## Gateway Configuration Examples - -### Envoy `ext_authz` - -??? "Envoy ext_authz filter configuration" - :icon: gear - - Replace placeholder values before applying this configuration. - - ```yaml - http_filters: - - name: envoy.filters.http.jwt_authn - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.jwt_authn.v3.JwtAuthentication - providers: - : - issuer: "" - remote_jwks: - http_uri: - uri: "" - cluster: - timeout: 5s - cache_duration: 600s - claim_to_headers: - - header_name: "X-NMP-Principal-Id" - claim_name: "sub" - - header_name: "X-NMP-Principal-Email" - claim_name: "email" - rules: - - match: - prefix: "/" - requires: - provider_name: "" - - name: envoy.filters.http.ext_authz - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz - grpc_service: - envoy_grpc: - cluster_name: - transport_api_version: V3 - failure_mode_allow: false - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - ``` - -### Header Stripping - -Configure your gateway to remove {{platform_name}} auth headers from incoming external requests. This prevents clients from forging identities. - -??? "Envoy header stripping" - :icon: shield - - Add `request_headers_to_remove` to your route configuration: - - ```yaml - route_config: - virtual_hosts: - - name: nmp_service - domains: ["*"] - request_headers_to_remove: - - "x-nmp-principal-id" - - "x-nmp-principal-email" - - "x-nmp-principal-groups" - - "x-nmp-principal-on-behalf-of" - - "x-nmp-scopes" - - "x-nmp-authorized" - routes: - - match: { prefix: "/" } - route: { cluster: nmp_backend } - ``` - -## Testing Gateway Auth - -After configuring gateway-level auth, verify: - -1. **Headers are set correctly** — Make a request through the gateway and check that `X-NMP-Authorized` and `X-NMP-Principal-Id` are present on the service side. -2. **Headers are stripped from external requests** — Try sending `X-NMP-Authorized: true` from outside; verify it is stripped by the gateway. - -## Related - -- [Auth Configuration](configuration.md) — Enabling auth and PDP provider (embedded vs OPA). -- [Security Model](../security-model.md) — Trust boundaries and gateway trust model. -- [Production Hardening](hardening.md) — Security checklist including gateway requirements. diff --git a/docs/auth/deployment/gateway.mdx b/docs/auth/deployment/gateway.mdx new file mode 100644 index 0000000000..70ab56e0d6 --- /dev/null +++ b/docs/auth/deployment/gateway.mdx @@ -0,0 +1,147 @@ +--- +title: "Gateway Integration" +description: "" +--- +In production, a gateway (reverse proxy, ingress controller, or service mesh) often sits in front of the NeMo Platform. This page explains how authorization works with and without gateway-level auth, what headers the gateway must set, and which paths skip authorization. + +For the security architecture, see [Security Model](/documentation/access-control/security-model). + +## Overview + +NeMo Platform's authorization middleware runs **inside each service**. Every request is evaluated there unless auth is disabled or the request matches a bypass path. Optionally, the **gateway** can perform the authorization check (e.g., via Envoy `ext_authz`) and forward the request with a special header so that services **trust the gateway's decision** and do not call the PDP again. That reduces latency and centralizes auth at the edge. + +## Two Authorization Models + +### Service-Level Auth (Default) + +- The gateway forwards requests unchanged (aside from routing/TLS). +- Each service's middleware validates the token (or principal headers) and calls the PDP. +- **No gateway auth configuration required.** Easiest to set up. + +### Gateway-Level Auth + +- The gateway calls the PDP (e.g., via Envoy `ext_authz`) before forwarding. +- If the PDP allows the request, the gateway adds headers and forwards; otherwise it returns 403. +- Services see `x-nmp-authorized: true` and the principal headers, and **skip** their own PDP call. +- **Benefit**: One auth check per request at the edge; lower latency and fewer PDP calls. + +To use gateway-level auth you must configure your gateway to call the NeMo Platform PDP and set the headers described below on allowed requests. + + + +**Security Requirement**: Your ingress/gateway **must** strip the following headers from all incoming external requests before forwarding to NeMo Platform: + +- `X-NMP-Principal-Id`, `X-NMP-Principal-Email`, `X-NMP-Principal-Groups`, `X-NMP-Principal-On-Behalf-Of` +- `X-NMP-Authorized`, `X-NMP-Scopes` + +If external clients can set these headers, they can forge any identity or bypass authorization entirely. The gateway should also block external access to `/internal/*` paths (used for service-to-service communication). + + +## Required Headers (Gateway-Level Auth) + +When the gateway has already authorized the request, it must set: + +| Header | Description | +|--------|-------------| +| `X-NMP-Authorized` | Must be `true` so services trust the gateway's decision and skip PDP. | +| `X-NMP-Principal-Id` | Principal identifier (e.g., user ID or email). Required. | +| `X-NMP-Principal-Email` | User email (optional but recommended). | +| `X-NMP-Principal-Groups` | Comma-separated group names (optional). | + +Header names are case-insensitive; services normalize them. + +## Bypass Paths + +The following are **not** subject to authorization checks; they are always allowed: + +- **Health and readiness**: `/health`, `/healthz`, `/ready`, `/readyz`, `/health/live`, `/health/ready`, `/metrics` +- **Discovery**: `/apis/auth/discovery` (for CLI/SDK OIDC discovery) +- **PDP endpoints**: Paths under `/apis/auth/v2/authz/` are restricted to service principals only. The middleware rejects external and regular-user requests automatically. +- **Studio**: Paths under `/studio` (the Studio UI handles its own OIDC login) + +Configure the gateway so these paths are not sent to the PDP (or are always allowed) when using gateway-level auth. + +## Gateway Configuration Examples + +### Envoy `ext_authz` + + + +Replace placeholder values before applying this configuration. + +```yaml +http_filters: + - name: envoy.filters.http.jwt_authn + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.jwt_authn.v3.JwtAuthentication + providers: + : + issuer: "" + remote_jwks: + http_uri: + uri: "" + cluster: + timeout: 5s + cache_duration: 600s + claim_to_headers: + - header_name: "X-NMP-Principal-Id" + claim_name: "sub" + - header_name: "X-NMP-Principal-Email" + claim_name: "email" + rules: + - match: + prefix: "/" + requires: + provider_name: "" + - name: envoy.filters.http.ext_authz + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz + grpc_service: + envoy_grpc: + cluster_name: + transport_api_version: V3 + failure_mode_allow: false + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router +``` + + +### Header Stripping + +Configure your gateway to remove NeMo Platform auth headers from incoming external requests. This prevents clients from forging identities. + + + +Add `request_headers_to_remove` to your route configuration: + +```yaml +route_config: + virtual_hosts: + - name: nmp_service + domains: ["*"] + request_headers_to_remove: + - "x-nmp-principal-id" + - "x-nmp-principal-email" + - "x-nmp-principal-groups" + - "x-nmp-principal-on-behalf-of" + - "x-nmp-scopes" + - "x-nmp-authorized" + routes: + - match: { prefix: "/" } + route: { cluster: nmp_backend } +``` + + +## Testing Gateway Auth + +After configuring gateway-level auth, verify: + +1. **Headers are set correctly** — Make a request through the gateway and check that `X-NMP-Authorized` and `X-NMP-Principal-Id` are present on the service side. +2. **Headers are stripped from external requests** — Try sending `X-NMP-Authorized: true` from outside; verify it is stripped by the gateway. + +## Related + +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Enabling auth and PDP provider (embedded vs OPA). +- [Security Model](/documentation/access-control/security-model) — Trust boundaries and gateway trust model. +- [Production Hardening](/documentation/access-control/deployment/production-hardening) — Security checklist including gateway requirements. diff --git a/docs/auth/deployment/hardening.md b/docs/auth/deployment/hardening.mdx similarity index 68% rename from docs/auth/deployment/hardening.md rename to docs/auth/deployment/hardening.mdx index c6a10dc7e9..60a3a49874 100644 --- a/docs/auth/deployment/hardening.md +++ b/docs/auth/deployment/hardening.mdx @@ -1,41 +1,43 @@ -# Production Hardening +--- +title: "Production Hardening" +description: "" +--- +Security checklist for deploying NeMo Platform to production. Work through each section and verify your deployment meets these requirements. -Security checklist for deploying {{platform_name}} to production. Work through each section and verify your deployment meets these requirements. - -For the security architecture, see [Security Model](../security-model.md). For configuration details, see [Auth Configuration](configuration.md). +For the security architecture, see [Security Model](/documentation/access-control/security-model). For configuration details, see [Auth Configuration](/documentation/access-control/deployment/configuration). ## Authentication - [ ] **Enable auth**: Set `auth.enabled: true` in platform config. -- [ ] **Configure OIDC**: Connect a production identity provider. See [OIDC Setup](../authentication/oidc.md). -- [ ] **Disable password grant in production**: Password grant bypasses MFA. If your IdP supports it, disable the resource owner password grant for the {{platform_name}} application registration. Restrict it to dedicated service accounts if CI/CD requires it. +- [ ] **Configure OIDC**: Connect a production identity provider. See [OIDC Setup](/documentation/access-control/authentication/oidc-setup). +- [ ] **Disable password grant in production**: Password grant bypasses MFA. If your IdP supports it, disable the resource owner password grant for the NeMo Platform application registration. Restrict it to dedicated service accounts if CI/CD requires it. - [ ] **Set `admin_email` to a real platform admin**: Use a specific person's email, not a shared mailbox. The PlatformAdmin role bypasses all authorization checks. - [ ] **Verify token lifetime**: Check your IdP's access token and refresh token lifetimes. Shorter access token lifetimes (1 hour or less) reduce the impact of token theft. -- [ ] **Review additional issuers**: If `additional_issuers` is configured, verify all listed issuers are trusted. Each issuer can produce tokens that {{platform_name}} will accept. +- [ ] **Review additional issuers**: If `additional_issuers` is configured, verify all listed issuers are trusted. Each issuer can produce tokens that NeMo Platform will accept. ## Authorization - [ ] **Review default workspace bindings**: The `default` workspace grants Editor to `*` (all authenticated users). If your deployment requires tighter control, restrict this after bootstrap. - [ ] **Restrict PlatformAdmin**: Only one email should have PlatformAdmin. This role bypasses all authorization — treat it like a root account. -- [ ] **Use scoped tokens for CI/CD**: Request `platform:read` only for pipelines that don't need to modify resources. See [API Scopes](../authorization/api-scopes.md). +- [ ] **Use scoped tokens for CI/CD**: Request `platform:read` only for pipelines that don't need to modify resources. See [API Scopes](/documentation/access-control/authorization/api-scopes). - [ ] **Audit workspace access**: Periodically review workspace members (`nemo workspaces members list --workspace `) and remove stale access. - [ ] **Use wildcard bindings carefully**: Only grant `*` (all users) a role when you intentionally want shared access. Prefer Viewer over Editor for public workspaces. ## Gateway and Network -- [ ] **Strip auth headers from external requests**: Configure your ingress/gateway to remove `X-NMP-Principal-Id`, `X-NMP-Principal-Email`, `X-NMP-Principal-Groups`, `X-NMP-Principal-On-Behalf-Of`, `X-NMP-Authorized`, and `X-NMP-Scopes` from all incoming external traffic. See [Gateway Integration](gateway.md). +- [ ] **Strip auth headers from external requests**: Configure your ingress/gateway to remove `X-NMP-Principal-Id`, `X-NMP-Principal-Email`, `X-NMP-Principal-Groups`, `X-NMP-Principal-On-Behalf-Of`, `X-NMP-Authorized`, and `X-NMP-Scopes` from all incoming external traffic. See [Gateway Integration](/documentation/access-control/deployment/gateway-integration). - [ ] **Enable TLS termination**: Terminate TLS at the ingress or load balancer. Tokens in `Authorization` headers are sent in the clear without TLS. -- [ ] **Consider gateway-level auth**: For reduced latency and centralized authorization, configure Envoy `ext_authz` to call the PDP at the edge. See [Gateway Integration](gateway.md). +- [ ] **Consider gateway-level auth**: For reduced latency and centralized authorization, configure Envoy `ext_authz` to call the PDP at the edge. See [Gateway Integration](/documentation/access-control/deployment/gateway-integration). ## Policy Engine -- [ ] **Choose the right PDP provider**: Use embedded (default) for new deployments. Use external OPA if you already run OPA for other services. See [Policy Engine](../authorization/policy-engine.md). +- [ ] **Choose the right PDP provider**: Use embedded (default) for new deployments. Use external OPA if you already run OPA for other services. See [Policy Engine](/documentation/access-control/authorization/policy-engine). - [ ] **Set appropriate refresh interval**: `policy_data_refresh_interval` (embedded) or `bundle_cache_seconds` (external OPA) controls how quickly role changes take effect. Lower values = faster propagation but more load on the entity store. - [ ] **Monitor PDP health**: Ensure the auth service (embedded) or OPA sidecar (external) is healthy. If the PDP is unreachable, the middleware fails closed (returns 503). ## Secrets and Credentials -- [ ] **Verify CLI config file permissions**: Token storage at `~/.config/nmp/config.yaml` should have permissions `0600` (owner read/write only). Avoid storing this file in cloud-synced or shared directories. See [Using Authentication — Config File](../authentication/using-authentication.md#config-file) for full guidance. +- [ ] **Verify CLI config file permissions**: Token storage at `~/.config/nmp/config.yaml` should have permissions `0600` (owner read/write only). Avoid storing this file in cloud-synced or shared directories. See [Using Authentication — Config File](/documentation/access-control/authentication/using-authentication#config-file) for full guidance. - [ ] **Rotate IdP client secrets**: If your OIDC application uses a client secret, rotate it periodically per your organization's policy. - [ ] **Avoid storing tokens in source code or CI configs**: Use environment variables or secret managers for tokens in CI/CD pipelines. @@ -61,7 +63,7 @@ nemo workspaces list ## Related -- [Security Model](../security-model.md) — Architecture, trust boundaries, and authorization layers. -- [Auth Configuration](configuration.md) — Full configuration reference. -- [Gateway Integration](gateway.md) — Gateway auth and header stripping. -- [Roles & Permissions](../authorization/roles-and-permissions.md) — Permission matrix for role auditing. +- [Security Model](/documentation/access-control/security-model) — Architecture, trust boundaries, and authorization layers. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Full configuration reference. +- [Gateway Integration](/documentation/access-control/deployment/gateway-integration) — Gateway auth and header stripping. +- [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions) — Permission matrix for role auditing. diff --git a/docs/auth/index.md b/docs/auth/index.md deleted file mode 100644 index b1dc51c90b..0000000000 --- a/docs/auth/index.md +++ /dev/null @@ -1,154 +0,0 @@ -# Authentication and Authorization - -{{platform_name}} includes a built-in security layer that lets you control who can access your platform and what they can do. When multiple teams or users share a {{platform_name}} deployment, authentication and authorization ensure that each user sees only the workspaces and resources they are permitted to access, and can only perform actions appropriate to their role. - -Access control has two layers: - -- **Authentication** — Prove your identity. {{platform_name}} validates a JWT issued by your OpenID Connect (OIDC) identity provider. -- **Authorization** — Control what you can do. Workspace-scoped RBAC with roles (Viewer, Editor, Admin) and optional API scopes on tokens. - -Both layers are opt-in. When `auth.enabled` is `false` (the default), all requests are allowed without checks. This lets you get started quickly and add security when you are ready for multi-user or production deployments. - -## How Authentication Works - -{{platform_name}} authenticates every request using a JWT from your OIDC identity provider. The token is sent in the `Authorization: Bearer ` header, and {{platform_name}} validates the signature, issuer, audience, and expiry. Refer to [OIDC Setup](authentication/oidc.md) to connect your identity provider. - -How you **obtain** the token depends on your context: - -- **CLI** — Run `nemo auth login` to authenticate using the browser-based device flow. The CLI stores and auto-refreshes the token. Refer to [Using Authentication](authentication/using-authentication.md). -- **SDK** — After `nemo auth login`, the Python SDK automatically reads stored tokens from the CLI config and refreshes them transparently. Refer to [Using Authentication](authentication/using-authentication.md#python-sdk). -- **HTTP** — For raw HTTP calls, fetch a token from your IdP (or from the CLI using `nemo auth token`) and pass it in the `Authorization: Bearer ` header. -- **Studio** — When auth is enabled, Studio automatically redirects you to your IdP to sign in and uses the resulting token for all API calls. - -!!! tip - **Quickstart shortcut** — When running {{platform_name}} quickstart without an OIDC provider, you can use an unsigned JWT: - - `nemo auth login --unsigned-token --email ` - - Quickstart-generated unsigned tokens expire after 24 hours. - - Unsigned JWT login only works for quickstart and must not be used in production. See [Getting Started](#quickstart-development) below. - -## Getting Started - - -### Quickstart / Development - -#### Step 1: Enable Authorization - -Run `nemo quickstart configure` and select **Configure advanced options** → **Yes** for authentication. Provide an admin email — it receives the **PlatformAdmin** role with full platform access. - -```bash -$ nemo quickstart configure -# Select: Configure advanced options → Enable authentication → Yes -# Enter admin email: admin@example.com -``` - -??? "Full quickstart configure output" - :icon: terminal - - ```text - {{platform_name}} Quickstart Configuration - ... - Step 3 of 3: Save Config - Save configuration? - 1. Save configuration - > 2. Configure advanced options - authentication, ports, registry - - • Platform Authorization - Enable auth to require authentication for API requests. - When enabled, you can set an admin email to bootstrap access. - - Enable authentication/authorization? - 1. No - Allow all requests without authentication - > 2. Yes - Require authentication for API access - - ✓ Authorization enabled - - Admin email (grants PlatformAdmin role): admin@example.com - ✓ Admin: admin@example.com - - ℹ All CLI requests will be authenticated as admin@example.com. - To use a different identity: nemo auth login --unsigned-token --email - ... - ✓ Configuration saved successfully! - ``` - -The CLI is automatically configured to authenticate as the admin email for all subsequent commands after setup. To switch identity, run: - -`nemo auth login --unsigned-token --email `. - -#### Step 2: Make Authenticated Calls - -After authorization is enabled, all API requests must include an identity. The CLI and SDK are already configured after Step 1 — they read the admin email from the CLI config automatically. - -=== "CLI" - - ```bash - # CLI is already configured after quickstart configure - # All commands are authenticated as the admin - nemo workspaces list - - # To use a different identity: - nemo auth login --unsigned-token --email other-user@example.com - ``` - -=== "Python SDK" - - ```python - from nemo_platform import NeMoPlatform - - # No arguments needed — the SDK reads base_url, workspace, and credentials - # from the active CLI context (set by `nemo auth login` or `nemo quickstart configure`). - # See: Initializing the CLI and SDK in the quickstart for other init options. - client = NeMoPlatform() - - workspaces = client.workspaces.list() - print(f"Found {len(workspaces.data)} workspaces") - ``` - -### Production / Helm Deployment - -For production or Helm-based deployments, enable auth by setting `platformConfig.auth.enabled: true` in your Helm values and configure the `auth:` section in platform config. Refer to [Auth Configuration](deployment/configuration.md) for the full reference and [OIDC Setup](authentication/oidc.md) to connect your identity provider. - -## Where to Go Next - -
- -- **[Security Model](security-model.md)** - - --- - - Understand how {{platform_name}} authentication and authorization work together — trust boundaries, principal model, and authorization layers. - -- **[Connect an Identity Provider](authentication/oidc.md)** - - --- - - Configure {{platform_name}} to authenticate users using your OIDC identity provider. - -- **[Manage Workspace Access](authorization/managing-access.md)** - - --- - - Add users to workspaces, assign roles, and control who can access your resources. - -- **[Configuration Reference](deployment/configuration.md)** - - --- - - Full configuration reference — enabling auth, PDP provider, OIDC settings, environment variables. - -- **[Harden for Production](deployment/hardening.md)** - - --- - - Security checklist for production deployments — OIDC, gateway headers, scoped tokens, TLS. - -- **[Troubleshooting](troubleshooting.md)** - - --- - - Fix common auth issues — 401/403 errors, login failures, role propagation delays. - -
diff --git a/docs/auth/index.mdx b/docs/auth/index.mdx new file mode 100644 index 0000000000..64a9ee7714 --- /dev/null +++ b/docs/auth/index.mdx @@ -0,0 +1,159 @@ +--- +title: "Authentication and Authorization" +description: "" +--- +NeMo Platform includes a built-in security layer that lets you control who can access your platform and what they can do. When multiple teams or users share a NeMo Platform deployment, authentication and authorization ensure that each user sees only the workspaces and resources they are permitted to access, and can only perform actions appropriate to their role. + +Access control has two layers: + +- **Authentication** — Prove your identity. NeMo Platform validates a JWT issued by your OpenID Connect (OIDC) identity provider. +- **Authorization** — Control what you can do. Workspace-scoped RBAC with roles (Viewer, Editor, Admin) and optional API scopes on tokens. + +Both layers are opt-in. When `auth.enabled` is `false` (the default), all requests are allowed without checks. This lets you get started quickly and add security when you are ready for multi-user or production deployments. + +## How Authentication Works + +NeMo Platform authenticates every request using a JWT from your OIDC identity provider. The token is sent in the `Authorization: Bearer ` header, and NeMo Platform validates the signature, issuer, audience, and expiry. Refer to [OIDC Setup](/documentation/access-control/authentication/oidc-setup) to connect your identity provider. + +How you **obtain** the token depends on your context: + +- **CLI** — Run `nemo auth login` to authenticate using the browser-based device flow. The CLI stores and auto-refreshes the token. Refer to [Using Authentication](/documentation/access-control/authentication/using-authentication). +- **SDK** — After `nemo auth login`, the Python SDK automatically reads stored tokens from the CLI config and refreshes them transparently. Refer to [Using Authentication](/documentation/access-control/authentication/using-authentication#python-sdk). +- **HTTP** — For raw HTTP calls, fetch a token from your IdP (or from the CLI using `nemo auth token`) and pass it in the `Authorization: Bearer ` header. +- **Studio** — When auth is enabled, Studio automatically redirects you to your IdP to sign in and uses the resulting token for all API calls. + + + +**Quickstart shortcut** — When running NeMo Platform quickstart without an OIDC provider, you can use an unsigned JWT: + +`nemo auth login --unsigned-token --email ` + +Quickstart-generated unsigned tokens expire after 24 hours. + +Unsigned JWT login only works for quickstart and must not be used in production. See [Getting Started](#quickstart-development) below. + + +## Getting Started + + +### Quickstart / Development + +#### Step 1: Enable Authorization + +Run `nemo quickstart configure` and select **Configure advanced options** → **Yes** for authentication. Provide an admin email — it receives the **PlatformAdmin** role with full platform access. + +```bash +$ nemo quickstart configure +# Select: Configure advanced options → Enable authentication → Yes +# Enter admin email: admin@example.com +``` + + + +```text +NeMo Platform Quickstart Configuration +... +Step 3 of 3: Save Config +Save configuration? +1. Save configuration +> 2. Configure advanced options - authentication, ports, registry + +• Platform Authorization +Enable auth to require authentication for API requests. +When enabled, you can set an admin email to bootstrap access. + +Enable authentication/authorization? +1. No - Allow all requests without authentication +> 2. Yes - Require authentication for API access + +✓ Authorization enabled + +Admin email (grants PlatformAdmin role): admin@example.com +✓ Admin: admin@example.com + +ℹ All CLI requests will be authenticated as admin@example.com. +To use a different identity: nemo auth login --unsigned-token --email +... +✓ Configuration saved successfully! +``` + + +The CLI is automatically configured to authenticate as the admin email for all subsequent commands after setup. To switch identity, run: + +`nemo auth login --unsigned-token --email `. + +#### Step 2: Make Authenticated Calls + +After authorization is enabled, all API requests must include an identity. The CLI and SDK are already configured after Step 1 — they read the admin email from the CLI config automatically. + + + + + +```bash +# CLI is already configured after quickstart configure +# All commands are authenticated as the admin +nemo workspaces list + +# To use a different identity: +nemo auth login --unsigned-token --email other-user@example.com +``` + + + + +```python +from nemo_platform import NeMoPlatform + +# No arguments needed — the SDK reads base_url, workspace, and credentials +# from the active CLI context (set by `nemo auth login` or `nemo quickstart configure`). +# See: Initializing the CLI and SDK in the quickstart for other init options. +client = NeMoPlatform() + +workspaces = client.workspaces.list() +print(f"Found {len(workspaces.data)} workspaces") +``` + + + + +### Production / Helm Deployment + +For production or Helm-based deployments, enable auth by setting `platformConfig.auth.enabled: true` in your Helm values and configure the `auth:` section in platform config. Refer to [Auth Configuration](/documentation/access-control/deployment/configuration) for the full reference and [OIDC Setup](/documentation/access-control/authentication/oidc-setup) to connect your identity provider. + +## Where to Go Next + + + + + +Understand how NeMo Platform authentication and authorization work together — trust boundaries, principal model, and authorization layers. + + + + +Configure NeMo Platform to authenticate users using your OIDC identity provider. + + + + +Add users to workspaces, assign roles, and control who can access your resources. + + + + +Full configuration reference — enabling auth, PDP provider, OIDC settings, environment variables. + + + + +Security checklist for production deployments — OIDC, gateway headers, scoped tokens, TLS. + + + + +Fix common auth issues — 401/403 errors, login failures, role propagation delays. + + + + diff --git a/docs/auth/security-model.md b/docs/auth/security-model.mdx similarity index 68% rename from docs/auth/security-model.md rename to docs/auth/security-model.mdx index aedb17d658..a16aed2993 100644 --- a/docs/auth/security-model.md +++ b/docs/auth/security-model.mdx @@ -1,8 +1,10 @@ -# Security Model +--- +title: "Security Model" +description: "" +--- +This page describes the security architecture of NeMo Platform authentication and authorization — how requests are authenticated, how access decisions are made, and where trust boundaries lie. It is intended for platform operators and security reviewers evaluating NeMo Platform for production deployment. -This page describes the security architecture of {{platform_name}} authentication and authorization — how requests are authenticated, how access decisions are made, and where trust boundaries lie. It is intended for platform operators and security reviewers evaluating {{platform_name}} for production deployment. - -For hands-on setup, see [Auth Configuration](deployment/configuration.md). For the authorization model, see [Concepts](concepts.md). +For hands-on setup, see [Auth Configuration](/documentation/access-control/deployment/configuration). For the authorization model, see [Concepts](/documentation/access-control/concepts). ## Architecture Overview @@ -55,20 +57,22 @@ The request flow: 3. **PDP** (Policy Decision Point) evaluates authorization — checks the principal's role bindings and token scopes against the operation's requirements. 4. If allowed, the service handles the request. In gateway deployments, the gateway forwards the request with trusted `X-NMP-Principal-*` headers so downstream services skip re-validation. -!!! note - In quickstart deployments without an OIDC provider, the `X-NMP-Principal-*` headers are set directly by the client instead of being derived from a validated JWT. See [Getting Started](index.md). + + +In quickstart deployments without an OIDC provider, the `X-NMP-Principal-*` headers are set directly by the client instead of being derived from a validated JWT. See [Getting Started](/documentation/access-control/overview). + ## Authentication Modes -{{platform_name}} supports two authentication modes: **service-level** (the service validates the JWT) and **gateway-level** (the gateway validates at the edge). +NeMo Platform supports two authentication modes: **service-level** (the service validates the JWT) and **gateway-level** (the gateway validates at the edge). -In both modes, identity arrives as a **JWT** from the client. The JWT is validated exactly once — either by the first {{platform_name}} service or by the gateway. After validation, the authenticated identity (email, subject, groups) is propagated to downstream services via **trusted `X-NMP-Principal-*` headers**. Downstream services accept these headers without re-validating the JWT. +In both modes, identity arrives as a **JWT** from the client. The JWT is validated exactly once — either by the first NeMo Platform service or by the gateway. After validation, the authenticated identity (email, subject, groups) is propagated to downstream services via **trusted `X-NMP-Principal-*` headers**. Downstream services accept these headers without re-validating the JWT. -This "validate once, propagate via headers" design means that **network perimeter security is critical**: anything inside the trust boundary that receives `X-NMP-Principal-*` headers will trust them unconditionally. The gateway must strip these headers from all incoming external requests to prevent clients from forging an identity. See [Gateway Integration](deployment/gateway.md). +This "validate once, propagate via headers" design means that **network perimeter security is critical**: anything inside the trust boundary that receives `X-NMP-Principal-*` headers will trust them unconditionally. The gateway must strip these headers from all incoming external requests to prevent clients from forging an identity. See [Gateway Integration](/documentation/access-control/deployment/gateway-integration). ### Service-Level Authentication -The first {{platform_name}} service that receives the request validates the JWT directly: +The first NeMo Platform service that receives the request validates the JWT directly: 1. Extracts the `Authorization: Bearer ` header 2. Validates the JWT signature, issuer, audience, and expiry against the configured OIDC provider @@ -94,9 +98,11 @@ A **principal** is an authenticated identity — typically a human user identifi When OIDC is enabled, the principal is resolved from JWT claims: the `sub` claim becomes the principal ID (or `oid` for Azure AD), the `email` claim provides the email (or `upn` for Azure AD), and group memberships come from the `groups` claim. These claim names are configurable. In gateway deployments, the gateway performs this extraction and forwards the result in `X-NMP-Principal-*` headers. -!!! note - **Quickstart shortcut** — When running without OIDC (`email-as-API-key` mode), the principal is the raw value of the `X-NMP-Principal-Id` header, without any token validation. This is intended for quick testing only. + + +**Quickstart shortcut** — When running without OIDC (`email-as-API-key` mode), the principal is the raw value of the `X-NMP-Principal-Id` header, without any token validation. This is intended for quick testing only. + ### Trusted Identity Headers The following headers carry the authenticated identity through the system: @@ -120,9 +126,11 @@ A service principal's ID has the form `service:` (e.g., `service:jobs`, `s **Internal endpoints** (`/internal/*`) are also auto-authorized — they bypass PDP checks entirely and are reserved for service-to-service communication. -!!! info - Service principals rely on perimeter security — the gateway strips `X-NMP-Principal-*` headers from incoming requests, so external callers cannot forge a `service:` identity. The gateway must also block external access to `/internal/*` paths. See [Gateway Integration](deployment/gateway.md). + +Service principals rely on perimeter security — the gateway strips `X-NMP-Principal-*` headers from incoming requests, so external callers cannot forge a `service:` identity. The gateway must also block external access to `/internal/*` paths. See [Gateway Integration](/documentation/access-control/deployment/gateway-integration). + + #### On-Behalf-Of Delegation Some operations require a service to act with its own credentials while preserving the original user's identity for attribution and audit purposes. Two key examples: @@ -147,40 +155,42 @@ The downstream service constructs a principal from this object and evaluates the #### Job Credential Propagation -Many {{platform_name}} operations — customization, evaluation, synthetic data generation — run as asynchronous jobs. When a user submits a job, the platform propagates the submitting user's identity into the job container via the `NMP_PRINCIPAL` environment variable. +Many NeMo Platform operations — customization, evaluation, synthetic data generation — run as asynchronous jobs. When a user submits a job, the platform propagates the submitting user's identity into the job container via the `NMP_PRINCIPAL` environment variable. + + -!!! info - Job containers need to run inside the trust boundary — they call {{platform_name}} APIs using the propagated identity headers and are subject to the same authorization checks as any other caller. The key distinction is that jobs act as the *user*, not as a privileged service principal, so they can only access resources the submitting user is permitted to reach. +Job containers need to run inside the trust boundary — they call NeMo Platform APIs using the propagated identity headers and are subject to the same authorization checks as any other caller. The key distinction is that jobs act as the *user*, not as a privileged service principal, so they can only access resources the submitting user is permitted to reach. + ## Authorization: Workspace-Scoped RBAC -{{platform_name}} uses **workspace-scoped Role-Based Access Control (RBAC)**. All resources (models, datasets, jobs, evaluations) belong to exactly one workspace, and access is controlled at the workspace level — not per-resource. +NeMo Platform uses **workspace-scoped Role-Based Access Control (RBAC)**. All resources (models, datasets, jobs, evaluations) belong to exactly one workspace, and access is controlled at the workspace level — not per-resource. - Users are granted roles (Viewer, Editor, Admin) per workspace via **role bindings** - The wildcard principal `*` binds a role for all authenticated users at once - Workspaces are private by default; the creator becomes Admin automatically -On top of RBAC, {{platform_name}} supports **API scopes** as a second authorization layer at the token level. Every authorized request passes through two independent checks: +On top of RBAC, NeMo Platform supports **API scopes** as a second authorization layer at the token level. Every authorized request passes through two independent checks: 1. **Scope check** (token level): The JWT must carry at least one of the scopes required by the endpoint (e.g., `platform:read` or `platform:write`). Scopes limit what the *token* can do. 2. **Permission check** (role level): The principal must have the necessary permissions via role bindings in the workspace. Roles limit what the *user* can do. Both must pass. This enables least-privilege token usage — for example, an Editor can create a read-only token (`platform:read` only) for monitoring scripts. -For details, see [Authorization Concepts](concepts.md), [Roles & Permissions](authorization/roles-and-permissions.md), and [API Scopes](authorization/api-scopes.md). +For details, see [Authorization Concepts](/documentation/access-control/concepts), [Roles & Permissions](/documentation/access-control/authorization/roles-and-permissions), and [API Scopes](/documentation/access-control/authorization/api-scopes). -## What {{platform_name}} Does NOT Do +## What NeMo Platform Does NOT Do The following are explicitly out of scope for the current implementation: - **Multi-tenancy with database isolation.** Workspaces provide logical isolation, not separate databases or schemas per tenant. - **Service mesh mTLS.** Service-to-service encryption and mutual authentication are delegated to the customer's infrastructure (e.g., Istio, Linkerd). -- **Built-in audit logging.** The embedded PDP does not produce a dedicated audit trail. If you need structured decision logs, use [External OPA](authorization/policy-engine.md#external-opa) — OPA's [decision logging](https://www.openpolicyagent.org/docs/latest/management-decision-logs/) can export every authorization decision to your log aggregator. +- **Built-in audit logging.** The embedded PDP does not produce a dedicated audit trail. If you need structured decision logs, use [External OPA](/documentation/access-control/authorization/policy-engine#external-opa) — OPA's [decision logging](https://www.openpolicyagent.org/docs/latest/management-decision-logs/) can export every authorization decision to your log aggregator. - **Custom RBAC at runtime.** Custom roles can be defined at deployment time via YAML configuration, but cannot be created or modified at runtime through the API. ## Related -- [Authorization Concepts](concepts.md) — Workspaces, roles, bindings, and the RBAC model. -- [Auth Configuration](deployment/configuration.md) — Enabling auth, PDP provider, OIDC settings. -- [Gateway Integration](deployment/gateway.md) — Gateway-level auth and header configuration. -- [Production Hardening](deployment/hardening.md) — Security checklist for production deployments. +- [Authorization Concepts](/documentation/access-control/concepts) — Workspaces, roles, bindings, and the RBAC model. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Enabling auth, PDP provider, OIDC settings. +- [Gateway Integration](/documentation/access-control/deployment/gateway-integration) — Gateway-level auth and header configuration. +- [Production Hardening](/documentation/access-control/deployment/production-hardening) — Security checklist for production deployments. diff --git a/docs/auth/troubleshooting.md b/docs/auth/troubleshooting.mdx similarity index 89% rename from docs/auth/troubleshooting.md rename to docs/auth/troubleshooting.mdx index bfd0c5de00..7de925a613 100644 --- a/docs/auth/troubleshooting.md +++ b/docs/auth/troubleshooting.mdx @@ -1,5 +1,7 @@ -# Troubleshooting - +--- +title: "Troubleshooting" +description: "" +--- When something goes wrong with authentication or authorization, start here. Problems are organized by symptom. ## "I Get 401 Unauthorized on Every Request" @@ -81,7 +83,7 @@ The requested scope is not configured or admin consent has not been granted. ### "Device flow not enabled" or "Public client flows not allowed" -Your IdP doesn't have device flow enabled for the {{platform_name}} application. +Your IdP doesn't have device flow enabled for the NeMo Platform application. **Fix**: @@ -91,7 +93,7 @@ Your IdP doesn't have device flow enabled for the {{platform_name}} application. ### Client ID Mismatch -The `client_id` in {{platform_name}} config doesn't match the application in your IdP. +The `client_id` in NeMo Platform config doesn't match the application in your IdP. **Fix**: Verify `auth.oidc.client_id` matches the client ID in your IdP exactly. @@ -160,7 +162,7 @@ If these steps don't resolve the issue: ## Related -- [Auth Configuration](deployment/configuration.md) — Configuration reference. -- [Security Model](security-model.md) — Architecture and trust boundaries. -- [OIDC Setup](authentication/oidc.md) — IdP configuration. -- [Gateway Integration](deployment/gateway.md) — Gateway auth and headers. +- [Auth Configuration](/documentation/access-control/deployment/configuration) — Configuration reference. +- [Security Model](/documentation/access-control/security-model) — Architecture and trust boundaries. +- [OIDC Setup](/documentation/access-control/authentication/oidc-setup) — IdP configuration. +- [Gateway Integration](/documentation/access-control/deployment/gateway-integration) — Gateway auth and headers. diff --git a/docs/cli/configuration.md b/docs/cli/configuration.mdx similarity index 98% rename from docs/cli/configuration.md rename to docs/cli/configuration.mdx index d33defdccb..eb403c55dc 100644 --- a/docs/cli/configuration.md +++ b/docs/cli/configuration.mdx @@ -1,5 +1,7 @@ -# Configuration - +--- +title: "Configuration" +description: "" +--- The NeMo CLI uses a configuration file to store connection settings, credentials, and preferences. This allows you to work with multiple environments and switch between them easily. ## Quick Setup diff --git a/docs/cli/index.md b/docs/cli/index.mdx similarity index 59% rename from docs/cli/index.md rename to docs/cli/index.mdx index d303bc6493..b48fc037ff 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.mdx @@ -1,7 +1,10 @@ +--- +title: "NeMo Platform CLI" +description: "" +--- -# {{platform_name}} CLI -The {{platform_name}} CLI (`nemo`) is a command-line tool for interacting with {{platform_name}}. It provides a unified interface for managing models, running jobs, deploying inference endpoints, and working with a local setup. +The NeMo Platform CLI (`nemo`) is a command-line tool for interacting with NeMo Platform. It provides a unified interface for managing models, running jobs, deploying inference endpoints, and working with a local setup. ## Key Capabilities @@ -13,10 +16,18 @@ The {{platform_name}} CLI (`nemo`) is a command-line tool for interacting with { ## Installation -!!! note "This package downloads and installs additional third-party open source software projects. Review the license terms of these open source projects before use." - If you previously installed the `nemo-microservices` package, uninstall it first to avoid conflicts: + + +This package downloads and installs additional third-party open-source software projects. Review the license terms of these open-source projects before use. + + + +If you previously installed the `nemo-microservices` package, uninstall it first to avoid conflicts: + +```bash +pip uninstall nemo-microservices +``` - pip uninstall nemo-microservices ### Install in a Virtual Environment ```bash @@ -29,7 +40,11 @@ Or with uv: uv pip install nemo-platform[all] ``` -!!! warning "When installed in a virtual environment, the `nemo` command is only available when the environment is activated." + + +When installed in a virtual environment, the `nemo` command is only available when the environment is activated. + + ### Verify Installation @@ -37,7 +52,7 @@ uv pip install nemo-platform[all] nemo --help ``` -If you see `Unknown command: nemo`, see the [troubleshooting](troubleshooting.md) page. +If you see `Unknown command: nemo`, see the [troubleshooting](/documentation/reference/cli-reference/troubleshooting) page. ## Getting Started @@ -56,7 +71,7 @@ nemo auth login nemo workspaces list ``` -This command outputs a list of available workspaces. If the connection fails, see [troubleshooting](troubleshooting.md) for common issues and solutions. +This command outputs a list of available workspaces. If the connection fails, see [troubleshooting](/documentation/reference/cli-reference/troubleshooting) for common issues and solutions. ## Command Structure @@ -66,8 +81,7 @@ The CLI follows a consistent pattern: nemo [GLOBAL OPTIONS] [...] [OPTIONS] ``` ---8<-- "_snippets/cli-summary.md" - + ## Common Workflows ### Exploring Commands @@ -123,7 +137,7 @@ nemo models list -f csv --all-pages --no-truncate --output-columns all > models. ``` ## Next Steps -- [configuration](configuration.md) - Contexts, authentication, environment variables, and shell completion -- [working-with-resources](working-with-resources.md) - Output formats, pagination, and input methods -- [troubleshooting](troubleshooting.md) - Common issues and solutions -- [reference](reference.md) - Full command reference +- [configuration](/documentation/reference/cli-reference/configuration) - Contexts, authentication, environment variables, and shell completion +- [working-with-resources](/documentation/reference/cli-reference/working-with-resources) - Output formats, pagination, and input methods +- [troubleshooting](/documentation/reference/cli-reference/troubleshooting) - Common issues and solutions +- [reference](/documentation/reference/cli-reference/full-cli-reference) - Full command reference diff --git a/docs/cli/reference.md b/docs/cli/reference.mdx similarity index 99% rename from docs/cli/reference.md rename to docs/cli/reference.mdx index bf68ddee37..a1ba332f8a 100644 --- a/docs/cli/reference.md +++ b/docs/cli/reference.mdx @@ -1,5 +1,7 @@ -# Full CLI Reference - +--- +title: "Full CLI Reference" +description: "" +--- Command-line interface for NeMo Platform. **Getting started:** @@ -1163,8 +1165,8 @@ nemo files filesets list [OPTIONS] * `--filter FILTER_JSON`: Use --filter with JSON for complex/nested queries, or --filter. FIELD options for simple fields. Both can be combined, with field options taking precedence. JSON-only fields: - created_at: {gte: str, lte: str} - updated_at: {gte: str, lte: str} + created_at: \{gte: str, lte: str} + updated_at: \{gte: str, lte: str} Filter filesets by name, description, purpose, storage_type, created_at, and updated_at. * `--filter.description` @@ -1544,8 +1546,8 @@ nemo inference deployment-configs list [OPTIONS] * `--filter FILTER_JSON`: Use --filter with JSON for complex/nested queries, or --filter. FIELD options for simple fields. Both can be combined, with field options taking precedence. JSON-only fields: - created_at: {gte: str, lte: str} - updated_at: {gte: str, lte: str} + created_at: \{gte: str, lte: str} + updated_at: \{gte: str, lte: str} Filter deployment configs by workspace, project, model_entity_id, name, description, created_at, and updated_at. * `--filter.description` @@ -1878,8 +1880,8 @@ nemo inference deployments list [OPTIONS] * `--filter FILTER_JSON`: Use --filter with JSON for complex/nested queries, or --filter. FIELD options for simple fields. Both can be combined, with field options taking precedence. JSON-only fields: - created_at: {gte: str, lte: str} - updated_at: {gte: str, lte: str} + created_at: \{gte: str, lte: str} + updated_at: \{gte: str, lte: str} Filter deployments by workspace, project, status, config, model_provider_id, name, status_message, created_at, and updated_at. * `--filter.config` @@ -2973,8 +2975,8 @@ nemo inference providers list [OPTIONS] * `--filter FILTER_JSON`: Use --filter with JSON for complex/nested queries, or --filter. FIELD options for simple fields. Both can be combined, with field options taking precedence. JSON-only fields: - created_at: {gte: str, lte: str} - updated_at: {gte: str, lte: str} + created_at: \{gte: str, lte: str} + updated_at: \{gte: str, lte: str} Filter model providers by workspace, project, status, model_deployment_id, name, description, host_url, created_at, and updated_at. * `--filter.description` @@ -3546,8 +3548,8 @@ nemo jobs list [OPTIONS] * `--filter FILTER_JSON`: Use --filter with JSON for complex/nested queries, or --filter. FIELD options for simple fields. Both can be combined, with field options taking precedence. JSON-only fields: - created_at: {gte: str, lte: str} - updated_at: {gte: str, lte: str} + created_at: \{gte: str, lte: str} + updated_at: \{gte: str, lte: str} Filter jobs by workspace, project, name, status, source, created_at, and updated_at. * `--filter.name` @@ -4172,7 +4174,7 @@ nemo models create [OPTIONS] [NAME] * `--base-model`: Link to another model which is used as a base for the current model * `--custom-fields`: Custom fields for additional metadata (JSON string) * `--description`: Optional description of the model -* `--fileset`: A set of checkpoint files, configs, and other auxiliary info associated with this model - expected format {workspace}/{fileset_name} +* `--fileset`: A set of checkpoint files, configs, and other auxiliary info associated with this model - expected format \{workspace}/\{fileset_name} * `--finetuning-type `: Finetuning types. [possible values: lora_merged, all_weights, last_layer, top_layers, gradual_unfreezing, bias_only, attention_only, lora, qlora, adalora, dora, lora_plus, prompt_tuning, prefix_tuning, p_tuning, p_tuning_v2, soft_prompt, ppo, dpo, cdpo, ipo, orpo, kto, rrhf, grpo] * `--model-providers`: List of ModelProvider workspace/name resource names that provide inference for this Model Entity (can be repeated) * `--ownership`: Ownership information for the model (JSON string) @@ -4246,8 +4248,8 @@ nemo models list [OPTIONS] * `--filter FILTER_JSON`: Use --filter with JSON for complex/nested queries, or --filter. FIELD options for simple fields. Both can be combined, with field options taking precedence. JSON-only fields: - created_at: {gte: str, lte: str} - updated_at: {gte: str, lte: str} + created_at: \{gte: str, lte: str} + updated_at: \{gte: str, lte: str} Filter models by name, project, workspace, base_model, adapters, finetuning_type, prompt, lora_enabled, description, created_at, and updated_at. * `--filter.adapters` @@ -4337,7 +4339,7 @@ nemo models update [OPTIONS] NAME * `--base-model`: Link to another model which is used as a base for the current model * `--custom-fields`: Custom fields for additional metadata (JSON string) * `--description`: Optional description of the model -* `--fileset`: A set of checkpoint files, configs, and other auxiliary info associated with this model - expected format {workspace}/{fileset_name} +* `--fileset`: A set of checkpoint files, configs, and other auxiliary info associated with this model - expected format \{workspace}/\{fileset_name} * `--finetuning-type `: Finetuning types. [possible values: lora_merged, all_weights, last_layer, top_layers, gradual_unfreezing, bias_only, attention_only, lora, qlora, adalora, dora, lora_plus, prompt_tuning, prefix_tuning, p_tuning, p_tuning_v2, soft_prompt, ppo, dpo, cdpo, ipo, orpo, kto, rrhf, grpo] * `--model-providers`: List of ModelProvider workspace/name resource names that provide inference for this Model Entity (can be repeated) * `--ownership`: Ownership information for the model (JSON string) @@ -4407,7 +4409,7 @@ nemo models adapters create [OPTIONS] MODEL_NAME [NAME] **Options:** * `--workspace` -* `--fileset`: Location where adapter files are stored - expected format {workspace}/{fileset_name} +* `--fileset`: Location where adapter files are stored - expected format \{workspace}/\{fileset_name} * `--finetuning-type `: Finetuning types. [possible values: lora_merged, all_weights, last_layer, top_layers, gradual_unfreezing, bias_only, attention_only, lora, qlora, adalora, dora, lora_plus, prompt_tuning, prefix_tuning, p_tuning, p_tuning_v2, soft_prompt, ppo, dpo, cdpo, ipo, orpo, kto, rrhf, grpo] * `--description`: Optional description of the adapter * `--enabled`: Whether to make this adapter available for inference post training @@ -4882,7 +4884,7 @@ nemo workspaces list [OPTIONS] **Options:** * `--filter`: Query filter expression. Supports text and JSON syntaxes: -- Text: name:"value" AND status>500 with operators : ~ > >= < <= IN NOT IN AND OR and negation prefix - +- Text: name:"value" AND status>500 with operators : ~ > >= \< \<= IN NOT IN AND OR and negation prefix - - Object (JSON): `{"name":{"$like":"value"}}` with operators `$eq`, `$like`, `$lt`, `$lte`, `$gt`, `$gte`, `$in`, `$nin`, `$and`, `$or`, `$not` * `--page `: Page number * `--page-size `: Items per page @@ -5221,7 +5223,7 @@ nemo guardrail check [OPTIONS] * `--messages`: A list of messages comprising the conversation so far (JSON string) * `--model`: The model to use for completion. Must be one of the available models. * `--frequency-penalty `: Positive values penalize new tokens based on their existing frequency in the text. -* `--function-call`: Deprecated in favor of tool_choice. 'none' means the model will not call a function and instead generates a message. 'auto' means the model can pick between generating a message or calling a function. Specifying a particular function via {'name': 'my_function'} forces the model to call that function. (JSON string) +* `--function-call`: Deprecated in favor of tool_choice. 'none' means the model will not call a function and instead generates a message. 'auto' means the model can pick between generating a message or calling a function. Specifying a particular function via \{'name': 'my_function'} forces the model to call that function. (JSON string) * `--guardrails`: Guardrails specific options for the request. (JSON string) * `--ignore-eos`: Ignore the eos when running * `--logit-bias`: Modify the likelihood of specified tokens appearing in the completion. Maps token IDs (as strings) to bias values from -100 to 100. (JSON string) @@ -5231,7 +5233,7 @@ nemo guardrail check [OPTIONS] * `--n `: How many chat completion choices to generate for each input message. * `--presence-penalty `: Positive values penalize new tokens based on whether they appear in the text so far. * `--reasoning-effort`: Constrains effort on reasoning for reasoning models. Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning in a response. -* `--response-format`: Format of the response. Use {'type': 'json_object'} for JSON mode or {'type': 'json_schema', 'json_schema': {...}} for structured outputs. (JSON string) +* `--response-format`: Format of the response. Use \{'type': 'json_object'} for JSON mode or \{'type': 'json_schema', 'json_schema': \{...}} for structured outputs. (JSON string) * `--seed `: If specified, attempts to sample deterministically. * `--stop`: Up to 4 sequences where the API will stop generating further tokens. (JSON string) * `--stream`: If set, partial message deltas will be sent, like in ChatGPT. @@ -5370,8 +5372,8 @@ nemo guardrail configs list [OPTIONS] * `--filter FILTER_JSON`: Use --filter with JSON for complex/nested queries, or --filter. FIELD options for simple fields. Both can be combined, with field options taking precedence. JSON-only fields: - created_at: {gte: str, lte: str} - updated_at: {gte: str, lte: str} + created_at: \{gte: str, lte: str} + updated_at: \{gte: str, lte: str} Filter guardrail configs by name, description, project, created_at, and updated_at. * `--filter.description` diff --git a/docs/cli/troubleshooting.md b/docs/cli/troubleshooting.mdx similarity index 86% rename from docs/cli/troubleshooting.md rename to docs/cli/troubleshooting.mdx index 29976d9bb9..9c5550e029 100644 --- a/docs/cli/troubleshooting.md +++ b/docs/cli/troubleshooting.mdx @@ -1,5 +1,7 @@ -# Troubleshooting - +--- +title: "Troubleshooting" +description: "" +--- This page covers common issues and how to resolve them. ## Command Not Found @@ -98,5 +100,5 @@ nemo config view ## Getting Help - Use `--help` on any command for usage information -- See the [reference](reference.md) for complete command documentation -- See [configuration](configuration.md) for detailed configuration options +- See the [reference](/documentation/reference/cli-reference/full-cli-reference) for complete command documentation +- See [configuration](/documentation/reference/cli-reference/configuration) for detailed configuration options diff --git a/docs/cli/working-with-resources.md b/docs/cli/working-with-resources.mdx similarity index 98% rename from docs/cli/working-with-resources.md rename to docs/cli/working-with-resources.mdx index 8ca8004f84..e1b1952770 100644 --- a/docs/cli/working-with-resources.md +++ b/docs/cli/working-with-resources.mdx @@ -1,5 +1,7 @@ -# Working with Resources - +--- +title: "Working with Resources" +description: "" +--- This page covers how to list, view, and create resources with the CLI. ## Output Formats diff --git a/docs/contributing/skills-spec.md b/docs/contributing/skills-spec.mdx similarity index 99% rename from docs/contributing/skills-spec.md rename to docs/contributing/skills-spec.mdx index 9f3df9f8db..fd77b55180 100644 --- a/docs/contributing/skills-spec.md +++ b/docs/contributing/skills-spec.mdx @@ -1,5 +1,7 @@ -# NeMo Platform Skills Spec - +--- +title: "NeMo Platform Skills Spec" +description: "" +--- **Status:** Draft. This document defines the conventions for skills shipped with NeMo Platform: how they're structured, what their frontmatter must contain, how they get tested, and where they live in the repo. diff --git a/docs/customizer/about.md b/docs/customizer/about.mdx similarity index 85% rename from docs/customizer/about.md rename to docs/customizer/about.mdx index 05ebdca962..b5952b5e1b 100644 --- a/docs/customizer/about.md +++ b/docs/customizer/about.mdx @@ -1,7 +1,10 @@ +--- +title: "Customization Concepts" +description: "" +--- -# Customization Concepts -This page provides an overview of the customization concepts for the {{platform_name}}. +This page provides an overview of the customization concepts for the NeMo Platform. ## Supervised Fine-Tuning @@ -58,7 +61,7 @@ flowchart TD style M fill:#90EE90 ``` -{{ncm_short_name}} supports supervised fine-tuning (SFT) with the PEFT method of Low-Rank Adaptation (LoRA). +NeMo Customizer supports supervised fine-tuning (SFT) with the PEFT method of Low-Rank Adaptation (LoRA). Fine-tuning with LoRA is the recommended starting point for most use cases. @@ -74,7 +77,7 @@ In comparison to full fine-tuning of a base model, Low-Rank Adaptation (LoRA) ha LoRA high level architecture: ```mermaid -{% raw %} + --- caption: Low Rank Adaptation (LoRA) --- @@ -92,7 +95,7 @@ flowchart LR class Input,API input class FW,RD,CM neural class Add add -{% endraw %} + ``` During LoRA training: @@ -117,7 +120,7 @@ Additional Resources: ## Training with Your Own Data -Use {{ncm_short_name}} to train custom models on your own data. The workflow can be carried out as follows: +Use NeMo Customizer to train custom models on your own data. The workflow can be carried out as follows: - Upload a dataset - Train a custom model - Perform inference with the trained model @@ -126,23 +129,27 @@ Use {{ncm_short_name}} to train custom models on your own data. The workflow can Long samples in the dataset are truncated during training if the total token length exceeds the context supported by the model. -!!! note - Refer to the model's documentation to see the maximum supported sequence length. + +Refer to the model's documentation to see the maximum supported sequence length. + + | Dataset Type | Token Counting | Length Management | |--------------|----------------|-------------------| -| Prompt Completion | • Total = prompt + completion tokens | • Truncates prompt tokens to fit limits
• Filters out entries that still exceed maximum length | -| Conversational | • Total = conversation turns + template tokens
• Templates are model-specific | • Truncates tokens beyond maximum limit
• Preserves template formatting | +| Prompt Completion | • Total = prompt + completion tokens | • Truncates prompt tokens to fit limits
• Filters out entries that still exceed maximum length | +| Conversational | • Total = conversation turns + template tokens
• Templates are model-specific | • Truncates tokens beyond maximum limit
• Preserves template formatting | ### Prompt Completion Datasets Below are some examples of how you might format your dataset to perform a handful of different tasks. -!!! note - When testing models trained with prompt/completion datasets, use the `/v1/completions` endpoint instead of `/v1/chat/completions`. + + +When testing models trained with prompt/completion datasets, use the `/v1/completions` endpoint instead of `/v1/chat/completions`. - For details, refer to the [Dataset Formatting tutorial](tutorials/format-training-dataset.md#format-a-prompt-completion-dataset). +For details, refer to the [Dataset Formatting tutorial](/documentation/fine-tune-models/tutorials/format-training-dataset#format-a-prompt-completion-dataset). + #### Document Classification Classify a document into predefined categories. Each training example consists of: @@ -190,7 +197,7 @@ completion: "" Most of the models support Instruction Templates for training, the expected dataset conforms with the standard [OpenAI messages format](https://platform.openai.com/docs/guides/fine-tuning#multi-turn-chat-examples). Additionally, some models support tool calling which have additional optional parameters of `tools` at the top level of each entry and `tool_calls` per message. -For more information refer to our [in-depth instructions](tutorials/format-training-dataset.md#format-a-conversation-dataset). +For more information refer to our [in-depth instructions](/documentation/fine-tune-models/tutorials/format-training-dataset#format-a-conversation-dataset). ## Hyperparameters @@ -208,22 +215,24 @@ Hyperparameters are configuration settings used to control the training process. ## Parallelism -{{platform_name}} Customizer supports various distributed training parallelization methods, which can be mixed together. +NeMo Platform Customizer supports various distributed training parallelization methods, which can be mixed together. ### Tensor Parallelism [Tensor Parallelism](https://docs.nvidia.com/nemo-framework/user-guide/latest/nemotoolkit/features/parallelisms.html#tensor-parallelism) (TP) distributes the parameter tensor of an individual layer across GPUs. In addition to reducing model state memory usage, it also saves activation memory as the per-GPU tensor sizes shrink. The tradeoff is increased CPU overhead. -TP can be configured via `parallelism.tensor_parallel_size` in the [training configuration](manage-customization-jobs/hyperparameters.md). +TP can be configured via `parallelism.tensor_parallel_size` in the [training configuration](/documentation/customizer-reference/manage-jobs/training-configuration). -!!! note - As of release 25.10.0, AutoModel engines including Phi-4, Qwen, and Gemma support tensor parallelism greater than 1 through the multi-GPU LoRA patch. Previous releases only supported `TP=1` for these models. + +As of release 25.10.0, AutoModel engines including Phi-4, Qwen, and Gemma support tensor parallelism greater than 1 through the multi-GPU LoRA patch. Previous releases only supported `TP=1` for these models. + + ### Pipeline Parallelism [Pipeline Parallelism](https://docs.nvidia.com/nemo-framework/user-guide/latest/nemotoolkit/features/parallelisms.html#pipeline-parallelism) (PP) distributes the layers of a neural network across GPUs. The GPUs then process the different layers sequentially. -PP can be configured via `parallelism.pipeline_parallel_size` in the [training configuration](manage-customization-jobs/hyperparameters.md). +PP can be configured via `parallelism.pipeline_parallel_size` in the [training configuration](/documentation/customizer-reference/manage-jobs/training-configuration). #### Configuration @@ -241,7 +250,7 @@ PP can be configured via `parallelism.pipeline_parallel_size` in the [training c [Sequence Parallelism](https://docs.nvidia.com/nemo-framework/user-guide/latest/nemotoolkit/features/parallelisms.html#sequence-parallelism) (SP) extends tensor-level model parallelism by distributing computing load and activation memory across multiple GPUs along the sequence dimension of transformer layers. This method is particularly useful when training on the datasets with longer sequences. It also benefits portions of the layer that have previously not been parallelized, enhancing overall model performance and efficiency. -Sequence Parallelism can be enabled/disabled using `parallelism.sequence_parallel` in the [training configuration](manage-customization-jobs/hyperparameters.md). +Sequence Parallelism can be enabled/disabled using `parallelism.sequence_parallel` in the [training configuration](/documentation/customizer-reference/manage-jobs/training-configuration). ## Sequence Packing @@ -265,9 +274,11 @@ When enabled, the `batch_size` and number of training steps update so that each - Chat prompt templates do not have support for sequence packing. -!!! note - If `training.sequence_packing` is enabled when using a model that does not support sequence packing, the fine-tuning will proceed _without_ sequence packing and a warning will be returned in the API response. + + +If `training.sequence_packing` is enabled when using a model that does not support sequence packing, the fine-tuning will proceed _without_ sequence packing and a warning will be returned in the API response. + ### Example of using in the API Example of creating a customization job with sequence packing enabled: diff --git a/docs/customizer/index.md b/docs/customizer/index.md deleted file mode 100644 index c55f165660..0000000000 --- a/docs/customizer/index.md +++ /dev/null @@ -1,178 +0,0 @@ -# About Fine-Tuning - - -Learn how to fine-tune models by making requests to NVIDIA NeMo Customizer through the API. Fine-tuned models you have created can be deployed using NVIDIA NIMs. - -## Fine-Tuning Workflow - - -At a high level, the fine-tuning workflow consists of the following steps: - -1. [Create a Model Entity](manage-model-entities/index.md) pointing to your base model checkpoint (stored as a FileSet). -1. Format a compatible [dataset](tutorials/format-training-dataset.md). -1. [Create a customization job](manage-customization-jobs/index.md) referencing the Model Entity. -1. Monitor the job until it completes. -1. The customization job automatically creates either: - - **LoRA jobs**: An adapter attached to the original Model Entity - - **Full fine-tuning jobs**: A new Model Entity with the customized weights -1. [Deploy the model](../run-inference/about.md) using the Deployment Management Service. -1. Move on to [Evaluate the output model](../evaluator/index.md). - ---- - -## Model Catalog - -Explore the model families and sizes supported by NVIDIA NeMo Customizer. - - -
- -- **[Llama Models](models/llama.md)** - - --- - - View the available Llama models in the model catalog. - -- **[Llama Nemotron Models](models/llama-nemotron.md)** - - --- - - View the available Llama Nemotron models from NVIDIA, including Nano and Super variants for efficient and advanced instruction tuning. - -- **[Phi Models](models/phi.md)** - - --- - - View the available Phi models from Microsoft, designed for strong reasoning capabilities with efficient deployment. - -- **[GPT-OSS Models](models/gpt-oss.md)** - - --- - - View the available GPT-OSS models supported for Full SFT customization. - -- **[Embedding Models](models/embedding.md)** - - --- - - View the available embedding models for question-answering and retrieval tasks. - -
- -## Task Guides - -Perform common fine-tuning tasks. - -
- -- **[Manage Customization Jobs](manage-customization-jobs/index.md)** - - --- - - Create, list, view, and cancel customization jobs. - -- **[Manage Model Entities](manage-model-entities/index.md)** - - --- - - Create FileSets and Model Entities to prepare base models for customization. - -- **[Manage Datasets](../get-started/concepts/manage-files.md)** - - --- - - Upload and manage datasets for training. - -
- ---- - -## Tutorials - -Follow these tutorials to learn how to accomplish common fine-tuning tasks. - -
- -- **[Format Training Datasets](tutorials/format-training-dataset.md)** - - --- - - Learn how to format datasets for different model types. - - datasets chat-models completion-models - -- **[Start a LoRA Customization Job](tutorials/lora-customization-job.ipynb)** - - --- - - Learn how to start a LoRA customization job using a custom dataset. - - nemo-customizer - -- **[Start a Full SFT Customization Job](tutorials/sft-customization-job.ipynb)** - - --- - - Learn how to start a SFT customization job using a custom dataset. - - nemo-customizer - -- **[Align a Model with DPO](tutorials/dpo-customization-job.ipynb)** - - --- - - Learn how to align a model with DPO (Direct Preference Optimization) using preference data. - - nemo-customizer dpo - -- **[Distill a Model with Knowledge Distillation](tutorials/distillation-customization-job.ipynb)** - - --- - - Learn how to compress a larger teacher model into a smaller student model. - - nemo-customizer knowledge-distillation - -- **[Check Customization Job Metrics](tutorials/metrics.md)** - - --- - - Learn how to check job metrics using MLFlow or Weights & Biases. - - nemo-customizer mlflow wandb - -- **[Optimize Tokens per GPU](tutorials/optimize-throughput.ipynb)** - - --- - - Learn how to optimize the token-per-GPU throughput for a LoRA optimization job. - - nemo-customizer wandb sequence-packing - -
- ---- - -## References - -
- -- **[Hyperparameters](manage-customization-jobs/hyperparameters.md)** - - --- - - View the available hyperparameters and their valid options that you can set when creating a customization job. - -- **[Customizer API](../api/index.md#tag-customizer)** - - --- - - View the OpenAPI specification for Customizer. - -- **[Troubleshoot Failed Jobs](../troubleshooting/customizer.md)** - - --- - - View troubleshooting tips for failed jobs. - -
diff --git a/docs/customizer/index.mdx b/docs/customizer/index.mdx new file mode 100644 index 0000000000..b5a587b617 --- /dev/null +++ b/docs/customizer/index.mdx @@ -0,0 +1,166 @@ +--- +title: "About Fine-Tuning" +description: "" +--- + +Learn how to fine-tune models by making requests to NVIDIA NeMo Customizer through the API. Fine-tuned models you have created can be deployed using NVIDIA NIMs. + +## Fine-Tuning Workflow + + +At a high level, the fine-tuning workflow consists of the following steps: + +1. [Create a Model Entity](/documentation/customizer-reference/manage-model-entities/overview) pointing to your base model checkpoint (stored as a FileSet). +1. Format a compatible [dataset](/documentation/fine-tune-models/tutorials/format-training-dataset). +1. [Create a customization job](/documentation/fine-tune-models/manage-customization-jobs) referencing the Model Entity. +1. Monitor the job until it completes. +1. The customization job automatically creates either: + - **LoRA jobs**: An adapter attached to the original Model Entity + - **Full fine-tuning jobs**: A new Model Entity with the customized weights +1. [Deploy the model](/documentation/models-and-inference) using the Deployment Management Service. +1. Move on to [Evaluate the output model](/documentation/evaluate-models). + +--- + +## Model Catalog + +Explore the model families and sizes supported by NVIDIA NeMo Customizer. + + + + + + +View the available Llama models in the model catalog. + + + + +View the available Llama Nemotron models from NVIDIA, including Nano and Super variants for efficient and advanced instruction tuning. + + + + +View the available Phi models from Microsoft, designed for strong reasoning capabilities with efficient deployment. + + + + +View the available GPT-OSS models supported for Full SFT customization. + + + + +View the available embedding models for question-answering and retrieval tasks. + + + + + +## Task Guides + +Perform common fine-tuning tasks. + + + + + +Create, list, view, and cancel customization jobs. + + + + +Create FileSets and Model Entities to prepare base models for customization. + + + + +Upload and manage datasets for training. + + + + + +--- + +## Tutorials + +Follow these tutorials to learn how to accomplish common fine-tuning tasks. + + + + + +Learn how to format datasets for different model types. + +datasets chat-models completion-models + + + + +Learn how to start a LoRA customization job using a custom dataset. + +nemo-customizer + + + + +Learn how to start a SFT customization job using a custom dataset. + +nemo-customizer + + + + +Learn how to align a model with DPO (Direct Preference Optimization) using preference data. + +nemo-customizer dpo + + + + +Learn how to compress a larger teacher model into a smaller student model. + +nemo-customizer knowledge-distillation + + + + +Learn how to check job metrics using MLFlow or Weights & Biases. + +nemo-customizer mlflow wandb + + + + +Learn how to optimize the token-per-GPU throughput for a LoRA optimization job. + +nemo-customizer wandb sequence-packing + + + + + +--- + +## References + + + + + +View the available hyperparameters and their valid options that you can set when creating a customization job. + + + + +View the OpenAPI specification for Customizer. + + + + +View troubleshooting tips for failed jobs. + + + + diff --git a/docs/customizer/manage-customization-jobs/cancel-job.md b/docs/customizer/manage-customization-jobs/cancel-job.md deleted file mode 100644 index 14d0d403bc..0000000000 --- a/docs/customizer/manage-customization-jobs/cancel-job.md +++ /dev/null @@ -1,76 +0,0 @@ -# Cancel Job - - -## Prerequisites - -Before you can cancel a customization job, make sure that you have: - -- Obtained the base URL of your {{platform_name}}. -- Set the `NMP_BASE_URL` environment variable to your {{platform_name}} endpoint - -```bash -export NMP_BASE_URL="https://your-nmp-base-url" -``` - ---- - -## To Cancel a Customization Job - -Running jobs may be cancelled. A cancelled job does not upload checkpoints. You need the job's name and workspace; you can get these from [List Active Jobs](list-active-jobs.md). - -Use the SDK to cancel a customization job: - -```python -import os -from nemo_platform import NeMoPlatform - -# Initialize the client -client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="default", -) - -# Cancel a customization job (use the job name and workspace from List Active Jobs) -job_name = "my-sft-job" -workspace = "default" -cancelled_job = client.customization.jobs.cancel(name=job_name, workspace=workspace) - -print(f"Job {cancelled_job.name} has been cancelled") -print(f"Current status: {cancelled_job.status}") -print(f"Updated at: {cancelled_job.updated_at}") -``` - -??? "Example Response" - :icon: code-square - :open: - - ```json - { - "name": "my-sft-job", - "workspace": "default", - "id": "job-abc123def456", - "status": "cancelled", - "spec": { - "model": "default/llama-3-2-1b", - "dataset": "fileset://default/my-training-dataset", - "training": { - "type": "sft", - "batch_size": 16, - "epochs": 3, - "learning_rate": 1e-05, - "max_seq_length": 4096, - "parallelism": { - "num_gpus_per_node": 2, - "tensor_parallel_size": 2 - } - }, - "output": { - "name": "my-finetuned-llama", - "type": "model", - "fileset": "my-finetuned-llama-a1b2c3d4e5f6" - } - }, - "created_at": "2026-02-09T10:30:00.000Z", - "updated_at": "2026-02-09T10:35:00.000Z" - } - ``` diff --git a/docs/customizer/manage-customization-jobs/cancel-job.mdx b/docs/customizer/manage-customization-jobs/cancel-job.mdx new file mode 100644 index 0000000000..472e45eb7f --- /dev/null +++ b/docs/customizer/manage-customization-jobs/cancel-job.mdx @@ -0,0 +1,80 @@ +--- +title: "Cancel Job" +description: "" +--- + +## Prerequisites + +Before you can cancel a customization job, make sure that you have: + +- Obtained the base URL of your NeMo Platform. +- Set the `NMP_BASE_URL` environment variable to your NeMo Platform endpoint + +```bash +export NMP_BASE_URL="https://your-nmp-base-url" +``` + +--- + +## To Cancel a Customization Job + +Running jobs may be cancelled. A cancelled job does not upload checkpoints. You need the job's name and workspace; you can get these from [List Active Jobs](/documentation/customizer-reference/manage-jobs/list-active-jobs). + +Use the SDK to cancel a customization job: + +```python +import os +from nemo_platform import NeMoPlatform + +# Initialize the client +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) + +# Cancel a customization job (use the job name and workspace from List Active Jobs) +job_name = "my-sft-job" +workspace = "default" +cancelled_job = client.customization.jobs.cancel(name=job_name, workspace=workspace) + +print(f"Job {cancelled_job.name} has been cancelled") +print(f"Current status: {cancelled_job.status}") +print(f"Updated at: {cancelled_job.updated_at}") +``` + + + +:open: + +```json +{ + "name": "my-sft-job", + "workspace": "default", + "id": "job-abc123def456", + "status": "cancelled", + "spec": { + "model": "default/llama-3-2-1b", + "dataset": "fileset://default/my-training-dataset", + "training": { + "type": "sft", + "batch_size": 16, + "epochs": 3, + "learning_rate": 1e-05, + "max_seq_length": 4096, + "parallelism": { + "num_gpus_per_node": 2, + "tensor_parallel_size": 2 + } + }, + "output": { + "name": "my-finetuned-llama", + "type": "model", + "fileset": "my-finetuned-llama-a1b2c3d4e5f6" + } + }, + "created_at": "2026-02-09T10:30:00.000Z", + "updated_at": "2026-02-09T10:35:00.000Z" +} +``` + + diff --git a/docs/customizer/manage-customization-jobs/create-job.md b/docs/customizer/manage-customization-jobs/create-job.mdx similarity index 66% rename from docs/customizer/manage-customization-jobs/create-job.md rename to docs/customizer/manage-customization-jobs/create-job.mdx index d51469aba6..36c8d29354 100644 --- a/docs/customizer/manage-customization-jobs/create-job.md +++ b/docs/customizer/manage-customization-jobs/create-job.mdx @@ -1,16 +1,18 @@ -# Create Job - +--- +title: "Create Job" +description: "" +--- ## Prerequisites Before you can create a customization job, make sure that you have: -- Obtained the base URL of your {{platform_name}}. -- Created a [FileSet and Model Entity](../manage-model-entities/index.md) for your base model. -- [Uploaded a dataset](../../get-started/concepts/manage-files.md) as a FileSet. -- Determined the [training configuration](hyperparameters.md) you want to use for the customization job. -- Verified that the platform has sufficient storage for the job. Full SFT jobs require approximately 3× the base model size in free disk space; LoRA jobs require approximately 1.5×. See [ft-tut-understand-models](../tutorials/understand-configurations-and-models.md) for details. If you are also deploying the model from a base checkpoint fileset, plan for ~2.5× model size overall for LoRA. -- Set the `NMP_BASE_URL` environment variable to your {{platform_name}} endpoint. +- Obtained the base URL of your NeMo Platform. +- Created a [FileSet and Model Entity](/documentation/customizer-reference/manage-model-entities/overview) for your base model. +- [Uploaded a dataset](/documentation/get-started/core-concepts/manage-files) as a FileSet. +- Determined the [training configuration](/documentation/customizer-reference/manage-jobs/training-configuration) you want to use for the customization job. +- Verified that the platform has sufficient storage for the job. Full SFT jobs require approximately 3× the base model size in free disk space; LoRA jobs require approximately 1.5×. See [ft-tut-understand-models](/documentation/customizer-reference/tutorials/understanding-models-and-training) for details. If you are also deploying the model from a base checkpoint fileset, plan for ~2.5× model size overall for LoRA. +- Set the `NMP_BASE_URL` environment variable to your NeMo Platform endpoint. ```bash export NMP_BASE_URL="https://your-nemo-platform-url" @@ -60,43 +62,44 @@ print(f"Created job: {job.name}") print(f"Job status: {job.status}") ``` -??? "Example Response" - :icon: code-square - :open: - - ```json - { - "name": "my-lora-job", - "workspace": "default", - "id": "job-abc123def456", - "status": "created", - "spec": { - "model": "default/llama-3-2-1b", - "dataset": "fileset://default/my-training-dataset", - "training": { - "type": "sft", - "peft": { - "type": "lora", - "rank": 8, - "alpha": 32, - "dropout": 0.0 - }, - "batch_size": 32, - "epochs": 3, - "learning_rate": 0.0001, - "max_seq_length": 2048 - }, - "output": { - "name": "my-custom-model", - "type": "adapter", - "fileset": "my-custom-model-a1b2c3d4e5f6" - } + + +:open: + +```json +{ + "name": "my-lora-job", + "workspace": "default", + "id": "job-abc123def456", + "status": "created", + "spec": { + "model": "default/llama-3-2-1b", + "dataset": "fileset://default/my-training-dataset", + "training": { + "type": "sft", + "peft": { + "type": "lora", + "rank": 8, + "alpha": 32, + "dropout": 0.0 }, - "created_at": "2026-02-09T10:30:00.000Z", - "updated_at": "2026-02-09T10:30:00.000Z" + "batch_size": 32, + "epochs": 3, + "learning_rate": 0.0001, + "max_seq_length": 2048 + }, + "output": { + "name": "my-custom-model", + "type": "adapter", + "fileset": "my-custom-model-a1b2c3d4e5f6" } - ``` + }, + "created_at": "2026-02-09T10:30:00.000Z", + "updated_at": "2026-02-09T10:30:00.000Z" +} +``` + --- ### Knowledge Distillation Example @@ -126,14 +129,16 @@ job = client.customization.jobs.create( ) ``` -!!! note - See [Knowledge Distillation constraints](hyperparameters.md) for requirements on model compatibility, tokenizer, and GPU memory. + + +See [Knowledge Distillation constraints](/documentation/customizer-reference/manage-jobs/training-configuration) for requirements on model compatibility, tokenizer, and GPU memory. + --- ## Job Configuration Reference -The job spec contains the model, dataset, training, output, deployment, and integration settings for the job. For key fields, the complete API schema, and W&B or MLflow integration options, see [Customization Job Reference](customization-job-reference.md). +The job spec contains the model, dataset, training, output, deployment, and integration settings for the job. For key fields, the complete API schema, and W&B or MLflow integration options, see [Customization Job Reference](/documentation/customizer-reference/manage-jobs/customization-job-reference). --- diff --git a/docs/customizer/manage-customization-jobs/customization-job-reference.md b/docs/customizer/manage-customization-jobs/customization-job-reference.mdx similarity index 54% rename from docs/customizer/manage-customization-jobs/customization-job-reference.md rename to docs/customizer/manage-customization-jobs/customization-job-reference.mdx index 14705aa0be..9cd62cc3df 100644 --- a/docs/customizer/manage-customization-jobs/customization-job-reference.md +++ b/docs/customizer/manage-customization-jobs/customization-job-reference.mdx @@ -1,9 +1,11 @@ -# Customization Job Reference - +--- +title: "Customization Job Reference" +description: "" +--- Use this page when you need field-level details for customization job specifications, the complete API schema, or integration options. -For concepts, see [Customization Job overview](index.md). +For concepts, see [Customization Job overview](/documentation/fine-tune-models/manage-customization-jobs). ## Key Fields @@ -15,7 +17,7 @@ All job configuration (model, dataset, training, and output) is specified in the | `workspace` | Yes | Workspace where the job runs. Determines what datasets and models are authorized to be used in the job. | | `spec.model` | Yes | Reference to the Model Entity (`workspace/name` format) | | `spec.dataset` | Yes | Dataset URI (`fileset://workspace/name`) | -| `spec.training` | Yes | Training method and hyperparameters (see [Training Configuration](hyperparameters.md)) | +| `spec.training` | Yes | Training method and hyperparameters (see [Training Configuration](/documentation/customizer-reference/manage-jobs/training-configuration)) | | `spec.training.type` | Yes | Training method: `sft`, `distillation`, or `dpo` | | `spec.training.peft` | No | PEFT adapter configuration (e.g., `{"type": "lora", ...}`). Omit for full-weight training | | `spec.output` | No | Output artifact configuration (`{"name": "..."}`). Auto-generated if not provided | @@ -25,7 +27,7 @@ All job configuration (model, dataset, training, and output) is specified in the ## Complete API Reference -For generated REST API details, see the [Customizer API Reference](../../api/index.md#tag-customizer) and +For generated REST API details, see the [Customizer API Reference](/documentation/reference/api-reference) and search for `CustomizationJobInput`. --- @@ -34,51 +36,57 @@ search for `CustomizationJobInput`. To enable W&B integration, add the `integrations` configuration: -=== "Python" - - ```python - job = client.customization.jobs.create( - name="my-job", - workspace="default", - spec={ - "model": "default/llama-3-2-1b", - "dataset": "fileset://default/my-dataset", - "training": {"type": "sft", "peft": {"type": "lora"}, "epochs": 3}, - "integrations": { - "wandb": { - "project": "my-finetuning-project", - "entity": "my-team", - "tags": ["fine-tuning", "llama"], - "api_key_secret": "my-wandb-key", - } - }, - }, - ) - ``` + -=== "CLI" + - ```bash - nemo customization jobs create my-job \ - --workspace default \ - --spec '{ +```python +job = client.customization.jobs.create( + name="my-job", + workspace="default", + spec={ "model": "default/llama-3-2-1b", "dataset": "fileset://default/my-dataset", "training": {"type": "sft", "peft": {"type": "lora"}, "epochs": 3}, "integrations": { - "wandb": { - "project": "my-finetuning-project", - "entity": "my-team", - "tags": ["fine-tuning", "llama"], - "api_key_secret": "my-wandb-key" - } - } - }' - ``` - + "wandb": { + "project": "my-finetuning-project", + "entity": "my-team", + "tags": ["fine-tuning", "llama"], + "api_key_secret": "my-wandb-key", + } + }, + }, +) +``` + + + + +```bash +nemo customization jobs create my-job \ + --workspace default \ + --spec '{ + "model": "default/llama-3-2-1b", + "dataset": "fileset://default/my-dataset", + "training": {"type": "sft", "peft": {"type": "lora"}, "epochs": 3}, + "integrations": { + "wandb": { + "project": "my-finetuning-project", + "entity": "my-team", + "tags": ["fine-tuning", "llama"], + "api_key_secret": "my-wandb-key" + } + } + }' +``` + + + + The `api_key_secret` field references a stored secret containing your `WANDB_API_KEY`. Use the secret name (e.g., `"my-wandb-key"`) to resolve it from the request workspace. -To create the secret, see [Weights & Biases Keys](../../get-started/concepts/manage-secrets.md). +To create the secret, see [Weights & Biases Keys](/documentation/get-started/core-concepts/manage-secrets). | Field | Description | |-------|-------------| @@ -90,7 +98,7 @@ To create the secret, see [Weights & Biases Keys](../../get-started/concepts/man | `notes` | Notes or description for the run | | `base_url` | Base URL for self-hosted W&B servers (e.g., `https://wandb.mycompany.com`). Omit to use W&B cloud | -To view your training metrics in W&B after the job starts, see [ft-tut-metrics-wandb](../tutorials/metrics.md). +To view your training metrics in W&B after the job starts, see [ft-tut-metrics-wandb](/documentation/customizer-reference/tutorials/job-metrics). --- @@ -98,44 +106,50 @@ To view your training metrics in W&B after the job starts, see [ft-tut-metrics-w To enable MLflow integration: -=== "Python" - - ```python - job = client.customization.jobs.create( - name="my-job", - workspace="default", - spec={ - "model": "default/llama-3-2-1b", - "dataset": "fileset://default/my-dataset", - "training": {"type": "sft", "peft": {"type": "lora"}, "epochs": 3}, - "integrations": { - "mlflow": { - "experiment_name": "llama-finetuning", - "tracking_uri": "http://mlflow.example.com:5000", - } - }, - }, - ) - ``` + -=== "CLI" + - ```bash - nemo customization jobs create my-job \ - --workspace default \ - --spec '{ +```python +job = client.customization.jobs.create( + name="my-job", + workspace="default", + spec={ "model": "default/llama-3-2-1b", "dataset": "fileset://default/my-dataset", "training": {"type": "sft", "peft": {"type": "lora"}, "epochs": 3}, "integrations": { - "mlflow": { - "experiment_name": "llama-finetuning", - "tracking_uri": "http://mlflow.example.com:5000" - } - } - }' - ``` - + "mlflow": { + "experiment_name": "llama-finetuning", + "tracking_uri": "http://mlflow.example.com:5000", + } + }, + }, +) +``` + + + + +```bash +nemo customization jobs create my-job \ + --workspace default \ + --spec '{ + "model": "default/llama-3-2-1b", + "dataset": "fileset://default/my-dataset", + "training": {"type": "sft", "peft": {"type": "lora"}, "epochs": 3}, + "integrations": { + "mlflow": { + "experiment_name": "llama-finetuning", + "tracking_uri": "http://mlflow.example.com:5000" + } + } + }' +``` + + + + | Field | Description | |-------|-------------| | `experiment_name` | MLflow experiment name. Defaults to `output.name` if not set | @@ -146,7 +160,7 @@ To enable MLflow integration: ## Next Steps -- [Create a customization job](create-job.md): Start a job with a model, dataset, training configuration, and optional integrations. -- [Monitor training metrics](../tutorials/metrics.md): View logs and metrics through MLflow or W&B. -- [Manage secrets](../../get-started/concepts/manage-secrets.md): Store credentials such as W&B API keys and provider tokens. -- [Troubleshooting MLflow integrations](../../troubleshooting/customizer.md): Diagnose failed or misconfigured customization jobs. +- [Create a customization job](/documentation/fine-tune-models/manage-customization-jobs/create-a-customization-job): Start a job with a model, dataset, training configuration, and optional integrations. +- [Monitor training metrics](/documentation/customizer-reference/tutorials/job-metrics): View logs and metrics through MLflow or W&B. +- [Manage secrets](/documentation/get-started/core-concepts/manage-secrets): Store credentials such as W&B API keys and provider tokens. +- [Troubleshooting MLflow integrations](/documentation/reference/troubleshooting/customizer): Diagnose failed or misconfigured customization jobs. diff --git a/docs/customizer/manage-customization-jobs/get-job-status.md b/docs/customizer/manage-customization-jobs/get-job-status.mdx similarity index 90% rename from docs/customizer/manage-customization-jobs/get-job-status.md rename to docs/customizer/manage-customization-jobs/get-job-status.mdx index 0744df61af..fcc700fa17 100644 --- a/docs/customizer/manage-customization-jobs/get-job-status.md +++ b/docs/customizer/manage-customization-jobs/get-job-status.mdx @@ -1,23 +1,27 @@ -# Get Job Status - +--- +title: "Get Job Status" +description: "" +--- Get detailed execution status for a customization job, including step-by-step progress and real-time training metrics. -!!! tip - This endpoint provides granular execution details including: + - - **Step-level status**: `model-and-dataset-download` → `customization-training-job` → `model-upload` → `model-entity-creation` - - **Training metrics**: `step`, `epoch`, `loss`, `lr` (learning rate), `grad_norm`, `val_loss` - - **Progress tracking**: `downloaded_files`, `uploaded_bytes`, `progress_pct` +This endpoint provides granular execution details including: - To list jobs or get job definitions (model entity, hyperparameters, spec), use [List Active Jobs](list-active-jobs.md) instead. +- **Step-level status**: `model-and-dataset-download` → `customization-training-job` → `model-upload` → `model-entity-creation` +- **Training metrics**: `step`, `epoch`, `loss`, `lr` (learning rate), `grad_norm`, `val_loss` +- **Progress tracking**: `downloaded_files`, `uploaded_bytes`, `progress_pct` +To list jobs or get job definitions (model entity, hyperparameters, spec), use [List Active Jobs](/documentation/customizer-reference/manage-jobs/list-active-jobs) instead. + + ## Prerequisites Before you can get the status of a customization job, make sure that you have: -- Obtained the base URL of your {{platform_name}}. -- Set the `NMP_BASE_URL` environment variable to your {{platform_name}} endpoint +- Obtained the base URL of your NeMo Platform. +- Set the `NMP_BASE_URL` environment variable to your NeMo Platform endpoint ```bash export NMP_BASE_URL="https://your-nmp-base-url" @@ -58,7 +62,7 @@ for step in status.steps or []: print(f" Progress: {current_step}/{max_steps}") ``` -???+ "Example Response" + **Active Job (Training in Progress)** @@ -266,3 +270,5 @@ for step in status.steps or []: ] } ``` + + diff --git a/docs/customizer/manage-customization-jobs/hyperparameters.md b/docs/customizer/manage-customization-jobs/hyperparameters.mdx similarity index 76% rename from docs/customizer/manage-customization-jobs/hyperparameters.md rename to docs/customizer/manage-customization-jobs/hyperparameters.mdx index 716b121057..20233e7ca7 100644 --- a/docs/customizer/manage-customization-jobs/hyperparameters.md +++ b/docs/customizer/manage-customization-jobs/hyperparameters.mdx @@ -1,12 +1,16 @@ -# Training Configuration - +--- +title: "Training Configuration" +description: "" +--- -!!! tip - Want to learn about training concepts at a high level? Check out the [Customization concepts](../about.md) page. + +Want to learn about training concepts at a high level? Check out the [Customization concepts](/documentation/customizer-reference/customization-concepts) page. + + ## Complete Schema Reference -For generated REST API details, see the [Customizer API Reference](../../api/index.md#tag-customizer) and +For generated REST API details, see the [Customizer API Reference](/documentation/reference/api-reference) and search for `CustomizationJobInput`. Training is configured in the job's `spec.training` object. @@ -22,7 +26,7 @@ The `training` field is a discriminated union on the `type` field. Each training | `training.type` | `sft`, `dpo`, `distillation` | Training method (discriminated union) | | `training.peft` | `{ type: "lora", rank: 8, ... }` or omit | PEFT adapter configuration. If set, trains an adapter; if omitted, performs full-weight training | -For generated SFT schema details, see the [Customizer API Reference](../../api/index.md#tag-customizer) +For generated SFT schema details, see the [Customizer API Reference](/documentation/reference/api-reference) and search for `SFTTrainingInput`. ### DPO Configuration @@ -37,15 +41,19 @@ When `training.type` is `"dpo"`, additional DPO-specific fields are available: | `preference_loss_weight` | Weight for preference loss | `1.0` | | `sft_loss_weight` | Weight for SFT loss | `0.0` | -For generated DPO schema details, see the [Customizer API Reference](../../api/index.md#tag-customizer) +For generated DPO schema details, see the [Customizer API Reference](/documentation/reference/api-reference) and search for `DPOTrainingInput`. -!!! note - PEFT (LoRA) is not yet supported with DPO training. Use full-weight training by omitting the `peft` field. + + +PEFT (LoRA) is not yet supported with DPO training. Use full-weight training by omitting the `peft` field. -!!! tip - When setting `val_check_interval` for DPO, use a fractional value (e.g., `0.5` for twice per epoch) or omit it entirely (validates once at end of epoch). Avoid integer step counts — they may not divide evenly into the total training steps, which can prevent validation from running on the final step. + + +When setting `val_check_interval` for DPO, use a fractional value (e.g., `0.5` for twice per epoch) or omit it entirely (validates once at end of epoch). Avoid integer step counts — they may not divide evenly into the total training steps, which can prevent validation from running on the final step. + + ### Parallelism Configuration Parallelism parameters are grouped inside `training.parallelism`: @@ -60,11 +68,13 @@ Parallelism parameters are grouped inside `training.parallelism`: | `parallelism.expert_parallel_size` | Expert parallelism for MoE models | Must divide number of experts | | `parallelism.sequence_parallel` | Enable sequence parallelism | Memory optimization for long sequences | -!!! note - **GPU Relationship**: `total_gpus = num_gpus_per_node x num_nodes` + + +**GPU Relationship**: `total_gpus = num_gpus_per_node x num_nodes` - `data_parallel_size` is automatically derived as `total_gpus / (TP × PP × CP)`. +`data_parallel_size` is automatically derived as `total_gpus / (TP × PP × CP)`. + ### PEFT / LoRA Configuration To train a LoRA adapter, set `training.peft`: @@ -104,16 +114,18 @@ When `training.type` is `"distillation"`, additional KD-specific fields are avai | `distillation_temperature` | Softmax temperature for KD. Higher = softer probability distributions | `1.0–5.0` (start with `2.0`) | For generated distillation schema details, see the -[Customizer API Reference](../../api/index.md#tag-customizer) and search for +[Customizer API Reference](/documentation/reference/api-reference) and search for `DistillationTrainingInput`. -!!! note - - Knowledge distillation uses **logit-pair distillation only** — the student learns to match the teacher's output probability distribution. - - Both student and teacher models must be **full-weight Model Entities**. LoRA adapters cannot be used as teacher models. - - Student and teacher must **share the same tokenizer and vocabulary**. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct). - - Both models are loaded during training. Plan GPU memory accordingly. + + +- Knowledge distillation uses **logit-pair distillation only** — the student learns to match the teacher's output probability distribution. +- Both student and teacher models must be **full-weight Model Entities**. LoRA adapters cannot be used as teacher models. +- Student and teacher must **share the same tokenizer and vocabulary**. Use models from the same family (e.g., Llama 3.2 1B Instruct + Llama 3.2 3B Instruct). +- Both models are loaded during training. Plan GPU memory accordingly. + --- ## Common Tuning Scenarios @@ -172,5 +184,8 @@ Estimated GPU requirements by model size: | 13B | 80GB | 4 × 80GB | | 70B | 2 × 80GB | 8+ × 80GB | -!!! tip - Use LoRA for most fine-tuning tasks. It's significantly more memory-efficient and often achieves comparable results to full fine-tuning. + + +Use LoRA for most fine-tuning tasks. It's significantly more memory-efficient and often achieves comparable results to full fine-tuning. + + diff --git a/docs/customizer/manage-customization-jobs/index.md b/docs/customizer/manage-customization-jobs/index.md deleted file mode 100644 index 93a2d7946d..0000000000 --- a/docs/customizer/manage-customization-jobs/index.md +++ /dev/null @@ -1,74 +0,0 @@ -# Manage Customization Jobs - - -Use customization jobs to fine-tune a [model](../models/index.md) using a [dataset](../../get-started/concepts/manage-files.md) and [hyperparameters](hyperparameters.md). - -## How It Works - -Customization jobs reference a **Model Entity** that contains the base model checkpoint. When training completes: - -- **LoRA jobs**: Create an **Adapter** attached to the original Model Entity. Adapters can be auto-deployed to NIMs. -- **Full fine-tuning jobs**: Create a **new Model Entity** with the customized weights, linked to the base model. - -This design keeps adapters organized with their parent models and simplifies deployment workflows. - -## Prerequisites - -Before you can customize a model using a customization job, make sure that you have `prepared and uploaded a dataset <../tutorials/format-training-dataset>` to the dataset repository. See also [format-training-dataset](../tutorials/format-training-dataset.md) for dataset formatting requirements. - ---- - -## Task Guides - -Perform common customization job tasks. - -!!! tip - The value for `NMP_BASE_URL` will depend on your deployment. After the standard [Setup](../../get-started/setup.md) flow, the default local URL is `http://localhost:8080`. Otherwise, consult with your cluster administrator. - -
- -- **[Create a Customization Job](create-job.md)** - - --- - - Create a customization job using SFT, DPO, or Knowledge Distillation. - -- **[Get Job Status](get-job-status.md)** - - --- - - Check the status of a customization job. - -- **[List Active Jobs](list-active-jobs.md)** - - --- - - List all active customization jobs to find a job name for use with Get Status or Cancel. - -- **[Cancel a Job](cancel-job.md)** - - --- - - Cancel a customization job using its name and workspace. - -
- -## References - -Refer to the following pages for more information on customization jobs. - -
- -- **[Hyperparameters](hyperparameters.md)** - - --- - - Review the hyperparameters that you can use to customize a model. - -- **[Troubleshoot Failed Jobs](../../troubleshooting/customizer.md)** - - --- - - View troubleshooting tips for failed jobs. - -
diff --git a/docs/customizer/manage-customization-jobs/index.mdx b/docs/customizer/manage-customization-jobs/index.mdx new file mode 100644 index 0000000000..631b40efad --- /dev/null +++ b/docs/customizer/manage-customization-jobs/index.mdx @@ -0,0 +1,74 @@ +--- +title: "Manage Customization Jobs" +description: "" +--- + +Use customization jobs to fine-tune a [model](/documentation/customizer-reference/models/model-catalog) using a [dataset](/documentation/get-started/core-concepts/manage-files) and [hyperparameters](/documentation/customizer-reference/manage-jobs/training-configuration). + +## How It Works + +Customization jobs reference a **Model Entity** that contains the base model checkpoint. When training completes: + +- **LoRA jobs**: Create an **Adapter** attached to the original Model Entity. Adapters can be auto-deployed to NIMs. +- **Full fine-tuning jobs**: Create a **new Model Entity** with the customized weights, linked to the base model. + +This design keeps adapters organized with their parent models and simplifies deployment workflows. + +## Prerequisites + +Before you can customize a model using a customization job, make sure that you have [prepared and uploaded a dataset](/documentation/fine-tune-models/tutorials/format-training-dataset) to the dataset repository. + +--- + +## Task Guides + +Perform common customization job tasks. + + + +The value for `NMP_BASE_URL` will depend on your deployment. After the standard [Setup](/documentation/get-started) flow, the default local URL is `http://localhost:8080`. Otherwise, consult with your cluster administrator. + + + + + + +Create a customization job using SFT, DPO, or Knowledge Distillation. + + + + +Check the status of a customization job. + + + + +List all active customization jobs to find a job name for use with Get Status or Cancel. + + + + +Cancel a customization job using its name and workspace. + + + + + +## References + +Refer to the following pages for more information on customization jobs. + + + + + +Review the hyperparameters that you can use to customize a model. + + + + +View troubleshooting tips for failed jobs. + + + + diff --git a/docs/customizer/manage-customization-jobs/list-active-jobs.md b/docs/customizer/manage-customization-jobs/list-active-jobs.md deleted file mode 100644 index a813c8b7a8..0000000000 --- a/docs/customizer/manage-customization-jobs/list-active-jobs.md +++ /dev/null @@ -1,116 +0,0 @@ -# List Active Jobs - - -List all customization jobs and their high-level status. This returns job definitions including the model, dataset, training configuration, and overall status. - -!!! tip - To get **detailed execution progress** (step-by-step status, training metrics like loss/epoch/step), use [Get Job Status](get-job-status.md) instead. - -## Prerequisites - -Before you can list active customization jobs, make sure that you have: - -- Obtained the base URL of your {{platform_name}}. -- Set the `NMP_BASE_URL` environment variable to your {{platform_name}} endpoint - -```bash -export NMP_BASE_URL="https://your-nmp-base-url" -``` - ---- - -## To List Active Customization Jobs - -Use the SDK to list customization jobs: - -```python -import os -from nemo_platform import NeMoPlatform - -# Initialize the client -client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="default", -) - -# List all customization jobs -jobs = client.customization.jobs.list( - workspace="default", page=1, page_size=10, sort="created_at" -) - -print(f"Found {len(jobs.data)} jobs") -for job in jobs.data: - print(f"Job {job.name}: {job.status}") - -# List jobs with filters (optional) -# Valid filter fields: name, status, project, workspace, created_at, updated_at -filtered_jobs = client.customization.jobs.list( - workspace="default", - filter={ - "status": "active", # Filter by job status - "name": "my-sft-job", # Filter by job name - }, - sort="-created_at", # Sort by created_at descending -) - -print(f"Found {len(filtered_jobs.data)} jobs") -for job in filtered_jobs.data: - print(f"Job {job.name}: {job.status}") -``` - -??? "Example Response" - :icon: code-square - :open: - - ```json - { - "data": [ - { - "id": "platform-job-QtyhRY5ub4t4tTLPY4sTkz", - "name": "my-sft-job-99da", - "workspace": "default", - "created_at": "2026-02-09T22:12:45", - "updated_at": "2026-02-09T22:12:45", - "status": "active", - "status_details": { - "message": "Job is running" - }, - "spec": { - "model": "default/llama-3-2-1b", - "dataset": "fileset://default/sft-dataset", - "training": { - "type": "sft", - "batch_size": 64, - "epochs": 2, - "learning_rate": 5e-05, - "weight_decay": 0.01, - "max_seq_length": 2048, - "parallelism": { - "num_gpus_per_node": 1, - "num_nodes": 1, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "context_parallel_size": 1 - } - }, - "output": { - "name": "customization-407790d32cfb", - "type": "model", - "fileset": "customization-407790d32cfb" - }, - "custom_fields": {} - } - } - ], - "pagination": { - "page": 1, - "page_size": 10, - "current_page_size": 1, - "total_pages": 1, - "total_results": 1 - }, - "sort": "created_at", - "filter": {}, - "search": {} - } - ``` diff --git a/docs/customizer/manage-customization-jobs/list-active-jobs.mdx b/docs/customizer/manage-customization-jobs/list-active-jobs.mdx new file mode 100644 index 0000000000..7a6b644507 --- /dev/null +++ b/docs/customizer/manage-customization-jobs/list-active-jobs.mdx @@ -0,0 +1,122 @@ +--- +title: "List Active Jobs" +description: "" +--- + +List all customization jobs and their high-level status. This returns job definitions including the model, dataset, training configuration, and overall status. + + + +To get **detailed execution progress** (step-by-step status, training metrics like loss/epoch/step), use [Get Job Status](/documentation/customizer-reference/manage-jobs/get-job-status) instead. + + +## Prerequisites + +Before you can list active customization jobs, make sure that you have: + +- Obtained the base URL of your NeMo Platform. +- Set the `NMP_BASE_URL` environment variable to your NeMo Platform endpoint + +```bash +export NMP_BASE_URL="https://your-nmp-base-url" +``` + +--- + +## To List Active Customization Jobs + +Use the SDK to list customization jobs: + +```python +import os +from nemo_platform import NeMoPlatform + +# Initialize the client +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) + +# List all customization jobs +jobs = client.customization.jobs.list( + workspace="default", page=1, page_size=10, sort="created_at" +) + +print(f"Found {len(jobs.data)} jobs") +for job in jobs.data: + print(f"Job {job.name}: {job.status}") + +# List jobs with filters (optional) +# Valid filter fields: name, status, project, workspace, created_at, updated_at +filtered_jobs = client.customization.jobs.list( + workspace="default", + filter={ + "status": "active", # Filter by job status + "name": "my-sft-job", # Filter by job name + }, + sort="-created_at", # Sort by created_at descending +) + +print(f"Found {len(filtered_jobs.data)} jobs") +for job in filtered_jobs.data: + print(f"Job {job.name}: {job.status}") +``` + + + +:open: + +```json +{ + "data": [ + { + "id": "platform-job-QtyhRY5ub4t4tTLPY4sTkz", + "name": "my-sft-job-99da", + "workspace": "default", + "created_at": "2026-02-09T22:12:45", + "updated_at": "2026-02-09T22:12:45", + "status": "active", + "status_details": { + "message": "Job is running" + }, + "spec": { + "model": "default/llama-3-2-1b", + "dataset": "fileset://default/sft-dataset", + "training": { + "type": "sft", + "batch_size": 64, + "epochs": 2, + "learning_rate": 5e-05, + "weight_decay": 0.01, + "max_seq_length": 2048, + "parallelism": { + "num_gpus_per_node": 1, + "num_nodes": 1, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "context_parallel_size": 1 + } + }, + "output": { + "name": "customization-407790d32cfb", + "type": "model", + "fileset": "customization-407790d32cfb" + }, + "custom_fields": {} + } + } + ], + "pagination": { + "page": 1, + "page_size": 10, + "current_page_size": 1, + "total_pages": 1, + "total_results": 1 + }, + "sort": "created_at", + "filter": {}, + "search": {} +} +``` + + diff --git a/docs/customizer/manage-model-entities/create-fileset.md b/docs/customizer/manage-model-entities/create-fileset.md deleted file mode 100644 index c9ffaea239..0000000000 --- a/docs/customizer/manage-model-entities/create-fileset.md +++ /dev/null @@ -1,266 +0,0 @@ -# Create a Model FileSet - - -Create a FileSet containing your base model checkpoint before creating a Model Entity. - -## Prerequisites - -- Obtained the base URL of your {{platform_name}}. -- For HuggingFace models: Created a secret with your HF token. Refer to [Manage Secrets](../../get-started/concepts/manage-secrets.md). -- Set the `NMP_BASE_URL` environment variable. - -```bash -export NMP_BASE_URL="https://your-nemo-platform-url" -``` - ---- - -## From HuggingFace Hub - -The most common method is downloading directly from HuggingFace: - -=== "Python SDK" - - ```python - import os - from nemo_platform import ConflictError, NeMoPlatform - from nemo_platform.types.files import HuggingfaceStorageConfigParam - - client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="default", - ) - - HF_REPO_ID = "meta-llama/Llama-3.2-1B-Instruct" - MODEL_NAME = "llama-3-2-1b" - HF_SECRET_NAME = "my-hf-token" - HF_TOKEN = os.environ.get("HF_TOKEN") - if not HF_TOKEN: - raise RuntimeError("Set HF_TOKEN before creating the HuggingFace secret.") - - # First, create a secret for your HF token (if not already created) - try: - hf_secret = client.secrets.create( - name=HF_SECRET_NAME, - workspace="default", - value=HF_TOKEN, - ) - print(f"Created secret: {HF_SECRET_NAME}") - except ConflictError: - print(f"Secret '{HF_SECRET_NAME}' already exists, continuing...") - hf_secret = client.secrets.retrieve(name=HF_SECRET_NAME, workspace="default") - - # Create FileSet from HuggingFace - try: - fileset = client.files.filesets.create( - workspace="default", - name=MODEL_NAME, - description="Llama 3.2 1B Instruct from HuggingFace", - purpose="model", - storage=HuggingfaceStorageConfigParam( - type="huggingface", - repo_id=HF_REPO_ID, - repo_type="model", - token_secret=hf_secret.name, - ), - ) - print(f"Created FileSet: {fileset.name}") - except ConflictError: - print(f"FileSet '{MODEL_NAME}' already exists, retrieving...") - fileset = client.files.filesets.retrieve(workspace="default", name=MODEL_NAME) - - print(f"FileSet ready: {fileset.name}") - ``` - -=== "CLI" - - ```bash - export WORKSPACE="default" - export HF_REPO_ID="meta-llama/Llama-3.2-1B-Instruct" - export MODEL_NAME="llama-3-2-1b" - export HF_SECRET_NAME="my-hf-token" - - # Export HF_TOKEN with your HuggingFace token before running this. - : "${HF_TOKEN:?Set HF_TOKEN before creating the HuggingFace secret.}" - - nemo secrets get "$HF_SECRET_NAME" --workspace "$WORKSPACE" >/dev/null 2>&1 || \ - printf '%s' "$HF_TOKEN" | nemo secrets create "$HF_SECRET_NAME" \ - --workspace "$WORKSPACE" \ - --from-file - - - nemo files filesets create "$MODEL_NAME" \ - --workspace "$WORKSPACE" \ - --description "Llama 3.2 1B Instruct from HuggingFace" \ - --purpose model \ - --exist-ok \ - --storage '{ - "type": "huggingface", - "repo_id": "'"$HF_REPO_ID"'", - "repo_type": "model", - "token_secret": "'"$HF_SECRET_NAME"'" - }' - - nemo files filesets get "$MODEL_NAME" --workspace "$WORKSPACE" - ``` - -!!! tip - For gated models (like Llama), you need to: - 1. Accept the model license on HuggingFace - 2. Create a HuggingFace token with read access - 3. Store the token as a secret in the platform - ---- - -## From NGC - -For models from NVIDIA NGC: - -=== "Python SDK" - - ```python - import os - from nemo_platform import ConflictError, NeMoPlatform - from nemo_platform.types.files import NGCStorageConfigParam - - client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="default", - ) - - MODEL_NAME = "nemotron-mini-4b" - NGC_RESOURCE = "nemotron-mini-4b-instruct" - NGC_ORG = "nvidia" - NGC_TEAM = "nemo" - NGC_VERSION = "1.0" - NGC_API_KEY_SECRET = "my-ngc-key" - NGC_API_KEY = os.environ.get("NGC_API_KEY") - if not NGC_API_KEY: - raise RuntimeError("Set NGC_API_KEY before creating the NGC secret.") - - # First, create a secret for your NGC API key (if not already created) - try: - ngc_secret = client.secrets.create( - name=NGC_API_KEY_SECRET, - workspace="default", - value=NGC_API_KEY, - ) - print(f"Created secret: {NGC_API_KEY_SECRET}") - except ConflictError: - print(f"Secret '{NGC_API_KEY_SECRET}' already exists, continuing...") - ngc_secret = client.secrets.retrieve(name=NGC_API_KEY_SECRET, workspace="default") - - # Create FileSet from NGC - try: - fileset = client.files.filesets.create( - workspace="default", - name=MODEL_NAME, - description="Nemotron Mini 4B from NGC", - purpose="model", - storage=NGCStorageConfigParam( - type="ngc", - org=NGC_ORG, - team=NGC_TEAM, - resource=NGC_RESOURCE, # NGC resource name - version=NGC_VERSION, - api_key_secret=ngc_secret.name, - ), - ) - print(f"Created FileSet: {fileset.name}") - except ConflictError: - print("FileSet already exists, retrieving...") - fileset = client.files.filesets.retrieve(workspace="default", name=MODEL_NAME) - ``` - -=== "CLI" - - ```bash - export WORKSPACE="default" - export MODEL_NAME="nemotron-mini-4b" - export NGC_RESOURCE="nemotron-mini-4b-instruct" - export NGC_ORG="nvidia" - export NGC_TEAM="nemo" - export NGC_VERSION="1.0" - export NGC_API_KEY_SECRET="my-ngc-key" - - # Export NGC_API_KEY with your NGC API key before running this. - : "${NGC_API_KEY:?Set NGC_API_KEY before creating the NGC secret.}" - - nemo secrets get "$NGC_API_KEY_SECRET" --workspace "$WORKSPACE" >/dev/null 2>&1 || \ - printf '%s' "$NGC_API_KEY" | nemo secrets create "$NGC_API_KEY_SECRET" \ - --workspace "$WORKSPACE" \ - --from-file - - - nemo files filesets create "$MODEL_NAME" \ - --workspace "$WORKSPACE" \ - --description "Nemotron Mini 4B from NGC" \ - --purpose model \ - --exist-ok \ - --storage '{ - "type": "ngc", - "org": "'"$NGC_ORG"'", - "team": "'"$NGC_TEAM"'", - "resource": "'"$NGC_RESOURCE"'", - "version": "'"$NGC_VERSION"'", - "api_key_secret": "'"$NGC_API_KEY_SECRET"'" - }' - - nemo files filesets get "$MODEL_NAME" --workspace "$WORKSPACE" - ``` - ---- - -## Check FileSet Status - -Files are downloaded in the background after you create a FileSet. Check the status: - -```python -# List files in the FileSet -files = client.files.list( - workspace="default", - fileset="llama-3-2-1b", -) - -print(f"Files in FileSet ({len(files)} total):") -for f in files[:10]: # Show first 10 - print(f" - {f.path} ({f.size:,} bytes)") -``` - -??? "Example Response" - :icon: code-square - - ```json - { - "data": [ - { - "file_ref": "/v2/workspaces/default/filesets/llama-3-2-1b/-/config.json", - "path": "config.json", - "size": 1234, - "cache_status": null - }, - { - "file_ref": "/v2/workspaces/default/filesets/llama-3-2-1b/-/model.safetensors", - "path": "model.safetensors", - "size": 2400000000, - "cache_status": null - }, - { - "file_ref": "/v2/workspaces/default/filesets/llama-3-2-1b/-/tokenizer.json", - "path": "tokenizer.json", - "size": 9085657, - "cache_status": null - }, - { - "file_ref": "/v2/workspaces/default/filesets/llama-3-2-1b/-/tokenizer_config.json", - "path": "tokenizer_config.json", - "size": 901, - "cache_status": null - } - ] - } - ``` - ---- - -## Next Steps - -Proceed to [create a Model Entity](create-model-entity.md). diff --git a/docs/customizer/manage-model-entities/create-fileset.mdx b/docs/customizer/manage-model-entities/create-fileset.mdx new file mode 100644 index 0000000000..94809e87a6 --- /dev/null +++ b/docs/customizer/manage-model-entities/create-fileset.mdx @@ -0,0 +1,282 @@ +--- +title: "Create a Model FileSet" +description: "" +--- + +Create a FileSet containing your base model checkpoint before creating a Model Entity. + +## Prerequisites + +- Obtained the base URL of your NeMo Platform. +- For HuggingFace models: Created a secret with your HF token. Refer to [Manage Secrets](/documentation/get-started/core-concepts/manage-secrets). +- Set the `NMP_BASE_URL` environment variable. + +```bash +export NMP_BASE_URL="https://your-nemo-platform-url" +``` + +--- + +## From HuggingFace Hub + +The most common method is downloading directly from HuggingFace: + + + + + +```python +import os +from nemo_platform import ConflictError, NeMoPlatform +from nemo_platform.types.files import HuggingfaceStorageConfigParam + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) + +HF_REPO_ID = "meta-llama/Llama-3.2-1B-Instruct" +MODEL_NAME = "llama-3-2-1b" +HF_SECRET_NAME = "my-hf-token" +HF_TOKEN = os.environ.get("HF_TOKEN") +if not HF_TOKEN: + raise RuntimeError("Set HF_TOKEN before creating the HuggingFace secret.") + +# First, create a secret for your HF token (if not already created) +try: + hf_secret = client.secrets.create( + name=HF_SECRET_NAME, + workspace="default", + value=HF_TOKEN, + ) + print(f"Created secret: {HF_SECRET_NAME}") +except ConflictError: + print(f"Secret '{HF_SECRET_NAME}' already exists, continuing...") + hf_secret = client.secrets.retrieve(name=HF_SECRET_NAME, workspace="default") + +# Create FileSet from HuggingFace +try: + fileset = client.files.filesets.create( + workspace="default", + name=MODEL_NAME, + description="Llama 3.2 1B Instruct from HuggingFace", + purpose="model", + storage=HuggingfaceStorageConfigParam( + type="huggingface", + repo_id=HF_REPO_ID, + repo_type="model", + token_secret=hf_secret.name, + ), + ) + print(f"Created FileSet: {fileset.name}") +except ConflictError: + print(f"FileSet '{MODEL_NAME}' already exists, retrieving...") + fileset = client.files.filesets.retrieve(workspace="default", name=MODEL_NAME) + +print(f"FileSet ready: {fileset.name}") +``` + + + + +```bash +export WORKSPACE="default" +export HF_REPO_ID="meta-llama/Llama-3.2-1B-Instruct" +export MODEL_NAME="llama-3-2-1b" +export HF_SECRET_NAME="my-hf-token" + +# Export HF_TOKEN with your HuggingFace token before running this. +: "${HF_TOKEN:?Set HF_TOKEN before creating the HuggingFace secret.}" + +nemo secrets get "$HF_SECRET_NAME" --workspace "$WORKSPACE" >/dev/null 2>&1 || \ + printf '%s' "$HF_TOKEN" | nemo secrets create "$HF_SECRET_NAME" \ + --workspace "$WORKSPACE" \ + --from-file - + +nemo files filesets create "$MODEL_NAME" \ + --workspace "$WORKSPACE" \ + --description "Llama 3.2 1B Instruct from HuggingFace" \ + --purpose model \ + --exist-ok \ + --storage '{ + "type": "huggingface", + "repo_id": "'"$HF_REPO_ID"'", + "repo_type": "model", + "token_secret": "'"$HF_SECRET_NAME"'" + }' + +nemo files filesets get "$MODEL_NAME" --workspace "$WORKSPACE" +``` + + + + + + +For gated models (like Llama), you need to: +1. Accept the model license on HuggingFace +2. Create a HuggingFace token with read access +3. Store the token as a secret in the platform + + +--- + +## From NGC + +For models from NVIDIA NGC: + + + + + +```python +import os +from nemo_platform import ConflictError, NeMoPlatform +from nemo_platform.types.files import NGCStorageConfigParam + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) + +MODEL_NAME = "nemotron-mini-4b" +NGC_RESOURCE = "nemotron-mini-4b-instruct" +NGC_ORG = "nvidia" +NGC_TEAM = "nemo" +NGC_VERSION = "1.0" +NGC_API_KEY_SECRET = "my-ngc-key" +NGC_API_KEY = os.environ.get("NGC_API_KEY") +if not NGC_API_KEY: + raise RuntimeError("Set NGC_API_KEY before creating the NGC secret.") + +# First, create a secret for your NGC API key (if not already created) +try: + ngc_secret = client.secrets.create( + name=NGC_API_KEY_SECRET, + workspace="default", + value=NGC_API_KEY, + ) + print(f"Created secret: {NGC_API_KEY_SECRET}") +except ConflictError: + print(f"Secret '{NGC_API_KEY_SECRET}' already exists, continuing...") + ngc_secret = client.secrets.retrieve(name=NGC_API_KEY_SECRET, workspace="default") + +# Create FileSet from NGC +try: + fileset = client.files.filesets.create( + workspace="default", + name=MODEL_NAME, + description="Nemotron Mini 4B from NGC", + purpose="model", + storage=NGCStorageConfigParam( + type="ngc", + org=NGC_ORG, + team=NGC_TEAM, + resource=NGC_RESOURCE, # NGC resource name + version=NGC_VERSION, + api_key_secret=ngc_secret.name, + ), + ) + print(f"Created FileSet: {fileset.name}") +except ConflictError: + print("FileSet already exists, retrieving...") + fileset = client.files.filesets.retrieve(workspace="default", name=MODEL_NAME) +``` + + + + +```bash +export WORKSPACE="default" +export MODEL_NAME="nemotron-mini-4b" +export NGC_RESOURCE="nemotron-mini-4b-instruct" +export NGC_ORG="nvidia" +export NGC_TEAM="nemo" +export NGC_VERSION="1.0" +export NGC_API_KEY_SECRET="my-ngc-key" + +# Export NGC_API_KEY with your NGC API key before running this. +: "${NGC_API_KEY:?Set NGC_API_KEY before creating the NGC secret.}" + +nemo secrets get "$NGC_API_KEY_SECRET" --workspace "$WORKSPACE" >/dev/null 2>&1 || \ + printf '%s' "$NGC_API_KEY" | nemo secrets create "$NGC_API_KEY_SECRET" \ + --workspace "$WORKSPACE" \ + --from-file - + +nemo files filesets create "$MODEL_NAME" \ + --workspace "$WORKSPACE" \ + --description "Nemotron Mini 4B from NGC" \ + --purpose model \ + --exist-ok \ + --storage '{ + "type": "ngc", + "org": "'"$NGC_ORG"'", + "team": "'"$NGC_TEAM"'", + "resource": "'"$NGC_RESOURCE"'", + "version": "'"$NGC_VERSION"'", + "api_key_secret": "'"$NGC_API_KEY_SECRET"'" + }' + +nemo files filesets get "$MODEL_NAME" --workspace "$WORKSPACE" +``` + + + + +--- + +## Check FileSet Status + +Files are downloaded in the background after you create a FileSet. Check the status: + +```python +# List files in the FileSet +files = client.files.list( + workspace="default", + fileset="llama-3-2-1b", +) + +print(f"Files in FileSet ({len(files)} total):") +for f in files[:10]: # Show first 10 + print(f" - {f.path} ({f.size:,} bytes)") +``` + + + +```json +{ + "data": [ + { + "file_ref": "/v2/workspaces/default/filesets/llama-3-2-1b/-/config.json", + "path": "config.json", + "size": 1234, + "cache_status": null + }, + { + "file_ref": "/v2/workspaces/default/filesets/llama-3-2-1b/-/model.safetensors", + "path": "model.safetensors", + "size": 2400000000, + "cache_status": null + }, + { + "file_ref": "/v2/workspaces/default/filesets/llama-3-2-1b/-/tokenizer.json", + "path": "tokenizer.json", + "size": 9085657, + "cache_status": null + }, + { + "file_ref": "/v2/workspaces/default/filesets/llama-3-2-1b/-/tokenizer_config.json", + "path": "tokenizer_config.json", + "size": 901, + "cache_status": null + } + ] +} +``` + + +--- + +## Next Steps + +Proceed to [create a Model Entity](/documentation/customizer-reference/manage-model-entities/create-a-model-entity). diff --git a/docs/customizer/manage-model-entities/create-model-entity.md b/docs/customizer/manage-model-entities/create-model-entity.mdx similarity index 74% rename from docs/customizer/manage-model-entities/create-model-entity.md rename to docs/customizer/manage-model-entities/create-model-entity.mdx index 6c129a0835..7c540799ad 100644 --- a/docs/customizer/manage-model-entities/create-model-entity.md +++ b/docs/customizer/manage-model-entities/create-model-entity.mdx @@ -1,11 +1,13 @@ -# Create a Model Entity - +--- +title: "Create a Model Entity" +description: "" +--- Create a Model Entity that references your FileSet to enable customization jobs. ## Prerequisites -- Created a FileSet containing your model checkpoint (refer to [Create a Model FileSet](create-fileset.md)). +- Created a FileSet containing your model checkpoint (refer to [Create a Model FileSet](/documentation/fine-tune-models/manage-model-entities/create-a-model-file-set)). - Set the `NMP_BASE_URL` environment variable. ```bash @@ -39,25 +41,25 @@ except ConflictError: model = client.models.retrieve(workspace="default", name="llama-3-2-1b") ``` -??? "Example Response" - :icon: code-square - - ```json - { - "id": "model-abc123def456", - "name": "llama-3-2-1b", - "workspace": "default", - "description": "Llama 3.2 1B base model for customization", - "fileset": "default/llama-3-2-1b", - "spec": null, - "adapters": [], - "created_at": "2026-02-09T10:30:00Z", - "updated_at": "2026-02-09T10:30:00Z" - } - ``` - - Note: `spec` is initially `null` and will be auto-populated by the Models Controller. + + +```json +{ + "id": "model-abc123def456", + "name": "llama-3-2-1b", + "workspace": "default", + "description": "Llama 3.2 1B base model for customization", + "fileset": "default/llama-3-2-1b", + "spec": null, + "adapters": [], + "created_at": "2026-02-09T10:30:00Z", + "updated_at": "2026-02-09T10:30:00Z" +} +``` + +Note: `spec` is initially `null` and will be auto-populated by the Models Controller. + --- ## Wait for Model Spec Auto-Population @@ -84,24 +86,24 @@ print(f" Layers: {model.spec.num_layers}") print(f" Attention Heads: {model.spec.num_attention_heads}") ``` -??? "Example Model Spec" - :icon: code-square - - ```json - { - "spec": { - "family": "llama", - "base_num_parameters": 1235814400, - "hidden_size": 2048, - "num_layers": 16, - "num_attention_heads": 32, - "num_key_value_heads": 8, - "vocab_size": 128256, - "max_sequence_length": 131072 - } - } - ``` + + +```json +{ + "spec": { + "family": "llama", + "base_num_parameters": 1235814400, + "hidden_size": 2048, + "num_layers": 16, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "vocab_size": 128256, + "max_sequence_length": 131072 + } +} +``` + --- ## Verify the Model Entity @@ -139,7 +141,7 @@ job = client.customization.jobs.create( ) ``` -Refer to [create-job](../manage-customization-jobs/create-job.md) for complete job creation details. +Refer to [create-job](/documentation/fine-tune-models/manage-customization-jobs/create-a-customization-job) for complete job creation details. --- @@ -167,5 +169,5 @@ else: ## Next Steps -- [Create a customization job](../manage-customization-jobs/create-job.md) -- [Understand hyperparameters](../manage-customization-jobs/hyperparameters.md) +- [Create a customization job](/documentation/fine-tune-models/manage-customization-jobs/create-a-customization-job) +- [Understand hyperparameters](/documentation/customizer-reference/manage-jobs/training-configuration) diff --git a/docs/customizer/manage-model-entities/index.md b/docs/customizer/manage-model-entities/index.mdx similarity index 80% rename from docs/customizer/manage-model-entities/index.md rename to docs/customizer/manage-model-entities/index.mdx index f499bbf44c..a9f3dd2876 100644 --- a/docs/customizer/manage-model-entities/index.md +++ b/docs/customizer/manage-model-entities/index.mdx @@ -1,25 +1,26 @@ -# Manage Model Entities for Customization - +--- +title: "Manage Model Entities for Customization" +description: "" +--- Before running a customization job, you need to set up a **Model Entity** that points to your base model checkpoint. This section covers creating the required FileSet and Model Entity. ## Task Guides -
+ -- **[Create a Model FileSet](create-fileset.md)** + - --- +Create a FileSet containing your base model checkpoint from HuggingFace, NGC, or local storage. - Create a FileSet containing your base model checkpoint from HuggingFace, NGC, or local storage. + + -- **[Create a Model Entity](create-model-entity.md)** +Create a Model Entity that references your FileSet and enables customization. - --- + - Create a Model Entity that references your FileSet and enables customization. - -
+ --- @@ -53,9 +54,11 @@ A **Model Entity** is the platform's representation of a model. It contains: Complete example of setting up a model for customization: -!!! tip - **HuggingFace Token**: If downloading from a gated HuggingFace repository (like Llama models), you will need to create a secret containing your HuggingFace API token first. Refer to [Manage Secrets](../../get-started/concepts/manage-secrets.md) for instructions. + + +**HuggingFace Token**: If downloading from a gated HuggingFace repository (like Llama models), you will need to create a secret containing your HuggingFace API token first. Refer to [Manage Secrets](/documentation/get-started/core-concepts/manage-secrets) for instructions. + ```python import os import time diff --git a/docs/customizer/models/data-format.md b/docs/customizer/models/data-format.mdx similarity index 99% rename from docs/customizer/models/data-format.md rename to docs/customizer/models/data-format.mdx index 0a276b8508..2e1b2f2d41 100644 --- a/docs/customizer/models/data-format.md +++ b/docs/customizer/models/data-format.mdx @@ -1,5 +1,8 @@ +--- +title: "Dataset Format Requirements" +description: "" +--- -# Dataset Format Requirements Use the following guidelines to prepare your training dataset for the supported models. @@ -8,7 +11,7 @@ Use the following guidelines to prepare your training dataset for the supported - **File Format**: Save your training data as `.jsonl` files (one JSON object per line). - **Validation**: Each record is automatically validated against the appropriate schema when training begins. The required format depends on the `training_type` (SFT, DPO) specified in your job configuration. -For dataset creation tutorials, refer to [Format Training Dataset](../tutorials/format-training-dataset.md). +For dataset creation tutorials, refer to [Format Training Dataset](/documentation/fine-tune-models/tutorials/format-training-dataset). ## Dataset Formats diff --git a/docs/customizer/models/embedding.md b/docs/customizer/models/embedding.mdx similarity index 85% rename from docs/customizer/models/embedding.md rename to docs/customizer/models/embedding.mdx index 7705bc1bd2..f22bb5e5a2 100644 --- a/docs/customizer/models/embedding.md +++ b/docs/customizer/models/embedding.mdx @@ -1,7 +1,10 @@ +--- +title: "Embedding Models" +description: "" +--- -# Embedding Models -This page provides detailed technical specifications for the embedding model family supported by {{ncm_short_name}}. For information about supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the embedding model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Llama Nemotron Embedding 1B v2 @@ -37,9 +40,11 @@ Create a Model Entity for this embedding model: - **LoRA (merged)**: 1x 80GB GPU, tensor parallel size 1 - **Full SFT**: 1x 80GB GPU, tensor parallel size 1 -!!! note - Embedding models only support **merged LoRA** (`peft` with `merge=True`). Unmerged LoRA adapters are not supported because the embedding NIM requires ONNX format, which cannot represent standalone adapters. + +Embedding models only support **merged LoRA** (`peft` with `merge=True`). Unmerged LoRA adapters are not supported because the embedding NIM requires ONNX format, which cannot represent standalone adapters. + + ### Resource Requirements - **Minimum GPU Memory**: 80GB @@ -71,7 +76,7 @@ NVIDIA recommends evaluating fine-tuned embedding models against the baseline to This model supports inference deployment through NVIDIA Inference Microservices (NIM). After customization, access your model through the **Inference Gateway**: -1. **Deploy the model**: Create a ModelDeploymentConfig and ModelDeployment to deploy your fine-tuned model. See [about](../../run-inference/about.md) for details. +1. **Deploy the model**: Create a ModelDeploymentConfig and ModelDeployment to deploy your fine-tuned model. See [about](/documentation/models-and-inference) for details. 2. **Access through Inference Gateway**: The Inference Gateway provides unified access to all deployed models via three routing patterns: - **Model Entity routing**: `/v2/workspaces/{workspace}/inference/gateway/model/{name}/-/v1/embeddings` @@ -90,9 +95,11 @@ client = NeMoPlatform( oai_client = client.models.get_openai_client() ``` -!!! note - The embedding model requires NIM container images that support embedding inference. When the deployment reaches `READY` state, a ModelProvider is automatically created for routing inference requests. + + +The embedding model requires NIM container images that support embedding inference. When the deployment reaches `READY` state, a ModelProvider is automatically created for routing inference requests. + ### Example Usage After fine-tuning and deployment, you can use the model for embedding tasks: @@ -127,4 +134,4 @@ print(f"Embedding dimension: {len(embedding)}") For detailed fine-tuning instructions, refer to the [Embedding Customization tutorial](../tutorials/embedding-customization-job.ipynb). -For more information about formatting training datasets for the embedding model, refer to [Dataset Format Requirements](data-format.md). +For more information about formatting training datasets for the embedding model, refer to [Dataset Format Requirements](/documentation/customizer-reference/models/dataset-format). diff --git a/docs/customizer/models/gpt-oss.md b/docs/customizer/models/gpt-oss.mdx similarity index 74% rename from docs/customizer/models/gpt-oss.md rename to docs/customizer/models/gpt-oss.mdx index cff3cabc6e..39f58491ca 100644 --- a/docs/customizer/models/gpt-oss.md +++ b/docs/customizer/models/gpt-oss.mdx @@ -1,11 +1,14 @@ +--- +title: "GPT-OSS Models" +description: "" +--- -# GPT-OSS Models -This page provides detailed technical specifications for the OpenAI GPT-OSS model family supported by {{ncm_short_name}}. For supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the OpenAI GPT-OSS model family supported by NeMo Customizer. For supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Before You Start -These models require a HuggingFace token to download. Create a secret with your HuggingFace API key, then create a FileSet and Model Entity referencing the model. See [index](../manage-model-entities/index.md) for setup instructions. +These models require a HuggingFace token to download. Create a secret with your HuggingFace API key, then create a FileSet and Model Entity referencing the model. See [index](/documentation/customizer-reference/manage-model-entities/overview) for setup instructions. --- @@ -61,11 +64,13 @@ Example: Set reasoning level using "Reasoning: high" in the system prompt. GPT-OSS models use a Mixture of Experts (MoE) architecture and benefit from specialized parallelization across expert layers for optimal performance. -!!! note - **MoE Parallelism Constraints** + + +**MoE Parallelism Constraints** - MoE models only support expert parallelism for distributing experts across GPUs. When `expert_parallel_size > 1`, `tensor_parallel_size` must be set to 1. Additionally, `expert_parallel_size` must evenly divide the number of GPUs. These constraints apply to training parallelism only and NIM deployment may use different GPU counts optimized for inference. +MoE models only support expert parallelism for distributing experts across GPUs. When `expert_parallel_size > 1`, `tensor_parallel_size` must be set to 1. Additionally, `expert_parallel_size` must evenly divide the number of GPUs. These constraints apply to training parallelism only and NIM deployment may use different GPU counts optimized for inference. + ### Model Selection Guidelines - **GPT-OSS 20B**: Ideal for lower latency requirements, local deployment, specialized use cases, and consumer hardware (with Ollama support) @@ -75,5 +80,8 @@ GPT-OSS models use a Mixture of Experts (MoE) architecture and benefit from spec Both models use the harmony response format and require this format for proper functionality. -!!! note - Sequence packing is not supported for GPT-OSS models in {{ncm_short_name}}. + + +Sequence packing is not supported for GPT-OSS models in NeMo Customizer. + + diff --git a/docs/customizer/models/index.md b/docs/customizer/models/index.mdx similarity index 57% rename from docs/customizer/models/index.md rename to docs/customizer/models/index.mdx index 6e13051327..ec9acd03c9 100644 --- a/docs/customizer/models/index.md +++ b/docs/customizer/models/index.mdx @@ -1,70 +1,69 @@ +--- +title: "Model Catalog" +description: "" +--- -# Model Catalog -Explore the model families and sizes supported by {{ncm_long_name}}. +Explore the model families and sizes supported by NVIDIA NeMo Customizer. + + -!!! note - For information on setting up model entities for customization, see the [Manage Model Entities](../manage-model-entities/index.md) guide. - For fine-tuning and deployment tutorials, see the [Tutorials](../tutorials/index.md) guide. +For information on setting up model entities for customization, see the [Manage Model Entities](/documentation/customizer-reference/manage-model-entities/overview) guide. +For fine-tuning and deployment tutorials, see the [Tutorials](/documentation/fine-tune-models/tutorials) guide. + ## Before You Start -If downloading models hosted on Hugging Face, create a secret with your HuggingFace API key, then create a FileSet and Model Entity referencing the model. See [index](../manage-model-entities/index.md) for setup instructions. +If downloading models hosted on Hugging Face, create a secret with your HuggingFace API key, then create a FileSet and Model Entity referencing the model. See [index](/documentation/customizer-reference/manage-model-entities/overview) for setup instructions. --- ## Model Families -
- -- **[Llama Models](llama.md)** - - --- - - View the available Llama models from Meta, ranging from 8 billion to 70 billion parameters. - -- **[Llama Nemotron Models](llama-nemotron.md)** - - --- - - View the available Llama Nemotron models from NVIDIA, including Nano and Super variants for efficient and advanced instruction tuning. + -- **[Phi Models](phi.md)** + - --- +View the available Llama models from Meta, ranging from 8 billion to 70 billion parameters. - View the available Phi models from Microsoft, designed for strong reasoning capabilities with efficient deployment. + + -- **[Embedding Models](embedding.md)** +View the available Llama Nemotron models from NVIDIA, including Nano and Super variants for efficient and advanced instruction tuning. - --- + + - View the available embedding models optimized for retrieval and question-answering tasks. +View the available Phi models from Microsoft, designed for strong reasoning capabilities with efficient deployment. -- **[GPT-OSS Models](gpt-oss.md)** + + - --- +View the available embedding models optimized for retrieval and question-answering tasks. - View the available GPT-OSS models supported for customization. + + -- **[Qwen Models](qwen.md)** +View the available GPT-OSS models supported for customization. - --- + + - View the available Qwen models from Alibaba Cloud, including compact variants for efficient customization. +View the available Qwen models from Alibaba Cloud, including compact variants for efficient customization. -- **[Mistral Models](mistral.md)** + + - --- +View the available Mistral models, including Mistral and Ministral variants for instruction-following and reasoning tasks. - View the available Mistral models, including Mistral and Ministral variants for instruction-following and reasoning tasks. + -
+ ## Tested Models -The following table lists models that NVIDIA tested and their available features. While {{ncm_short_name}} works with all LLM NIM microservices, the table lists the models that NVIDIA tested. Models available for fine-tuning with {{ncm_short_name}} are not limited to those listed. +The following table lists models that NVIDIA tested and their available features. While NeMo Customizer works with all LLM NIM microservices, the table lists the models that NVIDIA tested. Models available for fine-tuning with NeMo Customizer are not limited to those listed. For detailed technical specifications of each model such as architecture, parameters, and token limits, refer to the [model family](#model-families) pages. @@ -97,4 +96,4 @@ The following models support both chat and completion model training. |--|--|--| | [nvidia/llama-nemotron-embed-1b-v2](https://huggingface.co/nvidia/llama-nemotron-embed-1b-v2) | Full SFT, LoRA (merged) | Supported | -For detailed technical specifications and configuration information for embedding models, see the [Embedding Models](embedding.md) page. +For detailed technical specifications and configuration information for embedding models, see the [Embedding Models](/documentation/fine-tune-models/models/embedding) page. diff --git a/docs/customizer/models/llama-nemotron.md b/docs/customizer/models/llama-nemotron.mdx similarity index 80% rename from docs/customizer/models/llama-nemotron.md rename to docs/customizer/models/llama-nemotron.mdx index 3ed124791e..091b420d7c 100644 --- a/docs/customizer/models/llama-nemotron.md +++ b/docs/customizer/models/llama-nemotron.mdx @@ -1,7 +1,10 @@ +--- +title: "Llama Nemotron Models" +description: "" +--- -# Llama Nemotron Models -This page provides detailed technical specifications for the Nemotron model family supported by {{ncm_short_name}}. For information about supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the Nemotron model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Llama 3.1 Nemotron Nano 8B v1 @@ -84,20 +87,24 @@ This page provides detailed technical specifications for the Nemotron model fami - **LoRA**: 2x 80GB GPU, tensor parallel size 1, expert parallel size 2, pipeline parallel size 1 - **Full SFT**: 8x 80GB GPU, tensor parallel size 1, expert parallel size 8, pipeline parallel size 1 -!!! note - **MoE Parallelism Constraints** + - MoE models only support expert parallelism for distributing experts across GPUs. When `expert_parallel_size > 1`, `tensor_parallel_size` must be set to 1. Additionally, `expert_parallel_size` must evenly divide the number of GPUs. These constraints apply to training parallelism only and NIM deployment may use different GPU counts optimized for inference. +**MoE Parallelism Constraints** +MoE models only support expert parallelism for distributing experts across GPUs. When `expert_parallel_size > 1`, `tensor_parallel_size` must be set to 1. Additionally, `expert_parallel_size` must evenly divide the number of GPUs. These constraints apply to training parallelism only and NIM deployment may use different GPU counts optimized for inference. + + ### Deployment Configuration - **Full SFT**: - NIM Image: `nvcr.io/nim/nvidia/nemotron-3-nano:1.7.0-variant` - GPU Count: 2x 80GB -!!! note - Deployment for LoRA using NIM is not supported for this model. + + +Deployment for LoRA using NIM is not supported for this model. + ## NVIDIA Nemotron 3 Super 120B A12B | Property | Value | @@ -114,11 +121,13 @@ This page provides detailed technical specifications for the Nemotron model fami - **LoRA**: 8x 80GB GPU, tensor parallel size 1, expert parallel size 8, pipeline parallel size 1 -!!! note - **MoE Parallelism Constraints** + + +**MoE Parallelism Constraints** - MoE models only support expert parallelism for distributing experts across GPUs. When `expert_parallel_size > 1`, `tensor_parallel_size` must be set to 1. Additionally, `expert_parallel_size` must evenly divide the number of GPUs. These constraints apply to training parallelism only and NIM deployment may use different GPU counts optimized for inference. +MoE models only support expert parallelism for distributing experts across GPUs. When `expert_parallel_size > 1`, `tensor_parallel_size` must be set to 1. Additionally, `expert_parallel_size` must evenly divide the number of GPUs. These constraints apply to training parallelism only and NIM deployment may use different GPU counts optimized for inference. + ### Deployment Configuration - **LoRA**: diff --git a/docs/customizer/models/llama.md b/docs/customizer/models/llama.mdx similarity index 92% rename from docs/customizer/models/llama.md rename to docs/customizer/models/llama.mdx index e226b227d5..31573aa778 100644 --- a/docs/customizer/models/llama.md +++ b/docs/customizer/models/llama.mdx @@ -1,7 +1,9 @@ -# Llama Models - +--- +title: "Llama Models" +description: "" +--- -This page provides detailed technical specifications for the Llama model family supported by {{ncm_short_name}}. For information about supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the Llama model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Llama-3.2-3B Instruct diff --git a/docs/customizer/models/mistral.md b/docs/customizer/models/mistral.mdx similarity index 85% rename from docs/customizer/models/mistral.md rename to docs/customizer/models/mistral.mdx index dd0104b4f1..dc3850f9c7 100644 --- a/docs/customizer/models/mistral.md +++ b/docs/customizer/models/mistral.mdx @@ -1,7 +1,10 @@ +--- +title: "Mistral Models" +description: "" +--- -# Mistral Models -This page provides detailed technical specifications for the Mistral model family supported by {{ncm_short_name}}. For information about supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the Mistral model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Mistral-7B-Instruct-v0.3 @@ -50,9 +53,11 @@ This page provides detailed technical specifications for the Mistral model famil - **LoRA**: 1x 80GB GPU, tensor parallel size 1 - **Full SFT**: 2x 80GB GPU, tensor parallel size 1 -!!! note - Deployment using NIM is not supported for this model. + +Deployment using NIM is not supported for this model. + + ## Ministral-3-3B-Reasoning-2512 | Property | Value | @@ -71,5 +76,8 @@ This page provides detailed technical specifications for the Mistral model famil - **LoRA**: 1x 80GB GPU, tensor parallel size 1 - **Full SFT**: 2x 80GB GPU, tensor parallel size 1 -!!! note - Deployment using NIM is not supported for this model. + + +Deployment using NIM is not supported for this model. + + diff --git a/docs/customizer/models/phi.md b/docs/customizer/models/phi.mdx similarity index 81% rename from docs/customizer/models/phi.md rename to docs/customizer/models/phi.mdx index 11968d4262..caeaf972a1 100644 --- a/docs/customizer/models/phi.md +++ b/docs/customizer/models/phi.mdx @@ -1,7 +1,9 @@ -# Phi Models - +--- +title: "Phi Models" +description: "" +--- -This page provides detailed technical specifications for the Phi model family supported by {{ncm_short_name}}. For information about supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the Phi model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Microsoft Phi-4 diff --git a/docs/customizer/models/qwen.md b/docs/customizer/models/qwen.mdx similarity index 88% rename from docs/customizer/models/qwen.md rename to docs/customizer/models/qwen.mdx index 1fc506bc06..3d414a3d3d 100644 --- a/docs/customizer/models/qwen.md +++ b/docs/customizer/models/qwen.mdx @@ -1,7 +1,10 @@ +--- +title: "Qwen Models" +description: "" +--- -# Qwen Models -This page provides detailed technical specifications for the Qwen model family supported by {{ncm_short_name}}. For information about supported features and capabilities, refer to [Tested Models](index.md). +This page provides detailed technical specifications for the Qwen model family supported by NeMo Customizer. For information about supported features and capabilities, refer to [Tested Models](/documentation/customizer-reference/models/model-catalog). ## Qwen2.5-1.5B-Instruct diff --git a/docs/customizer/tutorials/_snippets/customizer-prereqs.md b/docs/customizer/tutorials/_snippets/customizer-prereqs.md deleted file mode 100644 index 61e3c3a3f7..0000000000 --- a/docs/customizer/tutorials/_snippets/customizer-prereqs.md +++ /dev/null @@ -1,32 +0,0 @@ -# NeMo Customizer Prerequisites - -??? "Platform Setup Requirements and Environment Variables" - :icon: gear - - Before starting, make sure you have: - - - {{platform_name}} installed and deployed (see [Setup](../../get-started/setup.md)) - - The `nemo-platform` Python SDK installed (`pip install nemo-platform[all]`) - - (Optional) Weights & Biases account and API key for enhanced visualization - - **Set up environment variables:** - - ```bash - # Set the base URL for {{platform_name}} - export NMP_BASE_URL="http://localhost:8080" # Or your deployed platform URL - - # Optional: Weights & Biases for experiment tracking - export WANDB_API_KEY="" - ``` - - **Initialize the SDK:** - - ```python - import os - from nemo_platform import NeMoPlatform - - client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="default", - ) - ``` diff --git a/docs/customizer/tutorials/distillation-customization-job.mdx b/docs/customizer/tutorials/distillation-customization-job.mdx new file mode 100644 index 0000000000..3553b95f6b --- /dev/null +++ b/docs/customizer/tutorials/distillation-customization-job.mdx @@ -0,0 +1,9 @@ +--- +title: "Knowledge Distillation Customization" +description: "" +--- + + diff --git a/docs/customizer/tutorials/dpo-customization-job.mdx b/docs/customizer/tutorials/dpo-customization-job.mdx new file mode 100644 index 0000000000..a6732520c8 --- /dev/null +++ b/docs/customizer/tutorials/dpo-customization-job.mdx @@ -0,0 +1,9 @@ +--- +title: "DPO Customization" +description: "" +--- + + diff --git a/docs/customizer/tutorials/embedding-customization-job.mdx b/docs/customizer/tutorials/embedding-customization-job.mdx new file mode 100644 index 0000000000..8ad23791a8 --- /dev/null +++ b/docs/customizer/tutorials/embedding-customization-job.mdx @@ -0,0 +1,9 @@ +--- +title: "Embedding Model Customization" +description: "" +--- + + diff --git a/docs/customizer/tutorials/format-training-dataset.md b/docs/customizer/tutorials/format-training-dataset.mdx similarity index 71% rename from docs/customizer/tutorials/format-training-dataset.md rename to docs/customizer/tutorials/format-training-dataset.mdx index a5e6badd28..dc675e400a 100644 --- a/docs/customizer/tutorials/format-training-dataset.md +++ b/docs/customizer/tutorials/format-training-dataset.mdx @@ -1,5 +1,8 @@ +--- +title: "Format Training Dataset" +description: "" +--- -# Format Training Dataset Learn how to format a training dataset to work with the model type you want to train, such as a **chat** or **completion** model. @@ -10,30 +13,29 @@ Customizer expects _all_ datasets to use JSONL format, where each line in the da ## Prerequisites ---8<-- "_snippets/tutorials/prereqs.md" - ---8<-- "_snippets/customizer-prereqs.md" - + ## Dataset Best Practices Before formatting your dataset, follow these principles to ensure high-quality training: -!!! tip - **Quality Principles:** + + +**Quality Principles:** - - **Quality > Quantity:** 100 high-quality examples beat 1,000 poor ones - - **Diversity:** Cover different scenarios, edge cases, and variations - - **Consistency:** Maintain uniform format, tone, and style across examples - - **Balance:** Include both common and rare cases relevant to your use case - - **Validation split:** Reserve 10-20% for validation to detect overfitting +- **Quality > Quantity:** 100 high-quality examples beat 1,000 poor ones +- **Diversity:** Cover different scenarios, edge cases, and variations +- **Consistency:** Maintain uniform format, tone, and style across examples +- **Balance:** Include both common and rare cases relevant to your use case +- **Validation split:** Reserve 10-20% for validation to detect overfitting - **Recommended dataset sizes:** +**Recommended dataset sizes:** - - **Minimum:** 50-100 examples for simple tasks - - **Typical:** 500-2,000 examples for most use cases - - **Large scale:** 10,000+ for complex domains with high variation +- **Minimum:** 50-100 examples for simple tasks +- **Typical:** 500-2,000 examples for most use cases +- **Large scale:** 10,000+ for complex domains with high variation + ### Data Quality Checklist Before uploading your dataset, verify: @@ -84,41 +86,76 @@ A conversational dataset contains a sequence of `messages` that represent intera - A `role` field to categorize the message text. Options include `system`, `user`, and `assistant`. - A `content` field for the actual body of information communicated by that role. -!!! tip - For best training results, the `assistant` role should be the last message in each training example. + -Example entry formatted for JSONL dataset file: - ---8<-- "_snippets/output/chat-basic-format-example.jsonl" - -??? "Expanded JSON example" +For best training results, the `assistant` role should be the last message in each training example. - For illustrative purposes only, we show an example entry as multi-line JSON. + +Example entry formatted for JSONL dataset file: - --8<-- "_snippets/output/chat-expanded-format-example.json" + + +For illustrative purposes only, we show an example entry as multi-line JSON. + +```json +{ + "messages": [ + { + "role": "system", + "content": "" + }, { + "role": "user", + "content": "" + }, { + "role": "assistant", + "content": "" + } + ] +} +``` + ### Reasoning Considerations -Some models (such as [Llama Nemotron](../models/llama-nemotron.md)) support a **detailed thinking** mode, which you can toggle in the system message. This setting controls whether the model is encouraged to show step-by-step reasoning in its responses. +Some models (such as [Llama Nemotron](/documentation/customizer-reference/models/llama-nemotron)) support a **detailed thinking** mode, which you can toggle in the system message. This setting controls whether the model is encouraged to show step-by-step reasoning in its responses. - **Training data without reasoning:** Use `detailed thinking off` in the system message. - **Training data with reasoning:** Use `detailed thinking on` in the system message. -!!! note - If you have an existing system message that must be preserved, prepend `detailed thinking on` or `detailed thinking off` to the beginning of your system message. + - For example, if your original system message is `You are a helpful assistant.`, you should use `detailed thinking on\nYou are a helpful assistant.` or `detailed thinking off\nYou are a helpful assistant.` +If you have an existing system message that must be preserved, prepend `detailed thinking on` or `detailed thinking off` to the beginning of your system message. +For example, if your original system message is `You are a helpful assistant.`, you should use `detailed thinking on\nYou are a helpful assistant.` or `detailed thinking off\nYou are a helpful assistant.` -=== "Detailed Thinking Off" + + - --8<-- "_snippets/output/chat-thinking-off-example.jsonl" + -=== "Detailed Thinking On" +```json +{"messages": [ + {"role": "system", "content": "detailed thinking off"}, + {"role": "user", "content": "What is 2 + 2?"}, + {"role": "assistant", "content": "4"} +]} +``` + + + - --8<-- "_snippets/output/chat-thinking-on-example.jsonl" +```json +{"messages": [ + {"role": "system", "content": "detailed thinking on"}, + {"role": "user", "content": "What is 2 + 2?"}, + {"role": "assistant", "content": "To solve 2 + 2, add 2 and 2 together. The answer is 4."} +]} +``` + + + You can adjust the system message for each training example to match the style of your data and the behavior you want the model to learn. #### Schema with Tool Calling @@ -136,14 +173,49 @@ To train a model with tool calling capabilities, use the conversational dataset Every sample must be a single line as in this example below. ---8<-- "_snippets/output/chat-tool-calling-basic-example.jsonl" - -??? "Expanded JSON example" + - For illustrative purposes only, we show an example entry as multi-line JSON. +For illustrative purposes only, we show an example entry as multi-line JSON. - --8<-- "_snippets/output/chat-tool-calling-expanded-example.json" +```json +{ + "messages": [ + { + "role": "user", + "content": "" + }, + { + "role": "assistant", + "content": "", + "tool_calls": [{ + "type": "function", + "function": { + "name": "fibonacci", + "arguments": {"n": 20} + } + }] + } + ], + "tools": [{ + "type": "function", + "function": { + "name": "fibonacci", + "description": "Calculates the nth Fibonacci number.", + "parameters": { + "type": "object", + "properties": { + "n": { + "description": "The position of the Fibonacci number.", + "type": "integer" + } + } + } + } + }] +} +``` + ##### Shared Tools When your dataset uses the same set of tools across all examples, you can streamline your configuration by omitting the `tools` field from individual dataset entries. Instead, specify these tools once in the `dataset_parameters` section during job creation. @@ -225,9 +297,11 @@ if model.spec: To run inference with a chat model, you first need to deploy the model, then use the Inference Gateway to send requests. -!!! note - Before running inference, ensure your model is deployed. See [about](../../run-inference/about.md) for details on creating a ModelDeploymentConfig and ModelDeployment. + + +Before running inference, ensure your model is deployed. See [about](/documentation/models-and-inference) for details on creating a ModelDeploymentConfig and ModelDeployment. + ```python import os from nemo_platform import NeMoPlatform @@ -276,15 +350,15 @@ Prompt completion datasets have a simple schema. Each datum has: - A `prompt` field for the body of information provided by the user. - A `completion` field for output of the model. ---8<-- "_snippets/output/completion-format-example.jsonl" - ### Prompt the Model To run inference with a completion model, use the Inference Gateway to send requests to the `/completions` endpoint. -!!! note - Before running inference, ensure your model is deployed. See [about](../../run-inference/about.md) for details on creating a ModelDeploymentConfig and ModelDeployment. + + +Before running inference, ensure your model is deployed. See [about](/documentation/models-and-inference) for details on creating a ModelDeploymentConfig and ModelDeployment. + ```python import os from nemo_platform import NeMoPlatform diff --git a/docs/customizer/tutorials/import-hf-model.md b/docs/customizer/tutorials/import-hf-model.md deleted file mode 100644 index 60d90b5aa6..0000000000 --- a/docs/customizer/tutorials/import-hf-model.md +++ /dev/null @@ -1,729 +0,0 @@ - - -# Import and Fine-Tune Private HuggingFace Models - -Use this tutorial to learn how to import a private HuggingFace model into NeMo Customizer, fine-tune it with LoRA, and deploy it for inference. - -## Prerequisites - ---8<-- "_snippets/tutorials/prereqs.md" - ---8<-- "_snippets/customizer-prereqs.md" - -### Tutorial-Specific Prerequisites - -- Access to Data Store and Deployment Manager service -- `hf` cli installed on a machine with internet access [(installation instructions)](https://huggingface.co/docs/huggingface_hub/main/en/guides/cli). -- A HuggingFace model with a **compatible architecture**. Not all HuggingFace models are compatible with NeMo Customizer. This tutorial uses `gemma-2-2b-it` as an example, but success depends on architectural compatibility. -- A HuggingFace API token and proper authentication setup. -- Sufficient storage space for the model files (typically 5-50GB depending on model size) -- At least 8GB GPU memory for smaller models, more for larger models - -!!! note - Verify that all required services are running and accessible before proceeding. You can check service health using the health endpoints documented in each service's API specification. - -### Known Issues - -!!! warning - **Conv1D Model Architecture Limitation**: Models that use `Conv1D` layers are not compatible with NeMo Customizer AutoModel LoRA. - - **Error signature**: `AttributeError: 'Conv1D' object has no attribute 'config'` - - **Affected models include:** - - - `microsoft/DialoGPT-*` series - - `openai-gpt` models - - Some older `gpt2` variants - - Other models with Conv1D-based architectures - - **Root cause**: These models use Conv1D layers that lack the linear layers expected by NeMo's LoRA transformation utilities. - - **Solution**: Use modern transformer architectures instead: - - - ✅ **Recommended**: Llama models (3.1, 3.2, 3.3 series) - - ✅ **Recommended**: Nemotron models - - ✅ **Recommended**: Phi models - - ✅ **Alternative**: Gemma models (used in this tutorial) - - For a complete list of tested models, see the [Model Catalog](../models/index.md). - ---- - -## Download Model From HuggingFace Hub - -1. Authenticate to HuggingFace using `hf auth login`. -2. Download the model. - - --8<-- "_snippets/input/common.sh" - ---- - -## Create Model in Data Store - -Next, create a model repository in the NeMo Data Store and upload the downloaded model files. - -### Create Namespace and Model Repository - -=== "Python SDK" - - --8<-- "_snippets/input/common.py" - - --8<-- "_snippets/input/common.py" - -=== "cURL" - - ```bash - # Set environment variables - Update these to match your deployment - export NAMESPACE="my-org" - export MODEL_NAME="gemma-2-2b-it" - export MODEL_VERSION="$(date +%Y%m%d-%H%M%S)" # Unique version for this run - export NEMO_BASE_URL="http://nemo.test" - export DATASTORE_URL="http://data-store.test" - export REPO_ID="${NAMESPACE}/${MODEL_NAME}" - export DATASET_NAME="${MODEL_NAME}-training-data" - - # Create namespace - curl -X POST "${DATASTORE_URL}/v1/datastore/namespaces" \ - -H 'Content-Type: application/json' \ - -d '{"namespace": "'${NAMESPACE}'"}' - - # Create model repository in datastore - curl -X POST "${DATASTORE_URL}/v1/hf/api/repos/create" \ - -H 'Content-Type: application/json' \ - -d '{ - "organization": "'${NAMESPACE}'", - "name": "'${MODEL_NAME}'", - "type": "model" - }' - ``` - -### Upload Model Files to Data Store - -Upload the downloaded model files to the Data Store repository: - ---8<-- "_snippets/input/common.py" - -## Create Model Entity in Entity Store - -After uploading the model files to the Data Store, create a model entity in the Entity Store to register the model with its metadata and specifications for use in customization jobs. - -=== "Python SDK" - - --8<-- "_snippets/input/common.py" - -=== "cURL" - - ```bash - # Create model entity in Entity Store - curl -X POST "${NEMO_BASE_URL}/v1/models" \ - -H 'Content-Type: application/json' \ - -d '{ - "name": "'${MODEL_NAME}'", - "namespace": "'${NAMESPACE}'", - "description": "Private '${MODEL_NAME}' model imported for customization", - "artifact": { - "files_url": "hf://models/'${REPO_ID}'", - "backend_engine": "hugging_face", - "status": "upload_completed" - }, - "spec": { - "num_parameters": 200000000, - "context_size": 1024, - "is_chat": true, - "num_virtual_tokens": -1 - }, - "peft": { - "finetuning_type": "all_weights" - } - }' | jq . - ``` - -## Deploy the Base Model - -Deploy the base model for inference with LoRA adapter support enabled, allowing it to load fine-tuned adapters from customization jobs. - -=== "Python SDK" - - --8<-- "_snippets/input/common.py" - -=== "cURL" - - ```bash - curl "${NEMO_BASE_URL}/v1/deployment/configs" \ - -X POST \ - -H 'Content-Type: application/json' \ - --data-binary '{ - "model": "'${MODEL_NAME}'", - "name": "'${MODEL_NAME}'-deployment-config", - "namespace": "'${NAMESPACE}'", - "nim_deployment": { - "additional_envs": { - "NIM_FT_MODEL": "", - "NIM_GUIDED_DECODING_BACKEND": "outlines", - "NIM_JSONL_LOGGING": "0", - "NIM_MODEL_NAME": "/model-store", - "NIM_PEFT_REFRESH_INTERVAL": "30", - "NIM_PEFT_SOURCE": "http://nemo-entity-store:8000", - "UVICORN_LOG_LEVEL": "DEBUG", - "VLLM_NVEXT_LOG_LEVEL": "DEBUG" - }, - "gpu": 1, - "image_name": "nvcr.io/nim/nvidia/llm-nim", - "image_tag": "1.13.1", - "disable_lora_support": false - } - }' | jq . - - - curl "${NEMO_BASE_URL}/v1/deployment/model-deployments" \ - -X POST \ - -H 'Content-Type: application/json' \ - -d '{ - "name": "'${MODEL_NAME}'-deployment", - "namespace": "'${NAMESPACE}'", - "config": "'${NAMESPACE}'/'${MODEL_NAME}'-deployment-config" - }' | jq . - - ``` - -## Create Customization Target - -Create a customization target that references the uploaded model in the Data Store. - -=== "Python SDK" - - --8<-- "_snippets/input/common.py" - -=== "cURL" - - ```bash - - # Create customization target - curl -X POST \ - "${NEMO_BASE_URL}/v1/customization/targets" \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ - "name": "'${MODEL_NAME}'@v'${MODEL_VERSION}'", - "namespace": "'${NAMESPACE}'", - "description": "Customization target for '${MODEL_NAME}'", - "enabled": true, - "model_uri": "hf://'${NAMESPACE}'/'${MODEL_NAME}'", - "num_parameters": 200000000, - "precision": "bf16-mixed" - }' | jq . - ``` - -Wait for the model to be downloaded and ready: - -=== "Python SDK" - - --8<-- "_snippets/input/common.py" - -=== "CLI" - - ```bash - # Check target status with comprehensive handling - while true; do - RESPONSE=$(curl -s -X GET \ - "${NEMO_BASE_URL}/v1/customization/targets/${NAMESPACE}/${MODEL_NAME}@v${MODEL_VERSION}" \ - -H 'accept: application/json') - - STATUS=$(echo "$RESPONSE" | jq -r '.status') - echo "Target status: $STATUS" - - if [ "$STATUS" = "ready" ]; then - echo "Model is ready for customization!" - break - elif [ "$STATUS" = "failed" ] || [ "$STATUS" = "cancelled" ] || [ "$STATUS" = "unknown" ] || [ "$STATUS" = "delete_failed" ]; then - echo "Model download failed with status: $STATUS" - echo "Contact your administrator for assistance." - break - elif [ "$STATUS" = "created" ] || [ "$STATUS" = "pending" ] || [ "$STATUS" = "downloading" ]; then - echo "Model is still being prepared, waiting..." - elif [ "$STATUS" = "deleted" ] || [ "$STATUS" = "deleting" ]; then - echo "Model is being deleted (status: $STATUS)" - echo "This target cannot be used for customization." - break - else - echo "Unknown status: $STATUS" - fi - - sleep 30 - done - - ``` - -## Create Customization Configuration - -Create a configuration for LoRA fine-tuning: - -=== "Python SDK" - - --8<-- "_snippets/input/common.py" - -=== "cURL" - - ```bash - # Create customization configuration - curl -X POST \ - "${NEMO_BASE_URL}/v1/customization/configs" \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ - "name": "'${MODEL_NAME}'-lora-config@v'${MODEL_VERSION}'", - "namespace": "'${NAMESPACE}'", - "target": "'${NAMESPACE}'/'${MODEL_NAME}'@v'${MODEL_VERSION}'", - "description": "LoRA configuration for '${MODEL_NAME}'", - "training_options": [ - { - "training_type": "sft", - "finetuning_type": "lora", - "num_gpus": 1, - "num_nodes": 1, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "micro_batch_size": 1 - } - ], - "training_precision": "bf16-mixed", - "max_seq_length": 1024, - "prompt_template": "{prompt} {completion}" - }' | jq . - ``` - ---- - -## Prepare Training and Validation Datasets - -Before starting the customization job, prepare both training and validation datasets. The validation dataset helps track training progress and reduce overfitting. - -Create datasets in JSONL format: - -```python -import json -# Note: This assumes MODEL_NAME, NAMESPACE, and hf_api are already defined from previous code blocks - -# Example training data for Gemma 2 (instruction format) -# Gemma 2 uses standard chat message format -training_data = [ - { - "messages": [ - {"role": "user", "content": "Hello, how are you?"}, - { - "role": "assistant", - "content": "I'm doing well, thank you for asking! How can I assist you today?", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "What's the weather like?"}, - { - "role": "assistant", - "content": "I don't have access to current weather data, but I can help you find weather information or discuss weather patterns.", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "Tell me a joke"}, - { - "role": "assistant", - "content": "Why don't scientists trust atoms? Because they make up everything!", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "Can you help me with coding?"}, - { - "role": "assistant", - "content": "Absolutely! I'd be happy to help you with coding. What programming language or specific problem are you working on?", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "What is machine learning?"}, - { - "role": "assistant", - "content": "Machine learning is a subset of artificial intelligence that enables computers to learn and make decisions from data without being explicitly programmed.", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "How do I write a function in Python?"}, - { - "role": "assistant", - "content": "To write a function in Python, use the 'def' keyword followed by the function name and parameters: def my_function(param): return result", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "What are the benefits of exercise?"}, - { - "role": "assistant", - "content": "Exercise offers numerous benefits including improved cardiovascular health, stronger muscles, better mental health, and increased energy levels.", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "Explain the solar system"}, - { - "role": "assistant", - "content": "The solar system consists of the Sun and all celestial objects that orbit it, including eight planets, moons, asteroids, and comets.", - }, - ] - }, -] - -# Expand training data by repeating examples to create a larger dataset -training_data = training_data * 10 # 80 total examples (8 unique * 10 repetitions) - -# Save training data to JSONL file -with open("training_data.jsonl", "w") as f: - for item in training_data: - f.write(json.dumps(item) + "\n") - -# Create dataset in datastore (similar to model creation) -dataset_name = f"{MODEL_NAME}-training-data" -dataset_repo_id = f"{NAMESPACE}/{dataset_name}" - -# Create dataset repository -hf_api.create_repo(dataset_repo_id, repo_type="dataset", exist_ok=True) - -# Upload training data -hf_api.upload_file( - path_or_fileobj="training_data.jsonl", - path_in_repo="training.jsonl", - repo_id=dataset_repo_id, - repo_type="dataset", - commit_message="Upload training data", -) - -validation_data = [ - { - "messages": [ - {"role": "user", "content": "How do you stay motivated?"}, - { - "role": "assistant", - "content": "Staying motivated involves setting clear goals, celebrating small wins, and maintaining a positive mindset even when facing challenges.", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "What's the capital of France?"}, - { - "role": "assistant", - "content": "The capital of France is Paris, a beautiful city known for its art, culture, and iconic landmarks like the Eiffel Tower.", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "Explain quantum physics"}, - { - "role": "assistant", - "content": "Quantum physics is the branch of physics that studies matter and energy at the smallest scales, where particles behave in ways that seem strange compared to our everyday experience.", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "What is photosynthesis?"}, - { - "role": "assistant", - "content": "Photosynthesis is the process by which plants convert sunlight, carbon dioxide, and water into glucose and oxygen, providing energy for the plant.", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "How does the internet work?"}, - { - "role": "assistant", - "content": "The internet works through a global network of interconnected computers that communicate using standardized protocols to share information and resources.", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "What is artificial intelligence?"}, - { - "role": "assistant", - "content": "Artificial intelligence is the simulation of human intelligence in machines that are programmed to think, learn, and solve problems like humans.", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "Describe the water cycle"}, - { - "role": "assistant", - "content": "The water cycle is the continuous movement of water through evaporation, condensation, precipitation, and collection that sustains life on Earth.", - }, - ] - }, - { - "messages": [ - {"role": "user", "content": "What are renewable energy sources?"}, - { - "role": "assistant", - "content": "Renewable energy sources include solar, wind, hydroelectric, geothermal, and biomass energy that can be naturally replenished and don't deplete over time.", - }, - ] - }, -] - -# Expand validation data by repeating examples to create a larger dataset -validation_data = validation_data * 10 # 80 total examples (8 unique * 10 repetitions) - -# Save validation data to JSONL file -with open("validation_data.jsonl", "w") as f: - for item in validation_data: - f.write(json.dumps(item) + "\n") - -# Upload validation data -hf_api.upload_file( - path_or_fileobj="validation_data.jsonl", - path_in_repo="validation.jsonl", - repo_id=dataset_repo_id, - repo_type="dataset", - commit_message="Upload validation data", -) - -# Create dataset entity in Entity Store -dataset = client.datasets.create( - name=dataset_name, namespace=NAMESPACE, files_url=f"hf://datasets/{dataset_repo_id}" -) -print(f"Created dataset entity: {dataset.namespace}/{dataset.name}") -``` - ---- - -## Start Customization Job - -Start the LoRA fine-tuning job. The job will create an output artifact with the name specified in `output`, which you'll use later to access your fine-tuned model for inference. - -=== "Python SDK" - - --8<-- "_snippets/input/common.py" - -=== "cURL" - - ```bash - # Create job and capture job ID - RESPONSE=$(curl -s -X POST \ - "${NEMO_BASE_URL}/v1/customization/jobs" \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ - "name": "'${MODEL_NAME}'-lora-job", - "config": "'${NAMESPACE}'/'${MODEL_NAME}'-lora-config@v'${MODEL_VERSION}'", - "dataset": "'${NAMESPACE}'/'${DATASET_NAME}'", - "output": {"name": "'${NAMESPACE}'/'${MODEL_NAME}'-lora@v'${MODEL_VERSION}'"}, - "description": "LoRA fine-tuning job for '${MODEL_NAME}'", - "training": { - "type": "sft", - "peft": { - "type": "lora", - "rank": 16, - "alpha": 32, - "dropout": 0.01 - }, - "epochs": 3, - "batch_size": 8, - "learning_rate": 5e-5 - } - }') - - JOB_ID=$(echo "$RESPONSE" | jq -r '.id') - OUTPUT_NAME=$(echo "$RESPONSE" | jq -r '.spec.output.name') - echo "Started job with ID: $JOB_ID" - echo "Output name: $OUTPUT_NAME" - ``` - -Copy the following values from the response: - -- `id` (Job ID) -- `spec.output.name` - -We'll need them later to monitor the job's status and access the fine-tuned model. - -Check job progress: - -=== "Python SDK" - - --8<-- "_snippets/input/job-monitoring.py" - -=== "cURL" - - ```bash - # Monitor job status with comprehensive handling - while true; do - RESPONSE=$(curl -s -X GET \ - "${NEMO_BASE_URL}/v1/customization/jobs/${JOB_ID}" \ - -H 'accept: application/json') - - STATUS=$(echo "$RESPONSE" | jq -r '.status') - echo "Job status: $STATUS" - - if [ "$STATUS" = "completed" ]; then - echo "Training completed successfully!" - break - elif [ "$STATUS" = "failed" ] || [ "$STATUS" = "cancelled" ]; then - echo "Training finished with status: $STATUS" - if [ "$STATUS" = "failed" ]; then - echo "Check the job logs for error details." - fi - break - elif [ "$STATUS" = "created" ] || [ "$STATUS" = "pending" ]; then - echo "Job is queued and waiting to start..." - elif [ "$STATUS" = "running" ]; then - echo "Training is in progress..." - # Optionally show progress if available - PROGRESS=$(echo "$RESPONSE" | jq -r '.status_details.percentage_done // "N/A"') - if [ "$PROGRESS" != "N/A" ] && [ "$PROGRESS" != "null" ]; then - echo "Progress: ${PROGRESS}%" - fi - elif [ "$STATUS" = "cancelling" ]; then - echo "Job is being cancelled..." - elif [ "$STATUS" = "ready" ] || [ "$STATUS" = "unknown" ]; then - echo "Job finished with status: $STATUS" - break - else - echo "Unknown status: $STATUS" - fi - - sleep 60 # Wait 1 minute before checking again - done - ``` - ---- - -## Test the Deployed Model - -After the customization job has been completed, you can use the `output.name` to access the fine-tuned model and evaluate its performance. The base model NIM deployment you created earlier will automatically load the LoRA adapter when you specify the LoRA model ID in your inference requests. - -!!! note - The inference endpoints use the `inference_base_url` configured during client initialization (typically the NIM proxy URL). The base model deployment must be running before you can test inference with LoRA adapters. - -!!! tip - If you included a WandB API key, you can view your training results at [wandb.ai](https://wandb.ai/home) under the `nvidia-nemo-customizer` project. - -=== "Python SDK" - - ```python - base_model_id = f"{NAMESPACE}/{MODEL_NAME}" - - # Option 1: If you still have the job object from creation - # lora_model_id = job.spec.output.name - - # Option 2: Construct from job parameters (use this if running in a new session) - lora_model_id = f"{NAMESPACE}/{MODEL_NAME}-lora@v{MODEL_VERSION}" - - # First, check if the models are available for inference - # Note: The base model deployment must be running and registered with the NIM proxy - try: - # List models from the entity store (shows all registered models) - models_response = client.models.list() - available_models = models_response.data - - print("Registered models:") - for model in available_models: - print(f" - {model.id}") - - # Test base model - print(f"\nTesting base model: {base_model_id}") - base_response = client.chat.completions.create( - model=base_model_id, - messages=[ - { - "role": "user", - "content": "Hello, can you help me?" - } - ], - max_tokens=100, - temperature=0.7 - ) - - print("Base model response:") - print(base_response.choices[0].message.content) - - # Test LoRA-adapted model (if available) - print(f"\nTesting LoRA-adapted model: {lora_model_id}") - lora_response = client.chat.completions.create( - model=lora_model_id, - messages=[ - { - "role": "user", - "content": "Hello, can you help me?" - } - ], - max_tokens=100, - temperature=0.7 - ) - - print("LoRA-adapted model response:") - print(lora_response.choices[0].message.content) - - except Exception as e: - print(f"Error testing model: {e}") - ``` - -=== "cURL" - - ```bash - # Set the NIM proxy URL - export NIM_PROXY_URL="http://nim.test" - - # Option 1: If you captured OUTPUT_MODEL from job creation (from earlier in tutorial) - # export OUTPUT_MODEL="" - - # Option 2: Construct from environment variables (use this if running in a new session) - export LORA_MODEL_ID="${NAMESPACE}/${MODEL_NAME}-lora@v${MODEL_VERSION}" - - # Test model availability - curl -X GET "${NIM_PROXY_URL}/models" | jq - - # Test inference against the base model - curl -X POST "${NIM_PROXY_URL}/v1/chat/completions" \ - -H 'Content-Type: application/json' \ - -d '{ - "model": "'${NAMESPACE}'/'${MODEL_NAME}'", - "messages": [ - {"role": "user", "content": "Hello! How are you?"}, - {"role": "assistant", "content": "Hi! I am quite well, how can I help you today?"}, - {"role": "user", "content": "Can you write me a song?"} - ], - "top_p": 1, - "n": 1, - "max_tokens": 50, - "frequency_penalty": 1.0 - }' - - # Test inference against a LoRA-adapted model (after customization completes) - curl -X POST "${NIM_PROXY_URL}/v1/chat/completions" \ - -H 'Content-Type: application/json' \ - -d '{ - "model": "'${LORA_MODEL_ID}'", - "messages": [ - {"role": "user", "content": "Hello! How are you?"}, - {"role": "assistant", "content": "Hi! I am quite well, how can I help you today?"}, - {"role": "user", "content": "Can you write me a song?"} - ], - "top_p": 1, - "n": 1, - "max_tokens": 50, - "frequency_penalty": 1.0 - }' - ``` - -## Next Steps - -Learn how to [check customization job metrics](metrics.md) to monitor the training progress and performance of your fine-tuned model. diff --git a/docs/customizer/tutorials/import-hf-model.mdx b/docs/customizer/tutorials/import-hf-model.mdx new file mode 100644 index 0000000000..a4a43ba4f8 --- /dev/null +++ b/docs/customizer/tutorials/import-hf-model.mdx @@ -0,0 +1,688 @@ +--- +title: "Import and Fine-Tune Private HuggingFace Models" +description: "" +--- + + +Use this tutorial to learn how to import a private HuggingFace model into NeMo Customizer, fine-tune it with LoRA, and deploy it for inference. + +## Prerequisites + + +### Tutorial-Specific Prerequisites + +- Access to Data Store and Deployment Manager service +- `hf` cli installed on a machine with internet access [(installation instructions)](https://huggingface.co/docs/huggingface_hub/main/en/guides/cli). +- A HuggingFace model with a **compatible architecture**. Not all HuggingFace models are compatible with NeMo Customizer. This tutorial uses `gemma-2-2b-it` as an example, but success depends on architectural compatibility. +- A HuggingFace API token and proper authentication setup. +- Sufficient storage space for the model files (typically 5-50GB depending on model size) +- At least 8GB GPU memory for smaller models, more for larger models + + + +Verify that all required services are running and accessible before proceeding. You can check service health using the health endpoints documented in each service's API specification. + + +### Known Issues + + + +**Conv1D Model Architecture Limitation**: Models that use `Conv1D` layers are not compatible with NeMo Customizer AutoModel LoRA. + +**Error signature**: `AttributeError: 'Conv1D' object has no attribute 'config'` + +**Affected models include:** + +- `microsoft/DialoGPT-*` series +- `openai-gpt` models +- Some older `gpt2` variants +- Other models with Conv1D-based architectures + +**Root cause**: These models use Conv1D layers that lack the linear layers expected by NeMo's LoRA transformation utilities. + +**Solution**: Use modern transformer architectures instead: + +- ✅ **Recommended**: Llama models (3.1, 3.2, 3.3 series) +- ✅ **Recommended**: Nemotron models +- ✅ **Recommended**: Phi models +- ✅ **Alternative**: Gemma models (used in this tutorial) + +For a complete list of tested models, see the [Model Catalog](/documentation/customizer-reference/models/model-catalog). + + +--- + +## Download Model From HuggingFace Hub + +1. Authenticate to HuggingFace using `hf auth login`. +2. Download the model. + +--- + +## Create Model in Data Store + +Next, create a model repository in the NeMo Data Store and upload the downloaded model files. + +### Create Namespace and Model Repository + +```bash +# Set environment variables - Update these to match your deployment +export NAMESPACE="my-org" +export MODEL_NAME="gemma-2-2b-it" +export MODEL_VERSION="$(date +%Y%m%d-%H%M%S)" # Unique version for this run +export NEMO_BASE_URL="http://nemo.test" +export DATASTORE_URL="http://data-store.test" +export REPO_ID="${NAMESPACE}/${MODEL_NAME}" +export DATASET_NAME="${MODEL_NAME}-training-data" + +# Create namespace +curl -X POST "${DATASTORE_URL}/v1/datastore/namespaces" \ +-H 'Content-Type: application/json' \ +-d '{"namespace": "'${NAMESPACE}'"}' + +# Create model repository in datastore +curl -X POST "${DATASTORE_URL}/v1/hf/api/repos/create" \ +-H 'Content-Type: application/json' \ +-d '{ +"organization": "'${NAMESPACE}'", +"name": "'${MODEL_NAME}'", +"type": "model" +}' +``` + +### Upload Model Files to Data Store + +Upload the downloaded model files to the Data Store repository: + +## Create Model Entity in Entity Store + +After uploading the model files to the Data Store, create a model entity in the Entity Store to register the model with its metadata and specifications for use in customization jobs. + +```bash +# Create model entity in Entity Store +curl -X POST "${NEMO_BASE_URL}/v1/models" \ +-H 'Content-Type: application/json' \ +-d '{ +"name": "'${MODEL_NAME}'", +"namespace": "'${NAMESPACE}'", +"description": "Private '${MODEL_NAME}' model imported for customization", +"artifact": { +"files_url": "hf://models/'${REPO_ID}'", +"backend_engine": "hugging_face", +"status": "upload_completed" +}, +"spec": { +"num_parameters": 200000000, +"context_size": 1024, +"is_chat": true, +"num_virtual_tokens": -1 +}, +"peft": { +"finetuning_type": "all_weights" +} +}' | jq . +``` + +## Deploy the Base Model + +Deploy the base model for inference with LoRA adapter support enabled, allowing it to load fine-tuned adapters from customization jobs. + +```bash +curl "${NEMO_BASE_URL}/v1/deployment/configs" \ +-X POST \ +-H 'Content-Type: application/json' \ +--data-binary '{ +"model": "'${MODEL_NAME}'", +"name": "'${MODEL_NAME}'-deployment-config", +"namespace": "'${NAMESPACE}'", +"nim_deployment": { +"additional_envs": { +"NIM_FT_MODEL": "", +"NIM_GUIDED_DECODING_BACKEND": "outlines", +"NIM_JSONL_LOGGING": "0", +"NIM_MODEL_NAME": "/model-store", +"NIM_PEFT_REFRESH_INTERVAL": "30", +"NIM_PEFT_SOURCE": "http://nemo-entity-store:8000", +"UVICORN_LOG_LEVEL": "DEBUG", +"VLLM_NVEXT_LOG_LEVEL": "DEBUG" +}, +"gpu": 1, +"image_name": "nvcr.io/nim/nvidia/llm-nim", +"image_tag": "1.13.1", +"disable_lora_support": false +} +}' | jq . + + +curl "${NEMO_BASE_URL}/v1/deployment/model-deployments" \ +-X POST \ +-H 'Content-Type: application/json' \ +-d '{ +"name": "'${MODEL_NAME}'-deployment", +"namespace": "'${NAMESPACE}'", +"config": "'${NAMESPACE}'/'${MODEL_NAME}'-deployment-config" +}' | jq . + +``` + +## Create Customization Target + +Create a customization target that references the uploaded model in the Data Store. + +```bash + +# Create customization target +curl -X POST \ +"${NEMO_BASE_URL}/v1/customization/targets" \ +-H 'accept: application/json' \ +-H 'Content-Type: application/json' \ +-d '{ +"name": "'${MODEL_NAME}'@v'${MODEL_VERSION}'", +"namespace": "'${NAMESPACE}'", +"description": "Customization target for '${MODEL_NAME}'", +"enabled": true, +"model_uri": "hf://'${NAMESPACE}'/'${MODEL_NAME}'", +"num_parameters": 200000000, +"precision": "bf16-mixed" +}' | jq . +``` + +Wait for the model to be downloaded and ready: + +```bash +# Check target status with comprehensive handling +while true; do +RESPONSE=$(curl -s -X GET \ +"${NEMO_BASE_URL}/v1/customization/targets/${NAMESPACE}/${MODEL_NAME}@v${MODEL_VERSION}" \ +-H 'accept: application/json') + +STATUS=$(echo "$RESPONSE" | jq -r '.status') +echo "Target status: $STATUS" + +if [ "$STATUS" = "ready" ]; then +echo "Model is ready for customization!" +break +elif [ "$STATUS" = "failed" ] || [ "$STATUS" = "cancelled" ] || [ "$STATUS" = "unknown" ] || [ "$STATUS" = "delete_failed" ]; then +echo "Model download failed with status: $STATUS" +echo "Contact your administrator for assistance." +break +elif [ "$STATUS" = "created" ] || [ "$STATUS" = "pending" ] || [ "$STATUS" = "downloading" ]; then +echo "Model is still being prepared, waiting..." +elif [ "$STATUS" = "deleted" ] || [ "$STATUS" = "deleting" ]; then +echo "Model is being deleted (status: $STATUS)" +echo "This target cannot be used for customization." +break +else +echo "Unknown status: $STATUS" +fi + +sleep 30 +done + +``` + +## Create Customization Configuration + +Create a configuration for LoRA fine-tuning: + +```bash +# Create customization configuration +curl -X POST \ +"${NEMO_BASE_URL}/v1/customization/configs" \ +-H 'accept: application/json' \ +-H 'Content-Type: application/json' \ +-d '{ +"name": "'${MODEL_NAME}'-lora-config@v'${MODEL_VERSION}'", +"namespace": "'${NAMESPACE}'", +"target": "'${NAMESPACE}'/'${MODEL_NAME}'@v'${MODEL_VERSION}'", +"description": "LoRA configuration for '${MODEL_NAME}'", +"training_options": [ +{ +"training_type": "sft", +"finetuning_type": "lora", +"num_gpus": 1, +"num_nodes": 1, +"tensor_parallel_size": 1, +"pipeline_parallel_size": 1, +"micro_batch_size": 1 +} +], +"training_precision": "bf16-mixed", +"max_seq_length": 1024, +"prompt_template": "{prompt} {completion}" +}' | jq . +``` + +--- + +## Prepare Training and Validation Datasets + +Before starting the customization job, prepare both training and validation datasets. The validation dataset helps track training progress and reduce overfitting. + +Create datasets in JSONL format: + +```python +import json +# Note: This assumes MODEL_NAME, NAMESPACE, and hf_api are already defined from previous code blocks + +# Example training data for Gemma 2 (instruction format) +# Gemma 2 uses standard chat message format +training_data = [ + { + "messages": [ + {"role": "user", "content": "Hello, how are you?"}, + { + "role": "assistant", + "content": "I'm doing well, thank you for asking! How can I assist you today?", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "What's the weather like?"}, + { + "role": "assistant", + "content": "I don't have access to current weather data, but I can help you find weather information or discuss weather patterns.", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "Tell me a joke"}, + { + "role": "assistant", + "content": "Why don't scientists trust atoms? Because they make up everything!", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "Can you help me with coding?"}, + { + "role": "assistant", + "content": "Absolutely! I'd be happy to help you with coding. What programming language or specific problem are you working on?", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "What is machine learning?"}, + { + "role": "assistant", + "content": "Machine learning is a subset of artificial intelligence that enables computers to learn and make decisions from data without being explicitly programmed.", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "How do I write a function in Python?"}, + { + "role": "assistant", + "content": "To write a function in Python, use the 'def' keyword followed by the function name and parameters: def my_function(param): return result", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "What are the benefits of exercise?"}, + { + "role": "assistant", + "content": "Exercise offers numerous benefits including improved cardiovascular health, stronger muscles, better mental health, and increased energy levels.", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "Explain the solar system"}, + { + "role": "assistant", + "content": "The solar system consists of the Sun and all celestial objects that orbit it, including eight planets, moons, asteroids, and comets.", + }, + ] + }, +] + +# Expand training data by repeating examples to create a larger dataset +training_data = training_data * 10 # 80 total examples (8 unique * 10 repetitions) + +# Save training data to JSONL file +with open("training_data.jsonl", "w") as f: + for item in training_data: + f.write(json.dumps(item) + "\n") + +# Create dataset in datastore (similar to model creation) +dataset_name = f"{MODEL_NAME}-training-data" +dataset_repo_id = f"{NAMESPACE}/{dataset_name}" + +# Create dataset repository +hf_api.create_repo(dataset_repo_id, repo_type="dataset", exist_ok=True) + +# Upload training data +hf_api.upload_file( + path_or_fileobj="training_data.jsonl", + path_in_repo="training.jsonl", + repo_id=dataset_repo_id, + repo_type="dataset", + commit_message="Upload training data", +) + +validation_data = [ + { + "messages": [ + {"role": "user", "content": "How do you stay motivated?"}, + { + "role": "assistant", + "content": "Staying motivated involves setting clear goals, celebrating small wins, and maintaining a positive mindset even when facing challenges.", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "What's the capital of France?"}, + { + "role": "assistant", + "content": "The capital of France is Paris, a beautiful city known for its art, culture, and iconic landmarks like the Eiffel Tower.", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "Explain quantum physics"}, + { + "role": "assistant", + "content": "Quantum physics is the branch of physics that studies matter and energy at the smallest scales, where particles behave in ways that seem strange compared to our everyday experience.", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "What is photosynthesis?"}, + { + "role": "assistant", + "content": "Photosynthesis is the process by which plants convert sunlight, carbon dioxide, and water into glucose and oxygen, providing energy for the plant.", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "How does the internet work?"}, + { + "role": "assistant", + "content": "The internet works through a global network of interconnected computers that communicate using standardized protocols to share information and resources.", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "What is artificial intelligence?"}, + { + "role": "assistant", + "content": "Artificial intelligence is the simulation of human intelligence in machines that are programmed to think, learn, and solve problems like humans.", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "Describe the water cycle"}, + { + "role": "assistant", + "content": "The water cycle is the continuous movement of water through evaporation, condensation, precipitation, and collection that sustains life on Earth.", + }, + ] + }, + { + "messages": [ + {"role": "user", "content": "What are renewable energy sources?"}, + { + "role": "assistant", + "content": "Renewable energy sources include solar, wind, hydroelectric, geothermal, and biomass energy that can be naturally replenished and don't deplete over time.", + }, + ] + }, +] + +# Expand validation data by repeating examples to create a larger dataset +validation_data = validation_data * 10 # 80 total examples (8 unique * 10 repetitions) + +# Save validation data to JSONL file +with open("validation_data.jsonl", "w") as f: + for item in validation_data: + f.write(json.dumps(item) + "\n") + +# Upload validation data +hf_api.upload_file( + path_or_fileobj="validation_data.jsonl", + path_in_repo="validation.jsonl", + repo_id=dataset_repo_id, + repo_type="dataset", + commit_message="Upload validation data", +) + +# Create dataset entity in Entity Store +dataset = client.datasets.create( + name=dataset_name, namespace=NAMESPACE, files_url=f"hf://datasets/{dataset_repo_id}" +) +print(f"Created dataset entity: {dataset.namespace}/{dataset.name}") +``` + +--- + +## Start Customization Job + +Start the LoRA fine-tuning job. The job will create an output artifact with the name specified in `output`, which you'll use later to access your fine-tuned model for inference. + +```bash +# Create job and capture job ID +RESPONSE=$(curl -s -X POST \ +"${NEMO_BASE_URL}/v1/customization/jobs" \ +-H 'accept: application/json' \ +-H 'Content-Type: application/json' \ +-d '{ +"name": "'${MODEL_NAME}'-lora-job", +"config": "'${NAMESPACE}'/'${MODEL_NAME}'-lora-config@v'${MODEL_VERSION}'", +"dataset": "'${NAMESPACE}'/'${DATASET_NAME}'", +"output": {"name": "'${NAMESPACE}'/'${MODEL_NAME}'-lora@v'${MODEL_VERSION}'"}, +"description": "LoRA fine-tuning job for '${MODEL_NAME}'", +"training": { +"type": "sft", +"peft": { +"type": "lora", +"rank": 16, +"alpha": 32, +"dropout": 0.01 +}, +"epochs": 3, +"batch_size": 8, +"learning_rate": 5e-5 +} +}') + +JOB_ID=$(echo "$RESPONSE" | jq -r '.id') +OUTPUT_NAME=$(echo "$RESPONSE" | jq -r '.spec.output.name') +echo "Started job with ID: $JOB_ID" +echo "Output name: $OUTPUT_NAME" +``` + +Copy the following values from the response: + +- `id` (Job ID) +- `spec.output.name` + +We'll need them later to monitor the job's status and access the fine-tuned model. + +Check job progress: + +```bash +# Monitor job status with comprehensive handling +while true; do +RESPONSE=$(curl -s -X GET \ +"${NEMO_BASE_URL}/v1/customization/jobs/${JOB_ID}" \ +-H 'accept: application/json') + +STATUS=$(echo "$RESPONSE" | jq -r '.status') +echo "Job status: $STATUS" + +if [ "$STATUS" = "completed" ]; then +echo "Training completed successfully!" +break +elif [ "$STATUS" = "failed" ] || [ "$STATUS" = "cancelled" ]; then +echo "Training finished with status: $STATUS" +if [ "$STATUS" = "failed" ]; then +echo "Check the job logs for error details." +fi +break +elif [ "$STATUS" = "created" ] || [ "$STATUS" = "pending" ]; then +echo "Job is queued and waiting to start..." +elif [ "$STATUS" = "running" ]; then +echo "Training is in progress..." +# Optionally show progress if available +PROGRESS=$(echo "$RESPONSE" | jq -r '.status_details.percentage_done // "N/A"') +if [ "$PROGRESS" != "N/A" ] && [ "$PROGRESS" != "null" ]; then +echo "Progress: ${PROGRESS}%" +fi +elif [ "$STATUS" = "cancelling" ]; then +echo "Job is being cancelled..." +elif [ "$STATUS" = "ready" ] || [ "$STATUS" = "unknown" ]; then +echo "Job finished with status: $STATUS" +break +else +echo "Unknown status: $STATUS" +fi + +sleep 60 # Wait 1 minute before checking again +done +``` + +--- + +## Test the Deployed Model + +After the customization job has been completed, you can use the `output.name` to access the fine-tuned model and evaluate its performance. The base model NIM deployment you created earlier will automatically load the LoRA adapter when you specify the LoRA model ID in your inference requests. + + + +The inference endpoints use the `inference_base_url` configured during client initialization (typically the NIM proxy URL). The base model deployment must be running before you can test inference with LoRA adapters. + + + + +If you included a WandB API key, you can view your training results at [wandb.ai](https://wandb.ai/home) under the `nvidia-nemo-customizer` project. + + + + + + +```python +base_model_id = f"{NAMESPACE}/{MODEL_NAME}" + +# Option 1: If you still have the job object from creation +# lora_model_id = job.spec.output.name + +# Option 2: Construct from job parameters (use this if running in a new session) +lora_model_id = f"{NAMESPACE}/{MODEL_NAME}-lora@v{MODEL_VERSION}" + +# First, check if the models are available for inference +# Note: The base model deployment must be running and registered with the NIM proxy +try: + # List models from the entity store (shows all registered models) + models_response = client.models.list() + available_models = models_response.data + + print("Registered models:") + for model in available_models: + print(f" - {model.id}") + + # Test base model + print(f"\nTesting base model: {base_model_id}") + base_response = client.chat.completions.create( + model=base_model_id, + messages=[ + { + "role": "user", + "content": "Hello, can you help me?" + } + ], + max_tokens=100, + temperature=0.7 + ) + + print("Base model response:") + print(base_response.choices[0].message.content) + + # Test LoRA-adapted model (if available) + print(f"\nTesting LoRA-adapted model: {lora_model_id}") + lora_response = client.chat.completions.create( + model=lora_model_id, + messages=[ + { + "role": "user", + "content": "Hello, can you help me?" + } + ], + max_tokens=100, + temperature=0.7 + ) + + print("LoRA-adapted model response:") + print(lora_response.choices[0].message.content) + +except Exception as e: + print(f"Error testing model: {e}") +``` + + + + +```bash +# Set the NIM proxy URL +export NIM_PROXY_URL="http://nim.test" + +# Option 1: If you captured OUTPUT_MODEL from job creation (from earlier in tutorial) +# export OUTPUT_MODEL="" + +# Option 2: Construct from environment variables (use this if running in a new session) +export LORA_MODEL_ID="${NAMESPACE}/${MODEL_NAME}-lora@v${MODEL_VERSION}" + +# Test model availability +curl -X GET "${NIM_PROXY_URL}/models" | jq + +# Test inference against the base model +curl -X POST "${NIM_PROXY_URL}/v1/chat/completions" \ +-H 'Content-Type: application/json' \ +-d '{ +"model": "'${NAMESPACE}'/'${MODEL_NAME}'", +"messages": [ +{"role": "user", "content": "Hello! How are you?"}, +{"role": "assistant", "content": "Hi! I am quite well, how can I help you today?"}, +{"role": "user", "content": "Can you write me a song?"} +], +"top_p": 1, +"n": 1, +"max_tokens": 50, +"frequency_penalty": 1.0 +}' + +# Test inference against a LoRA-adapted model (after customization completes) +curl -X POST "${NIM_PROXY_URL}/v1/chat/completions" \ +-H 'Content-Type: application/json' \ +-d '{ +"model": "'${LORA_MODEL_ID}'", +"messages": [ +{"role": "user", "content": "Hello! How are you?"}, +{"role": "assistant", "content": "Hi! I am quite well, how can I help you today?"}, +{"role": "user", "content": "Can you write me a song?"} +], +"top_p": 1, +"n": 1, +"max_tokens": 50, +"frequency_penalty": 1.0 +}' +``` + + + + +## Next Steps + +Learn how to [check customization job metrics](/documentation/customizer-reference/tutorials/job-metrics) to monitor the training progress and performance of your fine-tuned model. diff --git a/docs/customizer/tutorials/index.md b/docs/customizer/tutorials/index.md deleted file mode 100644 index c6fe716138..0000000000 --- a/docs/customizer/tutorials/index.md +++ /dev/null @@ -1,103 +0,0 @@ -# Fine-Tuning Tutorials - -Use the tutorials in this section to gain a deeper understanding of how the NVIDIA NeMo Customizer microservice enables fine-tuning tasks. - -!!! tip "Tutorials are organized by complexity and typically build on one another. The tutorials reference `NMP_BASE_URL`, which is the base URL of your {{platform_name}} deployment. Refer to the [Setup guide](../../get-started/setup.md) for installation, setup, and platform URL guidance." - ---- - -## Getting Started - -
- -- **[Understanding Model Entities and Adapters](understand-configurations-and-models.md)** - - --- - - Learn the fundamentals of how NeMo Customizer works with Model Entities and Adapters, and how to choose the right approach for your project. - - model-entities adapters training-types - -
- -## Dataset Preparation - -
- -- **[Format Training Datasets](format-training-dataset.md)** - - --- - - Learn how to format datasets for different model types. - - datasets chat-models completion-models - -
- -## Customization Jobs - -
- -- **[Fine-Tune a Model with Custom Data Using LoRA](lora-customization-job.ipynb)** - - --- - - Learn how to perform supervised fine-tuning with LoRA adapters using custom data. - - nemo-customizer - -- **[Fine-Tune a Model with Custom Data Processing All Weights](sft-customization-job.ipynb)** - - --- - - Learn how to perform supervised fine-tuning using custom data by modifying the all training parameters. - - nemo-customizer - -- **[Align a Model with DPO and Preference Data](dpo-customization-job.ipynb)** - - --- - - Learn how to align a model with DPO (Direct Preference Optimization) to prefer certain kinds of responses over others. - - nemo-customizer dpo - -- **[Distill a Large Model into a Smaller One with Knowledge Distillation](distillation-customization-job.ipynb)** - - --- - - Learn how to compress a larger teacher model into a smaller student model using knowledge distillation. - - nemo-customizer knowledge-distillation - -- **[Fine-Tune an Embedding Model With Positive and Negative Samples Using LoRA](embedding-customization-job.ipynb)** - - --- - - Learn how to fine-tune embedding models using LoRA merged training for improved question-answering and retrieval tasks. - - embedding-models lora-merged nemo-customizer - -
- -## Monitoring & Optimization - -
- -- **[Check Customization Job Metrics](metrics.md)** - - --- - - Learn how to check job metrics using MLflow or Weights & Biases. - - nemo-customizer mlflow wandb - -- **[Optimize Tokens per GPU](optimize-throughput.ipynb)** - - --- - - Learn how to optimize the token-per-GPU throughput for a LoRA optimization job. - - nemo-customizer wandb sequence-packing - -
diff --git a/docs/customizer/tutorials/index.mdx b/docs/customizer/tutorials/index.mdx new file mode 100644 index 0000000000..f24ff9c074 --- /dev/null +++ b/docs/customizer/tutorials/index.mdx @@ -0,0 +1,104 @@ +--- +title: "Fine-Tuning Tutorials" +description: "" +--- +Use the tutorials in this section to gain a deeper understanding of how the NVIDIA NeMo Customizer microservice enables fine-tuning tasks. + + + +Tutorials are organized by complexity and typically build on one another. The tutorials reference `NMP_BASE_URL`, which is the base URL of your NeMo Platform deployment. Refer to the [Setup guide](/documentation/get-started) for installation, setup, and platform URL guidance. + + + +--- + +## Getting Started + + + + + +Learn the fundamentals of how NeMo Customizer works with Model Entities and Adapters, and how to choose the right approach for your project. + +model-entities adapters training-types + + + + + +## Dataset Preparation + + + + + +Learn how to format datasets for different model types. + +datasets chat-models completion-models + + + + + +## Customization Jobs + + + + + +Learn how to perform supervised fine-tuning with LoRA adapters using custom data. + +nemo-customizer + + + + +Learn how to perform supervised fine-tuning using custom data by modifying the all training parameters. + +nemo-customizer + + + + +Learn how to align a model with DPO (Direct Preference Optimization) to prefer certain kinds of responses over others. + +nemo-customizer dpo + + + + +Learn how to compress a larger teacher model into a smaller student model using knowledge distillation. + +nemo-customizer knowledge-distillation + + + + +Learn how to fine-tune embedding models using LoRA merged training for improved question-answering and retrieval tasks. + +embedding-models lora-merged nemo-customizer + + + + + +## Monitoring & Optimization + + + + + +Learn how to check job metrics using MLflow or Weights & Biases. + +nemo-customizer mlflow wandb + + + + +Learn how to optimize the token-per-GPU throughput for a LoRA optimization job. + +nemo-customizer wandb sequence-packing + + + + diff --git a/docs/customizer/tutorials/lora-customization-job.mdx b/docs/customizer/tutorials/lora-customization-job.mdx new file mode 100644 index 0000000000..d4b3e1eaa0 --- /dev/null +++ b/docs/customizer/tutorials/lora-customization-job.mdx @@ -0,0 +1,9 @@ +--- +title: "LoRA Model Customization" +description: "" +--- + + diff --git a/docs/customizer/tutorials/metrics.md b/docs/customizer/tutorials/metrics.mdx similarity index 78% rename from docs/customizer/tutorials/metrics.md rename to docs/customizer/tutorials/metrics.mdx index 2c3a594a3e..ddf9f2bef6 100644 --- a/docs/customizer/tutorials/metrics.md +++ b/docs/customizer/tutorials/metrics.mdx @@ -1,5 +1,8 @@ +--- +title: "Checking Your Customization Job Metrics" +description: "" +--- -# Checking Your Customization Job Metrics After completing a customization job, you can monitor its performance through training and validation metrics. You can access these metrics in three ways: @@ -7,14 +10,16 @@ After completing a customization job, you can monitor its performance through tr 2. Through MLflow (optional) 3. Using Weights & Biases (optional) -!!! note "The time to complete this tutorial is approximately 10 minutes." + -## Prerequisites +The time to complete this tutorial is approximately 10 minutes. ---8<-- "_snippets/tutorials/prereqs.md" + ---8<-- "customizer/tutorials/_snippets/customizer-prereqs.md" +## Prerequisites + + ### Tutorial-Specific Prerequisites - Completed customization job with a valid ID @@ -74,25 +79,28 @@ If your deployment has MLflow tracking enabled: 3. Find the run using your customization job ID 4. View detailed metrics, including training and validation loss curves, under the "Metrics" tab -!!! note "MLflow integration is configured at the cluster level. Contact your administrator if you need access to the MLflow UI or if MLflow tracking is not enabled for your deployment." + + +MLflow integration is configured at the cluster level. Contact your administrator if you need access to the MLflow UI or if MLflow tracking is not enabled for your deployment. + + ### Using Weights & Biases -If your customization job was created with W&B integration enabled (see [Weights & Biases Integration](../manage-customization-jobs/create-job.md)): +If your customization job was created with W&B integration enabled (see [Weights & Biases Integration](/documentation/fine-tune-models/manage-customization-jobs/create-a-customization-job)): 1. Go to [wandb.ai](https://wandb.ai/home) and navigate to your project 2. Find the run corresponding to your customization job 3. View training and validation loss curves, learning rate schedules, and other metrics under the run's dashboard +```python client = NeMoPlatform( base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), workspace="default", ) # Create a customization job with W&B integration - -```python job = client.customization.jobs.create( name="my-wandb-job", workspace="default", @@ -123,10 +131,13 @@ print(f"Status: {job.status}") The `api_key_secret` field references a stored secret containing your `WANDB_API_KEY`. Use the secret name (e.g., `"my-wandb-key"`) to resolve it from the request workspace. -To create the secret, see [Weights & Biases Keys](../../get-started/concepts/manage-secrets.md). +To create the secret, see [Weights & Biases Keys](/documentation/get-started/core-concepts/manage-secrets). Then view your results at [wandb.ai](https://wandb.ai/home) under your project. ![W&B charts example](../_images/wandb_charts_example.png) -!!! note - The W&B integration is optional and must be configured when [creating the customization job](../manage-customization-jobs/create-job.md). When enabled, training metrics are sent to W&B using your API key. While we encrypt your API key and don't log it internally, please review W&B's terms of service before use. + + +The W&B integration is optional and must be configured when [creating the customization job](/documentation/fine-tune-models/manage-customization-jobs/create-a-customization-job). When enabled, training metrics are sent to W&B using your API key. While we encrypt your API key and don't log it internally, please review W&B's terms of service before use. + + diff --git a/docs/customizer/tutorials/optimize-throughput.mdx b/docs/customizer/tutorials/optimize-throughput.mdx new file mode 100644 index 0000000000..7cb9e80b75 --- /dev/null +++ b/docs/customizer/tutorials/optimize-throughput.mdx @@ -0,0 +1,9 @@ +--- +title: "Optimize for Tokens/GPU Throughput" +description: "" +--- + + diff --git a/docs/customizer/tutorials/sft-customization-job.mdx b/docs/customizer/tutorials/sft-customization-job.mdx new file mode 100644 index 0000000000..836bb4ce04 --- /dev/null +++ b/docs/customizer/tutorials/sft-customization-job.mdx @@ -0,0 +1,9 @@ +--- +title: "Full SFT Customization" +description: "" +--- + + diff --git a/docs/customizer/tutorials/understand-configurations-and-models.md b/docs/customizer/tutorials/understand-configurations-and-models.mdx similarity index 81% rename from docs/customizer/tutorials/understand-configurations-and-models.md rename to docs/customizer/tutorials/understand-configurations-and-models.mdx index daac191bd4..910dcd9c51 100644 --- a/docs/customizer/tutorials/understand-configurations-and-models.md +++ b/docs/customizer/tutorials/understand-configurations-and-models.mdx @@ -1,25 +1,30 @@ +--- +title: "Understanding NeMo Customizer: Models, Training, and Resources" +description: "" +--- -# Understanding NeMo Customizer: Models, Training, and Resources Learn the fundamentals of how NeMo Customizer works to make informed decisions about your fine-tuning projects. This tutorial covers how models are organized, how adapters attach to base models, training types and GPU requirements, and how to choose the right approach for your use case. Understanding these basics will help you navigate the fine-tuning process more effectively and avoid common issues. If you're ready to start fine-tuning immediately, you can jump to [SFT Customization Job](sft-customization-job.ipynb) after completing this tutorial. -!!! note "The time to complete this tutorial is approximately 15 minutes." - This tutorial focuses on understanding and discovery—no actual training jobs are created. -## Prerequisites + ---8<-- "_snippets/tutorials/prereqs.md" +The time to complete this tutorial is approximately 15 minutes. ---8<-- "customizer/tutorials/_snippets/customizer-prereqs.md" + + This tutorial focuses on understanding and discovery—no actual training jobs are created. +## Prerequisites + + --- ## Core Concepts ### What is a Model Entity? -A **Model Entity** represents a model registered in the {{platform_name}}. It contains: +A **Model Entity** represents a model registered in the NeMo Platform. It contains: 1. **FileSet Reference**: Points to the model checkpoint files (weights, config, tokenizer) 2. **Model Spec**: Auto-populated metadata about the model architecture (layers, parameters, etc.) @@ -50,7 +55,7 @@ A **FileSet** is a collection of files stored in the platform's file service. Fo ## The Customization Workflow ```mermaid -{% raw %} + flowchart LR A[1. Create FileSet
with model files] --> B[2. Create Model Entity
pointing to FileSet] B --> C[3. Create Customization Job
referencing Model Entity] @@ -58,7 +63,7 @@ flowchart LR D -->{{ LoRA }} E[Adapter created and
attached to Model Entity] D -->|Full SFT| F[New Model Entity
with customized weights] E --> G[Auto-deploy to NIM
if enabled] -{% endraw %} + ``` ### Step-by-Step Breakdown @@ -240,15 +245,17 @@ Customization jobs consume disk space on the platform's shared persistent volume | LoRA | ~1.5× base model size | Stores base model + small adapter weights | | Full SFT | ~3× base model size | Stores base model + full checkpoint + output model | -!!! info - These estimates cover model weights only and do not include training dataset size. - If the platform disk fills during a job, the job fails with an I/O - error and the job service may return a ``500`` status when you retrieve logs. + - Ensure your platform's shared persistent volume has at least **3× the base model size** - of free space before starting a full SFT job, or **1.5×** for LoRA jobs. +These estimates cover model weights only and do not include training dataset size. +If the platform disk fills during a job, the job fails with an I/O +error and the job service may return a ``500`` status when you retrieve logs. -For troubleshooting disk-related failures, see [customizer](../../troubleshooting/customizer.md). +Ensure your platform's shared persistent volume has at least **3× the base model size** +of free space before starting a full SFT job, or **1.5×** for LoRA jobs. + + +For troubleshooting disk-related failures, see [customizer](/documentation/reference/troubleshooting/customizer). ### Parallelism Parameters Explained @@ -264,30 +271,34 @@ Parallelism is configured via `training.parallelism`. These parameters control h `data_parallel_size` is automatically derived as `total_gpus / (TP × PP × CP)` and is not set directly. -!!! info - **Recommended parallelism for Experts (MoE) Models**: + + +**Recommended parallelism for Experts (MoE) Models**: - The `expert_parallel_size` parameter is used to parallelize a Mixture of Experts (MoE) model's experts across GPUs. For non-MoE models, this parameter is ignored. A model's model card will indicate if it is a Mixture of Experts model and specifies its number of experts. +The `expert_parallel_size` parameter is used to parallelize a Mixture of Experts (MoE) model's experts across GPUs. For non-MoE models, this parameter is ignored. A model's model card will indicate if it is a Mixture of Experts model and specifies its number of experts. - The number of experts in the model must be divisible by `expert_parallel_size`. For example, if a model has 8 experts, setting `expert_parallel_size=4` results in each GPU processing 2 experts. +The number of experts in the model must be divisible by `expert_parallel_size`. For example, if a model has 8 experts, setting `expert_parallel_size=4` results in each GPU processing 2 experts. - Also, the value of `expert_parallel_size` must evenly divide the derived `data_parallel_size`, which is automatically calculated as `data_parallel_size = total GPUs / (tensor_parallel_size × pipeline_parallel_size × context_parallel_size)`. +Also, the value of `expert_parallel_size` must evenly divide the derived `data_parallel_size`, which is automatically calculated as `data_parallel_size = total GPUs / (tensor_parallel_size × pipeline_parallel_size × context_parallel_size)`. - For example, with 8 total GPUs, `tensor_parallel_size=2`, and `pipeline_parallel_size=1`: - - Derived `data_parallel_size = 8 / (2 × 1 × 1) = 4` - - Valid `expert_parallel_size` values: `1`, `2`, or `4` (must evenly divide 4) - - Invalid `expert_parallel_size` value: `3` (does not evenly divide 4) +For example, with 8 total GPUs, `tensor_parallel_size=2`, and `pipeline_parallel_size=1`: +- Derived `data_parallel_size = 8 / (2 × 1 × 1) = 4` +- Valid `expert_parallel_size` values: `1`, `2`, or `4` (must evenly divide 4) +- Invalid `expert_parallel_size` value: `3` (does not evenly divide 4) + ### Resource Allocation Rules Training configurations must satisfy mathematical constraints to work properly: -!!! info - **GPU Allocation Rule**: The total number of GPUs (`num_gpus_per_node x num_nodes`) must be a multiple of: - `tensor_parallel_size × pipeline_parallel_size × context_parallel_size` + - If this constraint isn't met, your training job will fail with a validation error. +**GPU Allocation Rule**: The total number of GPUs (`num_gpus_per_node x num_nodes`) must be a multiple of: +`tensor_parallel_size × pipeline_parallel_size × context_parallel_size` +If this constraint isn't met, your training job will fail with a validation error. + + **Example Calculations**: - 8 GPUs with `tensor_parallel_size=4, pipeline_parallel_size=2` ✅ Valid (8 = 4 × 2 × 1) @@ -300,7 +311,7 @@ Training configurations must satisfy mathematical constraints to work properly: ### Decision Framework ```mermaid -{% raw %} + flowchart TD A[What's your goal?] --> B{Need maximum
performance?} B -->{{ Yes }} C{Have 4+ GPUs?} @@ -316,7 +327,7 @@ flowchart TD E --> I[New Model Entity
with full weights] G --> J[Multiple adapters
on same Model Entity] H --> J -{% endraw %} + ``` ### When to Use LoRA @@ -380,7 +391,7 @@ model = client.models.create( ) ``` -For detailed guidance, see [Import HuggingFace Model](import-hf-model.md). +For detailed guidance, see [Import HuggingFace Model](/documentation/customizer-reference/tutorials/import-hugging-face-models). --- @@ -388,33 +399,30 @@ For detailed guidance, see [Import HuggingFace Model](import-hf-model.md). Now that you understand how Model Entities and Adapters work, you're ready to proceed: -
- -- **[Format Training Dataset](format-training-dataset.md)** - - --- - - Learn how to prepare your data for fine-tuning. + -- **[Start a LoRA Job](lora-customization-job.ipynb)** + - --- +Learn how to prepare your data for fine-tuning. - Create a parameter-efficient LoRA adapter. + + -- **[Start a Full SFT Job](sft-customization-job.ipynb)** +Create a parameter-efficient LoRA adapter. - --- + + - Use full supervised fine-tuning for maximum performance. +Use full supervised fine-tuning for maximum performance. -- **[Import Custom Models](import-hf-model.md)** + + - --- +Import and fine-tune private HuggingFace models. - Import and fine-tune private HuggingFace models. + -
+ --- diff --git a/docs/data-designer/_snippets/job-results.md b/docs/data-designer/_snippets/job-results.md deleted file mode 100644 index 0294001b9d..0000000000 --- a/docs/data-designer/_snippets/job-results.md +++ /dev/null @@ -1,8 +0,0 @@ -??? "More about job results" - The Data Designer library writes several artifacts to disk when running a full generation job, including the final dataset as parquet. - When a Data Designer job runs through NeMo Services, the entire working directory of artifacts produced by the library is saved as a job result. - The `download_artifacts` method downloads this artifacts directory (stored as a `.tar.gz` archive), - unarchives it, and returns a `DataDesignerJobResults` object that can be used to load results into memory as DataFrames or other objects for programmatic inspection. - - By default, `download_artifacts` saves the artifacts to a relative local directory named after the job. - An alternative path can be passed to `download_artifacts`. diff --git a/docs/data-designer/_snippets/preview-results.md b/docs/data-designer/_snippets/preview-results.md deleted file mode 100644 index 3656b1594d..0000000000 --- a/docs/data-designer/_snippets/preview-results.md +++ /dev/null @@ -1,4 +0,0 @@ -??? "More about preview results" - The `PreviewResults` object returned by `client.data_designer.preview` stores all its fields in memory; nothing is persisted to disk by default. - Use standard Python methods to save any preview data you want to keep around longer term. - For example, the `dataset` is a regular Pandas DataFrame and can be saved to disk via methods like `to_csv` or `to_parquet`. diff --git a/docs/data-designer/cli.md b/docs/data-designer/cli.mdx similarity index 96% rename from docs/data-designer/cli.md rename to docs/data-designer/cli.mdx index 57772ba1a4..8ad1daf480 100644 --- a/docs/data-designer/cli.md +++ b/docs/data-designer/cli.mdx @@ -1,5 +1,8 @@ +--- +title: "Data Designer CLI" +description: "" +--- -# Data Designer CLI The NeMo Data Designer plugin adds the `nemo data-designer` command group. Use it to execute Data Designer workloads locally in the CLI process or submit them to NeMo Services. @@ -25,7 +28,7 @@ def load_config_builder() -> dd.DataDesignerConfigBuilder: return config_builder ``` -The same configuration source can usually be used with `run` or `submit`. Resource choices determine whether it is compatible with NeMo Services execution; see [Execution Modes](execution-modes.md). +The same configuration source can usually be used with `run` or `submit`. Resource choices determine whether it is compatible with NeMo Services execution; see [Execution Modes](/documentation/design-synthetic-data/execution-modes). ## Run Versus Submit diff --git a/docs/data-designer/execution-modes.md b/docs/data-designer/execution-modes.mdx similarity index 93% rename from docs/data-designer/execution-modes.md rename to docs/data-designer/execution-modes.mdx index e967937066..5dadea4cf6 100644 --- a/docs/data-designer/execution-modes.md +++ b/docs/data-designer/execution-modes.mdx @@ -1,5 +1,8 @@ +--- +title: "Execution Modes" +description: "" +--- -# Execution Modes After building Data Designer configurations with the library's `data_designer.config` APIs, you choose where the NeMo Data Designer plugin executes the workload. @@ -12,9 +15,11 @@ The important distinction is not simply "local" versus "remote". There are two s `run` versus `submit` primarily controls where the plugin workload execution happens. It does not necessarily determine whether the workload uses NeMo Services APIs. -!!! note - `nemo data-designer ... run` can be fully local, but it is not an offline-only mode. A local run can still use the Files API, Secrets API, and Inference Gateway API from a running NeMo Services cluster when the configuration references the corresponding resources. + +`nemo data-designer ... run` can be fully local, but it is not an offline-only mode. A local run can still use the Files API, Secrets API, and Inference Gateway API from a running NeMo Services cluster when the configuration references the corresponding resources. + + ## Terms | Term | Meaning | diff --git a/docs/data-designer/index.md b/docs/data-designer/index.mdx similarity index 70% rename from docs/data-designer/index.md rename to docs/data-designer/index.mdx index ee03774bc0..5b734ff691 100644 --- a/docs/data-designer/index.md +++ b/docs/data-designer/index.mdx @@ -1,7 +1,10 @@ +--- +title: "Data Designer" +description: "" +--- -# Data Designer -Data Designer on {{platform_name}} enables high-quality synthetic data generation through the NeMo Data Designer plugin. You can execute workloads locally from the CLI, submit them to a running NeMo Services cluster, or call the Data Designer API from the SDK. +Data Designer on NeMo Platform enables high-quality synthetic data generation through the NeMo Data Designer plugin. You can execute workloads locally from the CLI, submit them to a running NeMo Services cluster, or call the Data Designer API from the SDK. ## Overview @@ -13,10 +16,12 @@ The plugin is built on the open-source [NVIDIA NeMo Data Designer library](https Data Designer separates **configuration** from **execution**. -!!! note - The code snippets below are for conceptual demonstration purposes only. - For runnable examples, see the [tutorials](tutorials/index.md). + +The code snippets below are for conceptual demonstration purposes only. +For runnable examples, see the [tutorials](/documentation/design-synthetic-data/tutorials). + + ### 1. Build Configurations Use `data_designer.config` to define the dataset you want to generate: @@ -55,7 +60,7 @@ The same configuration can run through different plugin surfaces: `run` versus `submit` primarily controls where the plugin workload execution happens. A local `run` can be fully local, but it is not an offline-only mode: it can still use the Files API, Secrets API, and Inference Gateway API from a running NeMo Services cluster when the configuration references the corresponding resources. -See [Execution Modes](execution-modes.md) for the full model. +See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for the full model. ## NeMo Services Integration @@ -72,36 +77,32 @@ These integrations are required for `submit` and SDK execution. They are optiona ## Next Steps -
- -- **[Execution Modes](execution-modes.md)** - - --- - - Understand local execution, NeMo Services execution, and NeMo resources. - -- **[CLI](cli.md)** + - --- + - Run previews and create datasets with `nemo data-designer`. +Understand local execution, NeMo Services execution, and NeMo resources. -- **[Tutorials](tutorials/index.md)** + + - --- +Run previews and create datasets with `nemo data-designer`. - Learn through examples: basics, seeding, and more. + + -- **[Migration Guide](migration.md)** +Learn through examples: basics, seeding, and more. - --- + + - Move configurations between local CLI and NeMo Services execution. +Move configurations between local CLI and NeMo Services execution. -- **[Library Documentation](https://docs.nvidia.com/nemo/datadesigner/v0.6.1/getting-started/welcome)** + + - --- +Comprehensive guides on column types, constraints, and advanced features. - Comprehensive guides on column types, constraints, and advanced features. + -
+ diff --git a/docs/data-designer/migration.md b/docs/data-designer/migration.mdx similarity index 90% rename from docs/data-designer/migration.md rename to docs/data-designer/migration.mdx index 5aaef10286..4d3b6e2bec 100644 --- a/docs/data-designer/migration.md +++ b/docs/data-designer/migration.mdx @@ -1,9 +1,12 @@ +--- +title: "Moving Between Execution Modes" +description: "" +--- -# Moving Between Execution Modes Data Designer configurations are mostly portable across plugin execution modes. The main migration question is not "library versus service"; it is whether the workload executes locally in the CLI process or through NeMo Services, and which resources the configuration references. -See [Execution Modes](execution-modes.md) for the full two-axis model. +See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for the full two-axis model. ## What Usually Stays the Same @@ -77,7 +80,7 @@ Before switching execution modes, verify: ## Getting Help -- **Execution Modes:** See [Execution Modes](execution-modes.md) for the conceptual model. -- **CLI:** See [Data Designer CLI](cli.md) for `run`, `submit`, and persona commands. -- **Tutorials:** Follow the [tutorials](tutorials/index.md) for hands-on examples. +- **Execution Modes:** See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for the conceptual model. +- **CLI:** See [Data Designer CLI](/documentation/design-synthetic-data/cli) for `run`, `submit`, and persona commands. +- **Tutorials:** Follow the [tutorials](/documentation/design-synthetic-data/tutorials) for hands-on examples. - **Library Docs:** Refer to the [open-source library documentation](https://docs.nvidia.com/nemo/datadesigner/v0.6.1/getting-started/welcome) for configuration details. diff --git a/docs/data-designer/sdk-resources.md b/docs/data-designer/sdk-resources.mdx similarity index 90% rename from docs/data-designer/sdk-resources.md rename to docs/data-designer/sdk-resources.mdx index 7160797ee0..2beda3ef97 100644 --- a/docs/data-designer/sdk-resources.md +++ b/docs/data-designer/sdk-resources.mdx @@ -1,15 +1,19 @@ +--- +title: "Data Designer SDK Resources" +description: "" +--- -# Data Designer SDK Resources The `data_designer.config` module provides a consistent, context-agnostic experience for building Data Designer configs. Once you are ready to execute that config through NeMo Services APIs, you use objects from the `nemo_platform` SDK. This page explains the SDK objects used for Data Designer API execution. -!!! note - The SDK currently executes Data Designer workloads through the Data Designer API. Local SDK execution is planned, but not available yet. Use `nemo data-designer ... run` for local in-process execution today. + +The SDK currently executes Data Designer workloads through the Data Designer API. Local SDK execution is planned, but not available yet. Use `nemo data-designer ... run` for local in-process execution today. + ## DataDesignerResource The `DataDesignerResource` is the initial SDK object for working with Data Designer through the SDK. diff --git a/docs/data-designer/tutorials/basics.md b/docs/data-designer/tutorials/basics.mdx similarity index 92% rename from docs/data-designer/tutorials/basics.md rename to docs/data-designer/tutorials/basics.mdx index 03ee0c763c..e6f8bfe46d 100644 --- a/docs/data-designer/tutorials/basics.md +++ b/docs/data-designer/tutorials/basics.mdx @@ -1,7 +1,8 @@ - - +--- +title: "The Basics" +description: "" +--- -# The Basics This tutorial demonstrates the fundamentals of Data Designer by generating a product review dataset. @@ -9,15 +10,17 @@ For more detail about column behavior, see the [open-source library's version](h ## Prerequisites -Ensure you have completed the [tutorials prerequisites](index.md#prerequisites). This tutorial uses an Inference Gateway provider, so local CLI `run` and NeMo Services execution both need access to the Inference Gateway API in a running NeMo Services cluster. +Ensure you have completed the [tutorials prerequisites](/documentation/design-synthetic-data/tutorials#prerequisites). This tutorial uses an Inference Gateway provider, so local CLI `run` and NeMo Services execution both need access to the Inference Gateway API in a running NeMo Services cluster. ## Part 1: Build the Configuration Use the `data_designer.config` package to define your dataset schema. This configuration code is the same across the plugin execution modes. -!!! tip - Build the configuration once, then choose whether to execute with CLI `run`, CLI `submit`, or the SDK. + +Build the configuration once, then choose whether to execute with CLI `run`, CLI `submit`, or the SDK. + + ### Define Models Start by defining the models you want to use: @@ -46,7 +49,7 @@ config_builder = dd.DataDesignerConfigBuilder(model_configs) Define the columns for your dataset. The [library documentation](https://docs.nvidia.com/nemo/datadesigner/v0.6.1/tutorials/the-basics) explains these column types in detail. -{% raw %} + ```python # Product category sampler config_builder.add_column( @@ -184,7 +187,7 @@ config_builder.add_column( ) ) ``` -{% endraw %} + ## Part 2: Execute @@ -255,8 +258,7 @@ print(df.head()) preview.analysis.to_report() ``` ---8<-- "data-designer/_snippets/preview-results.md" - + **Iterate:** Adjust column configurations, prompts, or parameters in your `config_builder`, then run `preview` again until you're satisfied with the results. ### Scaling Up with Jobs @@ -282,8 +284,7 @@ analysis = results.load_analysis() analysis.to_report() ``` ---8<-- "data-designer/_snippets/job-results.md" - + ## What Happens Under the Hood When you use CLI `run`: @@ -302,7 +303,7 @@ When you use CLI `submit` or the SDK today: ## Next Steps -- **Seed data:** Learn how to use external datasets in the [seeding tutorial](seeding.md) -- **Execution modes:** Learn more about local and NeMo Services execution in [Execution Modes](../execution-modes.md) +- **Seed data:** Learn how to use external datasets in the [seeding tutorial](/documentation/design-synthetic-data/tutorials/seeding-with-external-datasets) +- **Execution modes:** Learn more about local and NeMo Services execution in [Execution Modes](/documentation/design-synthetic-data/execution-modes) - **Column types:** Explore all available column types in the [library documentation](https://docs.nvidia.com/nemo/datadesigner/v0.6.1/concepts/columns) - **Advanced features:** Learn about [processors](https://docs.nvidia.com/nemo/datadesigner/v0.6.1/concepts/processors) and [validation](https://docs.nvidia.com/nemo/datadesigner/v0.6.1/concepts/validators) diff --git a/docs/data-designer/tutorials/index.md b/docs/data-designer/tutorials/index.mdx similarity index 56% rename from docs/data-designer/tutorials/index.md rename to docs/data-designer/tutorials/index.mdx index 1a0d6c2bbe..9c7f6bffa7 100644 --- a/docs/data-designer/tutorials/index.md +++ b/docs/data-designer/tutorials/index.mdx @@ -1,12 +1,17 @@ +--- +title: "Tutorials" +description: "" +--- -# Tutorials These tutorials demonstrate how to build Data Designer configurations and execute them through the NeMo Data Designer plugin. -!!! note - The code snippets on this page are for conceptual demonstration purposes only. - For runnable examples, jump ahead to the [Basics](basics.md) or [Seeding](seeding.md) tutorial. + +The code snippets on this page are for conceptual demonstration purposes only. +For runnable examples, jump ahead to the [Basics](/documentation/design-synthetic-data/tutorials/the-basics) or [Seeding](/documentation/design-synthetic-data/tutorials/seeding-with-external-datasets) tutorial. + + ## Configuration and Execution Data Designer separates **configuration** (building dataset schemas) from **execution** (generating the data). @@ -47,9 +52,11 @@ preview = data_designer.preview(config_builder) job = data_designer.create(config_builder, num_records=1000) ``` -!!! tip - `run` versus `submit` primarily controls where the workload executes. Local `run` can still use the Files API, Secrets API, and Inference Gateway API from a running NeMo Services cluster when the configuration references the corresponding resources. See [Execution Modes](../execution-modes.md) for details. + + +`run` versus `submit` primarily controls where the workload executes. Local `run` can still use the Files API, Secrets API, and Inference Gateway API from a running NeMo Services cluster when the configuration references the corresponding resources. See [Execution Modes](/documentation/design-synthetic-data/execution-modes) for details. + ## Execution-Specific Considerations When running through the plugin, supported resources depend on the execution mode: @@ -63,28 +70,27 @@ When running through the plugin, supported resources depend on the execution mod ## Prerequisites -These tutorials use an [Inference Gateway](../../run-inference/about.md) provider for model calls, so a NeMo Services cluster must be running before you preview or create data — including with local CLI `run` (see [Execution Modes](../execution-modes.md#local-nemo-services-execution) for more about this distinction). -Complete [Setup](../../get-started/setup.md) to ensure you have the NeMo Services running locally and an inference provider available. +These tutorials use an [Inference Gateway](/documentation/models-and-inference) provider for model calls, so a NeMo Services cluster must be running before you preview or create data — including with local CLI `run` (see [Execution Modes](/documentation/design-synthetic-data/execution-modes#local-nemo-services-execution) for more about this distinction). +Complete [Setup](/documentation/get-started) to ensure you have the NeMo Services running locally and an inference provider available. These tutorials reference the default NVIDIA Build model provider, which is created as `default/nvidia-build` during setup. ## Tutorials -
- -- **[The Basics](basics.md)** + - --- + - Generate a product review dataset using samplers and LLM-generated text. Learn the fundamentals of building configurations and executing jobs. +Generate a product review dataset using samplers and LLM-generated text. Learn the fundamentals of building configurations and executing jobs. - beginner data-designer +beginner data-designer -- **[Seeding](seeding.md)** + + - --- +Use external datasets to ground synthetic data generation. Generate realistic patient medical notes from symptom-to-diagnosis data. - Use external datasets to ground synthetic data generation. Generate realistic patient medical notes from symptom-to-diagnosis data. +intermediate data-designer - intermediate data-designer + -
+ diff --git a/docs/data-designer/tutorials/seeding.md b/docs/data-designer/tutorials/seeding.mdx similarity index 92% rename from docs/data-designer/tutorials/seeding.md rename to docs/data-designer/tutorials/seeding.mdx index ba91662be4..38c4ba3b3a 100644 --- a/docs/data-designer/tutorials/seeding.md +++ b/docs/data-designer/tutorials/seeding.mdx @@ -1,7 +1,8 @@ - - +--- +title: "Seeding with External Datasets" +description: "" +--- -# Seeding with External Datasets This tutorial demonstrates how to use external datasets as seed data for synthetic data generation in Data Designer. @@ -17,9 +18,11 @@ Seed source support depends on where the workload executes: | **HuggingFace** | Supported | Supported | Publicly available datasets or private HuggingFace datasets. | | **Files API Filesets** | Supported when NeMo Services access is configured | Supported | Shared seed data stored through the Files API. | -!!! note - `run` versus `submit` controls where the workload executes. A local `run` can still read Files API Filesets if the configuration references them and NeMo Services access is configured. + +`run` versus `submit` controls where the workload executes. A local `run` can still read Files API Filesets if the configuration references them and NeMo Services access is configured. + + ### HuggingFace Datasets Use `HuggingFaceSeedSource` to load data from HuggingFace: @@ -55,7 +58,7 @@ FilesetFileSeedSource( ## Prerequisites -Ensure you have completed the [tutorials prerequisites](index.md#prerequisites). This tutorial uses an Inference Gateway provider, so local CLI `run` and NeMo Services execution both need access to the Inference Gateway API in a running NeMo Services cluster. +Ensure you have completed the [tutorials prerequisites](/documentation/design-synthetic-data/tutorials#prerequisites). This tutorial uses an Inference Gateway provider, so local CLI `run` and NeMo Services execution both need access to the Inference Gateway API in a running NeMo Services cluster. ## Example: Medical Notes from Symptom Data @@ -135,7 +138,7 @@ config_builder.with_seed_dataset( Add columns that reference and extend the seed data: -{% raw %} + ```python # Patient details config_builder.add_column( @@ -241,7 +244,7 @@ Respond with only the notes, no other text. ) ) ``` -{% endraw %} + **Note:** The `diagnosis` and `patient_summary` variables come from the seed dataset columns. @@ -302,8 +305,7 @@ print(df.head()) preview.analysis.to_report() ``` ---8<-- "data-designer/_snippets/preview-results.md" - + ### Generating the Full Dataset When you're satisfied with the preview, submit a larger generation job: @@ -327,8 +329,7 @@ analysis = results.load_analysis() analysis.to_report() ``` ---8<-- "data-designer/_snippets/job-results.md" - + ## How Seeding Works When you configure a seed dataset: @@ -342,6 +343,6 @@ When you configure a seed dataset: ## Next Steps -- **Execution modes:** Learn more about local and NeMo Services execution in [Execution Modes](../execution-modes.md) +- **Execution modes:** Learn more about local and NeMo Services execution in [Execution Modes](/documentation/design-synthetic-data/execution-modes) - **Column types:** Explore all available column types in the [library documentation](https://docs.nvidia.com/nemo/datadesigner/v0.6.1/concepts/columns) - **Processors:** Transform your data with processors in the [library documentation](https://docs.nvidia.com/nemo/datadesigner/v0.6.1/concepts/processors) diff --git a/docs/eula.md b/docs/eula.mdx similarity index 94% rename from docs/eula.md rename to docs/eula.mdx index 52fabb5845..c4df83cefd 100644 --- a/docs/eula.md +++ b/docs/eula.mdx @@ -1,5 +1,7 @@ -# Governing Terms - +--- +title: "Governing Terms" +description: "" +--- The source code in the NVIDIA NeMo Platform repository is licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). Some NVIDIA-distributed artifacts used with or alongside NeMo Platform, including NIM containers, model weights, hosted services, or other separately distributed binaries and materials, may be governed by additional or different terms. Review the license files and terms that accompany each artifact before use. These terms may include the [NVIDIA Software License Agreement](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-software-license-agreement/) and the [Product-Specific Terms for NVIDIA AI Products](https://www.nvidia.com/en-us/agreements/enterprise-software/product-specific-terms-for-ai-products/). diff --git a/docs/evaluator/benchmarks/agentic.md b/docs/evaluator/benchmarks/agentic.mdx similarity index 58% rename from docs/evaluator/benchmarks/agentic.md rename to docs/evaluator/benchmarks/agentic.mdx index 84f78ecdfc..bebdec6506 100644 --- a/docs/evaluator/benchmarks/agentic.md +++ b/docs/evaluator/benchmarks/agentic.mdx @@ -1,5 +1,8 @@ +--- +title: "Agentic Benchmarks" +description: "" +--- -# Agentic Benchmarks Evaluate agent tool-calling capabilities using industry-standard benchmarks. @@ -54,58 +57,68 @@ benchmark_params = { } ``` -=== "Job" + - ```python - from nemo_platform.types.evaluation import SystemBenchmarkOnlineJobParam + - job = client.evaluation.benchmark_jobs.create( - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/bfclv3-live-simple", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - benchmark_params={}, - ) +```python +from nemo_platform.types.evaluation import SystemBenchmarkOnlineJobParam + +job = client.evaluation.benchmark_jobs.create( + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/bfclv3-live-simple", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + benchmark_params={}, ) - ``` -=== "Result" +) +``` + + + - Accuracy of tool call predictions +Accuracy of tool call predictions - - Score name: tool-calling-accuracy - - Value range: 0.0–1.0 +- Score name: tool-calling-accuracy +- Value range: 0.0–1.0 - ```json +```json +{ + "scores": [ { - "scores": [ - { - "name": "tool-calling-accuracy", - "score_type": "range", - "count": 1, - "nan_count": 0, - "sum": 1.0, - "mean": 1.0, - "min": 1.0, - "max": 1.0, - "std_dev": 0.0, - "variance": 0.0 - } - ] + "name": "tool-calling-accuracy", + "score_type": "range", + "count": 1, + "nan_count": 0, + "sum": 1.0, + "mean": 1.0, + "min": 1.0, + "max": 1.0, + "std_dev": 0.0, + "variance": 0.0 } - ``` + ] +} +``` + + + + --- ## Job Management -After creating a job, navigate to [Benchmark Job Management](job-management.md) to oversee its execution and monitor progress. +After creating a job, navigate to [Benchmark Job Management](/documentation/evaluator-benchmark-reference/benchmarks/job-management) to oversee its execution and monitor progress. --- -!!! info + + +- [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration) - Use agents (generic or NAT) as targets in online evaluation and benchmark jobs +- [Agentic Evaluation Metrics](/documentation/evaluate-models/metrics/agentic-metrics) - Detailed metric documentation for evaluating agentic workflows +- [Managing Secrets](/documentation/get-started/core-concepts/manage-secrets) - Store API keys for external APIs +- [Evaluation Results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results) - Understanding and downloading results - - [Agent Configuration](../metrics/agent-configuration.md) - Use agents (generic or NAT) as targets in online evaluation and benchmark jobs - - [Agentic Evaluation Metrics](../metrics/agentic.md) - Detailed metric documentation for evaluating agentic workflows - - [Managing Secrets](../../get-started/concepts/manage-secrets.md) - Store API keys for external APIs - - [Evaluation Results](results.md) - Understanding and downloading results + diff --git a/docs/evaluator/benchmarks/custom.md b/docs/evaluator/benchmarks/custom.mdx similarity index 76% rename from docs/evaluator/benchmarks/custom.md rename to docs/evaluator/benchmarks/custom.mdx index 1e8505ce44..0b1aa58d24 100644 --- a/docs/evaluator/benchmarks/custom.md +++ b/docs/evaluator/benchmarks/custom.mdx @@ -1,18 +1,23 @@ +--- +title: "Custom Benchmarks" +description: "" +--- -# Custom Benchmarks -Custom benchmarks allow you to create reusable evaluation suites tailored to your specific use case. A benchmark combines one or more [metrics](../metrics/index.md) with a dataset, enabling consistent evaluation across multiple models or pipeline versions. +Custom benchmarks allow you to create reusable evaluation suites tailored to your specific use case. A benchmark combines one or more [metrics](/documentation/evaluate-models/metrics) with a dataset, enabling consistent evaluation across multiple models or pipeline versions. -!!! note - Custom benchmarks can only include custom metrics that you create in your workspace. System metrics (in the `system` workspace) cannot be included in custom benchmarks at this time. To use system metrics, refer to [Industry Benchmarks](industry.md). + +Custom benchmarks can only include custom metrics that you create in your workspace. System metrics (in the `system` workspace) cannot be included in custom benchmarks at this time. To use system metrics, refer to [Industry Benchmarks](/documentation/evaluate-models/benchmarks/industry-benchmarks). + + ## Prerequisites Before creating a custom benchmark, ensure you have: -- A [workspace](../../get-started/concepts/workspaces.md) created for your project -- One or more [custom metrics](../metrics/index.md) defined in your workspace -- A dataset uploaded as a [fileset](../../get-started/concepts/manage-files.md) +- A [workspace](/documentation/get-started/core-concepts/workspaces) created for your project +- One or more [custom metrics](/documentation/evaluate-models/metrics) defined in your workspace +- A dataset uploaded as a [fileset](/documentation/get-started/core-concepts/manage-files) ```python import os @@ -24,12 +29,14 @@ client = NeMoPlatform( ) ``` -!!! tip - Set `NMP_BASE_URL` to your {{nem_short_name}} deployment endpoint. See [CLI and SDK initialization](../../get-started/setup.md#setup-init) for the full convention. + + +Set `NMP_BASE_URL` to your NeMo Evaluator deployment endpoint. See [CLI and SDK initialization](/documentation/get-started#setup-init) for the full convention. + ## Dataset Requirements -Your dataset must be compatible with all metrics in the benchmark. Each metric defines input templates (like `{%raw%}{{output}}{%endraw%}` and `{%raw%}{{reference}}{%endraw%}`) that map to columns in your dataset. +Your dataset must be compatible with all metrics in the benchmark. Each metric defines input templates (like `{{output}}` and `{{reference}}`) that map to columns in your dataset. For **offline evaluation**, your dataset should contain pre-generated model outputs as a JSON array: @@ -63,15 +70,19 @@ For **online evaluation**, your dataset contains inputs that will be sent to the ] ``` -!!! note - Upload your dataset to a fileset in any [supported format](../metrics/similarity.md). You can select specific files using a `#` fragment pattern (for example, `my-workspace/my-fileset#data.csv`). If no pattern is specified, all parsable files in the fileset are loaded. + + +Upload your dataset to a fileset in any [supported format](/documentation/evaluate-models/metrics/similarity-metrics). You can select specific files using a `#` fragment pattern (for example, `my-workspace/my-fileset#data.csv`). If no pattern is specified, all parsable files in the fileset are loaded. -!!! info - Ensure your dataset columns match both: + + - 1. The input templates defined in your metrics (for example, `{%raw%}{{output}}{%endraw%}`, `{%raw%}{{reference}}{%endraw%}`) - 2. The `prompt_template` used in online evaluation jobs (for example, `{%raw%}{{question}}{%endraw%}`) +Ensure your dataset columns match both: +1. The input templates defined in your metrics (for example, `{{output}}`, `{{reference}}`) +2. The `prompt_template` used in online evaluation jobs (for example, `{{question}}`) + + For schema-aware validation, prefer canonical evaluator fields in metric prompts and online job prompts: - use `{{input}}`, `{{output}}`, `{{reference}}`, and `{{context}}` in prompts @@ -80,7 +91,7 @@ For schema-aware validation, prefer canonical evaluator fields in metric prompts - add fileset metadata such as `dataset.schema` and `dataset.schemas_by_path` when you want benchmark creation and job submission to validate dataset compatibility before a run starts - remember that benchmark-level `field_mapping` applies to every metric in the benchmark -Legacy raw dataset variables such as `{% raw %}{{question}}{% endraw %}`, `{% raw %}{{item.question}}{% endraw %}`, or `{% raw %}{{item.response}}{% endraw %}` still work. In that case, prompt variable names are matched directly against dataset columns unless `field_mapping.custom` remaps them. +Legacy raw dataset variables such as `{{question}}`, `{{item.question}}`, or `{{item.response}}` still work. In that case, prompt variable names are matched directly against dataset columns unless `field_mapping.custom` remaps them. ## Create a Custom Benchmark @@ -117,7 +128,7 @@ benchmark = client.evaluation.benchmarks.create( print(f"Created benchmark: {benchmark.name}") ``` -Refer to [Manage Benchmarks](manage-benchmarks.md) for listing and managing custom benchmarks. +Refer to [Manage Benchmarks](/documentation/evaluator-benchmark-reference/benchmarks/manage-benchmarks) for listing and managing custom benchmarks. ## Run Benchmark Evaluation Jobs @@ -180,12 +191,12 @@ job = client.evaluation.benchmark_jobs.create( Online evaluation generates model responses at runtime, then evaluates them against your metrics. Use this to evaluate a model's live performance. -The `model` field accepts either an inline model definition or a model reference. Refer to [Model Configuration](../metrics/model-configuration.md) for details on both formats. +The `model` field accepts either an inline model definition or a model reference. Refer to [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) for details on both formats. #### With Inline Model ```python -{% raw %} + from nemo_platform.types.evaluation import ( BenchmarkOnlineJobParam, ModelParam, @@ -202,13 +213,13 @@ job = client.evaluation.benchmark_jobs.create( prompt_template="Answer the following customer question:\n\n{{input}}", ), ) -{% endraw %} + ``` #### With Model Reference ```python -{% raw %} + from nemo_platform.types.evaluation import BenchmarkOnlineJobParam job = client.evaluation.benchmark_jobs.create( @@ -218,24 +229,26 @@ job = client.evaluation.benchmark_jobs.create( prompt_template="Answer the following customer question:\n\n{{input}}", ), ) -{% endraw %} + ``` ## Job Management -After successfully creating a job, navigate to [Benchmark Job Management](job-management.md) to oversee its execution and monitor progress. +After successfully creating a job, navigate to [Benchmark Job Management](/documentation/evaluator-benchmark-reference/benchmarks/job-management) to oversee its execution and monitor progress. ## Retrieve Results -After the job completes, retrieve and analyze results. Refer to [Benchmark Results](results.md) for detailed examples of downloading aggregate scores, row-level scores, and analyzing results with Pandas. +After the job completes, retrieve and analyze results. Refer to [Benchmark Results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results) for detailed examples of downloading aggregate scores, row-level scores, and analyzing results with Pandas. ## Complete Example Here is a complete workflow for creating a benchmark and running an evaluation. -!!! note - This example assumes you have already created the metrics (`exact-match`, `f1-score`) and uploaded your dataset (`qa-test-data`). Refer to [Evaluation Metrics](../metrics/index.md) for how to create custom metrics. + + +This example assumes you have already created the metrics (`exact-match`, `f1-score`) and uploaded your dataset (`qa-test-data`). Refer to [Evaluation Metrics](/documentation/evaluate-models/metrics) for how to create custom metrics. + ```python import json import time @@ -330,4 +343,4 @@ if status.status == "completed": 3. **Validate dataset**: Ensure your dataset files are valid and columns/keys are consistent across rows 4. **Test metrics first**: Run individual metric evaluations before combining into a benchmark -For additional help, refer to [Troubleshooting](../../troubleshooting/evaluator.md). +For additional help, refer to [Troubleshooting](/documentation/reference/troubleshooting/evaluator). diff --git a/docs/evaluator/benchmarks/discover-industry-benchmarks.mdx b/docs/evaluator/benchmarks/discover-industry-benchmarks.mdx new file mode 100644 index 0000000000..6fd0dfb05c --- /dev/null +++ b/docs/evaluator/benchmarks/discover-industry-benchmarks.mdx @@ -0,0 +1,55 @@ +--- +title: "List benchmarks with pagination:" +description: "" +--- +### Discover Industry Benchmarks + +Discover industry benchmarks available to use for your evaluation job within the `system` workspace. List all industry benchmarks or filter by label category. + + + +The `system` workspace is a reserved workspace for NeMo Platform that contains ready-to-use benchmarks representing industry benchmarks with published datasets and metrics. + + + + +**Initialization:** This example uses `NeMoPlatform()` with no arguments so the SDK reads your active CLI context (set by `nemo auth login`). The `workspace="system"` is passed per-call to access the reserved system workspace. For the standard local initialization pattern, see [CLI and SDK initialization](/documentation/get-started#setup-init). + + +```python +from nemo_platform import NeMoPlatform + +client = NeMoPlatform() + +benchmarks = client.evaluation.benchmarks.list(workspace="system") + +print(f"{benchmarks.pagination.total_results} benchmarks") +for benchmark in benchmarks: + print(f"{benchmark.name}: {benchmark.description}") + +benchmarks = client.evaluation.benchmarks.list( + workspace="system", page=2, page_size=100 +) +for benchmark in benchmarks: + print(f"{benchmark.name}: {benchmark.description}") + +# Filter by evaluation category label +filtered_benchmarks = client.evaluation.benchmarks.list( + workspace="system", + extra_query={"filter[data.labels.eval_category]": "advanced_reasoning"}, +) +print(filtered_benchmarks) +``` + +| Category Label | Description | +|----------------|-------------| +| `agentic` | Evaluate the performance of agent-based or multi-step reasoning models, especially in scenarios requiring planning, tool use, and iterative reasoning. | +| `advanced_reasoning` | Evaluate reasoning capabilities of large language models through complex tasks. | +| `code` | Evaluate code generation capabilities using functional correctness benchmarks that test synthesis of working programs. | +| `content_safety` | Evaluate model safety risks including vulnerability to generate harmful, biased, or misleading content. | +| `instruction_following` | Evaluate the ability to follow explicit formatting and structural instructions | +| `language_understanding` | Evaluate knowledge and reasoning across diverse subjects in different languages. | +| `math` | Evaluate mathematical reasoning abilities. | +| `question_answering` | Evaluate the ability to generate answers to questions. | +| `rag` | Evaluate the quality of RAG pipelines by measuring both retrieval and answer generation performance. | +| `retrieval` | Evaluate the quality of document retriever pipelines. | diff --git a/docs/evaluator/benchmarks/hf-secret.mdx b/docs/evaluator/benchmarks/hf-secret.mdx new file mode 100644 index 0000000000..ac87458131 --- /dev/null +++ b/docs/evaluator/benchmarks/hf-secret.mdx @@ -0,0 +1,15 @@ +--- +title: "Create a Hugging Face Secret" +description: "" +--- +Most benchmarks require a Hugging Face token (`hf_token`) to access gated datasets. Create this secret before running evaluations: + +```python +import os + +client.secrets.create( + workspace=workspace, + name="hf_token", + value=os.getenv("HF_TOKEN", ""), +) +``` diff --git a/docs/evaluator/benchmarks/index.md b/docs/evaluator/benchmarks/index.mdx similarity index 60% rename from docs/evaluator/benchmarks/index.md rename to docs/evaluator/benchmarks/index.mdx index 6d2fec9fe5..2042810fdf 100644 --- a/docs/evaluator/benchmarks/index.md +++ b/docs/evaluator/benchmarks/index.mdx @@ -1,5 +1,8 @@ +--- +title: "Evaluation Benchmarks" +description: "" +--- -# Evaluation Benchmarks A benchmark is a reusable evaluation suite: one or more metrics paired with a dataset. Instead of redefining metrics and data inputs for every run, you define the benchmark once and run it repeatedly. @@ -10,7 +13,7 @@ Use benchmarks when you want to: - Compare multiple model versions using the same scoring criteria and dataset - Package validated metrics with domain-specific test data for repeatable evaluation -{{platform_name}} provides two types of benchmarks: +NeMo Platform provides two types of benchmarks: - **Industry Benchmarks**: Industry-standard academic benchmarks such as MMLU, HumanEval, and GSM8K for comparing model capabilities against published baselines - **Custom Benchmarks**: User-defined evaluation suites that combine your choice of metrics with domain-specific datasets @@ -21,14 +24,13 @@ Custom benchmarks are valuable for domain-specific evaluation where standard ben | Type | Use Case | Dataset | Metrics | |------|----------|---------|---------| -| [**Industry Benchmarks**](industry.md) | Compare against published baselines, regression testing, model selection | Canonical datasets (fixed) | Standardized metrics | -| [**Custom Benchmarks**](custom.md) | Domain-specific evaluation, production monitoring, task-specific assessment | Your evaluation data | Your choice of metrics | - ---8<-- "evaluator/benchmarks/discover-industry-benchmarks.md" +| [**Industry Benchmarks**](/documentation/evaluate-models/benchmarks/industry-benchmarks) | Compare against published baselines, regression testing, model selection | Canonical datasets (fixed) | Standardized metrics | +| [**Custom Benchmarks**](/documentation/evaluate-models/benchmarks/custom-benchmarks) | Domain-specific evaluation, production monitoring, task-specific assessment | Your evaluation data | Your choice of metrics | + ## Create Custom Benchmarks -Create a custom benchmark by combining metrics with your dataset. Before creating a benchmark, you will need to [create the metrics](../metrics/index.md) that define how to score your model's outputs. +Create a custom benchmark by combining metrics with your dataset. Before creating a benchmark, you will need to [create the metrics](/documentation/evaluate-models/metrics) that define how to score your model's outputs. ```python benchmark = client.evaluation.benchmarks.create( @@ -43,7 +45,7 @@ benchmark = client.evaluation.benchmarks.create( ) ``` -Refer to [Manage Benchmarks](manage-benchmarks.md) for listing and managing custom benchmarks. +Refer to [Manage Benchmarks](/documentation/evaluator-benchmark-reference/benchmarks/manage-benchmarks) for listing and managing custom benchmarks. ## Run Benchmark Jobs @@ -91,38 +93,36 @@ print(f"Job created: {job.name}") List, retrieve, and delete evaluation benchmarks using the Python SDK. You can discover industry benchmarks in the `system` workspace, list custom benchmarks in your workspace, retrieve detailed benchmark configurations, and delete custom benchmarks when no longer needed. -Refer to [Manage Benchmarks](manage-benchmarks.md) for complete SDK examples including pagination, sorting, filtering, and extended response options. +Refer to [Manage Benchmarks](/documentation/evaluator-benchmark-reference/benchmarks/manage-benchmarks) for complete SDK examples including pagination, sorting, filtering, and extended response options. ## Job Management -After successfully creating a job, refer to [Benchmark Job Management](job-management.md) to oversee its execution and monitor progress. +After successfully creating a job, refer to [Benchmark Job Management](/documentation/evaluator-benchmark-reference/benchmarks/job-management) to oversee its execution and monitor progress. ## Benchmark Categories -
- -- **[Custom Benchmarks](custom.md)** - - --- + - Compose a custom benchmark with a collection of metrics to evaluate tasks bespoke to your needs. + - RAGAS BFCL +Compose a custom benchmark with a collection of metrics to evaluate tasks bespoke to your needs. -- **[Agentic Benchmarks](agentic.md)** +RAGAS BFCL - --- + + - Evaluate agent workflows including tool calling, goal accuracy, topic adherence, and trajectory evaluation. +Evaluate agent workflows including tool calling, goal accuracy, topic adherence, and trajectory evaluation. - RAGAS BFCL +RAGAS BFCL -- **[Industry Benchmarks](industry.md)** + + - --- +Ready-to-use benchmarks for reasoning, code generation, safety, and language understanding with published datasets. - Ready-to-use benchmarks for reasoning, code generation, safety, and language understanding with published datasets. +LM Harness BigCode - LM Harness BigCode + -
\ No newline at end of file + diff --git a/docs/evaluator/benchmarks/industry.md b/docs/evaluator/benchmarks/industry.md deleted file mode 100644 index 7451ae8ef1..0000000000 --- a/docs/evaluator/benchmarks/industry.md +++ /dev/null @@ -1,1109 +0,0 @@ - -# Industry Benchmarks - -## Evaluate with Published Datasets - -{{platform_name}} provides a streamlined API to evaluate large language models with publicly available datasets, offering over 130 industry benchmarks to run with evaluation jobs. - -Benchmarks provide standardized methods for comparing model performance across different capabilities. These benchmarks are widely used in the research community and provide reliable, reproducible metrics for model assessment. - -Refer to the [Run an LLM Judge Evaluation](../tutorials/run-llm-judge-evaluation.md) tutorial for details on using evaluator jobs and result handling. - -- **Standard Datasets**: Most benchmarks include predefined datasets widely used in research. -- **Reproducible Metrics**: Use established methodologies to calculate metrics. -- **Community Standards**: You can compare results across different models and research groups. - ---8<-- "evaluator/benchmarks/discover-industry-benchmarks.md" - -### Choosing a Benchmark Variant - -Many benchmarks offer multiple variants optimized for different model types: - -| Variant | Endpoint | Description | -|---------|----------|-------------| -| `-instruct` | `/v1/chat/completions` | Zero-shot evaluation for instruction-tuned models | -| Base (no suffix) | `/v1/completions` | Few-shot evaluation for base models (requires `tokenizer` param) | -| `-nemo` | `/v1/chat/completions` | Optimized prompts for NVIDIA NeMo models; often no judge required | -| `-cot` | `/v1/chat/completions` | Chain-of-thought prompting for improved reasoning accuracy | - -### Common Parameters - -| Parameter | Description | -|-----------|-------------| -| `parallelism` | Number of concurrent inference requests. Higher values increase throughput but can hit rate limits. | -| `limit_samples` | Evaluate only the first N samples. Useful for testing before running full evaluations. | -| `hf_token` | Reference to a secret containing your Hugging Face token for accessing gated datasets. | -| `tokenizer` | Hugging Face tokenizer ID, required for completions-based benchmarks. | - -!!! tip - The `model` field in all benchmark examples below accepts either an inline model definition or a model reference string (for example, `"my-workspace/my-model"`). Refer to [Model Configuration](../metrics/model-configuration.md) for details. - -## Advanced Reasoning - -Evaluate reasoning capabilities of large language models through complex tasks with datasets like GPQA, BIG-Bench Hard (BBH), or Multistep Soft Reasoning (MuSR). - -* Label: `eval_category.advanced_reasoning` - -### Available Benchmarks - -To get the latest benchmarks available for your system, [filter with the `eval_category.advanced_reasoning` label](#discover-industry-benchmarks). - -| Benchmark | Description | Required Params | -|-----------|-------------|-----------------| -| `system/gpqa-diamond` | [GPQA](https://arxiv.org/abs/2311.12022) Diamond subset—198 graduate-level science questions | `hf_token` | -| `system/gpqa-extended` | GPQA Extended subset—546 questions in biology, physics, chemistry | `hf_token` | -| `system/gpqa-main` | GPQA Main subset—448 questions | `hf_token` | -| `system/gpqa-diamond-nemo` | GPQA Diamond with NeMo alignment template | `hf_token` | -| `system/gpqa-diamond-cot` | GPQA Diamond with chain-of-thought prompting | `hf_token` | -| `system/gpqa` | GPQA few-shot evaluation ¹ | `hf_token`, `tokenizer` | -| `system/bbh-instruct` | [BIG-Bench Hard](https://github.com/google/BIG-bench)—23 challenging reasoning tasks | `hf_token` | -| `system/bbh` | BIG-Bench Hard ¹ | `hf_token`, `tokenizer` | -| `system/musr` | [MuSR](https://arxiv.org/abs/2310.16049)—multistep reasoning through narrative problems ¹ | `hf_token`, `tokenizer` | - -**¹** Completions-only: Requires `/v1/completions` endpoint and `tokenizer` parameter. - -### Examples - -```python -import os -from nemo_platform import NeMoPlatform -from nemo_platform.types.evaluation import ( - SystemBenchmarkOnlineJobParam, - RunConfig, -) - -client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="my-workspace", # Replace with your workspace or use "default" -) -``` - -=== "GPQA Diamond" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="GPQA Diamond evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/gpqa-diamond", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -=== "GPQA Extended" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="GPQA Extended evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/gpqa-extended", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -=== "GPQA Main" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="GPQA Main evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/gpqa-main", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -=== "GPQA Diamond (NeMo)" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="GPQA Diamond evaluation with NeMo template", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/gpqa-diamond-nemo", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -=== "GPQA Diamond CoT" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="GPQA Diamond chain-of-thought evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/gpqa-diamond-cot", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -=== "GPQA (Completions)" - - Requires a `/v1/completions` endpoint. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="GPQA few-shot evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/gpqa", - model={ - "url": "/v1/completions", - "name": "", - }, - params=RunConfig(parallelism=16), - benchmark_params={ - "hf_token": "hf_token", - "tokenizer": "", - }, - ), - ) - ``` - -=== "BBH Instruct" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="BIG-Bench Hard evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/bbh-instruct", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -=== "BBH (Completions)" - - Requires a `/v1/completions` endpoint. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="BIG-Bench Hard evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/bbh", - model={ - "url": "/v1/completions", - "name": "", - }, - params=RunConfig(parallelism=16), - benchmark_params={ - "hf_token": "hf_token", - "tokenizer": "", - }, - ), - ) - ``` - -=== "MuSR (Completions)" - - Requires a `/v1/completions` endpoint. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="Multistep Soft Reasoning evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/musr", - model={ - "url": "/v1/completions", - "name": "", - }, - params=RunConfig(parallelism=16), - benchmark_params={ - "hf_token": "hf_token", - "tokenizer": "", - }, - ), - ) - ``` - -!!! note - - --8<-- "evaluator/benchmarks/hf-secret.md" - -### Results - -All advanced reasoning benchmarks produce an **accuracy** score (0.0–1.0) measuring the proportion of correct answers: - -- **GPQA**: Multiple-choice accuracy (random baseline = 25%) -- **BBH**: Exact match accuracy across 23 reasoning tasks -- **MuSR**: Accuracy on multistep reasoning narratives - -```python -# Get results after the job completes -aggregate = client.evaluation.benchmark_jobs.results.aggregate_scores.download( - job=job.name, -) - -# Print accuracy -for score in aggregate.scores: - print(f"{score.name}: {score.mean:.1%}") -``` - -For detailed results analysis, refer to [eval-benchmark-results](results.md). - -## Instruction Following - -Evaluate a model's ability to follow explicit formatting and structural instructions such as "include keyword x" or "use format y." - -* Label: `instruction_following` - -### Available Benchmarks - -| Benchmark | Description | Required Params | -|-----------|-------------|-----------------| -| `system/ifeval` | [IFEval](https://arxiv.org/abs/2311.07911)—500 prompts testing adherence to verifiable instructions | `hf_token` | - -### Examples - -```python -import os -from nemo_platform import NeMoPlatform -from nemo_platform.types.evaluation import ( - SystemBenchmarkOnlineJobParam, - RunConfig, -) - -client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="my-workspace", # Replace with your workspace or use "default" -) - -job = client.evaluation.benchmark_jobs.create( - description="IFEval instruction following evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/ifeval", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), -) -``` - -### Results - -IFEval produces multiple accuracy scores measuring instruction compliance: - -- **Prompt-level accuracy**: Percentage of prompts where all instructions were followed -- **Instruction-level accuracy**: Percentage of individual instructions followed across all prompts - -```python -aggregate = client.evaluation.benchmark_jobs.results.aggregate_scores.download( - job=job.name -) -for score in aggregate.scores: - print(f"{score.name}: {score.mean:.1%}") -``` - -For detailed results analysis, refer to [eval-benchmark-results](results.md). - -## Language Understanding - -Evaluate knowledge and reasoning across diverse subjects using MMLU (Massive Multitask Language Understanding) benchmarks covering 57 subjects across STEM, humanities, social sciences, and more. - -* Label: `language_understanding` - -### Available Benchmarks - -| Benchmark | Description | Required Params | -|-----------|-------------|-----------------| -| `system/mmlu` | [MMLU](https://arxiv.org/abs/2009.03300)—57 subjects, few-shot evaluation ¹ | `hf_token`, `tokenizer` | -| `system/mmlu-instruct` | MMLU zero-shot with single-letter response format | `hf_token` | -| `system/mmlu-pro` | [MMLU-Pro](https://arxiv.org/abs/2406.01574)—10 answer choices, more rigorous ¹ | `hf_token`, `tokenizer` | -| `system/mmlu-pro-instruct` | MMLU-Pro zero-shot with chat template | `hf_token` | -| `system/mmlu-redux` | [MMLU-Redux](https://arxiv.org/abs/2406.04127)—3,000 re-annotated questions ¹ | `hf_token`, `tokenizer` | -| `system/mmlu-redux-instruct` | MMLU-Redux zero-shot with chat template | `hf_token` | -| `system/wikilingua` | [WikiLingua](https://arxiv.org/abs/2010.03093)—cross-lingual summarization | `hf_token` | -| `system/mmlu-{lang}` | [Global-MMLU](https://huggingface.co/datasets/CohereForAI/Global-MMLU) in 30+ languages ² | `hf_token` | - -**¹** Completions-only: Requires `/v1/completions` endpoint and `tokenizer` parameter. - -**²** Supported languages: `am`, `ar`, `bn`, `cs`, `de`, `el`, `en`, `es`, `fa`, `fil`, `fr`, `ha`, `he`, `hi`, `id`, `ig`, `it`, `ja`, `ko`, `ky`, `lt`, `mg`, `ms`, `ne`, `nl`, `ny`, `pl`, `pt`, `ro`, `ru`, `si`, `sn`, `so`, `sr`, `sv`, `sw`, `te`, `tr`, `uk`, `vi`, `yo` - -### Examples - -```python -import os -from nemo_platform import NeMoPlatform -from nemo_platform.types.evaluation import ( - SystemBenchmarkOnlineJobParam, - RunConfig, -) - -client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="my-workspace", # Replace with your workspace or use "default" -) -``` - -=== "MMLU Instruct" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="MMLU zero-shot evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/mmlu-instruct", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -=== "MMLU (Completions)" - - Requires a `/v1/completions` endpoint. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="MMLU few-shot evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/mmlu", - model={ - "url": "/v1/completions", - "name": "", - }, - params=RunConfig(parallelism=16), - benchmark_params={ - "hf_token": "hf_token", - "tokenizer": "", - }, - ), - ) - ``` - -=== "MMLU-Pro Instruct" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="MMLU-Pro zero-shot evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/mmlu-pro-instruct", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -=== "MMLU-Pro (Completions)" - - Requires a `/v1/completions` endpoint. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="MMLU-Pro few-shot evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/mmlu-pro", - model={ - "url": "/v1/completions", - "name": "", - }, - params=RunConfig(parallelism=16), - benchmark_params={ - "hf_token": "hf_token", - "tokenizer": "", - }, - ), - ) - ``` - -=== "MMLU-Redux Instruct" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="MMLU-Redux zero-shot evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/mmlu-redux-instruct", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -=== "MMLU Spanish" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="Global-MMLU Spanish evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/mmlu-es", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -=== "WikiLingua" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="WikiLingua cross-lingual summarization", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/wikilingua", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -### Results - -Language understanding benchmarks produce accuracy scores: - -- **MMLU/MMLU-Pro/MMLU-Redux**: Multiple-choice accuracy across subjects (random baseline = 25% for MMLU, 10% for MMLU-Pro) -- **WikiLingua**: ROUGE scores for summarization quality - -```python -aggregate = client.evaluation.benchmark_jobs.results.aggregate_scores.download( - job=job.name -) -for score in aggregate.scores: - print(f"{score.name}: {score.mean:.1%}") -``` - -For detailed results analysis, refer to [eval-benchmark-results](results.md). - -## Math & Reasoning - -Evaluate mathematical reasoning abilities from grade school arithmetic to competition-level mathematics. - -* Label: `math` - -### Available Benchmarks - -| Benchmark | Description | Required Params | -|-----------|-------------|-----------------| -| `system/gsm8k` | [GSM8K](https://arxiv.org/abs/2110.14168)—1,319 grade school math problems ¹ | `hf_token`, `tokenizer` | -| `system/gsm8k-cot-instruct` | GSM8K with chain-of-thought zero-shot | `hf_token` | -| `system/mgsm` | [MGSM](https://arxiv.org/abs/2210.03057)—multilingual math (10 languages) ¹ | `hf_token`, `tokenizer` | -| `system/mgsm-cot` | MGSM with chain-of-thought prompting | `hf_token` | -| `system/aime-2024` | [AIME 2024](https://artofproblemsolving.com/wiki/index.php/2024_AIME_I)—competition math ² | `judge` | -| `system/aime-2025` | AIME 2025—competition math ² | `judge` | -| `system/aime-2024-nemo` | AIME 2024 with NeMo template | — | -| `system/aime-2025-nemo` | AIME 2025 with NeMo template | — | -| `system/math-test-500` | [MATH](https://arxiv.org/abs/2103.03874) test set (500 problems) ² | `judge` | -| `system/math-test-500-nemo` | MATH test with NeMo template | — | -| `system/aa-aime-2024` | AIME 2024 (Artificial Analysis setup) ² | `judge` | -| `system/aa-math-test-500` | MATH test (Artificial Analysis setup) ² | `judge` | - -**¹** Completions-only: Requires `/v1/completions` endpoint and `tokenizer` parameter. - -**²** Judge required: Requires a judge model to evaluate free-form math responses. - -!!! info - For math benchmarks requiring a judge, use a model with strong instruction-following capabilities (70B+ parameters recommended). Smaller models may produce malformed judge outputs. - -### Examples - -```python -import os -from nemo_platform import NeMoPlatform -from nemo_platform.types.evaluation import ( - SystemBenchmarkOnlineJobParam, - RunConfig, -) - -client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="my-workspace", # Replace with your workspace or use "default" -) -``` - -=== "GSM8K CoT Instruct" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="GSM8K chain-of-thought evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/gsm8k-cot-instruct", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -=== "GSM8K (Completions)" - - Requires a `/v1/completions` endpoint. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="GSM8K few-shot evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/gsm8k", - model={ - "url": "/v1/completions", - "name": "", - }, - params=RunConfig(parallelism=16), - benchmark_params={ - "hf_token": "hf_token", - "tokenizer": "", - }, - ), - ) - ``` - -=== "MGSM CoT" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="MGSM multilingual math evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/mgsm-cot", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), - ) - ``` - -=== "AIME 2025 (NeMo)" - - No judge required with NeMo template. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="AIME 2025 competition math", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/aime-2025-nemo", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - ), - ) - ``` - -=== "AIME 2025 (Judge)" - - Requires a judge model. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="AIME 2025 competition math with judge", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/aime-2025", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={ - "judge": { - "model": { - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - } - } - }, - ), - ) - ``` - -=== "MATH Test 500 (NeMo)" - - No judge required with NeMo template. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="MATH test set evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/math-test-500-nemo", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - ), - ) - ``` - -### Results - -Math benchmarks produce accuracy scores: - -- **GSM8K/MGSM**: Exact match accuracy on final numerical answers -- **AIME/MATH**: Correctness as judged by the judge model (or exact match for NeMo variants) - -```python -aggregate = client.evaluation.benchmark_jobs.results.aggregate_scores.download( - job=job.name -) -for score in aggregate.scores: - print(f"{score.name}: {score.mean:.1%}") -``` - -For detailed results analysis, refer to [eval-benchmark-results](results.md). - - -## Content Safety - -Evaluate model safety risks including vulnerability to generate harmful, biased, or misleading content. - -* Label: `content_safety` - -### Available Benchmarks - -| Benchmark | Description | Required Params | -|-----------|-------------|-----------------| -| `system/aegis-v2` | [AEGIS 2.0](https://aclanthology.org/2025.naacl-long.306/)—12 hazard categories using Nemotron Safety Guard | `hf_token`, `judge` | -| `system/wildguard` | [WildGuard](https://arxiv.org/abs/2406.18495)—privacy, misinformation, harmful language | `hf_token`, `judge` | - -!!! info - Safety benchmarks require specific judge models deployed with a `/v1/completions` endpoint (not chat/completions): - - | Benchmark | Required Judge Model | Model ID | - |-----------|---------------------|----------| - | `system/aegis-v2` | [Llama Nemotron Safety Guard V2](https://huggingface.co/nvidia/llama-3.1-nemoguard-8b-content-safety) | `nvidia/llama-3.1-nemoguard-8b-content-safety` | - | `system/wildguard` | [WildGuard](https://huggingface.co/allenai/wildguard) | `allenai/wildguard` | - - -The benchmarks can take up to 1-3 hours. - -### Examples - -```python -import os -from nemo_platform import NeMoPlatform -from nemo_platform.types.evaluation import ( - SystemBenchmarkOnlineJobParam, - RunConfig, -) - -client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="my-workspace", # Replace with your workspace or use "default" -) -``` - -=== "AEGIS-v2" - - Requires Llama Nemotron Safety Guard V2 judge. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="AEGIS-v2 content safety evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/aegis-v2", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={ - "hf_token": "hf_token", - "judge": { - "model": { - "url": "/v1/completions", - "name": "nvidia/llama-3.1-nemoguard-8b-content-safety", - } - }, - }, - ), - ) - ``` - -=== "WildGuard" - - Requires WildGuard judge. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="WildGuard content safety evaluation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/wildguard", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={ - "hf_token": "hf_token", - "judge": { - "model": { - "url": "/v1/completions", - "name": "allenai/wildguard", - } - }, - }, - ), - ) - ``` - -### Results - -Safety benchmarks produce category-level safety rates: - -- **AEGIS-v2**: Safe/unsafe classification across 12 hazard categories (violence, hate, sexual content, and so on) -- **WildGuard**: Safe/unsafe classification for privacy, misinformation, harmful language, malicious use - -```python -aggregate = client.evaluation.benchmark_jobs.results.aggregate_scores.download( - job=job.name -) -for score in aggregate.scores: - print(f"{score.name}: {score.mean:.1%}") -``` - -For detailed results analysis, refer to [eval-benchmark-results](results.md). - -### Troubleshooting Content Safety Benchmarks - -Refer to [Evaluator Troubleshooting](../../troubleshooting/evaluator.md) for general troubleshooting steps for failed evaluation jobs. - -
-This section covers common issues for the safety harness. - -#### Hugging Face Error - -Evaluations with safety harness requires Hugging Face access to the respective dataset and model tokenizer. If your job fails with the following errors, visit [https://huggingface.co/](https://huggingface.co/) and log in to request access to the dataset or model. - -```text -datasets.exceptions.DatasetNotFoundError: Dataset 'allenai/wildguardmix' is a gated dataset on the Hub. Visit the dataset page at https://huggingface.co/datasets/allenai/wildguardmix to ask for access. -``` - -```text -GatedRepoError: 403 Client Error. - -Cannot access gated repo for url https://huggingface.co//resolve/main/tokenizer_config.json. -Your request to access model is awaiting a review from the repo authors. -``` - -#### Incompatible Judge Model - -Using an unsupported judge model results in a job error. The `aegis-v2` evaluation requires Llama Nemotron Safety Guard V2 judge and `wildguard` evaluation requires `allenai/wildguard` judge. `KeyError` is an example error for the wrong judge model like the following error. - -```text -Metrics calculated - - - Evaluation Metrics -┏━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓ -┃ Safety Category ┃ Average Count ┃ -┡━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩ -│ ERROR │ 5.0 │ -└─────────────────┴───────────────┘ - -... - -Subprocess finished with return code: 0 -{'ERROR': 5.0} -Traceback (most recent call last): -... -"/usr/local/lib/python3.10/site-packages/core_evals/safety_eval/__init__.py", line 14, in parse_output - return parse_output(output_dir) - File "/usr/local/lib/python3.10/site-packages/core_evals/safety_eval/output.py", line 16, in parse_output - safety_rate = data['safe'] / sum(data.values()) -KeyError: 'safe' -``` - -#### Unexpected Reasoning Traces - -Safety evaluations do not support reasoning traces and may result in the job error below. - -```text -ERROR There are at least 2 MUT (model under test) responses that start with . Reasoning traces should not be evaluated. Exiting. -``` - -If the model or judge outputs reasoning traces like `reasoning contextanswer`, configure the job to only include the the final answer after the reasoning token (e.g. ``) with `reasoning.end_token`. Consider specifying `inference.max_tokens` to a reasonable limit for the model's chain of thought to conclude with the expected reasoning end token in order for the reasoning context to be properly omitted for evaluation. - -Additionally, if you are encountering this error, it could be caused by the model exceeding its token limit resulting in the full response being consumed by the model thinking. These results can be dropped by setting the `reasoning.include_if_not_finished` parameter. - -```python -from nemo_platform.types.evaluation import ( - RunConfig, - InferenceParams, - ReasoningParams, -) - -params = RunConfig( - inference=InferenceParams(max_tokens=1024), - reasoning=ReasoningParams( - end_token="", - include_if_not_finished=False, - ), -) -``` - -
- -## Code - -Evaluate code generation capabilities using functional correctness benchmarks that test synthesis of working programs. - -* Label: `code` - -### Available Benchmarks - -| Benchmark | Description | Required Params | -|-----------|-------------|-----------------| -| `system/humaneval` | [HumanEval](https://arxiv.org/abs/2107.03374)—164 Python problems ¹ | — | -| `system/humaneval-instruct` | HumanEval for instruction-tuned models | — | -| `system/humanevalplus` | [HumanEval+](https://arxiv.org/abs/2305.01210)—80x more test cases ¹ | — | -| `system/mbpp` | [MBPP](https://arxiv.org/abs/2108.07732)—Python programming problems | — | -| `system/mbppplus` | [MBPP+](https://arxiv.org/abs/2305.01210)—35x more test cases | — | -| `system/mbppplus-nemo` | MBPP+ with NeMo template | — | -| `system/multiple-{lang}` | [MultiPL-E](https://arxiv.org/abs/2208.08227)—HumanEval in 20+ languages ¹ | — | - -**¹** Completions-only: Requires `/v1/completions` endpoint. - -**MultiPL-E languages**: `clj` (Clojure), `cpp` (C++), `cs` (C#), `d`, `elixir`, `go`, `hs` (Haskell), `java`, `jl` (Julia), `js` (JavaScript), `lua`, `ml` (OCaml), `php`, `pl` (Perl), `r`, `rb` (Ruby), `rkt` (Racket), `rs` (Rust), `scala`, `sh` (Bash), `swift` - -The benchmarks can take up to 1–5 hours. - -### Examples - -```python -import os -from nemo_platform import NeMoPlatform -from nemo_platform.types.evaluation import ( - SystemBenchmarkOnlineJobParam, - RunConfig, -) - -client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="my-workspace", # Replace with your workspace or use "default" -) -``` - -=== "HumanEval Instruct" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="HumanEval code generation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/humaneval-instruct", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - ), - ) - ``` - -=== "HumanEval (Completions)" - - Requires a `/v1/completions` endpoint. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="HumanEval code generation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/humaneval", - model={ - "url": "/v1/completions", - "name": "", - }, - params=RunConfig(parallelism=16), - ), - ) - ``` - -=== "HumanEval+ (Completions)" - - Extended test suite with 80x more test cases. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="HumanEval+ code generation", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/humanevalplus", - model={ - "url": "/v1/completions", - "name": "", - }, - params=RunConfig(parallelism=16), - ), - ) - ``` - -=== "MBPP+ (NeMo)" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="MBPP+ Python programming", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/mbppplus-nemo", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - ), - ) - ``` - -=== "MBPP+" - - ```python - job = client.evaluation.benchmark_jobs.create( - description="MBPP+ Python programming", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/mbppplus", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - ), - ) - ``` - -=== "MultiPL-E JavaScript" - - Requires a `/v1/completions` endpoint. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="MultiPL-E JavaScript", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/multiple-js", - model={ - "url": "/v1/completions", - "name": "", - }, - params=RunConfig(parallelism=16), - ), - ) - ``` - -=== "MultiPL-E Rust" - - Requires a `/v1/completions` endpoint. - - ```python - job = client.evaluation.benchmark_jobs.create( - description="MultiPL-E Rust", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/multiple-rs", - model={ - "url": "/v1/completions", - "name": "", - }, - params=RunConfig(parallelism=16), - ), - ) - ``` - -### Results - -Code benchmarks produce pass@k metrics measuring functional correctness: - -- **pass@1**: Percentage of problems solved with one attempt -- **pass@10**: Percentage of problems solved within 10 attempts (when `n_samples` > 1) - -```python -aggregate = client.evaluation.benchmark_jobs.results.aggregate_scores.download( - job=job.name -) -for score in aggregate.scores: - print(f"{score.name}: {score.mean:.1%}") -``` - -For detailed results analysis, refer to [eval-benchmark-results](results.md). - -!!! tip - **Want to experiment first?** You can try these benchmarks using the [open-source NeMo Evaluator SDK](https://github.com/NVIDIA-NeMo/evaluator) before deploying the platform. The SDK provides a lightweight way to test evaluation workflows locally. - -## Run Benchmark Job - -Create a workspace if you have not already. - -```python -workspace = "my-workspace" - -client.workspaces.create(name=workspace) -``` - -Create an evaluation job with a benchmark that satisfy the required and optional parameters. - -!!! note - For benchmarks that require a Hugging Face token or other API keys for external services, create the secret to be referenced by the job. - - --8<-- "evaluator/benchmarks/hf-secret.md" - -```python -from nemo_platform.types.evaluation import ( - SystemBenchmarkOnlineJobParam, - RunConfig, -) - -job = client.evaluation.benchmark_jobs.create( - description="Example running system benchmark to evaluate my model's advanced reasoning capabilities.", - spec=SystemBenchmarkOnlineJobParam( - benchmark="system/gpqa-diamond-cot", - model={ - "url": "/v1", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - }, - params=RunConfig(parallelism=16), - benchmark_params={"hf_token": "hf_token"}, - ), -) -``` - -## Job Management - -After creating a job, navigate to [Benchmark Job Management](job-management.md) to oversee its execution and monitor progress. diff --git a/docs/evaluator/benchmarks/industry.mdx b/docs/evaluator/benchmarks/industry.mdx new file mode 100644 index 0000000000..247a512f04 --- /dev/null +++ b/docs/evaluator/benchmarks/industry.mdx @@ -0,0 +1,1172 @@ +--- +title: "Industry Benchmarks" +description: "" +--- + + +## Evaluate with Published Datasets + +NeMo Platform provides a streamlined API to evaluate large language models with publicly available datasets, offering over 130 industry benchmarks to run with evaluation jobs. + +Benchmarks provide standardized methods for comparing model performance across different capabilities. These benchmarks are widely used in the research community and provide reliable, reproducible metrics for model assessment. + +Refer to the [Run an LLM Judge Evaluation](/documentation/evaluate-models/tutorials/run-llm-as-a-judge-evaluation) tutorial for details on using evaluator jobs and result handling. + +- **Standard Datasets**: Most benchmarks include predefined datasets widely used in research. +- **Reproducible Metrics**: Use established methodologies to calculate metrics. +- **Community Standards**: You can compare results across different models and research groups. + + +### Choosing a Benchmark Variant + +Many benchmarks offer multiple variants optimized for different model types: + +| Variant | Endpoint | Description | +|---------|----------|-------------| +| `-instruct` | `/v1/chat/completions` | Zero-shot evaluation for instruction-tuned models | +| Base (no suffix) | `/v1/completions` | Few-shot evaluation for base models (requires `tokenizer` param) | +| `-nemo` | `/v1/chat/completions` | Optimized prompts for NVIDIA NeMo models; often no judge required | +| `-cot` | `/v1/chat/completions` | Chain-of-thought prompting for improved reasoning accuracy | + +### Common Parameters + +| Parameter | Description | +|-----------|-------------| +| `parallelism` | Number of concurrent inference requests. Higher values increase throughput but can hit rate limits. | +| `limit_samples` | Evaluate only the first N samples. Useful for testing before running full evaluations. | +| `hf_token` | Reference to a secret containing your Hugging Face token for accessing gated datasets. | +| `tokenizer` | Hugging Face tokenizer ID, required for completions-based benchmarks. | + + + +The `model` field in all benchmark examples below accepts either an inline model definition or a model reference string (for example, `"my-workspace/my-model"`). Refer to [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) for details. + + +## Advanced Reasoning + +Evaluate reasoning capabilities of large language models through complex tasks with datasets like GPQA, BIG-Bench Hard (BBH), or Multistep Soft Reasoning (MuSR). + +* Label: `eval_category.advanced_reasoning` + +### Available Benchmarks + +To get the latest benchmarks available for your system, [filter with the `eval_category.advanced_reasoning` label](#discover-industry-benchmarks). + +| Benchmark | Description | Required Params | +|-----------|-------------|-----------------| +| `system/gpqa-diamond` | [GPQA](https://arxiv.org/abs/2311.12022) Diamond subset—198 graduate-level science questions | `hf_token` | +| `system/gpqa-extended` | GPQA Extended subset—546 questions in biology, physics, chemistry | `hf_token` | +| `system/gpqa-main` | GPQA Main subset—448 questions | `hf_token` | +| `system/gpqa-diamond-nemo` | GPQA Diamond with NeMo alignment template | `hf_token` | +| `system/gpqa-diamond-cot` | GPQA Diamond with chain-of-thought prompting | `hf_token` | +| `system/gpqa` | GPQA few-shot evaluation ¹ | `hf_token`, `tokenizer` | +| `system/bbh-instruct` | [BIG-Bench Hard](https://github.com/google/BIG-bench)—23 challenging reasoning tasks | `hf_token` | +| `system/bbh` | BIG-Bench Hard ¹ | `hf_token`, `tokenizer` | +| `system/musr` | [MuSR](https://arxiv.org/abs/2310.16049)—multistep reasoning through narrative problems ¹ | `hf_token`, `tokenizer` | + +**¹** Completions-only: Requires `/v1/completions` endpoint and `tokenizer` parameter. + +### Examples + +```python +import os +from nemo_platform import NeMoPlatform +from nemo_platform.types.evaluation import ( + SystemBenchmarkOnlineJobParam, + RunConfig, +) + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="my-workspace", # Replace with your workspace or use "default" +) +``` + + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="GPQA Diamond evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/gpqa-diamond", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="GPQA Extended evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/gpqa-extended", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="GPQA Main evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/gpqa-main", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="GPQA Diamond evaluation with NeMo template", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/gpqa-diamond-nemo", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="GPQA Diamond chain-of-thought evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/gpqa-diamond-cot", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +Requires a `/v1/completions` endpoint. + +```python +job = client.evaluation.benchmark_jobs.create( + description="GPQA few-shot evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/gpqa", + model={ + "url": "/v1/completions", + "name": "", + }, + params=RunConfig(parallelism=16), + benchmark_params={ + "hf_token": "hf_token", + "tokenizer": "", + }, + ), +) +``` + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="BIG-Bench Hard evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/bbh-instruct", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +Requires a `/v1/completions` endpoint. + +```python +job = client.evaluation.benchmark_jobs.create( + description="BIG-Bench Hard evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/bbh", + model={ + "url": "/v1/completions", + "name": "", + }, + params=RunConfig(parallelism=16), + benchmark_params={ + "hf_token": "hf_token", + "tokenizer": "", + }, + ), +) +``` + + + + +Requires a `/v1/completions` endpoint. + +```python +job = client.evaluation.benchmark_jobs.create( + description="Multistep Soft Reasoning evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/musr", + model={ + "url": "/v1/completions", + "name": "", + }, + params=RunConfig(parallelism=16), + benchmark_params={ + "hf_token": "hf_token", + "tokenizer": "", + }, + ), +) +``` + + + + + + + + + +### Results + +All advanced reasoning benchmarks produce an **accuracy** score (0.0–1.0) measuring the proportion of correct answers: + +- **GPQA**: Multiple-choice accuracy (random baseline = 25%) +- **BBH**: Exact match accuracy across 23 reasoning tasks +- **MuSR**: Accuracy on multistep reasoning narratives + +```python +# Get results after the job completes +aggregate = client.evaluation.benchmark_jobs.results.aggregate_scores.download( + job=job.name, +) + +# Print accuracy +for score in aggregate.scores: + print(f"{score.name}: {score.mean:.1%}") +``` + +For detailed results analysis, refer to [eval-benchmark-results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results). + +## Instruction Following + +Evaluate a model's ability to follow explicit formatting and structural instructions such as "include keyword x" or "use format y." + +* Label: `instruction_following` + +### Available Benchmarks + +| Benchmark | Description | Required Params | +|-----------|-------------|-----------------| +| `system/ifeval` | [IFEval](https://arxiv.org/abs/2311.07911)—500 prompts testing adherence to verifiable instructions | `hf_token` | + +### Examples + +```python +import os +from nemo_platform import NeMoPlatform +from nemo_platform.types.evaluation import ( + SystemBenchmarkOnlineJobParam, + RunConfig, +) + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="my-workspace", # Replace with your workspace or use "default" +) + +job = client.evaluation.benchmark_jobs.create( + description="IFEval instruction following evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/ifeval", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + +### Results + +IFEval produces multiple accuracy scores measuring instruction compliance: + +- **Prompt-level accuracy**: Percentage of prompts where all instructions were followed +- **Instruction-level accuracy**: Percentage of individual instructions followed across all prompts + +```python +aggregate = client.evaluation.benchmark_jobs.results.aggregate_scores.download( + job=job.name +) +for score in aggregate.scores: + print(f"{score.name}: {score.mean:.1%}") +``` + +For detailed results analysis, refer to [eval-benchmark-results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results). + +## Language Understanding + +Evaluate knowledge and reasoning across diverse subjects using MMLU (Massive Multitask Language Understanding) benchmarks covering 57 subjects across STEM, humanities, social sciences, and more. + +* Label: `language_understanding` + +### Available Benchmarks + +| Benchmark | Description | Required Params | +|-----------|-------------|-----------------| +| `system/mmlu` | [MMLU](https://arxiv.org/abs/2009.03300)—57 subjects, few-shot evaluation ¹ | `hf_token`, `tokenizer` | +| `system/mmlu-instruct` | MMLU zero-shot with single-letter response format | `hf_token` | +| `system/mmlu-pro` | [MMLU-Pro](https://arxiv.org/abs/2406.01574)—10 answer choices, more rigorous ¹ | `hf_token`, `tokenizer` | +| `system/mmlu-pro-instruct` | MMLU-Pro zero-shot with chat template | `hf_token` | +| `system/mmlu-redux` | [MMLU-Redux](https://arxiv.org/abs/2406.04127)—3,000 re-annotated questions ¹ | `hf_token`, `tokenizer` | +| `system/mmlu-redux-instruct` | MMLU-Redux zero-shot with chat template | `hf_token` | +| `system/wikilingua` | [WikiLingua](https://arxiv.org/abs/2010.03093)—cross-lingual summarization | `hf_token` | +| `system/mmlu-{lang}` | [Global-MMLU](https://huggingface.co/datasets/CohereForAI/Global-MMLU) in 30+ languages ² | `hf_token` | + +**¹** Completions-only: Requires `/v1/completions` endpoint and `tokenizer` parameter. + +**²** Supported languages: `am`, `ar`, `bn`, `cs`, `de`, `el`, `en`, `es`, `fa`, `fil`, `fr`, `ha`, `he`, `hi`, `id`, `ig`, `it`, `ja`, `ko`, `ky`, `lt`, `mg`, `ms`, `ne`, `nl`, `ny`, `pl`, `pt`, `ro`, `ru`, `si`, `sn`, `so`, `sr`, `sv`, `sw`, `te`, `tr`, `uk`, `vi`, `yo` + +### Examples + +```python +import os +from nemo_platform import NeMoPlatform +from nemo_platform.types.evaluation import ( + SystemBenchmarkOnlineJobParam, + RunConfig, +) + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="my-workspace", # Replace with your workspace or use "default" +) +``` + + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="MMLU zero-shot evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/mmlu-instruct", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +Requires a `/v1/completions` endpoint. + +```python +job = client.evaluation.benchmark_jobs.create( + description="MMLU few-shot evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/mmlu", + model={ + "url": "/v1/completions", + "name": "", + }, + params=RunConfig(parallelism=16), + benchmark_params={ + "hf_token": "hf_token", + "tokenizer": "", + }, + ), +) +``` + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="MMLU-Pro zero-shot evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/mmlu-pro-instruct", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +Requires a `/v1/completions` endpoint. + +```python +job = client.evaluation.benchmark_jobs.create( + description="MMLU-Pro few-shot evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/mmlu-pro", + model={ + "url": "/v1/completions", + "name": "", + }, + params=RunConfig(parallelism=16), + benchmark_params={ + "hf_token": "hf_token", + "tokenizer": "", + }, + ), +) +``` + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="MMLU-Redux zero-shot evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/mmlu-redux-instruct", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="Global-MMLU Spanish evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/mmlu-es", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="WikiLingua cross-lingual summarization", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/wikilingua", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +### Results + +Language understanding benchmarks produce accuracy scores: + +- **MMLU/MMLU-Pro/MMLU-Redux**: Multiple-choice accuracy across subjects (random baseline = 25% for MMLU, 10% for MMLU-Pro) +- **WikiLingua**: ROUGE scores for summarization quality + +```python +aggregate = client.evaluation.benchmark_jobs.results.aggregate_scores.download( + job=job.name +) +for score in aggregate.scores: + print(f"{score.name}: {score.mean:.1%}") +``` + +For detailed results analysis, refer to [eval-benchmark-results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results). + +## Math & Reasoning + +Evaluate mathematical reasoning abilities from grade school arithmetic to competition-level mathematics. + +* Label: `math` + +### Available Benchmarks + +| Benchmark | Description | Required Params | +|-----------|-------------|-----------------| +| `system/gsm8k` | [GSM8K](https://arxiv.org/abs/2110.14168)—1,319 grade school math problems ¹ | `hf_token`, `tokenizer` | +| `system/gsm8k-cot-instruct` | GSM8K with chain-of-thought zero-shot | `hf_token` | +| `system/mgsm` | [MGSM](https://arxiv.org/abs/2210.03057)—multilingual math (10 languages) ¹ | `hf_token`, `tokenizer` | +| `system/mgsm-cot` | MGSM with chain-of-thought prompting | `hf_token` | +| `system/aime-2024` | [AIME 2024](https://artofproblemsolving.com/wiki/index.php/2024_AIME_I)—competition math ² | `judge` | +| `system/aime-2025` | AIME 2025—competition math ² | `judge` | +| `system/aime-2024-nemo` | AIME 2024 with NeMo template | — | +| `system/aime-2025-nemo` | AIME 2025 with NeMo template | — | +| `system/math-test-500` | [MATH](https://arxiv.org/abs/2103.03874) test set (500 problems) ² | `judge` | +| `system/math-test-500-nemo` | MATH test with NeMo template | — | +| `system/aa-aime-2024` | AIME 2024 (Artificial Analysis setup) ² | `judge` | +| `system/aa-math-test-500` | MATH test (Artificial Analysis setup) ² | `judge` | + +**¹** Completions-only: Requires `/v1/completions` endpoint and `tokenizer` parameter. + +**²** Judge required: Requires a judge model to evaluate free-form math responses. + + + +For math benchmarks requiring a judge, use a model with strong instruction-following capabilities (70B+ parameters recommended). Smaller models may produce malformed judge outputs. + + +### Examples + +```python +import os +from nemo_platform import NeMoPlatform +from nemo_platform.types.evaluation import ( + SystemBenchmarkOnlineJobParam, + RunConfig, +) + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="my-workspace", # Replace with your workspace or use "default" +) +``` + + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="GSM8K chain-of-thought evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/gsm8k-cot-instruct", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +Requires a `/v1/completions` endpoint. + +```python +job = client.evaluation.benchmark_jobs.create( + description="GSM8K few-shot evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/gsm8k", + model={ + "url": "/v1/completions", + "name": "", + }, + params=RunConfig(parallelism=16), + benchmark_params={ + "hf_token": "hf_token", + "tokenizer": "", + }, + ), +) +``` + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="MGSM multilingual math evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/mgsm-cot", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + + + + +No judge required with NeMo template. + +```python +job = client.evaluation.benchmark_jobs.create( + description="AIME 2025 competition math", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/aime-2025-nemo", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + ), +) +``` + + + + +Requires a judge model. + +```python +job = client.evaluation.benchmark_jobs.create( + description="AIME 2025 competition math with judge", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/aime-2025", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={ + "judge": { + "model": { + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + } + } + }, + ), +) +``` + + + + +No judge required with NeMo template. + +```python +job = client.evaluation.benchmark_jobs.create( + description="MATH test set evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/math-test-500-nemo", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + ), +) +``` + + + + +### Results + +Math benchmarks produce accuracy scores: + +- **GSM8K/MGSM**: Exact match accuracy on final numerical answers +- **AIME/MATH**: Correctness as judged by the judge model (or exact match for NeMo variants) + +```python +aggregate = client.evaluation.benchmark_jobs.results.aggregate_scores.download( + job=job.name +) +for score in aggregate.scores: + print(f"{score.name}: {score.mean:.1%}") +``` + +For detailed results analysis, refer to [eval-benchmark-results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results). + + +## Content Safety + +Evaluate model safety risks including vulnerability to generate harmful, biased, or misleading content. + +* Label: `content_safety` + +### Available Benchmarks + +| Benchmark | Description | Required Params | +|-----------|-------------|-----------------| +| `system/aegis-v2` | [AEGIS 2.0](https://aclanthology.org/2025.naacl-long.306/)—12 hazard categories using Nemotron Safety Guard | `hf_token`, `judge` | +| `system/wildguard` | [WildGuard](https://arxiv.org/abs/2406.18495)—privacy, misinformation, harmful language | `hf_token`, `judge` | + + + +Safety benchmarks require specific judge models deployed with a `/v1/completions` endpoint (not chat/completions): + +| Benchmark | Required Judge Model | Model ID | +|-----------|---------------------|----------| +| `system/aegis-v2` | [Llama Nemotron Safety Guard V2](https://huggingface.co/nvidia/llama-3.1-nemoguard-8b-content-safety) | `nvidia/llama-3.1-nemoguard-8b-content-safety` | +| `system/wildguard` | [WildGuard](https://huggingface.co/allenai/wildguard) | `allenai/wildguard` | + + +The benchmarks can take up to 1-3 hours. + +### Examples + +```python +import os +from nemo_platform import NeMoPlatform +from nemo_platform.types.evaluation import ( + SystemBenchmarkOnlineJobParam, + RunConfig, +) + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="my-workspace", # Replace with your workspace or use "default" +) +``` + + + + + +Requires Llama Nemotron Safety Guard V2 judge. + +```python +job = client.evaluation.benchmark_jobs.create( + description="AEGIS-v2 content safety evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/aegis-v2", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={ + "hf_token": "hf_token", + "judge": { + "model": { + "url": "/v1/completions", + "name": "nvidia/llama-3.1-nemoguard-8b-content-safety", + } + }, + }, + ), +) +``` + + + + +Requires WildGuard judge. + +```python +job = client.evaluation.benchmark_jobs.create( + description="WildGuard content safety evaluation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/wildguard", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={ + "hf_token": "hf_token", + "judge": { + "model": { + "url": "/v1/completions", + "name": "allenai/wildguard", + } + }, + }, + ), +) +``` + + + + +### Results + +Safety benchmarks produce category-level safety rates: + +- **AEGIS-v2**: Safe/unsafe classification across 12 hazard categories (violence, hate, sexual content, and so on) +- **WildGuard**: Safe/unsafe classification for privacy, misinformation, harmful language, malicious use + +```python +aggregate = client.evaluation.benchmark_jobs.results.aggregate_scores.download( + job=job.name +) +for score in aggregate.scores: + print(f"{score.name}: {score.mean:.1%}") +``` + +For detailed results analysis, refer to [eval-benchmark-results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results). + +### Troubleshooting Content Safety Benchmarks + +Refer to [Evaluator Troubleshooting](/documentation/reference/troubleshooting/evaluator) for general troubleshooting steps for failed evaluation jobs. + +
+This section covers common issues for the safety harness. + +#### Hugging Face Error + +Evaluations with safety harness requires Hugging Face access to the respective dataset and model tokenizer. If your job fails with the following errors, visit [https://huggingface.co/](https://huggingface.co/) and log in to request access to the dataset or model. + +```text +datasets.exceptions.DatasetNotFoundError: Dataset 'allenai/wildguardmix' is a gated dataset on the Hub. Visit the dataset page at https://huggingface.co/datasets/allenai/wildguardmix to ask for access. +``` + +```text +GatedRepoError: 403 Client Error. + +Cannot access gated repo for url https://huggingface.co//resolve/main/tokenizer_config.json. +Your request to access model is awaiting a review from the repo authors. +``` + +#### Incompatible Judge Model + +Using an unsupported judge model results in a job error. The `aegis-v2` evaluation requires Llama Nemotron Safety Guard V2 judge and `wildguard` evaluation requires `allenai/wildguard` judge. `KeyError` is an example error for the wrong judge model like the following error. + +```text +Metrics calculated + + + Evaluation Metrics +┏━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓ +┃ Safety Category ┃ Average Count ┃ +┡━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩ +│ ERROR │ 5.0 │ +└─────────────────┴───────────────┘ + +... + +Subprocess finished with return code: 0 +{'ERROR': 5.0} +Traceback (most recent call last): +... +"/usr/local/lib/python3.10/site-packages/core_evals/safety_eval/__init__.py", line 14, in parse_output + return parse_output(output_dir) + File "/usr/local/lib/python3.10/site-packages/core_evals/safety_eval/output.py", line 16, in parse_output + safety_rate = data['safe'] / sum(data.values()) +KeyError: 'safe' +``` + +#### Unexpected Reasoning Traces + +Safety evaluations do not support reasoning traces and may result in the job error below. + +```text +ERROR There are at least 2 MUT (model under test) responses that start with . Reasoning traces should not be evaluated. Exiting. +``` + +If the model or judge outputs reasoning traces like `reasoning contextanswer`, configure the job to only include the the final answer after the reasoning token (e.g. ``) with `reasoning.end_token`. Consider specifying `inference.max_tokens` to a reasonable limit for the model's chain of thought to conclude with the expected reasoning end token in order for the reasoning context to be properly omitted for evaluation. + +Additionally, if you are encountering this error, it could be caused by the model exceeding its token limit resulting in the full response being consumed by the model thinking. These results can be dropped by setting the `reasoning.include_if_not_finished` parameter. + +```python +from nemo_platform.types.evaluation import ( + RunConfig, + InferenceParams, + ReasoningParams, +) + +params = RunConfig( + inference=InferenceParams(max_tokens=1024), + reasoning=ReasoningParams( + end_token="", + include_if_not_finished=False, + ), +) +``` + +
+ +## Code + +Evaluate code generation capabilities using functional correctness benchmarks that test synthesis of working programs. + +* Label: `code` + +### Available Benchmarks + +| Benchmark | Description | Required Params | +|-----------|-------------|-----------------| +| `system/humaneval` | [HumanEval](https://arxiv.org/abs/2107.03374)—164 Python problems ¹ | — | +| `system/humaneval-instruct` | HumanEval for instruction-tuned models | — | +| `system/humanevalplus` | [HumanEval+](https://arxiv.org/abs/2305.01210)—80x more test cases ¹ | — | +| `system/mbpp` | [MBPP](https://arxiv.org/abs/2108.07732)—Python programming problems | — | +| `system/mbppplus` | [MBPP+](https://arxiv.org/abs/2305.01210)—35x more test cases | — | +| `system/mbppplus-nemo` | MBPP+ with NeMo template | — | +| `system/multiple-{lang}` | [MultiPL-E](https://arxiv.org/abs/2208.08227)—HumanEval in 20+ languages ¹ | — | + +**¹** Completions-only: Requires `/v1/completions` endpoint. + +**MultiPL-E languages**: `clj` (Clojure), `cpp` (C++), `cs` (C#), `d`, `elixir`, `go`, `hs` (Haskell), `java`, `jl` (Julia), `js` (JavaScript), `lua`, `ml` (OCaml), `php`, `pl` (Perl), `r`, `rb` (Ruby), `rkt` (Racket), `rs` (Rust), `scala`, `sh` (Bash), `swift` + +The benchmarks can take up to 1–5 hours. + +### Examples + +```python +import os +from nemo_platform import NeMoPlatform +from nemo_platform.types.evaluation import ( + SystemBenchmarkOnlineJobParam, + RunConfig, +) + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="my-workspace", # Replace with your workspace or use "default" +) +``` + + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="HumanEval code generation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/humaneval-instruct", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + ), +) +``` + + + + +Requires a `/v1/completions` endpoint. + +```python +job = client.evaluation.benchmark_jobs.create( + description="HumanEval code generation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/humaneval", + model={ + "url": "/v1/completions", + "name": "", + }, + params=RunConfig(parallelism=16), + ), +) +``` + + + + +Extended test suite with 80x more test cases. + +```python +job = client.evaluation.benchmark_jobs.create( + description="HumanEval+ code generation", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/humanevalplus", + model={ + "url": "/v1/completions", + "name": "", + }, + params=RunConfig(parallelism=16), + ), +) +``` + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="MBPP+ Python programming", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/mbppplus-nemo", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + ), +) +``` + + + + +```python +job = client.evaluation.benchmark_jobs.create( + description="MBPP+ Python programming", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/mbppplus", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + ), +) +``` + + + + +Requires a `/v1/completions` endpoint. + +```python +job = client.evaluation.benchmark_jobs.create( + description="MultiPL-E JavaScript", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/multiple-js", + model={ + "url": "/v1/completions", + "name": "", + }, + params=RunConfig(parallelism=16), + ), +) +``` + + + + +Requires a `/v1/completions` endpoint. + +```python +job = client.evaluation.benchmark_jobs.create( + description="MultiPL-E Rust", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/multiple-rs", + model={ + "url": "/v1/completions", + "name": "", + }, + params=RunConfig(parallelism=16), + ), +) +``` + + + + +### Results + +Code benchmarks produce pass@k metrics measuring functional correctness: + +- **pass@1**: Percentage of problems solved with one attempt +- **pass@10**: Percentage of problems solved within 10 attempts (when `n_samples` > 1) + +```python +aggregate = client.evaluation.benchmark_jobs.results.aggregate_scores.download( + job=job.name +) +for score in aggregate.scores: + print(f"{score.name}: {score.mean:.1%}") +``` + +For detailed results analysis, refer to [eval-benchmark-results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results). + + + +**Want to experiment first?** You can try these benchmarks using the [open-source NeMo Evaluator SDK](https://github.com/NVIDIA-NeMo/evaluator) before deploying the platform. The SDK provides a lightweight way to test evaluation workflows locally. + + +## Run Benchmark Job + +Create a workspace if you have not already. + +```python +workspace = "my-workspace" + +client.workspaces.create(name=workspace) +``` + +Create an evaluation job with a benchmark that satisfy the required and optional parameters. + + + +For benchmarks that require a Hugging Face token or other API keys for external services, create the secret to be referenced by the job. + + + + +```python +from nemo_platform.types.evaluation import ( + SystemBenchmarkOnlineJobParam, + RunConfig, +) + +job = client.evaluation.benchmark_jobs.create( + description="Example running system benchmark to evaluate my model's advanced reasoning capabilities.", + spec=SystemBenchmarkOnlineJobParam( + benchmark="system/gpqa-diamond-cot", + model={ + "url": "/v1", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + }, + params=RunConfig(parallelism=16), + benchmark_params={"hf_token": "hf_token"}, + ), +) +``` + +## Job Management + +After creating a job, navigate to [Benchmark Job Management](/documentation/evaluator-benchmark-reference/benchmarks/job-management) to oversee its execution and monitor progress. diff --git a/docs/evaluator/benchmarks/job-management.md b/docs/evaluator/benchmarks/job-management.mdx similarity index 82% rename from docs/evaluator/benchmarks/job-management.md rename to docs/evaluator/benchmarks/job-management.mdx index 3c8a18905a..aa540dcd80 100644 --- a/docs/evaluator/benchmarks/job-management.md +++ b/docs/evaluator/benchmarks/job-management.mdx @@ -1,11 +1,16 @@ +--- +title: "Benchmark Job Management" +description: "" +--- -# Benchmark Job Management Manage your evaluation job. -!!! note - **Performance Tuning**: You can improve evaluation performance by setting `job.params.parallelism` to control the number of concurrent requests. A typical default value is 16, but you might need to adjust based on your model capacity and rate limits. + +**Performance Tuning**: You can improve evaluation performance by setting `job.params.parallelism` to control the number of concurrent requests. A typical default value is 16, but you might need to adjust based on your model capacity and rate limits. + + ## Monitor Job Monitor the status and progress of a job in real-time. @@ -19,7 +24,7 @@ while job_status.status in ("active", "pending", "created"): print(job_status) ``` -Refer to [Evaluator Troubleshooting](../../troubleshooting/evaluator.md) for help troubleshooting job failures. +Refer to [Evaluator Troubleshooting](/documentation/reference/troubleshooting/evaluator) for help troubleshooting job failures. ## Fetch Job Logs @@ -42,7 +47,7 @@ while logs_response.next_page: ## View Evaluation Results -Evaluation results are available after the evaluation job completes. Refer to [Benchmark Results](results.md) for details on fetching evaluation results to analyze the job output. +Evaluation results are available after the evaluation job completes. Refer to [Benchmark Results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results) for details on fetching evaluation results to analyze the job output. ### Aggregate Scores diff --git a/docs/evaluator/benchmarks/manage-benchmarks.md b/docs/evaluator/benchmarks/manage-benchmarks.mdx similarity index 85% rename from docs/evaluator/benchmarks/manage-benchmarks.md rename to docs/evaluator/benchmarks/manage-benchmarks.mdx index fb1afc9fab..cf2749cff5 100644 --- a/docs/evaluator/benchmarks/manage-benchmarks.md +++ b/docs/evaluator/benchmarks/manage-benchmarks.mdx @@ -1,7 +1,10 @@ +--- +title: "Manage Benchmarks" +description: "" +--- -# Manage Benchmarks -List, retrieve, and delete evaluation benchmarks using the {{platform_name}} Python SDK. +List, retrieve, and delete evaluation benchmarks using the NeMo Platform Python SDK. ```python import os @@ -130,9 +133,11 @@ benchmarks = client.evaluation.benchmarks.list( Delete a custom evaluation benchmark. Industry benchmarks in the `system` workspace cannot be deleted. -!!! warning - Deleting a benchmark is permanent and cannot be undone. Ensure the benchmark is not being used by any active evaluations before deletion. + +Deleting a benchmark is permanent and cannot be undone. Ensure the benchmark is not being used by any active evaluations before deletion. + + ```python client.evaluation.benchmarks.delete(name="my-custom-benchmark") ``` @@ -140,6 +145,6 @@ client.evaluation.benchmarks.delete(name="my-custom-benchmark") ## Related Topics -- [Custom Benchmarks](custom.md) - Create custom benchmarks -- [Benchmark Job Management](job-management.md) - Manage benchmark evaluation jobs -- [Benchmark Results](results.md) - View and download benchmark evaluation results +- [Custom Benchmarks](/documentation/evaluate-models/benchmarks/custom-benchmarks) - Create custom benchmarks +- [Benchmark Job Management](/documentation/evaluator-benchmark-reference/benchmarks/job-management) - Manage benchmark evaluation jobs +- [Benchmark Results](/documentation/evaluator-benchmark-reference/benchmarks/benchmark-results) - View and download benchmark evaluation results diff --git a/docs/evaluator/benchmarks/results.md b/docs/evaluator/benchmarks/results.mdx similarity index 99% rename from docs/evaluator/benchmarks/results.md rename to docs/evaluator/benchmarks/results.mdx index 2344afdca6..dec4bd1208 100644 --- a/docs/evaluator/benchmarks/results.md +++ b/docs/evaluator/benchmarks/results.mdx @@ -1,5 +1,8 @@ +--- +title: "Benchmark Results" +description: "" +--- -# Benchmark Results When a benchmark job completes, the platform automatically creates a **benchmark job result** — a persistent, queryable entity that captures the outcome of that run. Each result is created exactly once per completed job and shares the job's name. diff --git a/docs/evaluator/index.md b/docs/evaluator/index.mdx similarity index 55% rename from docs/evaluator/index.md rename to docs/evaluator/index.mdx index ccd6fb3955..cf4b63c444 100644 --- a/docs/evaluator/index.md +++ b/docs/evaluator/index.mdx @@ -1,12 +1,16 @@ +--- +title: "About Evaluating" +description: "" +--- + -# About Evaluating -Evaluation is powered by {{platform_name}}, a cloud-native platform for evaluating large language models (LLMs), RAG pipelines, and AI agents at enterprise scale. The evaluation API provides automated workflows for over 100 industry benchmarks, LLM-as-a-judge scoring, and specialized metrics for RAG and agent systems. +Evaluation is powered by NeMo Platform, a cloud-native platform for evaluating large language models (LLMs), RAG pipelines, and AI agents at enterprise scale. The evaluation API provides automated workflows for over 100 industry benchmarks, LLM-as-a-judge scoring, and specialized metrics for RAG and agent systems. -{{platform_name}} enables real-time evaluations of your LLM application through APIs, guiding you in refining and optimizing LLMs for enhanced performance and real-world applicability. The {{nem_short_name}} APIs can be seamlessly automated within development pipelines, enabling faster iterations without the need for live data. It is cost-effective and suitable for pre-deployment checks and regression testing. +NeMo Platform enables real-time evaluations of your LLM application through APIs, guiding you in refining and optimizing LLMs for enhanced performance and real-world applicability. The NeMo Evaluator APIs can be seamlessly automated within development pipelines, enabling faster iterations without the need for live data. It is cost-effective and suitable for pre-deployment checks and regression testing. -[**Tutorials**](tutorials/index.md){ .md-button } -[**Open Source SDK**](https://github.com/NVIDIA-NeMo/evaluator){ .md-button } + + --- @@ -14,15 +18,17 @@ Evaluation is powered by {{platform_name}}, a cloud-native platform for evaluati Evaluator separates **evaluation definition** from **execution**. -!!! note - The code snippets below are for conceptual demonstration purposes only. - For runnable examples see the [tutorials](tutorials/index.md) and [SDK resources](sdk-resources.md). + +The code snippets below are for conceptual demonstration purposes only. +For runnable examples see the [tutorials](/documentation/evaluate-models/tutorials) and [SDK resources](/documentation/evaluate-models/sdk-resources). + + ### 1. Build RunConfig with the Library Use the `nemo_evaluator_sdk` package to define your metric, dataset rows, runtime configuration, and optional model or agent target: -{% raw %} + ```python from nemo_evaluator_sdk import RunConfig, ExactMatchMetric @@ -42,14 +48,14 @@ dataset = [ config = RunConfig(limit_samples=100, parallelism=8) ``` -{% endraw %} + **The library handles:** Metric definitions, dataset row schemas, prompt templates, model and agent targets, runtime parameters, retries, aggregation, and typed result objects. ### 2. Execute on the Platform -Submit your evaluation to the Evaluator service using the {{platform_name}} SDK: +Submit your evaluation to the Evaluator service using the NeMo Platform SDK: ```python from nemo_evaluator.sdk import Evaluator @@ -68,25 +74,25 @@ job.wait_until_done() result = job.get_result() ``` -**The platform handles:** Job orchestration, inference routing through {{platform_name}}'s Inference Gateway, Fileset-based datasets, distributed execution, artifact storage, status monitoring, and result download. +**The platform handles:** Job orchestration, inference routing through NeMo Platform's Inference Gateway, Fileset-based datasets, distributed execution, artifact storage, status monitoring, and result download. ## Key Differences from Standalone Library -When using Evaluator as a {{platform_name}} plugin : +When using Evaluator as a NeMo Platform plugin : -| Feature | Standalone Library | {{platform_name}} Plugin | +| Feature | Standalone Library | NeMo Platform Plugin | |---------|-------------------|-------------| | **Execution** | Local Python process | Local plugin runs for local experimentation and durable platform jobs for production | -| **Inference** | Direct model or agent endpoint calls | The same as standalone and can also route through {{platform_name}} Inference Gateway and platform-managed endpoints | -| **Datasets** | Inline rows and local files | Inline rows, local paths resolved at submission time, and {{platform_name}} [Filesets](../get-started/concepts/manage-files.md) | -| **Results Artifacts** | Results stored in memory | {{platform_name}} artifact storage with typed result download | -| **Authentication** | Local environment variables | Local environment variables for local runs and {{platform_name}} Secrets service for remote jobs | +| **Inference** | Direct model or agent endpoint calls | The same as standalone and can also route through NeMo Platform Inference Gateway and platform-managed endpoints | +| **Datasets** | Inline rows and local files | Inline rows, local paths resolved at submission time, and NeMo Platform [Filesets](/documentation/get-started/core-concepts/manage-files) | +| **Results Artifacts** | Results stored in memory | NeMo Platform artifact storage with typed result download | +| **Authentication** | Local environment variables | Local environment variables for local runs and NeMo Platform Secrets service for remote jobs | --- ## Evaluation Concepts -{{platform_name}} supports two core evaluation primitives: +NeMo Platform supports two core evaluation primitives: - **Metrics**: Scoring logic that evaluates model outputs. Use metrics when you need flexible, reusable scoring for your own datasets and task-specific criteria. @@ -98,33 +104,33 @@ There are two execution modes and two evaluation patterns: - **Offline evaluation**: Score existing dataset rows (for example, model outputs already generated). - **Online evaluation**: Generate outputs from a model as part of evaluation, then score them. -For deeper details, see [Evaluation Metrics](metrics/index.md). +For deeper details, see [Evaluation Metrics](/documentation/evaluate-models/metrics). --- ## Tutorials -After [setting up a local instance of the platform](../get-started/setup.md), use the following tutorials to learn how to accomplish common evaluation tasks. These step-by-step guides help you evaluate models using different benchmarks and metrics. +After [setting up a local instance of the platform](/documentation/get-started), use the following tutorials to learn how to accomplish common evaluation tasks. These step-by-step guides help you evaluate models using different benchmarks and metrics. -
+ -- **[Run an LLM Judge Eval](tutorials/run-llm-judge-evaluation.md)** + - --- +Learn how to evaluate a fine-tuned model using the LLM Judge metric with a custom dataset. - Learn how to evaluate a fine-tuned model using the LLM Judge metric with a custom dataset. +custom-dataset - custom-dataset + -- **[Define and Run Custom Python Metrics](tutorials/define-run-custom-python-metrics.md)** + - --- +Learn how to write a domain-specific Python metric, test it locally, and run it through the Evaluator service. - Learn how to write a domain-specific Python metric, test it locally, and run it through the Evaluator service. +custom-metric - custom-metric + -
+ --- @@ -133,12 +139,12 @@ After [setting up a local instance of the platform](../get-started/setup.md), us Most teams get the best results by starting metric-first, then moving to benchmarks: 1. **Develop and validate your metrics first** - - Start with [Metrics](metrics/index.md) to define how quality should be scored for your use case. + - Start with [Metrics](/documentation/evaluate-models/metrics) to define how quality should be scored for your use case. - Use live evaluation (`POST /v2/workspaces/{workspace}/evaluation/metric-evaluate`) with small `DatasetRows` payloads to iterate quickly. 1. **Scale metric evaluation to jobs** - When metrics are validated, run async metric jobs (`/evaluation/metric-jobs`) on larger datasets. - - Use filesets for production-scale inputs. See [Manage Files](../get-started/concepts/manage-files.md). + - Use filesets for production-scale inputs. See [Manage Files](/documentation/get-started/core-concepts/manage-files). 1. **Monitor and analyze results** - Track job status and progress with job management APIs. @@ -148,8 +154,8 @@ Most teams get the best results by starting metric-first, then moving to benchma ## Where to Go Next -- For metric workflows, see [Metric Jobs](metrics/index.md) and [Metric Results](metrics/results.md). -- For full endpoint details, see the [Evaluator API Reference](../api/index.md#tag-evaluator). +- For metric workflows, see [Metric Jobs](/documentation/evaluate-models/metrics) and Metric Results. +- For full endpoint details, see the [Evaluator API Reference](/documentation/reference/api-reference). --- diff --git a/docs/evaluator/metrics/agent-configuration.md b/docs/evaluator/metrics/agent-configuration.mdx similarity index 85% rename from docs/evaluator/metrics/agent-configuration.md rename to docs/evaluator/metrics/agent-configuration.mdx index 0c00f565e4..012bc52d59 100644 --- a/docs/evaluator/metrics/agent-configuration.md +++ b/docs/evaluator/metrics/agent-configuration.mdx @@ -1,5 +1,8 @@ +--- +title: "Agent Configuration" +description: "" +--- -# Agent Configuration Online evaluations can target an **agent** instead of a model. An agent is an HTTP endpoint that accepts a request and returns a response, optionally with a trajectory of intermediate steps. Use agents when you want to evaluate an agentic system end to end rather than a standalone LLM endpoint. @@ -34,7 +37,7 @@ evaluator: Evaluator = client.evaluator # this object is an Evaluator resource If your agent endpoint requires authentication, configure `api_key_secret` on the `Agent`. -For local `evaluator.run(...)` calls, `api_key_secret` must name an environment variable available to the local Python process. For remote `evaluator.submit(...)` jobs, it must name a NeMo platform secret in the target workspace. See [Model API Authentication](model-configuration.md#model-api-authentication) for the local-versus-remote behavior. +For local `evaluator.run(...)` calls, `api_key_secret` must name an environment variable available to the local Python process. For remote `evaluator.submit(...)` jobs, it must name a NeMo platform secret in the target workspace. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication) for the local-versus-remote behavior. For remote `evaluator.submit(...)` jobs, create the secret in the platform workspace before submitting the job: @@ -63,14 +66,14 @@ You control the request shape with `body` and extract values from the response w | `url` | Yes | string | Base URL of the agent endpoint. | | `name` | Yes | string | Agent name or identifier. | | `format` | No | string | `generic` (default) or `nemo_agent_toolkit`. | -| `api_key_secret` | No | string | API key reference. See [Model API Authentication](model-configuration.md#model-api-authentication). | -| `body` | Yes | dict | Jinja template for the request payload. Use `{%raw%}{{ prompt }}{%endraw%}`, `{%raw%}{{ messages }}{%endraw%}`, or fields from the rendered prompt context. | +| `api_key_secret` | No | string | API key reference. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). | +| `body` | Yes | dict | Jinja template for the request payload. Use `{{ prompt }}`, `{{ messages }}`, or fields from the rendered prompt context. | | `response_path` | Yes | string | [JSONPath](https://datatracker.ietf.org/doc/html/rfc9535) expression to extract the response text. | | `trajectory_path` | No | string | JSONPath expression to extract the trajectory. | ### Run a Generic Agent Evaluation -{% raw %} + ```python from nemo_evaluator_sdk import Agent, RunConfigOnline @@ -99,7 +102,7 @@ for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") ``` -{% endraw %} + Use `evaluator.submit(...)` with the same argument shape when you want a durable remote job, but set `api_key_secret` to a platform secret name for the target workspace. @@ -150,11 +153,11 @@ Use the `nemo_agent_toolkit` format when evaluating agents built with the [NeMo | `url` | Yes | string | Base URL of the agent endpoint. | | `name` | Yes | string | Agent name or identifier. | | `format` | Yes | string | Set to `nemo_agent_toolkit`. | -| `api_key_secret` | No | string | API key reference. See [Model API Authentication](model-configuration.md#model-api-authentication). | +| `api_key_secret` | No | string | API key reference. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). | ### Run a NAT Agent Evaluation -{% raw %} + ```python from nemo_evaluator_sdk import Agent, RunConfigOnline @@ -186,7 +189,7 @@ job.wait_until_done() result = job.get_result() ``` -{% endraw %} + ## Model vs Agent: When to Use Which @@ -198,12 +201,14 @@ result = job.get_result() | Evaluate a custom HTTP endpoint with non-standard response format | | x | | Use a standard chat completions API | x | | -!!! info - Online evaluations accept **either** a model **or** an agent as the request target, never both. + + +Online evaluations accept **either** a model **or** an agent as the request target, never both. + ## Related -- [Model Configuration](model-configuration.md) - Inline model targets for LLM endpoints. -- [Agentic Evaluation Metrics](agentic.md) - Metrics for evaluating agent tool calling, goal accuracy, and trajectory. -- [LLM-as-a-Judge](llm-as-a-judge.md) - Custom judge-based evaluation with flexible scoring criteria. -- [Bring Your Own Metric](remote.md) - Integrate custom evaluation endpoints. +- [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) - Inline model targets for LLM endpoints. +- [Agentic Evaluation Metrics](/documentation/evaluate-models/metrics/agentic-metrics) - Metrics for evaluating agent tool calling, goal accuracy, and trajectory. +- [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge) - Custom judge-based evaluation with flexible scoring criteria. +- [Bring Your Own Metric](/documentation/evaluate-models/metrics/bring-your-own-metric) - Integrate custom evaluation endpoints. diff --git a/docs/evaluator/metrics/agentic.md b/docs/evaluator/metrics/agentic.md deleted file mode 100644 index 0f6a433303..0000000000 --- a/docs/evaluator/metrics/agentic.md +++ /dev/null @@ -1,1235 +0,0 @@ - -# Agentic Evaluation Metrics - -Evaluate agent-based and multi-step reasoning models using metrics powered by [RAGAS](https://github.com/explodinggradients/ragas). These metrics assess tool calling accuracy, goal completion, topic adherence, and answer correctness in agentic workflows. - -## Agent Workflow Evaluation Stages - -Key stages of agent workflow evaluation: - -![Agent Evaluation Framework](../images/agent_eval_framework.png) - -**1. Intermediate Steps Evaluation** -Assesses the correctness of intermediate steps during agent execution: - -- **Tool Use**: Validates that the agent invoked the right tools with correct arguments at each step. Refer to [Tool Call Accuracy](#tool-call-accuracy) for implementation details. - -**2. Final-Step Evaluation** -Evaluates the quality of the agent's final output using: - -- **Agent Goal Accuracy**: Measures whether the agent successfully completed the requested task. Refer to [Agent Goal Accuracy](#agent-goal-accuracy). -- **Topic Adherence**: Assesses how well the agent maintained focus on the assigned topic throughout the conversation. Refer to [Topic Adherence](#topic-adherence). -- **Answer Accuracy**: Evaluates the factual correctness of agent answers. Refer to [Answer Accuracy](#answer-accuracy). -- **Custom Metrics**: For domain-specific or custom evaluation criteria, use [LLM-as-a-Judge](llm-as-a-judge.md) with the `data` task type. - -**3. Trajectory Evaluation** -Evaluates the agent's decision-making process by analyzing the entire sequence of actions taken to accomplish a goal. This includes assessing whether the agent chose appropriate tools in the correct order. Refer to [Trajectory Evaluation](#trajectory-evaluation) for the expected data format and current plugin SDK support. - ---- - -## Online vs Offline Evaluation - -Agentic metrics support two execution patterns through the Evaluator plugin SDK: - -### Offline Evaluation - -Offline evaluation scores pre-generated responses or tool calls already present in the dataset: - -- Dataset rows are passed inline with `dataset=[...]`, as a file Path, or as a FilesetRef. -- No model or agent generation is performed. -- Use this mode to evaluate existing agent outputs or compare different response strategies. - -### Online Target Generation - -Online target generation first calls a model or agent target, then evaluates the generated output: - -- Configure the target with `Model` or `Agent` from `nemo_evaluator_sdk`. -- Pass the target through `target=...`. -- Use `RunConfigOnlineModel` for model targets and `RunConfigOnline` for agent targets. -- Include a `prompt_template` when the dataset row must be transformed into a model or agent request. - -!!! note - **Response Usage**: In online target generation, the generated response is used as the metric response. A dataset `response` column is optional and is superseded by the generated response for that run. - ---- - -## Overview - -Agentic metrics evaluate different aspects of agent behavior: - -| Metric | Use Case | Requires Judge | Plugin SDK Execution | -|--------|----------|----------------|----------------------| -| [**Tool Call Accuracy**](#tool-call-accuracy) | Evaluates tool/function call correctness | No | `run` + `submit` | -| [**Tool Calling** (template)](#tool-calling-template) | Evaluates tool/function call correctness using Jinja templates | No | `run` + `submit` | -| [**Topic Adherence**](#topic-adherence) | Measures topic focus in multi-turn conversations | Yes | `run` + `submit` | -| [**Agent Goal Accuracy**](#agent-goal-accuracy) | Assesses goal completion, with or without reference. | Yes | `run` + `submit` | -| [**Answer Accuracy**](#answer-accuracy) | Checks factual correctness | Yes | `run` + `submit` | -| [**Trajectory Evaluation**](#trajectory-evaluation) | Evaluates decision-making across action sequence | Yes | Not exposed as a typed plugin SDK metric | - -!!! note - Use `evaluator.run(...)` for local in-process evaluation and `evaluator.submit(...)` for durable remote platform jobs. The examples below use inline dataset rows through `dataset=[...]`, but you can use a file Path or a FilesetRef instead. - -## Prerequisites - -Before running agentic evaluations: - -1. **Workspace**: Have a workspace created. All remote resources, including secrets and jobs, are scoped to a workspace. -2. **Judge LLM endpoint** *(for most metrics)*: Have access to an LLM that will serve as your judge. -3. **API key secret** *(if judge requires auth)*: If your judge endpoint requires authentication, [create a secret](../../get-started/concepts/manage-secrets.md) to store the API key. For local `run` versus remote `submit` behavior, see [Model API Authentication](model-configuration.md#model-api-authentication). -4. **Initialize the SDK**: - -```python -import os -from nemo_evaluator.sdk import Evaluator -from nemo_platform import NeMoPlatform - - -client = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="default", -) -evaluator: Evaluator = client.evaluator # this object is an Evaluator resource -``` - -### Creating a Secret for API Keys - -If using external endpoints, such as [Build NVIDIA](https://build.nvidia.com/models) API endpoints (https://integrate.api.nvidia.com/v1), create a secret first: - -```python -client.secrets.create( - name="nvidia-api-key", - value="nvapi-YOUR_API_KEY_HERE", - description="NVIDIA Build API key for RAGAS metrics", -) -``` - -### SDK Types Reference - -The plugin SDK examples use context-agnostic metric classes and runtime value classes: - -```python -from nemo_evaluator_sdk import ( - ToolCallingMetric, -) -from nemo_evaluator_sdk.metrics.ragas import ( - AgentGoalAccuracyMetric, - AnswerAccuracyMetric, - ToolCallAccuracyMetric, - TopicAdherenceMetric, -) -from nemo_evaluator_sdk import ( - Agent, - RunConfigOnlineModel, - RunConfigOnline, - RunConfig, - InferenceParams, - Model, - ReasoningParams, -) -``` - -Use `dataset=[...]` for inline rows. For offline scoring options, use `config=RunConfig(parallelism=...)`. Whenever outputs must be generated before scoring, pass `target=Model(...)` or `target=Agent(...)` plus the corresponding online parameters. Use the same `dataset`, `config`, and `target` arguments for both `evaluator.run(...)` and `evaluator.submit(...)`; durable jobs follow the identical pattern as local runs and only differ in waiting for and fetching results. - ---- - -## Agentic Metrics - -### Tool Call Accuracy - -Evaluates whether the agent invoked the correct tools with the correct arguments. This metric **does not require a judge LLM**. - -!!! note - **Online/offline support**: Tool Call Accuracy supports scoring existing tool calls directly. It can also score tool calls produced during online target generation when the target response includes the required tool-call fields. - -#### Data Format - -```json -{ - "user_input": [ - { - "content": "What's the weather like in New York?", - "type": "human" - }, - { - "content": "Let me check that for you.", - "type": "ai", - "tool_calls": [ - { - "name": "weather_check", - "args": { - "location": "New York" - } - } - ] - }, - { - "content": "It's 75°F and partly cloudy.", - "type": "tool" - }, - { - "content": "The weather in New York is 75°F and partly cloudy.", - "type": "ai" - } - ], - "reference_tool_calls": [ - { - "name": "weather_check", - "args": { - "location": "New York" - } - } - ] -} -``` - -=== "Run Locally" - - ```python - from nemo_evaluator_sdk.metrics.ragas import ToolCallAccuracyMetric - metric = ToolCallAccuracyMetric() - - result = evaluator.run( - metric=metric, - dataset=[ - { - "user_input": [ - {"content": "What's the weather in Paris?", "type": "human"}, - { - "content": "Let me check.", - "type": "ai", - "tool_calls": [{"name": "weather_api", "args": {"city": "Paris"}}], - }, - {"content": "Sunny, 22°C", "type": "tool"}, - {"content": "It's sunny and 22°C in Paris.", "type": "ai"}, - ], - "reference_tool_calls": [ - {"name": "weather_api", "args": {"city": "Paris"}} - ], - } - ], - ) - print(result.aggregate_scores) - ``` - -=== "Submit Job" - - ```python - from nemo_evaluator_sdk import RunConfig - from nemo_evaluator_sdk.metrics.ragas import ToolCallAccuracyMetric - metric = ToolCallAccuracyMetric() - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "user_input": [ - {"content": "What's the weather in Paris?", "type": "human"}, - { - "content": "Let me check.", - "type": "ai", - "tool_calls": [{"name": "weather_api", "args": {"city": "Paris"}}], - }, - {"content": "Sunny, 22°C", "type": "tool"}, - {"content": "It's sunny and 22°C in Paris.", "type": "ai"}, - ], - "reference_tool_calls": [ - {"name": "weather_api", "args": {"city": "Paris"}} - ], - } - ], - config=RunConfig(parallelism=4), - ) - job.wait_until_done() - result = job.get_result() - print(result.aggregate_scores) - ``` - -=== "Result" - - ```json - { - "aggregate_scores": [ - { - "name": "tool_call_accuracy", - "count": 1, - "mean": 1.0, - "min": 1.0, - "max": 1.0 - } - ], - "row_scores": [ - { - "index": 0, - "scores": { - "tool_call_accuracy": 1.0 - } - } - ] - } - ``` - ---- - -### Tool Calling (Template) - -A template-based metric for evaluating tool/function call accuracy. Unlike the RAGAS Tool Call Accuracy metric, this metric uses configurable templates and produces multiple scores. - -!!! note - **Online/offline support**: Tool Calling supports scoring existing tool-call outputs directly. Use online target generation when you want the model or agent target to produce the response first. - -#### Scores Produced - -- `function_name_accuracy` - Accuracy of function names only -- `function_name_and_args_accuracy` - Accuracy of both function names and arguments - -#### Data Format - -Data must use OpenAI-compliant tool calling format: - -```json -{ - "messages": [ - { - "role": "user", - "content": "Book a table for 2 at 7pm." - }, - { - "role": "assistant", - "content": "Booking a table...", - "tool_calls": [ - { - "function": { - "name": "book_table", - "arguments": { - "people": 2, - "time": "7pm" - } - } - } - ] - } - ], - "tool_calls": [ - { - "function": { - "name": "book_table", - "arguments": { - "people": 2, - "time": "7pm" - } - } - } - ], - "response": { - "choices": [ - { - "message": { - "tool_calls": [ - { - "function": { - "name": "book_table", - "arguments": "{\"people\": 2, \"time\": \"7pm\"}" - } - } - ] - } - } - ] - } -} -``` - -!!! note - - Function names with dots (`.`) must be replaced with underscores (`_`). - - Comparison is case-sensitive. - - Order of tool calls is ignored, which supports parallel tool calling. - -=== "Run Locally" - - {% raw %} - ```python - - - from nemo_evaluator_sdk import ToolCallingMetric - metric = ToolCallingMetric(reference="{{item.tool_calls}}") - - result = evaluator.run( - metric=metric, - dataset=[ - { - "messages": [ - {"role": "user", "content": "Book a table for 2 at 7pm."}, - { - "role": "assistant", - "content": "Booking...", - "tool_calls": [ - { - "function": { - "name": "book_table", - "arguments": {"people": 2, "time": "7pm"}, - } - } - ], - }, - ], - "tool_calls": [ - { - "function": { - "name": "book_table", - "arguments": {"people": 2, "time": "7pm"}, - } - } - ], - "response": { - "choices": [ - { - "message": { - "tool_calls": [ - { - "function": { - "name": "book_table", - "arguments": '{"people": 2, "time": "7pm"}', - } - } - ] - } - } - ] - }, - } - ], - ) - print(result.aggregate_scores) - ``` - {% endraw %} - -=== "Submit Job" - - {% raw %} - ```python - from nemo_evaluator_sdk import RunConfig, ToolCallingMetric - - metric = ToolCallingMetric(reference="{{item.tool_calls}}") - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "tool_calls": [ - { - "function": { - "name": "book_table", - "arguments": {"people": 2, "time": "7pm"}, - } - } - ], - "response": { - "choices": [ - { - "message": { - "tool_calls": [ - { - "function": { - "name": "book_table", - "arguments": '{"people": 2, "time": "7pm"}', - } - } - ] - } - } - ] - }, - } - ], - config=RunConfig(parallelism=4), - ) - job.wait_until_done() - result = job.get_result() - print(result.aggregate_scores) - ``` - {% endraw %} - -=== "Result" - - ```json - { - "aggregate_scores": [ - { - "name": "function_name_accuracy", - "count": 1, - "mean": 1.0, - "min": 1.0, - "max": 1.0 - }, - { - "name": "function_name_and_args_accuracy", - "count": 1, - "mean": 1.0, - "min": 1.0, - "max": 1.0 - } - ] - } - ``` - ---- - -### Topic Adherence - -Measures how well the agent maintained focus on assigned topics throughout a conversation. Supports F1, precision, or recall scoring modes. - -#### Data Format - -```json -{ - "user_input": [ - { - "content": "How do I stay healthy?", - "type": "human" - }, - { - "content": "Eat more fruits and vegetables, and exercise regularly.", - "type": "ai" - } - ], - "reference_topics": [ - "health", - "nutrition", - "fitness" - ] -} -``` - -#### Configuration Options - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `metric_mode` | string | `"f1"` | Scoring mode: `"f1"`, `"precision"`, or `"recall"` | - -=== "Run Locally" - - ```python - from nemo_evaluator_sdk import Model - from nemo_evaluator_sdk.metrics.ragas import TopicAdherenceMetric - - judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="meta/llama-3.1-70b-instruct", - api_key_secret="nvidia-api-key", - ) - metric = TopicAdherenceMetric(metric_mode="f1", judge_model=judge_model) - - result = evaluator.run( - metric=metric, - dataset=[ - { - "user_input": [ - {"content": "Tell me about healthy eating", "type": "human"}, - { - "content": "Eating fruits and vegetables is essential for good health.", - "type": "ai", - }, - ], - "reference_topics": ["health", "nutrition", "diet"], - } - ], - ) - print(result.aggregate_scores) - ``` - -=== "Submit Job" - - ```python - from nemo_evaluator_sdk import RunConfig, Model - from nemo_evaluator_sdk.metrics.ragas import TopicAdherenceMetric - - judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="meta/llama-3.1-70b-instruct", - api_key_secret="nvidia-api-key", - ) - metric = TopicAdherenceMetric(metric_mode="f1", judge_model=judge_model) - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "user_input": [ - {"content": "Tell me about healthy eating", "type": "human"}, - { - "content": "Eating fruits and vegetables is essential for good health.", - "type": "ai", - }, - ], - "reference_topics": ["health", "nutrition", "diet"], - } - ], - config=RunConfig(parallelism=4), - ) - job.wait_until_done() - result = job.get_result() - print(result.aggregate_scores) - ``` - -=== "Online Target Generation" - - {% raw %} - ```python - from nemo_evaluator_sdk import RunConfigOnlineModel, InferenceParams, Model - from nemo_evaluator_sdk.metrics.ragas import TopicAdherenceMetric - - judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="meta/llama-3.1-70b-instruct", - api_key_secret="nvidia-api-key", - ) - target_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="nvidia/llama-3.3-nemotron-super-49b-v1", - api_key_secret="nvidia-api-key", - ) - metric = TopicAdherenceMetric(metric_mode="f1", judge_model=judge_model) - - result = evaluator.run( - metric=metric, - dataset=[ - { - "user_input": "Tell me about healthy eating", - "reference_topics": ["health", "nutrition", "diet"], - } - ], - config=RunConfigOnlineModel( - parallelism=4, - inference=InferenceParams(temperature=0.7, max_tokens=1024), - ), - target=target_model, - prompt_template={ - "messages": [ - { - "role": "user", - "content": "{{item.user_input}}", - } - ] - }, - ) - print(result.aggregate_scores) - ``` - {% endraw %} - -=== "Result" - - ```json - { - "aggregate_scores": [ - { - "name": "topic_adherence(mode=f1)", - "count": 1, - "mean": 0.85, - "min": 0.85, - "max": 0.85 - } - ], - "row_scores": [ - { - "index": 0, - "scores": { - "topic_adherence(mode=f1)": 0.85 - } - } - ] - } - ``` - ---- - -### Agent Goal Accuracy - -Evaluates whether the agent successfully completed the requested task. Returns a binary score (0 or 1). Supports evaluation with or without a reference outcome. - -#### With Reference - -Compare the agent's outcome against a known reference: - -**Data Format** - -```json -{ - "user_input": [ - { - "content": "Book a table at a Chinese restaurant for 8pm", - "type": "human" - }, - { - "content": "I'll search for restaurants.", - "type": "ai", - "tool_calls": [ - { - "name": "restaurant_search", - "args": {} - } - ] - }, - { - "content": "Found: Italian Place", - "type": "tool" - }, - { - "content": "Your table at Italian Place is booked for 8pm.", - "type": "ai" - } - ], - "reference": "Successfully booked a table at a restaurant for 8pm" -} -``` - -**Configuration Options** - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `use_reference` | boolean | `True` | Whether to compare against a reference outcome | - -=== "Run Locally" - - ```python - from nemo_evaluator_sdk import Model - from nemo_evaluator_sdk.metrics.ragas import AgentGoalAccuracyMetric - - judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="meta/llama-3.1-70b-instruct", - api_key_secret="nvidia-api-key", - ) - metric = AgentGoalAccuracyMetric(use_reference=True, judge_model=judge_model) - - result = evaluator.run( - metric=metric, - dataset=[ - { - "user_input": [ - {"content": "Book a table at a restaurant for 8pm", "type": "human"}, - { - "content": "I'll search for restaurants.", - "type": "ai", - "tool_calls": [{"name": "restaurant_search", "args": {}}], - }, - {"content": "Found: Italian Place", "type": "tool"}, - { - "content": "Your table at Italian Place is booked for 8pm.", - "type": "ai", - }, - ], - "reference": "Successfully booked a table at a restaurant for 8pm", - } - ], - ) - print(result.aggregate_scores) - ``` - -=== "Submit Job" - - ```python - from nemo_evaluator_sdk import RunConfig, Model - from nemo_evaluator_sdk.metrics.ragas import AgentGoalAccuracyMetric - - judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="meta/llama-3.1-70b-instruct", - api_key_secret="nvidia-api-key", - ) - metric = AgentGoalAccuracyMetric(use_reference=True, judge_model=judge_model) - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "user_input": [ - {"content": "Book a table at a restaurant for 8pm", "type": "human"}, - { - "content": "I'll search for restaurants.", - "type": "ai", - "tool_calls": [{"name": "restaurant_search", "args": {}}], - }, - {"content": "Found: Italian Place", "type": "tool"}, - { - "content": "Your table at Italian Place is booked for 8pm.", - "type": "ai", - }, - ], - "reference": "Successfully booked a table at a restaurant for 8pm", - } - ], - config=RunConfig(parallelism=4), - ) - job.wait_until_done() - result = job.get_result() - print(result.aggregate_scores) - ``` - -=== "Result" - - ```json - { - "aggregate_scores": [ - { - "name": "agent_goal_accuracy", - "count": 1, - "mean": 1.0, - "min": 1.0, - "max": 1.0 - } - ], - "row_scores": [ - { - "index": 0, - "scores": { - "agent_goal_accuracy": 1.0 - } - } - ] - } - ``` - -#### Without Reference - -The judge LLM infers the goal from the conversation context: - -**Data Format** - -```json -{ - "user_input": [ - { - "content": "Set a reminder for my dentist appointment tomorrow at 2pm", - "type": "human" - }, - { - "content": "I'll set that reminder for you.", - "type": "ai", - "tool_calls": [ - { - "name": "set_reminder", - "args": { - "title": "Dentist appointment", - "date": "tomorrow", - "time": "2pm" - } - } - ] - }, - { - "content": "Reminder set successfully.", - "type": "tool" - }, - { - "content": "Your reminder for the dentist appointment tomorrow at 2pm has been set.", - "type": "ai" - } - ] -} -``` - -=== "Run Locally" - - ```python - from nemo_evaluator_sdk import Model - from nemo_evaluator_sdk.metrics.ragas import AgentGoalAccuracyMetric - - judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="meta/llama-3.1-70b-instruct", - api_key_secret="nvidia-api-key", - ) - metric = AgentGoalAccuracyMetric(use_reference=False, judge_model=judge_model) - - result = evaluator.run( - metric=metric, - dataset=[ - { - "user_input": [ - { - "content": "Set a reminder for my dentist appointment tomorrow at 2pm", - "type": "human", - }, - { - "content": "I'll set that reminder for you.", - "type": "ai", - "tool_calls": [ - { - "name": "set_reminder", - "args": { - "title": "Dentist appointment", - "date": "tomorrow", - "time": "2pm", - }, - } - ], - }, - {"content": "Reminder set successfully.", "type": "tool"}, - {"content": "Your reminder has been set.", "type": "ai"}, - ], - } - ], - ) - print(result.aggregate_scores) - ``` - -=== "Submit Job" - - ```python - from nemo_evaluator_sdk import RunConfig, Model - from nemo_evaluator_sdk.metrics.ragas import AgentGoalAccuracyMetric - - judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="meta/llama-3.1-70b-instruct", - api_key_secret="nvidia-api-key", - ) - metric = AgentGoalAccuracyMetric(use_reference=False, judge_model=judge_model) - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "user_input": [ - { - "content": "Set a reminder for my dentist appointment tomorrow at 2pm", - "type": "human", - }, - { - "content": "I'll set that reminder for you.", - "type": "ai", - "tool_calls": [ - { - "name": "set_reminder", - "args": { - "title": "Dentist appointment", - "date": "tomorrow", - "time": "2pm", - }, - } - ], - }, - {"content": "Reminder set successfully.", "type": "tool"}, - {"content": "Your reminder has been set.", "type": "ai"}, - ], - } - ], - config=RunConfig(parallelism=4), - ) - job.wait_until_done() - result = job.get_result() - print(result.aggregate_scores) - ``` - ---- - -### Answer Accuracy - -Evaluates the factual correctness of an agent's answer by comparing it against a reference answer. Two LLM judges independently rate the agreement, and scores are averaged. Scores range from 0 (incorrect) to 0.5 (partial match) to 1 (exact match). - -#### Data Format - -```json -{ - "user_input": "What is the capital of France?", - "response": "The capital of France is Paris.", - "reference": "Paris" -} -``` - -=== "Run Locally" - - ```python - from nemo_evaluator_sdk import Model - from nemo_evaluator_sdk.metrics.ragas import AnswerAccuracyMetric - - judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="meta/llama-3.1-70b-instruct", - api_key_secret="nvidia-api-key", - ) - metric = AnswerAccuracyMetric(judge_model=judge_model) - - result = evaluator.run( - metric=metric, - dataset=[ - { - "user_input": "What is the capital of France?", - "response": "The capital of France is Paris.", - "reference": "Paris", - } - ], - ) - print(result.aggregate_scores) - ``` - -=== "Submit Job" - - ```python - from nemo_evaluator_sdk import RunConfig, Model - from nemo_evaluator_sdk.metrics.ragas import AnswerAccuracyMetric - - judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="meta/llama-3.1-70b-instruct", - api_key_secret="nvidia-api-key", - ) - metric = AnswerAccuracyMetric(judge_model=judge_model) - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "user_input": "What is the capital of France?", - "response": "The capital of France is Paris.", - "reference": "Paris", - } - ], - config=RunConfig(parallelism=4), - ) - job.wait_until_done() - result = job.get_result() - print(result.aggregate_scores) - ``` - -=== "Online Target Generation" - - {% raw %} - ```python - from nemo_evaluator_sdk import RunConfigOnlineModel, InferenceParams, Model - from nemo_evaluator_sdk.metrics.ragas import AnswerAccuracyMetric - - judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="meta/llama-3.1-70b-instruct", - api_key_secret="nvidia-api-key", - ) - target_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="nvidia/llama-3.3-nemotron-super-49b-v1", - api_key_secret="nvidia-api-key", - ) - metric = AnswerAccuracyMetric(judge_model=judge_model) - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "user_input": "What is the capital of France?", - "reference": "Paris", - } - ], - config=RunConfigOnlineModel( - parallelism=4, - inference=InferenceParams(temperature=0.7, max_tokens=1024), - ), - target=target_model, - prompt_template={ - "messages": [ - { - "role": "user", - "content": "{{item.user_input}}", - } - ] - }, - ) - - job.wait_until_done() - result = job.get_result() - print(result.aggregate_scores) - ``` - {% endraw %} - -=== "Result" - - ```json - { - "aggregate_scores": [ - { - "name": "nv_accuracy", - "count": 1, - "mean": 1.0, - "min": 1.0, - "max": 1.0 - } - ], - "row_scores": [ - { - "index": 0, - "scores": { - "nv_accuracy": 1.0 - } - } - ] - } - ``` - ---- - -### Trajectory Evaluation - -Evaluates the agent's decision-making process by analyzing the entire sequence of actions (trajectory) taken to accomplish a goal. This **system metric** assesses whether the agent chose appropriate tools in the correct order. - -!!! info - **NAT Format Requirement**: This metric supports the NVIDIA Agent Toolkit format with `intermediate_steps` containing detailed event traces. - - **Current plugin SDK support**: The current plugin SDK does not expose a typed trajectory-evaluation metric class. Use the data-format details below when preparing datasets for environments where the system metric is enabled, but do not use the old generated evaluator job APIs for plugin SDK execution. - -#### Data Format - -Each data entry must follow the [NeMo Agent Toolkit](https://docs.nvidia.com/nemo/agent-toolkit/latest/index.html) format: - -```json -{ - "question": "What are LLMs", - "generated_answer": "LLMs, or Large Language Models, are a type of artificial intelligence designed to process and generate human-like language. They are trained on vast amounts of text data and can be fine-tuned for specific tasks or guided by prompt engineering.", - "answer": "LLMs stand for Large Language Models, which are a type of machine learning model designed for natural language processing tasks such as language generation.", - "intermediate_steps": [ - { - "payload": { - "event_type": "LLM_END", - "name": "nvidia/llama-3.3-nemotron-super-49b-v1", - "data": { - "input": "\nPrevious conversation history:\n\n\nQuestion: What are LLMs\n", - "output": "Thought: I need to find information about LLMs to answer this question.\n\nAction: wikipedia_search\nAction Input: {'question': 'LLMs'}\n\n" - } - } - }, - { - "payload": { - "event_type": "TOOL_END", - "name": "wikipedia_search", - "data": { - "input": "{'question': 'LLMs'}", - "output": "\nA large language model (LLM) is a language model trained with self-supervised machine learning..." - } - } - }, - { - "payload": { - "event_type": "LLM_END", - "name": "meta/llama-3.1-70b-instruct", - "data": { - "input": "...", - "output": "Thought: I now know the final answer\n\nFinal Answer: LLMs, or Large Language Models, are a type of artificial intelligence..." - } - } - } - ] -} -``` - -#### Parameters - -| Parameter | Required | Type | Description | -|-----------|----------|------|-------------| -| `judge` | required | object | Judge LLM configuration | -| `trajectory_used_tools` | required | string | Comma-separated list of tools available to the agent. Example: `"wikipedia_search,current_datetime,code_generation"` | -| `trajectory_custom_tools` | optional | object | JSON mapping custom tool names to descriptions for non-default tools | - ---- - -## Judge Configuration - -Most agentic metrics require a judge LLM. Configure the judge model in the metric definition: - -```python -from nemo_evaluator_sdk import Model - - -judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="meta/llama-3.1-70b-instruct", - api_key_secret="nvidia-api-key", -) -``` - -!!! info - **Recommended model size**: Use a 70B+ parameter model as the judge for reliable results. Smaller models may fail to follow the required output schema, causing parsing errors. - -### Using Reasoning Models - -For models that support extended reasoning, such as `nvidia/llama-3.3-nemotron-super-49b-v1`, add system prompt and reasoning parameters to online model generation: - -```python -from nemo_evaluator_sdk import RunConfigOnlineModel, InferenceParams, Model, ReasoningParams - - -judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="nvidia/llama-3.3-nemotron-super-49b-v1", - api_key_secret="nvidia-api-key", -) - -config = RunConfigOnlineModel( - system_prompt="detailed thinking on", - reasoning=ReasoningParams(end_token=""), - inference=InferenceParams(max_tokens=1024), -) -``` - -### Managing Secrets for Authenticated Endpoints - -If your judge endpoint requires an API key, store it as a secret: - -```python -from nemo_evaluator_sdk import Model - -client.secrets.create(name="nvidia-api-key", value="nvapi-YOUR_API_KEY_HERE") - -judge_model = Model( - url="https://integrate.api.nvidia.com/v1/chat/completions", - name="meta/llama-3.1-70b-instruct", - api_key_secret="nvidia-api-key", -) -``` - -For more details on secret management, refer to [Managing Secrets](../../get-started/concepts/manage-secrets.md). - -For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](model-configuration.md#model-api-authentication). - ---- - -## Job Management - -After submitting a durable remote job with `evaluator.submit(metric=metric, dataset=dataset)`, use the returned job resource to monitor execution and retrieve results: - -```python -job.wait_until_done() -result = job.get_result() -print(result.aggregate_scores) -``` - -Navigate to [Metrics Job Management](job-management.md) for more job lifecycle details. - ---- - -## Dataset Notes - -The current plugin SDK examples use inline dataset rows through `dataset=[...]`. Keep each row shaped for the selected metric, including fields such as `user_input`, `response`, `reference`, `reference_topics`, `reference_tool_calls`, or OpenAI-style `tool_calls` as required. - -Use `RunConfig(limit_samples=...)` when you want to test a small slice of a larger inline dataset before submitting the full request. - ---- - -## Important Notes - -1. **Execution choice**: Use `run` for local in-process evaluation and `submit` for durable remote jobs with `wait_until_done()` and `get_result()`. - -2. **Column Names**: RAGAS metrics use specific column names: - - `user_input` (not `question`) - - `response` (not `answer`) - - `reference` (not `ground_truth`) - -3. **Judge Model Quality**: For metrics requiring a judge, evaluation quality depends on the judge model's ability to follow instructions. Larger models (70B+) produce more consistent results. - -4. **RAGAS Dependency**: These metrics are powered by RAGAS and may have version-specific behavior. - -5. **NaN troubleshooting for judge-based metrics**: If you see `nan_count > 0` with `mean = null`, check judge model authentication first (API key secret, endpoint access, and model permissions). See [Model API Authentication](model-configuration.md#model-api-authentication) for `api_key_secret` behavior. Some RAGAS metrics are known to convert auth failures into `NaN` scores instead of raising a hard error. - -!!! info - - [Agent Configuration](agent-configuration.md) - Use agents (generic or NAT) as targets in online evaluation jobs - - [Agentic Benchmarks](../benchmarks/agentic.md) - BFCL benchmark for tool-calling evaluation - - [LLM-as-a-Judge](llm-as-a-judge.md) - Custom judge-based evaluation - - [Evaluation Results](results.md) - Understanding results - - [RAG Metrics](rag.md) - RAGAS metrics for RAG pipelines diff --git a/docs/evaluator/metrics/agentic.mdx b/docs/evaluator/metrics/agentic.mdx new file mode 100644 index 0000000000..269c1caea6 --- /dev/null +++ b/docs/evaluator/metrics/agentic.mdx @@ -0,0 +1,1290 @@ +--- +title: "Agentic Evaluation Metrics" +description: "" +--- + + +Evaluate agent-based and multi-step reasoning models using metrics powered by [RAGAS](https://github.com/explodinggradients/ragas). These metrics assess tool calling accuracy, goal completion, topic adherence, and answer correctness in agentic workflows. + +## Agent Workflow Evaluation Stages + +Key stages of agent workflow evaluation: + +![Agent Evaluation Framework](../images/agent_eval_framework.png) + +**1. Intermediate Steps Evaluation** +Assesses the correctness of intermediate steps during agent execution: + +- **Tool Use**: Validates that the agent invoked the right tools with correct arguments at each step. Refer to [Tool Call Accuracy](#tool-call-accuracy) for implementation details. + +**2. Final-Step Evaluation** +Evaluates the quality of the agent's final output using: + +- **Agent Goal Accuracy**: Measures whether the agent successfully completed the requested task. Refer to [Agent Goal Accuracy](#agent-goal-accuracy). +- **Topic Adherence**: Assesses how well the agent maintained focus on the assigned topic throughout the conversation. Refer to [Topic Adherence](#topic-adherence). +- **Answer Accuracy**: Evaluates the factual correctness of agent answers. Refer to [Answer Accuracy](#answer-accuracy). +- **Custom Metrics**: For domain-specific or custom evaluation criteria, use [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge) with the `data` task type. + +**3. Trajectory Evaluation** +Evaluates the agent's decision-making process by analyzing the entire sequence of actions taken to accomplish a goal. This includes assessing whether the agent chose appropriate tools in the correct order. Refer to [Trajectory Evaluation](#trajectory-evaluation) for the expected data format and current plugin SDK support. + +--- + +## Online vs Offline Evaluation + +Agentic metrics support two execution patterns through the Evaluator plugin SDK: + +### Offline Evaluation + +Offline evaluation scores pre-generated responses or tool calls already present in the dataset: + +- Dataset rows are passed inline with `dataset=[...]`, as a file Path, or as a FilesetRef. +- No model or agent generation is performed. +- Use this mode to evaluate existing agent outputs or compare different response strategies. + +### Online Target Generation + +Online target generation first calls a model or agent target, then evaluates the generated output: + +- Configure the target with `Model` or `Agent` from `nemo_evaluator_sdk`. +- Pass the target through `target=...`. +- Use `RunConfigOnlineModel` for model targets and `RunConfigOnline` for agent targets. +- Include a `prompt_template` when the dataset row must be transformed into a model or agent request. + + + +**Response Usage**: In online target generation, the generated response is used as the metric response. A dataset `response` column is optional and is superseded by the generated response for that run. + + +--- + +## Overview + +Agentic metrics evaluate different aspects of agent behavior: + +| Metric | Use Case | Requires Judge | Plugin SDK Execution | +|--------|----------|----------------|----------------------| +| [**Tool Call Accuracy**](#tool-call-accuracy) | Evaluates tool/function call correctness | No | `run` + `submit` | +| [**Tool Calling** (template)](#tool-calling-template) | Evaluates tool/function call correctness using Jinja templates | No | `run` + `submit` | +| [**Topic Adherence**](#topic-adherence) | Measures topic focus in multi-turn conversations | Yes | `run` + `submit` | +| [**Agent Goal Accuracy**](#agent-goal-accuracy) | Assesses goal completion, with or without reference. | Yes | `run` + `submit` | +| [**Answer Accuracy**](#answer-accuracy) | Checks factual correctness | Yes | `run` + `submit` | +| [**Trajectory Evaluation**](#trajectory-evaluation) | Evaluates decision-making across action sequence | Yes | Not exposed as a typed plugin SDK metric | + + + +Use `evaluator.run(...)` for local in-process evaluation and `evaluator.submit(...)` for durable remote platform jobs. The examples below use inline dataset rows through `dataset=[...]`, but you can use a file Path or a FilesetRef instead. + + +## Prerequisites + +Before running agentic evaluations: + +1. **Workspace**: Have a workspace created. All remote resources, including secrets and jobs, are scoped to a workspace. +2. **Judge LLM endpoint** *(for most metrics)*: Have access to an LLM that will serve as your judge. +3. **API key secret** *(if judge requires auth)*: If your judge endpoint requires authentication, [create a secret](/documentation/get-started/core-concepts/manage-secrets) to store the API key. For local `run` versus remote `submit` behavior, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). +4. **Initialize the SDK**: + +```python +import os +from nemo_evaluator.sdk import Evaluator +from nemo_platform import NeMoPlatform + + +client = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) +evaluator: Evaluator = client.evaluator # this object is an Evaluator resource +``` + +### Creating a Secret for API Keys + +If using external endpoints, such as [Build NVIDIA](https://build.nvidia.com/models) API endpoints (https://integrate.api.nvidia.com/v1), create a secret first: + +```python +client.secrets.create( + name="nvidia-api-key", + value="nvapi-YOUR_API_KEY_HERE", + description="NVIDIA Build API key for RAGAS metrics", +) +``` + +### SDK Types Reference + +The plugin SDK examples use context-agnostic metric classes and runtime value classes: + +```python +from nemo_evaluator_sdk import ( + ToolCallingMetric, +) +from nemo_evaluator_sdk.metrics.ragas import ( + AgentGoalAccuracyMetric, + AnswerAccuracyMetric, + ToolCallAccuracyMetric, + TopicAdherenceMetric, +) +from nemo_evaluator_sdk import ( + Agent, + RunConfigOnlineModel, + RunConfigOnline, + RunConfig, + InferenceParams, + Model, + ReasoningParams, +) +``` + +Use `dataset=[...]` for inline rows. For offline scoring options, use `config=RunConfig(parallelism=...)`. Whenever outputs must be generated before scoring, pass `target=Model(...)` or `target=Agent(...)` plus the corresponding online parameters. Use the same `dataset`, `config`, and `target` arguments for both `evaluator.run(...)` and `evaluator.submit(...)`; durable jobs follow the identical pattern as local runs and only differ in waiting for and fetching results. + +--- + +## Agentic Metrics + +### Tool Call Accuracy + +Evaluates whether the agent invoked the correct tools with the correct arguments. This metric **does not require a judge LLM**. + + + +**Online/offline support**: Tool Call Accuracy supports scoring existing tool calls directly. It can also score tool calls produced during online target generation when the target response includes the required tool-call fields. + + +#### Data Format + +```json +{ + "user_input": [ + { + "content": "What's the weather like in New York?", + "type": "human" + }, + { + "content": "Let me check that for you.", + "type": "ai", + "tool_calls": [ + { + "name": "weather_check", + "args": { + "location": "New York" + } + } + ] + }, + { + "content": "It's 75°F and partly cloudy.", + "type": "tool" + }, + { + "content": "The weather in New York is 75°F and partly cloudy.", + "type": "ai" + } + ], + "reference_tool_calls": [ + { + "name": "weather_check", + "args": { + "location": "New York" + } + } + ] +} +``` + + + + + +```python +from nemo_evaluator_sdk.metrics.ragas import ToolCallAccuracyMetric +metric = ToolCallAccuracyMetric() + +result = evaluator.run( + metric=metric, + dataset=[ + { + "user_input": [ + {"content": "What's the weather in Paris?", "type": "human"}, + { + "content": "Let me check.", + "type": "ai", + "tool_calls": [{"name": "weather_api", "args": {"city": "Paris"}}], + }, + {"content": "Sunny, 22°C", "type": "tool"}, + {"content": "It's sunny and 22°C in Paris.", "type": "ai"}, + ], + "reference_tool_calls": [ + {"name": "weather_api", "args": {"city": "Paris"}} + ], + } + ], +) +print(result.aggregate_scores) +``` + + + + +```python +from nemo_evaluator_sdk import RunConfig +from nemo_evaluator_sdk.metrics.ragas import ToolCallAccuracyMetric +metric = ToolCallAccuracyMetric() + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "user_input": [ + {"content": "What's the weather in Paris?", "type": "human"}, + { + "content": "Let me check.", + "type": "ai", + "tool_calls": [{"name": "weather_api", "args": {"city": "Paris"}}], + }, + {"content": "Sunny, 22°C", "type": "tool"}, + {"content": "It's sunny and 22°C in Paris.", "type": "ai"}, + ], + "reference_tool_calls": [ + {"name": "weather_api", "args": {"city": "Paris"}} + ], + } + ], + config=RunConfig(parallelism=4), +) +job.wait_until_done() +result = job.get_result() +print(result.aggregate_scores) +``` + + + + +```json +{ + "aggregate_scores": [ + { + "name": "tool_call_accuracy", + "count": 1, + "mean": 1.0, + "min": 1.0, + "max": 1.0 + } + ], + "row_scores": [ + { + "index": 0, + "scores": { + "tool_call_accuracy": 1.0 + } + } + ] +} +``` + + + + +--- + +### Tool Calling (Template) + +A template-based metric for evaluating tool/function call accuracy. Unlike the RAGAS Tool Call Accuracy metric, this metric uses configurable templates and produces multiple scores. + + + +**Online/offline support**: Tool Calling supports scoring existing tool-call outputs directly. Use online target generation when you want the model or agent target to produce the response first. + + +#### Scores Produced + +- `function_name_accuracy` - Accuracy of function names only +- `function_name_and_args_accuracy` - Accuracy of both function names and arguments + +#### Data Format + +Data must use OpenAI-compliant tool calling format: + +```json +{ + "messages": [ + { + "role": "user", + "content": "Book a table for 2 at 7pm." + }, + { + "role": "assistant", + "content": "Booking a table...", + "tool_calls": [ + { + "function": { + "name": "book_table", + "arguments": { + "people": 2, + "time": "7pm" + } + } + } + ] + } + ], + "tool_calls": [ + { + "function": { + "name": "book_table", + "arguments": { + "people": 2, + "time": "7pm" + } + } + } + ], + "response": { + "choices": [ + { + "message": { + "tool_calls": [ + { + "function": { + "name": "book_table", + "arguments": "{\"people\": 2, \"time\": \"7pm\"}" + } + } + ] + } + } + ] + } +} +``` + + + +- Function names with dots (`.`) must be replaced with underscores (`_`). +- Comparison is case-sensitive. +- Order of tool calls is ignored, which supports parallel tool calling. + + + + + + +```python + + +from nemo_evaluator_sdk import ToolCallingMetric +metric = ToolCallingMetric(reference="{{item.tool_calls}}") + +result = evaluator.run( + metric=metric, + dataset=[ + { + "messages": [ + {"role": "user", "content": "Book a table for 2 at 7pm."}, + { + "role": "assistant", + "content": "Booking...", + "tool_calls": [ + { + "function": { + "name": "book_table", + "arguments": {"people": 2, "time": "7pm"}, + } + } + ], + }, + ], + "tool_calls": [ + { + "function": { + "name": "book_table", + "arguments": {"people": 2, "time": "7pm"}, + } + } + ], + "response": { + "choices": [ + { + "message": { + "tool_calls": [ + { + "function": { + "name": "book_table", + "arguments": '{"people": 2, "time": "7pm"}', + } + } + ] + } + } + ] + }, + } + ], +) +print(result.aggregate_scores) +``` + + + + +```python +from nemo_evaluator_sdk import RunConfig, ToolCallingMetric + +metric = ToolCallingMetric(reference="{{item.tool_calls}}") + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "tool_calls": [ + { + "function": { + "name": "book_table", + "arguments": {"people": 2, "time": "7pm"}, + } + } + ], + "response": { + "choices": [ + { + "message": { + "tool_calls": [ + { + "function": { + "name": "book_table", + "arguments": '{"people": 2, "time": "7pm"}', + } + } + ] + } + } + ] + }, + } + ], + config=RunConfig(parallelism=4), +) +job.wait_until_done() +result = job.get_result() +print(result.aggregate_scores) +``` + + + + +```json +{ + "aggregate_scores": [ + { + "name": "function_name_accuracy", + "count": 1, + "mean": 1.0, + "min": 1.0, + "max": 1.0 + }, + { + "name": "function_name_and_args_accuracy", + "count": 1, + "mean": 1.0, + "min": 1.0, + "max": 1.0 + } + ] +} +``` + + + + +--- + +### Topic Adherence + +Measures how well the agent maintained focus on assigned topics throughout a conversation. Supports F1, precision, or recall scoring modes. + +#### Data Format + +```json +{ + "user_input": [ + { + "content": "How do I stay healthy?", + "type": "human" + }, + { + "content": "Eat more fruits and vegetables, and exercise regularly.", + "type": "ai" + } + ], + "reference_topics": [ + "health", + "nutrition", + "fitness" + ] +} +``` + +#### Configuration Options + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `metric_mode` | string | `"f1"` | Scoring mode: `"f1"`, `"precision"`, or `"recall"` | + + + + + +```python +from nemo_evaluator_sdk import Model +from nemo_evaluator_sdk.metrics.ragas import TopicAdherenceMetric + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="meta/llama-3.1-70b-instruct", + api_key_secret="nvidia-api-key", +) +metric = TopicAdherenceMetric(metric_mode="f1", judge_model=judge_model) + +result = evaluator.run( + metric=metric, + dataset=[ + { + "user_input": [ + {"content": "Tell me about healthy eating", "type": "human"}, + { + "content": "Eating fruits and vegetables is essential for good health.", + "type": "ai", + }, + ], + "reference_topics": ["health", "nutrition", "diet"], + } + ], +) +print(result.aggregate_scores) +``` + + + + +```python +from nemo_evaluator_sdk import RunConfig, Model +from nemo_evaluator_sdk.metrics.ragas import TopicAdherenceMetric + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="meta/llama-3.1-70b-instruct", + api_key_secret="nvidia-api-key", +) +metric = TopicAdherenceMetric(metric_mode="f1", judge_model=judge_model) + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "user_input": [ + {"content": "Tell me about healthy eating", "type": "human"}, + { + "content": "Eating fruits and vegetables is essential for good health.", + "type": "ai", + }, + ], + "reference_topics": ["health", "nutrition", "diet"], + } + ], + config=RunConfig(parallelism=4), +) +job.wait_until_done() +result = job.get_result() +print(result.aggregate_scores) +``` + + + + +```python +from nemo_evaluator_sdk import RunConfigOnlineModel, InferenceParams, Model +from nemo_evaluator_sdk.metrics.ragas import TopicAdherenceMetric + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="meta/llama-3.1-70b-instruct", + api_key_secret="nvidia-api-key", +) +target_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="nvidia/llama-3.3-nemotron-super-49b-v1", + api_key_secret="nvidia-api-key", +) +metric = TopicAdherenceMetric(metric_mode="f1", judge_model=judge_model) + +result = evaluator.run( + metric=metric, + dataset=[ + { + "user_input": "Tell me about healthy eating", + "reference_topics": ["health", "nutrition", "diet"], + } + ], + config=RunConfigOnlineModel( + parallelism=4, + inference=InferenceParams(temperature=0.7, max_tokens=1024), + ), + target=target_model, + prompt_template={ + "messages": [ + { + "role": "user", + "content": "{{item.user_input}}", + } + ] + }, +) +print(result.aggregate_scores) +``` + + + + +```json +{ + "aggregate_scores": [ + { + "name": "topic_adherence(mode=f1)", + "count": 1, + "mean": 0.85, + "min": 0.85, + "max": 0.85 + } + ], + "row_scores": [ + { + "index": 0, + "scores": { + "topic_adherence(mode=f1)": 0.85 + } + } + ] +} +``` + + + + +--- + +### Agent Goal Accuracy + +Evaluates whether the agent successfully completed the requested task. Returns a binary score (0 or 1). Supports evaluation with or without a reference outcome. + +#### With Reference + +Compare the agent's outcome against a known reference: + +**Data Format** + +```json +{ + "user_input": [ + { + "content": "Book a table at a Chinese restaurant for 8pm", + "type": "human" + }, + { + "content": "I'll search for restaurants.", + "type": "ai", + "tool_calls": [ + { + "name": "restaurant_search", + "args": {} + } + ] + }, + { + "content": "Found: Italian Place", + "type": "tool" + }, + { + "content": "Your table at Italian Place is booked for 8pm.", + "type": "ai" + } + ], + "reference": "Successfully booked a table at a restaurant for 8pm" +} +``` + +**Configuration Options** + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `use_reference` | boolean | `True` | Whether to compare against a reference outcome | + + + + + +```python +from nemo_evaluator_sdk import Model +from nemo_evaluator_sdk.metrics.ragas import AgentGoalAccuracyMetric + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="meta/llama-3.1-70b-instruct", + api_key_secret="nvidia-api-key", +) +metric = AgentGoalAccuracyMetric(use_reference=True, judge_model=judge_model) + +result = evaluator.run( + metric=metric, + dataset=[ + { + "user_input": [ + {"content": "Book a table at a restaurant for 8pm", "type": "human"}, + { + "content": "I'll search for restaurants.", + "type": "ai", + "tool_calls": [{"name": "restaurant_search", "args": {}}], + }, + {"content": "Found: Italian Place", "type": "tool"}, + { + "content": "Your table at Italian Place is booked for 8pm.", + "type": "ai", + }, + ], + "reference": "Successfully booked a table at a restaurant for 8pm", + } + ], +) +print(result.aggregate_scores) +``` + + + + +```python +from nemo_evaluator_sdk import RunConfig, Model +from nemo_evaluator_sdk.metrics.ragas import AgentGoalAccuracyMetric + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="meta/llama-3.1-70b-instruct", + api_key_secret="nvidia-api-key", +) +metric = AgentGoalAccuracyMetric(use_reference=True, judge_model=judge_model) + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "user_input": [ + {"content": "Book a table at a restaurant for 8pm", "type": "human"}, + { + "content": "I'll search for restaurants.", + "type": "ai", + "tool_calls": [{"name": "restaurant_search", "args": {}}], + }, + {"content": "Found: Italian Place", "type": "tool"}, + { + "content": "Your table at Italian Place is booked for 8pm.", + "type": "ai", + }, + ], + "reference": "Successfully booked a table at a restaurant for 8pm", + } + ], + config=RunConfig(parallelism=4), +) +job.wait_until_done() +result = job.get_result() +print(result.aggregate_scores) +``` + + + + +```json +{ + "aggregate_scores": [ + { + "name": "agent_goal_accuracy", + "count": 1, + "mean": 1.0, + "min": 1.0, + "max": 1.0 + } + ], + "row_scores": [ + { + "index": 0, + "scores": { + "agent_goal_accuracy": 1.0 + } + } + ] +} +``` + + + + +#### Without Reference + +The judge LLM infers the goal from the conversation context: + +**Data Format** + +```json +{ + "user_input": [ + { + "content": "Set a reminder for my dentist appointment tomorrow at 2pm", + "type": "human" + }, + { + "content": "I'll set that reminder for you.", + "type": "ai", + "tool_calls": [ + { + "name": "set_reminder", + "args": { + "title": "Dentist appointment", + "date": "tomorrow", + "time": "2pm" + } + } + ] + }, + { + "content": "Reminder set successfully.", + "type": "tool" + }, + { + "content": "Your reminder for the dentist appointment tomorrow at 2pm has been set.", + "type": "ai" + } + ] +} +``` + + + + + +```python +from nemo_evaluator_sdk import Model +from nemo_evaluator_sdk.metrics.ragas import AgentGoalAccuracyMetric + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="meta/llama-3.1-70b-instruct", + api_key_secret="nvidia-api-key", +) +metric = AgentGoalAccuracyMetric(use_reference=False, judge_model=judge_model) + +result = evaluator.run( + metric=metric, + dataset=[ + { + "user_input": [ + { + "content": "Set a reminder for my dentist appointment tomorrow at 2pm", + "type": "human", + }, + { + "content": "I'll set that reminder for you.", + "type": "ai", + "tool_calls": [ + { + "name": "set_reminder", + "args": { + "title": "Dentist appointment", + "date": "tomorrow", + "time": "2pm", + }, + } + ], + }, + {"content": "Reminder set successfully.", "type": "tool"}, + {"content": "Your reminder has been set.", "type": "ai"}, + ], + } + ], +) +print(result.aggregate_scores) +``` + + + + +```python +from nemo_evaluator_sdk import RunConfig, Model +from nemo_evaluator_sdk.metrics.ragas import AgentGoalAccuracyMetric + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="meta/llama-3.1-70b-instruct", + api_key_secret="nvidia-api-key", +) +metric = AgentGoalAccuracyMetric(use_reference=False, judge_model=judge_model) + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "user_input": [ + { + "content": "Set a reminder for my dentist appointment tomorrow at 2pm", + "type": "human", + }, + { + "content": "I'll set that reminder for you.", + "type": "ai", + "tool_calls": [ + { + "name": "set_reminder", + "args": { + "title": "Dentist appointment", + "date": "tomorrow", + "time": "2pm", + }, + } + ], + }, + {"content": "Reminder set successfully.", "type": "tool"}, + {"content": "Your reminder has been set.", "type": "ai"}, + ], + } + ], + config=RunConfig(parallelism=4), +) +job.wait_until_done() +result = job.get_result() +print(result.aggregate_scores) +``` + + + + +--- + +### Answer Accuracy + +Evaluates the factual correctness of an agent's answer by comparing it against a reference answer. Two LLM judges independently rate the agreement, and scores are averaged. Scores range from 0 (incorrect) to 0.5 (partial match) to 1 (exact match). + +#### Data Format + +```json +{ + "user_input": "What is the capital of France?", + "response": "The capital of France is Paris.", + "reference": "Paris" +} +``` + + + + + +```python +from nemo_evaluator_sdk import Model +from nemo_evaluator_sdk.metrics.ragas import AnswerAccuracyMetric + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="meta/llama-3.1-70b-instruct", + api_key_secret="nvidia-api-key", +) +metric = AnswerAccuracyMetric(judge_model=judge_model) + +result = evaluator.run( + metric=metric, + dataset=[ + { + "user_input": "What is the capital of France?", + "response": "The capital of France is Paris.", + "reference": "Paris", + } + ], +) +print(result.aggregate_scores) +``` + + + + +```python +from nemo_evaluator_sdk import RunConfig, Model +from nemo_evaluator_sdk.metrics.ragas import AnswerAccuracyMetric + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="meta/llama-3.1-70b-instruct", + api_key_secret="nvidia-api-key", +) +metric = AnswerAccuracyMetric(judge_model=judge_model) + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "user_input": "What is the capital of France?", + "response": "The capital of France is Paris.", + "reference": "Paris", + } + ], + config=RunConfig(parallelism=4), +) +job.wait_until_done() +result = job.get_result() +print(result.aggregate_scores) +``` + + + + +```python +from nemo_evaluator_sdk import RunConfigOnlineModel, InferenceParams, Model +from nemo_evaluator_sdk.metrics.ragas import AnswerAccuracyMetric + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="meta/llama-3.1-70b-instruct", + api_key_secret="nvidia-api-key", +) +target_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="nvidia/llama-3.3-nemotron-super-49b-v1", + api_key_secret="nvidia-api-key", +) +metric = AnswerAccuracyMetric(judge_model=judge_model) + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "user_input": "What is the capital of France?", + "reference": "Paris", + } + ], + config=RunConfigOnlineModel( + parallelism=4, + inference=InferenceParams(temperature=0.7, max_tokens=1024), + ), + target=target_model, + prompt_template={ + "messages": [ + { + "role": "user", + "content": "{{item.user_input}}", + } + ] + }, +) + +job.wait_until_done() +result = job.get_result() +print(result.aggregate_scores) +``` + + + + +```json +{ + "aggregate_scores": [ + { + "name": "nv_accuracy", + "count": 1, + "mean": 1.0, + "min": 1.0, + "max": 1.0 + } + ], + "row_scores": [ + { + "index": 0, + "scores": { + "nv_accuracy": 1.0 + } + } + ] +} +``` + + + + +--- + +### Trajectory Evaluation + +Evaluates the agent's decision-making process by analyzing the entire sequence of actions (trajectory) taken to accomplish a goal. This **system metric** assesses whether the agent chose appropriate tools in the correct order. + + + +**NAT Format Requirement**: This metric supports the NVIDIA Agent Toolkit format with `intermediate_steps` containing detailed event traces. + +**Current plugin SDK support**: The current plugin SDK does not expose a typed trajectory-evaluation metric class. Use the data-format details below when preparing datasets for environments where the system metric is enabled, but do not use the old generated evaluator job APIs for plugin SDK execution. + + +#### Data Format + +Each data entry must follow the [NeMo Agent Toolkit](https://docs.nvidia.com/nemo/agent-toolkit/latest/index.html) format: + +```json +{ + "question": "What are LLMs", + "generated_answer": "LLMs, or Large Language Models, are a type of artificial intelligence designed to process and generate human-like language. They are trained on vast amounts of text data and can be fine-tuned for specific tasks or guided by prompt engineering.", + "answer": "LLMs stand for Large Language Models, which are a type of machine learning model designed for natural language processing tasks such as language generation.", + "intermediate_steps": [ + { + "payload": { + "event_type": "LLM_END", + "name": "nvidia/llama-3.3-nemotron-super-49b-v1", + "data": { + "input": "\nPrevious conversation history:\n\n\nQuestion: What are LLMs\n", + "output": "Thought: I need to find information about LLMs to answer this question.\n\nAction: wikipedia_search\nAction Input: {'question': 'LLMs'}\n\n" + } + } + }, + { + "payload": { + "event_type": "TOOL_END", + "name": "wikipedia_search", + "data": { + "input": "{'question': 'LLMs'}", + "output": "\nA large language model (LLM) is a language model trained with self-supervised machine learning..." + } + } + }, + { + "payload": { + "event_type": "LLM_END", + "name": "meta/llama-3.1-70b-instruct", + "data": { + "input": "...", + "output": "Thought: I now know the final answer\n\nFinal Answer: LLMs, or Large Language Models, are a type of artificial intelligence..." + } + } + } + ] +} +``` + +#### Parameters + +| Parameter | Required | Type | Description | +|-----------|----------|------|-------------| +| `judge` | required | object | Judge LLM configuration | +| `trajectory_used_tools` | required | string | Comma-separated list of tools available to the agent. Example: `"wikipedia_search,current_datetime,code_generation"` | +| `trajectory_custom_tools` | optional | object | JSON mapping custom tool names to descriptions for non-default tools | + +--- + +## Judge Configuration + +Most agentic metrics require a judge LLM. Configure the judge model in the metric definition: + +```python +from nemo_evaluator_sdk import Model + + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="meta/llama-3.1-70b-instruct", + api_key_secret="nvidia-api-key", +) +``` + + + +**Recommended model size**: Use a 70B+ parameter model as the judge for reliable results. Smaller models may fail to follow the required output schema, causing parsing errors. + + +### Using Reasoning Models + +For models that support extended reasoning, such as `nvidia/llama-3.3-nemotron-super-49b-v1`, add system prompt and reasoning parameters to online model generation: + +```python +from nemo_evaluator_sdk import RunConfigOnlineModel, InferenceParams, Model, ReasoningParams + + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="nvidia/llama-3.3-nemotron-super-49b-v1", + api_key_secret="nvidia-api-key", +) + +config = RunConfigOnlineModel( + system_prompt="detailed thinking on", + reasoning=ReasoningParams(end_token=""), + inference=InferenceParams(max_tokens=1024), +) +``` + +### Managing Secrets for Authenticated Endpoints + +If your judge endpoint requires an API key, store it as a secret: + +```python +from nemo_evaluator_sdk import Model + +client.secrets.create(name="nvidia-api-key", value="nvapi-YOUR_API_KEY_HERE") + +judge_model = Model( + url="https://integrate.api.nvidia.com/v1/chat/completions", + name="meta/llama-3.1-70b-instruct", + api_key_secret="nvidia-api-key", +) +``` + +For more details on secret management, refer to [Managing Secrets](/documentation/get-started/core-concepts/manage-secrets). + +For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). + +--- + +## Job Management + +After submitting a durable remote job with `evaluator.submit(metric=metric, dataset=dataset)`, use the returned job resource to monitor execution and retrieve results: + +```python +job.wait_until_done() +result = job.get_result() +print(result.aggregate_scores) +``` + +Navigate to Metrics Job Management for more job lifecycle details. + +--- + +## Dataset Notes + +The current plugin SDK examples use inline dataset rows through `dataset=[...]`. Keep each row shaped for the selected metric, including fields such as `user_input`, `response`, `reference`, `reference_topics`, `reference_tool_calls`, or OpenAI-style `tool_calls` as required. + +Use `RunConfig(limit_samples=...)` when you want to test a small slice of a larger inline dataset before submitting the full request. + +--- + +## Important Notes + +1. **Execution choice**: Use `run` for local in-process evaluation and `submit` for durable remote jobs with `wait_until_done()` and `get_result()`. + +2. **Column Names**: RAGAS metrics use specific column names: + - `user_input` (not `question`) + - `response` (not `answer`) + - `reference` (not `ground_truth`) + +3. **Judge Model Quality**: For metrics requiring a judge, evaluation quality depends on the judge model's ability to follow instructions. Larger models (70B+) produce more consistent results. + +4. **RAGAS Dependency**: These metrics are powered by RAGAS and may have version-specific behavior. + +5. **NaN troubleshooting for judge-based metrics**: If you see `nan_count > 0` with `mean = null`, check judge model authentication first (API key secret, endpoint access, and model permissions). See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication) for `api_key_secret` behavior. Some RAGAS metrics are known to convert auth failures into `NaN` scores instead of raising a hard error. + + + +- [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration) - Use agents (generic or NAT) as targets in online evaluation jobs +- Agentic Benchmarks - BFCL benchmark for tool-calling evaluation +- [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge) - Custom judge-based evaluation +- Evaluation Results - Understanding results +- [RAG Metrics](/documentation/evaluate-models/metrics/rag-metrics) - RAGAS metrics for RAG pipelines + + diff --git a/docs/evaluator/metrics/index.md b/docs/evaluator/metrics/index.mdx similarity index 61% rename from docs/evaluator/metrics/index.md rename to docs/evaluator/metrics/index.mdx index 85b5ad529e..f2110f754e 100644 --- a/docs/evaluator/metrics/index.md +++ b/docs/evaluator/metrics/index.mdx @@ -1,5 +1,8 @@ +--- +title: "Evaluation Metrics" +description: "" +--- -# Evaluation Metrics Metrics define how to score the outputs of your models, agents, or pipelines. @@ -11,7 +14,11 @@ A metric is a scoring definition that evaluates model or agent outputs. In the E - **Outputs**: Row-level scores and aggregate statistics. - **Execution**: Metric objects run with `dataset`, optional runtime configuration, and an optional model or agent target. -!!! note "Terminology on this page:" + + +Terminology on this page: + + - **Metric definition**: The reusable scoring configuration. - **Metric type**: The metric family (for example exact-match, BLEU, LLM-as-a-judge). - **Metric score**: The numeric or rubric output produced at evaluation time. @@ -34,7 +41,7 @@ A metric is a scoring definition that evaluates model or agent outputs. In the E Minimal sync evaluation with a built-in metric: -{% raw %} + ```python import os @@ -60,7 +67,7 @@ result = evaluator.run( print(result.aggregate_scores) ``` -{% endraw %} + ## Execution Modes @@ -80,15 +87,15 @@ Online evaluation jobs can target either a **model** (an OpenAI-compatible chat | **Model** | Standalone LLM endpoints using a standard chat completions API. | | **Agent** | Agentic systems, NeMo Agent Toolkit workflows, or custom HTTP endpoints with non-standard response formats. | -See [Model Configuration](model-configuration.md) and [Agent Configuration](agent-configuration.md) for setup details. +See [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) and [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration) for setup details. ## Built-in vs. Custom Metrics -- **Built-in metrics**: Ready-to-use metrics provided by {{platform_name}} (for example `exact-match`, `bleu`, `rouge`). +- **Built-in metrics**: Ready-to-use metrics provided by NeMo Platform (for example `exact-match`, `bleu`, `rouge`). - **Custom metrics**: Metrics you define for domain-specific evaluation needs. -To configure inline metric objects, see [Manage Metrics](manage-metrics.md). -For custom metric creation guides, start with [Similarity Metrics](similarity.md), [LLM-as-a-Judge](llm-as-a-judge.md), or [Bring Your Own Metric](remote.md). +To configure inline metric objects, see [Manage Metrics](/documentation/evaluate-models/metrics/manage-metrics). +For custom metric creation guides, start with [Similarity Metrics](/documentation/evaluate-models/metrics/similarity-metrics), [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge), or [Bring Your Own Metric](/documentation/evaluate-models/metrics/bring-your-own-metric). ## Datasets @@ -97,7 +104,7 @@ Evaluation jobs need dataset input. You can provide data in two ways: | Dataset Source | Description | Best For | |------|-------------|----------| | **DatasetRows** | Inline rows sent directly in the request | Quick testing and live evaluation | -| **FilesetRef** | Reference to a persisted [fileset](../../get-started/concepts/manage-files.md) (`workspace/fileset-name`) | Production jobs and reusable datasets | +| **FilesetRef** | Reference to a persisted [fileset](/documentation/get-started/core-concepts/manage-files) (`workspace/fileset-name`) | Production jobs and reusable datasets | Example of providing a `FilesetRef` to reference specific files or globs: @@ -122,57 +129,52 @@ dataset = "my-workspace/my-dataset#subdir/path/*.jsonl" Use the metric-type pages below to create and configure custom metrics. -
- -- **[LLM-as-a-Judge](llm-as-a-judge.md)** - - --- - - Use another LLM to evaluate outputs with flexible scoring criteria. Define custom rubrics or numerical ranges. - - custom-scoring rubrics - -- **[Agentic Metrics](agentic.md)** + - --- + - Evaluate agent workflows including tool calling accuracy, goal completion, and topic adherence. +Use another LLM to evaluate outputs with flexible scoring criteria. Define custom rubrics or numerical ranges. - RAGAS tool-calling +custom-scoring rubrics -- **[RAG Metrics](rag.md)** + + - --- +Evaluate agent workflows including tool calling accuracy, goal completion, and topic adherence. - Evaluate RAG pipelines for retrieval quality and answer generation using RAGAS metrics. +RAGAS tool-calling - faithfulness relevancy + + -- **[Similarity Metrics](similarity.md)** +Evaluate RAG pipelines for retrieval quality and answer generation using RAGAS metrics. - --- +faithfulness relevancy - Create metrics for text similarity, exact matching, and standard NLP evaluations using Jinja2 templating. + + - F1 ROUGE BLEU +Create metrics for text similarity, exact matching, and standard NLP evaluations using Jinja2 templating. -- **[Bring Your Own Metric](remote.md)** +F1 ROUGE BLEU - --- + + - Integrate custom evaluation endpoints for domain-specific scoring. +Integrate custom evaluation endpoints for domain-specific scoring. - remote custom +remote custom -- **[Agent Configuration](agent-configuration.md)** + + - --- +Configure agent endpoints (generic or NeMo Agent Toolkit) as targets for online evaluation jobs. - Configure agent endpoints (generic or NeMo Agent Toolkit) as targets for online evaluation jobs. +agent NAT - agent NAT + -
+ ## Understanding Scores @@ -185,4 +187,4 @@ Scores are the metric outputs produced during evaluation: ## Manage Metric Definitions -Create inline metric objects that can be reused from Python helpers or modules. See [Manage Metrics](manage-metrics.md) for SDK patterns. +Create inline metric objects that can be reused from Python helpers or modules. See [Manage Metrics](/documentation/evaluate-models/metrics/manage-metrics) for SDK patterns. diff --git a/docs/evaluator/metrics/job-management.md b/docs/evaluator/metrics/job-management.mdx similarity index 82% rename from docs/evaluator/metrics/job-management.md rename to docs/evaluator/metrics/job-management.mdx index bf263b242b..65fa3fa576 100644 --- a/docs/evaluator/metrics/job-management.md +++ b/docs/evaluator/metrics/job-management.mdx @@ -1,14 +1,19 @@ +--- +title: "Metric Job Management" +description: "" +--- -# Metric Job Management -!!! note - **Performance Tuning**: You can improve evaluation speed by passing `RunConfig(parallelism=...)` in `config=...`. Adjust parallelism based on your model capacity and rate limits. + +**Performance Tuning**: You can improve evaluation speed by passing `RunConfig(parallelism=...)` in `config=...`. Adjust parallelism based on your model capacity and rate limits. + + ## Submit a Job Submit a metric evaluation with the Evaluator plugin SDK. The `submit` method returns an `EvaluatorJobResource` that you can use to monitor the job and download results. -{% raw %} + ```python import os @@ -38,7 +43,7 @@ print("Submitted job:", job.name) job.wait_until_done() result = job.get_result() ``` -{% endraw %} + ## Monitor Job @@ -54,7 +59,7 @@ if not job.check_if_complete(): `wait_until_done()` raises if the job reaches a terminal failure status or times out. -Refer to [Evaluator Troubleshooting](../../troubleshooting/evaluator.md) for help troubleshooting job failures. +Refer to [Evaluator Troubleshooting](/documentation/reference/troubleshooting/evaluator) for help troubleshooting job failures. ## Reconnect to a Job @@ -79,7 +84,7 @@ for row in result.row_scores: print(row.item, row.metrics) ``` -See [Evaluation Results](results.md) for details on aggregate and row-level result fields. +See [Evaluation Results](/documentation/evaluate-models/metrics/metric-results) for details on aggregate and row-level result fields. ## Download Job Artifacts diff --git a/docs/evaluator/metrics/llm-as-a-judge.md b/docs/evaluator/metrics/llm-as-a-judge.mdx similarity index 85% rename from docs/evaluator/metrics/llm-as-a-judge.md rename to docs/evaluator/metrics/llm-as-a-judge.mdx index 8ed2b74a0a..a22caec4b2 100644 --- a/docs/evaluator/metrics/llm-as-a-judge.md +++ b/docs/evaluator/metrics/llm-as-a-judge.mdx @@ -1,5 +1,8 @@ +--- +title: "Evaluate with LLM-as-a-Judge" +description: "" +--- -# Evaluate with LLM-as-a-Judge Use another LLM to evaluate outputs from your model or dataset with flexible scoring criteria. LLM-as-a-Judge is useful for creative, complex, or domain-specific tasks where traditional metrics do not capture the behavior you care about. @@ -11,7 +14,7 @@ LLM-as-a-Judge evaluation sends each dataset row to a judge LLM and parses the j - **Pre-generated data**: Score existing question-answer pairs or conversations. - **Custom criteria**: Define range scores, rubric scores, prompt templates, and parser behavior. -{{nem_short_name}} supports two execution modes through the Evaluator plugin SDK: +NeMo Evaluator supports two execution modes through the Evaluator plugin SDK: | Mode | Use Case | SDK Call | |------|----------|----------| @@ -45,16 +48,18 @@ evaluator: Evaluator = client.evaluator # this object is an Evaluator resource ## Local Execution -!!! tip - The `model` field accepts both inline model definitions and model references (for example, `"my-workspace/my-model"`). Refer to [Model Configuration](model-configuration.md) for details. + + +The `model` field accepts both inline model definitions and model references (for example, `"my-workspace/my-model"`). Refer to [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) for details. + Live evaluation is designed for rapid iteration when developing and refining your evaluation metrics. Use it to quickly test different judge prompts, scoring criteria, and data formats before committing to a full evaluation job. Results return immediately, making it easy to experiment and debug. ### Basic Example with Range Scores Evaluate responses using numerical range scores, such as a 1-5 scale: -{% raw %} + ```python from nemo_evaluator_sdk import ( InferenceParams, @@ -128,41 +133,41 @@ for score in result.aggregate_scores.scores: for row in result.row_scores: print(row.row_index, row.item, row.metrics) ``` -{% endraw %} + The result includes aggregate scores and row scores. Row scores are useful when you are debugging the prompt or parser because they show how each individual row was scored. -??? "Example Response" - :icon: code-square + - ```python - { - "aggregate_scores": { - "scores": [ - {"name": "helpfulness", "count": 2, "mean": 4.5, "min": 4.0, "max": 5.0}, - {"name": "accuracy", "count": 2, "mean": 4.0, "min": 3.0, "max": 5.0}, - ] - }, - "row_scores": [ - { - "row_index": 0, - "item": { - "input": "What is the capital of France?", - "output": "The capital of France is Paris.", - }, - "metrics": { - "llm-judge": {"scores": [{"name": "helpfulness", "value": 5.0}]} - }, - } - ], - } - ``` +```python +{ + "aggregate_scores": { + "scores": [ + {"name": "helpfulness", "count": 2, "mean": 4.5, "min": 4.0, "max": 5.0}, + {"name": "accuracy", "count": 2, "mean": 4.0, "min": 3.0, "max": 5.0}, + ] + }, + "row_scores": [ + { + "row_index": 0, + "item": { + "input": "What is the capital of France?", + "output": "The capital of France is Paris.", + }, + "metrics": { + "llm-judge": {"scores": [{"name": "helpfulness", "value": 5.0}]} + }, + } + ], +} +``` + ### Example with Rubric Scores Use rubric scores when you want categorical labels with explicit descriptions: -{% raw %} + ```python from nemo_evaluator_sdk import JSONScoreParser, Model, RubricScore, LLMJudgeMetric @@ -256,13 +261,13 @@ result = evaluator.run( print(result.aggregate_scores.model_dump(exclude_none=True)) ``` -{% endraw %} + ### Custom Aggregate Fields By default, aggregate scores include `count`, `mean`, `min`, and `max`. Request additional statistics with `aggregate_fields`: -{% raw %} + ```python result = evaluator.run( @@ -282,7 +287,7 @@ for score in result.aggregate_scores.scores: print(f" p50: {score.percentiles.p50:.3f}") print(f" p90: {score.percentiles.p90:.3f}") ``` -{% endraw %} + --- @@ -290,7 +295,7 @@ for score in result.aggregate_scores.scores: For production workloads, submit the same metric and dataset as a durable platform job. The SDK returns a job resource that can wait for completion and download the final `EvaluationResult`. -{% raw %} + ```python from nemo_evaluator_sdk import RunConfig, JSONScoreParser, Model, RubricScore, LLMJudgeMetric @@ -350,7 +355,7 @@ result = job.get_result() for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}, count={score.count}") ``` -{% endraw %} + --- @@ -411,9 +416,11 @@ Use rubric scores for categorical evaluations with explicit criteria: } ``` -!!! tip - Rubric scores use [structured outputs](https://docs.nvidia.com/nim/large-language-models/latest/structured-generation.html) by default, which constrains the judge model to output valid JSON. This significantly reduces parsing errors. + + +Rubric scores use [structured outputs](https://docs.nvidia.com/nim/large-language-models/latest/structured-generation.html) by default, which constrains the judge model to output valid JSON. This significantly reduces parsing errors. + ### Score Parsers Configure how scores are extracted from judge responses: @@ -436,20 +443,22 @@ By default, the JSON parser is used for range and rubric scores, with the score "parser": {"type": "regex", "pattern": "SCORE: (\\d+)", "method": "search"} ``` -!!! tip - **Regex method options:** + - - `match` (default): Matches the pattern only at the **beginning** of the text. Use when your prompt instructs the judge to output the score first. - - `search`: Finds the pattern **anywhere** in the text. Uses the first match of the regex found in the judge output. +**Regex method options:** - For example, with `method: "search"` and pattern `SCORE: (\d+)`, the parser can extract the score from: +- `match` (default): Matches the pattern only at the **beginning** of the text. Use when your prompt instructs the judge to output the score first. +- `search`: Finds the pattern **anywhere** in the text. Uses the first match of the regex found in the judge output. - ```text - The response is accurate and well-written. SCORE: 5 - ``` +For example, with `method: "search"` and pattern `SCORE: (\d+)`, the parser can extract the score from: - This would fail with the default `match` method since "SCORE:" is not at the beginning. If multiple matches exist, `search` returns the first occurrence. +```text +The response is accurate and well-written. SCORE: 5 +``` +This would fail with the default `match` method since "SCORE:" is not at the beginning. If multiple matches exist, `search` returns the first occurrence. + + --- ## Custom Judge Prompts @@ -460,21 +469,21 @@ Customize the judge prompt to match your evaluation criteria. Use Jinja2 templat | Variable | Description | |----------|-------------| -| `{%raw%}{{input}}{%endraw%}` | Input field from dataset row | -| `{%raw%}{{output}}{%endraw%}` | Output field from dataset row | -| `{%raw%}{{context}}{%endraw%}`, `{%raw%}{{reference}}{%endraw%}`, `{%raw%}{{messages}}{%endraw%}`, `{%raw%}{{tool_calls}}{%endraw%}`, `{%raw%}{{tools}}{%endraw%}` | Other canonical evaluator fields | +| `{{input}}` | Input field from dataset row | +| `{{output}}` | Output field from dataset row | +| `{{context}}`, `{{reference}}`, `{{messages}}`, `{{tool_calls}}`, `{{tools}}` | Other canonical evaluator fields | | `item.` | Any field from the dataset row | | `sample.output_text` | Model-generated response (when evaluating a model) | -| `scores` | Dictionary of score definitions (typically used in expressions/loops, for example `{% raw %}{{ scores.keys() | join(", ") }}{% endraw %}`) | +| `scores` | Dictionary of score definitions (typically used in expressions/loops, for example `{{ scores.keys() \| join(", ") }}`) | ### Canonical vs Legacy Prompt Variables LLM judge prompt variables define the fields required from the evaluation context: -{% raw %} + - Prefer canonical evaluator variables such as `{{input}}`, `{{output}}`, `{{context}}`, and `{{reference}}` for reusable metrics. - Raw dataset variables such as `{{item.question}}`, `{{item.response}}`, `{{question}}`, or `{{sample.output_text}}` continue to work for backward compatibility. -{% endraw %} + When your dataset uses different field names, keep the metric prompt stable and map dataset columns at job or benchmark submission time with `field_mapping`: @@ -505,14 +514,14 @@ With the mapping above, a dataset row like this: renders the prompt template variables as: -{% raw %} + - `{{input}}` -> `question` -> `"What is the capital of France?"` - `{{output}}` -> `response` -> `"Paris"` Custom prompt variables are also allowed. For example, `{{input}} {{output}} {{custom_value}}` produces a required schema with all three fields, and `field_mapping.custom.custom_value` can bind that prompt variable to a dataset column when needed. When no `field_mapping` is provided, prompt variable names are matched directly against dataset columns. That means a prompt using `{{question}}` and `{{response}}` expects dataset rows with `question` and `response` fields unless you remap them explicitly. -{% endraw %} + If a prompt field should be available when present but not required in every row, add it to `optional_fields` on the metric. This is useful for prompts that can use `reference` when available but should still validate against datasets that only provide `input` and `output`. @@ -537,7 +546,7 @@ metric = { ### Schema-Aware Validation -{{nem_short_name}} derives the required prompt fields directly from the prompt variables used by the metric and validates them against dataset metadata during benchmark and job creation. +NeMo Evaluator derives the required prompt fields directly from the prompt variables used by the metric and validates them against dataset metadata during benchmark and job creation. - Add fileset metadata `dataset.schema` for a default row schema. - Add `dataset.schemas_by_path` when different files in the same fileset have different row shapes. @@ -551,7 +560,7 @@ Benchmark-level `field_mapping` is shared by every metric in that benchmark. If ### Example: Custom Judge Template ```python -{% raw %} + JUDGE_TEMPLATE = """You are an expert evaluator assessing AI assistant responses. Evaluate the response on these criteria: @@ -591,7 +600,7 @@ metric = { ] } } -{% endraw %} + ``` --- @@ -600,7 +609,7 @@ metric = { If your judge model endpoint requires an API key, store it as a secret. The secret is automatically resolved from the same workspace as your evaluation. -For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](model-configuration.md#model-api-authentication). +For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). ### Create a Secret @@ -631,7 +640,7 @@ metric = { Control judge model behavior with inference parameters: ```python -{% raw %} + "prompt_template": { "messages": [...], "temperature": 0.1, # Lower for more consistent scoring @@ -639,10 +648,14 @@ Control judge model behavior with inference parameters: "timeout": 30, # Request timeout in seconds "stop": ["<{{ end_of_text }}>"] # Stop sequences } -{% endraw %} + ``` -!!! note "The default value for `max_tokens` for judge models is set to `1024`. It is highly recommended to set an appropriate value for your judge model based on the expected outputs (for example, `structured_output` is used by default to format model output, ensure your `max_tokens` is set to accommodate the full JSON output). Incomplete JSON outputs will cause parsing errors and result in NaN score values." + + +The default value for `max_tokens` for judge models is set to `1024`. It is highly recommended to set an appropriate value for your judge model based on the expected outputs (for example, `structured_output` is used by default to format model output, ensure your `max_tokens` is set to accommodate the full JSON output). Incomplete JSON outputs will cause parsing errors and result in NaN score values. + + ### Reasoning Model Configuration @@ -679,9 +692,11 @@ metric = { 4. **Live Evaluation Limits**: Live evaluations are limited to 10 rows. Use job-based evaluation for larger datasets. -!!! info + + +- [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) - Inline models vs model references +- Evaluation Results - Understanding and downloading results +- [Agentic Evaluation](/documentation/evaluate-models/metrics/agentic-metrics) - Evaluate agent workflows +- [RAG Evaluation](/documentation/evaluate-models/metrics/rag-metrics) - Evaluate retrieval-augmented generation - - [Model Configuration](model-configuration.md) - Inline models vs model references - - [Evaluation Results](results.md) - Understanding and downloading results - - [Agentic Evaluation](agentic.md) - Evaluate agent workflows - - [RAG Evaluation](rag.md) - Evaluate retrieval-augmented generation + diff --git a/docs/evaluator/metrics/manage-metrics.md b/docs/evaluator/metrics/manage-metrics.mdx similarity index 85% rename from docs/evaluator/metrics/manage-metrics.md rename to docs/evaluator/metrics/manage-metrics.mdx index a0c88e7e88..0784c869c5 100644 --- a/docs/evaluator/metrics/manage-metrics.md +++ b/docs/evaluator/metrics/manage-metrics.mdx @@ -1,5 +1,8 @@ +--- +title: "Manage Metrics" +description: "" +--- -# Manage Metrics Instantiate the metric class you want to run and pass it with `dataset` and optional configuration to `evaluator.run(...)` or `evaluator.submit(...)`. @@ -23,7 +26,7 @@ evaluator: Evaluator = client.evaluator # this object is an Evaluator resource Metric objects are normal Python objects from `nemo_evaluator_sdk.metrics.*`. Keep them close to the evaluation code so the definition, dataset fields, and execution request stay in sync. -{% raw %} + ```python from nemo_evaluator_sdk import ExactMatchMetric @@ -43,7 +46,7 @@ result = evaluator.run( for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}") ``` -{% endraw %} + Use `run` for fast local execution while developing a metric. Use `submit` for durable remote execution through the platform job service. @@ -51,7 +54,7 @@ Use `run` for fast local execution while developing a metric. Use `submit` for d Because metrics are inline objects, reuse is usually just a Python helper function or module-level factory. -{% raw %} + ```python from nemo_evaluator_sdk import F1Metric @@ -65,7 +68,7 @@ def answer_f1_metric() -> F1Metric: metric = answer_f1_metric() ``` -{% endraw %} + ## Choose Metric Classes @@ -88,11 +91,11 @@ from nemo_evaluator_sdk import RunConfig config = RunConfig(parallelism=4, limit_samples=100) ``` -For online evaluations, provide a model or agent target and use the online parameter classes described in [Model Configuration](model-configuration.md) and [Agent Configuration](agent-configuration.md). +For online evaluations, provide a model or agent target and use the online parameter classes described in [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) and [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration). ## Submit a Durable Job -{% raw %} + ```python from nemo_evaluator_sdk import RunConfig, ExactMatchMetric @@ -110,10 +113,10 @@ job = evaluator.submit( job.wait_until_done() result = job.get_result() ``` -{% endraw %} + ## Related Topics -- [Metric Results](results.md) - Work with `EvaluationResult`, aggregate scores, and row scores -- [Manage Metric Jobs](job-management.md) - Submit, monitor, reconnect to, and download job results -- [Similarity Metrics](similarity.md) - Configure exact match, F1, BLEU, ROUGE, and string/number checks +- Metric Results - Work with `EvaluationResult`, aggregate scores, and row scores +- Manage Metric Jobs - Submit, monitor, reconnect to, and download job results +- [Similarity Metrics](/documentation/evaluate-models/metrics/similarity-metrics) - Configure exact match, F1, BLEU, ROUGE, and string/number checks diff --git a/docs/evaluator/metrics/model-configuration.md b/docs/evaluator/metrics/model-configuration.mdx similarity index 95% rename from docs/evaluator/metrics/model-configuration.md rename to docs/evaluator/metrics/model-configuration.mdx index efb1444d1c..9691698531 100644 --- a/docs/evaluator/metrics/model-configuration.md +++ b/docs/evaluator/metrics/model-configuration.mdx @@ -1,5 +1,8 @@ +--- +title: "Model Configuration" +description: "" +--- -# Model Configuration Online evaluations use `Model` objects for model endpoints. A model can be the evaluation target that produces outputs, or it can be part of a judge-style metric such as LLM-as-a-Judge, RAG, or agentic metrics. @@ -65,7 +68,7 @@ client.secrets.create( Use `target=model` when the evaluator should call the model to generate the sample output before scoring. -{% raw %} + ```python from nemo_evaluator_sdk import ( @@ -99,13 +102,13 @@ result = evaluator.run( ) ``` -{% endraw %} + ## Model on a Judge Metric Use a model field on the metric when the metric itself calls an LLM to score existing outputs. -{% raw %} + ```python from nemo_evaluator_sdk import Model, RangeScore, LLMJudgeMetric @@ -152,7 +155,7 @@ result = evaluator.run( ) ``` -{% endraw %} + ## Runtime Parameters @@ -182,5 +185,8 @@ Use plain `RunConfig` for offline evaluations where the dataset already contains The plugin SDK examples on this page use inline `Model` objects. If your deployment resolves platform model entities into model endpoint details, perform that lookup before constructing the `Model`, then pass the resulting inline model to the metric or request. -!!! info - For evaluating agentic systems, use an `Agent` request target instead of a `Model`. See [Agent Configuration](agent-configuration.md). + + +For evaluating agentic systems, use an `Agent` request target instead of a `Model`. See [Agent Configuration](/documentation/evaluate-models/metrics/agent-configuration). + + diff --git a/docs/evaluator/metrics/rag.md b/docs/evaluator/metrics/rag.mdx similarity index 61% rename from docs/evaluator/metrics/rag.md rename to docs/evaluator/metrics/rag.mdx index 1691c911e7..d559118574 100644 --- a/docs/evaluator/metrics/rag.md +++ b/docs/evaluator/metrics/rag.mdx @@ -1,6 +1,9 @@ +--- +title: "RAG Evaluation Metrics" +description: "" +--- -# RAG Evaluation Metrics RAG (Retrieval Augmented Generation) metrics evaluate the quality of RAG pipelines by measuring both retrieval and answer generation performance. These metrics use [RAGAS](https://github.com/explodinggradients/ragas) to assess how well retrieved contexts support generated answers. @@ -13,8 +16,8 @@ RAGAS metrics evaluate RAG components by measuring how well retrieved contexts s 1. **Job's model and prompt_template** are used to generate responses 2. **Generated response** (in `sample["output_text"]`) is automatically used as `response` in RAGAS evaluation 3. **RAG context variables** can be included in the job's `prompt_template`: - - `{%raw%}{{user_input}}{%endraw%}` - User question/input from dataset - - `{%raw%}{{retrieved_contexts}}{%endraw%}` - Retrieved context passages from dataset + - `{{user_input}}` - User question/input from dataset + - `{{retrieved_contexts}}` - Retrieved context passages from dataset RAGAS metrics require: @@ -75,7 +78,7 @@ client.secrets.create( ) ``` -Reference secrets by name in your model configuration. For local `run` versus remote `submit` behavior, see [Model API Authentication](model-configuration.md#model-api-authentication). +Reference secrets by name in your model configuration. For local `run` versus remote `submit` behavior, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). ```python judge_model = Model( @@ -85,8 +88,12 @@ judge_model = Model( ) ``` -!!! tip "RAGAS metrics accept inline model definitions for `judge_model` and, where required, `embeddings_model`." - See [Model Configuration](model-configuration.md) for details. + + +RAGAS metrics accept inline model definitions for `judge_model` and, where required, `embeddings_model`. + + + See [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) for details. --- @@ -113,7 +120,7 @@ judge_model = Model( The metric examples below use these inline values: -For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](model-configuration.md#model-api-authentication). +For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). ```python judge_model = Model( @@ -150,7 +157,7 @@ offline_rows = [ Use online arguments when the evaluator should generate the response first: -{% raw %} + ```python online_dataset = [ @@ -173,7 +180,7 @@ online_config = RunConfigOnlineModel( inference=InferenceParams(temperature=0.2, max_tokens=1024), ) ``` -{% endraw %} + --- @@ -196,50 +203,57 @@ Measures the fraction of relevant content retrieved compared to the total releva } ``` -=== "Local Evaluation" + + + - ```python - metric = ContextRecallMetric(judge_model=judge_model) +```python +metric = ContextRecallMetric(judge_model=judge_model) - result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` -=== "Remote Job" + + - ```python - metric = ContextRecallMetric(judge_model=judge_model) +```python +metric = ContextRecallMetric(judge_model=judge_model) - job = evaluator.submit(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) - job.wait_until_done() - result = job.get_result() - ``` +job = evaluator.submit(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job.wait_until_done() +result = job.get_result() +``` -=== "Result" + + - ```json +```json +{ + "scores": [ { - "scores": [ - { - "count": 1, - "histogram": {}, - "name": "context_recall", - "nan_count": 0, - "max": 1.0, - "mean": 1.0, - "min": 1.0, - "percentiles": {}, - "score_type": "range", - "std_dev": 0.0, - "sum": 1.0, - "variance": 0.0 - } - ] + "count": 1, + "histogram": {}, + "name": "context_recall", + "nan_count": 0, + "max": 1.0, + "mean": 1.0, + "min": 1.0, + "percentiles": {}, + "score_type": "range", + "std_dev": 0.0, + "sum": 1.0, + "variance": 0.0 } - ``` + ] +} +``` + + + --- ## Context Precision @@ -261,45 +275,52 @@ Measures the proportion of relevant chunks in the retrieved contexts (precision@ } ``` -=== "Local Evaluation" + - ```python - metric = ContextPrecisionMetric(judge_model=judge_model) + - result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +```python +metric = ContextPrecisionMetric(judge_model=judge_model) - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` +result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` -=== "Remote Job" + + - ```python - metric = ContextPrecisionMetric(judge_model=judge_model) +```python +metric = ContextPrecisionMetric(judge_model=judge_model) - job = evaluator.submit(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) - job.wait_until_done() - result = job.get_result() - ``` +job = evaluator.submit(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +job.wait_until_done() +result = job.get_result() +``` -=== "Result" + + - ```json +```json +{ + "scores": [ { - "scores": [ - { - "count": 1, - "histogram": {}, - "name": "context_precision", - "nan_count": 0, - "max": 1.0, - "mean": 1.0, - "min": 1.0 - } - ] + "count": 1, + "histogram": {}, + "name": "context_precision", + "nan_count": 0, + "max": 1.0, + "mean": 1.0, + "min": 1.0 } - ``` + ] +} +``` + + + --- ## Context Relevance @@ -320,45 +341,51 @@ Measures how relevant the retrieved contexts are to the user input. } ``` -=== "Local Evaluation" - - ```python - metric = ContextRelevanceMetric(judge_model=judge_model) - - result = evaluator.run( - metric=metric, - dataset=[ - { - "user_input": "What is the capital of France?", - "retrieved_contexts": ["Paris is the capital and largest city of France."], - } - ], - config=RunConfig(parallelism=8), - ) - - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` - -=== "Remote Job" - - ```python - metric = ContextRelevanceMetric(judge_model=judge_model) - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "user_input": "What is the capital of France?", - "retrieved_contexts": ["Paris is the capital and largest city of France."], - } - ], - config=RunConfig(parallelism=8), - ) - job.wait_until_done() - result = job.get_result() - ``` + + + + +```python +metric = ContextRelevanceMetric(judge_model=judge_model) + +result = evaluator.run( + metric=metric, + dataset=[ + { + "user_input": "What is the capital of France?", + "retrieved_contexts": ["Paris is the capital and largest city of France."], + } + ], + config=RunConfig(parallelism=8), +) + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` + + + + +```python +metric = ContextRelevanceMetric(judge_model=judge_model) + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "user_input": "What is the capital of France?", + "retrieved_contexts": ["Paris is the capital and largest city of France."], + } + ], + config=RunConfig(parallelism=8), +) +job.wait_until_done() +result = job.get_result() +``` + + + --- ## Context Entity Recall @@ -379,45 +406,51 @@ Measures how many important entities from the reference are present in the retri } ``` -=== "Local Evaluation" - - ```python - metric = ContextEntityRecallMetric(judge_model=judge_model) - - result = evaluator.run( - metric=metric, - dataset=[ - { - "retrieved_contexts": ["Paris is the capital and largest city of France."], - "reference": "Paris is the capital of France.", - } - ], - config=RunConfig(parallelism=8), - ) - - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` - -=== "Remote Job" - - ```python - metric = ContextEntityRecallMetric(judge_model=judge_model) - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "retrieved_contexts": ["Paris is the capital and largest city of France."], - "reference": "Paris is the capital of France.", - } - ], - config=RunConfig(parallelism=8), - ) - job.wait_until_done() - result = job.get_result() - ``` + + + + +```python +metric = ContextEntityRecallMetric(judge_model=judge_model) + +result = evaluator.run( + metric=metric, + dataset=[ + { + "retrieved_contexts": ["Paris is the capital and largest city of France."], + "reference": "Paris is the capital of France.", + } + ], + config=RunConfig(parallelism=8), +) + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` + + + + +```python +metric = ContextEntityRecallMetric(judge_model=judge_model) + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "retrieved_contexts": ["Paris is the capital and largest city of France."], + "reference": "Paris is the capital of France.", + } + ], + config=RunConfig(parallelism=8), +) +job.wait_until_done() +result = job.get_result() +``` + + + --- ## Faithfulness @@ -439,50 +472,57 @@ Measures factual consistency of the response with the retrieved context. } ``` -=== "Local Evaluation" + - ```python - metric = FaithfulnessMetric(judge_model=judge_model) + - result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +```python +metric = FaithfulnessMetric(judge_model=judge_model) - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` +result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) -=== "Online Evaluation" +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` - ```python - metric = FaithfulnessMetric(judge_model=judge_model) + + - result = evaluator.run( - metric=metric, - dataset=online_dataset, - config=online_config, - target=generation_model, - prompt_template=online_prompt_template, - ) +```python +metric = FaithfulnessMetric(judge_model=judge_model) + +result = evaluator.run( + metric=metric, + dataset=online_dataset, + config=online_config, + target=generation_model, + prompt_template=online_prompt_template, +) - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` -=== "Remote Job" + + - ```python - metric = FaithfulnessMetric(judge_model=judge_model) +```python +metric = FaithfulnessMetric(judge_model=judge_model) + +job = evaluator.submit( + metric=metric, + dataset=online_dataset, + config=online_config, + target=generation_model, + prompt_template=online_prompt_template, +) +job.wait_until_done() +result = job.get_result() +``` - job = evaluator.submit( - metric=metric, - dataset=online_dataset, - config=online_config, - target=generation_model, - prompt_template=online_prompt_template, - ) - job.wait_until_done() - result = job.get_result() - ``` + + --- ## Response Groundedness @@ -503,50 +543,57 @@ Evaluates whether the response is grounded in the retrieved context without hall } ``` -=== "Local Evaluation" + - ```python - metric = ResponseGroundednessMetric(judge_model=judge_model) + - result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) +```python +metric = ResponseGroundednessMetric(judge_model=judge_model) - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` +result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) -=== "Online Evaluation" +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` - ```python - metric = ResponseGroundednessMetric(judge_model=judge_model) + + - result = evaluator.run( - metric=metric, - dataset=online_dataset, - config=online_config, - target=generation_model, - prompt_template=online_prompt_template, - ) +```python +metric = ResponseGroundednessMetric(judge_model=judge_model) + +result = evaluator.run( + metric=metric, + dataset=online_dataset, + config=online_config, + target=generation_model, + prompt_template=online_prompt_template, +) - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` -=== "Remote Job" + + - ```python - metric = ResponseGroundednessMetric(judge_model=judge_model) +```python +metric = ResponseGroundednessMetric(judge_model=judge_model) + +job = evaluator.submit( + metric=metric, + dataset=online_dataset, + config=online_config, + target=generation_model, + prompt_template=online_prompt_template, +) +job.wait_until_done() +result = job.get_result() +``` - job = evaluator.submit( - metric=metric, - dataset=online_dataset, - config=online_config, - target=generation_model, - prompt_template=online_prompt_template, - ) - job.wait_until_done() - result = job.get_result() - ``` + + --- ## Noise Sensitivity @@ -570,55 +617,61 @@ Measures robustness when retrieved contexts contain noisy or irrelevant informat } ``` -=== "Local Evaluation" - - ```python - metric = NoiseSensitivityMetric(judge_model=judge_model) - - result = evaluator.run( - metric=metric, - dataset=[ - { - "user_input": "What is the capital of France?", - "retrieved_contexts": [ - "Paris is the capital and largest city of France.", - "Berlin is the capital of Germany.", - ], - "response": "The capital of France is Paris.", - "reference": "Paris is the capital of France.", - } - ], - config=RunConfig(parallelism=8), - ) - - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` - -=== "Remote Job" - - ```python - metric = NoiseSensitivityMetric(judge_model=judge_model) - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "user_input": "What is the capital of France?", - "retrieved_contexts": [ - "Paris is the capital and largest city of France.", - "Berlin is the capital of Germany.", - ], - "response": "The capital of France is Paris.", - "reference": "Paris is the capital of France.", - } - ], - config=RunConfig(parallelism=8), - ) - job.wait_until_done() - result = job.get_result() - ``` + + + + +```python +metric = NoiseSensitivityMetric(judge_model=judge_model) + +result = evaluator.run( + metric=metric, + dataset=[ + { + "user_input": "What is the capital of France?", + "retrieved_contexts": [ + "Paris is the capital and largest city of France.", + "Berlin is the capital of Germany.", + ], + "response": "The capital of France is Paris.", + "reference": "Paris is the capital of France.", + } + ], + config=RunConfig(parallelism=8), +) + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` + + + + +```python +metric = NoiseSensitivityMetric(judge_model=judge_model) + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "user_input": "What is the capital of France?", + "retrieved_contexts": [ + "Paris is the capital and largest city of France.", + "Berlin is the capital of Germany.", + ], + "response": "The capital of France is Paris.", + "reference": "Paris is the capital of France.", + } + ], + config=RunConfig(parallelism=8), +) +job.wait_until_done() +result = job.get_result() +``` + + + --- ## Response Relevancy @@ -646,62 +699,69 @@ Measures how relevant a response is to the user input using generated questions |-----------|------|---------|-------------| | `strictness` | int | `1` | Number of parallel questions generated. NIM supports `1`. | -=== "Local Evaluation" - - ```python - metric = ResponseRelevancyMetric( - judge_model=judge_model, - embeddings_model=embeddings_model, - strictness=1, - ) - - result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) - - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` - -=== "Online Evaluation" - - ```python - metric = ResponseRelevancyMetric( - judge_model=judge_model, - embeddings_model=embeddings_model, - strictness=1, - ) - - result = evaluator.run( - metric=metric, - dataset=online_dataset, - config=online_config, - target=generation_model, - prompt_template=online_prompt_template, - ) - - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` - -=== "Remote Job" - - ```python - metric = ResponseRelevancyMetric( - judge_model=judge_model, - embeddings_model=embeddings_model, - strictness=1, - ) - - job = evaluator.submit( - metric=metric, - dataset=online_dataset, - config=online_config, - target=generation_model, - prompt_template=online_prompt_template, - ) - job.wait_until_done() - result = job.get_result() - ``` + + + + +```python +metric = ResponseRelevancyMetric( + judge_model=judge_model, + embeddings_model=embeddings_model, + strictness=1, +) + +result = evaluator.run(metric=metric, dataset=offline_rows, config=RunConfig(parallelism=8)) + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` + + + + +```python +metric = ResponseRelevancyMetric( + judge_model=judge_model, + embeddings_model=embeddings_model, + strictness=1, +) +result = evaluator.run( + metric=metric, + dataset=online_dataset, + config=online_config, + target=generation_model, + prompt_template=online_prompt_template, +) + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` + + + + +```python +metric = ResponseRelevancyMetric( + judge_model=judge_model, + embeddings_model=embeddings_model, + strictness=1, +) + +job = evaluator.submit( + metric=metric, + dataset=online_dataset, + config=online_config, + target=generation_model, + prompt_template=online_prompt_template, +) +job.wait_until_done() +result = job.get_result() +``` + + + + --- ## Dataset Format @@ -715,9 +775,11 @@ RAGAS metrics use specific column names: | `response` | string | Some metrics | Generated answer. Required for offline response-quality metrics; generated as `sample.output_text` for online model requests. | | `reference` | string | Some metrics | Reference answer or ground truth | -!!! note - Different metrics require different columns. Check the metric documentation for specific requirements. + +Different metrics require different columns. Check the metric documentation for specific requirements. + + ### Example Dataset ```json @@ -798,7 +860,7 @@ client.secrets.create(name="judge-api-key", value="") client.secrets.create(name="embedding-api-key", value="") ``` -Reference secrets by name in your metric configuration. For local `run` versus remote `submit` behavior, see [Model API Authentication](model-configuration.md#model-api-authentication). +Reference secrets by name in your metric configuration. For local `run` versus remote `submit` behavior, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). ```python judge_model = Model( @@ -831,12 +893,16 @@ print(f"Saved artifacts under {artifacts_dir}") |-------|-------|----------| | `judge_model` is required | Missing judge LLM config for metric | Add `judge_model` to metric configuration | | `embeddings_model` is required | Using `response_relevancy` without embeddings | Add `embeddings_model` to metric configuration | -| Job stuck in "pending" | Model endpoint not accessible | Verify endpoint URLs and API key secrets. See [Model API Authentication](model-configuration.md#model-api-authentication) | -| Authentication failed | Invalid or missing API key | Check `api_key_secret` for the execution mode. See [Model API Authentication](model-configuration.md#model-api-authentication) | +| Job stuck in "pending" | Model endpoint not accessible | Verify endpoint URLs and API key secrets. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication) | +| Authentication failed | Invalid or missing API key | Check `api_key_secret` for the execution mode. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication) | | `nan_count > 0` and `mean = null` | Judge/model call failures, such as auth, endpoint, quota, or timeout. Some RAGAS metrics are known to return `NaN` instead of raising on these failures. | Inspect row-level `error`; verify API key, endpoint, and model access | | Low faithfulness scores | Context doesn't support the response | Improve retrieval or response generation | -!!! warning "If you see `nan_count > 0` with `mean = null`, first validate judge model authentication." + + +If you see `nan_count > 0` with `mean = null`, first validate judge model authentication. + + For some RAGAS metrics, auth failures can be converted to `NaN` scores instead of surfacing as a hard error. ### Tips for Better Results @@ -851,7 +917,7 @@ print(f"Saved artifacts under {artifacts_dir}") ## Important Notes -1. **Secret Management**: API keys should be referenced through `api_key_secret`, with different local `run` and remote `submit` behavior. See [Model API Authentication](model-configuration.md#model-api-authentication). Never pass API keys directly in the request. +1. **Secret Management**: API keys should be referenced through `api_key_secret`, with different local `run` and remote `submit` behavior. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). Never pass API keys directly in the request. 2. **Column Names**: RAGAS metrics use specific column names: - `user_input` (not `question`) - `response` (not `answer`) @@ -865,7 +931,9 @@ print(f"Saved artifacts under {artifacts_dir}") 2. **Dataset Format**: RAGAS metrics use specific column names (`user_input`, `retrieved_contexts`, `response`, `reference`). Ensure your data matches this structure. -!!! info + + +- [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge) - Custom judge-based evaluation +- [Agentic Metrics](/documentation/evaluate-models/metrics/agentic-metrics) - Evaluate agent workflows - - [LLM-as-a-Judge](llm-as-a-judge.md) - Custom judge-based evaluation - - [Agentic Metrics](agentic.md) - Evaluate agent workflows + diff --git a/docs/evaluator/metrics/remote.md b/docs/evaluator/metrics/remote.mdx similarity index 74% rename from docs/evaluator/metrics/remote.md rename to docs/evaluator/metrics/remote.mdx index bac8007710..8886217f4d 100644 --- a/docs/evaluator/metrics/remote.md +++ b/docs/evaluator/metrics/remote.mdx @@ -1,7 +1,10 @@ +--- +title: "Bring Your Own Metric" +description: "" +--- -# Bring Your Own Metric -{{platform_name}} offers [built-in metrics](index.md) that can be configured to evaluate on your custom data. Remote metrics let you bring your own metric logic into the {{platform_name}} evaluation workflow by serving that logic from a REST API. +NeMo Platform offers [built-in metrics](/documentation/evaluate-models/metrics) that can be configured to evaluate on your custom data. Remote metrics let you bring your own metric logic into the NeMo Platform evaluation workflow by serving that logic from a REST API. A remote metric gives you control over the evaluation logic, request payload, and reported scores while the Evaluator plugin SDK handles dataset iteration, result aggregation, retries, and job execution. @@ -14,7 +17,7 @@ Remote metrics support two types: | **Generic Remote** (`remote`) | Custom endpoints with configurable request body and score extraction | User-defined Jinja template | | **NeMo Agent Toolkit Remote** (`nemo-agent-toolkit-remote`) | NeMo Agent Toolkit evaluator endpoints | Fixed: `{evaluator_name, item}` | -{{nem_short_name}} supports two execution modes through the Evaluator plugin SDK: +NeMo Evaluator supports two execution modes through the Evaluator plugin SDK: | Mode | Use Case | SDK Call | |------|----------|----------| @@ -58,7 +61,7 @@ Local execution provides immediate results for rapid iteration when developing a Use a generic remote metric when you need full control over the request payload and score extraction: -{% raw %} + ```python from nemo_evaluator_sdk import JSONScoreParser, RemoteScore, RemoteMetric @@ -93,10 +96,10 @@ result = evaluator.run( for score in result.aggregate_scores.scores: print(f"{score.name}: mean={score.mean}, count={score.count}") ``` -{% endraw %} + **Key configuration:** -- `body`: Jinja template for the request payload. Use `{% raw %}{{ item. }}{% endraw %}` to access dataset columns. +- `body`: Jinja template for the request payload. Use `{{ item. }}` to access dataset columns. - `scores`: List of score definitions with a `parser` object containing [JSONPath](https://datatracker.ietf.org/doc/html/rfc9535) expression for extracting values from the response. ### NeMo Agent Toolkit Remote Metric @@ -144,94 +147,96 @@ The NAT metric automatically: For production workloads, submit the same metric and dataset as a durable platform job. The returned job resource can wait for completion and download the final `EvaluationResult`. - - -=== "Generic Remote Metric" - - {% raw %} - ```python - from nemo_evaluator_sdk import RunConfig, JSONScoreParser, RemoteScore, RemoteMetric - - metric = RemoteMetric( - url="https://my-evaluation-server.test/evaluate", - body={ - "reference": "{{item.reference}}", - "response": "{{item.output}}", - }, - scores=[ - RemoteScore( - name="accuracy", - parser=JSONScoreParser(json_path="$.result.accuracy"), - minimum=0.0, - maximum=1.0, - ) - ], - timeout_seconds=30.0, - max_retries=3, - ) - - - job = evaluator.submit( - metric=metric, - dataset=[ - {"reference": "Paris", "output": "Paris"}, - {"reference": "2", "output": "2"}, - ], - config=RunConfig(parallelism=8), - ) - print("Submitted job:", job.name) - - job.wait_until_done() - result = job.get_result() - - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}, count={score.count}") - ``` - {% endraw %} - -=== "NAT Remote Metric" - - ```python - from nemo_evaluator_sdk import RunConfig, NemoAgentToolkitRemoteMetric - - metric = NemoAgentToolkitRemoteMetric( - url="http://localhost:8001/evaluate_item", - evaluator_name="similarity_eval", - timeout_seconds=30.0, - max_retries=3, - ) - - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "id": "item_1", - "input_obj": "What is the capital of France?", - "expected_output_obj": "The capital of France is Paris.", - "output_obj": "Paris is the capital.", - "trajectory": [], - "expected_trajectory": [], - "full_dataset_entry": {}, - } - ], - config=RunConfig(parallelism=4), - ) - job.wait_until_done() - result = job.get_result() - ``` - - + + + + + +```python +from nemo_evaluator_sdk import RunConfig, JSONScoreParser, RemoteScore, RemoteMetric + +metric = RemoteMetric( + url="https://my-evaluation-server.test/evaluate", + body={ + "reference": "{{item.reference}}", + "response": "{{item.output}}", + }, + scores=[ + RemoteScore( + name="accuracy", + parser=JSONScoreParser(json_path="$.result.accuracy"), + minimum=0.0, + maximum=1.0, + ) + ], + timeout_seconds=30.0, + max_retries=3, +) + + +job = evaluator.submit( + metric=metric, + dataset=[ + {"reference": "Paris", "output": "Paris"}, + {"reference": "2", "output": "2"}, + ], + config=RunConfig(parallelism=8), +) +print("Submitted job:", job.name) + +job.wait_until_done() +result = job.get_result() + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}, count={score.count}") +``` + + + + +```python +from nemo_evaluator_sdk import RunConfig, NemoAgentToolkitRemoteMetric + +metric = NemoAgentToolkitRemoteMetric( + url="http://localhost:8001/evaluate_item", + evaluator_name="similarity_eval", + timeout_seconds=30.0, + max_retries=3, +) + + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "id": "item_1", + "input_obj": "What is the capital of France?", + "expected_output_obj": "The capital of France is Paris.", + "output_obj": "Paris is the capital.", + "trajectory": [], + "expected_trajectory": [], + "full_dataset_entry": {}, + } + ], + config=RunConfig(parallelism=4), +) +job.wait_until_done() +result = job.get_result() +``` + + + + --- ## Using API Key Secrets If your remote endpoint requires authentication, store the API key as a platform secret and reference it from your metric: -For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](model-configuration.md#model-api-authentication). +For local `run` versus remote `submit` behavior of `api_key_secret`, see [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication). + -{% raw %} ```python from nemo_evaluator_sdk import JSONScoreParser, RemoteScore, SecretRef, RemoteMetric @@ -244,7 +249,7 @@ metric = RemoteMetric( result = evaluator.run(metric=metric, dataset=[{"input": "test"}]) ``` -{% endraw %} + The API key is sent in the `Authorization: Bearer ` header. For local execution, the SDK resolves the key according to the local `api_key_secret` behavior. For durable remote jobs, the job runtime receives the secret securely. @@ -330,7 +335,7 @@ And must return: | `body` | dict | (Generic only) Jinja template for request payload | | `scores` | list | (Generic only) List of score configuration objects (refer to Score Configuration section) | | `evaluator_name` | string | (NAT only) Name of the NAT evaluator | -| `api_key_secret` | `SecretRef` | Optional API key reference. See [Model API Authentication](model-configuration.md#model-api-authentication) | +| `api_key_secret` | `SecretRef` | Optional API key reference. See [Model API Authentication](/documentation/evaluate-models/metrics/model-configuration#model-api-authentication) | | `timeout_seconds` | float | Request timeout (default: 30.0) | | `max_retries` | int | Max retry attempts (default: 3) | @@ -377,7 +382,10 @@ RemoteScore( 3. **Live evaluation limits**: Live evaluations are limited to 10 rows. Use job-based evaluation for larger datasets. -!!! info - - [Evaluation Results](results.md) - Understanding and downloading results - - [LLM-as-a-Judge](llm-as-a-judge.md) - Use an LLM to evaluate outputs - - [Agentic Evaluation](agentic.md) - Evaluate agent workflows + + +- Evaluation Results - Understanding and downloading results +- [LLM-as-a-Judge](/documentation/evaluate-models/metrics/llm-as-a-judge) - Use an LLM to evaluate outputs +- [Agentic Evaluation](/documentation/evaluate-models/metrics/agentic-metrics) - Evaluate agent workflows + + diff --git a/docs/evaluator/metrics/results.md b/docs/evaluator/metrics/results.mdx similarity index 98% rename from docs/evaluator/metrics/results.md rename to docs/evaluator/metrics/results.mdx index 9732170bfc..07a1b333ea 100644 --- a/docs/evaluator/metrics/results.md +++ b/docs/evaluator/metrics/results.mdx @@ -1,5 +1,8 @@ +--- +title: "Metric Results" +description: "" +--- -# Metric Results When a metric job completes, the platform automatically creates a **metric job result** — a persistent, queryable entity that captures the outcome of that run. Each result is created exactly once per completed job and shares the job's name. @@ -12,7 +15,7 @@ An `EvaluationResult` contains: ## Get Results from a Local Run -{% raw %} + ```python import os @@ -36,11 +39,11 @@ result = evaluator.run( ], ) ``` -{% endraw %} + ## Get Results from a Submitted Job -{% raw %} + ```python from nemo_evaluator_sdk import RunConfig, ExactMatchMetric @@ -58,7 +61,7 @@ job = evaluator.submit( job.wait_until_done() ``` -{% endraw %} + ### Get Results in Memory diff --git a/docs/evaluator/metrics/similarity.md b/docs/evaluator/metrics/similarity.md deleted file mode 100644 index ca88c25c46..0000000000 --- a/docs/evaluator/metrics/similarity.md +++ /dev/null @@ -1,601 +0,0 @@ - -# Similarity Metrics - -{{platform_name}} offers [built-in metrics](index.md) that can be configured to evaluate on your custom data. Similarity metrics compare generated or precomputed text against references, labels, or numeric/string expectations. They support Jinja templates so you can map your dataset columns to the values each metric evaluates. - -Template functionality provides maximum flexibility for evaluating your models on proprietary, domain-specific, or novel tasks. You can bring your own datasets, define your own prompts and templates using [Jinja](https://github.com/pallets/jinja), and select the metrics that matter most for your use case. This approach is ideal when: - -- You want to evaluate on tasks, data, or formats not covered by [industry benchmarks](../benchmarks/industry.md) or built-in metrics. -- You need to measure model performance using custom or business-specific criteria. -- You want to experiment with new evaluation methodologies, metrics, or workflows. -- You need to create custom prompts and templates for specific use cases. - -## Setup - -```python -import os - -from nemo_evaluator.sdk import Evaluator -from nemo_platform import NeMoPlatform - -sdk = NeMoPlatform( - base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), - workspace="default", -) -evaluator: Evaluator = sdk.evaluator # this object is an Evaluator resource -``` - -Use `evaluator.run(metric=metric, dataset=dataset)` for a local synchronous evaluation. Use `evaluator.submit(metric=metric, dataset=dataset)` when you need a durable remote job: - -```python -job = evaluator.submit(metric=metric, dataset=dataset) -job.wait_until_done() -result = job.get_result() -``` - -## Template Variables - -All similarity metrics support Jinja templating with these variables: - -- `{%raw%}{{item}}{%endraw%}` - Access dataset columns (e.g., `{%raw%}{{item.question}}{%endraw%}`, `{%raw%}{{item.answer}}{%endraw%}`) -- `{%raw%}{{sample.output_text}}{%endraw%}` - The model's generated output for online runs -- Jinja filters: `lower`, `upper`, `trim`, `replace`, etc. - -Use Jinja filters to normalize text before comparison: - -{% raw %} -```python - -from nemo_evaluator_sdk import ExactMatchMetric -metric = ExactMatchMetric( - reference="{{item.expected | lower | trim}}", - candidate="{{item.output | lower | trim}}", -) -``` -{% endraw %} - -## BLEU Metric - -BLEU (Bilingual Evaluation Understudy) measures the similarity between machine-generated text and reference translations by comparing n-gram overlap. It's commonly used for evaluating machine translation and text generation tasks. - -**Use BLEU when:** -- Evaluating machine translation quality -- Measuring text generation similarity to references -- Comparing multiple reference texts - -**Metric Output:** A score between 0 and 100, where 100 indicates perfect match with references. - -=== "Local Evaluation" - - {% raw %} - ```python - from nemo_evaluator_sdk import BLEUMetric - - metric = BLEUMetric( - references=["{{item.reference_1}}", "{{item.reference_2}}"], - candidate="{{item.model_output}}", - ) - - result = evaluator.run( - metric=metric, - dataset=[ - { - "reference_1": "The cat sits on the mat.", - "reference_2": "A cat is sitting on the mat.", - "model_output": "The cat is on the mat.", - }, - { - "reference_1": "Hello world!", - "reference_2": "Hi world!", - "model_output": "Hello world!", - }, - ], - ) - - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` - {% endraw %} - -=== "Remote Job" - - {% raw %} - ```python - from nemo_evaluator_sdk import BLEUMetric - - metric = BLEUMetric( - references=["{{item.reference_1}}", "{{item.reference_2}}"], - candidate="{{item.model_output}}", - description="BLEU score for translation quality", - ) - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "reference_1": "The cat sits on the mat.", - "reference_2": "A cat is sitting on the mat.", - "model_output": "The cat is on the mat.", - }, - { - "reference_1": "Hello world!", - "reference_2": "Hi world!", - "model_output": "Hello world!", - }, - ], - ) - job.wait_until_done() - result = job.get_result() - ``` - {% endraw %} - -=== "Example Result" - {% raw %} - ```json - { - "scores": [ - { - "name": "sentence", - "count": 2, - "mean": 76.86, - "min": 53.73, - "max": 100.0 - }, - { - "name": "corpus", - "count": 1, - "mean": 53.895 - } - ] - } - ``` - {% endraw %} - -## Exact Match Metric - -Exact Match compares the candidate text with the reference text for perfect equality. This metric returns 1 if the strings match exactly and 0 otherwise. - -**Use Exact Match when:** -- Evaluating classification tasks with discrete labels -- Checking for exact answer correctness -- Validating structured output formats - -**Metric Output:** Binary score (0 or 1). - -=== "Local Evaluation" - - {% raw %} - ```python - from nemo_evaluator_sdk import ExactMatchMetric - - metric = ExactMatchMetric( - reference="{{item.correct_answer | lower | trim}}", - candidate="{{item.model_answer | lower | trim}}", - description="Exact match for question answering", - ) - - result = evaluator.run( - metric=metric, - dataset=[ - {"correct_answer": "Paris", "model_answer": "Paris"}, - {"correct_answer": "London", "model_answer": "london "}, - {"correct_answer": "Berlin", "model_answer": "Munich"}, - ], - ) - - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` - {% endraw %} - -=== "Remote Job" - - {% raw %} - ```python - from nemo_evaluator_sdk import ExactMatchMetric - - metric = ExactMatchMetric( - reference="{{item.correct_answer | lower | trim}}", - candidate="{{item.model_answer | lower | trim}}", - ) - - job = evaluator.submit( - metric=metric, - dataset=[ - {"correct_answer": "Paris", "model_answer": "Paris"}, - {"correct_answer": "London", "model_answer": "london "}, - {"correct_answer": "Berlin", "model_answer": "Munich"}, - ], - ) - job.wait_until_done() - result = job.get_result() - ``` - {% endraw %} - -=== "Example Result" - - ```json - { - "scores": [ - { - "name": "exact-match", - "count": 3, - "mean": 0.667, - "min": 0.0, - "max": 1.0 - } - ] - } - ``` - -## F1 Metric - -F1 measures token-level overlap between candidate and reference text. It balances precision and recall, making it useful when there are multiple acceptable ways to phrase a response. - -**Use F1 when:** -- Evaluating extractive question answering -- Comparing short free-form answers -- Measuring partial matches where exact match is too strict - -**Metric Output:** A score between 0 and 1. - -=== "Local Evaluation" - - {% raw %} - ```python - from nemo_evaluator_sdk import F1Metric - - metric = F1Metric( - reference="{{item.reference}}", - candidate="{{item.answer}}", - ) - - result = evaluator.run( - metric=metric, - dataset=[ - { - "reference": "the capital of France is Paris", - "answer": "Paris is the capital of France", - }, - {"reference": "a red apple", "answer": "red apple"}, - ], - ) - - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` - {% endraw %} - -=== "Remote Job" - - {% raw %} - ```python - from nemo_evaluator_sdk import F1Metric - - metric = F1Metric( - reference="{{item.reference}}", - candidate="{{item.answer}}", - ) - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "reference": "the capital of France is Paris", - "answer": "Paris is the capital of France", - }, - {"reference": "a red apple", "answer": "red apple"}, - ], - ) - job.wait_until_done() - result = job.get_result() - ``` - {% endraw %} - -=== "Example Result" - - ```json - { - "scores": [ - { - "name": "f1", - "count": 2, - "mean": 0.75, - "min": 0.5, - "max": 1.0 - } - ] - } - ``` - -## Number Check Metric - -Number Check performs numerical comparisons and operations on extracted values. Supports equality, inequality, comparison operators, and absolute difference calculations. - -**Use Number Check when:** -- Validating numerical outputs (calculations, counts, scores) -- Checking value ranges or thresholds -- Comparing predicted vs expected numbers - -**Metric Output:** 1 if the condition is true, 0 otherwise. If either value cannot be parsed as a number, the row score is `NaN`. - -### Supported Operations - -- Equality: `"equals"`, `"=="` -- Inequality: `"!="`, `"<>"`, `"not equals"` -- Comparisons: `">"`, `"gt"`, `">="`, `"gte"`, `"<"`, `"lt"`, `"<="`, `"lte"` -- Absolute difference: `"absolute difference"` (requires `epsilon` parameter) - -=== "Local Evaluation" - - {% raw %} - ```python - from nemo_evaluator_sdk import NumberCheckMetric - - metric = NumberCheckMetric( - operation="absolute difference", - epsilon=0.5, - left_template="{{item.expected}}", - right_template="{{item.predicted}}", - description="Check if values match within tolerance", - ) - - result = evaluator.run( - metric=metric, - dataset=[ - {"expected": "100", "predicted": "100"}, - {"expected": "42.5", "predicted": "42.3"}, - {"expected": "99", "predicted": "101"}, - ], - ) - - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` - {% endraw %} - -=== "Remote Job" - - {% raw %} - ```python - from nemo_evaluator_sdk import NumberCheckMetric - - metric = NumberCheckMetric( - operation=">", - left_template="{{item.predicted}}", - right_template="0.5", - description="Score must be greater than 0.5", - ) - - job = evaluator.submit( - metric=metric, - dataset=[ - {"predicted": "1"}, - {"predicted": "0.75"}, - {"predicted": "0.5"}, - {"predicted": "0.1"}, - ], - ) - job.wait_until_done() - result = job.get_result() - ``` - {% endraw %} - -=== "Example Result" - - ```json - { - "scores": [ - { - "name": "number-check", - "count": 3, - "mean": 0.6667, - "min": 0.0, - "max": 1.0 - } - ] - } - ``` - -## ROUGE Metric - -ROUGE (Recall-Oriented Understudy for Gisting Evaluation) measures overlap between generated text and reference text. It is commonly used for summarization and long-form generation quality checks. - -**Use ROUGE when:** -- Evaluating summarization quality -- Measuring overlap with reference passages -- Comparing generated text against longer expected answers - -**Metric Output:** ROUGE-1, ROUGE-2, ROUGE-3, and ROUGE-L F1 scores between 0 and 1. - -=== "Local Evaluation" - - {% raw %} - ```python - from nemo_evaluator_sdk import ROUGEMetric - - metric = ROUGEMetric( - reference="{{item.reference_summary}}", - candidate="{{item.model_summary}}", - ) - - result = evaluator.run( - metric=metric, - dataset=[ - { - "reference_summary": "The cat sat on the mat and looked out the window.", - "model_summary": "A cat sat on a mat near the window.", - }, - { - "reference_summary": "The launch was postponed because of high winds.", - "model_summary": "High winds delayed the launch.", - }, - ], - ) - - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` - {% endraw %} - -=== "Remote Job" - - {% raw %} - ```python - from nemo_evaluator_sdk import ROUGEMetric - - metric = ROUGEMetric( - reference="{{item.reference_summary}}", - candidate="{{item.model_summary}}", - ) - - job = evaluator.submit( - metric=metric, - dataset=[ - { - "reference_summary": "The cat sat on the mat and looked out the window.", - "model_summary": "A cat sat on a mat near the window.", - }, - { - "reference_summary": "The launch was postponed because of high winds.", - "model_summary": "High winds delayed the launch.", - }, - ], - ) - job.wait_until_done() - result = job.get_result() - ``` - {% endraw %} - -=== "Example Result" - - ```json - { - "scores": [ - { - "name": "rouge_1_score", - "count": 2, - "mean": 0.72 - }, - { - "name": "rouge_2_score", - "count": 2, - "mean": 0.43 - }, - { - "name": "rouge_3_score", - "count": 2, - "mean": 0.31 - }, - { - "name": "rouge_L_score", - "count": 2, - "mean": 0.67 - } - ] - } - ``` - -## String Check Metric - -String Check performs various string operations and comparisons. Supports equality, containment, and prefix/suffix checks. - -**Use String Check when:** -- Validating text format or structure -- Checking for keyword presence -- Pattern matching in generated text -- String-based classification - -**Metric Output:** Binary score (1 if condition is true, 0 otherwise). - -### Supported Operations - -- Equality: `"equals"`, `"=="` -- Inequality: `"!="`, `"<>"`, `"not equals"` -- Containment: `"contains"`, `"not contains"` -- Pattern: `"startswith"`, `"endswith"` - -=== "Local Evaluation" - - {% raw %} - ```python - from nemo_evaluator_sdk import StringCheckMetric - - metric = StringCheckMetric( - operation="contains", - left_template="{{item.output | trim}}", - right_template="{{item.must_contain}}", - ) - - result = evaluator.run( - metric=metric, - dataset=[ - {"output": "The answer is: 42", "must_contain": "answer"}, - {"output": "Result: Success", "must_contain": "Success"}, - {"output": "Error occurred", "must_contain": "Success"}, - ], - ) - - for score in result.aggregate_scores.scores: - print(f"{score.name}: mean={score.mean}") - ``` - {% endraw %} - -=== "Remote Job" - - {% raw %} - ```python - from nemo_evaluator_sdk import StringCheckMetric - - metric = StringCheckMetric( - operation="startswith", - left_template="{{item.output}}", - right_template="Answer:", - description="Check if output starts with 'Answer:'", - ) - - job = evaluator.submit( - metric=metric, - dataset=[ - {"output": "Answer: 42"}, - {"output": "Answer: Success"}, - {"output": "Error occurred"}, - ], - ) - job.wait_until_done() - result = job.get_result() - ``` - {% endraw %} - -=== "Example Result" - - ```json - { - "scores": [ - { - "name": "string-check", - "count": 3, - "mean": 0.667, - "min": 0.0, - "max": 1.0 - } - ] - } - ``` - - -## Dataset Format - -The examples on this page use inline dataset rows with `dataset=[...]`. Template fields determine the columns required by each metric: - -- `reference`, `references`, `left_template`, and `right_template` read from `item` fields in the dataset. -- `candidate` reads from an `item` field for offline rows when configured. -- If `candidate` is omitted for BLEU, Exact Match, F1, or ROUGE, the metric uses `sample.output_text`, which is populated during online evaluations. - -Keep field names consistent between the dataset rows and the templates you configure. For example, `{%raw%}{{item.expected}}{%endraw%}` requires each row to include an `expected` field. - -## Related Topics - -- [Metric Overview](index.md) -- [Model Configuration](model-configuration.md) -- [RAG Evaluation Metrics](rag.md) diff --git a/docs/evaluator/metrics/similarity.mdx b/docs/evaluator/metrics/similarity.mdx new file mode 100644 index 0000000000..75138076e1 --- /dev/null +++ b/docs/evaluator/metrics/similarity.mdx @@ -0,0 +1,621 @@ +--- +title: "Similarity Metrics" +description: "" +--- + + +NeMo Platform offers [built-in metrics](/documentation/evaluate-models/metrics) that can be configured to evaluate on your custom data. Similarity metrics compare generated or precomputed text against references, labels, or numeric/string expectations. They support Jinja templates so you can map your dataset columns to the values each metric evaluates. + +Template functionality provides maximum flexibility for evaluating your models on proprietary, domain-specific, or novel tasks. You can bring your own datasets, define your own prompts and templates using [Jinja](https://github.com/pallets/jinja), and select the metrics that matter most for your use case. This approach is ideal when: + +- You want to evaluate on tasks, data, or formats not covered by industry benchmarks or built-in metrics. +- You need to measure model performance using custom or business-specific criteria. +- You want to experiment with new evaluation methodologies, metrics, or workflows. +- You need to create custom prompts and templates for specific use cases. + +## Setup + +```python +import os + +from nemo_evaluator.sdk import Evaluator +from nemo_platform import NeMoPlatform + +sdk = NeMoPlatform( + base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), + workspace="default", +) +evaluator: Evaluator = sdk.evaluator # this object is an Evaluator resource +``` + +Use `evaluator.run(metric=metric, dataset=dataset)` for a local synchronous evaluation. Use `evaluator.submit(metric=metric, dataset=dataset)` when you need a durable remote job: + +```python +job = evaluator.submit(metric=metric, dataset=dataset) +job.wait_until_done() +result = job.get_result() +``` + +## Template Variables + +All similarity metrics support Jinja templating with these variables: + +- `{{item}}` - Access dataset columns (e.g., `{{item.question}}`, `{{item.answer}}`) +- `{{sample.output_text}}` - The model's generated output for online runs +- Jinja filters: `lower`, `upper`, `trim`, `replace`, etc. + +Use Jinja filters to normalize text before comparison: + + +```python + +from nemo_evaluator_sdk import ExactMatchMetric +metric = ExactMatchMetric( + reference="{{item.expected | lower | trim}}", + candidate="{{item.output | lower | trim}}", +) +``` + + +## BLEU Metric + +BLEU (Bilingual Evaluation Understudy) measures the similarity between machine-generated text and reference translations by comparing n-gram overlap. It's commonly used for evaluating machine translation and text generation tasks. + +**Use BLEU when:** +- Evaluating machine translation quality +- Measuring text generation similarity to references +- Comparing multiple reference texts + +**Metric Output:** A score between 0 and 100, where 100 indicates perfect match with references. + + + + + +```python +from nemo_evaluator_sdk import BLEUMetric + +metric = BLEUMetric( + references=["{{item.reference_1}}", "{{item.reference_2}}"], + candidate="{{item.model_output}}", +) + +result = evaluator.run( + metric=metric, + dataset=[ + { + "reference_1": "The cat sits on the mat.", + "reference_2": "A cat is sitting on the mat.", + "model_output": "The cat is on the mat.", + }, + { + "reference_1": "Hello world!", + "reference_2": "Hi world!", + "model_output": "Hello world!", + }, + ], +) + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` + + + + +```python +from nemo_evaluator_sdk import BLEUMetric + +metric = BLEUMetric( + references=["{{item.reference_1}}", "{{item.reference_2}}"], + candidate="{{item.model_output}}", + description="BLEU score for translation quality", +) + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "reference_1": "The cat sits on the mat.", + "reference_2": "A cat is sitting on the mat.", + "model_output": "The cat is on the mat.", + }, + { + "reference_1": "Hello world!", + "reference_2": "Hi world!", + "model_output": "Hello world!", + }, + ], +) +job.wait_until_done() +result = job.get_result() +``` + + + + +```json +{ + "scores": [ + { + "name": "sentence", + "count": 2, + "mean": 76.86, + "min": 53.73, + "max": 100.0 + }, + { + "name": "corpus", + "count": 1, + "mean": 53.895 + } + ] +} +``` + + + + +## Exact Match Metric + +Exact Match compares the candidate text with the reference text for perfect equality. This metric returns 1 if the strings match exactly and 0 otherwise. + +**Use Exact Match when:** +- Evaluating classification tasks with discrete labels +- Checking for exact answer correctness +- Validating structured output formats + +**Metric Output:** Binary score (0 or 1). + + + + + +```python +from nemo_evaluator_sdk import ExactMatchMetric + +metric = ExactMatchMetric( + reference="{{item.correct_answer | lower | trim}}", + candidate="{{item.model_answer | lower | trim}}", + description="Exact match for question answering", +) + +result = evaluator.run( + metric=metric, + dataset=[ + {"correct_answer": "Paris", "model_answer": "Paris"}, + {"correct_answer": "London", "model_answer": "london "}, + {"correct_answer": "Berlin", "model_answer": "Munich"}, + ], +) + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` + + + + +```python +from nemo_evaluator_sdk import ExactMatchMetric + +metric = ExactMatchMetric( + reference="{{item.correct_answer | lower | trim}}", + candidate="{{item.model_answer | lower | trim}}", +) + +job = evaluator.submit( + metric=metric, + dataset=[ + {"correct_answer": "Paris", "model_answer": "Paris"}, + {"correct_answer": "London", "model_answer": "london "}, + {"correct_answer": "Berlin", "model_answer": "Munich"}, + ], +) +job.wait_until_done() +result = job.get_result() +``` + + + + +```json +{ + "scores": [ + { + "name": "exact-match", + "count": 3, + "mean": 0.667, + "min": 0.0, + "max": 1.0 + } + ] +} +``` + + + + +## F1 Metric + +F1 measures token-level overlap between candidate and reference text. It balances precision and recall, making it useful when there are multiple acceptable ways to phrase a response. + +**Use F1 when:** +- Evaluating extractive question answering +- Comparing short free-form answers +- Measuring partial matches where exact match is too strict + +**Metric Output:** A score between 0 and 1. + + + + + +```python +from nemo_evaluator_sdk import F1Metric + +metric = F1Metric( + reference="{{item.reference}}", + candidate="{{item.answer}}", +) + +result = evaluator.run( + metric=metric, + dataset=[ + { + "reference": "the capital of France is Paris", + "answer": "Paris is the capital of France", + }, + {"reference": "a red apple", "answer": "red apple"}, + ], +) + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` + + + + +```python +from nemo_evaluator_sdk import F1Metric + +metric = F1Metric( + reference="{{item.reference}}", + candidate="{{item.answer}}", +) + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "reference": "the capital of France is Paris", + "answer": "Paris is the capital of France", + }, + {"reference": "a red apple", "answer": "red apple"}, + ], +) +job.wait_until_done() +result = job.get_result() +``` + + + + +```json +{ + "scores": [ + { + "name": "f1", + "count": 2, + "mean": 0.75, + "min": 0.5, + "max": 1.0 + } + ] +} +``` + + + + +## Number Check Metric + +Number Check performs numerical comparisons and operations on extracted values. Supports equality, inequality, comparison operators, and absolute difference calculations. + +**Use Number Check when:** +- Validating numerical outputs (calculations, counts, scores) +- Checking value ranges or thresholds +- Comparing predicted vs expected numbers + +**Metric Output:** 1 if the condition is true, 0 otherwise. If either value cannot be parsed as a number, the row score is `NaN`. + +### Supported Operations + +- Equality: `"equals"`, `"=="` +- Inequality: `"!="`, `"<>"`, `"not equals"` +- Comparisons: `">"`, `"gt"`, `">="`, `"gte"`, `"<"`, `"lt"`, `"<="`, `"lte"` +- Absolute difference: `"absolute difference"` (requires `epsilon` parameter) + + + + + +```python +from nemo_evaluator_sdk import NumberCheckMetric + +metric = NumberCheckMetric( + operation="absolute difference", + epsilon=0.5, + left_template="{{item.expected}}", + right_template="{{item.predicted}}", + description="Check if values match within tolerance", +) + +result = evaluator.run( + metric=metric, + dataset=[ + {"expected": "100", "predicted": "100"}, + {"expected": "42.5", "predicted": "42.3"}, + {"expected": "99", "predicted": "101"}, + ], +) + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` + + + + +```python +from nemo_evaluator_sdk import NumberCheckMetric + +metric = NumberCheckMetric( + operation=">", + left_template="{{item.predicted}}", + right_template="0.5", + description="Score must be greater than 0.5", +) + +job = evaluator.submit( + metric=metric, + dataset=[ + {"predicted": "1"}, + {"predicted": "0.75"}, + {"predicted": "0.5"}, + {"predicted": "0.1"}, + ], +) +job.wait_until_done() +result = job.get_result() +``` + + + + +```json +{ + "scores": [ + { + "name": "number-check", + "count": 3, + "mean": 0.6667, + "min": 0.0, + "max": 1.0 + } + ] +} +``` + + + + +## ROUGE Metric + +ROUGE (Recall-Oriented Understudy for Gisting Evaluation) measures overlap between generated text and reference text. It is commonly used for summarization and long-form generation quality checks. + +**Use ROUGE when:** +- Evaluating summarization quality +- Measuring overlap with reference passages +- Comparing generated text against longer expected answers + +**Metric Output:** ROUGE-1, ROUGE-2, ROUGE-3, and ROUGE-L F1 scores between 0 and 1. + + + + + +```python +from nemo_evaluator_sdk import ROUGEMetric + +metric = ROUGEMetric( + reference="{{item.reference_summary}}", + candidate="{{item.model_summary}}", +) + +result = evaluator.run( + metric=metric, + dataset=[ + { + "reference_summary": "The cat sat on the mat and looked out the window.", + "model_summary": "A cat sat on a mat near the window.", + }, + { + "reference_summary": "The launch was postponed because of high winds.", + "model_summary": "High winds delayed the launch.", + }, + ], +) + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` + + + + +```python +from nemo_evaluator_sdk import ROUGEMetric + +metric = ROUGEMetric( + reference="{{item.reference_summary}}", + candidate="{{item.model_summary}}", +) + +job = evaluator.submit( + metric=metric, + dataset=[ + { + "reference_summary": "The cat sat on the mat and looked out the window.", + "model_summary": "A cat sat on a mat near the window.", + }, + { + "reference_summary": "The launch was postponed because of high winds.", + "model_summary": "High winds delayed the launch.", + }, + ], +) +job.wait_until_done() +result = job.get_result() +``` + + + + +```json +{ + "scores": [ + { + "name": "rouge_1_score", + "count": 2, + "mean": 0.72 + }, + { + "name": "rouge_2_score", + "count": 2, + "mean": 0.43 + }, + { + "name": "rouge_3_score", + "count": 2, + "mean": 0.31 + }, + { + "name": "rouge_L_score", + "count": 2, + "mean": 0.67 + } + ] +} +``` + + + + +## String Check Metric + +String Check performs various string operations and comparisons. Supports equality, containment, and prefix/suffix checks. + +**Use String Check when:** +- Validating text format or structure +- Checking for keyword presence +- Pattern matching in generated text +- String-based classification + +**Metric Output:** Binary score (1 if condition is true, 0 otherwise). + +### Supported Operations + +- Equality: `"equals"`, `"=="` +- Inequality: `"!="`, `"<>"`, `"not equals"` +- Containment: `"contains"`, `"not contains"` +- Pattern: `"startswith"`, `"endswith"` + + + + + +```python +from nemo_evaluator_sdk import StringCheckMetric + +metric = StringCheckMetric( + operation="contains", + left_template="{{item.output | trim}}", + right_template="{{item.must_contain}}", +) + +result = evaluator.run( + metric=metric, + dataset=[ + {"output": "The answer is: 42", "must_contain": "answer"}, + {"output": "Result: Success", "must_contain": "Success"}, + {"output": "Error occurred", "must_contain": "Success"}, + ], +) + +for score in result.aggregate_scores.scores: + print(f"{score.name}: mean={score.mean}") +``` + + + + +```python +from nemo_evaluator_sdk import StringCheckMetric + +metric = StringCheckMetric( + operation="startswith", + left_template="{{item.output}}", + right_template="Answer:", + description="Check if output starts with 'Answer:'", +) + +job = evaluator.submit( + metric=metric, + dataset=[ + {"output": "Answer: 42"}, + {"output": "Answer: Success"}, + {"output": "Error occurred"}, + ], +) +job.wait_until_done() +result = job.get_result() +``` + + + + +```json +{ + "scores": [ + { + "name": "string-check", + "count": 3, + "mean": 0.667, + "min": 0.0, + "max": 1.0 + } + ] +} +``` + + + + + +## Dataset Format + +The examples on this page use inline dataset rows with `dataset=[...]`. Template fields determine the columns required by each metric: + +- `reference`, `references`, `left_template`, and `right_template` read from `item` fields in the dataset. +- `candidate` reads from an `item` field for offline rows when configured. +- If `candidate` is omitted for BLEU, Exact Match, F1, or ROUGE, the metric uses `sample.output_text`, which is populated during online evaluations. + +Keep field names consistent between the dataset rows and the templates you configure. For example, `{{item.expected}}` requires each row to include an `expected` field. + +## Related Topics + +- [Metric Overview](/documentation/evaluate-models/metrics) +- [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) +- [RAG Evaluation Metrics](/documentation/evaluate-models/metrics/rag-metrics) diff --git a/docs/evaluator/sdk-resources.md b/docs/evaluator/sdk-resources.mdx similarity index 77% rename from docs/evaluator/sdk-resources.md rename to docs/evaluator/sdk-resources.mdx index 53ce5db735..ca0ee6d708 100644 --- a/docs/evaluator/sdk-resources.md +++ b/docs/evaluator/sdk-resources.mdx @@ -1,15 +1,18 @@ +--- +title: "Evaluator NeMo Platform SDK Resources" +description: "" +--- -# Evaluator {{platform_name}} SDK Resources The `nemo_evaluator_sdk` package provides context-agnostic objects for defining metrics, datasets, evaluation configuration, and result handling. -When you want to execute those evaluations through the {{platform_name}} Evaluator plugin, use the Evaluator SDK resource mounted on the `nemo_platform` SDK. -This page explains the {{platform_name}}-specific objects used to run local plugin jobs, submit durable platform jobs, and retrieve evaluator job results. +When you want to execute those evaluations through the NeMo Platform Evaluator plugin, use the Evaluator SDK resource mounted on the `nemo_platform` SDK. +This page explains the NeMo Platform-specific objects used to run local plugin jobs, submit durable platform jobs, and retrieve evaluator job results. ## Evaluator -The `Evaluator` resource is the sync SDK object for working with the Evaluator plugin on {{platform_name}}. +The `Evaluator` resource is the sync SDK object for working with the Evaluator plugin on NeMo Platform. It is accessed directly from a `NeMoPlatform` instance: ```python @@ -34,7 +37,7 @@ Use `submit` when you want to create a durable remote platform job and manage th | `run()` | Runs one metric locally through the Evaluator plugin job runtime. | `EvaluationResult` | | `submit()` | Submits one metric evaluation as a durable platform job. | `EvaluatorJobResource` | | `plugin_status()` | Returns Evaluator plugin health information from the service. | `dict[str, object]` | -| `get_job_resource(job_name: str, workspace: str \| None = None)` | Returns a resource for an existing Evaluator plugin job. | `EvaluatorJobResource` | +| `get_job_resource(job_name: str, workspace: str \\| None = None)` | Returns a resource for an existing Evaluator plugin job. | `EvaluatorJobResource` | The `dataset` argument accepts inline rows, local dataset paths, local glob paths, and fileset references with optional fragment selectors. Use `config` for evaluator runtime settings, `aggregate_fields` on result-returning calls to shape aggregate scores, and `target` plus `prompt_template` when the evaluator should generate model or agent responses before scoring. @@ -44,10 +47,10 @@ The `dataset` argument accepts inline rows, local dataset paths, local glob path |----------|------|----------|-------------| | `metric` | `Metric` | Yes | Metric configuration used to score each row. | | `dataset` | `PluginDatasetInput` | Yes | Inline rows, local dataset paths, local glob paths, or fileset references with optional fragment selectors. | -| `config` | `RunConfig \| RunConfigOnline \| RunConfigOnlineModel \| None` | No | Runtime settings such as sample limits, parallelism, timeouts, and retry behavior. | -| `aggregate_fields` | `tuple[AggregateFieldName, ...] \| None` | No | Aggregate score fields to include in the returned result. | -| `target` | `Model \| Agent \| None` | No | Model or agent target used when the evaluator should generate outputs before scoring. | -| `prompt_template` | `str \| dict[str, Any] \| None` | No | Prompt template used with `target` for online model or agent evaluation. | +| `config` | `RunConfig \\| RunConfigOnline \\| RunConfigOnlineModel \\| None` | No | Runtime settings such as sample limits, parallelism, timeouts, and retry behavior. | +| `aggregate_fields` | `tuple[AggregateFieldName, ...] \\| None` | No | Aggregate score fields to include in the returned result. | +| `target` | `Model \\| Agent \\| None` | No | Model or agent target used when the evaluator should generate outputs before scoring. | +| `prompt_template` | `str \\| dict[str, Any] \\| None` | No | Prompt template used with `target` for online model or agent evaluation. | ### `submit()` arguments @@ -55,13 +58,13 @@ The `dataset` argument accepts inline rows, local dataset paths, local glob path |----------|------|----------|-------------| | `metric` | `Metric` | Yes | Metric configuration serialized into the durable platform job. | | `dataset` | `PluginDatasetInput` | Yes | Inline rows, local dataset paths, local glob paths, or fileset references with optional fragment selectors. | -| `config` | `RunConfig \| RunConfigOnline \| RunConfigOnlineModel \| None` | No | Runtime settings applied when the submitted job executes. | -| `target` | `Model \| ModelRef \| Agent \| None` | No | Model, model reference, or agent target used when the submitted job should generate outputs before scoring. | -| `prompt_template` | `str \| dict[str, Any] \| None` | No | Prompt template used with `target` for online model or agent evaluation. | +| `config` | `RunConfig \\| RunConfigOnline \\| RunConfigOnlineModel \\| None` | No | Runtime settings applied when the submitted job executes. | +| `target` | `Model \\| ModelRef \\| Agent \\| None` | No | Model, model reference, or agent target used when the submitted job should generate outputs before scoring. | +| `prompt_template` | `str \\| dict[str, Any] \\| None` | No | Prompt template used with `target` for online model or agent evaluation. | ### Run locally -{% raw %} + ```python from nemo_evaluator_sdk import ExactMatchMetric @@ -77,11 +80,11 @@ result = evaluator.run(metric=metric, dataset=dataset) print(result.aggregate_scores) ``` -{% endraw %} + ### Submit a platform job -{% raw %} + ```python from nemo_evaluator_sdk import ExactMatchMetric @@ -99,7 +102,7 @@ result = job.get_result() print(result.aggregate_scores) ``` -{% endraw %} + ## AsyncEvaluator @@ -125,11 +128,11 @@ evaluator: AsyncEvaluator = client.evaluator | `run()` | Runs one metric locally through the Evaluator plugin job runtime. | `EvaluationResult` | | `submit()` | Submits one metric evaluation as a durable platform job. | `AsyncEvaluatorJobResource` | | `plugin_status()` | Returns Evaluator plugin health information from the service. | `dict[str, object]` | -| `get_job_resource(job_name: str, workspace: str \| None = None)` | Returns a resource for an existing Evaluator plugin job. | `AsyncEvaluatorJobResource` | +| `get_job_resource(job_name: str, workspace: str \\| None = None)` | Returns a resource for an existing Evaluator plugin job. | `AsyncEvaluatorJobResource` | `AsyncEvaluator.run()` and `AsyncEvaluator.submit()` accept the same arguments as the sync methods [above](#run-arguments). -{% raw %} + ```python import asyncio @@ -154,7 +157,7 @@ async def main() -> None: asyncio.run(main()) ``` -{% endraw %} + ## EvaluatorJobResource diff --git a/docs/evaluator/tutorials/define-run-custom-python-metrics.md b/docs/evaluator/tutorials/define-run-custom-python-metrics.mdx similarity index 93% rename from docs/evaluator/tutorials/define-run-custom-python-metrics.md rename to docs/evaluator/tutorials/define-run-custom-python-metrics.mdx index 8057fcbdcc..065134acf2 100644 --- a/docs/evaluator/tutorials/define-run-custom-python-metrics.md +++ b/docs/evaluator/tutorials/define-run-custom-python-metrics.mdx @@ -1,5 +1,9 @@ - - +--- +title: "@nemo-nb: hide" +description: "" +--- + + # Define and Run Custom Python Metrics @@ -17,8 +21,11 @@ This tutorial shows how to evaluate a model that solves arithmetic word problems - Submit a durable Evaluator service job with `FilesetRef` and `ModelRef` - Inspect aggregate and row-level metric results -!!! tip - Keep custom metric code dependency-light. Local execution runs your metric in the current Python environment, and remote execution hydrates the serialized metric in the evaluator job runtime. This tutorial uses only the Python standard library. + + +Keep custom metric code dependency-light. Local execution runs your metric in the current Python environment, and remote execution hydrates the serialized metric in the evaluator job runtime. This tutorial uses only the Python standard library. + + ## Prerequisites @@ -37,7 +44,7 @@ import nemo_evaluator_sdk print(nemo_evaluator_sdk.version) ``` -You do not need a running {{platform_name}} instance to define the metric or run it locally. Start {{platform_name}} and configure platform resources before submitting the durable remote job. +You do not need a running NeMo Platform instance to define the metric or run it locally. Start NeMo Platform and configure platform resources before submitting the durable remote job. ## 1. Understand the Metric Contract @@ -275,9 +282,9 @@ With this mapping in place, every row still preserves its original `model_expres ## 6. Prepare Platform Resources -To run the same metric remotely, install and start {{platform_name}} using the [Setup guide](../../get-started/setup.md). You also need: +To run the same metric remotely, install and start NeMo Platform using the [Setup guide](/documentation/get-started). You also need: -- A model entity that can be referenced as `workspace/model-name`. See [Model Configuration](../metrics/model-configuration.md) for model setup details. +- A model entity that can be referenced as `workspace/model-name`. See [Model Configuration](/documentation/evaluate-models/metrics/model-configuration) for model setup details. - A platform secret for the model, if the model requires an API key. ```python @@ -389,7 +396,6 @@ The `nemo_evaluator_sdk` package provides metric and runtime types, such as `Met Pass `CloudpickleMetricBundlePackager()` to `client.evaluator.submit(...)` so the SDK serializes the metric object into the evaluator job spec. The job runtime hydrates the metric bundle before scoring rows. Use `ModelRef` to reference the platform model entity you configured with `NMP_EVAL_MODEL_REF`. -{% raw %} ```python from nemo_evaluator.shared.metric_bundles.cloudpickle import CloudpickleMetricBundlePackager @@ -425,10 +431,12 @@ remote_result = job.get_result() remote_result.print_summary() ``` -{% endraw %} -!!! warning - Cloudpickle metric bundles execute serialized Python code when the job hydrates the metric. Use this path only for metric code that you fully understand and trust. + + +Cloudpickle metric bundles execute serialized Python code when the job hydrates the metric. Use this path only for metric code that you fully understand and trust. + + ## 8. Inspect Results @@ -464,6 +472,6 @@ Use the local `result` object from Step 5 the same way if you want to inspect th ## Next Steps -- Learn about built-in metric types in [Evaluation Metrics](../metrics/index.md). -- Learn how to configure model targets in [Model Configuration](../metrics/model-configuration.md). -- Learn how to inspect result artifacts in [Evaluation Results](../metrics/results.md). +- Learn about built-in metric types in [Evaluation Metrics](/documentation/evaluate-models/metrics). +- Learn how to configure model targets in [Model Configuration](/documentation/evaluate-models/metrics/model-configuration). +- Learn how to inspect result artifacts in Evaluation Results. diff --git a/docs/evaluator/tutorials/index.md b/docs/evaluator/tutorials/index.md deleted file mode 100644 index 413df93a6e..0000000000 --- a/docs/evaluator/tutorials/index.md +++ /dev/null @@ -1,32 +0,0 @@ - -# Evaluation Tutorials - -Use these tutorials to become familiar with [evaluation with {{platform_name}}](../index.md). - -## Before You Start - -Set up [a local instance of the platform](../../get-started/setup.md) for the following tutorials. - -
- -- **[Run an LLM Judge Eval](run-llm-judge-evaluation.md)** - - --- - - Learn how to evaluate a fine-tuned model using the LLM Judge metric with a custom dataset. - - intermediate custom evaluation llm judge nemo-evaluator - -- **[Define and Run Custom Python Metrics](define-run-custom-python-metrics.md)** - - --- - - Learn how to write a domain-specific Python metric, test it locally, and run it through the Evaluator service. - - intermediate custom metric remote execution nemo-evaluator - -
- -## How It Works - -For the conceptual overview of how Evaluator separates definition (library) from execution (platform), see [About Evaluating → How It Works](../index.md#how-it-works-library-platform). For runnable SDK examples, see [SDK Resources](../sdk-resources.md). diff --git a/docs/evaluator/tutorials/index.mdx b/docs/evaluator/tutorials/index.mdx new file mode 100644 index 0000000000..1b2dbb5b12 --- /dev/null +++ b/docs/evaluator/tutorials/index.mdx @@ -0,0 +1,35 @@ +--- +title: "Evaluation Tutorials" +description: "" +--- + + +Use these tutorials to become familiar with [evaluation with NeMo Platform](/documentation/evaluate-models). + +## Before You Start + +Set up [a local instance of the platform](/documentation/get-started) for the following tutorials. + + + + + +Learn how to evaluate a fine-tuned model using the LLM Judge metric with a custom dataset. + +intermediate custom evaluation llm judge nemo-evaluator + + + + + +Learn how to write a domain-specific Python metric, test it locally, and run it through the Evaluator service. + +intermediate custom metric remote execution nemo-evaluator + + + + + +## How It Works + +For the conceptual overview of how Evaluator separates definition (library) from execution (platform), see [About Evaluating → How It Works](/documentation/evaluate-models#how-it-works-library-platform). For runnable SDK examples, see [SDK Resources](/documentation/evaluate-models/sdk-resources). diff --git a/docs/evaluator/tutorials/run-llm-judge-evaluation.md b/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx similarity index 87% rename from docs/evaluator/tutorials/run-llm-judge-evaluation.md rename to docs/evaluator/tutorials/run-llm-judge-evaluation.mdx index c7c8aadce5..27bb0113a1 100644 --- a/docs/evaluator/tutorials/run-llm-judge-evaluation.md +++ b/docs/evaluator/tutorials/run-llm-judge-evaluation.mdx @@ -1,13 +1,11 @@ - - - - - +--- +title: "@nemo-nb: hide" +description: "" +--- + + + + # Evaluate Response Quality with LLM-as-a-Judge @@ -33,12 +31,14 @@ This tutorial shows you how to build, validate, and iterate on LLM judge metrics | Complexity | Sophistication and depth of the response | 0-4 | | Verbosity | Appropriate level of detail | 0-4 | -!!! tip - This tutorial takes approximately **20 minutes** to complete. + + +This tutorial takes approximately **20 minutes** to complete. + ## Prerequisites -1. Install and start {{platform_name}} using the [Setup guide](../../get-started/setup.md). +1. Install and start NeMo Platform using the [Setup guide](/documentation/get-started). ```bash ! pip install nemo-platform[all] @@ -59,7 +59,7 @@ Before you begin, here is a quick overview of the resources you will use: - **Evaluator resource**: The plugin SDK resource mounted at `client.evaluator`. Use it to run metrics locally or submit durable platform jobs. - **Metric**: An inline Python object that defines how to score model outputs. In this tutorial, we create LLM judge metrics that prompt a model to rate responses. -- **Fileset**: A dataset registered with {{platform_name}}. The evaluator plugin SDK accepts fileset references directly, so this tutorial passes the registered HelpSteer2 split to evaluations as a `FilesetRef`. +- **Fileset**: A dataset registered with NeMo Platform. The evaluator plugin SDK accepts fileset references directly, so this tutorial passes the registered HelpSteer2 split to evaluations as a `FilesetRef`. - **Workspace**: A workspace that isolates your resources. Secrets, filesets, and jobs belong to a workspace. - **Job**: A durable remote platform task created with `evaluator.submit(...)`. - **Evaluation**: The process of scoring model outputs using one or more metrics. Use `evaluator.run(...)` for local in-process execution, `evaluator.submit(...)` for durable jobs @@ -80,7 +80,7 @@ from nemo_platform import ConflictError, NeMoPlatform # Workspace for this tutorial. WORKSPACE = "llm-as-judge-tutorial" -# Create an SDK instance to interact with the {{platform_name}}. +# Create an SDK instance to interact with the NeMo Platform. client = NeMoPlatform( base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"), workspace=WORKSPACE, @@ -139,9 +139,11 @@ print( ) ``` -!!! note - Local runs resolve `Model(api_key_secret="NVIDIA_API_KEY")` from your local environment. Remote jobs run in the platform job runtime, so they use the platform secret name created above. + +Local runs resolve `Model(api_key_secret="NVIDIA_API_KEY")` from your local environment. Remote jobs run in the platform job runtime, so they use the platform secret name created above. + + --- ## 3. Configure the Judge Models @@ -178,9 +180,11 @@ REMOTE_JUDGE_MODEL = Model( ) ``` -!!! tip - When using hosted APIs, keep `parallelism` low to avoid rate-limit errors. You can increase it for locally deployed models. + + +When using hosted APIs, keep `parallelism` low to avoid rate-limit errors. You can increase it for locally deployed models. + --- ## 4. Register and Load the Dataset @@ -227,9 +231,11 @@ except ConflictError: print(f"{fileset.workspace}/{fileset.name} dataset already registered") ``` -!!! note - HelpSteer2 contains prompt-response pairs with human ratings for helpfulness, correctness, coherence, complexity, and verbosity. Each rating is on a 0-4 scale. We'll use these human scores as ground truth to validate our LLM judge. + + +HelpSteer2 contains prompt-response pairs with human ratings for helpfulness, correctness, coherence, complexity, and verbosity. Each rating is on a 0-4 scale. We'll use these human scores as ground truth to validate our LLM judge. + Create a fileset reference for the validation split. The evaluator SDK resolves the selected fileset path when it runs, so you do not need to download the split into memory yourself: ```python @@ -249,11 +255,11 @@ Now let's create our first LLM judge metric. We'll start with a simple prompt fo A metric definition includes: - **Model**: Which LLM to use as the judge -- **Prompt template**: Instructions for the judge, with `{% raw %}{{item...}}{% endraw %}` fields filled from your dataset rows +- **Prompt template**: Instructions for the judge, with `{{item...}}` fields filled from your dataset rows - **Score definition**: The name, scale, and how to parse the judge's output ```python -{% raw %} + def create_helpfulness_metric(prompt_template: str, judge_model: Model) -> LLMJudgeMetric: """Create a helpfulness metric with the given system prompt.""" score = RangeScore( @@ -291,12 +297,14 @@ Respond with JSON only: {"helpfulness": <0-4>}""" metric_v1_local = create_helpfulness_metric(PROMPT_V1, LOCAL_JUDGE_MODEL) metric_v1_remote = create_helpfulness_metric(PROMPT_V1, REMOTE_JUDGE_MODEL) -{% endraw %} + ``` -!!! tip - Use low temperature for evaluation tasks. Low or zero temperature produces outputs with less variability, which is critical for reproducible scoring. This ensures the same response gets the same score across runs, making it easier to validate your judge and compare prompt versions. + + +Use low temperature for evaluation tasks. Low or zero temperature produces outputs with less variability, which is critical for reproducible scoring. This ensures the same response gets the same score across runs, making it easier to validate your judge and compare prompt versions. + --- ## 6. Test with Local Evaluation @@ -304,7 +312,7 @@ metric_v1_remote = create_helpfulness_metric(PROMPT_V1, REMOTE_JUDGE_MODEL) Before running a durable job, test your metric with a few examples using `evaluator.run(...)`. This runs locally in-process and returns results immediately, which is useful for prompt iteration. ```python -{% raw %} + quick_test_result = evaluator.run( metric=metric_v1_local, dataset=[ @@ -338,7 +346,7 @@ print("Quick test results:") for row in quick_test_result.row_scores: helpfulness = score_value(row, "helpfulness") print(f" Row {row.row_index}: helpfulness = {helpfulness}") -{% endraw %} + ``` **Expected output:** @@ -531,9 +539,11 @@ else: print("\nPrompt V1 performs better; simpler prompts can work well.") ``` -!!! note - More complex prompts do not always perform better. The best prompt depends on the model, task, and how well it aligns with the original annotation guidelines. If your V1 prompt outperforms V2, that's a valid result. Use what works best for your use case. + + +More complex prompts do not always perform better. The best prompt depends on the model, task, and how well it aligns with the original annotation guidelines. If your V1 prompt outperforms V2, that's a valid result. Use what works best for your use case. + --- ## 9. Visualize Score Distributions @@ -593,18 +603,20 @@ plt.show() print("Saved visualization to score_distributions.png") ``` -!!! tip - If you run this tutorial as a headless script instead of a notebook, configure a non-interactive Matplotlib backend before importing `pyplot`: + - ```python - import matplotlib +If you run this tutorial as a headless script instead of a notebook, configure a non-interactive Matplotlib backend before importing `pyplot`: - matplotlib.use("Agg") - import matplotlib.pyplot as plt - ``` +```python +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +``` - In that mode, save the figure with `plt.savefig(...)` and skip `plt.show()`. +In that mode, save the figure with `plt.savefig(...)` and skip `plt.show()`. + *The code above generates a chart showing score distributions. Your results will vary depending on the model used and the specific samples evaluated.* ### Percentile Analysis @@ -630,9 +642,11 @@ for name, scores in [ ) ``` -!!! tip - If your judge's distribution looks very different from humans, such as always scoring 3-4 while humans use the full range, adjust your prompt to calibrate the scoring criteria. + + +If your judge's distribution looks very different from humans, such as always scoring 3-4 while humans use the full range, adjust your prompt to calibrate the scoring criteria. + --- --- @@ -668,9 +682,11 @@ client.workspaces.delete(name=WORKSPACE) print("Cleanup complete!") ``` -!!! note - Workspaces cannot be deleted while they contain resources. The code above deletes resources in dependency order. + + +Workspaces cannot be deleted while they contain resources. The code above deletes resources in dependency order. + --- ## Troubleshooting @@ -804,12 +820,12 @@ In this tutorial, you learned how to: ## Next Steps -- **Experiment with rubric scores**: Use [categorical rubrics](../metrics/llm-as-a-judge.md) instead of numeric ranges for more interpretable criteria +- **Experiment with rubric scores**: Use [categorical rubrics](/documentation/evaluate-models/metrics/llm-as-a-judge) instead of numeric ranges for more interpretable criteria - **Try different judge models**: Larger models often correlate better with human judgment -- **Explore other evaluation types**: [RAG evaluation](../metrics/rag.md) or [agentic evaluation](../metrics/agentic.md) +- **Explore other evaluation types**: [RAG evaluation](/documentation/evaluate-models/metrics/rag-metrics) or [agentic evaluation](/documentation/evaluate-models/metrics/agentic-metrics) ## Related -- [LLM-as-a-Judge Reference](../metrics/llm-as-a-judge.md) - Complete guide to judge configuration -- [SDK Resources](../sdk-resources.md) - Evaluator plugin SDK resource reference -- [Manage Metrics](../metrics/manage-metrics.md) - Using evaluator SDK metric objects +- [LLM-as-a-Judge Reference](/documentation/evaluate-models/metrics/llm-as-a-judge) - Complete guide to judge configuration +- [SDK Resources](/documentation/evaluate-models/sdk-resources) - Evaluator plugin SDK resource reference +- [Manage Metrics](/documentation/evaluate-models/metrics/manage-metrics) - Using evaluator SDK metric objects diff --git a/docs/example-applications/about.md b/docs/example-applications/about.md deleted file mode 100644 index a36c3da430..0000000000 --- a/docs/example-applications/about.md +++ /dev/null @@ -1,3 +0,0 @@ -# Example Applications - -{{platform_name}} allows you to build complex AI applications by combining multiple components together. We've compiled a few examples that can be applied directly, extended and customized, or used as inspiration. \ No newline at end of file diff --git a/docs/example-applications/about.mdx b/docs/example-applications/about.mdx new file mode 100644 index 0000000000..603647f49f --- /dev/null +++ b/docs/example-applications/about.mdx @@ -0,0 +1,5 @@ +--- +title: "Example Applications" +description: "" +--- +NeMo Platform allows you to build complex AI applications by combining multiple components together. We've compiled a few examples that can be applied directly, extended and customized, or used as inspiration. diff --git a/docs/example-applications/tool-calling.mdx b/docs/example-applications/tool-calling.mdx new file mode 100644 index 0000000000..610c5aefb4 --- /dev/null +++ b/docs/example-applications/tool-calling.mdx @@ -0,0 +1,9 @@ +--- +title: "Tool-Calling Fine-Tuning with Synthetic Data" +description: "" +--- + + diff --git a/docs/fern/README.md b/docs/fern/README.md new file mode 100644 index 0000000000..0a4499b3c2 --- /dev/null +++ b/docs/fern/README.md @@ -0,0 +1,130 @@ +# NeMo Platform Fern Docs + +This directory holds the Fern **configuration** for the NeMo Platform documentation site. The page content (`.mdx`) lives in the parent `docs/` tree (e.g. `docs/get-started/setup.mdx`); this `docs/fern/` directory holds the navigation, theme, components, snippets, and OpenAPI wiring. The nav references pages with relative paths (`../..//.mdx`). + +## Quick Links + +| What | Where | +| --- | --- | +| Fern dashboard | https://dashboard.buildwithfern.com (NVIDIA org) | +| Contributor/agent guide | [`../AGENTS.md`](../AGENTS.md) | +| Make targets | `make docs`, `docs-check`, `docs-broken-links`, `docs-fix-links`, `docs-preview` (repo root) | +| CI workflows | [`../../.github/workflows/`](../../.github/workflows/) (`fern-docs-*.yaml`) | +| Publish workflow | [`../../.github/workflows/publish-fern-docs.yaml`](../../.github/workflows/publish-fern-docs.yaml) | + +## Quickstart + +From the repo root (these wrap `cd docs/fern && npm run …`): + +```bash +make docs-deps # one-time: install docs/fern tooling (needed for MDX validation) +make docs-login # one-time per machine: Fern CLI auth for the nvidia org +make docs-check # validate: fern check + MDX validation + gated-link check (what CI runs) +make docs # start local preview (prints a localhost URL) +``` + +`package.json` shells out to `npx -y fern-api@latest` for the Fern CLI itself, but `make docs-deps` (`npm ci`) is required once because the MDX validator (`@mdx-js/mdx`) is a local dependency. + +## Layout + +```text +docs/ # page content (.mdx), one tree per product area +├── get-started/ , agents/ , evaluator/ , ... # published pages +└── fern/ # <- this directory: Fern site config + ├── fern.config.json # Fern organization + CLI version + ├── package.json # npm run check|dev|generate|preview|broken-links|*-gated-links + ├── docs.yml # Site config, theme, css, redirects, versions + ├── generators.yml # OpenAPI spec wiring for the API reference + ├── openapi/openapi.yaml # symlink -> repo-root openapi/openapi.yaml (generated) + ├── styles/ # CSS (notebook-viewer.css, button.css) + ├── assets/ # logos, shared images + ├── components/ # custom TSX MDX components + ├── snippets/ # reusable fragments + ├── scripts/ # validate-mdx.mjs, delink-gated.mjs, ipynb-to-fern-json.py + ├── gated-nav.yml # reference nav blocks for gated (unready) features + └── versions/latest.yml # navigation tree (defines what gets built) +``` + +The site uses a single `Latest` version. `versions/latest.yml` defines the sidebar and maps each page file to its canonical route — and, because Fern only builds pages listed there, it is also what gates unready content (see below). + +## Authoring + +Add pages under the relevant `docs//` directory and wire them into `versions/latest.yml`. + +Use front matter for the rendered page title: + +```yaml +--- +title: "Page Title" +description: "" +--- +``` + +Do not add a duplicate first `# Page Title` heading when it matches the front matter `title`; Fern renders that title automatically. + +Use Fern-native MDX components such as ``, ``, ``, ``, ``, and ``. Do not reintroduce MkDocs Material syntax like `!!! note`, `=== "Tab"`, `--8<--`, or `
`. + +## Internal Links + +Use Fern's nav-derived canonical URLs: + +```mdx +[Workspaces](/documentation/get-started/core-concepts/workspaces) +``` + +Avoid source-path links such as `/get-started/concepts/workspaces`, `/latest/get-started/concepts/workspaces`, and relative `.md` links. If a public URL changes, add a redirect in `docs.yml`. + +## API Reference + +The REST API reference is generated natively by Fern from the OpenAPI spec — no `` embed. Two pieces wire it up: + +- `generators.yml` declares the spec: `api.specs[].openapi: ./openapi/openapi.yaml`. +- `versions/latest.yml` surfaces it with an `- api: API Reference` navigation node (under the **Reference** section). + +`openapi/openapi.yaml` is a symlink to the repo-root `openapi/openapi.yaml` (the generated source of truth), so the reference tracks the Platform API automatically. Regenerate the spec with `make refresh-openapi` from the repo root, then run `npm run check`. + +Fern groups endpoints by their OpenAPI tag in the sidebar (Customizer, Evaluator, Guardrails, …), which replaces the old per-service filter chips. Link to it from other pages with the nav URL `/documentation/reference/api-reference`. + +## Gated (unready) features + +Some features are not shipped yet and must be **fully excluded from the build** — not just hidden from the sidebar. Fern's `hidden: true` still builds and serves the page (reachable by direct URL and indexable), so it is **not** used for this. Instead, the gated pages are simply **left out of `versions/latest.yml`**: Fern only builds pages referenced in the navigation, so an omitted page is never built (it 404s and is not indexed). This matches the old MkDocs `hide_unready_docs` hook, which dropped the same files from the build. + +The gated `.mdx` files stay in the repo so they remain maintained. The gated trees today are: `auth/`, `customizer/`, `safe-synthesizer/`, `set-up/` + `helm/`, `evaluator/benchmarks/`, plus individual pages (`evaluator/metrics/{job-management,results}`, `run-inference/tutorials/deploy-models`, `example-applications/`, `troubleshooting/{cluster-setup,customizer}`, `get-started/quickstart`). + +Inbound links from visible pages into gated pages are **delinked to plain text** (not rewritten URLs), since the target is not built — otherwise they would be broken links. + +### Automated delinking + +The old MkDocs hook delinked those references automatically. Fern has no build hook, so `scripts/delink-gated.mjs` reproduces it. "Gated" is derived from the nav (any `.mdx` not in `versions/latest.yml`), so there is no second list to maintain. + +- `npm run check:gated-links` — fails if any **published** page links into a gated page (runs as part of `npm run check`, so CI enforces it). +- `npm run fix:gated-links` — rewrites those links to plain text in place. + +So if someone links into a gated section, CI flags it and one command delinks it — no hunting for dead links by hand. `npm run broken-links` (also run in CI) is the broader backstop for any other dead link. + +One difference from the old MkDocs hook: that hook ran at build time and kept the link in the source, so re-publishing a page reactivated its inbound links automatically. This script delinks in the **source**, so the references become plain text. When you publish a feature, re-add the links you want from those plain-text references as part of the same step that moves the nav block (`check:gated-links` won't block you — it only flags live links that still point at gated pages). This keeps local `fern docs dev` honest (no build-time source rewriting) at the cost of re-linking on publish. + +`gated-nav.yml` holds the ready-to-paste navigation blocks for the gated features (it is a reference only; Fern does not read it). **To publish a feature when it ships:** move its block from `gated-nav.yml` into `versions/latest.yml`, re-add any inbound links you want, then run `npm run check` and `npm run broken-links`. + +## CI and Publishing + +| Workflow | Trigger | Purpose | +| --- | --- | --- | +| `fern-docs-ci.yaml` | `pull_request` touching `docs/**` | `npm run check` (fern check + MDX + gated-link check) and `npm run broken-links` | +| `fern-docs-preview-build.yaml` | `pull_request` touching `docs/**` | Upload PR `docs/` sources as an artifact (no secrets — fork-safe) | +| `fern-docs-preview-comment.yaml` | successful preview build (`workflow_run`) | Build a Fern preview with `DOCS_FERN_TOKEN` and post/update the PR comment | +| `publish-fern-docs.yaml` | push to `main` touching `docs/**`, `docs/v*` tag, or manual dispatch | Publish the Fern docs site | + +Required secret: `DOCS_FERN_TOKEN` (org-level), from `fern token` for an account that can publish to the NVIDIA Fern organization. + +PRs that touch `docs/**` get a shared preview URL posted as a comment after the two-part preview workflow finishes. Note: the `workflow_run`-triggered comment job only runs once these workflows are on the default branch (`main`), so the first preview appears after this PR merges. + +## Troubleshooting + +| Symptom | Fix | +| --- | --- | +| `HTTP 403` or organization access error | Sign in at https://dashboard.buildwithfern.com, then run `npx -y fern-api@latest login` again | +| `fern check` YAML error | Use 2-space indentation; make sure `path:` values are relative to `versions/latest.yml` | +| Page 404 in preview | Check that `versions/latest.yml` lists the page (gated pages are intentionally omitted and *will* 404) | +| Broken internal link | Rewrite to the nav URL `/documentation/...`; if it targets a gated page, run `make docs-fix-links` to delink it. `make docs-broken-links` reports them all | +| JSX or MDX parse error | Escape raw `{}`, `<`, or `>` in prose, and use Fern components instead of raw MkDocs syntax | diff --git a/docs/fern/assets/images/agent_eval_framework.png b/docs/fern/assets/images/agent_eval_framework.png new file mode 100644 index 0000000000..044bfebc87 --- /dev/null +++ b/docs/fern/assets/images/agent_eval_framework.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4683db588a0d8a00e462a5f8c452184c941d7d5c6a42361149c50fa1432c8984 +size 100009 diff --git a/docs/fern/assets/images/attribute-inference-protection.png b/docs/fern/assets/images/attribute-inference-protection.png new file mode 100644 index 0000000000..51835ae4b3 --- /dev/null +++ b/docs/fern/assets/images/attribute-inference-protection.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a4cdefe61df6630387b2c5fa92bfdc275a14d97130f007e819355d90ce34bca +size 79411 diff --git a/docs/fern/assets/images/car-audio-theft.jpg b/docs/fern/assets/images/car-audio-theft.jpg new file mode 100644 index 0000000000..3995f5be16 --- /dev/null +++ b/docs/fern/assets/images/car-audio-theft.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ed605f0ed8e5e2c3f35cdf21894646f82f6df407aae1c527110a913df3253b7 +size 29606 diff --git a/docs/fern/assets/images/column-correlation-stability.png b/docs/fern/assets/images/column-correlation-stability.png new file mode 100644 index 0000000000..9187b2897b --- /dev/null +++ b/docs/fern/assets/images/column-correlation-stability.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bf8796bbcbe7c03d1acde5494d3110c35d5299c62f08056d0755853ebd96178 +size 270914 diff --git a/docs/fern/assets/images/column-distribution-stability.png b/docs/fern/assets/images/column-distribution-stability.png new file mode 100644 index 0000000000..1fd663e219 --- /dev/null +++ b/docs/fern/assets/images/column-distribution-stability.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e340503e29c05d47e1e7156a2efe2d6c403f4b1e12619938653d477a9ba50ed7 +size 77566 diff --git a/docs/fern/assets/images/customizations.png b/docs/fern/assets/images/customizations.png new file mode 100644 index 0000000000..bb5dc5dc81 --- /dev/null +++ b/docs/fern/assets/images/customizations.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25dc8dc4d79105232e46d4ee4af42a1e0aa1a6a34fce63a1dec67a29adc91ec5 +size 66483 diff --git a/docs/fern/assets/images/dashboard.png b/docs/fern/assets/images/dashboard.png new file mode 100644 index 0000000000..1ede1b98db --- /dev/null +++ b/docs/fern/assets/images/dashboard.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afafac13cda48a9a0c91e3d0164c7dcc7a16ef12cf80edbdcb8047915c72ea5f +size 131900 diff --git a/docs/fern/assets/images/datasets.png b/docs/fern/assets/images/datasets.png new file mode 100644 index 0000000000..e62cfae2e5 --- /dev/null +++ b/docs/fern/assets/images/datasets.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb226eb3adb2d1f6896b7c7f6a1e52b527deac3ab16ee0412cd3616b1d26e1e3 +size 60171 diff --git a/docs/fern/assets/images/deep-structure-stability.png b/docs/fern/assets/images/deep-structure-stability.png new file mode 100644 index 0000000000..e62a10bf92 --- /dev/null +++ b/docs/fern/assets/images/deep-structure-stability.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96249fa81a0a9a40f552dac2ce6a8fe6b7f8d863ffb443ceb2b9d0ba53875158 +size 121483 diff --git a/docs/fern/assets/images/dps-subscores.png b/docs/fern/assets/images/dps-subscores.png new file mode 100644 index 0000000000..ca899410c4 --- /dev/null +++ b/docs/fern/assets/images/dps-subscores.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27c407368d08b5aa1efe9939e7f2c86eccea140e7469757dfcfd6cd979787f67 +size 148059 diff --git a/docs/fern/assets/images/dps-usability.png b/docs/fern/assets/images/dps-usability.png new file mode 100644 index 0000000000..e89f6bc6b0 --- /dev/null +++ b/docs/fern/assets/images/dps-usability.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07a1d60023e4f8801e71b1f2c5381f9808b5592a4fc44230fa7aced976cc9c69 +size 51168 diff --git a/docs/fern/assets/images/evaluation_result_dir.png b/docs/fern/assets/images/evaluation_result_dir.png new file mode 100644 index 0000000000..a616d08f3c --- /dev/null +++ b/docs/fern/assets/images/evaluation_result_dir.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa6a64998f2f2a4a5af65683c646d9226680e4e9187e41900536c10e4999f275 +size 74039 diff --git a/docs/fern/assets/images/evaluations.png b/docs/fern/assets/images/evaluations.png new file mode 100644 index 0000000000..755c2b9d71 --- /dev/null +++ b/docs/fern/assets/images/evaluations.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbd197dd7dd5304094e379e1c4af64935cc9a76b586c24da64121b77cea655dc +size 57931 diff --git a/docs/fern/assets/images/evaluator_interactions.png b/docs/fern/assets/images/evaluator_interactions.png new file mode 100644 index 0000000000..79c10a6981 --- /dev/null +++ b/docs/fern/assets/images/evaluator_interactions.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea680b65c386858426ed0b05c83d500d4cf224acba7b3115c387450a616f67e5 +size 94633 diff --git a/docs/images/favicon.ico b/docs/fern/assets/images/favicon.ico similarity index 100% rename from docs/images/favicon.ico rename to docs/fern/assets/images/favicon.ico diff --git a/docs/fern/assets/images/gpu_memory.png b/docs/fern/assets/images/gpu_memory.png new file mode 100644 index 0000000000..0c5c31d9c1 --- /dev/null +++ b/docs/fern/assets/images/gpu_memory.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dbfd9fb92f561fe1439cde258cec7ec1e776862ca94bf6d7a77ece92d981c5d +size 103607 diff --git a/docs/fern/assets/images/gpu_utilization.png b/docs/fern/assets/images/gpu_utilization.png new file mode 100644 index 0000000000..fab38d6067 --- /dev/null +++ b/docs/fern/assets/images/gpu_utilization.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58036162ad51e43d8dbee45a70459ec37a10ab06614a4088417de9db6a9c8f64 +size 186900 diff --git a/docs/fern/assets/images/jobs.png b/docs/fern/assets/images/jobs.png new file mode 100644 index 0000000000..b5a6efff3f --- /dev/null +++ b/docs/fern/assets/images/jobs.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67e04e0f6f8ee07d3b22126610a071db0b966d4fd09fc3c740fee294b88ec4f4 +size 51380 diff --git a/docs/images/main_nv_logo_square.png b/docs/fern/assets/images/main_nv_logo_square.png similarity index 100% rename from docs/images/main_nv_logo_square.png rename to docs/fern/assets/images/main_nv_logo_square.png diff --git a/docs/fern/assets/images/membership-inference-protection.png b/docs/fern/assets/images/membership-inference-protection.png new file mode 100644 index 0000000000..03c108e0bd --- /dev/null +++ b/docs/fern/assets/images/membership-inference-protection.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e202fff24fec9baab76a980dcb3d98793a0573271ccda0c83172258e8c19e7ef +size 388868 diff --git a/docs/fern/assets/images/nemo-eval-diagram-2510.png b/docs/fern/assets/images/nemo-eval-diagram-2510.png new file mode 100644 index 0000000000..45bb6cb09b --- /dev/null +++ b/docs/fern/assets/images/nemo-eval-diagram-2510.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70d36b0e73afc7a63de97beb3795d3b7d39373386bf12b83a56ebb254fd4230c +size 286432 diff --git a/docs/fern/assets/images/nemo-platform-architecture.svg b/docs/fern/assets/images/nemo-platform-architecture.svg new file mode 100644 index 0000000000..3f54d60505 --- /dev/null +++ b/docs/fern/assets/images/nemo-platform-architecture.svg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c6ba483df553bf9dd0bf750359c27de7d34478054023894635084ae0e17a9d8 +size 132304 diff --git a/docs/images/nvidia-logo-white.png b/docs/fern/assets/images/nvidia-logo-white.png similarity index 100% rename from docs/images/nvidia-logo-white.png rename to docs/fern/assets/images/nvidia-logo-white.png diff --git a/docs/fern/assets/images/overall-sqs-dps.png b/docs/fern/assets/images/overall-sqs-dps.png new file mode 100644 index 0000000000..82639ab173 --- /dev/null +++ b/docs/fern/assets/images/overall-sqs-dps.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:240532d46d8ed57992bb87dd160c7f471bcb0f365411c23cec61ab930a02799b +size 49850 diff --git a/docs/fern/assets/images/packed_vs_not_packed_tables.png b/docs/fern/assets/images/packed_vs_not_packed_tables.png new file mode 100644 index 0000000000..4d85735cdf --- /dev/null +++ b/docs/fern/assets/images/packed_vs_not_packed_tables.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98ec568543654899b3de6f20e90306f369aa083f5d41e5e92b04bf1dedcb863d +size 51068 diff --git a/docs/fern/assets/images/packed_vs_not_packed_val_loss.png b/docs/fern/assets/images/packed_vs_not_packed_val_loss.png new file mode 100644 index 0000000000..64716a20f8 --- /dev/null +++ b/docs/fern/assets/images/packed_vs_not_packed_val_loss.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46aa81bd023f5ac8a05b3afb14e42c178efeae20ebfce990699324767bc51b72 +size 59932 diff --git a/docs/fern/assets/images/pca.png b/docs/fern/assets/images/pca.png new file mode 100644 index 0000000000..0284847598 --- /dev/null +++ b/docs/fern/assets/images/pca.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf0da92c1242110b4c89d01eb16c3fa6d0fdcfda2dea1482093ee0c191653d20 +size 244797 diff --git a/docs/fern/assets/images/pii-replay.png b/docs/fern/assets/images/pii-replay.png new file mode 100644 index 0000000000..ed585a3a64 --- /dev/null +++ b/docs/fern/assets/images/pii-replay.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1730ad81b8527927269948253b8a1a594c65ff37ae3af5e5b31586f759fc492 +size 502381 diff --git a/docs/fern/assets/images/pod_logs.png b/docs/fern/assets/images/pod_logs.png new file mode 100644 index 0000000000..d9bd08b145 --- /dev/null +++ b/docs/fern/assets/images/pod_logs.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99c4fd813f8417e0f10f8470944648f3daeadd679e46509e7f361a976363785e +size 85826 diff --git a/docs/fern/assets/images/prompt-tuned-models.png b/docs/fern/assets/images/prompt-tuned-models.png new file mode 100644 index 0000000000..5d320cf844 --- /dev/null +++ b/docs/fern/assets/images/prompt-tuned-models.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7478e25256510953156ab50827caaa751ab80732ae33b010c904f780fc33ad80 +size 93980 diff --git a/docs/fern/assets/images/run-evaluation.png b/docs/fern/assets/images/run-evaluation.png new file mode 100644 index 0000000000..b094b83a8c --- /dev/null +++ b/docs/fern/assets/images/run-evaluation.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02b2248cba2109fcbfb0301fbc8b09c6308ebad94f649111bf02181d93ba334f +size 75450 diff --git a/docs/fern/assets/images/runtime.png b/docs/fern/assets/images/runtime.png new file mode 100644 index 0000000000..08e93044b9 --- /dev/null +++ b/docs/fern/assets/images/runtime.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55e3ed1c7abc734af7f1ae800586c70f0fc8a95bba94153c95a1110c348bd351 +size 40022 diff --git a/docs/fern/assets/images/safe-synthesizer.png b/docs/fern/assets/images/safe-synthesizer.png new file mode 100644 index 0000000000..6c5eef9aec --- /dev/null +++ b/docs/fern/assets/images/safe-synthesizer.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f90ab8fa51a8d3281697f6437d7470c413ad0979f7ed67ee29b6be89bbd315a +size 92404 diff --git a/docs/fern/assets/images/secrets.png b/docs/fern/assets/images/secrets.png new file mode 100644 index 0000000000..0c9d1549c0 --- /dev/null +++ b/docs/fern/assets/images/secrets.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dc00607d2227df31a53f8a249fdf931e8f40f41564afda8db7e70c45f104c5f +size 56868 diff --git a/docs/fern/assets/images/seq_packing_stats.png b/docs/fern/assets/images/seq_packing_stats.png new file mode 100644 index 0000000000..f58d188ce1 --- /dev/null +++ b/docs/fern/assets/images/seq_packing_stats.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:928c2eb1aeaab3c28e81716f600ef250eeb5b781548e7d3ad879c1cc9b58ccd1 +size 34452 diff --git a/docs/fern/assets/images/sqs-subscores.png b/docs/fern/assets/images/sqs-subscores.png new file mode 100644 index 0000000000..5dce6af893 --- /dev/null +++ b/docs/fern/assets/images/sqs-subscores.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac180e353c03edeee07a829935bc3ce93cd39a9cd4c16da86ed1babfba282f85 +size 210286 diff --git a/docs/fern/assets/images/sqs-usability.png b/docs/fern/assets/images/sqs-usability.png new file mode 100644 index 0000000000..e597068b23 --- /dev/null +++ b/docs/fern/assets/images/sqs-usability.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ef0663067d0e5e1da919cf6108d32e26e80f815257310ba401723befde533c2 +size 56886 diff --git a/docs/fern/assets/images/street-scene.jpg b/docs/fern/assets/images/street-scene.jpg new file mode 100644 index 0000000000..fdfe810082 --- /dev/null +++ b/docs/fern/assets/images/street-scene.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce84c961f2ade3aa431ad77a10b8d6fc24ffba6b79a6c4d63c2f0cb4371993c9 +size 29771 diff --git a/docs/fern/assets/images/text-semantic-similarity.png b/docs/fern/assets/images/text-semantic-similarity.png new file mode 100644 index 0000000000..d4b474b7a7 --- /dev/null +++ b/docs/fern/assets/images/text-semantic-similarity.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eb77b97d7aa0c2fcc5170dbbce8c65b56e229d964af21fad3abb1b2337bd276 +size 161834 diff --git a/docs/fern/assets/images/text-structure-similarity.png b/docs/fern/assets/images/text-structure-similarity.png new file mode 100644 index 0000000000..daddded6d8 --- /dev/null +++ b/docs/fern/assets/images/text-structure-similarity.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03c4d27552697ee19f56aa0f1b6f1bb93dedcd14356732021809941a7e648f35 +size 106838 diff --git a/docs/fern/assets/images/wandb_charts_example.png b/docs/fern/assets/images/wandb_charts_example.png new file mode 100644 index 0000000000..8337792d11 --- /dev/null +++ b/docs/fern/assets/images/wandb_charts_example.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3afb36a6b944926559e07713756a5b27bb8e350a4f4bd2dd23ac810aced0dfcc +size 161897 diff --git a/docs/fern/assets/images/workflow_logs.png b/docs/fern/assets/images/workflow_logs.png new file mode 100644 index 0000000000..8c1dacd126 --- /dev/null +++ b/docs/fern/assets/images/workflow_logs.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4a563bc0eb19d27350bb55b8dda269fa054a8d058c790222a82f18616e01359 +size 67084 diff --git a/docs/fern/assets/images/workspaces.png b/docs/fern/assets/images/workspaces.png new file mode 100644 index 0000000000..ac3e56c3ea --- /dev/null +++ b/docs/fern/assets/images/workspaces.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a60d05fefbbea4ea30545f6de9aba7a8fe772c2a74e21815a88243b499fe28df +size 110153 diff --git a/docs/fern/components/Authors.tsx b/docs/fern/components/Authors.tsx new file mode 100644 index 0000000000..0c71197953 --- /dev/null +++ b/docs/fern/components/Authors.tsx @@ -0,0 +1,56 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Authors - Renders author byline with avatars for dev notes / blog posts. + * + * Uses authors data from components/devnotes/authors-data.ts (synced with .authors.yml). + * NOTE: Fern's custom component pipeline uses the automatic JSX runtime. + * + * Usage in MDX (authors from frontmatter): + * --- + * authors: + * - jdoe + * - asmith + * --- + * + * import { Authors } from "@/components/Authors"; + * + */ + +import { authors } from "./devnotes/authors-data"; + +export interface AuthorsProps { + /** Author IDs from .authors.yml. From frontmatter: ids={authors} */ + ids?: string[]; +} + +export const Authors = ({ ids }: AuthorsProps) => { + const validAuthors = (ids ?? []) + .map((id) => authors[id]) + .filter(Boolean); + + if (validAuthors.length === 0) return null; + + return ( +
+ {validAuthors.map((author, i) => ( +
+ +
+ {author.name} + {author.description} +
+
+ ))} +
+ ); +}; diff --git a/docs/fern/components/BadgeLinks.tsx b/docs/fern/components/BadgeLinks.tsx new file mode 100644 index 0000000000..c4a5949c33 --- /dev/null +++ b/docs/fern/components/BadgeLinks.tsx @@ -0,0 +1,37 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Badge links for GitHub, License, PyPI, etc. + * Uses a custom wrapper to avoid Fern's external-link icon stacking under badges. + * + * `badges` is required — there is intentionally no default. A previous + * version shipped placeholder URLs that could land in production for + * sites that rendered the component without props. See README-BadgeLinks.md. + */ +export type BadgeItem = { + href: string; + src: string; + alt: string; +}; + +export interface BadgeLinksProps { + badges: BadgeItem[]; +} + +export function BadgeLinks({ badges }: BadgeLinksProps) { + return ( +
+ {badges.map((b) => ( + + {b.alt} + + ))} +
+ ); +} diff --git a/docs/fern/components/Button.tsx b/docs/fern/components/Button.tsx new file mode 100644 index 0000000000..5945e6df42 --- /dev/null +++ b/docs/fern/components/Button.tsx @@ -0,0 +1,47 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Button — a styled link rendered as a CTA button. + * + * Replaces MkDocs Material's `{ .md-button }` / `{ .md-button--primary }` + * attribute list. The converter emits this wrapper when it sees that + * pattern in the docs source. + * + * NOTE: Fern's custom component pipeline uses the automatic JSX runtime; + * we do not import React. + * + * Usage in MDX: + * import { Button } from "@/components/Button"; + * + * + * + * + * In MDX you can also pass a markdown link via children if you prefer: + * + */ + +import type { ReactNode } from "react"; + +export interface ButtonProps { + href: string; + variant?: "primary" | "secondary"; + children: ReactNode; +} + +export function Button({ href, variant = "secondary", children }: ButtonProps) { + const external = /^https?:\/\//.test(href); + return ( + + {children} + + ); +} + +export default Button; diff --git a/docs/fern/components/Icon.tsx b/docs/fern/components/Icon.tsx new file mode 100644 index 0000000000..154fcaf8cb --- /dev/null +++ b/docs/fern/components/Icon.tsx @@ -0,0 +1,88 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Icon — render a small inline icon, mapping MkDocs Material icon names to + * inline SVGs. + * + * The MkDocs source uses `:icon: code-square` (admonition option) and + * `:material-foo:` (inline). The converter strips the admonition option + * (Accordion / Note don't take an icon prop yet), so this component is + * here for hand-authored MDX pages that want to render an icon next to + * text. Add new names by extending the ICONS map. + * + * Usage: + * import { Icon } from "@/components/Icon"; + * + * + * + */ + +import type { ReactNode } from "react"; + +// Minimal Lucide-style stroke icons. Lucide is MIT and we trace just the +// names actually used in our docs to avoid a runtime icon dependency. +const ICONS: Record = { + "code-square": ( + <> + + + + + ), + terminal: ( + <> + + + + ), + gear: ( + <> + + + + ), + info: ( + <> + + + + + ), + shield: ( + + ), +}; + +export interface IconProps { + name: string; + size?: number; +} + +export function Icon({ name, size = 16 }: IconProps) { + const glyph = ICONS[name]; + if (!glyph) { + return null; + } + return ( + + {glyph} + + ); +} + +export default Icon; diff --git a/docs/fern/components/MetricsTable.tsx b/docs/fern/components/MetricsTable.tsx new file mode 100644 index 0000000000..f95dc18646 --- /dev/null +++ b/docs/fern/components/MetricsTable.tsx @@ -0,0 +1,106 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * MetricsTable - Styled comparison table for benchmark results. + * + * Optional: highlights best values per column (bold). + * NOTE: Fern's custom component pipeline uses the automatic JSX runtime. + * Do NOT import React -- the `react` module is not resolvable in Fern's build. + * + * Usage in MDX: + * import { MetricsTable } from "@/components/MetricsTable"; + * + * + */ + +export interface MetricsTableProps { + headers: string[]; + rows: (string | number)[][]; + /** Column indices where lower is better (for highlighting) */ + lowerIsBetter?: number[]; + /** Column indices where higher is better (default for non-lowerIsBetter) */ + higherIsBetter?: number[]; +} + +function findBestIndices( + rows: (string | number)[][], + colIndex: number, + lowerIsBetter: boolean +): Set { + const values = rows.map((r) => { + const v = r[colIndex]; + if (typeof v === "number") return v; + const parsed = parseFloat(String(v)); + return isNaN(parsed) ? (lowerIsBetter ? Infinity : -Infinity) : parsed; + }); + const best = lowerIsBetter ? Math.min(...values) : Math.max(...values); + const bestIndices = new Set(); + values.forEach((v, i) => { + if (v === best) bestIndices.add(i); + }); + return bestIndices; +} + +export const MetricsTable = ({ + headers, + rows, + lowerIsBetter = [], + higherIsBetter = [], +}: MetricsTableProps) => { + const lowerSet = new Set(lowerIsBetter); + const bestByCol: Record> = {}; + + for (let c = 0; c < headers.length; c++) { + if (lowerSet.has(c)) { + bestByCol[c] = findBestIndices(rows, c, true); + } else if (higherIsBetter.includes(c)) { + bestByCol[c] = findBestIndices(rows, c, false); + } else { + const numLike = rows.every((r) => { + const v = r[c]; + return typeof v === "number" || !isNaN(parseFloat(String(v))); + }); + if (numLike) { + bestByCol[c] = findBestIndices(rows, c, false); + } + } + } + + return ( +
+ + + + {headers.map((h, i) => ( + + ))} + + + + {rows.map((row, rowIdx) => ( + + {row.map((cell, colIdx) => { + const isBest = bestByCol[colIdx]?.has(rowIdx); + return ( + + ); + })} + + ))} + +
{h}
+ {cell} +
+
+ ); +}; diff --git a/docs/fern/components/NotebookViewer.tsx b/docs/fern/components/NotebookViewer.tsx new file mode 100644 index 0000000000..e05f48ec41 --- /dev/null +++ b/docs/fern/components/NotebookViewer.tsx @@ -0,0 +1,456 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { ReactNode } from "react"; + +import distillationCustomizationJob from "./notebooks/distillation-customization-job"; +import dpoCustomizationJob from "./notebooks/dpo-customization-job"; +import embeddingCustomizationJob from "./notebooks/embedding-customization-job"; +import loraCustomizationJob from "./notebooks/lora-customization-job"; +import optimizeThroughput from "./notebooks/optimize-throughput"; +import sftCustomizationJob from "./notebooks/sft-customization-job"; +import toolCalling from "./notebooks/tool-calling"; + +// Pages mounted from outside docs/fern/ cannot use `@/` imports (Fern's +// bundler refuses `..` escapes), so notebook data is registered here and +// pages look it up by name: ``. +const notebooks: Record = { + "distillation-customization-job": distillationCustomizationJob, + "dpo-customization-job": dpoCustomizationJob, + "embedding-customization-job": embeddingCustomizationJob, + "lora-customization-job": loraCustomizationJob, + "optimize-throughput": optimizeThroughput, + "sft-customization-job": sftCustomizationJob, + "tool-calling": toolCalling, +}; + +/** + * NotebookViewer - Renders Jupyter notebook content in Fern docs. + * + * Uses Fern's code block structure (fern-code, fern-code-block, etc.) so input + * and output cells match the default Fern code block styling. + * + * Accepts notebook cells (markdown + code) and optionally a Colab URL. + * Designed to work with notebooks converted via `fern/scripts/ipynb-to-fern-json.py`. + * + * NOTE: Fern's custom component pipeline uses the automatic JSX runtime. + * Only type-only imports from "react" are used (erased at compile time). + * + * SECURITY / TRUST MODEL: + * -------- + * Notebook output cells of `format: "html"` (typically pandas DataFrame `_repr_html_`, + * matplotlib HTML, or similar) are rendered with `dangerouslySetInnerHTML` and + * are NOT sanitized — see the renderer near the bottom of this file. + * + * The trust boundary is the converter pipeline: + * + * docs/notebook_source/*.py (jupytext source — code-reviewed at PR time) + * └─> jupytext --execute (runs in CI/maintainer shell with NVIDIA_API_KEY) + * └─> *.ipynb (real outputs captured) + * └─> fern/scripts/ipynb-to-fern-json.py + * └─> fern/components/notebooks/*.{json,ts} (committed) + * + * Both ends of the pipeline (the .py source and the generated *.ts) are + * code-reviewed before merge. Fern's bundle is then static. + * + * If a future tutorial ever embeds output that incorporates LLM-generated HTML + * or arbitrary user-controlled content, switch the `format === "html"` branch + * to a sanitizer (e.g. DOMPurify) before merging. + * + * Usage in MDX: + * import { NotebookViewer } from "@/components/NotebookViewer"; + * import notebook from "@/components/notebooks/1-the-basics"; + * + * + */ + +export interface CellOutput { + type: "text" | "image"; + data: string; + format?: "plain" | "html"; + /** MIME for image outputs. Defaults to image/png; the converter sets + * image/jpeg when it re-encodes large outputs to keep the .ts payload small. */ + mime?: string; +} + +export interface NotebookCell { + type: "markdown" | "code"; + source: string; + /** Pre-rendered syntax-highlighted HTML (from Pygments). When present, used instead of escaped source. */ + source_html?: string; + language?: string; + outputs?: CellOutput[]; +} + +export interface NotebookData { + cells: NotebookCell[]; +} + +export interface NotebookViewerProps { + /** Notebook name registered in the registry above (preferred for MDX usage). */ + name?: string; + /** Notebook data with cells array. Used when no `name` is provided. */ + notebook?: NotebookData | null; + /** Optional Colab URL for "Run in Colab" badge */ + colabUrl?: string; + /** Show code cell outputs (default: true) */ + showOutputs?: boolean; +} + +function NotebookViewerError({ message, detail }: { message: string; detail?: string }) { + return ( +
+ NotebookViewer error: {message} + {detail && ( + 
+          {detail}
+
+ )} +
+ ); +} + +function escapeHtml(text: string): string { + if (typeof text !== "string") return ""; + return text + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + + +function handleCopy(content: string, button: HTMLButtonElement) { + navigator.clipboard.writeText(content).catch(() => {}); + const originalHtml = button.innerHTML; + const originalLabel = button.getAttribute("aria-label") ?? "Copy code"; + button.innerHTML = "Copied!"; + button.setAttribute("aria-label", "Copied to clipboard"); + setTimeout(() => { + button.innerHTML = originalHtml; + button.setAttribute("aria-label", originalLabel); + }, 1500); +} + +const FLAG_ICON = ( + + + + +); + +const SCROLL_AREA_STYLE = `[data-radix-scroll-area-viewport]{scrollbar-width:thin;scrollbar-color:var(--grayscale-a7) transparent;-webkit-overflow-scrolling:touch;}[data-radix-scroll-area-viewport]::-webkit-scrollbar{height:8px;width:8px;}[data-radix-scroll-area-viewport]::-webkit-scrollbar-track{background:transparent;}[data-radix-scroll-area-viewport]::-webkit-scrollbar-thumb{background:var(--grayscale-a7);border-radius:9999px;}`; + +const BUTTON_BASE_CLASS = + "focus-visible:ring-(color:--accent) rounded-2 inline-flex items-center justify-center gap-2 whitespace-nowrap text-sm font-medium transition-colors hover:transition-none focus-visible:outline-none focus-visible:ring-1 disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0 text-(color:--grayscale-a11) hover:bg-(color:--accent-a3) hover:text-(color:--accent-11) pointer-coarse:size-9 size-7"; + +/** Fern code block structure – matches Fern docs (header with language + buttons, pre with scroll area). */ +function FernCodeBlock({ + title, + children, + className = "", + asPre = true, + copyContent, + showLineNumbers = false, + codeHtml, +}: { + title: string; + children: ReactNode; + className?: string; + /** Use div instead of pre for content (needed when children include block elements like img/div). */ + asPre?: boolean; + /** Raw text to copy when copy button is clicked. When provided, shows a copy button. */ + copyContent?: string; + /** Show line numbers in a table layout (matches Fern's code block structure). */ + showLineNumbers?: boolean; + /** Pre-rendered HTML for each line when showLineNumbers is true. Lines are split by newline. */ + codeHtml?: string; +}) { + const headerLabel = title === "Output" ? "Output" : title.charAt(0).toUpperCase() + title.slice(1); + const wrapperClasses = + "fern-code fern-code-block bg-card-background border-card-border rounded-3 shadow-card-grayscale relative mb-6 mt-4 flex w-full min-w-0 max-w-full flex-col border first:mt-0"; + const preStyle = { + backgroundColor: "rgb(255, 255, 255)", + ["--shiki-dark-bg" as string]: "#212121", + color: "rgb(36, 41, 46)", + ["--shiki-dark" as string]: "#EEFFFF", + }; + + const scrollAreaContent = () => { + if (codeHtml == null) return null; + const lines = codeHtml.split("\n"); + return ( +
+ +``` +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) + +For deployment guide, see [Admin Setup](/documentation/self-managed-deployment/setup/about) in the NeMo Platform documentation. + +## Values + +The following is the complete `values.yaml` file for the NeMo Platform Helm Chart. +All configuration options are documented inline with comments. diff --git a/docs/index.md b/docs/index.mdx similarity index 58% rename from docs/index.md rename to docs/index.mdx index eae8c12333..cebc2f0af4 100644 --- a/docs/index.md +++ b/docs/index.mdx @@ -1,8 +1,10 @@ -# {{platform_name}} - +--- +title: "NeMo Platform" +description: "" +--- Make the agents you ship faster, more accurate, and safer. -{{platform_name}} brings NVIDIA NeMo libraries together under one CLI, Python SDK, and web UI. Hardening, evaluation, and tuning for the agents you put in production. +NeMo Platform brings NVIDIA NeMo libraries together under one CLI, Python SDK, and web UI. Hardening, evaluation, and tuning for the agents you put in production. ## What You Can Do @@ -17,18 +19,18 @@ Make the agents you ship faster, more accurate, and safer. The platform provides several shared capabilities to NeMo libraries. -- [Models and Inference](run-inference/about.md) for model providers, virtual +- [Models and Inference](/documentation/models-and-inference) for model providers, virtual models, and gateway calls. -- [Files](get-started/concepts/manage-files.md), [Secrets](get-started/concepts/manage-secrets.md), - [Entities](get-started/concepts/entities.md), and jobs for storing state and +- [Files](/documentation/get-started/core-concepts/manage-files), [Secrets](/documentation/get-started/core-concepts/manage-secrets), + [Entities](/documentation/get-started/core-concepts/entities), and jobs for storing state and running local work. ## Where to Go Next -- [Setup](get-started/setup.md) - install, configure providers, and run +- [Setup](/documentation/get-started) - install, configure providers, and run local services. -- [About Agents](agents/index.md) - learn the managed agent lifecycle. -- [Optimize Agents](agents/optimization.md) - improve cost, quality, and model +- [About Agents](/documentation/agents) - learn the managed agent lifecycle. +- [Optimize Agents](/documentation/agents/optimize-agents) - improve cost, quality, and model routing. -- [Secure Agents](agents/security.md) - harden agents with guardrails and data +- [Secure Agents](/documentation/agents/secure-agents) - harden agents with guardrails and data safety checks. diff --git a/docs/javascripts/api-filter.js b/docs/javascripts/api-filter.js deleted file mode 100644 index b42ea02938..0000000000 --- a/docs/javascripts/api-filter.js +++ /dev/null @@ -1,185 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// API Reference page: chip-based tag filtering for Swagger UI iframe -(function() { - function initApiFilter() { - var chips = document.querySelectorAll('.api-chip'); - var chipContainer = document.querySelector('.api-filter-chips'); - var iframe = document.querySelector('iframe.swagger-ui-iframe'); - if (!chipContainer || !iframe) return; - var hiddenTags = {}; - if (chipContainer.dataset.hiddenTags) { - chipContainer.dataset.hiddenTags.split(',').forEach(function(tag) { - tag = tag.trim(); - if (tag) hiddenTags[tag] = true; - }); - } - var hiddenTagIds = {}; - Object.keys(hiddenTags).forEach(function(tag) { - hiddenTagIds['operations-tag-' + tag.replace(/ /g, '_')] = true; - }); - var retryTimer = null; - var swaggerObserver = null; - - function tagSlug(tag) { - return tag.toLowerCase().trim().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, ''); - } - - function chipHash(chip) { - var id = chip.getAttribute('id'); - if (id) return '#' + id; - - var tag = chip.getAttribute('data-tag') || ''; - return tag ? '#tag-' + tagSlug(tag) : '#tag-all'; - } - - function tagFromHash() { - var hash = window.location.hash; - if (!hash) return ''; - - var selectedTag = ''; - var normalizedHash = decodeURIComponent(hash).toLowerCase(); - chips.forEach(function(chip) { - var tag = chip.getAttribute('data-tag') || ''; - if (chipHash(chip).toLowerCase() === normalizedHash) { - selectedTag = tag; - } - }); - - return hiddenTags[selectedTag] ? '' : selectedTag; - } - - function getDoc() { - try { return iframe.contentDocument || iframe.contentWindow.document; } - catch(e) { return null; } - } - - function applyFilter(tag) { - var doc = getDoc(); - if (!doc || !doc.querySelectorAll('.opblock-tag-section').length) return false; - if (hiddenTags[tag]) tag = ''; - var tagId = tag ? 'operations-tag-' + tag.replace(/ /g, '_') : ''; - doc.querySelectorAll('.opblock-tag-section').forEach(function(s) { - var heading = s.querySelector('.opblock-tag'); - var id = heading ? heading.getAttribute('id') : ''; - s.style.display = (!hiddenTagIds[id] && (!tag || id === tagId)) ? '' : 'none'; - }); - iframe.style.visibility = ''; - return true; - } - - function setActiveChip(tag) { - chips.forEach(function(chip) { - var isActive = (chip.getAttribute('data-tag') || '') === tag; - chip.classList.toggle('active', isActive); - }); - } - - function updateUrl(chip) { - var nextUrl = window.location.pathname + window.location.search + chipHash(chip); - if (window.location.href !== window.location.origin + nextUrl) { - window.history.pushState(null, '', nextUrl); - } - } - - function selectTag(tag, chip, shouldUpdateUrl) { - if (hiddenTags[tag]) tag = ''; - setActiveChip(tag); - if (!applyFilter(tag)) scheduleTryInit(50); - if (shouldUpdateUrl && chip) updateUrl(chip); - } - - function syncFromHash() { - selectTag(tagFromHash(), null, false); - } - - function bindChips() { - chips.forEach(function(chip) { - if (hiddenTags[chip.getAttribute('data-tag')]) { - chip.style.display = 'none'; - return; - } - if (chip.dataset.apiFilterBound) return; - chip.dataset.apiFilterBound = 'true'; - chip.addEventListener('click', function() { - selectTag(chip.getAttribute('data-tag') || '', chip, true); - }); - }); - } - - function scheduleTryInit(delayMs) { - if (retryTimer) return; - retryTimer = setTimeout(function() { - retryTimer = null; - tryInit(); - }, delayMs); - } - - function disconnectObserver() { - if (!swaggerObserver) return; - swaggerObserver.disconnect(); - swaggerObserver = null; - } - - function watchSwaggerDoc(doc) { - if (swaggerObserver || !doc || !doc.body || typeof MutationObserver === 'undefined') return; - swaggerObserver = new MutationObserver(function() { - if (doc.querySelectorAll('.opblock-tag-section').length) { - disconnectObserver(); - tryInit(); - } - }); - swaggerObserver.observe(doc.body, { childList: true, subtree: true }); - } - - function tryInit() { - var doc = getDoc(); - if (!doc || !doc.body) { - scheduleTryInit(50); - return; - } - if (!doc.querySelectorAll('.opblock-tag-section').length) { - watchSwaggerDoc(doc); - scheduleTryInit(250); - return; - } - - disconnectObserver(); - var filterBar = doc.querySelector('.filter-container'); - if (filterBar) filterBar.style.display = 'none'; - - bindChips(); - syncFromHash(); - } - - window.__apiFilterSyncFromHash = syncFromHash; - if (!window.__apiFilterHashListenerBound) { - window.__apiFilterHashListenerBound = true; - window.addEventListener('hashchange', function() { - if (window.__apiFilterSyncFromHash) window.__apiFilterSyncFromHash(); - }); - window.addEventListener('popstate', function() { - if (window.__apiFilterSyncFromHash) window.__apiFilterSyncFromHash(); - }); - } - - if (tagFromHash()) iframe.style.visibility = 'hidden'; - bindChips(); - syncFromHash(); - iframe.addEventListener('load', function() { - disconnectObserver(); - scheduleTryInit(0); - }); - scheduleTryInit(0); - } - - // MkDocs Material instant navigation re-renders pages without full reloads - if (typeof document$ !== 'undefined') { - document$.subscribe(function() { setTimeout(initApiFilter, 0); }); - } else { - document.addEventListener('DOMContentLoaded', function() { - setTimeout(initApiFilter, 0); - }); - } -})(); diff --git a/docs/notebooks/ndd_evaluator.md b/docs/notebooks/ndd_evaluator.mdx similarity index 97% rename from docs/notebooks/ndd_evaluator.md rename to docs/notebooks/ndd_evaluator.mdx index 7b7623981b..48e61d3a0a 100644 --- a/docs/notebooks/ndd_evaluator.md +++ b/docs/notebooks/ndd_evaluator.mdx @@ -1,5 +1,7 @@ -# Emergency Triage LLM Evaluation - +--- +title: "Emergency Triage LLM Evaluation" +description: "" +--- **Problem:** Accurately labeling Emergency Severity Index (ESI) levels from nurse triage notes is critical for clinical research and model development, but real-world data is often sensitive and access is restricted. Additionally, obtaining high-quality human annotations is costly and slow, making it difficult to build large, diverse datasets for robust evaluation. **Opportunity:** Synthetic data generation offers a scalable, privacy-preserving alternative. By simulating realistic triage notes and ESI labels, we can create rich datasets without exposing patient information or relying on scarce human annotators. This enables rapid iteration, benchmarking, and model improvement in a domain where data scarcity is a major bottleneck. @@ -37,7 +39,7 @@ Tip: Run the cells in order. You can re-run preview/generation to explore differ ## **Step 1**: 🎨 NeMo Data Designer -First, create a client for interacting with {{platform_name}}. +First, create a client for interacting with NeMo Platform. ```python import os @@ -52,7 +54,7 @@ client = NeMoPlatform(base_url=BASE_URL, workspace=WORKSPACE) ``` Create a Secret and Model Provider to use models hosted on `build.nvidia.com`. -Note: these resources may already exist if you are running {{platform_name}} Quickstart. +Note: these resources may already exist if you are running NeMo Platform Quickstart. ```python SECRET_NAME = "nvidia-build-api-key" @@ -224,7 +226,7 @@ config_builder.add_column( ## 🦜 LLM-generated columns ```python -{% raw %} + # LLM-generated triage note config_builder.add_column( dd.LLMTextColumnConfig( @@ -247,13 +249,13 @@ config_builder.add_column( model_alias=MODEL_ALIAS_GENERATOR, ) ) -{% endraw %} + ``` ## ⚖️ LLM-as-a-Judge Evaluation Step ```python -{% raw %} + # Rubric: clinical coherence clinical_coherence_rubric = dd.Score( name="Clinical Coherence", @@ -302,7 +304,7 @@ config_builder.add_column( scores=[clinical_coherence_rubric, esi_level_complexity_rubric], ) ) -{% endraw %} + ``` ### 🧪 Generate & Preview @@ -490,7 +492,7 @@ for level, df in df_complexities.items(): This section defines the evaluation configuration used to assess model performance on triage note classification using a custom evaluator. ```python -{% raw %} + EVALUATOR_CONFIG = { "prompt_template": { "messages": [ @@ -515,7 +517,7 @@ EVALUATOR_CONFIG = { } } -{% endraw %} + ``` ## 🔍 Model evaluation loop and configuration @@ -529,7 +531,7 @@ The models evaluated are: For *each* complexity level, the accuracy score for each model is printed, allowing for side-by-side evaluation of how each model performs at every complexity. ```python -{% raw %} + import time import pandas as pd from typing import Any @@ -637,7 +639,7 @@ for complexity in COMPLEXITIES: results_dict[spec['name']][complexity.capitalize()] = 100 * accuracy # Store as percentage print(f" --> DONE: {spec['name']}, {complexity.capitalize()} (Accuracy: {100*accuracy:.2f}%)\n") -{% endraw %} + ``` ```python @@ -650,6 +652,6 @@ display(df_results.style.format("{:.2f}")) ## Next Steps -- Explore more [Evaluator metrics](../evaluator/metrics/index.md) for additional evaluation scenarios. -- Learn more [Data Designer workflows](../data-designer/tutorials/index.md) for advanced synthetic generation pipelines. -- Review [Files service management](../get-started/concepts/manage-files.md) for organizing datasets and artifacts. +- Explore more [Evaluator metrics](/documentation/evaluate-models/metrics) for additional evaluation scenarios. +- Learn more [Data Designer workflows](/documentation/design-synthetic-data/tutorials) for advanced synthetic generation pipelines. +- Review [Files service management](/documentation/get-started/core-concepts/manage-files) for organizing datasets and artifacts. diff --git a/docs/pysdk/client/index.md b/docs/pysdk/client/index.mdx similarity index 93% rename from docs/pysdk/client/index.md rename to docs/pysdk/client/index.mdx index 404403934c..4e4fae8658 100644 --- a/docs/pysdk/client/index.md +++ b/docs/pysdk/client/index.mdx @@ -1,6 +1,8 @@ -# Client APIs - -The following reference provides detailed documentation for the synchronous and asynchronous clients of the {{platform_name}} Python SDK. +--- +title: "Client APIs" +description: "" +--- +The following reference provides detailed documentation for the synchronous and asynchronous clients of the NeMo Platform Python SDK. ## Synchronous Client @@ -39,7 +41,7 @@ NeMoPlatform( | Parameter | Description | | --- | --- | | `workspace` | Workspace name used by workspace-scoped routes. You can set it on the client or pass it to individual methods that accept a workspace argument. | -| `base_url` | Base URL for the {{platform_name}} API. If omitted, the client reads it from the active CLI context. | +| `base_url` | Base URL for the NeMo Platform API. If omitted, the client reads it from the active CLI context. | | `inference_base_url` | Optional override for inference gateway requests. Defaults to `base_url`. | | `config_path` | Path to the CLI config file. Defaults to `~/.config/nmp/config.yaml`. | | `context_name` | Named CLI context to read from the config file. | @@ -145,8 +147,8 @@ for SDK-level errors. ## Client Attributes -The {{platform_name}} clients provide access to API resources through the following attributes. -For endpoint details, see the [REST API Reference](../../api/index.md). +The NeMo Platform clients provide access to API resources through the following attributes. +For endpoint details, see the [REST API Reference](/documentation/reference/api-reference). ### Organization diff --git a/docs/pysdk/index.md b/docs/pysdk/index.md deleted file mode 100644 index 27eddca53a..0000000000 --- a/docs/pysdk/index.md +++ /dev/null @@ -1,25 +0,0 @@ - -# {{platform_name}} Python SDK Reference - -The [{{platform_name}} Python SDK](https://pypi.org/project/nemo-platform/) is a library for building and deploying AI models, abstracting the underlying infrastructure and providing a high-level interface for the {{platform_name}} APIs. - -## Installation - -Install the {{platform_name}} Python SDK using `pip`: - -```bash -pip install nemo-platform[all] -``` - -!!! note "This project will download and install additional third-party open source software projects. Review the license terms of these open source projects before use." - If you previously installed the `nemo-microservices` package, uninstall it first to avoid conflicts: - - ```bash - pip uninstall nemo-microservices - ``` ---8<-- "sdk/python/overrides/nemo-platform/README/03_usage.md" - -## Next Steps - -- Read more about connecting with the [client APIs](./client/index.md). -- Browse the [REST API Reference](../api/index.md). diff --git a/docs/pysdk/index.mdx b/docs/pysdk/index.mdx new file mode 100644 index 0000000000..af250136d5 --- /dev/null +++ b/docs/pysdk/index.mdx @@ -0,0 +1,31 @@ +--- +title: "NeMo Platform Python SDK Reference" +description: "" +--- + + +The [NeMo Platform Python SDK](https://pypi.org/project/nemo-platform/) is a library for building and deploying AI models, abstracting the underlying infrastructure and providing a high-level interface for the NeMo Platform APIs. + +## Installation + +Install the NeMo Platform Python SDK using `pip`: + +```bash +pip install nemo-platform[all] +``` + + + +This project will download and install additional third-party open source software projects. Review the license terms of these open source projects before use. + + + If you previously installed the `nemo-microservices` package, uninstall it first to avoid conflicts: + + ```bash + pip uninstall nemo-microservices + ``` + +## Next Steps + +- Read more about connecting with the [client APIs](/documentation/reference/python-sdk/client-ap-is). +- Browse the [REST API Reference](/documentation/reference/api-reference). diff --git a/docs/requirements-mkdocs.txt b/docs/requirements-mkdocs.txt deleted file mode 100644 index ddbfa947e5..0000000000 --- a/docs/requirements-mkdocs.txt +++ /dev/null @@ -1,41 +0,0 @@ -# MkDocs and core theme -mkdocs>=1.6.0 -mkdocs-material>=9.5.0 - -# Navigation -mkdocs-awesome-pages-plugin>=2.9.0 - -# Python API reference -mkdocstrings[python]>=0.25.0 -mkdocs-autorefs>=1.0.0 - -# Jupyter notebook rendering -mkdocs-jupyter>=0.24.0 -jupyter>=1.0.0 -nbconvert>=7.0.0 - -# Macros / substitutions -mkdocs-macros-plugin>=1.0.0 - -# Redirects -mkdocs-redirects>=1.2.0 - -# Versioning (replaces pydata version switcher) -mike>=2.0.0 - -# OpenAPI / Swagger -mkdocs-swagger-ui-tag>=0.6.0 - -# Live reload -watchdog>=4.0.0 - -# Code block formatting -ruff==0.15.7 - -# SDK dependencies (needed for mkdocstrings to import the SDK) -httpx>=0.23.0 -pydantic>=1.9.0,<3 -typing-extensions>=4.10 -anyio>=3.5.0 -distro>=1.7.0 -sniffio diff --git a/docs/requirements.md b/docs/requirements.mdx similarity index 78% rename from docs/requirements.md rename to docs/requirements.mdx index f12d2c1c83..9c87b7118e 100644 --- a/docs/requirements.md +++ b/docs/requirements.mdx @@ -1,8 +1,10 @@ -# Hardware and Software Requirements for {{platform_name}} +--- +title: "Hardware and Software Requirements for NeMo Platform" +description: "" +--- +This page lists the requirements for the OSS local-install path for NeMo Platform 0.1.0. For the full compatibility table, see the [Support Matrix](/documentation/reference/support-matrix). -This page lists the requirements for the OSS local-install path for {{platform_name}} {{ release }}. For the full compatibility table, see the [Support Matrix](support-matrix.md). - -The OSS {{ release }} documentation is scoped to local setup with the Python package and `nemo setup`. Docker Compose, Helm, Kubernetes, and OpenShift deployment guides are not part of this release scope. +The OSS 0.1.0 documentation is scoped to local setup with the Python package and `nemo setup`. Docker Compose, Helm, Kubernetes, and OpenShift deployment guides are not part of this release scope. ## Local Setup Requirements @@ -10,7 +12,7 @@ The OSS {{ release }} documentation is scoped to local setup with the Python pac |-----------|-------------|-------| | Python | 3.11, 3.12, or 3.13 (`>=3.11,<3.14`) | Use an isolated virtual environment. Python 3.14 and later are not part of the OSS local-install support range. | | Package installer | `uv` recommended; `pip` supported | The one-line installer sets up `uv` automatically when needed. | -| Operating system | Recent Linux or macOS release | See the [Support Matrix](support-matrix.md) for the supported OS list. | +| Operating system | Recent Linux or macOS release | See the [Support Matrix](/documentation/reference/support-matrix) for the supported OS list. | | Memory | 8 GB RAM minimum | 16 GB or more is recommended for larger local workflows. | | Disk space | 16 GB free disk space minimum | Additional space is needed for datasets, job outputs, and local model artifacts. | | Network | Outbound HTTPS access | Required for package installation and hosted model-provider APIs. | @@ -36,4 +38,4 @@ nemo --help curl -s http://localhost:8080/health/ready ``` -For setup instructions, see [Setup](get-started/setup.md). +For setup instructions, see [Setup](/documentation/get-started). diff --git a/docs/run-inference/about.md b/docs/run-inference/about.mdx similarity index 55% rename from docs/run-inference/about.md rename to docs/run-inference/about.mdx index 45332831c0..1939a6bd05 100644 --- a/docs/run-inference/about.md +++ b/docs/run-inference/about.mdx @@ -1,9 +1,10 @@ - - +--- +title: "About Models and Inference" +description: "" +--- -# About Models and Inference -The {{platform_name}} provides APIs for registering external model providers and routing inference requests through a unified gateway. +The NeMo Platform provides APIs for registering external model providers and routing inference requests through a unified gateway. ```mermaid flowchart LR @@ -30,7 +31,7 @@ Models service manages model entities and model providers. Model providers connect the platform to external inference APIs such as [NVIDIA Build](https://build.nvidia.com/) or OpenAI. The workflow is: -1. **Store the API key** as a [secret](../get-started/concepts/manage-secrets.md) in the platform +1. **Store the API key** as a [secret](/documentation/get-started/core-concepts/manage-secrets) in the platform 2. **Create a model provider** pointing to the external API with the secret reference 3. **Route inference** through the gateway using provider or model entity routing @@ -42,8 +43,7 @@ flowchart LR GW --> Client ``` ---8<-- "_snippets/nvidia-build-model-provider.md" - + ### Add External Providers @@ -57,138 +57,156 @@ The example below demonstrates how to recreate it in your own workspace. For disambiguation purposes, this example names the manually-created version `my-nvidia-build`. -=== "CLI" + - ```bash - # Store API key - echo "$NVIDIA_API_KEY" | nemo secrets create "nvidia-api-key" --from-file - + - # Create provider - nemo inference providers create "my-nvidia-build" \ - --host-url "https://integrate.api.nvidia.com" \ - --api-key-secret-name "nvidia-api-key" - - nemo wait inference provider my-nvidia-build +```bash +# Store API key +echo "$NVIDIA_API_KEY" | nemo secrets create "nvidia-api-key" --from-file - - # Test using interactive chat - nemo chat nvidia/llama-3.3-nemotron-super-49b-v1 'Hello!' \ - --provider my-nvidia-build - ``` +# Create provider +nemo inference providers create "my-nvidia-build" \ + --host-url "https://integrate.api.nvidia.com" \ + --api-key-secret-name "nvidia-api-key" -=== "Python SDK" +nemo wait inference provider my-nvidia-build - ```python - # Store API key - client.secrets.create(name="nvidia-api-key", data=os.environ["NVIDIA_API_KEY"]) +# Test using interactive chat +nemo chat nvidia/llama-3.3-nemotron-super-49b-v1 'Hello!' \ + --provider my-nvidia-build +``` - # Create provider - provider = client.inference.providers.create( - name="my-nvidia-build", - host_url="https://integrate.api.nvidia.com", - api_key_secret_name="nvidia-api-key", - ) + + - client.models.wait_for_provider("my-nvidia-build") +```python +# Store API key +client.secrets.create(name="nvidia-api-key", data=os.environ["NVIDIA_API_KEY"]) + +# Create provider +provider = client.inference.providers.create( + name="my-nvidia-build", + host_url="https://integrate.api.nvidia.com", + api_key_secret_name="nvidia-api-key", +) + +client.models.wait_for_provider("my-nvidia-build") + +# Use provider routing +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="my-nvidia-build", + body={ + "model": "meta/llama-3.1-8b-instruct", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` - # Use provider routing - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="my-nvidia-build", - body={ - "model": "meta/llama-3.1-8b-instruct", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` + + #### OpenAI -=== "CLI" - - ```bash - # Store API key - echo "$OPENAI_API_KEY" | nemo secrets create "openai-api-key" --from-file - + - # Create provider with enabled models - nemo inference providers create "openai" \ - --host-url "https://api.openai.com/v1" \ - --api-key-secret-name "openai-api-key" \ - --enabled-models "gpt-4" \ - --enabled-models "gpt-3.5-turbo" + - nemo wait inference provider openai +```bash +# Store API key +echo "$OPENAI_API_KEY" | nemo secrets create "openai-api-key" --from-file - - # Test using interactive chat - nemo chat gpt-4 'Hello!' \ - --provider openai - ``` +# Create provider with enabled models +nemo inference providers create "openai" \ + --host-url "https://api.openai.com/v1" \ + --api-key-secret-name "openai-api-key" \ + --enabled-models "gpt-4" \ + --enabled-models "gpt-3.5-turbo" -=== "Python SDK" +nemo wait inference provider openai - ```python - client.secrets.create(name="openai-api-key", data=os.environ["OPENAI_API_KEY"]) +# Test using interactive chat +nemo chat gpt-4 'Hello!' \ + --provider openai +``` - provider = client.inference.providers.create( - name="openai", - host_url="https://api.openai.com/v1", - api_key_secret_name="openai-api-key", - enabled_models=["gpt-4", "gpt-3.5-turbo"], - ) + + - client.models.wait_for_provider("openai") +```python +client.secrets.create(name="openai-api-key", data=os.environ["OPENAI_API_KEY"]) + +provider = client.inference.providers.create( + name="openai", + host_url="https://api.openai.com/v1", + api_key_secret_name="openai-api-key", + enabled_models=["gpt-4", "gpt-3.5-turbo"], +) + +client.models.wait_for_provider("openai") + +# Use provider routing +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="openai", + body={ + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` - # Use provider routing - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="openai", - body={ - "model": "gpt-4", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` + + #### Anthropic Anthropic's `/v1/messages` API expects the API key in an `X-Api-Key:` header (not `Authorization: Bearer`) and requires an `anthropic-version` header on every request. Use `--auth-header-format` (Jinja2 template, must contain exactly one `{{ auth_secret }}` variable) to override the default `Authorization: Bearer {{ auth_secret }}` and pass the API-version pin via `--default-extra-headers`. Without these, Anthropic rejects every request with 401. -=== "CLI" + - ```bash - # Store API key - echo "$ANTHROPIC_API_KEY" | nemo secrets create "anthropic-api-key" --from-file - + - # Create provider — override the default Bearer header and pin the API version - nemo inference providers create "anthropic" \ - --host-url "https://api.anthropic.com" \ - --api-key-secret-name "anthropic-api-key" \ - --auth-header-format "X-Api-Key: {{ auth_secret }}" \ - --default-extra-headers '{"anthropic-version": "2023-06-01"}' +```bash +# Store API key +echo "$ANTHROPIC_API_KEY" | nemo secrets create "anthropic-api-key" --from-file - - nemo wait inference provider anthropic - ``` +# Create provider — override the default Bearer header and pin the API version +nemo inference providers create "anthropic" \ + --host-url "https://api.anthropic.com" \ + --api-key-secret-name "anthropic-api-key" \ + --auth-header-format "X-Api-Key: {{ auth_secret }}" \ + --default-extra-headers '{"anthropic-version": "2023-06-01"}' -=== "Python SDK" +nemo wait inference provider anthropic +``` - ```python - # Store API key - client.secrets.create(name="anthropic-api-key", data=os.environ["ANTHROPIC_API_KEY"]) + + - # Create provider — override the default Bearer header and pin the API version - provider = client.inference.providers.create( - name="anthropic", - host_url="https://api.anthropic.com", - api_key_secret_name="anthropic-api-key", - auth_header_format="X-Api-Key: {{ auth_secret }}", - default_extra_headers={"anthropic-version": "2023-06-01"}, - ) +```python +# Store API key +client.secrets.create(name="anthropic-api-key", data=os.environ["ANTHROPIC_API_KEY"]) + +# Create provider — override the default Bearer header and pin the API version +provider = client.inference.providers.create( + name="anthropic", + host_url="https://api.anthropic.com", + api_key_secret_name="anthropic-api-key", + auth_header_format="X-Api-Key: {{ auth_secret }}", + default_extra_headers={"anthropic-version": "2023-06-01"}, +) + +client.models.wait_for_provider("anthropic") +``` - client.models.wait_for_provider("anthropic") - ``` + + `{{ auth_secret }}` is substituted with the resolved secret value at request time. --- @@ -228,8 +246,7 @@ without hand-assembling the path. Add `--provider ` or Set up the CLI or Python SDK first: ---8<-- "_snippets/tutorials/cli-sdk-setup.md" - + The SDK provides convenience methods for OpenAI compatibility: ```python @@ -277,4 +294,4 @@ working as designed — use `/health/ready` instead. ## API Reference -For complete API details, refer to the [Inference Gateway API Reference](../api/index.md#tag-inference-gateway) and [SDK Reference](../pysdk/index.md). +For complete API details, refer to the [Inference Gateway API Reference](/documentation/reference/api-reference) and [SDK Reference](/documentation/reference/python-sdk). diff --git a/docs/run-inference/tutorials/deploy-models.md b/docs/run-inference/tutorials/deploy-models.md deleted file mode 100644 index f73d322fa5..0000000000 --- a/docs/run-inference/tutorials/deploy-models.md +++ /dev/null @@ -1,933 +0,0 @@ - - - - -# Deploy Models - -Deploy models from NGC or HuggingFace. Register external providers like OpenAI or NVIDIA Build. - -!!! note "Resource names for deployments, deployment configs, and providers must contain only letters (a-z, A-Z), digits (0-9), underscores, hyphens, and dots. For example: `llama-3.1-8b`, `my-custom-model`, `qwen-fs-config`." - ---8<-- "_snippets/tutorials/cli-sdk-setup.md" - ---- - -## Add External Providers - -Register external inference APIs like NVIDIA Build or OpenAI. - -### NVIDIA Build - -By default, the platform pre-configures an external provider for NVIDIA Build named `nvidia-build` in the `system` workspace. -The example below demonstrates how to recreate it in your own workspace. -For disambiguation purposes, this example names the manually-created version `my-nvidia-build`. - - -=== "CLI" - - ```bash - # Store API key - echo "$NVIDIA_API_KEY" | nemo secrets create "nvidia-api-key" --from-file - - - # Create provider - nemo inference providers create "my-nvidia-build" \ - --host-url "https://integrate.api.nvidia.com" \ - --api-key-secret-name "nvidia-api-key" - - nemo wait inference provider my-nvidia-build - - # Test using interactive chat - nemo chat nvidia/llama-3.3-nemotron-super-49b-v1 'Hello!' \ - --provider my-nvidia-build - ``` - -=== "Python SDK" - - ```python - # Store API key - client.secrets.create(name="nvidia-api-key", value=os.environ["NVIDIA_API_KEY"]) - - # Create provider - provider = client.inference.providers.create( - name="my-nvidia-build", - host_url="https://integrate.api.nvidia.com", - api_key_secret_name="nvidia-api-key", - ) - - client.models.wait_for_provider("my-nvidia-build") - - # Use provider routing - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="my-nvidia-build", - body={ - "model": "meta/llama-3.1-8b-instruct", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` - -### OpenAI - - -=== "CLI" - - ```bash - # Store API key - echo "$OPENAI_API_KEY" | nemo secrets create "openai-api-key" --from-file - - - # Create provider with enabled models - nemo inference providers create "openai" \ - --host-url "https://api.openai.com/v1" \ - --api-key-secret-name "openai-api-key" \ - --enabled-models "gpt-4" \ - --enabled-models "gpt-3.5-turbo" - - nemo wait inference provider openai - - # Test using interactive chat - nemo chat gpt-4 'Hello!' \ - --provider openai - ``` - -=== "Python SDK" - - ```python - client.secrets.create(name="openai-api-key", value=os.environ["OPENAI_API_KEY"]) - - provider = client.inference.providers.create( - name="openai", - host_url="https://api.openai.com/v1", - api_key_secret_name="openai-api-key", - enabled_models=["gpt-4", "gpt-3.5-turbo"], - ) - - client.models.wait_for_provider("openai") - - # Use provider routing - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="openai", - body={ - "model": "gpt-4", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` - -### Anthropic - -Anthropic's `/v1/messages` API expects the API key in an `X-Api-Key:` header (not `Authorization: Bearer`) and requires an `anthropic-version` header on every request. Use `--auth-header-format` (Jinja2 template, must contain exactly one `{{ auth_secret }}` variable) to override the default `Authorization: Bearer {{ auth_secret }}` and pass the API-version pin via `--default-extra-headers`. Without these, Anthropic rejects every request with 401. - -=== "CLI" - - ```bash - # Store API key - echo "$ANTHROPIC_API_KEY" | nemo secrets create "anthropic-api-key" --from-file - - - # Create provider — override the default Bearer header and pin the API version - nemo inference providers create "anthropic" \ - --host-url "https://api.anthropic.com" \ - --api-key-secret-name "anthropic-api-key" \ - --auth-header-format "X-Api-Key: {{ auth_secret }}" \ - --default-extra-headers '{"anthropic-version": "2023-06-01"}' - - nemo wait inference provider anthropic - ``` - -=== "Python SDK" - - ```python - # Store API key - client.secrets.create(name="anthropic-api-key", data=os.environ["ANTHROPIC_API_KEY"]) - - # Create provider — override the default Bearer header and pin the API version - provider = client.inference.providers.create( - name="anthropic", - host_url="https://api.anthropic.com", - api_key_secret_name="anthropic-api-key", - auth_header_format="X-Api-Key: {{ auth_secret }}", - default_extra_headers={"anthropic-version": "2023-06-01"}, - ) - - client.models.wait_for_provider("anthropic") - ``` - -`{{ auth_secret }}` is substituted with the resolved secret value at request time. - ---- - -## Deploy from NGC - -Deploy pre-built NIM containers from NGC. - -### Deploy Llama 3.2 1B - - -=== "CLI" - - ```bash - nemo inference deployment-configs create \ - --name "llama-3-2-1b-config" \ - --nim-deployment '{ - "gpu": 1, - "image_name": "nvcr.io/nim/meta/llama-3.2-1b-instruct", - "image_tag": "1.8.6", - "model_name": "meta/llama-3.2-1b-instruct" - }' - - nemo inference deployments create \ - --name "llama-3-2-1b-deployment" \ - --config "llama-3-2-1b-config" - - nemo wait inference deployment llama-3-2-1b-deployment - - nemo chat meta/llama-3.2-1b-instruct 'Hello!' \ - --provider llama-3-2-1b-deployment \ - --max-tokens 100 - ``` - -=== "Python SDK" - - ```python - config = client.inference.deployment_configs.create( - name="llama-3-2-1b-config", - nim_deployment={ - "gpu": 1, - "image_name": "nvcr.io/nim/meta/llama-3.2-1b-instruct", - "image_tag": "1.8.6", - "model_name": "meta/llama-3.2-1b-instruct", - }, - ) - - deployment = client.inference.deployments.create( - name="llama-3-2-1b-deployment", config="llama-3-2-1b-config" - ) - - client.models.wait_for_status( - deployment_name="llama-3-2-1b-deployment", desired_status="READY" - ) - - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="llama-3-2-1b-deployment", - body={ - "model": "meta/llama-3.2-1b-instruct", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` - -### Deploy NemoGuard JailbreakDetect - -Deploy classification NIMs like NemoGuard for content safety. Uses the `/v1/classify` endpoint instead of chat completions. - - -=== "CLI" - - ```bash - nemo inference deployment-configs create \ - --name "nemoguard-jailbreak-config" \ - --nim-deployment '{ - "gpu": 1, - "image_name": "nvcr.io/nim/nvidia/nemoguard-jailbreak-detect", - "image_tag": "1.10.1" - }' - - nemo inference deployments create \ - --name "nemoguard-jailbreak-deployment" \ - --config "nemoguard-jailbreak-config" - - nemo wait inference deployment nemoguard-jailbreak-deployment - - nemo inference gateway provider post v1/classify \ - --name "nemoguard-jailbreak-deployment" \ - --body '{"input": "Tell me about vacation spots in Hawaii."}' - ``` - -=== "Python SDK" - - ```python - config = client.inference.deployment_configs.create( - name="nemoguard-jailbreak-config", - nim_deployment={ - "gpu": 1, - "image_name": "nvcr.io/nim/nvidia/nemoguard-jailbreak-detect", - "image_tag": "1.10.1", - }, - ) - - deployment = client.inference.deployments.create( - name="nemoguard-jailbreak-deployment", config="nemoguard-jailbreak-config" - ) - - client.models.wait_for_status( - deployment_name="nemoguard-jailbreak-deployment", desired_status="READY" - ) - - response = client.inference.gateway.provider.post( - "v1/classify", - name="nemoguard-jailbreak-deployment", - body={"input": "Tell me about vacation spots in Hawaii."}, - ) - ``` - ---- - -## Deploy from HuggingFace - -!!! warning "HuggingFace deployments use the Multi-LLM NIM (`nvcr.io/nim/nvidia/llm-nim:1.13.1`) by default, which only supports specific model architectures. Check the [supported architectures list](https://docs.nvidia.com/nim/large-language-models/1.13.0/supported-architectures.html) before deploying. If your model architecture is not listed, you will need a model-specific NIM image — see [Deploy from NGC](#deploy-from-ngc) for that approach." - -You can register a HuggingFace model through the Files service. This creates a fileset that acts as a proxy. The Files service handles authentication and caches the weights on first download, so subsequent deployments start faster. - - -=== "CLI" - - ```bash - # (Optional) Create a HuggingFace token secret for private models. - # Public models like Qwen do not require a token. - echo "$HF_TOKEN" | nemo secrets create "hf-token-secret" --from-file - - - # Create a fileset pointing to the HuggingFace model. - # "token_secret" is optional — only needed for private/gated models. - nemo files filesets create "qwen-2-5-1-5b" \ - --storage '{ - "type": "huggingface", - "repo_id": "Qwen/Qwen2.5-1.5B-Instruct", - "repo_type": "model", - "token_secret": "hf-token-secret" - }' - - # Register a model entity referencing the fileset - nemo models create "qwen-2-5-1-5b" \ - --fileset "default/qwen-2-5-1-5b" - - # Create deployment config pointing to the model entity - nemo inference deployment-configs create "qwen-fs-config" \ - --nim-deployment '{ - "model_namespace": "default", - "model_name": "qwen-2-5-1-5b", - "gpu": 1 - }' - - nemo inference deployments create "qwen-fs-deployment" \ - --config "qwen-fs-config" - - nemo wait inference deployment qwen-fs-deployment - - nemo chat default/qwen-2-5-1-5b 'Hello!' \ - --provider qwen-fs-deployment \ - --max-tokens 100 - ``` - -=== "Python SDK" - - ```python - # (Optional) Create a HuggingFace token secret for private models. - # Public models like Qwen do not require a token. - client.secrets.create(name="hf-token-secret", value=os.environ["HF_TOKEN"]) - - # Create a fileset pointing to the HuggingFace model. - # "token_secret" is optional — only needed for private/gated models. - client.files.filesets.create( - name="qwen-2-5-1-5b", - storage={ - "type": "huggingface", - "repo_id": "Qwen/Qwen2.5-1.5B-Instruct", - "repo_type": "model", - "token_secret": "hf-token-secret", - }, - ) - - # Register a model entity referencing the fileset - client.models.create(name="qwen-2-5-1-5b", fileset="default/qwen-2-5-1-5b") - - # Create deployment config pointing to the model entity - config = client.inference.deployment_configs.create( - name="qwen-fs-config", - nim_deployment={ - "model_namespace": "default", - "model_name": "qwen-2-5-1-5b", - "gpu": 1, - }, - ) - - # Deploy — no hf_token_secret_name needed - deployment = client.inference.deployments.create( - name="qwen-fs-deployment", config="qwen-fs-config" - ) - - client.models.wait_for_status( - deployment_name="qwen-fs-deployment", desired_status="READY" - ) - - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="qwen-fs-deployment", - body={ - "model": "default/qwen-2-5-1-5b", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` - -!!! tip - - The `fileset` format is `/`. This tells the deployment system to pull weights from the Files service, which proxies the download from HuggingFace using the fileset `token_secret`. For public models like Qwen, the `token_secret` on the fileset is optional. - ---- - - -## Deployment Cleanup - - -=== "CLI" - - ```bash - # Note: Deleting the deployment will free up its GPU(s) when complete - nemo inference deployments delete - nemo wait inference deployment --status DELETED - nemo inference deployment-configs delete - - # For external providers - nemo inference providers delete - nemo secrets delete - ``` - -=== "Python SDK" - - ```python - # Note: Deleting the deployment will free up its GPU(s) when complete - client.inference.deployments.delete(name="") - client.models.wait_for_status( - deployment_name="", desired_status="DELETED" - ) - client.inference.deployment_configs.delete(name="") - - # For external providers - client.inference.providers.delete(name="") - client.secrets.delete(name="") - ``` - ---- - -## Multi-GPU Deployments - -For larger models requiring multiple GPUs, parallelism configuration depends on the NIM type. - -### Parallelism Strategies - -- **Tensor Parallel (TP)**: Splits model layers across GPUs → best for latency -- **Pipeline Parallel (PP)**: Splits model depth across GPUs → best for throughput -- **Formula**: `gpu` = `tp_size` × `pp_size` - -### Model-Specific NIMs - -Model-specific NIMs (for example, `nvcr.io/nim/meta/llama-3.1-70b-instruct`) have TP/PP settings derived from manifest profiles in the container. Configure enough GPUs and the NIM selects the appropriate profile automatically. - -### Multi-LLM NIM - -The multi-LLM NIM (`nvcr.io/nim/nvidia/llm-nim:1.13.1`) requires explicit parallelism configuration via environment variables (`NIM_TENSOR_PARALLEL_SIZE`, `NIM_PIPELINE_PARALLEL_SIZE`). By default, it uses all GPUs for tensor parallelism (TP=gpu, PP=1). - -This example deploys [Qwen2.5-14B-Instruct](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct) across 2 GPUs using tensor parallelism. - - -=== "CLI" - - ```bash - # Create a fileset pointing to the HuggingFace model - # Qwen models are public — token_secret is optional - nemo files filesets create \ - --name "qwen-2-5-14b" \ - --storage '{ - "type": "huggingface", - "repo_id": "Qwen/Qwen2.5-14B-Instruct", - "repo_type": "model" - }' - - # Register a model entity referencing the fileset - nemo models create \ - --name "qwen-2-5-14b" \ - --fileset "default/qwen-2-5-14b" - - # Create deployment config with 2 GPUs (TP=2 by default) - nemo inference deployment-configs create \ - --name "qwen-14b-config" \ - --nim-deployment '{ - "model_name": "default/qwen-2-5-14b", - "gpu": 2 - }' - - # Deploy - nemo inference deployments create \ - --name "qwen-14b-deployment" \ - --config "qwen-14b-config" - - nemo wait inference deployment qwen-14b-deployment - - nemo chat default/qwen-2-5-14b 'Hello!' \ - --max-tokens 100 - ``` - -=== "Python SDK" - - ```python - # Create a fileset pointing to the HuggingFace model - # Qwen models are public — token_secret is optional - client.files.filesets.create( - name="qwen-2-5-14b", - storage={ - "type": "huggingface", - "repo_id": "Qwen/Qwen2.5-14B-Instruct", - "repo_type": "model", - }, - ) - - # Register a model entity referencing the fileset - client.models.create(name="qwen-2-5-14b", fileset="default/qwen-2-5-14b") - - # Create deployment config with 2 GPUs (TP=2 by default) - config = client.inference.deployment_configs.create( - name="qwen-14b-config", - nim_deployment={"model_name": "default/qwen-2-5-14b", "gpu": 2}, - ) - - # Deploy - deployment = client.inference.deployments.create( - name="qwen-14b-deployment", config="qwen-14b-config" - ) - - client.models.wait_for_status( - deployment_name="qwen-14b-deployment", desired_status="READY" - ) - - response = client.inference.gateway.model.post( - "v1/chat/completions", - name="qwen-2-5-14b", - body={ - "model": "default/qwen-2-5-14b", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - # NIM sets NIM_TENSOR_PARALLEL_SIZE=2 automatically - ``` - -### Custom Parallelism Configuration - -For larger models requiring more GPUs, you can configure specific TP/PP splits using `additional_envs`. The formula is: `gpu` = `NIM_TENSOR_PARALLEL_SIZE` × `NIM_PIPELINE_PARALLEL_SIZE`. - - -=== "CLI" - - ```bash - nemo inference deployment-configs create \ - --name "multi-gpu-custom-config" \ - --nim-deployment '{ - "model_name": "default/qwen-2-5-14b", - "gpu": 4, - "additional_envs": { - "NIM_TENSOR_PARALLEL_SIZE": "2", - "NIM_PIPELINE_PARALLEL_SIZE": "2" - } - }' - ``` - -=== "Python SDK" - - ```python - config = client.inference.deployment_configs.create( - name="multi-gpu-custom-config", - nim_deployment={ - "model_name": "default/qwen-2-5-14b", - "gpu": 4, - "additional_envs": { - "NIM_TENSOR_PARALLEL_SIZE": "2", - "NIM_PIPELINE_PARALLEL_SIZE": "2", - }, - }, - ) - ``` - -!!! tip - **Choosing Parallelism Strategy** - - - **TP=8, PP=1** (default): Lowest latency, best for real-time applications - - **TP=4, PP=2**: Balanced latency and throughput - - **TP=2, PP=4**: Highest throughput, best for batch processing - - For custom models, match deployment parallelism to training parallelism for optimal performance. - ---- - - -## Chat Templates and Tool Calling - -Configure custom chat templates and tool calling for NIM deployments. These settings control how the model formats chat messages and handles function/tool calling. - -For more information on chat templates, see the [Hugging Face chat templating guide](https://huggingface.co/docs/transformers/en/chat_templating) and the [NeMo chat templates documentation](https://docs.nvidia.com/nemo/rl/latest/design-docs/chat-datasets.html#chat-templates). - -!!! warning - **Security consideration:** Chat templates are Jinja2 programs that execute on every inference call. While NIM uses a sandboxed Jinja2 environment (mitigating arbitrary code execution), a malicious or misconfigured template can still alter model behavior — for example, by injecting hidden instructions, rewriting messages, or degrading output quality. Grant chat template permissions only to trusted users, and review templates before deploying to production. See [Inference-Time Backdoors via Chat Templates (IEEE S&P 2026)](https://arxiv.org/abs/2602.04653) for further background. - -### Configuration Sources - -There are two ways to configure chat templates and tool calling, with deployment-level settings taking highest priority: - -| Source | Priority | When to Use | -|--------|----------|-------------| -| **Fileset `metadata.model.tool_calling`** | Base | Set once per model — applies to all deployments using that model | -| **Deployment config (`nim_deployment`)** | Override | Per-deployment overrides — useful for A/B testing or deployment-specific behavior | - -When both are set, deployment config values override fileset values. - -### How It Works - -1. User sets `chat_template`, `tool_call_parser`, `tool_call_plugin`, and/or `auto_tool_choice` on the fileset via `metadata.model.tool_calling` -2. The model-spec background task reads the fileset's `metadata.model.tool_calling` and writes them into the model entity's `spec` -3. At deployment time, the platform reads `model_entity.spec` and any deployment-level overrides to set NIM environment variables - -```mermaid -flowchart LR - FS["Fileset
metadata.model.tool_calling"] -->|model-spec task| ME["Model Entity
spec"] - ME --> ENV["NIM Container
env vars"] - DC["Deployment Config
nim_deployment"] -->|override| ENV -``` - -### Option 1: Set via Fileset (Recommended) - -Set `chat_template` and tool calling configuration with fileset `metadata.model.tool_calling`. The platform automatically propagates these into the model entity spec when the model-spec background task runs. - - -=== "CLI" - - ```bash -{% raw %} - # Create fileset with chat template and tool calling config via metadata - nemo files filesets create \ - --name "llama-3-2-1b-tool" \ - --storage '{ - "type": "huggingface", - "repo_id": "meta-llama/Llama-3.2-1B-Instruct", - "repo_type": "model" - }' \ - --metadata '{ - "model": { - "tool_calling": { - "chat_template": "{%- for message in messages %}{%- set content = '\''<{{ start_header_id }}>'\'' + message['\''role'\''] + '\''<{{ end_header_id }}>\n\n'\'' + message['\''content'\''] | trim + '\''<{{ eot_id }}>'\'' %}{%- if loop.index0 == 0 %}{%- set content = '\''<{{ begin_of_text }}>'\'' + content %}{%- endif %}{{ content }}{%- endfor %}{%- if add_generation_prompt %}{{ '\''<{{ start_header_id }}>assistant<{{ end_header_id }}>\n\n'\'' }}{%- endif %}", - "tool_call_parser": "llama3_json", - "auto_tool_choice": true - } - } - }' - - # Register model entity referencing the fileset - nemo models create \ - --name "llama-3-2-1b-tool" \ - --fileset "default/llama-3-2-1b-tool" - - # Deploy - nemo inference deployment-configs create \ - --name "llama-tool-config" \ - --nim-deployment '{ - "model_name": "default/llama-3-2-1b-tool", - "gpu": 1 - }' - - nemo inference deployments create \ - --name "llama-tool-deployment" \ - --config "llama-tool-config" - - nemo wait inference deployment llama-tool-deployment -{% endraw %} - ``` - -=== "Python SDK" - - ```python - {% raw %} - # Create fileset with chat template and tool calling config via metadata - tool_metadata: dict[str, object] = { - "model": { - "tool_calling": { - "chat_template": ( - "{%- for message in messages %}" - "{%- set content = '<{{ start_header_id }}>' + message['role'] + '<{{ end_header_id }}>\n\n'" - " + message['content'] | trim + '<{{ eot_id }}>' %}" - "{%- if loop.index0 == 0 %}{%- set content = '<{{ begin_of_text }}>' + content %}{%- endif %}" - "{{ content }}{%- endfor %}" - "{%- if add_generation_prompt %}{{ '<{{ start_header_id }}>assistant<{{ end_header_id }}>\n\n' }}{%- endif %}" - ), - "tool_call_parser": "llama3_json", - "auto_tool_choice": True, - } - } - } - - client.files.filesets.create( - name="llama-3-2-1b-tool", - storage={ - "type": "huggingface", - "repo_id": "meta-llama/Llama-3.2-1B-Instruct", - "repo_type": "model" - }, - metadata=tool_metadata, - ) - - # Register model entity referencing the fileset - client.models.create( - name="llama-3-2-1b-tool", - fileset="default/llama-3-2-1b-tool", - ) - - # Deploy — chat_template and tool_call_config are inherited from the fileset - config = client.inference.deployment_configs.create( - name="llama-tool-config", - nim_deployment={ - "model_name": "default/llama-3-2-1b-tool", - "gpu": 1 - } - ) - - deployment = client.inference.deployments.create( - name="llama-tool-deployment", - config="llama-tool-config" - ) - - client.models.wait_for_status( - deployment_name="llama-tool-deployment", - desired_status="READY" - ) - {% endraw %} - ``` - -### Option 2: Set via Deployment Config (Override) - -Set `chat_template` and `tool_call_config` directly on the deployment config. These override any values from the fileset. - - -=== "CLI" - - ```bash -{% raw %} - nemo inference deployment-configs create \ - --name "llama-tool-override-config" \ - --nim-deployment '{ - "model_name": "default/llama-3-2-1b-tool", - "gpu": 1, - "chat_template": "{%- for message in messages %}{{ message[\"content\"] }}{%- endfor %}", - "tool_call_config": { - "tool_call_parser": "hermes", - "auto_tool_choice": false - } - }' -{% endraw %} - ``` - -=== "Python SDK" - - ```python - {% raw %} - config = client.inference.deployment_configs.create( - name="llama-tool-override-config", - nim_deployment={ - "model_name": "default/llama-3-2-1b-tool", - "gpu": 1, - "chat_template": "{%- for message in messages %}{{ message['content'] }}{%- endfor %}", - "tool_call_config": { - "tool_call_parser": "hermes", - "auto_tool_choice": False, - } - } - ) - {% endraw %} - ``` - -### Change Tool Calling Config for an Existing Model - -Updating a fileset's `metadata.model.tool_calling` does **not** propagate changes to an existing model entity. The model entity's `spec` is populated from the fileset only at creation time. To change the tool calling configuration, create a new fileset with the updated config and a new model entity that references it. - - -=== "CLI" - - ```bash - # Create a new fileset with the updated tool calling config - nemo files filesets create \ - --name "llama-3-2-1b-tool-v2" \ - --storage '{ - "type": "huggingface", - "repo_id": "meta-llama/Llama-3.2-1B-Instruct", - "repo_type": "model" - }' \ - --metadata '{ - "model": { - "tool_calling": { - "tool_call_parser": "mistral", - "auto_tool_choice": true - } - } - }' - - # Create a new model entity referencing the new fileset - nemo models create \ - --name "llama-3-2-1b-tool-v2" \ - --fileset "default/llama-3-2-1b-tool-v2" - - # Update deployment config to use the new model, or create a new one - nemo inference deployment-configs create \ - --name "llama-tool-config-v2" \ - --nim-deployment '{ - "model_name": "default/llama-3-2-1b-tool-v2", - "gpu": 1 - }' - - nemo inference deployments create \ - --name "llama-tool-deployment-v2" \ - --config "llama-tool-config-v2" - - nemo wait inference deployment llama-tool-deployment-v2 - ``` - -=== "Python SDK" - - ```python - # Create a new fileset with the updated tool calling config via metadata - tool_metadata_v2: dict[str, object] = { - "model": { - "tool_calling": { - "tool_call_parser": "mistral", - "auto_tool_choice": True, - } - } - } - - client.files.filesets.create( - name="llama-3-2-1b-tool-v2", - storage={ - "type": "huggingface", - "repo_id": "meta-llama/Llama-3.2-1B-Instruct", - "repo_type": "model", - }, - metadata=tool_metadata_v2, - ) - - # Create a new model entity referencing the new fileset - client.models.create( - name="llama-3-2-1b-tool-v2", - fileset="default/llama-3-2-1b-tool-v2", - ) - - # Create a new deployment config and deployment - config = client.inference.deployment_configs.create( - name="llama-tool-config-v2", - nim_deployment={"model_name": "default/llama-3-2-1b-tool-v2", "gpu": 1}, - ) - - deployment = client.inference.deployments.create( - name="llama-tool-deployment-v2", config="llama-tool-config-v2" - ) - - client.models.wait_for_status( - deployment_name="llama-tool-deployment-v2", desired_status="READY" - ) - ``` - -### Custom Tool Call Plugin - -For custom tool calling parsers, store the plugin Python file in a separate fileset and reference it via `tool_call_plugin`. - -!!! info - Because plugins execute arbitrary Python code inside the NIM container, `tool_call_plugin` is **disabled by default** at the platform level. - To enable it, set `models.tool_call_plugin.enabled: true` in the platform configuration and ensure the user has the `models.tool-call-plugin.set` permission (granted to Admin and PlatformAdmin roles by default). - - -=== "Python SDK" - - ```python - # 1. Create a fileset for the plugin file - client.files.filesets.create(name="my-tool-plugin") - client.files.upload( - fileset="my-tool-plugin", - local_path="my_parser.py", - remote_path="my_parser.py", - ) - - # 2. Reference the plugin fileset in the model's fileset metadata - plugin_tool_metadata: dict[str, object] = { - "model": { - "tool_calling": { - "tool_call_parser": "custom_parser", - "tool_call_plugin": "default/my-tool-plugin", - "auto_tool_choice": True, - } - } - } - - client.files.filesets.update( - "llama-3-2-1b-tool", - metadata=plugin_tool_metadata, - ) - ``` - -The platform downloads the plugin fileset at deployment time and passes the `.py` file path to NIM via the `NIM_TOOL_PARSER_PLUGIN` environment variable. - -### Tool Call Config Reference - -| Field | Type | Description | -|-------|------|-------------| -| `tool_call_parser` | `string` | Parser name: `llama3_json`, `hermes`, `mistral`, `pythonic`, `openai`, or a custom parser name | -| `tool_call_plugin` | `string` | Fileset reference (`workspace/fileset-name`) containing a custom plugin `.py` file | -| `auto_tool_choice` | `boolean` | When `true`, the model can decide to call tools without explicit user instruction | - -### NIM Environment Variable Mapping - -The platform translates tool calling configuration into NIM environment variables: - -| Metadata Field | NIM Environment Variable | Example Value | -|--------------|--------------------------|---------------| -| `tool_calling.chat_template` | `NIM_CHAT_TEMPLATE` | Jinja2 template string | -| `tool_calling.tool_call_parser` | `NIM_TOOL_CALL_PARSER` | `llama3_json` | -| `tool_calling.tool_call_plugin` | `NIM_TOOL_PARSER_PLUGIN` | `/path/to/my_parser.py` | -| `tool_calling.auto_tool_choice` | `NIM_ENABLE_AUTO_TOOL_CHOICE` | `1` or `0` | - ---- - -## Controlling Reasoning at Request Level - -Some models (for example, `nvidia/nemotron-3-nano-30b-a3b`) enable reasoning/thinking by default. (From the model card: "[nemotron-3] responds to user queries and tasks by first generating a reasoning trace and then concluding with a final response") You can disable it on a per-request basis by passing `chat_template_kwargs` directly in the request body: - - -=== "CLI" - - ```bash - nemo inference gateway provider post v1/chat/completions \ - --name "my-deployment" \ - --body '{ - "model": "default/my-model", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - "chat_template_kwargs": {"thinking": false} - }' - ``` - -=== "Python SDK" - - ```python - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="my-deployment", - body={ - "model": "default/my-model", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - "chat_template_kwargs": {"thinking": False}, - }, - ) - ``` - -!!! note "`chat_template_kwargs` is passed directly in the request body, not nested under `extra_body`. For more details on request-level reasoning overrides, see the [vLLM documentation](https://docs.vllm.ai/en/latest/features/reasoning_outputs/#request-level-override)." - This parameter only applies to models which use vLLM under the hood. For non-vLLM providers such as OpenAI or NVIDIA Build, the parameter that controls reasoning differs. Consult the provider's documentation for the specific parameter. diff --git a/docs/run-inference/tutorials/deploy-models.mdx b/docs/run-inference/tutorials/deploy-models.mdx new file mode 100644 index 0000000000..b93343555e --- /dev/null +++ b/docs/run-inference/tutorials/deploy-models.mdx @@ -0,0 +1,1033 @@ +--- +title: "Deploy Models" +description: "" +--- + + +Deploy models from NGC or HuggingFace. Register external providers like OpenAI or NVIDIA Build. + + + +Resource names for deployments, deployment configs, and providers must contain only letters (a-z, A-Z), digits (0-9), underscores, hyphens, and dots. For example: `llama-3.1-8b`, `my-custom-model`, `qwen-fs-config`. + + + + +--- + +## Add External Providers + +Register external inference APIs like NVIDIA Build or OpenAI. + +### NVIDIA Build + +By default, the platform pre-configures an external provider for NVIDIA Build named `nvidia-build` in the `system` workspace. +The example below demonstrates how to recreate it in your own workspace. +For disambiguation purposes, this example names the manually-created version `my-nvidia-build`. + + + + + + +```bash +# Store API key +echo "$NVIDIA_API_KEY" | nemo secrets create "nvidia-api-key" --from-file - + +# Create provider +nemo inference providers create "my-nvidia-build" \ +--host-url "https://integrate.api.nvidia.com" \ +--api-key-secret-name "nvidia-api-key" + +nemo wait inference provider my-nvidia-build + +# Test using interactive chat +nemo chat nvidia/llama-3.3-nemotron-super-49b-v1 'Hello!' \ +--provider my-nvidia-build +``` + + + + +```python +# Store API key +client.secrets.create(name="nvidia-api-key", value=os.environ["NVIDIA_API_KEY"]) + +# Create provider +provider = client.inference.providers.create( + name="my-nvidia-build", + host_url="https://integrate.api.nvidia.com", + api_key_secret_name="nvidia-api-key", +) + +client.models.wait_for_provider("my-nvidia-build") + +# Use provider routing +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="my-nvidia-build", + body={ + "model": "meta/llama-3.1-8b-instruct", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` + + + + +### OpenAI + + + + + + +```bash +# Store API key +echo "$OPENAI_API_KEY" | nemo secrets create "openai-api-key" --from-file - + +# Create provider with enabled models +nemo inference providers create "openai" \ +--host-url "https://api.openai.com/v1" \ +--api-key-secret-name "openai-api-key" \ +--enabled-models "gpt-4" \ +--enabled-models "gpt-3.5-turbo" + +nemo wait inference provider openai + +# Test using interactive chat +nemo chat gpt-4 'Hello!' \ +--provider openai +``` + + + + +```python +client.secrets.create(name="openai-api-key", value=os.environ["OPENAI_API_KEY"]) + +provider = client.inference.providers.create( + name="openai", + host_url="https://api.openai.com/v1", + api_key_secret_name="openai-api-key", + enabled_models=["gpt-4", "gpt-3.5-turbo"], +) + +client.models.wait_for_provider("openai") + +# Use provider routing +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="openai", + body={ + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` + + + + +### Anthropic + +Anthropic's `/v1/messages` API expects the API key in an `X-Api-Key:` header (not `Authorization: Bearer`) and requires an `anthropic-version` header on every request. Use `--auth-header-format` (Jinja2 template, must contain exactly one `{{ auth_secret }}` variable) to override the default `Authorization: Bearer {{ auth_secret }}` and pass the API-version pin via `--default-extra-headers`. Without these, Anthropic rejects every request with 401. + + + + + +```bash +# Store API key +echo "$ANTHROPIC_API_KEY" | nemo secrets create "anthropic-api-key" --from-file - + +# Create provider — override the default Bearer header and pin the API version +nemo inference providers create "anthropic" \ +--host-url "https://api.anthropic.com" \ +--api-key-secret-name "anthropic-api-key" \ +--auth-header-format "X-Api-Key: {{ auth_secret }}" \ +--default-extra-headers '{"anthropic-version": "2023-06-01"}' + +nemo wait inference provider anthropic +``` + + + + +```python +# Store API key +client.secrets.create(name="anthropic-api-key", data=os.environ["ANTHROPIC_API_KEY"]) + +# Create provider — override the default Bearer header and pin the API version +provider = client.inference.providers.create( + name="anthropic", + host_url="https://api.anthropic.com", + api_key_secret_name="anthropic-api-key", + auth_header_format="X-Api-Key: {{ auth_secret }}", + default_extra_headers={"anthropic-version": "2023-06-01"}, +) + +client.models.wait_for_provider("anthropic") +``` + + + + +`{{ auth_secret }}` is substituted with the resolved secret value at request time. + +--- + +## Deploy from NGC + +Deploy pre-built NIM containers from NGC. + +### Deploy Llama 3.2 1B + + + + + + +```bash +nemo inference deployment-configs create \ +--name "llama-3-2-1b-config" \ +--nim-deployment '{ +"gpu": 1, +"image_name": "nvcr.io/nim/meta/llama-3.2-1b-instruct", +"image_tag": "1.8.6", +"model_name": "meta/llama-3.2-1b-instruct" +}' + +nemo inference deployments create \ +--name "llama-3-2-1b-deployment" \ +--config "llama-3-2-1b-config" + +nemo wait inference deployment llama-3-2-1b-deployment + +nemo chat meta/llama-3.2-1b-instruct 'Hello!' \ +--provider llama-3-2-1b-deployment \ +--max-tokens 100 +``` + + + + +```python +config = client.inference.deployment_configs.create( + name="llama-3-2-1b-config", + nim_deployment={ + "gpu": 1, + "image_name": "nvcr.io/nim/meta/llama-3.2-1b-instruct", + "image_tag": "1.8.6", + "model_name": "meta/llama-3.2-1b-instruct", + }, +) + +deployment = client.inference.deployments.create( + name="llama-3-2-1b-deployment", config="llama-3-2-1b-config" +) + +client.models.wait_for_status( + deployment_name="llama-3-2-1b-deployment", desired_status="READY" +) + +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="llama-3-2-1b-deployment", + body={ + "model": "meta/llama-3.2-1b-instruct", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` + + + + +### Deploy NemoGuard JailbreakDetect + +Deploy classification NIMs like NemoGuard for content safety. Uses the `/v1/classify` endpoint instead of chat completions. + + + + + + +```bash +nemo inference deployment-configs create \ +--name "nemoguard-jailbreak-config" \ +--nim-deployment '{ +"gpu": 1, +"image_name": "nvcr.io/nim/nvidia/nemoguard-jailbreak-detect", +"image_tag": "1.10.1" +}' + +nemo inference deployments create \ +--name "nemoguard-jailbreak-deployment" \ +--config "nemoguard-jailbreak-config" + +nemo wait inference deployment nemoguard-jailbreak-deployment + +nemo inference gateway provider post v1/classify \ +--name "nemoguard-jailbreak-deployment" \ +--body '{"input": "Tell me about vacation spots in Hawaii."}' +``` + + + + +```python +config = client.inference.deployment_configs.create( + name="nemoguard-jailbreak-config", + nim_deployment={ + "gpu": 1, + "image_name": "nvcr.io/nim/nvidia/nemoguard-jailbreak-detect", + "image_tag": "1.10.1", + }, +) + +deployment = client.inference.deployments.create( + name="nemoguard-jailbreak-deployment", config="nemoguard-jailbreak-config" +) + +client.models.wait_for_status( + deployment_name="nemoguard-jailbreak-deployment", desired_status="READY" +) + +response = client.inference.gateway.provider.post( + "v1/classify", + name="nemoguard-jailbreak-deployment", + body={"input": "Tell me about vacation spots in Hawaii."}, +) +``` + + + + +--- + +## Deploy from HuggingFace + + + +HuggingFace deployments use the Multi-LLM NIM (`nvcr.io/nim/nvidia/llm-nim:1.13.1`) by default, which only supports specific model architectures. Check the [supported architectures list](https://docs.nvidia.com/nim/large-language-models/1.13.0/supported-architectures.html) before deploying. If your model architecture is not listed, you will need a model-specific NIM image — see [Deploy from NGC](#deploy-from-ngc) for that approach. + + + +You can register a HuggingFace model through the Files service. This creates a fileset that acts as a proxy. The Files service handles authentication and caches the weights on first download, so subsequent deployments start faster. + + + + + + +```bash +# (Optional) Create a HuggingFace token secret for private models. +# Public models like Qwen do not require a token. +echo "$HF_TOKEN" | nemo secrets create "hf-token-secret" --from-file - + +# Create a fileset pointing to the HuggingFace model. +# "token_secret" is optional — only needed for private/gated models. +nemo files filesets create "qwen-2-5-1-5b" \ +--storage '{ +"type": "huggingface", +"repo_id": "Qwen/Qwen2.5-1.5B-Instruct", +"repo_type": "model", +"token_secret": "hf-token-secret" +}' + +# Register a model entity referencing the fileset +nemo models create "qwen-2-5-1-5b" \ +--fileset "default/qwen-2-5-1-5b" + +# Create deployment config pointing to the model entity +nemo inference deployment-configs create "qwen-fs-config" \ +--nim-deployment '{ +"model_namespace": "default", +"model_name": "qwen-2-5-1-5b", +"gpu": 1 +}' + +nemo inference deployments create "qwen-fs-deployment" \ +--config "qwen-fs-config" + +nemo wait inference deployment qwen-fs-deployment + +nemo chat default/qwen-2-5-1-5b 'Hello!' \ +--provider qwen-fs-deployment \ +--max-tokens 100 +``` + + + + +```python +# (Optional) Create a HuggingFace token secret for private models. +# Public models like Qwen do not require a token. +client.secrets.create(name="hf-token-secret", value=os.environ["HF_TOKEN"]) + +# Create a fileset pointing to the HuggingFace model. +# "token_secret" is optional — only needed for private/gated models. +client.files.filesets.create( + name="qwen-2-5-1-5b", + storage={ + "type": "huggingface", + "repo_id": "Qwen/Qwen2.5-1.5B-Instruct", + "repo_type": "model", + "token_secret": "hf-token-secret", + }, +) + +# Register a model entity referencing the fileset +client.models.create(name="qwen-2-5-1-5b", fileset="default/qwen-2-5-1-5b") + +# Create deployment config pointing to the model entity +config = client.inference.deployment_configs.create( + name="qwen-fs-config", + nim_deployment={ + "model_namespace": "default", + "model_name": "qwen-2-5-1-5b", + "gpu": 1, + }, +) + +# Deploy — no hf_token_secret_name needed +deployment = client.inference.deployments.create( + name="qwen-fs-deployment", config="qwen-fs-config" +) + +client.models.wait_for_status( + deployment_name="qwen-fs-deployment", desired_status="READY" +) + +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="qwen-fs-deployment", + body={ + "model": "default/qwen-2-5-1-5b", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` + + + + + + +The `fileset` format is `/`. This tells the deployment system to pull weights from the Files service, which proxies the download from HuggingFace using the fileset `token_secret`. For public models like Qwen, the `token_secret` on the fileset is optional. + + +--- + + +## Deployment Cleanup + + + + + + +```bash +# Note: Deleting the deployment will free up its GPU(s) when complete +nemo inference deployments delete +nemo wait inference deployment --status DELETED +nemo inference deployment-configs delete + +# For external providers +nemo inference providers delete +nemo secrets delete +``` + + + + +```python +# Note: Deleting the deployment will free up its GPU(s) when complete +client.inference.deployments.delete(name="") +client.models.wait_for_status( + deployment_name="", desired_status="DELETED" +) +client.inference.deployment_configs.delete(name="") + +# For external providers +client.inference.providers.delete(name="") +client.secrets.delete(name="") +``` + + + + +--- + +## Multi-GPU Deployments + +For larger models requiring multiple GPUs, parallelism configuration depends on the NIM type. + +### Parallelism Strategies + +- **Tensor Parallel (TP)**: Splits model layers across GPUs → best for latency +- **Pipeline Parallel (PP)**: Splits model depth across GPUs → best for throughput +- **Formula**: `gpu` = `tp_size` × `pp_size` + +### Model-Specific NIMs + +Model-specific NIMs (for example, `nvcr.io/nim/meta/llama-3.1-70b-instruct`) have TP/PP settings derived from manifest profiles in the container. Configure enough GPUs and the NIM selects the appropriate profile automatically. + +### Multi-LLM NIM + +The multi-LLM NIM (`nvcr.io/nim/nvidia/llm-nim:1.13.1`) requires explicit parallelism configuration via environment variables (`NIM_TENSOR_PARALLEL_SIZE`, `NIM_PIPELINE_PARALLEL_SIZE`). By default, it uses all GPUs for tensor parallelism (TP=gpu, PP=1). + +This example deploys [Qwen2.5-14B-Instruct](https://huggingface.co/Qwen/Qwen2.5-14B-Instruct) across 2 GPUs using tensor parallelism. + + + + + + +```bash +# Create a fileset pointing to the HuggingFace model +# Qwen models are public — token_secret is optional +nemo files filesets create \ +--name "qwen-2-5-14b" \ +--storage '{ +"type": "huggingface", +"repo_id": "Qwen/Qwen2.5-14B-Instruct", +"repo_type": "model" +}' + +# Register a model entity referencing the fileset +nemo models create \ +--name "qwen-2-5-14b" \ +--fileset "default/qwen-2-5-14b" + +# Create deployment config with 2 GPUs (TP=2 by default) +nemo inference deployment-configs create \ +--name "qwen-14b-config" \ +--nim-deployment '{ +"model_name": "default/qwen-2-5-14b", +"gpu": 2 +}' + +# Deploy +nemo inference deployments create \ +--name "qwen-14b-deployment" \ +--config "qwen-14b-config" + +nemo wait inference deployment qwen-14b-deployment + +nemo chat default/qwen-2-5-14b 'Hello!' \ +--max-tokens 100 +``` + + + + +```python +# Create a fileset pointing to the HuggingFace model +# Qwen models are public — token_secret is optional +client.files.filesets.create( + name="qwen-2-5-14b", + storage={ + "type": "huggingface", + "repo_id": "Qwen/Qwen2.5-14B-Instruct", + "repo_type": "model", + }, +) + +# Register a model entity referencing the fileset +client.models.create(name="qwen-2-5-14b", fileset="default/qwen-2-5-14b") + +# Create deployment config with 2 GPUs (TP=2 by default) +config = client.inference.deployment_configs.create( + name="qwen-14b-config", + nim_deployment={"model_name": "default/qwen-2-5-14b", "gpu": 2}, +) + +# Deploy +deployment = client.inference.deployments.create( + name="qwen-14b-deployment", config="qwen-14b-config" +) + +client.models.wait_for_status( + deployment_name="qwen-14b-deployment", desired_status="READY" +) + +response = client.inference.gateway.model.post( + "v1/chat/completions", + name="qwen-2-5-14b", + body={ + "model": "default/qwen-2-5-14b", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +# NIM sets NIM_TENSOR_PARALLEL_SIZE=2 automatically +``` + + + + +### Custom Parallelism Configuration + +For larger models requiring more GPUs, you can configure specific TP/PP splits using `additional_envs`. The formula is: `gpu` = `NIM_TENSOR_PARALLEL_SIZE` × `NIM_PIPELINE_PARALLEL_SIZE`. + + + + + + +```bash +nemo inference deployment-configs create \ +--name "multi-gpu-custom-config" \ +--nim-deployment '{ +"model_name": "default/qwen-2-5-14b", +"gpu": 4, +"additional_envs": { +"NIM_TENSOR_PARALLEL_SIZE": "2", +"NIM_PIPELINE_PARALLEL_SIZE": "2" +} +}' +``` + + + + +```python +config = client.inference.deployment_configs.create( + name="multi-gpu-custom-config", + nim_deployment={ + "model_name": "default/qwen-2-5-14b", + "gpu": 4, + "additional_envs": { + "NIM_TENSOR_PARALLEL_SIZE": "2", + "NIM_PIPELINE_PARALLEL_SIZE": "2", + }, + }, +) +``` + + + + + + +**Choosing Parallelism Strategy** + +- **TP=8, PP=1** (default): Lowest latency, best for real-time applications +- **TP=4, PP=2**: Balanced latency and throughput +- **TP=2, PP=4**: Highest throughput, best for batch processing + +For custom models, match deployment parallelism to training parallelism for optimal performance. + + +--- + + +## Chat Templates and Tool Calling + +Configure custom chat templates and tool calling for NIM deployments. These settings control how the model formats chat messages and handles function/tool calling. + +For more information on chat templates, see the [Hugging Face chat templating guide](https://huggingface.co/docs/transformers/en/chat_templating) and the [NeMo chat templates documentation](https://docs.nvidia.com/nemo/rl/latest/design-docs/chat-datasets.html#chat-templates). + + + +**Security consideration:** Chat templates are Jinja2 programs that execute on every inference call. While NIM uses a sandboxed Jinja2 environment (mitigating arbitrary code execution), a malicious or misconfigured template can still alter model behavior — for example, by injecting hidden instructions, rewriting messages, or degrading output quality. Grant chat template permissions only to trusted users, and review templates before deploying to production. See [Inference-Time Backdoors via Chat Templates (IEEE S&P 2026)](https://arxiv.org/abs/2602.04653) for further background. + + +### Configuration Sources + +There are two ways to configure chat templates and tool calling, with deployment-level settings taking highest priority: + +| Source | Priority | When to Use | +|--------|----------|-------------| +| **Fileset `metadata.model.tool_calling`** | Base | Set once per model — applies to all deployments using that model | +| **Deployment config (`nim_deployment`)** | Override | Per-deployment overrides — useful for A/B testing or deployment-specific behavior | + +When both are set, deployment config values override fileset values. + +### How It Works + +1. User sets `chat_template`, `tool_call_parser`, `tool_call_plugin`, and/or `auto_tool_choice` on the fileset via `metadata.model.tool_calling` +2. The model-spec background task reads the fileset's `metadata.model.tool_calling` and writes them into the model entity's `spec` +3. At deployment time, the platform reads `model_entity.spec` and any deployment-level overrides to set NIM environment variables + +```mermaid +flowchart LR + FS["Fileset
metadata.model.tool_calling"] -->|model-spec task| ME["Model Entity
spec"] + ME --> ENV["NIM Container
env vars"] + DC["Deployment Config
nim_deployment"] -->|override| ENV +``` + +### Option 1: Set via Fileset (Recommended) + +Set `chat_template` and tool calling configuration with fileset `metadata.model.tool_calling`. The platform automatically propagates these into the model entity spec when the model-spec background task runs. + + + + + + +```bash + + # Create fileset with chat template and tool calling config via metadata + nemo files filesets create \ + --name "llama-3-2-1b-tool" \ + --storage '{ + "type": "huggingface", + "repo_id": "meta-llama/Llama-3.2-1B-Instruct", + "repo_type": "model" + }' \ + --metadata '{ + "model": { + "tool_calling": { + "chat_template": "{%- for message in messages %}{%- set content = '\''<{{ start_header_id }}>'\'' + message['\''role'\''] + '\''<{{ end_header_id }}>\n\n'\'' + message['\''content'\''] | trim + '\''<{{ eot_id }}>'\'' %}{%- if loop.index0 == 0 %}{%- set content = '\''<{{ begin_of_text }}>'\'' + content %}{%- endif %}{{ content }}{%- endfor %}{%- if add_generation_prompt %}{{ '\''<{{ start_header_id }}>assistant<{{ end_header_id }}>\n\n'\'' }}{%- endif %}", + "tool_call_parser": "llama3_json", + "auto_tool_choice": true + } + } + }' + + # Register model entity referencing the fileset + nemo models create \ + --name "llama-3-2-1b-tool" \ + --fileset "default/llama-3-2-1b-tool" + + # Deploy + nemo inference deployment-configs create \ + --name "llama-tool-config" \ + --nim-deployment '{ + "model_name": "default/llama-3-2-1b-tool", + "gpu": 1 + }' + + nemo inference deployments create \ + --name "llama-tool-deployment" \ + --config "llama-tool-config" + + nemo wait inference deployment llama-tool-deployment + + ``` + + + + +```python + +# Create fileset with chat template and tool calling config via metadata +tool_metadata: dict[str, object] = { + "model": { + "tool_calling": { + "chat_template": ( + "{%- for message in messages %}" + "{%- set content = '<{{ start_header_id }}>' + message['role'] + '<{{ end_header_id }}>\n\n'" + " + message['content'] | trim + '<{{ eot_id }}>' %}" + "{%- if loop.index0 == 0 %}{%- set content = '<{{ begin_of_text }}>' + content %}{%- endif %}" + "{{ content }}{%- endfor %}" + "{%- if add_generation_prompt %}{{ '<{{ start_header_id }}>assistant<{{ end_header_id }}>\n\n' }}{%- endif %}" + ), + "tool_call_parser": "llama3_json", + "auto_tool_choice": True, + } + } +} + +client.files.filesets.create( + name="llama-3-2-1b-tool", + storage={ + "type": "huggingface", + "repo_id": "meta-llama/Llama-3.2-1B-Instruct", + "repo_type": "model" + }, + metadata=tool_metadata, +) + +# Register model entity referencing the fileset +client.models.create( + name="llama-3-2-1b-tool", + fileset="default/llama-3-2-1b-tool", +) + +# Deploy — chat_template and tool_call_config are inherited from the fileset +config = client.inference.deployment_configs.create( + name="llama-tool-config", + nim_deployment={ + "model_name": "default/llama-3-2-1b-tool", + "gpu": 1 + } +) + +deployment = client.inference.deployments.create( + name="llama-tool-deployment", + config="llama-tool-config" +) + +client.models.wait_for_status( + deployment_name="llama-tool-deployment", + desired_status="READY" +) + + ``` + + + + +### Option 2: Set via Deployment Config (Override) + +Set `chat_template` and `tool_call_config` directly on the deployment config. These override any values from the fileset. + + + + + + +```bash + + nemo inference deployment-configs create \ + --name "llama-tool-override-config" \ + --nim-deployment '{ + "model_name": "default/llama-3-2-1b-tool", + "gpu": 1, + "chat_template": "{%- for message in messages %}{{ message[\"content\"] }}{%- endfor %}", + "tool_call_config": { + "tool_call_parser": "hermes", + "auto_tool_choice": false + } + }' + + ``` + + + + +```python + +config = client.inference.deployment_configs.create( + name="llama-tool-override-config", + nim_deployment={ + "model_name": "default/llama-3-2-1b-tool", + "gpu": 1, + "chat_template": "{%- for message in messages %}{{ message['content'] }}{%- endfor %}", + "tool_call_config": { + "tool_call_parser": "hermes", + "auto_tool_choice": False, + } + } +) + + ``` + + + + +### Change Tool Calling Config for an Existing Model + +Updating a fileset's `metadata.model.tool_calling` does **not** propagate changes to an existing model entity. The model entity's `spec` is populated from the fileset only at creation time. To change the tool calling configuration, create a new fileset with the updated config and a new model entity that references it. + + + + + + +```bash +# Create a new fileset with the updated tool calling config +nemo files filesets create \ +--name "llama-3-2-1b-tool-v2" \ +--storage '{ +"type": "huggingface", +"repo_id": "meta-llama/Llama-3.2-1B-Instruct", +"repo_type": "model" +}' \ +--metadata '{ +"model": { +"tool_calling": { +"tool_call_parser": "mistral", +"auto_tool_choice": true +} +} +}' + +# Create a new model entity referencing the new fileset +nemo models create \ +--name "llama-3-2-1b-tool-v2" \ +--fileset "default/llama-3-2-1b-tool-v2" + +# Update deployment config to use the new model, or create a new one +nemo inference deployment-configs create \ +--name "llama-tool-config-v2" \ +--nim-deployment '{ +"model_name": "default/llama-3-2-1b-tool-v2", +"gpu": 1 +}' + +nemo inference deployments create \ +--name "llama-tool-deployment-v2" \ +--config "llama-tool-config-v2" + +nemo wait inference deployment llama-tool-deployment-v2 +``` + + + + +```python +# Create a new fileset with the updated tool calling config via metadata +tool_metadata_v2: dict[str, object] = { + "model": { + "tool_calling": { + "tool_call_parser": "mistral", + "auto_tool_choice": True, + } + } +} + +client.files.filesets.create( + name="llama-3-2-1b-tool-v2", + storage={ + "type": "huggingface", + "repo_id": "meta-llama/Llama-3.2-1B-Instruct", + "repo_type": "model", + }, + metadata=tool_metadata_v2, +) + +# Create a new model entity referencing the new fileset +client.models.create( + name="llama-3-2-1b-tool-v2", + fileset="default/llama-3-2-1b-tool-v2", +) + +# Create a new deployment config and deployment +config = client.inference.deployment_configs.create( + name="llama-tool-config-v2", + nim_deployment={"model_name": "default/llama-3-2-1b-tool-v2", "gpu": 1}, +) + +deployment = client.inference.deployments.create( + name="llama-tool-deployment-v2", config="llama-tool-config-v2" +) + +client.models.wait_for_status( + deployment_name="llama-tool-deployment-v2", desired_status="READY" +) +``` + + + + +### Custom Tool Call Plugin + +For custom tool calling parsers, store the plugin Python file in a separate fileset and reference it via `tool_call_plugin`. + + + +Because plugins execute arbitrary Python code inside the NIM container, `tool_call_plugin` is **disabled by default** at the platform level. +To enable it, set `models.tool_call_plugin.enabled: true` in the platform configuration and ensure the user has the `models.tool-call-plugin.set` permission (granted to Admin and PlatformAdmin roles by default). + + + + + + +```python +# 1. Create a fileset for the plugin file +client.files.filesets.create(name="my-tool-plugin") +client.files.upload( + fileset="my-tool-plugin", + local_path="my_parser.py", + remote_path="my_parser.py", +) + +# 2. Reference the plugin fileset in the model's fileset metadata +plugin_tool_metadata: dict[str, object] = { + "model": { + "tool_calling": { + "tool_call_parser": "custom_parser", + "tool_call_plugin": "default/my-tool-plugin", + "auto_tool_choice": True, + } + } +} + +client.files.filesets.update( + "llama-3-2-1b-tool", + metadata=plugin_tool_metadata, +) +``` + + + + +The platform downloads the plugin fileset at deployment time and passes the `.py` file path to NIM via the `NIM_TOOL_PARSER_PLUGIN` environment variable. + +### Tool Call Config Reference + +| Field | Type | Description | +|-------|------|-------------| +| `tool_call_parser` | `string` | Parser name: `llama3_json`, `hermes`, `mistral`, `pythonic`, `openai`, or a custom parser name | +| `tool_call_plugin` | `string` | Fileset reference (`workspace/fileset-name`) containing a custom plugin `.py` file | +| `auto_tool_choice` | `boolean` | When `true`, the model can decide to call tools without explicit user instruction | + +### NIM Environment Variable Mapping + +The platform translates tool calling configuration into NIM environment variables: + +| Metadata Field | NIM Environment Variable | Example Value | +|--------------|--------------------------|---------------| +| `tool_calling.chat_template` | `NIM_CHAT_TEMPLATE` | Jinja2 template string | +| `tool_calling.tool_call_parser` | `NIM_TOOL_CALL_PARSER` | `llama3_json` | +| `tool_calling.tool_call_plugin` | `NIM_TOOL_PARSER_PLUGIN` | `/path/to/my_parser.py` | +| `tool_calling.auto_tool_choice` | `NIM_ENABLE_AUTO_TOOL_CHOICE` | `1` or `0` | + +--- + +## Controlling Reasoning at Request Level + +Some models (for example, `nvidia/nemotron-3-nano-30b-a3b`) enable reasoning/thinking by default. (From the model card: "[nemotron-3] responds to user queries and tasks by first generating a reasoning trace and then concluding with a final response") You can disable it on a per-request basis by passing `chat_template_kwargs` directly in the request body: + + + + + + +```bash +nemo inference gateway provider post v1/chat/completions \ +--name "my-deployment" \ +--body '{ +"model": "default/my-model", +"messages": [{"role": "user", "content": "Hello!"}], +"max_tokens": 100, +"chat_template_kwargs": {"thinking": false} +}' +``` + + + + +```python +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="my-deployment", + body={ + "model": "default/my-model", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + "chat_template_kwargs": {"thinking": False}, + }, +) +``` + + + + + + +`chat_template_kwargs` is passed directly in the request body, not nested under `extra_body`. For more details on request-level reasoning overrides, see the [vLLM documentation](https://docs.vllm.ai/en/latest/features/reasoning_outputs/#request-level-override). + + + This parameter only applies to models which use vLLM under the hood. For non-vLLM providers such as OpenAI or NVIDIA Build, the parameter that controls reasoning differs. Consult the provider's documentation for the specific parameter. diff --git a/docs/run-inference/tutorials/index.md b/docs/run-inference/tutorials/index.md deleted file mode 100644 index bbd61d8d7d..0000000000 --- a/docs/run-inference/tutorials/index.md +++ /dev/null @@ -1,13 +0,0 @@ - -# Tutorials - -Learn how to run inference through the {{platform_name}}. - -## Prerequisites - -- `NMP_BASE_URL` environment variable set to your platform URL -- Appropriate API credentials for an external provider (e.g. an NGC API key for NVIDIA Build) - -## Guides - -- [Run Inference](run-inference.md) — Route requests via model entity, provider, or OpenAI routing diff --git a/docs/run-inference/tutorials/index.mdx b/docs/run-inference/tutorials/index.mdx new file mode 100644 index 0000000000..3041231029 --- /dev/null +++ b/docs/run-inference/tutorials/index.mdx @@ -0,0 +1,16 @@ +--- +title: "Tutorials" +description: "" +--- + + +Learn how to run inference through the NeMo Platform. + +## Prerequisites + +- `NMP_BASE_URL` environment variable set to your platform URL +- Appropriate API credentials for an external provider (e.g. an NGC API key for NVIDIA Build) + +## Guides + +- [Run Inference](/documentation/models-and-inference/tutorials/run-inference) — Route requests via model entity, provider, or OpenAI routing diff --git a/docs/run-inference/tutorials/run-inference.md b/docs/run-inference/tutorials/run-inference.mdx similarity index 50% rename from docs/run-inference/tutorials/run-inference.md rename to docs/run-inference/tutorials/run-inference.mdx index 2ce4cabc5b..858c74a849 100644 --- a/docs/run-inference/tutorials/run-inference.md +++ b/docs/run-inference/tutorials/run-inference.mdx @@ -1,15 +1,17 @@ - - +--- +title: "Run Inference" +description: "" +--- -# Run Inference Route inference requests through the gateway using model entity routing, provider routing, or OpenAI-compatible routing. -!!! note - This tutorial assumes you have an external provider registered. The platform pre-configures a `system/nvidia-build` provider on startup, which most of the examples below use. To register your own provider, see [About Models and Inference](../about.md#model-providers). + ---8<-- "_snippets/tutorials/cli-sdk-setup.md" +This tutorial assumes you have an external provider registered. The platform pre-configures a `system/nvidia-build` provider on startup, which most of the examples below use. To register your own provider, see [About Models and Inference](/documentation/models-and-inference#model-providers). + + --- ## Listing Models @@ -24,29 +26,35 @@ There are two distinct ways to list models, depending on what you need: Use `nemo models list` to manage model configurations. Use `nemo inference models list` to discover what's ready to serve requests right now. -=== "CLI" + - ```bash - # All registered model entities (may include models without active deployments) - nemo models list + - # Models available for inference via the gateway (OpenAI-compatible IDs) - nemo inference models list - ``` +```bash +# All registered model entities (may include models without active deployments) +nemo models list -=== "Python SDK" +# Models available for inference via the gateway (OpenAI-compatible IDs) +nemo inference models list +``` - ```python - # All registered model entities - for model in client.models.list(): - print(model.name) + + - # Models available for inference via the gateway - models = client.inference.models.list() - for model in models.data: - print(model.id) # format: workspace/model_entity_name - ``` +```python +# All registered model entities +for model in client.models.list(): + print(model.name) + +# Models available for inference via the gateway +models = client.inference.models.list() +for model in models.data: + print(model.id) # format: workspace/model_entity_name +``` + + + --- ## Route by Model Entity @@ -56,45 +64,57 @@ Route requests using the model entity name. The gateway selects an available pro Find available model entities: -=== "CLI" + - ```bash - nemo models list - ``` + -=== "Python SDK" +```bash +nemo models list +``` - ```python - for model in client.models.list(): - print(model.name) - ``` + + +```python +for model in client.models.list(): + print(model.name) +``` + + + + Run inference by passing the Model Entity. -=== "CLI" + + + + +```bash +# Model entities are auto-discovered from deployments +# Use the model entity name from 'nemo models list' +nemo chat meta-llama-3-2-1b-instruct 'Hello!' --max-tokens 100 +``` - ```bash - # Model entities are auto-discovered from deployments - # Use the model entity name from 'nemo models list' - nemo chat meta-llama-3-2-1b-instruct 'Hello!' --max-tokens 100 - ``` + + -=== "Python SDK" +```python +# Model entities are auto-discovered from deployments +response = client.inference.gateway.model.post( + "v1/chat/completions", + name="meta-llama-3-2-1b-instruct", # Model entity name + body={ + "model": "meta/llama-3.2-1b-instruct", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` - ```python - # Model entities are auto-discovered from deployments - response = client.inference.gateway.model.post( - "v1/chat/completions", - name="meta-llama-3-2-1b-instruct", # Model entity name - body={ - "model": "meta/llama-3.2-1b-instruct", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` + + --- ## Route by Provider @@ -104,63 +124,79 @@ Route to a specific provider instance. Use for A/B testing or targeting specific Find available providers: -=== "CLI" + - ```bash - nemo inference providers list + - # List models available on a provider if its API is OpenAI compliant - nemo inference gateway provider get v1/models --name llama-3-2-1b-deployment - ``` +```bash +nemo inference providers list -=== "Python SDK" +# List models available on a provider if its API is OpenAI compliant +nemo inference gateway provider get v1/models --name llama-3-2-1b-deployment +``` + + + + +```python +for provider in client.inference.providers.list(): + print(f"{provider.name}: {provider.status}") - ```python - for provider in client.inference.providers.list(): - print(f"{provider.name}: {provider.status}") +# List models available on a provider if its API is OpenAI compliant +models = client.inference.gateway.provider.get( + "v1/models", + name="llama-3-2-1b-deployment", +) +print(models) +``` - # List models available on a provider if its API is OpenAI compliant - models = client.inference.gateway.provider.get( - "v1/models", - name="llama-3-2-1b-deployment", - ) - print(models) - ``` + + Pass inference request using provider routing. -=== "CLI" + - ```bash - # Provider name matches deployment name for auto-created providers - nemo chat meta/llama-3.2-1b-instruct 'Hello!' \ - --provider llama-3-2-1b-deployment \ - --max-tokens 100 - ``` + -=== "Python SDK" +```bash +# Provider name matches deployment name for auto-created providers +nemo chat meta/llama-3.2-1b-instruct 'Hello!' \ +--provider llama-3-2-1b-deployment \ +--max-tokens 100 +``` - ```python - # Provider name matches deployment name for auto-created providers - response = client.inference.gateway.provider.post( - "v1/chat/completions", - name="llama-3-2-1b-deployment", # Provider name - body={ - "model": "meta/llama-3.2-1b-instruct", - "messages": [{"role": "user", "content": "Hello!"}], - "max_tokens": 100, - }, - ) - ``` + + +```python +# Provider name matches deployment name for auto-created providers +response = client.inference.gateway.provider.post( + "v1/chat/completions", + name="llama-3-2-1b-deployment", # Provider name + body={ + "model": "meta/llama-3.2-1b-instruct", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 100, + }, +) +``` + + + + --- ## Route using OpenAI SDK Use the OpenAI-compatible endpoint for drop-in SDK replacement. The model field uses the format `{workspace}/{model_entity}`. -!!! note "Model entity naming" + + +Model entity naming + + The `{model_entity}` segment is validated against a strict regex (lowercase letters, digits, and hyphens; 2–63 characters; **no slashes**). Vendor-style IDs such as `meta/llama-3.3-70b-instruct` are rejected with @@ -174,23 +210,29 @@ Use the OpenAI-compatible endpoint for drop-in SDK replacement. The model field List models currently reachable via the Inference Gateway. Returns results in OpenAI-compatible format; each `id` is `workspace/model_entity_name`. -=== "CLI" + + + - ```bash - nemo inference models list - ``` +```bash +nemo inference models list +``` -=== "Python SDK" + + - ```python - models = client.inference.models.list() - for model in models.data: - print(model.id) # format: workspace/model_entity_name - ``` +```python +models = client.inference.models.list() +for model in models.data: + print(model.id) # format: workspace/model_entity_name +``` + + + ### Using SDK -Make requests to the OpenAI-compatible Inference Gateway route with the {{platform_name}} SDK. +Make requests to the OpenAI-compatible Inference Gateway route with the NeMo Platform SDK. ```python response = client.inference.gateway.openai.post( @@ -205,7 +247,7 @@ response = client.inference.gateway.openai.post( ### Using OpenAI Python SDK -Models service provides a convenient helper method `client.models.get_openai_client()` that can generate an OpenAI SDK client for the configured workspace. The SDK also offers helper methods for generating OpenAI-compatible URL strings for Inference Gateway. Refer to [SDK Helper Methods](../about.md#sdk-helper-methods) for more info. +Models service provides a convenient helper method `client.models.get_openai_client()` that can generate an OpenAI SDK client for the configured workspace. The SDK also offers helper methods for generating OpenAI-compatible URL strings for Inference Gateway. Refer to [SDK Helper Methods](/documentation/models-and-inference#sdk-helper-methods) for more info. ```python # Get pre-configured OpenAI client @@ -223,50 +265,55 @@ response = oai_client.chat.completions.create( If you construct third-party clients manually, pass the headers returned by `client.models.get_client_default_headers()` so auth and identity headers are preserved when authorization is enabled. -=== "OpenAI (manual client)" + + + + +```python +from openai import OpenAI - ```python - from openai import OpenAI +base_url = client.models.get_openai_route_base_url() +headers = client.models.get_client_default_headers() - base_url = client.models.get_openai_route_base_url() - headers = client.models.get_client_default_headers() +oai_client = OpenAI( + base_url=base_url, + api_key="not-needed", + default_headers=headers, +) - oai_client = OpenAI( - base_url=base_url, - api_key="not-needed", - default_headers=headers, - ) +response = oai_client.chat.completions.create( + model="default/meta-llama-3-2-1b-instruct", + messages=[{"role": "user", "content": "Hello!"}], +) +``` - response = oai_client.chat.completions.create( - model="default/meta-llama-3-2-1b-instruct", - messages=[{"role": "user", "content": "Hello!"}], - ) - ``` + + -=== "LiteLLM" +```bash +# make sure litellm is installed +pip install litellm +``` - ```bash - # make sure litellm is installed - pip install litellm - ``` +```python +from litellm import completion - - ```python - from litellm import completion +base_url = client.models.get_openai_route_base_url() +headers = client.models.get_client_default_headers() - base_url = client.models.get_openai_route_base_url() - headers = client.models.get_client_default_headers() +response = completion( + model="openai/meta-llama-3-2-1b-instruct", + messages=[{"role": "user", "content": "Hello!"}], + api_base=base_url, + api_key="not-needed", + extra_headers=headers, +) +``` - response = completion( - model="openai/meta-llama-3-2-1b-instruct", - messages=[{"role": "user", "content": "Hello!"}], - api_base=base_url, - api_key="not-needed", - extra_headers=headers, - ) - ``` + + ### Using curl The OpenAI-compatible chat-completions endpoint is reachable directly over HTTP. diff --git a/docs/safe-synthesizer/about/data-synthesis.md b/docs/safe-synthesizer/about/data-synthesis.mdx similarity index 75% rename from docs/safe-synthesizer/about/data-synthesis.md rename to docs/safe-synthesizer/about/data-synthesis.mdx index 395e350cab..9f40619f6f 100644 --- a/docs/safe-synthesizer/about/data-synthesis.md +++ b/docs/safe-synthesizer/about/data-synthesis.mdx @@ -1,11 +1,14 @@ +--- +title: "Data Synthesis" +description: "" +--- -# Data Synthesis -The synthesizer component is the main component of the {{nss_short_name}} product. It uses LLM-based fine-tuning to generate realistic synthetic data that maintains the utility of your original dataset while providing privacy protection. +The synthesizer component is the main component of the NeMo Safe Synthesizer product. It uses LLM-based fine-tuning to generate realistic synthetic data that maintains the utility of your original dataset while providing privacy protection. ## How It Works -{{nss_short_name}} employs a novel approach to synthetic data generation: +NeMo Safe Synthesizer employs a novel approach to synthetic data generation: 1. **Tabular Fine-Tuning**: Fine-tunes a language model on your tabular data to learn patterns, correlations, and statistical properties 2. **Generation**: Uses the fine-tuned model to generate new synthetic records that maintain data utility @@ -17,7 +20,7 @@ Creating synthetic versions of private data allows you to unlock insights withou ### LLM-Based Fine-Tuning -{{nss_short_name}} adapts language models to understand and generate tabular data: +NeMo Safe Synthesizer adapts language models to understand and generate tabular data: - Converts tabular data into text sequences suitable for LLM training - Fine-tunes on your dataset to capture patterns and correlations @@ -26,7 +29,7 @@ Creating synthetic versions of private data allows you to unlock insights withou ### Supported Data Types -{{nss_short_name}} supports diverse tabular data: +NeMo Safe Synthesizer supports diverse tabular data: - **Numeric**: Continuous and discrete numerical values - **Categorical**: Text labels and categories @@ -37,7 +40,7 @@ Creating synthetic versions of private data allows you to unlock insights withou A high level of privacy protection is achieved simply through the process of generating synthetic data, and is often a sufficient balance between privacy and utility. For use cases that require maximum privacy, you can fine-tune with differential privacy. -Differential privacy (DP) is the gold standard for privacy protection, providing mathematical guarantees that individual records cannot be identified. When you enable DP, {{nss_short_name}} introduces calibrated noise during model training to ensure that the synthetic data generation process is provably private. +Differential privacy (DP) is the gold standard for privacy protection, providing mathematical guarantees that individual records cannot be identified. When you enable DP, NeMo Safe Synthesizer introduces calibrated noise during model training to ensure that the synthetic data generation process is provably private. #### Mathematical Guarantee @@ -57,7 +60,7 @@ Where: #### DP-SGD Implementation -{{nss_short_name}} uses Differentially Private Stochastic Gradient Descent (DP-SGD) to add privacy guarantees during model training: +NeMo Safe Synthesizer uses Differentially Private Stochastic Gradient Descent (DP-SGD) to add privacy guarantees during model training: 1. **Per-sample gradient computation** - Calculate gradients for each training example individually 2. **Gradient clipping** - Clip L2 norm to `per_sample_max_grad_norm` to bound sensitivity @@ -92,7 +95,7 @@ Enabling DP provides strong privacy guarantees but affects synthetic data qualit **Data size:** DP performs best with 10,000+ training records. Smaller datasets may experience significant quality degradation due to the noise required for privacy guarantees. -For hands-on guidance, refer to [differential-privacy](../tutorials/differential-privacy.md). For complete parameter documentation, refer to [reference](reference.md). +For hands-on guidance, refer to [differential-privacy](/documentation/synthesize-safe-data/tutorials/differential-privacy). For complete parameter documentation, refer to [reference](/documentation/synthesize-safe-data/about/parameters-reference). ## Configuration @@ -102,10 +105,10 @@ Synthesis behavior is controlled through configuration parameters: - **Generation**: Number of records, temperature, sampling strategies - **Privacy**: Differential privacy parameters (epsilon, delta, clipping) -For a complete list of all available parameters and their defaults, refer to [reference](reference.md). +For a complete list of all available parameters and their defaults, refer to [reference](/documentation/synthesize-safe-data/about/parameters-reference). ## Related Topics -- [reference](reference.md): Complete parameter reference -- [differential-privacy](../tutorials/differential-privacy.md): Learn about differential privacy in practice -- [index](../tutorials/index.md): More tutorials +- [reference](/documentation/synthesize-safe-data/about/parameters-reference): Complete parameter reference +- [differential-privacy](/documentation/synthesize-safe-data/tutorials/differential-privacy): Learn about differential privacy in practice +- [index](/documentation/synthesize-safe-data/tutorials): More tutorials diff --git a/docs/safe-synthesizer/about/evaluation.md b/docs/safe-synthesizer/about/evaluation.mdx similarity index 89% rename from docs/safe-synthesizer/about/evaluation.md rename to docs/safe-synthesizer/about/evaluation.mdx index 8ba95c4f90..f2ccf53b67 100644 --- a/docs/safe-synthesizer/about/evaluation.md +++ b/docs/safe-synthesizer/about/evaluation.mdx @@ -1,8 +1,11 @@ - +--- +title: "Evaluation" +description: "" +--- + -# Evaluation -Evaluation is a critical component of {{nss_short_name}} that helps you understand both the utility and privacy of your synthetic data. The evaluation step is enabled by default and provides comprehensive reports comparing your training and synthetic datasets across multiple dimensions. +Evaluation is a critical component of NeMo Safe Synthesizer that helps you understand both the utility and privacy of your synthetic data. The evaluation step is enabled by default and provides comprehensive reports comparing your training and synthetic datasets across multiple dimensions. ## How It Works @@ -85,11 +88,11 @@ You should expect some PII replay, and it is often not a cause for concern. We t - Lower epsilon (stronger privacy) generally yields higher DPS scores - Enabling DP can reduce **SQS** due to privacy-utility tradeoff (noise affects quality) -For differential privacy configuration and privacy-utility tradeoffs, refer to [data-synthesis](data-synthesis.md) and [differential-privacy](../tutorials/differential-privacy.md). +For differential privacy configuration and privacy-utility tradeoffs, refer to [data-synthesis](/documentation/synthesize-safe-data/about/data-synthesis) and [differential-privacy](/documentation/synthesize-safe-data/tutorials/differential-privacy). ## Evaluation Reports -Every {{nss_short_name}} job automatically generates an HTML evaluation report containing: +Every NeMo Safe Synthesizer job automatically generates an HTML evaluation report containing: - Overall SQS and DPS scores - Detailed subscores for each metric @@ -121,6 +124,6 @@ We enable evaluation by default. ## Related Topics -- [safe-synthesizer-101](../tutorials/safe-synthesizer-101.md): Get started with evaluation -- [differential-privacy](../tutorials/differential-privacy.md): Learn about privacy metrics -- [index](../tutorials/index.md): More tutorials +- [safe-synthesizer-101](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101): Get started with evaluation +- [differential-privacy](/documentation/synthesize-safe-data/tutorials/differential-privacy): Learn about privacy metrics +- [index](/documentation/synthesize-safe-data/tutorials): More tutorials diff --git a/docs/safe-synthesizer/about/host-local-development.md b/docs/safe-synthesizer/about/host-local-development.mdx similarity index 88% rename from docs/safe-synthesizer/about/host-local-development.md rename to docs/safe-synthesizer/about/host-local-development.mdx index 7868815e7d..c5b5b2ee06 100644 --- a/docs/safe-synthesizer/about/host-local-development.md +++ b/docs/safe-synthesizer/about/host-local-development.mdx @@ -1,16 +1,18 @@ - - - +--- +title: "Local and Subprocess Execution" +description: "" +--- + -# Local and Subprocess Execution + -Run {{nss_short_name}} on your machine's GPU with `nemo safe-synthesizer run-local`. The public command is a local subprocess wrapper: the main NeMo CLI starts a separate Safe Synthesizer runtime Python, and that runtime executes the synthesis task module. +Run NeMo Safe Synthesizer on your machine's GPU with `nemo safe-synthesizer run-local`. The public command is a local subprocess wrapper: the main NeMo CLI starts a separate Safe Synthesizer runtime Python, and that runtime executes the synthesis task module. This page covers local execution only. Platform job submission uses the Jobs API or SDK; the `nemo safe-synthesizer` CLI exposes `run-local` and `runtime`. ## Prerequisites -- CUDA-capable NVIDIA GPU on the host (80GB+ VRAM recommended; check with `nvidia-smi`). See [Getting Started](../getting-started.md). +- CUDA-capable NVIDIA GPU on the host (80GB+ VRAM recommended; check with `nvidia-smi`). See [Getting Started](/documentation/synthesize-safe-data). - NeMo Platform repository checkout with the Safe Synthesizer plugin installed. - **No running platform required** for a typical local run when you pass `--data-source` — the NSS runtime can download base models from Hugging Face directly. @@ -35,7 +37,7 @@ There are two local paths: | Mode | Command | Use it when | |------|---------|-------------| | Managed local subprocess | `uv run nemo safe-synthesizer run-local ...` | You want the supported plugin CLI. This creates the parent CLI process, then launches the runtime Python subprocess. | -| Direct local task | ` -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local ...` | You are debugging the task process itself and want to bypass the parent CLI wrapper. | +| Direct local task | `\ -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local ...` | You are debugging the task process itself and want to bypass the parent CLI wrapper. | Both modes run on the host GPU and write artifacts to the local filesystem. Both accept the same task arguments: `--spec-file`, `--workspace`, `--output-dir`, and optional `--data-source`. @@ -61,7 +63,7 @@ uv run nemo safe-synthesizer run-local \ The parent command launches a subprocess equivalent to: ```bash - -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local \ +\ -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local \ --workspace default \ --spec-file ./nss-job.json \ --data-source ./input.csv \ @@ -79,7 +81,7 @@ uv run nemo safe-synthesizer runtime info Direct task execution is useful when you need to reproduce a subprocess failure without the parent CLI wrapper. ```bash -$(uv run nemo safe-synthesizer runtime info | awk -F': ' '/^python:/ {print $2}') \ +$(uv run nemo safe-synthesizer runtime info | awk -F': ' '/^python:/ \{print $2\}') \ -m nemo_safe_synthesizer_plugin.tasks.safe_synthesizer run-local \ --workspace default \ --spec-file ./nss-job.json \ @@ -208,5 +210,5 @@ Requires `RUN_NSS_LOCAL_E2E=1`, CUDA, and `nemo safe-synthesizer runtime setup`. ## Related topics -- [Getting Started](../getting-started.md) — GPU and local runtime prerequisites -- Plugin README: `plugins/nemo-safe-synthesizer/README.md` +- [Getting Started](/documentation/synthesize-safe-data) — GPU and local runtime prerequisites +- Plugin README: `plugins/nemo-safe-synthesizer/README.md` \ No newline at end of file diff --git a/docs/safe-synthesizer/about/index.md b/docs/safe-synthesizer/about/index.md deleted file mode 100644 index 4edd6d815a..0000000000 --- a/docs/safe-synthesizer/about/index.md +++ /dev/null @@ -1,121 +0,0 @@ - - -# About Generating Safe Synthetic Data - -{{nss_long_name}} enables you to create private versions of sensitive tabular datasets. The resulting data is entirely synthetic, with no one-to-one mapping to your original records. {{nss_short_name}} is purpose-built for privacy compliance and data protection while preserving data utility for downstream AI tasks. - -[**Quickstart**](../getting-started.md){ .md-button .md-button--primary } -[**Tutorials**](../tutorials/index.md){ .md-button } - ---- - -{{nss_short_name}} allows you to generate synthetic data that maintains the statistical properties of your original dataset without exposing sensitive information about individual records. - -{{nss_short_name}} is best when you have the data you need, but because it is private or sensitive in nature you cannot use it as-is. {{nss_short_name}} interpolates from existing data to generate a private, synthetic version, where new records have no one-to-one mapping to original records. If you do not have any data or want to extrapolate based on a very small set of examples, refer to [index](../../data-designer/index.md). NeMo Data Designer supports synthetic data creation from scratch or small seed for AI training and development use cases. - - -## {{nss_short_name}} Job - -A complete {{nss_short_name}} job consists of the following steps: - -1. [Upload Data](../../get-started/concepts/manage-files.md): Add your tabular data to the Files API -2. Prepare Data: - - [Configure PII Replacement](pii-replacement.md): Set up detection and replacement of sensitive information (recommended prior to the Synthesis step to ensure the model has no chance of learning the most sensitive information like names and addresses) - - Configure training data organization and holdout splits -3. Configure Synthesis: - - [Training](data-synthesis.md): Set model selection and training parameters including differential privacy - - Generate synthetic records - - [Evaluation](evaluation.md): Assess quality and privacy -4. Execute and Review: - - [Run and Monitor Job](jobs.md): Execute the job and track progress - - Download synthetic data and evaluation reports - -Find all Safe Synthesizer configuration parameters in [Parameters Reference](reference.md). - ---- - -## Installation Options - -Try out this early access API using Docker Compose or deploying the {{platform_name}} Helm chart. - -
- -- **[Quickstart](../getting-started.md)** - - --- - - Get started with the {{nss_short_name}} microservice locally using the {{platform_name}} CLI. Easiest for local testing. - - standalone - -- **[Helm Chart](../../set-up/helm/index.md)** - - --- - - Deploy the {{platform_name}} Helm Chart, which includes {{nss_short_name}}. - - helm-chart - -
- ---- - -## Tutorials - -Get hands-on experience with Safe Synthesizer through step-by-step tutorials. - -
- -- **[Tutorials](../tutorials/index.md)** - - --- - - Learn how to use {{nss_short_name}} with hands-on tutorials covering basics to advanced topics. - - beginner intermediate - -
- ---- - -## Core Concepts - -
- -- **[Data Synthesis](data-synthesis.md)** - - --- - - Learn about LLM-based synthesis, differential privacy, and tabular fine-tuning for generating synthetic data. - -- **[PII Replacement](pii-replacement.md)** - - --- - - Understand how PII detection and replacement works to protect sensitive information before synthesis. - -- **[Evaluation](evaluation.md)** - - --- - - Learn about quality and privacy metrics used to assess synthetic data including SQS and DPS scores. - -- **[Jobs](jobs.md)** - - --- - - Understand the job lifecycle, configuration, and execution for Safe Synthesizer pipelines. - -- **[Local and Subprocess Execution](host-local-development.md)** - - --- - - Run on a host GPU with `nemo safe-synthesizer run-local` and `runtime` commands; reuse local adapters and run plugin tests. - -- **[Parameters Reference](reference.md)** - - --- - - Reference all configuration parameters available when creating Safe Synthesizer jobs. - -
\ No newline at end of file diff --git a/docs/safe-synthesizer/about/index.mdx b/docs/safe-synthesizer/about/index.mdx new file mode 100644 index 0000000000..46c08cad48 --- /dev/null +++ b/docs/safe-synthesizer/about/index.mdx @@ -0,0 +1,118 @@ +--- +title: "About Generating Safe Synthetic Data" +description: "" +--- + + + +NVIDIA NeMo Safe Synthesizer enables you to create private versions of sensitive tabular datasets. The resulting data is entirely synthetic, with no one-to-one mapping to your original records. NeMo Safe Synthesizer is purpose-built for privacy compliance and data protection while preserving data utility for downstream AI tasks. + + + + +--- + +NeMo Safe Synthesizer allows you to generate synthetic data that maintains the statistical properties of your original dataset without exposing sensitive information about individual records. + +NeMo Safe Synthesizer is best when you have the data you need, but because it is private or sensitive in nature you cannot use it as-is. NeMo Safe Synthesizer interpolates from existing data to generate a private, synthetic version, where new records have no one-to-one mapping to original records. If you do not have any data or want to extrapolate based on a very small set of examples, refer to [index](/documentation/design-synthetic-data). NeMo Data Designer supports synthetic data creation from scratch or small seed for AI training and development use cases. + + +## NeMo Safe Synthesizer Job + +A complete NeMo Safe Synthesizer job consists of the following steps: + +1. [Upload Data](/documentation/get-started/core-concepts/manage-files): Add your tabular data to the Files API +2. Prepare Data: + - [Configure PII Replacement](/documentation/synthesize-safe-data/about/pii-replacement): Set up detection and replacement of sensitive information (recommended prior to the Synthesis step to ensure the model has no chance of learning the most sensitive information like names and addresses) + - Configure training data organization and holdout splits +3. Configure Synthesis: + - [Training](/documentation/synthesize-safe-data/about/data-synthesis): Set model selection and training parameters including differential privacy + - Generate synthetic records + - [Evaluation](/documentation/synthesize-safe-data/about/evaluation): Assess quality and privacy +4. Execute and Review: + - [Run and Monitor Job](/documentation/synthesize-safe-data/about/jobs): Execute the job and track progress + - Download synthetic data and evaluation reports + +Find all Safe Synthesizer configuration parameters in [Parameters Reference](/documentation/synthesize-safe-data/about/parameters-reference). + +--- + +## Installation Options + +Try out this early access API using Docker Compose or deploying the NeMo Platform Helm chart. + + + + + +Get started with the NeMo Safe Synthesizer microservice locally using the NeMo Platform CLI. Easiest for local testing. + +standalone + + + + +Deploy the NeMo Platform Helm Chart, which includes NeMo Safe Synthesizer. + +helm-chart + + + + + +--- + +## Tutorials + +Get hands-on experience with Safe Synthesizer through step-by-step tutorials. + + + + + +Learn how to use NeMo Safe Synthesizer with hands-on tutorials covering basics to advanced topics. + +beginner intermediate + + + + + +--- + +## Core Concepts + + + + + +Learn about LLM-based synthesis, differential privacy, and tabular fine-tuning for generating synthetic data. + + + + +Understand how PII detection and replacement works to protect sensitive information before synthesis. + + + + +Learn about quality and privacy metrics used to assess synthetic data including SQS and DPS scores. + + + + +Understand the job lifecycle, configuration, and execution for Safe Synthesizer pipelines. + + + + +Run on a host GPU with `nemo safe-synthesizer run-local` and `runtime` commands; reuse local adapters and run plugin tests. + + + + +Reference all configuration parameters available when creating Safe Synthesizer jobs. + + + + \ No newline at end of file diff --git a/docs/safe-synthesizer/about/jobs.md b/docs/safe-synthesizer/about/jobs.mdx similarity index 88% rename from docs/safe-synthesizer/about/jobs.md rename to docs/safe-synthesizer/about/jobs.mdx index 0945abe5c8..8cc79f99df 100644 --- a/docs/safe-synthesizer/about/jobs.md +++ b/docs/safe-synthesizer/about/jobs.mdx @@ -1,15 +1,18 @@ - - +--- +title: "Safe Synthesizer Jobs" +description: "" +--- + + -# Safe Synthesizer Jobs -{{nss_short_name}} jobs orchestrate the complete pipeline from data preparation through synthesis to evaluation. Understanding the job lifecycle and configuration options is essential for effective use of the platform. +NeMo Safe Synthesizer jobs orchestrate the complete pipeline from data preparation through synthesis to evaluation. Understanding the job lifecycle and configuration options is essential for effective use of the platform. + - ## Job Lifecycle -A {{nss_short_name}} job progresses through several states: +A NeMo Safe Synthesizer job progresses through several states: ### Job States @@ -109,7 +112,7 @@ When the job completes, access: For **platform jobs**, set `pretrained_model_job` in the job spec to a completed job that has an **`adapter`** result in Files. Reuse is generation-only (no retraining). Use either `pretrained_model_job` or `config.training.pretrained_model`, not both. -For **host-local** development (`nemo safe-synthesizer run-local`), set `config.training.pretrained_model` to a local adapter or work directory from an earlier run. See [Local and Subprocess Execution](host-local-development.md). +For **host-local** development (`nemo safe-synthesizer run-local`), set `config.training.pretrained_model` to a local adapter or work directory from an earlier run. See [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution). ## Job Builder API @@ -193,7 +196,7 @@ for log in job.fetch_logs(): ### Docker Compose Deployments -When running {{nss_short_name}} using Docker Compose, view container logs directly: +When running NeMo Safe Synthesizer using Docker Compose, view container logs directly: ```bash # View safe-synthesizer service logs @@ -309,7 +312,7 @@ kubectl get events -n --sort-by='.lastTimestamp' ## Related Topics -- [Local and Subprocess Execution](host-local-development.md): `run-local`, adapter reuse, and plugin tests -- [safe-synthesizer-101](../tutorials/safe-synthesizer-101.md): Get started with {{nss_short_name}} jobs -- [index](../tutorials/index.md): More hands-on tutorials -- [reference](reference.md): Full parameter reference +- [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution): `run-local`, adapter reuse, and plugin tests +- [safe-synthesizer-101](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101): Get started with NeMo Safe Synthesizer jobs +- [index](/documentation/synthesize-safe-data/tutorials): More hands-on tutorials +- [reference](/documentation/synthesize-safe-data/about/parameters-reference): Full parameter reference \ No newline at end of file diff --git a/docs/safe-synthesizer/about/pii-replacement.md b/docs/safe-synthesizer/about/pii-replacement.mdx similarity index 93% rename from docs/safe-synthesizer/about/pii-replacement.md rename to docs/safe-synthesizer/about/pii-replacement.mdx index 21601831a5..20b3bc2b3f 100644 --- a/docs/safe-synthesizer/about/pii-replacement.md +++ b/docs/safe-synthesizer/about/pii-replacement.mdx @@ -1,7 +1,10 @@ - - +--- +title: "PII Replacement" +description: "" +--- + + -# PII Replacement PII (Personally Identifiable Information) replacement is a critical privacy protection step that detects and replaces sensitive information in your datasets before synthesis. This ensures that the model has no chance of learning the most sensitive information like names, addresses, and other identifiers. @@ -16,7 +19,7 @@ The PII replacement pipeline operates in multiple stages: ## Detection Methods -{{nss_short_name}} supports multiple PII detection approaches: +NeMo Safe Synthesizer supports multiple PII detection approaches: ### Nemotron PII Detection @@ -172,7 +175,7 @@ Beyond these built-in types, you can define custom entities using: ## Configuration -PII replacement is configured through the `replace_pii` section. For the full schema, refer to [reference](reference.md). +PII replacement is configured through the `replace_pii` section. For the full schema, refer to [reference](/documentation/synthesize-safe-data/about/parameters-reference). ```json { @@ -214,5 +217,5 @@ PII replacement is always recommended as a preprocessing step before synthesis. ## Related Topics -- [safe-synthesizer-101](../tutorials/safe-synthesizer-101.md): Getting started tutorial with PII replacement -- [index](../tutorials/index.md): More tutorials +- [safe-synthesizer-101](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101): Getting started tutorial with PII replacement +- [index](/documentation/synthesize-safe-data/tutorials): More tutorials diff --git a/docs/safe-synthesizer/about/reference.md b/docs/safe-synthesizer/about/reference.mdx similarity index 83% rename from docs/safe-synthesizer/about/reference.md rename to docs/safe-synthesizer/about/reference.mdx index c049eba082..6fb8454c0a 100644 --- a/docs/safe-synthesizer/about/reference.md +++ b/docs/safe-synthesizer/about/reference.mdx @@ -1,11 +1,14 @@ - - +--- +title: "Parameters Reference" +description: "" +--- + # Parameters Reference This page summarizes the main configuration groups available when creating -{{nss_short_name}} jobs. For generated REST API schema details, see the -[Safe Synthesizer API Reference](../../api/index.md#tag-safe-synthesizer). +NeMo Safe Synthesizer jobs. For generated REST API schema details, see the +[Safe Synthesizer API Reference](/documentation/reference/api-reference). ## Job Spec (Plugin / REST) @@ -17,7 +20,7 @@ Top-level fields on the Safe Synthesizer job spec (alongside `config`): | `pretrained_model_job` | Prior completed job whose **`adapter`** result in Files is reused for **generation-only** synthesis. Format: `` or `/`. Mutually exclusive with `config.training.pretrained_model`. | | `hf_token_secret` | Platform secret name for Hugging Face token during model initialization | -For host-local runs, see [Local and Subprocess Execution](host-local-development.md). Reuse a local adapter with `config.training.pretrained_model`, not `pretrained_model_job`. +For host-local runs, see [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution). Reuse a local adapter with `config.training.pretrained_model`, not `pretrained_model_job`. ## Top-Level Configuration @@ -26,7 +29,7 @@ The `SafeSynthesizerParameters` schema defines the main configuration structure ### SafeSynthesizerParameters All fields are optional at the top level. For nested field constraints, see the -[Safe Synthesizer API Reference](../../api/index.md#tag-safe-synthesizer) and +[Safe Synthesizer API Reference](/documentation/reference/api-reference) and search for the schema name in the **Type** column. | Field | Type | Constraints / description | @@ -76,7 +79,7 @@ Configuration for synthetic data quality and privacy assessment, including MIA, ## PII Replacement Configuration -Configuration for PII detection and replacement. See [pii-replacement](../about/pii-replacement.md) for conceptual documentation. +Configuration for PII detection and replacement. See [pii-replacement](/documentation/synthesize-safe-data/about/pii-replacement) for conceptual documentation. ### Column Classification Config (`replace_pii.globals.classify`) @@ -128,6 +131,6 @@ job = builder.create_job(name="my-job", project="my-project") ## Related Topics -- [data-synthesis](../about/data-synthesis.md) - Learn about synthesis concepts -- [evaluation](../about/evaluation.md) - Learn about evaluation metrics -- [index](../tutorials/index.md) - Hands-on tutorials +- [data-synthesis](/documentation/synthesize-safe-data/about/data-synthesis) - Learn about synthesis concepts +- [evaluation](/documentation/synthesize-safe-data/about/evaluation) - Learn about evaluation metrics +- [index](/documentation/synthesize-safe-data/tutorials) - Hands-on tutorials diff --git a/docs/safe-synthesizer/getting-started.md b/docs/safe-synthesizer/getting-started.mdx similarity index 51% rename from docs/safe-synthesizer/getting-started.md rename to docs/safe-synthesizer/getting-started.mdx index 0b3f607351..95ffdd1f5d 100644 --- a/docs/safe-synthesizer/getting-started.md +++ b/docs/safe-synthesizer/getting-started.mdx @@ -1,26 +1,28 @@ +--- +title: "Getting Started with NeMo Safe Synthesizer" +description: "" +--- -# Getting Started with {{nss_short_name}} -Get started with {{nss_short_name}} for generating private synthetic versions of sensitive tabular datasets on a host GPU. +Get started with NeMo Safe Synthesizer for generating private synthetic versions of sensitive tabular datasets on a host GPU. ## Prerequisites -Before using {{nss_short_name}}, complete [Setup](../get-started/setup.md) to install the CLI/SDK. +Before using NeMo Safe Synthesizer, complete [Setup](/documentation/get-started) to install the CLI/SDK. -{{nss_short_name}} has the following additional requirements: +NeMo Safe Synthesizer has the following additional requirements: - An NVIDIA GPU **on the host machine** with 80GB+ VRAM (check with `nvidia-smi`). This is separate from any GPU inside a NIM container; Safe Synthesizer training runs directly on the host. - Sufficient disk space for generated datasets (50GB+ recommended) -For general platform troubleshooting (port conflicts, health checks, and so on), refer to [Setup](../get-started/setup.md). - ---8<-- "_snippets/nvidia-build-model-provider.md" +For general platform troubleshooting (port conflicts, health checks, and so on), refer to [Setup](/documentation/get-started). + --- ## Host-local CLI -For GPU development on your machine, install the Safe Synthesizer plugin from this repository and use `nemo safe-synthesizer run-local` (see [Local and Subprocess Execution](about/host-local-development.md)): +For GPU development on your machine, install the Safe Synthesizer plugin from this repository and use `nemo safe-synthesizer run-local` (see [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution)): ```shell BOOTSTRAP_LOCAL_PLUGIN_DIRS=plugins/nemo-safe-synthesizer make bootstrap-python @@ -39,8 +41,8 @@ The `run-local` command launches the Safe Synthesizer task in a separate runtime Create your first synthetic dataset: -- [Safe Synthesizer 101 Tutorial](tutorials/safe-synthesizer-101.md) - a beginner-friendly introduction -- [Local and Subprocess Execution](about/host-local-development.md) - local CLI and runtime task details -- [SDK Resources](sdk-resources.md) - Python SDK methods for jobs, builders, logs, and results +- [Safe Synthesizer 101 Tutorial](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101) - a beginner-friendly introduction +- [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution) - local CLI and runtime task details +- [SDK Resources](/documentation/synthesize-safe-data/sdk-resources) - Python SDK methods for jobs, builders, logs, and results ---- +--- \ No newline at end of file diff --git a/docs/safe-synthesizer/sdk-resources.md b/docs/safe-synthesizer/sdk-resources.mdx similarity index 78% rename from docs/safe-synthesizer/sdk-resources.md rename to docs/safe-synthesizer/sdk-resources.mdx index 736587227b..1f3c5ddaf9 100644 --- a/docs/safe-synthesizer/sdk-resources.md +++ b/docs/safe-synthesizer/sdk-resources.mdx @@ -1,7 +1,10 @@ +--- +title: "Safe Synthesizer NeMo Platform SDK Resources" +description: "" +--- -# Safe Synthesizer {{platform_name}} SDK Resources -The `nemo_safe_synthesizer_plugin.sdk` module provides {{platform_name}}-specific helpers for creating and monitoring {{nss_short_name}} jobs. Use these objects when you want to submit jobs through the platform Jobs service and retrieve platform-managed results. +The `nemo_safe_synthesizer_plugin.sdk` module provides NeMo Platform-specific helpers for creating and monitoring NeMo Safe Synthesizer jobs. Use these objects when you want to submit jobs through the platform Jobs service and retrieve platform-managed results. ## SafeSynthesizerResource @@ -23,13 +26,13 @@ An async variant with the same namespace is available as `AsyncNeMoPlatform.safe ## SafeSynthesizerJobsResource -`client.safe_synthesizer.jobs` calls the plugin API for {{nss_short_name}} jobs. +`client.safe_synthesizer.jobs` calls the plugin API for NeMo Safe Synthesizer jobs. | Method | Description | |--------|-------------| -| `create(*, spec, name=None, workspace=None, project=None, timeout=None, **params)` | Creates a {{nss_short_name}} platform job from a job spec. | -| `list(*, workspace=None, **params)` | Lists {{nss_short_name}} jobs in a workspace. | -| `retrieve(name, *, workspace=None)` | Retrieves one {{nss_short_name}} job by name. | +| `create(*, spec, name=None, workspace=None, project=None, timeout=None, **params)` | Creates a NeMo Safe Synthesizer platform job from a job spec. | +| `list(*, workspace=None, **params)` | Lists NeMo Safe Synthesizer jobs in a workspace. | +| `retrieve(name, *, workspace=None)` | Retrieves one NeMo Safe Synthesizer job by name. | | `get_status(name, *, workspace=None)` | Returns the job status from the platform Jobs service. | | `get_logs(name, *, workspace=None, **kwargs)` | Returns paginated job logs from the platform Jobs service. | @@ -92,6 +95,6 @@ job = ( ## Related Topics -- [Safe Synthesizer 101](tutorials/safe-synthesizer-101.md) - submit and monitor a first job -- [Safe Synthesizer Jobs](about/jobs.md) - understand job lifecycle and troubleshooting -- [Parameters Reference](about/reference.md) - review job spec and configuration fields +- [Safe Synthesizer 101](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101) - submit and monitor a first job +- [Safe Synthesizer Jobs](/documentation/synthesize-safe-data/about/jobs) - understand job lifecycle and troubleshooting +- [Parameters Reference](/documentation/synthesize-safe-data/about/parameters-reference) - review job spec and configuration fields diff --git a/docs/safe-synthesizer/tutorials/differential-privacy.md b/docs/safe-synthesizer/tutorials/differential-privacy.mdx similarity index 94% rename from docs/safe-synthesizer/tutorials/differential-privacy.md rename to docs/safe-synthesizer/tutorials/differential-privacy.mdx index 61c73521ac..3a47167ded 100644 --- a/docs/safe-synthesizer/tutorials/differential-privacy.md +++ b/docs/safe-synthesizer/tutorials/differential-privacy.mdx @@ -1,15 +1,18 @@ - - +--- +title: "Differential Privacy Tutorial" +description: "" +--- + + -# Differential Privacy Tutorial Learn how to apply differential privacy to achieve the maximum level of privacy with mathematical guarantees. This tutorial explores the privacy-utility tradeoff and demonstrates how to configure differential privacy parameters for optimal results. -If you have not yet completed the [Safe Synthesizer 101](safe-synthesizer-101.md) tutorial, consider starting there first. +If you have not yet completed the [Safe Synthesizer 101](/documentation/synthesize-safe-data/tutorials/safe-synthesizer-101) tutorial, consider starting there first. ## Prerequisites -- Understanding of [differential privacy](../about/data-synthesis.md) +- Understanding of [differential privacy](/documentation/synthesize-safe-data/about/data-synthesis) - Safe Synthesizer deployment with GPU resources --- @@ -46,7 +49,7 @@ Differential privacy (DP) provides mathematical guarantees that synthetic data d ### Record-Level vs Group-Level Privacy -By default, {{nss_short_name}} uses **record-level** differential privacy, which protects individual records. For datasets where multiple records belong to the same entity (e.g., a patient with multiple visits), you can use **group-level** privacy by setting `group_training_examples_by` to the column that identifies each entity. See [Group-Level Privacy](#group-level-privacy) in the Advanced Configuration section for a code example. +By default, NeMo Safe Synthesizer uses **record-level** differential privacy, which protects individual records. For datasets where multiple records belong to the same entity (e.g., a patient with multiple visits), you can use **group-level** privacy by setting `group_training_examples_by` to the column that identifies each entity. See [Group-Level Privacy](#group-level-privacy) in the Advanced Configuration section for a code example. **When to use group-level privacy:** - Multiple records per person/entity in your dataset @@ -57,7 +60,7 @@ By default, {{nss_short_name}} uses **record-level** differential privacy, which ## Setup -Install the {{platform_name}} SDK with Safe Synthesizer support: +Install the NeMo Platform SDK with Safe Synthesizer support: ```shell if command -v uv &> /dev/null; then @@ -381,7 +384,7 @@ check_data_requirements(len(df)) **Guidelines:** - **10,000+ records**: Ideal for differential privacy - **5,000-10,000 records**: May work, use higher epsilon -- **< 5,000 records**: Consider quality trade-offs carefully +- **\< 5,000 records**: Consider quality trade-offs carefully ### Choosing Epsilon diff --git a/docs/safe-synthesizer/tutorials/index.md b/docs/safe-synthesizer/tutorials/index.md deleted file mode 100644 index ab43168d4e..0000000000 --- a/docs/safe-synthesizer/tutorials/index.md +++ /dev/null @@ -1,76 +0,0 @@ - -# Tutorials - -Learn how to run {{nss_short_name}} jobs through hands-on tutorials to generate private synthetic versions of sensitive tabular datasets. Each tutorial provides step-by-step guidance with executable code examples. - -## Prerequisites - -Before starting any tutorial, ensure you have: - -- [{{nss_short_name}} deployed](../getting-started.md) using Docker Compose or Helm -- Python environment with `nemo-platform` SDK installed: - ```bash - pip install nemo-platform[all] - ``` -- Jupyter environment for running tutorial notebooks - ---- - -## Getting Started - -
- -- **[Safe Synthesizer 101](safe-synthesizer-101.md)** - - --- - - Learn the basics with your first Safe Synthesizer job, leveraging smart defaults. This tutorial covers uploading data, running a synthesis job with PII replacement, and reviewing evaluation reports. - - **Topics covered:** - - Installing the SDK - - Connecting to Safe Synthesizer - - Using `SafeSynthesizerJobBuilder` - - Monitoring job progress - - Retrieving synthetic data and evaluation reports - - beginner 20 minutes - -
- ---- - -## Advanced Topics - -
- -- **[Differential Privacy Tutorial](differential-privacy.md)** - - --- - - Apply differential privacy to achieve the maximum level of privacy with mathematical guarantees. This tutorial explores the privacy-utility tradeoff and how to configure differential privacy parameters. - - **Topics covered:** - - Understanding differential privacy concepts (epsilon, delta) - - Configuring privacy hyperparameters - - Privacy budget analysis - - Evaluating privacy-utility tradeoffs - - Interpreting privacy metrics - - intermediate 1.5 hours - -
- ---- - -## Additional Resources - -After completing these tutorials, explore: - -- **[index](../about/index.md)**: Understand core concepts and components - ---- - -## Need Help? - -- Check the [GitHub Issues](https://github.com/NVIDIA/GenerativeAIExamples/issues) for known issues -- Review the [jobs](../about/jobs.md) guide for job management diff --git a/docs/safe-synthesizer/tutorials/index.mdx b/docs/safe-synthesizer/tutorials/index.mdx new file mode 100644 index 0000000000..2f87ddfe8d --- /dev/null +++ b/docs/safe-synthesizer/tutorials/index.mdx @@ -0,0 +1,79 @@ +--- +title: "Tutorials" +description: "" +--- + + +Learn how to run NeMo Safe Synthesizer jobs through hands-on tutorials to generate private synthetic versions of sensitive tabular datasets. Each tutorial provides step-by-step guidance with executable code examples. + +## Prerequisites + +Before starting any tutorial, ensure you have: + +- [NeMo Safe Synthesizer deployed](/documentation/synthesize-safe-data) using Docker Compose or Helm +- Python environment with `nemo-platform` SDK installed: + ```bash + pip install nemo-platform[all] + ``` +- Jupyter environment for running tutorial notebooks + +--- + +## Getting Started + + + + + +Learn the basics with your first Safe Synthesizer job, leveraging smart defaults. This tutorial covers uploading data, running a synthesis job with PII replacement, and reviewing evaluation reports. + +**Topics covered:** +- Installing the SDK +- Connecting to Safe Synthesizer +- Using `SafeSynthesizerJobBuilder` +- Monitoring job progress +- Retrieving synthetic data and evaluation reports + +beginner 20 minutes + + + + + +--- + +## Advanced Topics + + + + + +Apply differential privacy to achieve the maximum level of privacy with mathematical guarantees. This tutorial explores the privacy-utility tradeoff and how to configure differential privacy parameters. + +**Topics covered:** +- Understanding differential privacy concepts (epsilon, delta) +- Configuring privacy hyperparameters +- Privacy budget analysis +- Evaluating privacy-utility tradeoffs +- Interpreting privacy metrics + +intermediate 1.5 hours + + + + + +--- + +## Additional Resources + +After completing these tutorials, explore: + +- **[index](/documentation/synthesize-safe-data/about)**: Understand core concepts and components + +--- + +## Need Help? + +- Check the [GitHub Issues](https://github.com/NVIDIA/GenerativeAIExamples/issues) for known issues +- Review the [jobs](/documentation/synthesize-safe-data/about/jobs) guide for job management diff --git a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx similarity index 73% rename from docs/safe-synthesizer/tutorials/safe-synthesizer-101.md rename to docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx index 5912bcccc3..010fedb346 100644 --- a/docs/safe-synthesizer/tutorials/safe-synthesizer-101.md +++ b/docs/safe-synthesizer/tutorials/safe-synthesizer-101.mdx @@ -1,15 +1,16 @@ - - +--- +title: "Safe Synthesizer 101" +description: "" +--- -# Safe Synthesizer 101 -Learn the fundamentals of {{nss_short_name}} by creating your first Safe Synthesizer job using provided defaults. In this tutorial, you'll upload sample customer data, replace personally identifiable information, fine-tune a model, generate synthetic records, and review the evaluation report. +Learn the fundamentals of NeMo Safe Synthesizer by creating your first Safe Synthesizer job using provided defaults. In this tutorial, you'll upload sample customer data, replace personally identifiable information, fine-tune a model, generate synthetic records, and review the evaluation report. ## Prerequisites Before you begin, make sure that you have: -- Access to a deployment of {{nss_short_name}} (see [getting-started](../getting-started.md)) +- Access to a deployment of NeMo Safe Synthesizer (see [getting-started](/documentation/synthesize-safe-data)) - **An NVIDIA GPU with 80 GB+ VRAM** — Safe Synthesizer requires GPU access for model training, even when using remote inference for other services. Verify with `nvidia-smi`. - Python environment with `nemo-platform` SDK installed - Basic understanding of Python and pandas @@ -29,7 +30,7 @@ By the end of this tutorial, you'll understand how to: ## Step 1: Install the SDK -Install the {{platform_name}} SDK with Safe Synthesizer support. Run the following command in a **terminal (shell)**: +Install the NeMo Platform SDK with Safe Synthesizer support. Run the following command in a **terminal (shell)**: ```shell if command -v uv &> /dev/null; then @@ -73,11 +74,8 @@ try: print("✅ Successfully connected to Safe Synthesizer service") print(f"Found {len(jobs.data)} existing jobs") except Exception as e: - raise RuntimeError( - "Cannot connect to the {{nss_short_name}} service. Restart the platform after installing " - "the {{nss_plugin_slug}} plugin, then verify that /apis/{{nss_plugin_slug}}/v2/workspaces/default/jobs " - "appears in the platform OpenAPI schema." - ) from e + print(f"❌ Cannot connect to service: {e}") + print("Please verify base_url and service status") ``` --- @@ -88,7 +86,7 @@ For this tutorial, we'll use a women's clothing reviews dataset from Kaggle that ```python import pandas as pd -import kagglehub +import kagglehub # type: ignore[import-not-found] # Download the dataset path = kagglehub.dataset_download("nicapotato/womens-ecommerce-clothing-reviews") @@ -111,10 +109,11 @@ print(df.head()) Before running jobs, set up column classification for accurate PII detection. -!!! tip - Column classification uses an LLM to automatically detect column types and improve PII detection accuracy. Without this setup, you may see classification errors and reduced detection quality. + ---8<-- "_snippets/nvidia-build-model-provider.md" +Column classification uses an LLM to automatically detect column types and improve PII detection accuracy. Without this setup, you may see classification errors and reduced detection quality. + + ```python # Use the pre-configured NVIDIA Build model provider @@ -129,8 +128,11 @@ else: print(f"✅ Using model provider: {provider_name}") ``` -!!! note - If you prefer not to send column data to `build.nvidia.com`, you can [deploy your own LLM](../../run-inference/tutorials/deploy-models.md) and create a custom model provider. Pass the fully-qualified provider name (`workspace/provider-name`) to `.with_classify_model_provider()` instead. + + +If you prefer not to send column data to `build.nvidia.com`, you can deploy your own LLM and create a custom model provider. Pass the fully-qualified provider name (`workspace/provider-name`) to `.with_classify_model_provider()` instead. + + --- @@ -237,8 +239,6 @@ except RuntimeError as e: If the job fails with **"No GPUs available on this system"**, ensure your quickstart is configured with GPU access: - - ```bash nemo setup --start-services ``` @@ -251,8 +251,6 @@ Verify GPU access with `nvidia-smi` on the host. Once the job is complete, retrieve the generated synthetic data: - - ```python synthetic_df = job.fetch_data() @@ -314,43 +312,6 @@ job.display_report_in_notebook() --- -## Step 11: Generate More Records from the Same Adapter - -The completed job stores its LoRA adapter as an `adapter` result in Files. You can submit a second job with `pretrained_model_job` to reuse that adapter and skip training. This is useful when you want more synthetic records from the same trained model. - -```python -reuse_job_name = f"synthesis-reuse-{pd.Timestamp.now().strftime('%Y%m%d-%H%M%S')}" -reuse_builder = ( - SafeSynthesizerJobBuilder(client) - .with_data_source(df) - .with_pretrained_model_job(job.job_name) - .with_generate(num_records=100) -) - -reuse_job = reuse_builder.create_job(name=reuse_job_name, project=project_name) -print(f"✅ Adapter reuse job created: {reuse_job.job_name}") -``` - -Wait for the generation-only job and fetch the additional records: - -```python -print("⏳ Waiting for adapter reuse job to complete...") -try: - reuse_job.wait_for_completion() - print("✅ Adapter reuse job completed!") -except RuntimeError as e: - print(f"❌ Adapter reuse job failed: {e}") - raise - -additional_synthetic_df = reuse_job.fetch_data() -print(f"✅ Generated {len(additional_synthetic_df)} additional synthetic records") -print(additional_synthetic_df.head()) -``` - -`pretrained_model_job` accepts either a job in the current workspace (`job.job_name`) or a fully-qualified `/` reference. Do not set `config.training.pretrained_model` when using `pretrained_model_job`; that local path field is only for `run-local` workflows. - ---- - ## Understanding the Results ### Interpreting Scores @@ -363,8 +324,8 @@ The evaluation report contains two high-level scores: Synthetic Quality Score (S Now that you've completed your first Safe Synthesizer job, try the local CLI path: -- [Local and Subprocess Execution](../about/host-local-development.md) - run Safe Synthesizer directly on a host GPU -- [Getting Started](../getting-started.md) - review local runtime prerequisites +- [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution) - run Safe Synthesizer directly on a host GPU +- [Getting Started](/documentation/synthesize-safe-data) - review local runtime prerequisites ### Try These Next @@ -413,7 +374,7 @@ print(f"Total jobs: {len(all_jobs.data)}") - Use smaller model (adjust `training.pretrained_model`) - Check GPU availability -For local CLI failures, see [Local and Subprocess Execution](../about/host-local-development.md). +For local CLI failures, see [Local and Subprocess Execution](/documentation/synthesize-safe-data/about/local-and-subprocess-execution). **Error: "Dataset must have at least 200 records to use holdout."** @@ -432,6 +393,9 @@ builder = ( ) ``` -!!! warning - Disabling holdout means you won't get quality metrics like privacy scores and synthetic data quality - scores. For production use, ensure your dataset has at least 200 records. + + +Disabling holdout means you won't get quality metrics like privacy scores and synthetic data quality +scores. For production use, ensure your dataset has at least 200 records. + + \ No newline at end of file diff --git a/docs/set-up/config-reference.md b/docs/set-up/config-reference.mdx similarity index 97% rename from docs/set-up/config-reference.md rename to docs/set-up/config-reference.mdx index 2b18566218..d629232cd8 100644 --- a/docs/set-up/config-reference.md +++ b/docs/set-up/config-reference.mdx @@ -1,6 +1,9 @@ +--- +title: "NeMo Platform configuration reference" +description: "" +--- (platform-config-reference)= -# NeMo Platform configuration reference This document describes the structure and defaults for the global config file for the NeMo Platform. All sections are shown in YAML format with inline comments for description, default, and possible values. @@ -14,15 +17,15 @@ Platform-wide configuration settings. platform: # Runtime used by the platform. Used to auto-detect default backends to use. | default: 'docker' | values: 'kubernetes' | 'docker' | 'none' runtime: docker - # Base URL for the {{platform_name}} api. Used as the default URL for all services. | default: 'http://localhost:8080' + # Base URL for the NeMo Platform api. Used as the default URL for all services. | default: 'http://localhost:8080' base_url: http://localhost:8080 # Optional loopback address override for job containers to reach platform services. If not specified, automatically determined based on platform: macOS uses 'host.docker.internal', Docker containers use container hostname, Linux host network uses no override. Can be set via config file or NMP_LOOPBACK_ADDRESS env var. loopback_address: # Global image pull secrets for the platform image_pull_secrets: [] - # Docker registry for {{platform_name}} images (e.g., 'nvcr.io/nvidia/nemo-microservices'). | default: 'my-registry' + # Docker registry for NeMo Platform images (e.g., 'nvcr.io/nvidia/nemo-microservices'). | default: 'my-registry' image_registry: my-registry - # Default tag for {{platform_name}} images. | default: 'local' + # Default tag for NeMo Platform images. | default: 'local' image_tag: local # Shared Docker configuration for services using Docker backends. docker: @@ -44,13 +47,13 @@ Common configuration shared by all services. service: # Format for logs generated by the service | default: 'plain' | values: 'json' | 'plain' log_format: plain - # Logging level for the {{platform_name}}. | default: 'INFO' | values: 'DEBUG' | 'INFO' | 'WARN' | 'ERROR' + # Logging level for the NeMo Platform. | default: 'INFO' | values: 'DEBUG' | 'INFO' | 'WARN' | 'ERROR' log_level: INFO - # Scheme for the {{platform_name}} service. | default: 'http' + # Scheme for the NeMo Platform service. | default: 'http' scheme: http - # Host for the {{platform_name}} service. | default: '127.0.0.1' + # Host for the NeMo Platform service. | default: '127.0.0.1' host: 127.0.0.1 - # Port for the {{platform_name}} service. | default: 8080 + # Port for the NeMo Platform service. | default: 8080 port: 8080 ``` @@ -80,7 +83,7 @@ auth: issuer: '' # Additional valid issuers for token validation. Useful for Azure AD where access tokens use v1.0 issuer format (https://sts.windows.net/{tenant}/) while endpoints use v2.0. additional_issuers: [] - # OAuth client ID for this {{platform_name}} deployment. Used for device flow and token audience validation. | default: '' + # OAuth client ID for this NeMo Platform deployment. Used for device flow and token audience validation. | default: '' client_id: '' # Override authorization endpoint (defaults to discovery). authorization_endpoint: @@ -442,7 +445,7 @@ models: docker_timeout: 600 # Networking mode for NIM containers: 'local' (port forwarding to localhost for local dev), 'dond' (container names on shared network for quickstart), 'dind' (port forwarding to docker service for DinD setups). | default: 'local' | values: 'local' | 'dond' | 'dind' models_docker_networking_mode: local - # Docker network name for 'dond' mode. NIMs will join this network to communicate with the {{platform_name}} container. Required when MODELS_DOCKER_NETWORKING_MODE='dond'. Quickstart sets this automatically via MODELS_DOCKER_NETWORK env var. | default: '' + # Docker network name for 'dond' mode. NIMs will join this network to communicate with the NeMo Platform container. Required when MODELS_DOCKER_NETWORKING_MODE='dond'. Quickstart sets this automatically via MODELS_DOCKER_NETWORK env var. | default: '' models_docker_network: '' # Container name for 'dond' mode. Used to replace localhost in URLs passed to NIMs so they can reach services via the Docker network. Quickstart sets this automatically via MODELS_DOCKER_CONTAINER_NAME env var. | default: '' models_docker_container_name: '' @@ -798,7 +801,7 @@ evaluator: results_dir: /jobs/results # Directory path in the job container for dataset files to be downloaded to and loaded from. | default: '/jobs/datasets' dataset_dir: /jobs/datasets - # Configuration for EvalFactory integration with {{platform_name}}. + # Configuration for EvalFactory integration with NeMo Platform. evalfactory: # default: 'nvcr.io/nvidia/eval-factory/agentic_eval:26.01' agentic_eval: nvcr.io/nvidia/eval-factory/agentic_eval:26.01 diff --git a/docs/set-up/helm/backup-and-restore.md b/docs/set-up/helm/backup-and-restore.mdx similarity index 94% rename from docs/set-up/helm/backup-and-restore.md rename to docs/set-up/helm/backup-and-restore.mdx index da35f6be22..8496004292 100644 --- a/docs/set-up/helm/backup-and-restore.md +++ b/docs/set-up/helm/backup-and-restore.mdx @@ -1,6 +1,8 @@ -# Backup and Restore - -{{platform_name}} use several storage systems that require backup consideration: +--- +title: "Backup and Restore" +description: "" +--- +NeMo Platform use several storage systems that require backup consideration: 1. **PostgreSQL database**: stores entity metadata and relationships 2. **Object storage**: stores model files, datasets, and other artifacts diff --git a/docs/set-up/helm/database-setup.md b/docs/set-up/helm/database-setup.mdx similarity index 83% rename from docs/set-up/helm/database-setup.md rename to docs/set-up/helm/database-setup.mdx index 92e231783f..c28de40e7b 100644 --- a/docs/set-up/helm/database-setup.md +++ b/docs/set-up/helm/database-setup.mdx @@ -1,10 +1,13 @@ +--- +title: "Database Setup" +description: "" +--- -# Database Setup -The {{platform_name}} uses a SQL-based database to store entities such as workspaces, jobs, and other records. +The NeMo Platform uses a SQL-based database to store entities such as workspaces, jobs, and other records. -By default, the {{platform_name}} Helm chart deploys an embedded PostgreSQL instance (a StatefulSet using the official Postgres image) with simplified authentication to enable quick installation. This should not be used for production use, and an external PostgreSQL instance is recommended. +By default, the NeMo Platform Helm chart deploys an embedded PostgreSQL instance (a StatefulSet using the official Postgres image) with simplified authentication to enable quick installation. This should not be used for production use, and an external PostgreSQL instance is recommended. ## External PostgreSQL Database diff --git a/docs/set-up/helm/file-storage.md b/docs/set-up/helm/file-storage.mdx similarity index 86% rename from docs/set-up/helm/file-storage.md rename to docs/set-up/helm/file-storage.mdx index 4849fb0d4f..dc155f1a82 100644 --- a/docs/set-up/helm/file-storage.md +++ b/docs/set-up/helm/file-storage.mdx @@ -1,7 +1,10 @@ +--- +title: "File Storage" +description: "" +--- -# File Storage -The {{platform_name}} Files service stores uploaded files and datasets. By default, it uses filesystem storage backed by a Kubernetes ReadWriteMany PersistentVolumeClaim (PVC). Alternatively, you can configure S3-compatible object storage. +The NeMo Platform Files service stores uploaded files and datasets. By default, it uses filesystem storage backed by a Kubernetes ReadWriteMany PersistentVolumeClaim (PVC). Alternatively, you can configure S3-compatible object storage. ## Prerequisites @@ -16,7 +19,7 @@ If using S3 storage, you must provision and manage your own bucket and credentia ## Local Storage (Default) -By default, the Files service uses local filesystem storage. Files are stored on the shared PVC configured in [Persistent Volumes](./persistent-volumes.md). +By default, the Files service uses local filesystem storage. Files are stored on the shared PVC configured in [Persistent Volumes](/documentation/self-managed-deployment/setup/helm/persistent-volumes). The default configuration is equivalent to: @@ -28,7 +31,7 @@ platformConfig: path: /vol/files ``` -You do not need to add this to your `values.yaml`. Once the PVC is set up as described in [Persistent Volumes](./persistent-volumes.md), local storage works out of the box. +You do not need to add this to your `values.yaml`. Once the PVC is set up as described in [Persistent Volumes](/documentation/self-managed-deployment/setup/helm/persistent-volumes), local storage works out of the box. ## S3 Object Storage @@ -127,9 +130,11 @@ platformConfig: use_sdk_auth: true ``` -!!! note - Some older S3-compatible systems may require `signature_version: s3` instead of the default `s3v4`. Only change this if you encounter signature-related errors. + +Some older S3-compatible systems may require `signature_version: s3` instead of the default `s3v4`. Only change this if you encounter signature-related errors. + + ### Configuration Reference | Field | Type | Default | Description | diff --git a/docs/set-up/helm/index.md b/docs/set-up/helm/index.md deleted file mode 100644 index 919d31b724..0000000000 --- a/docs/set-up/helm/index.md +++ /dev/null @@ -1,80 +0,0 @@ - -# Install {{platform_name}} with Helm - -The {{platform_name}} is bundled in an all-in-one Helm chart that supports end-to-end microservice deployment. Use the following guides to build and manage a complete data flywheel with the {{platform_name}} on your Kubernetes either on-prem or in the cloud. - -
- -- **[Prerequisites](prerequisites.md)** - - --- - - Review the prerequisites for installing the {{helm_chart_short_name}}. - - cluster-admin - -- **[Install }](install.md)** - - --- - - Install the {{platform_name}} using the Helm chart. - - cluster-admin on-prem cloud - -- **[Database Setup](database-setup.md)** - - --- - - Set up an external database for the {{platform_name}}. - - cluster-admin on-prem cloud - -- **[Ingress](ingress.md)** - - --- - - Set up Ingress for the {{platform_name}}. - - cluster-admin on-prem cloud - -- **[Persistent Volumes](persistent-volumes.md)** - - --- - - Set up persistent volumes for the {{platform_name}}. - - cluster-admin on-prem cloud - -- **[File Storage](file-storage.md)** - - --- - - Configure storage options for the Files service. - - cluster-admin on-prem cloud - -- **[Multinode Networking](multinode-networking.md)** - - --- - - Configure high-performance east-west networking (EFA, InfiniBand, TCP-XO, SR-IOV) for multi-node training. - - cluster-admin cloud - -- **[OpenShift](openshift.md)** - - --- - - Install with OpenShift-compatible configuration. - - cluster-admin openshift - -- **[Backup and Restore](backup-and-restore.md)** - - --- - - Set up backup and restore configurations for the {{platform_name}}. - - cluster-admin on-prem cloud - -
\ No newline at end of file diff --git a/docs/set-up/helm/index.mdx b/docs/set-up/helm/index.mdx new file mode 100644 index 0000000000..df9042d3ec --- /dev/null +++ b/docs/set-up/helm/index.mdx @@ -0,0 +1,75 @@ +--- +title: "Install NeMo Platform with Helm" +description: "" +--- + + +The NeMo Platform is bundled in an all-in-one Helm chart that supports end-to-end microservice deployment. Use the following guides to build and manage a complete data flywheel with the NeMo Platform on your Kubernetes either on-prem or in the cloud. + + + + + +Review the prerequisites for installing the NeMo Platform Helm Chart. + +cluster-admin + + + + +Install the NeMo Platform using the Helm chart. + +cluster-admin on-prem cloud + + + + +Set up an external database for the NeMo Platform. + +cluster-admin on-prem cloud + + + + +Set up Ingress for the NeMo Platform. + +cluster-admin on-prem cloud + + + + +Set up persistent volumes for the NeMo Platform. + +cluster-admin on-prem cloud + + + + +Configure storage options for the Files service. + +cluster-admin on-prem cloud + + + + +Configure high-performance east-west networking (EFA, InfiniBand, TCP-XO, SR-IOV) for multi-node training. + +cluster-admin cloud + + + + +Install with OpenShift-compatible configuration. + +cluster-admin openshift + + + + +Set up backup and restore configurations for the NeMo Platform. + +cluster-admin on-prem cloud + + + + diff --git a/docs/set-up/helm/ingress.md b/docs/set-up/helm/ingress.mdx similarity index 79% rename from docs/set-up/helm/ingress.md rename to docs/set-up/helm/ingress.mdx index 3e3589a41c..6b90953f98 100644 --- a/docs/set-up/helm/ingress.md +++ b/docs/set-up/helm/ingress.mdx @@ -1,9 +1,12 @@ +--- +title: "Ingress" +description: "" +--- -# Ingress -The {{platform_name}} Helm chart can expose the API service externally using **Kubernetes Ingress**, the **Gateway API HTTPRoute**, or on OpenShift an **OpenShift Route**. Choose one based on your cluster. +The NeMo Platform Helm chart can expose the API service externally using **Kubernetes Ingress**, the **Gateway API HTTPRoute**, or on OpenShift an **OpenShift Route**. Choose one based on your cluster. -**Prerequisites:** Complete [Prerequisites](./prerequisites.md) and [Install](./install.md) (or [OpenShift](./openshift.md)) so the platform is installed. Ensure your cluster has an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) controller (e.g. Traefik, OpenShift IngressController) or a [Gateway API](https://gateway-api.sigs.k8s.io/) Gateway configured. +**Prerequisites:** Complete [Prerequisites](/documentation/self-managed-deployment/setup/helm/prerequisites) and [Install](/documentation/self-managed-deployment/setup/helm/install) (or [OpenShift](/documentation/self-managed-deployment/setup/helm/open-shift)) so the platform is installed. Ensure your cluster has an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) controller (e.g. Traefik, OpenShift IngressController) or a [Gateway API](https://gateway-api.sigs.k8s.io/) Gateway configured. ## Kubernetes Ingress @@ -46,7 +49,7 @@ To use standard Kubernetes [Ingress](https://kubernetes.io/docs/concepts/service Install or upgrade with your values file so the Ingress resource is created. After installation, use the URL shown in the Helm notes (`helm status nemo-platform`) or the host you configured. -**Advanced:** For multiple hostnames or different paths per host, leave `defaultHost` unset and use `ingress.hosts` (each entry has `name` and `paths`). See the [{{helm_chart_short_name}} reference](../../helm/index.md) for the full structure. +**Advanced:** For multiple hostnames or different paths per host, leave `defaultHost` unset and use `ingress.hosts` (each entry has `name` and `paths`). See the [NeMo Platform Helm Chart reference](/documentation/self-managed-deployment/helm/helm-reference) for the full structure. ## Gateway API HTTPRoute @@ -66,7 +69,7 @@ On clusters that use the [Gateway API](https://gateway-api.sigs.k8s.io/), you ca 2. Install or upgrade with this values file. The HTTPRoute will be created and bound to the specified Gateway(s). -For full option details (e.g. `filters`, `labels`, `annotations`), see the [{{helm_chart_short_name}} reference](../../helm/index.md). +For full option details (e.g. `filters`, `labels`, `annotations`), see the [NeMo Platform Helm Chart reference](/documentation/self-managed-deployment/helm/helm-reference). ## OpenShift Route @@ -86,12 +89,12 @@ On Red Hat OpenShift you can expose the API using an [OpenShift Route](https://d 3. Install or upgrade with your values file. After installation, use the URL from the Helm notes (`helm status nemo-platform`) or the host you configured. -For all options (`targetPort`, `annotations`, `labels`), see the [{{helm_chart_short_name}} reference](../../helm/index.md). +For all options (`targetPort`, `annotations`, `labels`), see the [NeMo Platform Helm Chart reference](/documentation/self-managed-deployment/helm/helm-reference). ## Related -- [Install](./install.md) — Install steps and upgrade commands -- [OpenShift](./openshift.md) — OpenShift: security context, and optional Route (`openshiftRoute.enabled`), Ingress, or HTTPRoute +- [Install](/documentation/self-managed-deployment/setup/helm/install) — Install steps and upgrade commands +- [OpenShift](/documentation/self-managed-deployment/setup/helm/open-shift) — OpenShift: security context, and optional Route (`openshiftRoute.enabled`), Ingress, or HTTPRoute ## Cloud Provider Specific Ingress diff --git a/docs/set-up/helm/install.md b/docs/set-up/helm/install.mdx similarity index 67% rename from docs/set-up/helm/install.md rename to docs/set-up/helm/install.mdx index f82e878acf..ee5f98a825 100644 --- a/docs/set-up/helm/install.md +++ b/docs/set-up/helm/install.mdx @@ -1,11 +1,18 @@ +--- +title: "Install NeMo Platform Helm Chart" +description: "" +--- -# Install {{helm_chart_short_name}} -!!! tip "Note: This setup is the full enterprise platform, meant for advanced use. If you're just getting started, check out [setting up a local instance of the platform](../../get-started/setup.md) instead — it's faster and easier to explore the basics." + -To deploy the {{platform_name}}, follow these steps after completing the [Prerequisites](./prerequisites.md). +Note: This setup is the full enterprise platform, meant for advanced use. If you're just getting started, check out [setting up a local instance of the platform](/documentation/get-started) instead — it's faster and easier to explore the basics. -1. Add the {{helm_chart_short_name}} to your local Helm repositories. + + +To deploy the NeMo Platform, follow these steps after completing the [Prerequisites](/documentation/self-managed-deployment/setup/helm/prerequisites). + +1. Add the NeMo Platform Helm Chart to your local Helm repositories. ```sh helm repo add nmp https://helm.ngc.nvidia.com/nvidia/nemo-microservices \ @@ -16,24 +23,24 @@ To deploy the {{platform_name}}, follow these steps after completing the [Prereq helm repo update ``` -2. Review the default values in the [{{helm_chart_short_name}} reference](../../helm/index.md). To override the default values, create a custom values file. Review the following while creating your custom values file. +2. Review the default values in the [NeMo Platform Helm Chart reference](/documentation/self-managed-deployment/helm/helm-reference). To override the default values, create a custom values file. Review the following while creating your custom values file. - - To configure an external database, see [Database Setup](./database-setup.md). - - To configure persistent volumes for jobs and files storage, see [Persistent Volumes](./persistent-volumes.md). - - To configure file storage options, see [File Storage](./file-storage.md). - - To configure ingress, see [Ingress](./ingress.md). - - To configure multi-node networking, see [Multi-Node Networking](./multinode-networking.md). - - To configure OpenShift-compatible security context overrides, see [OpenShift](./openshift.md). + - To configure an external database, see [Database Setup](/documentation/self-managed-deployment/setup/helm/database-setup). + - To configure persistent volumes for jobs and files storage, see [Persistent Volumes](/documentation/self-managed-deployment/setup/helm/persistent-volumes). + - To configure file storage options, see [File Storage](/documentation/self-managed-deployment/setup/helm/file-storage). + - To configure ingress, see [Ingress](/documentation/self-managed-deployment/setup/helm/ingress). + - To configure multi-node networking, see [Multi-Node Networking](/documentation/self-managed-deployment/setup/helm/multinode-networking). + - To configure OpenShift-compatible security context overrides, see [OpenShift](/documentation/self-managed-deployment/setup/helm/open-shift). 3. Install the Volcano scheduler before installing the chart. This is required for customization jobs that leverage multiple nodes. ```sh -{% raw %} - kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v{{volcano_version}}/installer/volcano-development.yaml -{% endraw %} + + kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.9.0/installer/volcano-development.yaml + ``` - After applying, wait for the Volcano admission webhook to finish initializing before proceeding. The webhook registers immediately with `failurePolicy: Fail`, but TLS certificate generation runs asynchronously. If you proceed before the webhook is ready, all pod creation — including the {{platform_name}} Helm install — will fail with a certificate error. + After applying, wait for the Volcano admission webhook to finish initializing before proceeding. The webhook registers immediately with `failurePolicy: Fail`, but TLS certificate generation runs asynchronously. If you proceed before the webhook is ready, all pod creation — including the NeMo Platform Helm install — will fail with a certificate error. ```sh kubectl wait --for=condition=complete job/volcano-admission-init -n volcano-system --timeout=120s @@ -56,7 +63,7 @@ To deploy the {{platform_name}}, follow these steps after completing the [Prereq kubectl get pods ``` -For Red Hat OpenShift, use OpenShift-compatible security context overrides so pods satisfy the restricted SCC. See [OpenShift](./openshift.md). +For Red Hat OpenShift, use OpenShift-compatible security context overrides so pods satisfy the restricted SCC. See [OpenShift](/documentation/self-managed-deployment/setup/helm/open-shift). To upgrade the deployment with new configurations, use the following command: @@ -72,7 +79,11 @@ To uninstall the deployment, use the following command: helm uninstall nemo-platform ``` -!!! note "`helm uninstall` intentionally does **not** remove all resources:" + + +`helm uninstall` intentionally does **not** remove all resources: + + - **PVCs** are preserved to prevent accidental data loss. Delete them manually if no longer needed. - **CRDs** are not removed by Helm design ([upstream issue](https://github.com/helm/helm/issues/4840)) to avoid destroying custom resources across the cluster. - **Completed jobs, secrets, and the namespace** may also remain. @@ -86,7 +97,11 @@ kubectl delete namespace kubectl delete crd ``` -!!! warning "Deleting CRDs removes all custom resources of those types cluster-wide. Only do this if no other workloads depend on them." + + +Deleting CRDs removes all custom resources of those types cluster-wide. Only do this if no other workloads depend on them. + + ## Troubleshooting diff --git a/docs/set-up/helm/multinode-networking.md b/docs/set-up/helm/multinode-networking.mdx similarity index 88% rename from docs/set-up/helm/multinode-networking.md rename to docs/set-up/helm/multinode-networking.mdx index 097354de0a..ceb93decc6 100644 --- a/docs/set-up/helm/multinode-networking.md +++ b/docs/set-up/helm/multinode-networking.mdx @@ -1,5 +1,8 @@ +--- +title: "Multinode Networking" +description: "" +--- -# Multinode Networking Multi-node GPU training requires high-bandwidth, low-latency east-west networking between nodes. Without it, NCCL falls back to standard Ethernet and training times increase significantly. The Helm chart installs [Kyverno](https://kyverno.io/) policies that automatically inject the correct networking resources for four cloud providers. @@ -10,9 +13,11 @@ Multi-node GPU training requires high-bandwidth, low-latency east-west networkin - Nodes with high-performance networking hardware (EFA, InfiniBand, etc.) - The cloud provider's device plugin or network operator deployed so that networking resources are visible to Kubernetes -!!! note - The {{platform_name}} does not provision or manage the underlying cloud networking infrastructure. Each cloud provider section below lists the cluster-level prerequisites your administrator must configure. Refer to your cloud provider's documentation for setup instructions. + +The NeMo Platform does not provision or manage the underlying cloud networking infrastructure. Each cloud provider section below lists the cluster-level prerequisites your administrator must configure. Refer to your cloud provider's documentation for setup instructions. + + ## How It Works When a job requests more than one node, the jobs controller annotates the pod with `nmp.nvidia.com/enable-multi-node-networking: "true"`. Kyverno watches for this annotation and mutates the pod spec to inject the cloud-specific device requests, environment variables, and volume mounts that NCCL needs for high-performance inter-node communication. @@ -21,9 +26,11 @@ When a job requests more than one node, the jobs controller annotates the pod wi Enable **exactly one** cloud provider in the `multinodeNetworking` section of your `values.yaml`. Enabling more than one causes a Helm install error. -!!! info - Device-count parameters (`efaDevicesPerGPU`, `rdmaDevicesPerGPU`) must match the hardware ratio of your instance type. A mismatch causes jobs to either fail scheduling or silently run without high-speed networking. + + +Device-count parameters (`efaDevicesPerGPU`, `rdmaDevicesPerGPU`) must match the hardware ratio of your instance type. A mismatch causes jobs to either fail scheduling or silently run without high-speed networking. + ### AWS (EFA) Injects [Elastic Fabric Adapter](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html) device requests (`vpc.amazonaws.com/efa`), mounts the EFA OFI library, and configures shared memory. diff --git a/docs/set-up/helm/openshift.md b/docs/set-up/helm/openshift.mdx similarity index 64% rename from docs/set-up/helm/openshift.md rename to docs/set-up/helm/openshift.mdx index 727ef25034..4452f29d50 100644 --- a/docs/set-up/helm/openshift.md +++ b/docs/set-up/helm/openshift.mdx @@ -1,11 +1,14 @@ +--- +title: "OpenShift" +description: "" +--- -# OpenShift -The {{platform_name}} chart works on Red Hat OpenShift when security context overrides are applied. OpenShift’s **restricted** or **restricted-v2** Security Context Constraint (SCC) requires pods to run as non-root with explicit `runAsUser` and `runAsNonRoot`. +The NeMo Platform chart works on Red Hat OpenShift when security context overrides are applied. OpenShift’s **restricted** or **restricted-v2** Security Context Constraint (SCC) requires pods to run as non-root with explicit `runAsUser` and `runAsNonRoot`. ## Values -You can override the default values for the {{platform_name}} chart to make it compatible with OpenShift. +You can override the default values for the NeMo Platform chart to make it compatible with OpenShift. 1. **Create the OpenShift values file.** Save the following as `openshift-values.yaml`: @@ -34,7 +37,7 @@ You can override the default values for the {{platform_name}} chart to make it c fsGroup: 999 ``` -2. **Install** with your custom values and the OpenShift overrides (order matters; later files override earlier). Complete the [Prerequisites](./prerequisites.md) and follow [Install](./install.md), using the provided OpenShift values file. +2. **Install** with your custom values and the OpenShift overrides (order matters; later files override earlier). Complete the [Prerequisites](/documentation/self-managed-deployment/setup/helm/prerequisites) and follow [Install](/documentation/self-managed-deployment/setup/helm/install), using the provided OpenShift values file. ```sh helm upgrade --install --namespace \ @@ -47,9 +50,9 @@ You can override the default values for the {{platform_name}} chart to make it c You can expose the API using the following methods: -- **Kubernetes Ingress** — supported by OpenShift’s default IngressController; set `ingress.enabled: true` and `ingress.defaultHost` as in [Ingress](./ingress.md). -- **Gateway API HTTPRoute** — optional; configure `httpRoute` in values. See [Gateway API HTTPRoute](./ingress.md#gateway-api-httproute). -- **OpenShift Route** — enable the chart’s Route and set an optional hostname. See [OpenShift Route](./ingress.md#openshift-route) for steps. +- **Kubernetes Ingress** — supported by OpenShift’s default IngressController; set `ingress.enabled: true` and `ingress.defaultHost` as in [Ingress](/documentation/self-managed-deployment/setup/helm/ingress). +- **Gateway API HTTPRoute** — optional; configure `httpRoute` in values. See [Gateway API HTTPRoute](/documentation/self-managed-deployment/setup/helm/ingress#gateway-api-httproute). +- **OpenShift Route** — enable the chart’s Route and set an optional hostname. See [OpenShift Route](/documentation/self-managed-deployment/setup/helm/ingress#openshift-route) for steps. ## Troubleshooting diff --git a/docs/set-up/helm/persistent-volumes.md b/docs/set-up/helm/persistent-volumes.mdx similarity index 72% rename from docs/set-up/helm/persistent-volumes.md rename to docs/set-up/helm/persistent-volumes.mdx index 3baad49844..4d6c1d183a 100644 --- a/docs/set-up/helm/persistent-volumes.md +++ b/docs/set-up/helm/persistent-volumes.mdx @@ -1,7 +1,10 @@ +--- +title: "Persistent Volumes" +description: "" +--- -# Persistent Volumes -The {{platform_name}} uses persistent volume claims (PVCs) for jobs and files storage that can be mounted on multiple pods and nodes in read-write mode. This access mode is called ReadWriteMany (RWX) in Kubernetes. Using a ReadWriteMany-capable StorageClass is required for both jobs and files storage. +The NeMo Platform uses persistent volume claims (PVCs) for jobs and files storage that can be mounted on multiple pods and nodes in read-write mode. This access mode is called ReadWriteMany (RWX) in Kubernetes. Using a ReadWriteMany-capable StorageClass is required for both jobs and files storage. NVIDIA NIM microservices also scale, upgrade, and deploy more smoothly with an RWX-backed storage class. @@ -9,11 +12,13 @@ The platform does not manage storage classes. You must install an appropriate st ## Jobs and Files storage -The {{platform_name}} chart creates a single shared PVC for jobs and files storage, configured under `core.storage` in `values.yaml`. +The NeMo Platform chart creates a single shared PVC for jobs and files storage, configured under `core.storage` in `values.yaml`. -!!! note - As an alternative to PVC-based file storage, you can configure the Files service to use S3 object storage. See [File Storage](./file-storage.md) for S3 configuration options. When using S3 for files, the shared PVC is still required for jobs storage. + +As an alternative to PVC-based file storage, you can configure the Files service to use S3 object storage. See [File Storage](/documentation/self-managed-deployment/setup/helm/file-storage) for S3 configuration options. When using S3 for files, the shared PVC is still required for jobs storage. + + ### Option 1: Create a new PVC (default) 1. Confirm the `StorageClass` exists in your cluster: @@ -49,7 +54,7 @@ When set, the chart does not create a new PVC; pods mount the named volume. ## NIM storage class -For NIM deployments launched via the {{platform_name}}, you can set the default StorageClass used by NIM PVCs via platform config. In `values.yaml`, under `platformConfig`: +For NIM deployments launched via the NeMo Platform, you can set the default StorageClass used by NIM PVCs via platform config. In `values.yaml`, under `platformConfig`: ```yaml platformConfig: @@ -63,7 +68,7 @@ platformConfig: Replace `"nfs"` with your StorageClass name (e.g. `oci-nfs`, `gp3`). For NIM scaling and multi-node deployments, use a ReadWriteMany-capable StorageClass. -Refer to the [platform configuration documentation](../config-reference.md) for the full config reference. +Refer to the [platform configuration documentation](/documentation/reference/configuration-reference) for the full config reference. ## Persistent volume options @@ -91,7 +96,7 @@ For high-performance workloads: ## Azure persistent volumes -On AKS, Azure Disk (`managed-csi`) only supports ReadWriteOnce (RWO). Use **Azure Files** for the shared RWX volumes required by {{platform_name}}. PostgreSQL must use `managed-csi` because Azure Files does not support the POSIX permissions it requires. +On AKS, Azure Disk (`managed-csi`) only supports ReadWriteOnce (RWO). Use **Azure Files** for the shared RWX volumes required by NeMo Platform. PostgreSQL must use `managed-csi` because Azure Files does not support the POSIX permissions it requires. 1. Enable the [Azure Files CSI driver](https://learn.microsoft.com/en-us/azure/aks/azure-files-csi) if not already installed in your cluster. 2. Set `core.storage.storageClass` to `azurefile` (or `azurefile-csi` on AKS 1.29+) and `postgresql.persistence.storageClass` to `managed-csi`. diff --git a/docs/set-up/helm/prerequisites.md b/docs/set-up/helm/prerequisites.mdx similarity index 76% rename from docs/set-up/helm/prerequisites.md rename to docs/set-up/helm/prerequisites.mdx index 360f47efa1..44a7a09932 100644 --- a/docs/set-up/helm/prerequisites.md +++ b/docs/set-up/helm/prerequisites.mdx @@ -1,29 +1,33 @@ +--- +title: "Prerequisites" +description: "" +--- -# Prerequisites -Before installing the {{helm_chart_short_name}}, review requirements and create necessary secrets using your NGC API key. +Before installing the NeMo Platform Helm Chart, review requirements and create necessary secrets using your NGC API key. --- ## Review Requirements -To check the hardware and software specifications required for installing the {{helm_chart_short_name}}, review [](../../requirements.md). +To check the hardware and software specifications required for installing the NeMo Platform Helm Chart, review [](/documentation/reference/system-requirements). -For storage and persistent volume configuration (including ReadWriteMany and StorageClass configuration), see [](persistent-volumes.md). +For storage and persistent volume configuration (including ReadWriteMany and StorageClass configuration), see [](/documentation/self-managed-deployment/setup/helm/persistent-volumes). --- ## Create NGC API Key and Secrets -You need to have an NGC account, create an NGC API key, and create secrets using the key to access and download the {{helm_chart_short_name}} and Docker images. +You need to have an NGC account, create an NGC API key, and create secrets using the key to access and download the NeMo Platform Helm Chart and Docker images. To learn more about using Helm charts in the NGC Catalog console in general, refer to the [Helm Charts](https://docs.nvidia.com/ngc/gpu-cloud/ngc-catalog-user-guide/index.html#helm-charts) section in the _NGC Catalog User Guide_. 1. Create an NGC API key following the instructions at [Generating NGC API Keys](https://docs.nvidia.com/ngc/latest/ngc-private-registry-user-guide.html#generating-ngc-api-keys). - !!! tip + + While creating the key, make sure that you include the following permissions at a minimum: - NGC Catalog @@ -62,4 +66,4 @@ To learn more about using Helm charts in the NGC Catalog console in general, ref --namespace ``` -For more information about various secrets you might need to create and set up through the {{helm_chart_short_name}} depending on your use case, refer to [](../../get-started/concepts/manage-secrets.md). +For more information about various secrets you might need to create and set up through the NeMo Platform Helm Chart depending on your use case, refer to [](/documentation/get-started/core-concepts/manage-secrets). diff --git a/docs/set-up/index.md b/docs/set-up/index.md deleted file mode 100644 index 8d408d0984..0000000000 --- a/docs/set-up/index.md +++ /dev/null @@ -1,60 +0,0 @@ - -# About Platform Setup - -This section describes how to set up the {{platform_name}} on your Kubernetes cluster using the {{helm_chart_short_name}}. -With this chart, you can deploy the {{platform_name}} as a full deployment or a subset of the APIs as you need. - -This Platform Setup chapter is for the following personas. - -- **Cloud administrators**: Manage Kubernetes clusters and compute/storage resources. Deploy {{platform_name}} to the Kubernetes clusters on premises or cloud. - ---- - - -## {{helm_chart_short_name}} - -The [{{helm_chart_short_name}}](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo-microservices/helm-charts/nemo-platform) is an all-in-one Helm chart that bundles the complete {{platform_name}} ecosystem and all required dependencies for full platform deployment. - -You can also customize the configuration of your installation by updating the `values.yaml` file. You can also use the pre-configured tags to install only specific microservices that you need. - -For the chart assets and additional details, refer to the [{{platform_name}} Collection](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo-microservices/collections/nemo-microservices) page in the NVIDIA NGC Catalog. - ---- - -## Deploy the {{platform_name}} with Helm - -The following sections provide detailed instructions on how to deploy the {{platform_name}} using the {{helm_chart_short_name}}. - -
- -- **[Install }](helm/index.md)** - - --- - - Install the {{platform_name}} using the chart on your Kubernetes cluster. - - cluster-admin - -
- ---- - -## Other Configurations - -Review and manage other cluster settings. - -
- -- **[OpenTelemetry](opentelemetry.md)** - - --- - - Review and configure how {{platform_name}} use Open Telemetry for observability. - -- **[Milvus](milvus.md)** - - --- - - Review and configure how {{platform_name}} uses Milvus for vector database storage. - -
\ No newline at end of file diff --git a/docs/set-up/index.mdx b/docs/set-up/index.mdx new file mode 100644 index 0000000000..4b23de0546 --- /dev/null +++ b/docs/set-up/index.mdx @@ -0,0 +1,62 @@ +--- +title: "About Platform Setup" +description: "" +--- + + +This section describes how to set up the NeMo Platform on your Kubernetes cluster using the NeMo Platform Helm Chart. +With this chart, you can deploy the NeMo Platform as a full deployment or a subset of the APIs as you need. + +This Platform Setup chapter is for the following personas. + +- **Cloud administrators**: Manage Kubernetes clusters and compute/storage resources. Deploy NeMo Platform to the Kubernetes clusters on premises or cloud. + +--- + + +## NeMo Platform Helm Chart + +The [NeMo Platform Helm Chart](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo-microservices/helm-charts/nemo-platform) is an all-in-one Helm chart that bundles the complete NeMo Platform ecosystem and all required dependencies for full platform deployment. + +You can also customize the configuration of your installation by updating the `values.yaml` file. You can also use the pre-configured tags to install only specific microservices that you need. + +For the chart assets and additional details, refer to the [NeMo Platform Collection](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo-microservices/collections/nemo-microservices) page in the NVIDIA NGC Catalog. + +--- + +## Deploy the NeMo Platform with Helm + +The following sections provide detailed instructions on how to deploy the NeMo Platform using the NeMo Platform Helm Chart. + + + + + +Install the NeMo Platform using the chart on your Kubernetes cluster. + +cluster-admin + + + + + +--- + +## Other Configurations + +Review and manage other cluster settings. + + + + + +Review and configure how NeMo Platform use Open Telemetry for observability. + + + + +Review and configure how NeMo Platform uses Milvus for vector database storage. + + + + diff --git a/docs/set-up/manage-jobs.md b/docs/set-up/manage-jobs.mdx similarity index 73% rename from docs/set-up/manage-jobs.md rename to docs/set-up/manage-jobs.mdx index 45d59c0099..0e8c19d4f6 100644 --- a/docs/set-up/manage-jobs.md +++ b/docs/set-up/manage-jobs.mdx @@ -1,7 +1,10 @@ +--- +title: "Manage Jobs" +description: "" +--- -# Manage Jobs -This section describes how to configure jobs in {{platform_name}}. The Jobs service is responsible for scheduling batch jobs, collecting telemetry and managing job results. +This section describes how to configure jobs in NeMo Platform. The Jobs service is responsible for scheduling batch jobs, collecting telemetry and managing job results. ## Execution Profiles @@ -13,7 +16,7 @@ An execution profile is defined by the following attributes: - A compute provider (e.g. `cpu` or `gpu`) - An [execution backend](#execution-backends) (e.g. `docker`, `kubernetes_job`, `volcano_job`) -By default, the {{platform_name}} defines a default CPU and default GPU provider to launch CPU and GPU bound jobs. +By default, the NeMo Platform defines a default CPU and default GPU provider to launch CPU and GPU bound jobs. You can configure multiple execution profiles to suit the shape of your compute environment. For example, if you have a compute environment with heterogeneous infrastructure (e.g., two types of GPU hardware such as A100 and H200), you can define a list of execution profiles as follows: @@ -45,16 +48,16 @@ jobs: node-pool-name: h200-pool ``` -For full configuration details, see the [platform configuration reference](config-reference.md). +For full configuration details, see the [platform configuration reference](/documentation/reference/configuration-reference). ### Default Execution Profiles -The {{platform_name}} defines a default execution profile for each execution backend depending on the platform's control plane. The default execution profile is used when no specific execution profile is specified for a job. +The NeMo Platform defines a default execution profile for each execution backend depending on the platform's control plane. The default execution profile is used when no specific execution profile is specified for a job. - Default CPU Execution Profile (`cpu`): The default execution profile for CPU based jobs. - Default GPU Execution Profile (`gpu`): The default execution profile for GPU based jobs. -You may configure the default execution profiles by updating the `executor_defaults` section of the `jobs` section of the [platform configuration](config-reference.md). The structure of the `executor_defaults` matches the configuration of the execution backend configuration. +You may configure the default execution profiles by updating the `executor_defaults` section of the `jobs` section of the [platform configuration](/documentation/reference/configuration-reference). The structure of the `executor_defaults` matches the configuration of the execution backend configuration. ```yaml jobs: @@ -87,9 +90,9 @@ jobs: ## Execution Backends -Execution backends are the containerized job execution systems that {{platform_name}} jobs are scheduled to run on. Each execution backend is responsible for launching and managing the job containers, and the Jobs service communicates with the execution backend to schedule and manage jobs. +Execution backends are the containerized job execution systems that NeMo Platform jobs are scheduled to run on. Each execution backend is responsible for launching and managing the job containers, and the Jobs service communicates with the execution backend to schedule and manage jobs. -{{platform_name}} currently supports the following execution backends: +NeMo Platform currently supports the following execution backends: - Docker (`docker`) - Kubernetes Jobs (`kubernetes_job`) @@ -97,11 +100,15 @@ Execution backends are the containerized job execution systems that {{platform_n ### Docker -The {{platform_name}} supports Docker as an execution backend for CPU and GPU based jobs. When the shared GPU pool is configured (see below), Docker will use only the configured GPU devices and will not over-schedule jobs if there are not currently enough GPUs available. +The NeMo Platform supports Docker as an execution backend for CPU and GPU based jobs. When the shared GPU pool is configured (see below), Docker will use only the configured GPU devices and will not over-schedule jobs if there are not currently enough GPUs available. -**Note:** The {{platform_name}} supports Docker-based job execution by default when running the {{platform_name}} in quickstart mode, which requires no configuration. +**Note:** The NeMo Platform supports Docker-based job execution by default when running the NeMo Platform in quickstart mode, which requires no configuration. -!!! tip "If you are running GPU jobs with Docker, see [GPU Configuration](../get-started/setup.md) for information on configuring the shared GPU pool. This configuration is shared between the jobs and models services to prevent GPU resource conflicts." + + +If you are running GPU jobs with Docker, see [GPU Configuration](/documentation/get-started) for information on configuring the shared GPU pool. This configuration is shared between the jobs and models services to prevent GPU resource conflicts. + + ```yaml jobs: @@ -125,7 +132,7 @@ jobs: ### Kubernetes Jobs -The {{platform_name}} supports Kubernetes Jobs as an execution backend for CPU and GPU based jobs. +The NeMo Platform supports Kubernetes Jobs as an execution backend for CPU and GPU based jobs. ```yaml jobs: @@ -204,7 +211,7 @@ See the [KAI Scheduler documentation](https://github.com/kai-scheduler/KAI-Sched ### Volcano Jobs -The {{platform_name}} supports Volcano Jobs as an execution backend for launching distributed GPU jobs. +The NeMo Platform supports Volcano Jobs as an execution backend for launching distributed GPU jobs. Volcano Jobs are configured using the same configuration as Kubernetes Jobs, but with the following additional configuration: @@ -212,7 +219,7 @@ Volcano Jobs are configured using the same configuration as Kubernetes Jobs, but - `scheduler_name`: The Volcano scheduler to use for the job. - `plugins`: The Volcano plugins to use for the job. - `max_retry`: The maximum number of retries for the job. -- `enable_multi_node_networking`: Enable multi-node networking injection. Sets annotations to trigger Kyverno policy mutations. This is only available if the platform is configured to use multi-node networking (see [Multi-Node Networking](helm/multinode-networking.md)). +- `enable_multi_node_networking`: Enable multi-node networking injection. Sets annotations to trigger Kyverno policy mutations. This is only available if the platform is configured to use multi-node networking (see [Multi-Node Networking](/documentation/self-managed-deployment/setup/helm/multinode-networking)). ```yaml jobs: diff --git a/docs/set-up/milvus.md b/docs/set-up/milvus.md deleted file mode 100644 index 616a729530..0000000000 --- a/docs/set-up/milvus.md +++ /dev/null @@ -1,15 +0,0 @@ - -# Milvus - -{{nem_short_name}} uses Milvus for vector database storage for [Retrieval evaluations](../evaluator/metrics/retriever.md) and [RAG evaluations](../evaluator/metrics/rag.md). - -## Configuration - -To configure {{nem_short_name}} to use Milvus, set the `milvus_url` in the [platform configuration](config-reference.md): - -```yaml -evaluator: - milvus_url: "milvus-standalone.default.svc.cluster.local:19530" -``` - -See the [platform configuration reference](config-reference.md) for the complete {{nem_short_name}} configuration reference. diff --git a/docs/set-up/milvus.mdx b/docs/set-up/milvus.mdx new file mode 100644 index 0000000000..337da76d81 --- /dev/null +++ b/docs/set-up/milvus.mdx @@ -0,0 +1,18 @@ +--- +title: "Milvus" +description: "" +--- + + +NeMo Evaluator uses Milvus for vector database storage for [Retrieval evaluations](/documentation/evaluate-models/metrics) and [RAG evaluations](/documentation/evaluate-models/metrics/rag-metrics). + +## Configuration + +To configure NeMo Evaluator to use Milvus, set the `milvus_url` in the [platform configuration](/documentation/reference/configuration-reference): + +```yaml +evaluator: + milvus_url: "milvus-standalone.default.svc.cluster.local:19530" +``` + +See the [platform configuration reference](/documentation/reference/configuration-reference) for the complete NeMo Evaluator configuration reference. diff --git a/docs/set-up/opentelemetry.md b/docs/set-up/opentelemetry.md deleted file mode 100644 index 23a50bebc4..0000000000 --- a/docs/set-up/opentelemetry.md +++ /dev/null @@ -1,22 +0,0 @@ - -# OpenTelemetry Setup - -Set up OpenTelemetry configurations to gain visibility into the operations and performance of the {{platform_name}}. - -## Configuration - -The {{platform_name}} uses OpenTelemetry to collect telemetry data from the platform and services. It leverages common OpenTelemetry SDK [configuration options](https://opentelemetry.io/docs/languages/sdk-configuration/) to configure the platform deployment. - -## Helm Configuration - -The {{helm_chart_short_name}} `values.yaml` exposes OpenTelemetry SDK options to configure the platform deployment. For example, to enable OpenTelemetry for the platform: - -```yaml -telemetry: - OTEL_TRACES_EXPORTER: "otlp" - OTEL_METRICS_EXPORTER: "otlp" - OTEL_EXPORTER_OTLP_ENDPOINT: "opentelemetry-collector.monitoring:4317" - OTEL_EXPORTER_OTLP_INSECURE: true -``` - -For a complete list of the default values, refer to [Helm Configuration](../helm/index.md). diff --git a/docs/set-up/opentelemetry.mdx b/docs/set-up/opentelemetry.mdx new file mode 100644 index 0000000000..35984af0ec --- /dev/null +++ b/docs/set-up/opentelemetry.mdx @@ -0,0 +1,25 @@ +--- +title: "OpenTelemetry Setup" +description: "" +--- + + +Set up OpenTelemetry configurations to gain visibility into the operations and performance of the NeMo Platform. + +## Configuration + +The NeMo Platform uses OpenTelemetry to collect telemetry data from the platform and services. It leverages common OpenTelemetry SDK [configuration options](https://opentelemetry.io/docs/languages/sdk-configuration/) to configure the platform deployment. + +## Helm Configuration + +The NeMo Platform Helm Chart `values.yaml` exposes OpenTelemetry SDK options to configure the platform deployment. For example, to enable OpenTelemetry for the platform: + +```yaml +telemetry: + OTEL_TRACES_EXPORTER: "otlp" + OTEL_METRICS_EXPORTER: "otlp" + OTEL_EXPORTER_OTLP_ENDPOINT: "opentelemetry-collector.monitoring:4317" + OTEL_EXPORTER_OTLP_INSECURE: true +``` + +For a complete list of the default values, refer to [Helm Configuration](/documentation/self-managed-deployment/helm/helm-reference). diff --git a/docs/set-up/security.md b/docs/set-up/security.md deleted file mode 100644 index fc862f36c8..0000000000 --- a/docs/set-up/security.md +++ /dev/null @@ -1,63 +0,0 @@ - -# Security for {{platform_name}} - -This page provides security guidelines and best practices for deploying and managing {{platform_name}} in production environments. - -## Security Considerations - -- The {{platform_name}} does not impose rate limits. You must implement a rate-limiting strategy to restrict access to your application. -- The {{platform_name}} does not have an internal notion of a user. To restrict authorization to specific endpoints or users, implement an external mechanism such as an Envoy proxy. -- The {{nds_short_name}} microservice does not provide object-class-specific access controls. All items reside within a single access control boundary. -- The {{platform_name}}, by design, can access all content in the {{nds_short_name}} microservice, including LoRA adapters, training data, evaluation data, and evaluation results. - This access is required for model evaluation. - Carefully weigh the risk of data exposure from serving customized models directly to production against the overhead of a separate deployment. -- The {{platform_name}} is not intended to be internet-facing. Deploy them as the logic (middle) tier in a three-tier architecture. -- You are responsible for securing access to any application using the microservices. This includes: - - Implementing an authentication layer between users and your application - - Applying required authorization controls - - Securing communication between services in your application - -!!! note - Refer to the [NVIDIA Product Security](https://www.nvidia.com/en-us/security/psirt-policies/) page for information about subscribing to bulletins and updates, managing vulnerabilities, and reporting vulnerabilities. - -## Default Network Ports - -The following table lists the default network ports for each microservice or default database. You can override these port numbers during deployment. - -| Network Port | Microservice | -| --- | --- | -| 443/TCP | NeMo Admission Service API | -| 3000/TCP | {{nds_short_name}} API | -| 7331/TCP | {{nem_short_name}} API | -| 7331/TCP | {{ngm_short_name}} API | -| 8000/TCP | {{nim_short_name}} API | -| 8000/TCP | NeMo Retriever Text Embedding API | -| 8000/TCP | NeMo Retriever Text Reranking API | -| 8000/TCP | {{ncm_short_name}} API | -| 8000/TCP | {{nes_short_name}} API | -| 8443/TCP | {{nop_short_name}} metrics | -| 8080/TCP | Volcano Scheduler metrics | -| 9009/TCP | {{ncm_short_name}} callback | - -By default, the {{helm_chart_short_name}} configures databases with the following network ports. -Alternatively, you can configure each microservice to use an external database during installation. - -| Network Port | Database | -| --- | --- | -| 5432/TCP | {{ncm_short_name}} Database | -| 5432/TCP | {{nds_short_name}} Database | -| 5432/TCP | {{nem_short_name}} Database | -| 5432/TCP | {{nes_short_name}} Database | -| 9091/TCP | Milvus metrics | -| 19530/TCP | Milvus API | - -By default, the {{helm_chart_short_name}} installs an open telemetry collector, which uses the following network ports: - -- 4317/TCP -- 4318/TCP -- 6831/UDP -- 9411/TCP -- 14250/TCP -- 14268/TCP - -Refer to the [OpenTelemetry Documentation](https://opentelemetry.io/docs/) for more information. diff --git a/docs/set-up/security.mdx b/docs/set-up/security.mdx new file mode 100644 index 0000000000..9d9d554ca4 --- /dev/null +++ b/docs/set-up/security.mdx @@ -0,0 +1,68 @@ +--- +title: "Security for NeMo Platform" +description: "" +--- + + +This page provides security guidelines and best practices for deploying and managing NeMo Platform in production environments. + +## Security Considerations + +- The NeMo Platform does not impose rate limits. You must implement a rate-limiting strategy to restrict access to your application. +- The NeMo Platform does not have an internal notion of a user. To restrict authorization to specific endpoints or users, implement an external mechanism such as an Envoy proxy. +- The NeMo Data Store microservice does not provide object-class-specific access controls. All items reside within a single access control boundary. +- The NeMo Platform, by design, can access all content in the NeMo Data Store microservice, including LoRA adapters, training data, evaluation data, and evaluation results. + This access is required for model evaluation. + Carefully weigh the risk of data exposure from serving customized models directly to production against the overhead of a separate deployment. +- The NeMo Platform is not intended to be internet-facing. Deploy them as the logic (middle) tier in a three-tier architecture. +- You are responsible for securing access to any application using the microservices. This includes: + - Implementing an authentication layer between users and your application + - Applying required authorization controls + - Securing communication between services in your application + + + +Refer to the [NVIDIA Product Security](https://www.nvidia.com/en-us/security/psirt-policies/) page for information about subscribing to bulletins and updates, managing vulnerabilities, and reporting vulnerabilities. + + +## Default Network Ports + +The following table lists the default network ports for each microservice or default database. You can override these port numbers during deployment. + +| Network Port | Microservice | +| --- | --- | +| 443/TCP | NeMo Admission Service API | +| 3000/TCP | NeMo Data Store API | +| 7331/TCP | NeMo Evaluator API | +| 7331/TCP | NeMo Guardrails API | +| 8000/TCP | NIM API | +| 8000/TCP | NeMo Retriever Text Embedding API | +| 8000/TCP | NeMo Retriever Text Reranking API | +| 8000/TCP | NeMo Customizer API | +| 8000/TCP | NeMo Entity Store API | +| 8443/TCP | NeMo Operator metrics | +| 8080/TCP | Volcano Scheduler metrics | +| 9009/TCP | NeMo Customizer callback | + +By default, the NeMo Platform Helm Chart configures databases with the following network ports. +Alternatively, you can configure each microservice to use an external database during installation. + +| Network Port | Database | +| --- | --- | +| 5432/TCP | NeMo Customizer Database | +| 5432/TCP | NeMo Data Store Database | +| 5432/TCP | NeMo Evaluator Database | +| 5432/TCP | NeMo Entity Store Database | +| 9091/TCP | Milvus metrics | +| 19530/TCP | Milvus API | + +By default, the NeMo Platform Helm Chart installs an open telemetry collector, which uses the following network ports: + +- 4317/TCP +- 4318/TCP +- 6831/UDP +- 9411/TCP +- 14250/TCP +- 14268/TCP + +Refer to the [OpenTelemetry Documentation](https://opentelemetry.io/docs/) for more information. diff --git a/docs/studio/agents.md b/docs/studio/agents.mdx similarity index 72% rename from docs/studio/agents.md rename to docs/studio/agents.mdx index c87a32db00..74c9d0d9da 100644 --- a/docs/studio/agents.md +++ b/docs/studio/agents.mdx @@ -1,6 +1,8 @@ -# {{studio_short_name}} Agents - -Use **Agents** in the {{studio_short_name}} workspace sidebar to review and operate agent workflows managed by {{platform_name}}. +--- +title: "NeMo Studio Agents" +description: "" +--- +Use **Agents** in the NeMo Studio workspace sidebar to review and operate agent workflows managed by NeMo Platform. ## Agent List @@ -21,10 +23,10 @@ The agent side panel has two primary areas: - **Agent Details** shows the stored agent metadata and config format. - **Deployments** lists active and historical deployments for the agent, including endpoint and status. -Agents are created and updated through the `nemo agents` CLI or Agents API. {{studio_short_name}} reflects the current workspace state after the platform services refresh. +Agents are created and updated through the `nemo agents` CLI or Agents API. NeMo Studio reflects the current workspace state after the platform services refresh. ## Related Topics -- [About Agents](../agents/index.md) -- [Optimize Agents](../agents/optimization.md) -- [Secure Agents](../agents/security.md) +- [About Agents](/documentation/agents) +- [Optimize Agents](/documentation/agents/optimize-agents) +- [Secure Agents](/documentation/agents/secure-agents) diff --git a/docs/studio/index.md b/docs/studio/index.mdx similarity index 63% rename from docs/studio/index.md rename to docs/studio/index.mdx index b9e9322f81..ddb1811ab5 100644 --- a/docs/studio/index.md +++ b/docs/studio/index.mdx @@ -1,16 +1,23 @@ +--- +title: "About NVIDIA NeMo Studio" +description: "" +--- + -# About {{studio_long_name}} -!!! note - Studio is still in early development. Many features are missing or should be expected to change. + -{{studio_short_name}} is the web app for AI development with NVIDIA {{platform_name}}. It provides a workspace-oriented UI for managing local platform resources, reviewing agents, running agent optimization workflows, monitoring agent telemetry, and working with filesets, jobs, and secrets. +Studio is still in early development. Many features are missing or should be expected to change. + + + +NeMo Studio is the web app for AI development with NVIDIA NeMo Platform. It provides a workspace-oriented UI for managing local platform resources, reviewing agents, running agent optimization workflows, monitoring agent telemetry, and working with filesets, jobs, and secrets. --- ## Getting Started -{{studio_short_name}} is included with the platform. Follow the [Setup guide](../get-started/setup.md) to start {{platform_name}}, then access Studio at `/studio` on your running server. +NeMo Studio is included with the platform. Follow the [Setup guide](/documentation/get-started) to start NeMo Platform, then access Studio at `/studio` on your running server. ## Features @@ -18,25 +25,24 @@ Use the Studio Agents area to review platform-managed NeMo Agent Toolkit workflows, inspect agent details, deploy agents, open a chat session against a running deployment, and clean up deployments that are no longer needed. -For the full workflow, see [Studio Agents](agents.md). +For the full workflow, see [Studio Agents](/documentation/studio/agents). ### Suggestions -Use **Agents > Suggestions** to review optimizer suggestions for deployed agents. Studio groups suggestions by workspace or agent and lets you filter by type, priority, scope, and agent. +Use **Agents \> Suggestions** to review optimizer suggestions for deployed agents. Studio groups suggestions by workspace or agent and lets you filter by type, priority, scope, and agent. -For the full workflow, see [Studio Suggestions](suggestions.md). +For the full workflow, see [Studio Suggestions](/documentation/studio/suggestions). ### Monitor -Use **Agents > Monitor** to inspect agent telemetry stored by the platform, including recent inference logs and token usage summaries. +Use **Agents \> Monitor** to inspect agent telemetry stored by the platform, including recent inference logs and token usage summaries. -For the full workflow, see [Studio Monitor](monitor.md). +For the full workflow, see [Studio Monitor](/documentation/studio/monitor). ### Workspaces Keep models and data organized in dedicated workspaces. - ### Filesets Use **Filesets** to organize files by **purpose**, which determines which metadata fields are available. Purpose is chosen at creation time and can't be changed afterward. @@ -53,7 +59,7 @@ You can upload any file type into a fileset. However, each service supports spec | Service | Supported File Types | | --------------------------------------------------- | ------------------------------------- | -| [Data Designer](../data-designer/index.md) | `.json`, `.jsonl`, `.csv`, `.parquet` | +| [Data Designer](/documentation/design-synthetic-data) | `.json`, `.jsonl`, `.csv`, `.parquet` | ### Evaluations @@ -61,10 +67,10 @@ You can upload any file type into a fileset. However, each service supports spec | Location | What it shows | | -------- | ------------- | -| **Agents > Evaluations** | Evaluation runs scoped to a specific agent — results tied to agent deployments and versions. | -| **Evaluate > Evaluations** | All evaluation results across the workspace, regardless of which agent or model was evaluated. | +| **Agents \> Evaluations** | Evaluation runs scoped to a specific agent — results tied to agent deployments and versions. | +| **Evaluate \> Evaluations** | All evaluation results across the workspace, regardless of which agent or model was evaluated. | -Use **Agents > Evaluations** when reviewing how a specific agent performed. Use **Evaluate > Evaluations** for a broader view of evaluation history across the workspace. +Use **Agents \> Evaluations** when reviewing how a specific agent performed. Use **Evaluate \> Evaluations** for a broader view of evaluation history across the workspace. ### Jobs @@ -95,4 +101,4 @@ Navigate to **Jobs** in the workspace sidebar to see all jobs in the current wor ### Secrets -Store API keys and credentials to securely connect with external providers. See [manage-secrets](../get-started/concepts/manage-secrets.md) for details. +Store API keys and credentials to securely connect with external providers. See [manage-secrets](/documentation/get-started/core-concepts/manage-secrets) for details. diff --git a/docs/studio/monitor.md b/docs/studio/monitor.mdx similarity index 55% rename from docs/studio/monitor.md rename to docs/studio/monitor.mdx index 22d014de12..e6f994f287 100644 --- a/docs/studio/monitor.md +++ b/docs/studio/monitor.mdx @@ -1,10 +1,12 @@ -# {{studio_short_name}} Monitor - -Use {{studio_short_name}} **Agents > Monitor** to inspect recent agent activity for the selected workspace. +--- +title: "NeMo Studio Monitor" +description: "" +--- +Use NeMo Studio **Agents > Monitor** to inspect recent agent activity for the selected workspace. ## Telemetry Source -{{studio_short_name}} reads agent telemetry from the `nemo-agent-telemetry` fileset. Each telemetry file contributes run summaries, token usage, and request metadata that {{studio_short_name}} aggregates for the Monitor page. +NeMo Studio reads agent telemetry from the `nemo-agent-telemetry` fileset. Each telemetry file contributes run summaries, token usage, and request metadata that NeMo Studio aggregates for the Monitor page. ## Summary Cards @@ -20,5 +22,5 @@ The inference logs table shows recent agent requests from the loaded telemetry f ## Related Topics -- [Optimize Agents](../agents/optimization.md) -- [Secure Agents](../agents/security.md) +- [Optimize Agents](/documentation/agents/optimize-agents) +- [Secure Agents](/documentation/agents/secure-agents) diff --git a/docs/studio/suggestions.md b/docs/studio/suggestions.md deleted file mode 100644 index a373174b89..0000000000 --- a/docs/studio/suggestions.md +++ /dev/null @@ -1,39 +0,0 @@ -# {{studio_short_name}} Suggestions - -Use {{studio_short_name}} **Agents > Suggestions** to review optimizer suggestions for agents in the selected workspace. - -## What {{studio_short_name}} Shows - -{{studio_short_name}} loads the latest optimizer snapshot and suggestions from the platform files service. Suggestions can be scoped to a workspace or to a specific agent. - -The Suggestions page summarizes: - -- The number of agents and models observed in the latest optimizer snapshot. -- Suggestion counts by priority. -- Workspace-wide suggestions. -- Agent-specific suggestions grouped by agent. - -## Filter Suggestions - -Use the table filters to narrow suggestions by: - -| Filter | Purpose | -|--------|---------| -| Type | Focus on model optimization, guardrail, data safety, or new-model suggestions. | -| Priority | Show high-, medium-, or low-priority suggestions. | -| Scope | Separate workspace-wide suggestions from agent-specific suggestions. | -| Agent | Review suggestions for one agent at a time. | - -## Run an Optimization Pass - -If there are no suggestions, or if the latest snapshot is stale, {{studio_short_name}} starts a new optimizer pass for the workspace. The optimizer analyzes deployed agents and writes updated suggestions back to the platform files service. - -## Apply Suggestions - -{{studio_short_name}} can apply supported suggestions from the Suggestions page. Model optimization suggestions may ask you to choose an evaluation config before applying the change. - -## Next Steps - -- [{{studio_short_name}} Agents](agents.md): review, deploy, chat with, and delete agents from {{studio_short_name}}. -- [{{studio_short_name}} Monitor](monitor.md): inspect recent agent telemetry, token usage, and inference logs. -- [Optimize Agents](../agents/optimization.md): run CLI-driven optimization and review the underlying checks. diff --git a/docs/studio/suggestions.mdx b/docs/studio/suggestions.mdx new file mode 100644 index 0000000000..8ebbbd0299 --- /dev/null +++ b/docs/studio/suggestions.mdx @@ -0,0 +1,41 @@ +--- +title: "NeMo Studio Suggestions" +description: "" +--- +Use NeMo Studio **Agents > Suggestions** to review optimizer suggestions for agents in the selected workspace. + +## What NeMo Studio Shows + +NeMo Studio loads the latest optimizer snapshot and suggestions from the platform files service. Suggestions can be scoped to a workspace or to a specific agent. + +The Suggestions page summarizes: + +- The number of agents and models observed in the latest optimizer snapshot. +- Suggestion counts by priority. +- Workspace-wide suggestions. +- Agent-specific suggestions grouped by agent. + +## Filter Suggestions + +Use the table filters to narrow suggestions by: + +| Filter | Purpose | +|--------|---------| +| Type | Focus on model optimization, guardrail, data safety, or new-model suggestions. | +| Priority | Show high-, medium-, or low-priority suggestions. | +| Scope | Separate workspace-wide suggestions from agent-specific suggestions. | +| Agent | Review suggestions for one agent at a time. | + +## Run an Optimization Pass + +If there are no suggestions, or if the latest snapshot is stale, NeMo Studio starts a new optimizer pass for the workspace. The optimizer analyzes deployed agents and writes updated suggestions back to the platform files service. + +## Apply Suggestions + +NeMo Studio can apply supported suggestions from the Suggestions page. Model optimization suggestions may ask you to choose an evaluation config before applying the change. + +## Next Steps + +- [NeMo Studio Agents](/documentation/studio/agents): review, deploy, chat with, and delete agents from NeMo Studio. +- [NeMo Studio Monitor](/documentation/studio/monitor): inspect recent agent telemetry, token usage, and inference logs. +- [Optimize Agents](/documentation/agents/optimize-agents): run CLI-driven optimization and review the underlying checks. diff --git a/docs/stylesheets/nvidia.css b/docs/stylesheets/nvidia.css deleted file mode 100644 index 0085785cff..0000000000 --- a/docs/stylesheets/nvidia.css +++ /dev/null @@ -1,433 +0,0 @@ -/* SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ -/* SPDX-License-Identifier: Apache-2.0 */ - -/* - * NVIDIA NeMo Platform – Material for MkDocs theme overrides - * Primary: NVIDIA Green (#76b900) - * Supports light (default) and dark (slate) modes. - */ - -/* ── Google Fonts ──────────────────────────────────────────────────────────── */ -@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap'); - -/* ── Light mode – NVIDIA Green primary ─────────────────────────────────────── */ -[data-md-color-scheme="default"] { - --md-primary-fg-color: #76b900; - --md-primary-fg-color--light: #94d400; - --md-primary-fg-color--dark: #5a8c00; - --md-accent-fg-color: #76b900; - --md-accent-fg-color--transparent: rgba(118, 185, 0, 0.12); - - /* Typescale – slightly tighter */ - --md-typeset-font-size: 0.85rem; - - /* Admonition note → green tint */ - --md-admonition-fg-color: currentColor; -} - -/* ── Dark mode – NVIDIA near-black background ───────────────────────────────── */ -[data-md-color-scheme="slate"] { - /* Green accent */ - --md-primary-fg-color: #76b900; - --md-primary-fg-color--light: #94d400; - --md-primary-fg-color--dark: #5a8c00; - --md-accent-fg-color: #76b900; - --md-accent-fg-color--transparent: rgba(118, 185, 0, 0.12); - - /* Near-black backgrounds instead of the default blue-gray slate */ - --md-default-bg-color: #0e0e0e; - --md-default-bg-color--light: #161616; - --md-default-bg-color--lighter: #1e1e1e; - --md-default-bg-color--lightest: #282828; - - /* Text */ - --md-default-fg-color: hsla(0, 0%, 95%, 1); - --md-default-fg-color--light: hsla(0, 0%, 80%, 0.87); - --md-default-fg-color--lighter: hsla(0, 0%, 65%, 0.54); - --md-default-fg-color--lightest:hsla(0, 0%, 50%, 0.26); - - /* Code */ - --md-code-bg-color: #1a1a1a; - --md-code-fg-color: #d4d4d4; - - /* Typescale */ - --md-typeset-font-size: 0.85rem; -} - -/* ── Header: always dark/NVIDIA on both modes ───────────────────────────────── */ -.md-header { - background-color: #0e0e0e; - color: #ffffff; - box-shadow: 0 1px 0 0 rgba(118, 185, 0, 0.3); -} - -.md-header__button.md-logo { - padding: 0.1rem 0; - margin-right: 0.4rem; -} - -.md-header__button.md-logo img, -.md-header__button.md-logo svg { - height: 2rem; - width: auto; - max-width: none; -} - -/* Header links / search text */ -.md-header__title, -.md-header a { - color: #ffffff; -} - -/* Search bar */ -[data-md-color-scheme="slate"] .md-search__input { - background-color: rgba(255, 255, 255, 0.08); - color: #ffffff; -} - -[data-md-color-scheme="slate"] .md-search__input::placeholder { - color: rgba(255, 255, 255, 0.45); -} - -/* ── Navigation tabs removed: using sidebar-only nav ───────────────────────── */ - -/* ── Sidebar ─────────────────────────────────────────────────────────────────── */ -[data-md-color-scheme="slate"] .md-nav__title, -[data-md-color-scheme="slate"] .md-nav__link { - color: var(--md-default-fg-color--light); -} - -[data-md-color-scheme="slate"] .md-nav__link:hover, -[data-md-color-scheme="slate"] .md-nav__link--active { - color: #76b900; -} - -[data-md-color-scheme="slate"] .md-sidebar { - background-color: var(--md-default-bg-color); -} - -/* Active nav item indicator */ -.md-nav__link--active { - color: #76b900 !important; - font-weight: 600; -} - -/* ── Code blocks ─────────────────────────────────────────────────────────────── */ -[data-md-color-scheme="slate"] .md-typeset pre { - background-color: #161616; - border: 1px solid #2a2a2a; - border-radius: 6px; -} - -[data-md-color-scheme="default"] .md-typeset pre { - border: 1px solid #e8e8e8; - border-radius: 6px; -} - -/* Inline code */ -[data-md-color-scheme="slate"] .md-typeset code:not(.highlight code) { - background-color: #1e1e1e; - color: #94d400; - padding: 0.1em 0.35em; - border-radius: 3px; -} - -[data-md-color-scheme="default"] .md-typeset code:not(.highlight code) { - background-color: #f0f7e0; - color: #3d6200; - padding: 0.1em 0.35em; - border-radius: 3px; -} - -/* ── Admonitions ─────────────────────────────────────────────────────────────── */ -[data-md-color-scheme="slate"] .md-typeset .admonition, -[data-md-color-scheme="slate"] .md-typeset details { - border-color: rgba(118, 185, 0, 0.3); - background-color: #161616; -} - -/* Note → green */ -.md-typeset .admonition.note, -.md-typeset details.note { - border-left-color: #76b900; -} - -.md-typeset .admonition.note > .admonition-title, -.md-typeset details.note > summary { - background-color: rgba(118, 185, 0, 0.1); - color: #76b900; -} - -/* ── Tables ──────────────────────────────────────────────────────────────────── */ -[data-md-color-scheme="slate"] .md-typeset table:not([class]) { - background-color: #161616; -} - -[data-md-color-scheme="slate"] .md-typeset table:not([class]) th { - background-color: #1e1e1e; - color: #94d400; -} - -[data-md-color-scheme="slate"] .md-typeset table:not([class]) tr:hover { - background-color: #1e1e1e; -} - -/* ── Buttons / links ─────────────────────────────────────────────────────────── */ -.md-typeset a { - color: #76b900; -} - -.md-typeset a:hover { - color: #94d400; -} - -[data-md-color-scheme="slate"] .md-typeset a { - color: #94d400; -} - -[data-md-color-scheme="slate"] .md-typeset a:hover { - color: #b8e050; -} - -/* ── Footer ──────────────────────────────────────────────────────────────────── */ -.md-footer { - background-color: #0a0a0a; - color: rgba(255, 255, 255, 0.6); -} - -.md-footer-meta { - background-color: #080808; -} - -/* ── Content area ────────────────────────────────────────────────────────────── */ -[data-md-color-scheme="slate"] .md-content { - background-color: var(--md-default-bg-color); -} - -/* ── Headings ────────────────────────────────────────────────────────────────── */ -[data-md-color-scheme="slate"] .md-typeset h1 { - color: #ffffff; - font-weight: 700; -} - -[data-md-color-scheme="slate"] .md-typeset h2, -[data-md-color-scheme="slate"] .md-typeset h3 { - color: #e0e0e0; - font-weight: 600; -} - -/* H1 bottom border accent */ -.md-typeset h1 { - border-bottom: 2px solid #76b900; - padding-bottom: 0.3em; - margin-bottom: 1em; -} - -/* ── Top-of-page feedback widget ─────────────────────────────────────────────── */ -.md-feedback { - border-top: 1px solid rgba(118, 185, 0, 0.2); - padding-top: 1rem; - margin-top: 2rem; -} - -/* ── Version banner ──────────────────────────────────────────────────────────── */ -.md-version__current { - color: #76b900 !important; -} - -/* ── Scrollbar (dark mode) ───────────────────────────────────────────────────── */ -[data-md-color-scheme="slate"] ::-webkit-scrollbar { - width: 6px; - height: 6px; -} - -[data-md-color-scheme="slate"] ::-webkit-scrollbar-track { - background: #161616; -} - -[data-md-color-scheme="slate"] ::-webkit-scrollbar-thumb { - background: #3a3a3a; - border-radius: 3px; -} - -[data-md-color-scheme="slate"] ::-webkit-scrollbar-thumb:hover { - background: #76b900; -} - -/* ── Material tabs ───────────────────────────────────────────────────────────── */ -[data-md-color-scheme="slate"] .md-typeset .tabbed-labels > label { - color: rgba(255, 255, 255, 0.6); - border-bottom-color: transparent; -} - -[data-md-color-scheme="slate"] .md-typeset .tabbed-labels > label:hover, -[data-md-color-scheme="slate"] .md-typeset .tabbed-labels > input:checked + label { - color: #76b900; - border-bottom-color: #76b900; -} - -/* ── Search results ──────────────────────────────────────────────────────────── */ -[data-md-color-scheme="slate"] .md-search-result__article, -[data-md-color-scheme="slate"] .md-search-result__meta { - background-color: #161616; -} - -/* ── GitHub source widget in header ─────────────────────────────────────────── */ -.md-source { - background-color: #1a1a1a; - border-radius: 0.2rem; - color: #ffffff; -} - -.md-source:hover { - opacity: 0.8; -} - -.md-source__icon svg { - fill: #ffffff; -} - -.md-source__repository { - color: #ffffff; - font-weight: 500; -} - -.md-source__facts { - color: rgba(255, 255, 255, 0.7); -} - -/* ── Theme toggle → far right ───────────────────────────────────────────────── */ -.md-header__inner { - display: flex; - align-items: center; -} - -/* Push palette toggle after the source/repo widget */ -[data-md-component="palette"] { - order: 2; - margin-left: 0.4rem; -} - -[data-md-component="palette"] .md-header__button { - color: #ffffff; -} - -[data-md-component="palette"] .md-header__button:hover { - color: #76b900; -} - -/* ── Footer: remove social section ──────────────────────────────────────────── */ -.md-footer-meta__inner .md-social { - display: none; -} - -/* ── Grid cards: flex layout for bottom-aligned badges ───────────────────────── */ -.md-typeset .grid.cards > ol > li, -.md-typeset .grid.cards > ul > li { - display: flex; - flex-direction: column; - position: relative; - cursor: pointer; - transition: border-color 0.2s; -} - -.md-typeset .grid.cards > ol > li:hover, -.md-typeset .grid.cards > ul > li:hover { - border-color: #76b900; -} - -/* Stretch the title link to cover the entire card */ -.md-typeset .grid.cards > ol > li > p:first-child a, -.md-typeset .grid.cards > ul > li > p:first-child a { - text-decoration: none; -} - -.md-typeset .grid.cards > ol > li > p:first-child a::after, -.md-typeset .grid.cards > ul > li > p:first-child a::after { - content: ""; - position: absolute; - inset: 0; - z-index: 1; -} - -.md-typeset .grid.cards > ol > li > small:last-child, -.md-typeset .grid.cards > ul > li > small:last-child { - margin-top: auto; - padding-top: 0.6rem; - border-top: 1px solid rgba(118, 185, 0, 0.2); -} - -/* ── Card badge tags ─────────────────────────────────────────────────────────── */ -.md-typeset .md-tag { - display: inline-block; - padding: 0.15em 0.6em; - margin: 0.15em 0.15em; - font-size: 0.65rem; - font-weight: 600; - border-radius: 2px; - background-color: #1a3a00; - color: #94d400; - border: none; - letter-spacing: 0.02em; -} - -[data-md-color-scheme="default"] .md-typeset .md-tag { - background-color: #e8f5cc; - color: #2d5000; - border: none; -} - -/* ── API Reference: service filter chips ──────────────────────────────────── */ -.api-filter-chips { - display: flex; - flex-wrap: wrap; - gap: 0.35rem; - margin-bottom: 1.5rem; -} - -.api-chip { - display: inline-flex; - align-items: center; - padding: 0.15em 0.75em; - font-size: 0.7rem; - font-weight: 600; - line-height: 1.6; - border-radius: 999px; - border: 1.5px solid #76b900; - background: transparent; - color: #76b900; - cursor: pointer; - transition: background-color 0.2s, color 0.2s; -} - -.api-chip:hover { - background-color: rgba(118, 185, 0, 0.1); -} - -.api-chip.active { - background-color: #76b900; - color: #fff; -} - -[data-md-color-scheme="default"] .api-chip { - border-color: #76b900; - color: #76b900; -} - -[data-md-color-scheme="default"] .api-chip:hover { - background-color: rgba(118, 185, 0, 0.1); -} - -[data-md-color-scheme="default"] .api-chip.active { - background-color: #76b900; - color: #fff; -} - -/* ── Architecture diagram: clickable with hover effect ──────────────────────── */ -.md-typeset a > img { - transition: opacity 0.2s; -} - -.md-typeset a > img:hover { - opacity: 0.85; -} diff --git a/docs/support-matrix.md b/docs/support-matrix.mdx similarity index 88% rename from docs/support-matrix.md rename to docs/support-matrix.mdx index fe3c333f63..9324a4dcab 100644 --- a/docs/support-matrix.md +++ b/docs/support-matrix.mdx @@ -1,8 +1,10 @@ -# Support Matrix +--- +title: "Support Matrix" +description: "" +--- +This matrix defines the supported OSS local-install target for NeMo Platform 0.1.0. It applies to the Python package, CLI, SDK, and local services started by `nemo setup`. -This matrix defines the supported OSS local-install target for {{platform_name}} {{ release }}. It applies to the Python package, CLI, SDK, and local services started by `nemo setup`. - -Docker Compose, Helm, Kubernetes, and OpenShift deployment paths are not part of the OSS {{ release }} documentation scope. +Docker Compose, Helm, Kubernetes, and OpenShift deployment paths are not part of the OSS 0.1.0 documentation scope. ## Host Platforms @@ -13,7 +15,7 @@ Docker Compose, Helm, Kubernetes, and OpenShift deployment paths are not part of | Linux | Ubuntu 22.04 LTS, Ubuntu 24.04 LTS, RHEL 9, Rocky Linux 9, Debian 12 | Supported for CLI, SDK, local services, and NVIDIA GPU workloads. | | macOS | [macOS Tahoe 26](https://support.apple.com/en-us/122868) and macOS Sequoia 15 | Supported for CLI, SDK, local services, and cloud/provider workflows. Local NVIDIA GPU workloads are not supported on macOS. | | Architecture | x86_64 Linux; Apple Silicon and Intel macOS | NVIDIA GPU workloads require x86_64 Linux. | -| Windows and WSL | Not in the OSS {{ release }} support matrix | Use a supported Linux or macOS host for the local install path. | +| Windows and WSL | Not in the OSS 0.1.0 support matrix | Use a supported Linux or macOS host for the local install path. | ## Local Runtime @@ -53,7 +55,7 @@ Docker Compose, Helm, Kubernetes, and OpenShift deployment paths are not part of ## Out of Scope -The following are not part of the OSS {{ release }} local support matrix: +The following are not part of the OSS 0.1.0 local support matrix: - Native Windows local install. - Docker Compose, Helm, Kubernetes, and OpenShift deployment guides. diff --git a/docs/template/EULA.md b/docs/template/EULA.md deleted file mode 100644 index 4e8360e71f..0000000000 --- a/docs/template/EULA.md +++ /dev/null @@ -1,4 +0,0 @@ -# Eula - -By using MICROSERVICE container, you acknowledge that you have read and agreed to the -[license](https://registry.ngc.nvidia.com/orgs/ORG/teams/TEAM/resources/eula) diff --git a/docs/template/acknowledgements.md b/docs/template/acknowledgements.md deleted file mode 100644 index 7661efaecf..0000000000 --- a/docs/template/acknowledgements.md +++ /dev/null @@ -1,7 +0,0 @@ -# Acknowledgements - -## Software - -``` text -copyright notice from them -``` diff --git a/docs/template/getting_started/deploy-docker.md b/docs/template/getting_started/deploy-docker.md deleted file mode 100644 index d20ced6e01..0000000000 --- a/docs/template/getting_started/deploy-docker.md +++ /dev/null @@ -1,38 +0,0 @@ -# Deploying with Docker - - is intended to be run on a system with NVIDIA Datacenter GPUs, -with the exact requirements depending on the specific model and deployment options. - -For full systems hardware and software requirements see [Support -Matrix](./support-matrix). -For information about the models supported by the different containers, and the GPUs needed to run the models, see [Models](./models/models). - -## Pre-requisite Software - -To run LLM NIMs, you'll need a container runtime with support for NVIDIA GPUs. You can set this up with the following steps: - -1. Install [Docker](https://docs.docker.com/engine/install/) -1. Install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#installing-the-nvidia-container-toolkit) - -## Setting up the Environment - -Set the **NGC_CLI_API_KEY** environment variable to your NGC API key, as -shown in the following example. - -``` bash -export NGC_CLI_API_KEY="key from ngc" -``` - -If you have not set up NGC, see [NGC -Setup](https://catalog.ngc.nvidia.comsetup). Don't forget to download and -install the NCG CLI (the download is on that page). - -## Download Container - -## Download a Model - -## Launching the Container - -## Health and Liveness Checks - -## Stopping the Container diff --git a/docs/template/getting_started/deploy-helm.md b/docs/template/getting_started/deploy-helm.md deleted file mode 100644 index 1f9f3bdb1b..0000000000 --- a/docs/template/getting_started/deploy-helm.md +++ /dev/null @@ -1,35 +0,0 @@ -# Deploying with Helm - - is intended to be run on a system with NVIDIA Datacenter GPUs, -with the exact requirements depending on the specific model and deployment options. - -For full systems hardware and software requirements see [Support -Matrix](./support-matrix). -For information about the models supported by the different containers, and the GPUs needed to run the models, see [Models](./models/models). - -Since helm is deploying a container, we recommend that you become familiar with the general information in -[Deploying with Docker](./deploy-docker) before using helm and Kubernetes. - -## Pre-requisite Software - -To run LLM NIMs, you'll need a container runtime with support for NVIDIA GPUs. You can set this up with the following steps: - -1. Install [Docker](https://docs.docker.com/engine/install/) -1. Install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#installing-the-nvidia-container-toolkit) - -## Setting up the Environment - -Set the **NGC_CLI_API_KEY** environment variable to your NGC API key, as -shown in the following example. - -``` bash -export NGC_CLI_API_KEY="key from ngc" -``` - -If you have not set up NGC, see [NGC -Setup](https://catalog.ngc.nvidia.comsetup). Don't forget to download and -install the NCG CLI (the download is on that page). - -## Download a Model - -## Launching in Kubernetes diff --git a/docs/template/models.md b/docs/template/models.md deleted file mode 100644 index c2c908b55c..0000000000 --- a/docs/template/models.md +++ /dev/null @@ -1,20 +0,0 @@ -# Models - -## Optimized Models - -The following models are supported in the optimized CONTAINER container. - - -| Foundation Model | Strengths | Max I/O Tokens | Parameters | Training Data | GPU(s) | -| ---------------- | --------- | -------------- | ---------- | ------------- | ------ | -| **MODEL NAME** -Creator: CREATOR Architecture: Transformer | MODEL NAME is a ... | N | N billion | N billion | trillion tokens (up to MONTH YEAR) | (N) GPU - -## Supported Models - -The following models have been tested and validated for performance and accuracy with the CONTAINER container. Other models supported by vLLM may also be used. - -| Foundation Model | Strengths | Max I/O Tokens | Parameters | Training Data | GPU(s) | -| ---------------- | --------- | -------------- | ---------- | ------------- | ------ | -| **MODEL NAME** -Creator: CREATOR Architecture: Transformer | MODEL NAME is a ... | N | N billion | N billion tokens (up to MONTH YEAR) | (N) GPU diff --git a/docs/template/overview.md b/docs/template/overview.md deleted file mode 100644 index 4d7b9f3407..0000000000 --- a/docs/template/overview.md +++ /dev/null @@ -1,12 +0,0 @@ -# Overview - -() \... - -## Features - -## Applications - -## License - -By using MICROSERVICE, you acknowledge that you have read and agreed to the -[license](https://registry.ngc.nvidia.com/orgs/ORG/teams/TEAM/resources/eula/files). diff --git a/docs/template/playbooks/playbook.md b/docs/template/playbooks/playbook.md deleted file mode 100644 index e436935bbe..0000000000 --- a/docs/template/playbooks/playbook.md +++ /dev/null @@ -1,31 +0,0 @@ -# ??? Playbook - -The container used in this playbook: [](???). - -## Notebook Requirements - -- Access to -- Access to NGC ??? -- Docker - -## Getting - -To see the list of all available prebuilt models: - -``` bash -ngc registry model list "${inference_ngc_org_team}/*" -``` - -Once you see the model you want to use, -you can get information about the model, -as shown in the following example: - -```bash -ngc registry model info nvcr.io/ORG/TEAM/MODEL -``` - -And then download the model using the following command: - -``` bash -ngc registry model download-version "ORG/TEAM/MODEL" -``` diff --git a/docs/template/reference/api-reference.md b/docs/template/reference/api-reference.md deleted file mode 100644 index bcc736c789..0000000000 --- a/docs/template/reference/api-reference.md +++ /dev/null @@ -1,81 +0,0 @@ -# API Reference - -Use this page as a starting point for service-specific API reference content. -For a complete REST API page in the current MkDocs stack, render an OpenAPI file -with the `mkdocs-swagger-ui-tag` plugin: - -```html - -``` - -## Examples - -### List Models - -=== "CLI (cURL)" - - ```bash - curl "http://${HOSTNAME}:${SERVICE_PORT}/v1/models" \ - -H "Accept: application/json" - ``` - -=== "Python" - - ```python - import requests - - url = "http://:/v1/models" - headers = {"Accept": "application/json"} - response = requests.get(url, headers=headers, timeout=30) - print(response.text) - ``` - -**Response** - -```json -{ - "object": "list", - "data": [ - { - "id": "NV-Embed-QA", - "created": 0, - "object": "model", - "owned_by": "organization-owner" - } - ] -} -``` - -### Service-Specific Request - -=== "CLI (cURL)" - - ```bash - curl -X "VERB" \ - "http://${HOSTNAME}:${SERVICE_PORT}/v1/ENDPOINT" \ - -H "Accept: application/json" \ - -H "Content-Type: application/json" \ - -d '{ - "arg1": "value1", - "argN": "valueN" - }' - ``` - -=== "Python" - - ```python - import json - - import requests - - url = "http://:/v1/ENDPOINT" - payload = json.dumps( - { - "arg1": "value1", - "argN": "valueN", - } - ) - headers = {"Content-Type": "application/json"} - response = requests.request("VERB", url, headers=headers, data=payload, timeout=30) - print(response.text) - ``` diff --git a/docs/template/release-notes.md b/docs/template/release-notes.md deleted file mode 100644 index 95377ce19a..0000000000 --- a/docs/template/release-notes.md +++ /dev/null @@ -1,64 +0,0 @@ -# Release Notes - -## Release YY.MM - -### Summary - -This release ... - -### Key Features - -- Feature -- Feature - -### Language Models - -- Model -- Model - -### API Endpoints - -- OpenAI API endpoints - - `endpoint` - - `endpoint` -- NemoLLM API endpoints - - `endpoint` - - `endpoint` - -### OpenAI API Endpoints - -- endpoint - - Supported Parameters - - `parameter` - - `parameter` - - Unsupported Parameters - - `parameter` - - `parameter` -- endpoint ... - -### NemoLLM API Endpoints - -- endpoint - - Supported Parameters - - `parameter` - - `parameter` - - - Unsupported Parameters - - `parameter` - - `parameter` -- endpoint ... - -### Fixes - -- Fix -- Fix - -### Known Issues - -- Issue -- Issue - -#### Fixed bugs - -- Bug fix -- Bug fix diff --git a/docs/template/support-matrix.md b/docs/template/support-matrix.md deleted file mode 100644 index 04f6ef2162..0000000000 --- a/docs/template/support-matrix.md +++ /dev/null @@ -1,34 +0,0 @@ -# Support Matrix - -## Hardware - -The YY.MM release of MICROSERVICE has been tested on the following NVIDIA GPUs: - -- [NVIDIA A100](https://www.nvidia.com/en-us/data-center/a100/) -- [NVIDIA L4](https://www.nvidia.com/en-us/data-center/l4/) - -## Software - -### NVIDIA Driver - -Release YY.MM is based on [CUDA -12.2.2](https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/index.html) -which requires [NVIDIA -Driver](https://www.nvidia.com/Download/index.aspx?lang=en-us) release -535 or later. However, if you are running on a [data center -GPU](https://www.nvidia.com/en-us/data-center/products/), you can use -NVIDIA driver release 450.51 (or later R450), 470.57 (or later R470), -510.47 (or later R510), 515.65 (or later R515), or 525.85 (or later -R525), or 535.86 (or later R535). The CUDA driver's compatibility -package only supports particular drivers. Thus, users should upgrade -from all R418, R440, and R460 drivers, which are not forward-compatible -with CUDA 12.2. For a complete list of supported drivers, see [CUDA -Application -Compatibility](https://docs.nvidia.com/deploy/cuda-compatibility/index.html#use-the-right-compat-package). - -### NVIDIA Container Toolkit - -Your Docker environment must support NVIDIA GPUs. See the [NVIDIA -Container -Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) -for more information. diff --git a/docs/template/using-ms.md b/docs/template/using-ms.md deleted file mode 100644 index 367509df94..0000000000 --- a/docs/template/using-ms.md +++ /dev/null @@ -1,33 +0,0 @@ -# Client Examples - -The Python client is uploaded to the NGC Private Registry Resources. -Since the packages are not deployed to a public package registry, -you must download the `.whl` and `pip` files and install them from the following locations: - -- [EA Participants](https://registry.ngc.nvidia.com/orgs/ohlfw0olaadg/teams/ea-participants/resources/nemo-MICROSERVICE-python-client) -- [NVIDIAN/nemo-llm](https://registry.ngc.nvidia.com/orgs/nvidian/teams/nemo-llm/resources/nemo-MICROSERVICE-python-client) - -## MICROSERVICE - -You must modify the `base_url` if you did not start the MICROSERVICE using the default configuration. - -``` python -from nemo_MICROSERVICE_client import MICROSERVICEClient -from pprint import pprint - -NAME = MICROSERVICEClient(base_url="http://localhost:1984") - -# GET pipelines: list all pipeline options available -response = NAME.get_pipelines() - -# GET collections: list all created collections -response = NAME.get_collections() - -# CREATE a new collection - specify the pipeline type and name of the collection -response = NAME.create_collection(pipeline="dense_elasticsearch", name="testCollection") -created_collection_id = ( - response.collection.id -) # store ID of the newly created collection - -# other examples -``` diff --git a/docs/troubleshooting/cluster-setup.md b/docs/troubleshooting/cluster-setup.mdx similarity index 82% rename from docs/troubleshooting/cluster-setup.md rename to docs/troubleshooting/cluster-setup.mdx index d8fd4a012f..eabccc83d8 100644 --- a/docs/troubleshooting/cluster-setup.md +++ b/docs/troubleshooting/cluster-setup.mdx @@ -1,14 +1,10 @@ -# Troubleshooting {{platform_name}} Deployment on Kubernetes +--- +title: "Troubleshooting NeMo Platform Deployment on Kubernetes" +description: "" +--- +Use this documentation to troubleshoot issues that can arise while you deploy and run the NeMo Platform on Kubernetes. -Use this documentation to troubleshoot issues that can arise while you deploy and run the {{platform_name}} on Kubernetes. - ## Network Issues @@ -72,35 +68,4 @@ Sorry, home directories outside of /home needs configuration. To configure snap to use the correct home directory, follow the instructions at [Home directories outside of /home](https://snapcraft.io/docs/home-outside-home) in the Snap documentation. - diff --git a/docs/troubleshooting/customizer.md b/docs/troubleshooting/customizer.mdx similarity index 89% rename from docs/troubleshooting/customizer.md rename to docs/troubleshooting/customizer.mdx index ee4152986e..3b90b8b408 100644 --- a/docs/troubleshooting/customizer.md +++ b/docs/troubleshooting/customizer.mdx @@ -1,5 +1,7 @@ -# Troubleshooting {{ncm_short_name}} - +--- +title: "Troubleshooting NeMo Customizer" +description: "" +--- **Job fails during model download:** - Verify the HuggingFace token secret is configured correctly - Accept the model's license on the [HuggingFace model page](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) @@ -9,7 +11,7 @@ - The platform's shared persistent volume is likely full. Customization jobs require significant disk space: ~3× model size for full SFT, ~1.5× for LoRA. If you are also deploying the model from a base checkpoint fileset, plan for ~2.5× model size overall. - Clean up completed job artifacts or increase the PVC size (default: 200Gi at `/var/run/scratch/job`). - DPO/GRPO jobs also consume ephemeral node storage under `/tmp` via Ray workers — check node disk in addition to the PVC. -- See [ft-tut-understand-models](../customizer/tutorials/understand-configurations-and-models.md) for full storage requirement details. +- See [ft-tut-understand-models](/documentation/customizer-reference/tutorials/understanding-models-and-training) for full storage requirement details. **Job fails with OOM (Out of Memory):** 1. Reduce `micro_batch_size` to 1 diff --git a/docs/troubleshooting/data-designer.md b/docs/troubleshooting/data-designer.mdx similarity index 68% rename from docs/troubleshooting/data-designer.md rename to docs/troubleshooting/data-designer.mdx index 3256b19683..81b784751b 100644 --- a/docs/troubleshooting/data-designer.md +++ b/docs/troubleshooting/data-designer.mdx @@ -1,7 +1,10 @@ +--- +title: "Troubleshoot Data Designer" +description: "" +--- -# Troubleshoot Data Designer -Learn how to troubleshoot common issues with {{ndd_short_name}} in a local {{platform_name}} setup. +Learn how to troubleshoot common issues with NeMo Data Designer in a local NeMo Platform setup. ## Prerequisites @@ -46,8 +49,8 @@ Look for startup errors, provider authentication failures, port conflicts, or Da ## Next steps -- Tutorial: [Data Designer Tutorials](../data-designer/tutorials/index.md) covers common generation workflows. -- How-to: [Set Up {{platform_name}}](../get-started/setup.md) covers local setup and service startup. -- Explanation: [Data Designer Overview](../data-designer/index.md) explains configuration, execution, and service behavior. -- Reference: [API Reference](../api/index.md#tag-data-designer) lists Data Designer endpoints. -- Troubleshooting: [Troubleshooting Overview](index.md) links to other service troubleshooting pages. +- Tutorial: [Data Designer Tutorials](/documentation/design-synthetic-data/tutorials) covers common generation workflows. +- How-to: [Set Up NeMo Platform](/documentation/get-started) covers local setup and service startup. +- Explanation: [Data Designer Overview](/documentation/design-synthetic-data) explains configuration, execution, and service behavior. +- Reference: [API Reference](/documentation/reference/api-reference) lists Data Designer endpoints. +- Troubleshooting: [Troubleshooting Overview](/documentation/reference/troubleshooting) links to other service troubleshooting pages. diff --git a/docs/troubleshooting/evaluator.md b/docs/troubleshooting/evaluator.mdx similarity index 72% rename from docs/troubleshooting/evaluator.md rename to docs/troubleshooting/evaluator.mdx index 911734c0ce..4ef9790724 100644 --- a/docs/troubleshooting/evaluator.md +++ b/docs/troubleshooting/evaluator.mdx @@ -1,11 +1,16 @@ +--- +title: "Troubleshooting NeMo Evaluator" +description: "" +--- -# Troubleshooting {{nem_short_name}} -Use this documentation to troubleshoot issues that can arise when you work with [{{nem_long_name}}](../evaluator/index.md). +Use this documentation to troubleshoot issues that can arise when you work with [NVIDIA NeMo Evaluator](/documentation/evaluate-models). + + -!!! tip - You can [get metric logs](../evaluator/metrics/job-management.md) or [get benchmark logs](../evaluator/benchmarks/job-management.md) for `COMPLETED` or `FAILED` jobs and use them to help troubleshoot. +You can get metric logs or get benchmark logs for `COMPLETED` or `FAILED` jobs and use them to help troubleshoot. + --- @@ -35,12 +40,12 @@ Not all models make good judges. If the judge produces inconsistent output and d Incoming request body={'messages': [{'content': 'The output string did not satisfy the constraints given in the prompt. Fix the output string and return it.\nPlease return the output in a JSON format that complies with the following schema as specified in JSON Schema:\n{"properties": {"text": {"title": "Text", "type": "string"}}, "required": ["text"], "title": "StringIO", "type": "object"} ``` -## Dataset {dataset} is not in the expected format; it needs to have the files_url property set +## Dataset \{dataset} is not in the expected format; it needs to have the files_url property set This means that either the `files_url` is not provided as part of the dataset specification in the config, or that the `files_url` is not provided in the expected format. The dataset must be a JSON object with the `files_url` property set, -pointing to the path of the file in the {{nds_short_name}} in the format: `hf://datasets///`. +pointing to the path of the file in the NeMo Data Store in the format: `hf://datasets///`. ## Error connecting to inference server @@ -49,18 +54,18 @@ This means that for a custom evaluation, the target LLM endpoint is unable to co ## Inference SSL Error -An evaluation job that uses an HTTPS model endpoint can fail if the endpoint certificate or DNS name is not trusted by the local environment. Verify that the model URL is reachable from the host running {{platform_name}} and that the endpoint presents a valid certificate for its hostname. +An evaluation job that uses an HTTPS model endpoint can fail if the endpoint certificate or DNS name is not trusted by the local environment. Verify that the model URL is reachable from the host running NeMo Platform and that the endpoint presents a valid certificate for its hostname. ``` Error: HTTPSConnectionPool(host="", port=443): Max retries exceeded with url: /v1/chat/completions (Caused by SSLError(SSLError)) ``` -## Error occurred while checking the existence of file {file_ref} on {{nds_short_name}} +## Error occurred while checking the existence of file \{file_ref} on NeMo Data Store -This could mean that the dataset is not specified correctly, or that the {{nds_short_name}} itself is unresponsive. +This could mean that the dataset is not specified correctly, or that the NeMo Data Store itself is unresponsive. -- Verify that the files URL is correct and that the dataset and file exists in the {{nds_short_name}}. -- Verify that the {{nds_short_name}} is responsive and reachable. +- Verify that the files URL is correct and that the dataset and file exists in the NeMo Data Store. +- Verify that the NeMo Data Store is responsive and reachable. If the error contains the string `Dataset {file_ref} is not present on datastore`, it means that the datastore is responsive, but the file reference does not exist. @@ -91,7 +96,11 @@ If you changed the platform URL, use the value configured in `NMP_BASE_URL` or i To troubleshoot an evaluation job that has failed, download the evaluation result archive and inspect the job logs. -!!! warning "These are advanced troubleshooting steps that should only be done after all other troubleshooting fails." + + +These are advanced troubleshooting steps that should only be done after all other troubleshooting fails. + + ### Evaluation Job Logs @@ -114,36 +123,43 @@ Log files can be found in the results folder with the file extension `*.log`. ## Skip validation checks -When you launch an evaluation job, {{nem_short_name}} performs availability checks (for example, checking if the dataset and files exist in {{nds_short_name}}). +When you launch an evaluation job, NeMo Evaluator performs availability checks (for example, checking if the dataset and files exist in NeMo Data Store). To speed up job launch, or due to strict constraints of validation checks, you can pass the query parameter `skip_validation_checks` during job launch. Use the following code to create an evaluation job that skips validation checks. -=== "curl" - ```bash - curl -X 'POST' \ - 'https://${EVALUATOR_BASE_URL}/v1/evaluation/jobs?skip_validation_checks=True' \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ + + + + +```bash +curl -X 'POST' \ +'https://${EVALUATOR_BASE_URL}/v1/evaluation/jobs?skip_validation_checks=True' \ +-H 'accept: application/json' \ +-H 'Content-Type: application/json' \ +-d '{ +"namespace": "my-organization", +"target": "", +"config": "" +}' +``` + + + + +```python +data = { "namespace": "my-organization", "target": "", - "config": "" - }' - ``` + "config": "", +} -=== "Python" - ```python - data = { - "namespace": "my-organization", - "target": "", - "config": "", - } +endpoint = f"{EVALUATOR_BASE_URL}/v1/evaluation/jobs?skip_validation_checks=True" - endpoint = f"{EVALUATOR_BASE_URL}/v1/evaluation/jobs?skip_validation_checks=True" +response = requests.post(endpoint, json=data).json() +``` - response = requests.post(endpoint, json=data).json() - ``` + - + diff --git a/docs/troubleshooting/guardrails.md b/docs/troubleshooting/guardrails.mdx similarity index 52% rename from docs/troubleshooting/guardrails.md rename to docs/troubleshooting/guardrails.mdx index 3a75b24ee1..fdd32de1d0 100644 --- a/docs/troubleshooting/guardrails.md +++ b/docs/troubleshooting/guardrails.mdx @@ -1,10 +1,12 @@ -# Troubleshooting {{ngm_short_name}} - -Use this documentation to troubleshoot issues that can arise when you work with [{{ngm_long_name}}]. +--- +title: "Troubleshooting NeMo Guardrails" +description: "" +--- +Use this documentation to troubleshoot issues that can arise when you work with [NVIDIA NeMo Guardrails]. ## API Catalog Endpoint Issues -Several sample configurations in the documentation use NIMs with model endpoints hosted at . +Several sample configurations in the documentation use NIMs with model endpoints hosted at [https://integrate.api.nvidia.com/v1](https://integrate.api.nvidia.com/v1). The purpose of using the endpoints is to avoid deploying NIMs locally to reduce the initial effort to get started. Perform the following steps to troubleshoot configurations that use model endpoints from the API catalog: @@ -17,14 +19,91 @@ Perform the following steps to troubleshoot configurations that use model endpoi 1. Access a model, such as the Llama 3.1 8B NemoGuard Content Safety, from the model endpoint: - --8<-- "troubleshooting/_snippets/input/guardrails-cmds.sh" + ```bash + + # start-content-safety + + curl https://integrate.api.nvidia.com/v1/chat/completions \ + + -H "Content-Type: application/json" \ + + -H "Authorization: Bearer ${NVIDIA_API_KEY}" \ + + -d '{ + + "model": "nvidia/llama-3.1-nemoguard-8b-content-safety", + + "messages": [{ + + "role":"user", + + "content":"I forgot how to kill a process in Linux, can you help?" + + }], + + "stream": false + + }' + + # end-content-safety + + ``` + + + + ```json + + { + + "id": "chat-2aaefebc0963475d862f1fd202f172b2", + + "object": "chat.completion", - ??? "Example Output" + "created": 1743540105, - --8<-- "troubleshooting/_snippets/output/guardrails-content-safety.json" + "model": "nvidia/llama-3.1-nemoguard-8b-content-safety", + "choices": [ + + { + + "index": 0, + + "message": { + + "role": "assistant", + + "content": "{\"User Safety\": \"safe\"} " + + }, + + "logprobs": null, + + "finish_reason": "stop", + + "stop_reason": null + + } + + ], + + "usage": { + + "prompt_tokens": 408, + + "total_tokens": 416, + + "completion_tokens": 8 + + }, + + "prompt_logprobs": null + + } + + ``` If your request is not successful, such as a 401 or 403 HTTP status code, - go to to generate a new API key. + go to [https://build.nvidia.com/settings/api-keys](https://build.nvidia.com/settings/api-keys) to generate a new API key. 1. Access other models. In the preceding sample `curl` command, replace the `model` value with one of the following: diff --git a/docs/troubleshooting/index.md b/docs/troubleshooting/index.md deleted file mode 100644 index 4a1435fb9c..0000000000 --- a/docs/troubleshooting/index.md +++ /dev/null @@ -1,30 +0,0 @@ -# Troubleshooting Guide - -This section provides troubleshooting information for the {{platform_name}}. -
- -- **[Evaluator Troubleshooting](evaluator.md)** - - --- - - Learn how to troubleshoot common issues with {{nem_short_name}}. - -- **[Guardrails Troubleshooting](guardrails.md)** - - --- - - Learn how to troubleshoot common issues with {{ngm_short_name}}. - -- **[Data Designer Troubleshooting](data-designer.md)** - - --- - - Learn how to troubleshoot common issues with {{ndd_short_name}}. - -- **[Studio Troubleshooting](studio.md)** - - --- - - Learn how to troubleshoot common issues with Studio. - -
diff --git a/docs/troubleshooting/index.mdx b/docs/troubleshooting/index.mdx new file mode 100644 index 0000000000..ac76d18af7 --- /dev/null +++ b/docs/troubleshooting/index.mdx @@ -0,0 +1,29 @@ +--- +title: "Troubleshooting Guide" +description: "" +--- +This section provides troubleshooting information for the NeMo Platform. + + + + +Learn how to troubleshoot common issues with NeMo Evaluator. + + + + +Learn how to troubleshoot common issues with NeMo Guardrails. + + + + +Learn how to troubleshoot common issues with NeMo Data Designer. + + + + +Learn how to troubleshoot common issues with Studio. + + + + diff --git a/docs/troubleshooting/studio.md b/docs/troubleshooting/studio.mdx similarity index 81% rename from docs/troubleshooting/studio.md rename to docs/troubleshooting/studio.mdx index fdabb4eca2..1008015b28 100644 --- a/docs/troubleshooting/studio.md +++ b/docs/troubleshooting/studio.mdx @@ -1,7 +1,10 @@ +--- +title: "Troubleshooting Studio" +description: "" +--- -# Troubleshooting Studio -Learn how to troubleshoot common issues with {{studio_short_name}}. +Learn how to troubleshoot common issues with NeMo Studio. ## Filesets or Jobs Not Appearing in Studio @@ -37,4 +40,4 @@ curl -X POST http://localhost:8080/v1/datasets \ ``` All resource creation and update APIs support the `project` parameter. -Refer to the [Python SDK](../pysdk/index.md) and [API references](../api/index.md) for more details. +Refer to the [Python SDK](/documentation/reference/python-sdk) and [API references](/documentation/reference/api-reference) for more details. diff --git a/docs/work/guardrails/README.md b/docs/work/guardrails/README.mdx similarity index 87% rename from docs/work/guardrails/README.md rename to docs/work/guardrails/README.mdx index 47ddb7803b..bbe2e70db6 100644 --- a/docs/work/guardrails/README.md +++ b/docs/work/guardrails/README.mdx @@ -1,3 +1,7 @@ +--- +title: "" +description: "" +--- docker run --rm -it \ -v $(pwd)/work/guardrails:/config-store \ -e CONFIG_STORE_PATH=/config-store \ diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index fe7465f4fb..0000000000 --- a/mkdocs.yml +++ /dev/null @@ -1,513 +0,0 @@ -site_name: NeMo Platform -site_description: NVIDIA NeMo Platform Documentation -site_url: !ENV [NMP_DOCS_SITE_URL, "https://nvidia-nemo.github.io/nemo-platform/latest/"] -repo_url: https://github.com/NVIDIA-NeMo/nemo-platform -repo_name: NVIDIA-NeMo/nemo-platform -edit_uri: edit/main/docs/ -copyright: "Copyright © 2026 NVIDIA Corporation" - -docs_dir: docs -site_dir: site -use_directory_urls: true - -exclude_docs: | - _scripts/ - _generated/ - _hooks/ - _build/ - _overrides/ - _snippets/*.md - _snippets/**/*.md - **/_snippets/*.md - **/_snippets/**/*.md - .venv-mkdocs/ - template/ - work/ - notebooks/ - **/*.executed.ipynb - **/*.tmp.ipynb - **/*.py - **/*.pyc - README.md - CONTRIBUTING.md - CHANGELOG.md - MIGRATION.md - Makefile - requirements-mkdocs.txt - -# ─── Theme ─────────────────────────────────────────────────────────────────── -theme: - name: material - custom_dir: docs/_overrides - logo: assets/nvidia-logo-white.png - favicon: assets/favicon.ico - icon: - repo: fontawesome/brands/github - - palette: - # Light mode - - media: "(prefers-color-scheme: light)" - scheme: default - primary: custom - accent: custom - toggle: - icon: material/weather-sunny - name: Switch to dark mode - # Dark mode - - media: "(prefers-color-scheme: dark)" - scheme: slate - primary: custom - accent: custom - toggle: - icon: material/weather-night - name: Switch to light mode - - font: - text: Inter - code: JetBrains Mono - - features: - - navigation.indexes - - navigation.top - - navigation.path - - navigation.footer - - search.suggest - - search.highlight - - search.share - - content.code.copy - - content.code.annotate - - content.tabs.link - - content.tooltips - - toc.follow - - announce.dismiss - -# ─── Plugins ───────────────────────────────────────────────────────────────── -plugins: - - search: - lang: en - - - awesome-pages - - - macros: - on_error_fail: false - on_undefined: keep - - - mkdocstrings: - handlers: - python: - paths: [sdk/python/nemo-platform/src] - options: - docstring_style: google - show_source: false - show_root_heading: true - show_root_full_path: false - show_if_no_docstring: false - members_order: source - inherited_members: false - - - mkdocs-jupyter: - execute: false - include_source: true - ignore: - - "_scripts/*.py" - - "_extensions/**" - - "_templates/**" - - - redirects: - redirect_maps: - get-started/platform-prereq.md: get-started/setup.md - get-started/installation.md: get-started/setup.md - get-started/quickstart.md: get-started/setup.md - get-started/getting-started.md: index.md - generate-synthetic-data/index.md: data-designer/index.md - anonymizer/quickstart.md: anonymizer/tutorials/index.md - - - swagger-ui-tag: - docExpansion: list - filter: true - syntaxHighlightTheme: monokai - tryItOutEnabled: false - - - mike: - alias_type: redirect - canonical_version: !ENV [NMP_DOCS_CANONICAL_VERSION, latest] - -# ─── Hooks ─────────────────────────────────────────────────────────────────── -hooks: - # Reads extra.hidden_docs below to hide temporary pages from nav, output, and API filters. - - docs/_hooks/hide_unready_docs.py - -# ─── Markdown Extensions ───────────────────────────────────────────────────── -markdown_extensions: - - admonition - - pymdownx.details - - pymdownx.superfences: - custom_fences: - - name: mermaid - class: mermaid - format: !!python/name:pymdownx.superfences.fence_code_format - - pymdownx.tabbed: - alternate_style: true - slugify: !!python/object/apply:pymdownx.slugs.slugify - kwds: - case: lower - - pymdownx.highlight: - anchor_linenums: true - line_spans: __span - pygments_lang_class: true - - pymdownx.inlinehilite - - pymdownx.snippets: - base_path: - - docs - check_paths: false - - pymdownx.emoji: - emoji_index: !!python/name:material.extensions.emoji.twemoji - emoji_generator: !!python/name:material.extensions.emoji.to_svg - - attr_list - - md_in_html - - tables - - footnotes - - def_list - - toc: - permalink: true - title: On this page - - pymdownx.keys - - pymdownx.mark - - pymdownx.caret - - pymdownx.tilde - - pymdownx.tasklist: - custom_checkbox: true - -# ─── Extra ─────────────────────────────────────────────────────────────────── -extra: - social: [] - analytics: - feedback: - title: Was this page helpful? - ratings: - - icon: material/thumb-up-outline - name: This page was helpful - data: 1 - note: >- - Thanks for your feedback! - - icon: material/thumb-down-outline - name: This page could be improved - data: 0 - note: >- - Thanks for your feedback! Help us improve by - opening an issue. - - # Temporary gating for docs that are not ready for the 2026-05-12 launch. - # Used by docs/_hooks/hide_unready_docs.py. - # Set NMP_HIDE_UNREADY_DOCS=false, or run `make html-with-unready` / `make live-with-unready`, - # to build or serve these pages locally without editing nav. - hidden_docs: - enabled: !ENV [NMP_HIDE_UNREADY_DOCS, true] - sections: - - Platform - - Fine-tune Models - - Customizer - - Example Applications - paths: - - auth/** - - customizer/** - - evaluator/benchmarks/** - - evaluator/metrics/job-management.md - - evaluator/metrics/results.md - - evaluator/metrics/retriever.md - - evaluator/tutorials/run-an-evaluation.md - - example-applications/tool-calling.ipynb - - get-started/quickstart.md - - helm/** - - run-inference/tutorials/deploy-models.md - - set-up/** - - troubleshooting/cluster-setup.md - - troubleshooting/customizer.md - api_tags: - - Auditor - - Customizer - - Safe Synthesizer - - # Substitution variables (accessed as {{ release }}, {{ ncm_short_name }}, etc.) - release: "0.1.0" - platform_name: "NeMo Platform" - ncm_long_name: "NVIDIA NeMo Customizer" - ncm_short_name: "NeMo Customizer" - ndd_long_name: "NVIDIA NeMo Data Designer" - ndd_short_name: "NeMo Data Designer" - nds_long_name: "NVIDIA NeMo Data Store" - nds_short_name: "NeMo Data Store" - nes_long_name: "NVIDIA NeMo Entity Store" - nes_short_name: "NeMo Entity Store" - nem_long_name: "NVIDIA NeMo Evaluator" - nem_short_name: "NeMo Evaluator" - ngm_long_name: "NVIDIA NeMo Guardrails" - ngm_short_name: "NeMo Guardrails" - nim_long_name: "NVIDIA NIM" - nim_short_name: "NIM" - nim_llm_long_name_cap: "NVIDIA NIM for Large Language Models (LLMs)" - nim_llm_long_name: "NVIDIA NIM for large language models (LLMs)" - nim_llm_short_name: "NIM for LLMs" - nrm_long_name: "NVIDIA NeMo Retriever" - nrm_short_name: "NeMo Retriever" - nsm_long_name: "NVIDIA NeMo Studio" - nsm_short_name: "NeMo Studio" - nss_long_name: "NVIDIA NeMo Safe Synthesizer" - nss_short_name: "NeMo Safe Synthesizer" - nss_plugin_slug: "safe-synthesizer" - nop_long_name: "NVIDIA NeMo Operator" - nop_short_name: "NeMo Operator" - studio_long_name: "NVIDIA NeMo Studio" - studio_short_name: "NeMo Studio" - helm_chart_long_name: "NVIDIA NeMo Platform Helm Chart" - helm_chart_short_name: "NeMo Platform Helm Chart" - parent_helm_chart_latest_version: "1.0.0" - docker_compose_latest_version: "26.3" - build_nvidia_dot_com: "build.nvidia.com" - nemo_pysdk: "NeMo Platform Python SDK" - NEMO_HOST: "nemo.test" - inference_ngc_org: "nvidia" - deployment_management_long_name: "NVIDIA NeMo Deployment Management" - deployment_management_short_name: "NeMo Deployment Management" - proxy_short_name: "NIM Proxy" - __auditor_long_name: "NVIDIA NeMo Auditor" - __auditor_short_name: "NeMo Auditor" - __auditor_img_tag: "26.3" - volcano_version: "1.9.0" - -extra_javascript: - - javascripts/api-filter.js - -extra_css: - - stylesheets/nvidia.css - -# ─── Nav ───────────────────────────────────────────────────────────────────── -nav: - - Home: index.md - - Get Started: - - Setup: get-started/setup.md - - Core Concepts: - - Overview: get-started/concepts/index.md - - Entities: get-started/concepts/entities.md - - Entity References: get-started/concepts/entity-references.md - - Filtering: get-started/concepts/filtering.md - - Manage Files: get-started/concepts/manage-files.md - - Manage Secrets: get-started/concepts/manage-secrets.md - - Projects: get-started/concepts/projects.md - - Workspaces: get-started/concepts/workspaces.md - - Example Applications: - - About: example-applications/about.md - - Tool Calling: example-applications/tool-calling.ipynb - - Models and Inference: - - About: run-inference/about.md - - Tutorials: - - Overview: run-inference/tutorials/index.md - - Run Inference: run-inference/tutorials/run-inference.md - - Agents: - - About: agents/index.md - - Optimize Agents: agents/optimization.md - - Secure Agents: agents/security.md - - Plugins and Skills: agents/plugins.md - - Design Synthetic Data: - - About: data-designer/index.md - - Execution Modes: data-designer/execution-modes.md - - CLI: data-designer/cli.md - - Tutorials: - - Overview: data-designer/tutorials/index.md - - The Basics: data-designer/tutorials/basics.md - - Seeding with External Datasets: data-designer/tutorials/seeding.md - - SDK Resources: data-designer/sdk-resources.md - - Migrating from Standalone Library: data-designer/migration.md - - Synthesize Safe Data: - - Getting Started: safe-synthesizer/getting-started.md - - About: - - Overview: safe-synthesizer/about/index.md - - Data Synthesis: safe-synthesizer/about/data-synthesis.md - - PII Replacement: safe-synthesizer/about/pii-replacement.md - - Evaluation: safe-synthesizer/about/evaluation.md - - Jobs: safe-synthesizer/about/jobs.md - - Local and Subprocess Execution: safe-synthesizer/about/host-local-development.md - - Parameters Reference: safe-synthesizer/about/reference.md - - Tutorials: - - Overview: safe-synthesizer/tutorials/index.md - - Safe Synthesizer 101: safe-synthesizer/tutorials/safe-synthesizer-101.md - - Differential Privacy: safe-synthesizer/tutorials/differential-privacy.md - - SDK Resources: safe-synthesizer/sdk-resources.md - - Anonymize Data: - - About: anonymizer/index.md - - Tutorials: - - Overview: anonymizer/tutorials/index.md - - Preview a Config: anonymizer/tutorials/preview.md - - Run an Anonymizer Job: anonymizer/tutorials/run.md - - SDK Resources: anonymizer/sdk-resources.md - - CLI Reference: anonymizer/cli.md - - Fine-tune Models: - - About: customizer/index.md - - Customization Concepts: customizer/about.md - - Tutorials: - - Overview: customizer/tutorials/index.md - - Format Training Dataset: customizer/tutorials/format-training-dataset.md - - Import HuggingFace Models: customizer/tutorials/import-hf-model.md - - Job Metrics: customizer/tutorials/metrics.md - - Understanding Models and Training: customizer/tutorials/understand-configurations-and-models.md - - Models: - - Model Catalog: customizer/models/index.md - - Dataset Format: customizer/models/data-format.md - - Llama: customizer/models/llama.md - - Llama Nemotron: customizer/models/llama-nemotron.md - - Mistral: customizer/models/mistral.md - - Phi: customizer/models/phi.md - - Qwen: customizer/models/qwen.md - - GPT-OSS: customizer/models/gpt-oss.md - - Embedding: customizer/models/embedding.md - - Manage Model Entities: - - Overview: customizer/manage-model-entities/index.md - - Create a Model FileSet: customizer/manage-model-entities/create-fileset.md - - Create a Model Entity: customizer/manage-model-entities/create-model-entity.md - - Manage Jobs: - - Overview: customizer/manage-customization-jobs/index.md - - Create Job: customizer/manage-customization-jobs/create-job.md - - Customization Job Reference: customizer/manage-customization-jobs/customization-job-reference.md - - Get Job Status: customizer/manage-customization-jobs/get-job-status.md - - List Active Jobs: customizer/manage-customization-jobs/list-active-jobs.md - - Cancel Job: customizer/manage-customization-jobs/cancel-job.md - - Training Configuration: customizer/manage-customization-jobs/hyperparameters.md - - Evaluation: - - About: evaluator/index.md - - Tutorials: - - Overview: evaluator/tutorials/index.md - - Run LLM-as-a-Judge Evaluation: evaluator/tutorials/run-llm-judge-evaluation.md - - Define and Run Custom Python Metrics: evaluator/tutorials/define-run-custom-python-metrics.md - - SDK Resources: evaluator/sdk-resources.md - - Metrics: - - Overview: evaluator/metrics/index.md - - Manage Metrics: evaluator/metrics/manage-metrics.md - - LLM-as-a-Judge: evaluator/metrics/llm-as-a-judge.md - - RAG Metrics: evaluator/metrics/rag.md - - Similarity Metrics: evaluator/metrics/similarity.md - - Agentic Metrics: evaluator/metrics/agentic.md - - Bring Your Own Metric: evaluator/metrics/remote.md - - Agent Configuration: evaluator/metrics/agent-configuration.md - - Model Configuration: evaluator/metrics/model-configuration.md - - Job Management: evaluator/metrics/job-management.md - - Metric Results: evaluator/metrics/results.md - - Benchmarks: - - Overview: evaluator/benchmarks/index.md - - Industry Benchmarks: evaluator/benchmarks/industry.md - - Discover Industry Benchmarks: evaluator/benchmarks/discover-industry-benchmarks.md - - Agentic Benchmarks: evaluator/benchmarks/agentic.md - - Custom Benchmarks: evaluator/benchmarks/custom.md - - Manage Benchmarks: evaluator/benchmarks/manage-benchmarks.md - - HuggingFace Secret: evaluator/benchmarks/hf-secret.md - - Job Management: evaluator/benchmarks/job-management.md - - Benchmark Results: evaluator/benchmarks/results.md - - Vulnerability Scanning: - - About: auditor/index.md - - Tutorials: - - Overview: auditor/tutorials/index.md - - Run an Audit Locally: auditor/tutorials/run-audit-locally.md - - SDK Resources: auditor/sdk-resources.md - - Configurations: - - Overview: auditor/configs/index.md - - Selecting Probes: auditor/configs/probes.md - - Schema: auditor/configs/schema.md - - Targets: - - Overview: auditor/targets/index.md - - Inference Gateway: auditor/targets/inference-gateway.md - - Schema: auditor/targets/schema.md - - Guardrails: - - About: guardrails/index.md - - Core Concepts: - - Overview: guardrails/concepts/index.md - - Architecture: guardrails/concepts/architecture.md - - Configurations: - - Overview: guardrails/concepts/configurations/index.md - - Configuration Structure: guardrails/concepts/configurations/configuration-structure.md - - Default Configurations: guardrails/concepts/configurations/default-configs.md - - Manage Configurations: guardrails/concepts/configurations/manage-configs.md - - Running Inference: guardrails/concepts/inference.md - - Running Checks: guardrails/concepts/checks.md - - Tutorials: - - Overview: guardrails/tutorials/index.md - - Content Safety: guardrails/tutorials/content-safety.md - - Deploy NemoGuard NIMs: guardrails/tutorials/deploy-nemoguard-nims.md - - Injection Detection: guardrails/tutorials/injection-detection.md - - Multimodal Data: guardrails/tutorials/multimodal-data.md - - Parallel Rails: guardrails/tutorials/parallel-rails.md - - Terminology: guardrails/terminology.md - - Observability: guardrails/observability.md - - Platform: - - About: set-up/index.md - - Authentication & Authorization: - - Overview: auth/index.md - - Concepts: auth/concepts.md - - Security Model: auth/security-model.md - - Authentication: - - Overview: auth/authentication/index.md - - OIDC Setup: auth/authentication/oidc.md - - Using Authentication: auth/authentication/using-authentication.md - - Providers: - - Overview: auth/authentication/providers/index.md - - Azure AD (Entra ID): auth/authentication/providers/azure-ad.md - - Generic OIDC: auth/authentication/providers/generic.md - - Authorization: - - Overview: auth/authorization/index.md - - Roles and Permissions: auth/authorization/roles-and-permissions.md - - Managing Access: auth/authorization/managing-access.md - - Permissions Reference: auth/authorization/permissions-reference.md - - API Scopes: auth/authorization/api-scopes.md - - Policy Engine: auth/authorization/policy-engine.md - - Deployment: - - Configuration: auth/deployment/configuration.md - - Credential Propagation: auth/deployment/credential-propagation.md - - Gateway Integration: auth/deployment/gateway.md - - Production Hardening: auth/deployment/hardening.md - - Troubleshooting: auth/troubleshooting.md - - Deploying on Kubernetes: - - Overview: set-up/helm/index.md - - Prerequisites: set-up/helm/prerequisites.md - - Install: set-up/helm/install.md - - Ingress: set-up/helm/ingress.md - - Database Setup: set-up/helm/database-setup.md - - File Storage: set-up/helm/file-storage.md - - Persistent Volumes: set-up/helm/persistent-volumes.md - - Multinode Networking: set-up/helm/multinode-networking.md - - OpenShift: set-up/helm/openshift.md - - Backup and Restore: set-up/helm/backup-and-restore.md - - Jobs: set-up/manage-jobs.md - - Observability: set-up/opentelemetry.md - - Security: set-up/security.md - - Milvus: set-up/milvus.md - - Studio (alpha): - - About: studio/index.md - - Agents: studio/agents.md - - Suggestions: studio/suggestions.md - - Monitor: studio/monitor.md - - Reference: - - Release Notes: - - Overview: about/release-notes/index.md - - Current Release: about/release-notes/current-release.md - - System Requirements: requirements.md - - Support Matrix: support-matrix.md - - API Reference: api/index.md - - Python SDK: - - Overview: pysdk/index.md - - Client APIs: pysdk/client/index.md - - CLI Reference: - - Overview: cli/index.md - - Configuration: cli/configuration.md - - Working with Resources: cli/working-with-resources.md - - Full CLI Reference: cli/reference.md - - Troubleshooting: cli/troubleshooting.md - - Config Reference: set-up/config-reference.md - - Helm Reference: helm/index.md - - Troubleshooting: - - Overview: troubleshooting/index.md - - Cluster Setup: troubleshooting/cluster-setup.md - - Customizer: troubleshooting/customizer.md - - Data Designer: troubleshooting/data-designer.md - - Evaluator: troubleshooting/evaluator.md - - Guardrails: troubleshooting/guardrails.md - - Studio: troubleshooting/studio.md - - EULA: eula.md - - Acknowledgements: acknowledgements/index.md diff --git a/packages/nemo_platform/pyproject.toml b/packages/nemo_platform/pyproject.toml index bc065598c9..f89ab0ca09 100644 --- a/packages/nemo_platform/pyproject.toml +++ b/packages/nemo_platform/pyproject.toml @@ -660,7 +660,6 @@ inherit."optional-dependencies" = true inherit.scripts = true inherit."entry-points" = ["nemo.*"] force_include."../../../../../docs" = "nemo_platform/cli/docs" -force_include."../../../../../mkdocs.yml" = "nemo_platform/cli/mkdocs.yml" [tool.bundle-package.nemo-agents-example-calculator] source = "../../plugins/nemo-agents/examples/calculator-agent/src/calculator_agent" diff --git a/packages/nemo_platform_ext/pyproject.toml b/packages/nemo_platform_ext/pyproject.toml index e441095729..ec2b105cfd 100644 --- a/packages/nemo_platform_ext/pyproject.toml +++ b/packages/nemo_platform_ext/pyproject.toml @@ -77,7 +77,6 @@ packages = ["src/nemo_platform_ext"] [tool.hatch.build.targets.wheel.force-include] "../../docs" = "nemo_platform_ext/cli/docs" -"../../mkdocs.yml" = "nemo_platform_ext/cli/mkdocs.yml" [tool.vendor-package] package = "nemo_platform_ext" diff --git a/packages/nemo_platform_ext/scripts/docs_generator.py b/packages/nemo_platform_ext/scripts/docs_generator.py index a10dd5d3dc..fbc13a446f 100644 --- a/packages/nemo_platform_ext/scripts/docs_generator.py +++ b/packages/nemo_platform_ext/scripts/docs_generator.py @@ -312,10 +312,10 @@ def generate_command_docs( else: full_name = command_name - # Header - use "Full CLI Reference" for root, plain name for others + # Header - use Fern MDX frontmatter for root, plain name for others # Markdown only supports up to 6 levels of headers, use bold for deeper nesting if is_root: - docs += "# Full CLI Reference\n\n" + docs += '---\ntitle: "Full CLI Reference"\ndescription: ""\n---\n' elif indent <= 6: docs += "#" * indent + f" {full_name}\n\n" else: @@ -545,6 +545,12 @@ def generate_index_snippet(app: typer.Typer, name: str = "nemo") -> str: lines: list[str] = [] + # Fern MDX frontmatter (snippet is included into other pages). + lines.append("---") + lines.append('title: ""') + lines.append('description: ""') + lines.append("---") + # --- Global options table --- lines.append("**Global options** apply to all commands:") lines.append("") @@ -638,9 +644,54 @@ def generate_docs(app: typer.Typer, name: str = "nemo") -> str: "{ref}`Entity references `", ) + docs = _escape_mdx(docs) + return docs.strip() + "\n" +def _escape_mdx(text: str) -> str: + """Escape `{` and `<` outside fenced code blocks so MDX does not parse them as JSX.""" + lines = text.split("\n") + in_code_fence = False + escaped: list[str] = [] + for line in lines: + stripped = line.lstrip() + if stripped.startswith("```"): + in_code_fence = not in_code_fence + escaped.append(line) + continue + if in_code_fence: + escaped.append(line) + continue + escaped.append(_escape_mdx_line(line)) + return "\n".join(escaped) + + +def _escape_mdx_line(line: str) -> str: + """Escape `{` and `<` in `line`, preserving inline backtick spans.""" + out: list[str] = [] + i = 0 + n = len(line) + while i < n: + ch = line[i] + if ch == "`": + end = line.find("`", i + 1) + if end == -1: + out.append(line[i:]) + break + out.append(line[i : end + 1]) + i = end + 1 + continue + if ch == "{": + out.append("\\{") + elif ch == "<": + out.append("\\<") + else: + out.append(ch) + i += 1 + return "".join(out) + + def main() -> None: """Generate CLI documentation and print to stdout. diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/docs.py b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/docs.py index 6a62ec07f5..a7de5d5f51 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/docs.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/cli/commands/docs.py @@ -8,48 +8,12 @@ import os import posixpath import sys -from collections.abc import Sequence -from fnmatch import fnmatchcase from pathlib import Path from typing import Annotated import typer -import yaml - -class _MkDocsConfigLoader(yaml.SafeLoader): - """YAML loader that tolerates MkDocs-specific tags.""" - - -def _construct_env(loader: _MkDocsConfigLoader, node: yaml.Node) -> object: - if isinstance(node, yaml.SequenceNode): - values = loader.construct_sequence(node) - if len(values) == 1: - env_names = values - default = None - else: - env_names = values[:-1] - default = values[-1] if values else None - for env_name in env_names: - env_value = os.environ.get(str(env_name)) - if env_value is not None: - return env_value - return default - - env_name = loader.construct_scalar(node) - return os.environ.get(env_name, "") - - -def _construct_unknown(loader: _MkDocsConfigLoader, _tag_suffix: str, node: yaml.Node) -> object: - if isinstance(node, yaml.MappingNode): - return loader.construct_mapping(node) - if isinstance(node, yaml.SequenceNode): - return loader.construct_sequence(node) - return loader.construct_scalar(node) - - -_MkDocsConfigLoader.add_constructor("!ENV", _construct_env) -_MkDocsConfigLoader.add_multi_constructor("", _construct_unknown) +DOC_EXT = ".mdx" def _find_docs_root(module_file: Path | None = None) -> Path | None: @@ -65,116 +29,22 @@ def _find_docs_root(module_file: Path | None = None) -> Path | None: for parent in parents: docs_dir = parent / "docs" - mkdocs_config = parent / "mkdocs.yml" - if mkdocs_config.is_file() and docs_dir.is_dir(): - return docs_dir.resolve() - - for parent in parents: - docs_dir = parent / "docs" - if docs_dir.is_dir() and (docs_dir / "index.md").is_file(): + if docs_dir.is_dir() and any(docs_dir.rglob(f"*{DOC_EXT}")): return docs_dir.resolve() return None -def _normalize_path(path: str) -> str: - return posixpath.normpath(path.strip().replace("\\", "/")).lstrip("./") - - -def _normalize_pattern(pattern: object) -> str: - raw_pattern = str(pattern).strip().replace("\\", "/") - is_directory_pattern = raw_pattern.endswith("/") and not raw_pattern.endswith("/**") - normalized = _normalize_path(raw_pattern) - if is_directory_pattern and not normalized.endswith("/"): - return f"{normalized}/" - return normalized - - -def _is_enabled(value: object) -> bool: - if isinstance(value, str): - return value.strip().lower() not in {"", "0", "false", "no", "off"} - return bool(value) - - -def _load_mkdocs_config(docs_root: Path) -> dict[str, object]: - config_path = docs_root.parent / "mkdocs.yml" - if not config_path.is_file(): - return {} - - try: - loaded = yaml.load(config_path.read_text(encoding="utf-8"), Loader=_MkDocsConfigLoader) - except (OSError, yaml.YAMLError): - return {} - - return loaded if isinstance(loaded, dict) else {} - - -def _sequence_value(value: object) -> Sequence[object]: - if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): - return () - return value - - -def _exclude_patterns(config: dict[str, object]) -> tuple[str, ...]: - exclude_docs = config.get("exclude_docs", "") - if isinstance(exclude_docs, str): - return tuple( - _normalize_pattern(line) - for line in exclude_docs.splitlines() - if line.strip() and not line.lstrip().startswith("#") - ) - - return tuple(_normalize_pattern(pattern) for pattern in _sequence_value(exclude_docs) if str(pattern).strip()) - - -def _hidden_doc_patterns(config: dict[str, object]) -> tuple[str, ...]: - extra = config.get("extra", {}) - if not isinstance(extra, dict): - return () - - hidden_docs = extra.get("hidden_docs", {}) - if not isinstance(hidden_docs, dict) or not _is_enabled(hidden_docs.get("enabled")): - return () - - return tuple( - _normalize_pattern(pattern) for pattern in _sequence_value(hidden_docs.get("paths", ())) if str(pattern).strip() - ) - - -def _matches_pattern(path: str, pattern: str) -> bool: - if pattern.endswith("/"): - directory = pattern.rstrip("/") - return path == directory or path.startswith(f"{directory}/") - - if pattern.endswith("/**"): - directory = pattern[:-3] - return path == directory or path.startswith(f"{directory}/") - - return fnmatchcase(path, pattern) - - -def _matches_any(path: str, patterns: tuple[str, ...]) -> bool: - return any(_matches_pattern(path, pattern) for pattern in patterns) - - -def _is_visible_doc(rel_path: Path, exclude_patterns: tuple[str, ...], hidden_patterns: tuple[str, ...]) -> bool: - if any(part.startswith("_") or part.startswith(".") for part in rel_path.parts): - return False - - posix_rel_path = rel_path.as_posix() - return not _matches_any(posix_rel_path, exclude_patterns) and not _matches_any(posix_rel_path, hidden_patterns) +def _is_visible_doc(rel_path: Path) -> bool: + return not any(part.startswith("_") or part.startswith(".") for part in rel_path.parts) def _list_docs(docs_root: Path) -> list[str]: - """List doc topics that match the rendered MkDocs site.""" - config = _load_mkdocs_config(docs_root) - exclude = _exclude_patterns(config) - hidden = _hidden_doc_patterns(config) - + """List doc topics under the docs root.""" paths = [] - for md_file in sorted(docs_root.rglob("*.md")): - rel = md_file.relative_to(docs_root) - if not _is_visible_doc(rel, exclude, hidden): + for doc_file in sorted(docs_root.rglob(f"*{DOC_EXT}")): + rel = doc_file.relative_to(docs_root) + if not _is_visible_doc(rel): continue paths.append(rel.with_suffix("").as_posix()) return paths @@ -190,8 +60,10 @@ def _topic_from_user_path(path: str) -> str | None: return None topic = normalized.removeprefix("./") - if topic.endswith(".md"): - topic = topic[:-3] + for ext in (DOC_EXT, ".md"): + if topic.endswith(ext): + topic = topic[: -len(ext)] + break return topic or None @@ -205,7 +77,7 @@ def _resolve_doc_path(docs_root: Path, path: str) -> Path | None: if topic not in topics: return None - return (docs_root / f"{topic}.md").resolve() + return (docs_root / f"{topic}{DOC_EXT}").resolve() def docs_command( diff --git a/packages/nemo_platform_ext/tests/cli/test_docs.py b/packages/nemo_platform_ext/tests/cli/test_docs.py index d0ddc40e48..50474ddfec 100644 --- a/packages/nemo_platform_ext/tests/cli/test_docs.py +++ b/packages/nemo_platform_ext/tests/cli/test_docs.py @@ -41,7 +41,7 @@ def test_docs_list(self): def test_docs_list_independent_of_cwd(self, tmp_path: Path, monkeypatch): unrelated_docs = tmp_path / "docs" unrelated_docs.mkdir() - (unrelated_docs / "jenkins-pipeline-guide.md").write_text("# Wrong docs root\n") + (unrelated_docs / "jenkins-pipeline-guide.mdx").write_text("# Wrong docs root\n") monkeypatch.delenv("NMP_DOCS_ROOT", raising=False) monkeypatch.chdir(tmp_path) @@ -52,19 +52,15 @@ def test_docs_list_independent_of_cwd(self, tmp_path: Path, monkeypatch): assert "get-started/setup" in result.stdout assert "jenkins-pipeline-guide" not in result.stdout - def test_docs_root_prefers_repo_docs_over_package_snapshot(self, tmp_path: Path, monkeypatch): + def test_docs_root_prefers_repo_docs_with_mdx(self, tmp_path: Path, monkeypatch): from nemo_platform_ext.cli.commands import docs monkeypatch.delenv("NMP_DOCS_ROOT", raising=False) repo_docs = tmp_path / "docs" - package_docs = tmp_path / "src" / "nemo_platform_ext" / "cli" / "docs" commands_dir = tmp_path / "src" / "nemo_platform_ext" / "cli" / "commands" repo_docs.mkdir(parents=True) - package_docs.mkdir(parents=True) commands_dir.mkdir(parents=True) - (tmp_path / "mkdocs.yml").write_text("site_name: Test\n") - (repo_docs / "index.md").write_text("# Repo docs\n") - (package_docs / "index.md").write_text("# Package snapshot\n") + (repo_docs / "index.mdx").write_text("# Repo docs\n") result = docs._find_docs_root(commands_dir / "docs.py") @@ -78,77 +74,24 @@ def test_docs_root_uses_package_snapshot_without_repo_docs(self, tmp_path: Path, commands_dir = tmp_path / "site-packages" / "nemo_platform" / "cli" / "commands" package_docs.mkdir(parents=True) commands_dir.mkdir(parents=True) - (package_docs / "index.md").write_text("# Packaged docs\n") + (package_docs / "index.mdx").write_text("# Packaged docs\n") result = docs._find_docs_root(commands_dir / "docs.py") assert result == package_docs.resolve() - def test_packaged_docs_are_filtered_by_packaged_mkdocs_config(self, tmp_path: Path): + def test_list_docs_skips_underscore_and_hidden_paths(self, tmp_path: Path): from nemo_platform_ext.cli.commands import docs - package_root = tmp_path / "site-packages" / "nemo_platform" / "cli" - package_docs = package_root / "docs" - package_docs.mkdir(parents=True) - (package_docs / "index.md").write_text("# Home\n") - (package_docs / "customizer").mkdir() - (package_docs / "customizer" / "about.md").write_text("# Hidden\n") - (package_docs / "template").mkdir() - (package_docs / "template" / "EULA.md").write_text("# Excluded\n") - (package_root / "mkdocs.yml").write_text( - """ -exclude_docs: | - template/ - -extra: - hidden_docs: - enabled: true - paths: - - customizer/** -""".lstrip() - ) - - assert docs._list_docs(package_docs) == ["index"] - - def test_single_item_env_sequence_is_not_treated_as_default(self, tmp_path: Path, monkeypatch): - from nemo_platform_ext.cli.commands import docs - - monkeypatch.delenv("NMP_HIDE_TEST_DOCS", raising=False) - package_root = tmp_path / "site-packages" / "nemo_platform" / "cli" - package_docs = package_root / "docs" - package_docs.mkdir(parents=True) - (package_docs / "index.md").write_text("# Home\n") - (package_docs / "hidden").mkdir() - (package_docs / "hidden" / "topic.md").write_text("# Hidden\n") - (package_root / "mkdocs.yml").write_text( - """ -extra: - hidden_docs: - enabled: !ENV [NMP_HIDE_TEST_DOCS] - paths: - - hidden/** -""".lstrip() - ) - - assert docs._list_docs(package_docs) == ["hidden/topic", "index"] - - def test_docs_list_filters_unrendered_topics(self): - result = _invoke("docs", "--list") + (tmp_path / "index.mdx").write_text("# Home\n") + (tmp_path / "guide").mkdir() + (tmp_path / "guide" / "topic.mdx").write_text("# Topic\n") + (tmp_path / "_snippets").mkdir() + (tmp_path / "_snippets" / "fragment.mdx").write_text("snippet") + (tmp_path / ".hidden").mkdir() + (tmp_path / ".hidden" / "secret.mdx").write_text("secret") - assert result.exit_code == 0 - topics = _listed_topics(result.stdout) - assert "get-started/setup" in topics - assert "cli/configuration" in topics - assert "pysdk/client/index" in topics - assert "audit/index" not in topics - assert "auth/concepts" not in topics - assert "customizer/about" not in topics - assert "evaluator/metrics/job-management" not in topics - assert "helm/index" not in topics - assert "CONTRIBUTING" not in topics - assert "README" not in topics - assert "template/EULA" not in topics - assert "work/guardrails/README" not in topics + assert docs._list_docs(tmp_path) == ["guide/topic", "index"] def test_docs_no_args_shows_list(self): result = _invoke("docs") @@ -158,13 +101,13 @@ def test_docs_no_args_shows_list(self): def test_docs_read_setup(self): result = _invoke("docs", "get-started/setup") assert result.exit_code == 0 - assert "# Setup" in result.stdout + assert "Setup" in result.stdout assert "nemo-setup" in result.stdout - def test_docs_read_with_md_extension(self): - result = _invoke("docs", "get-started/setup.md") + def test_docs_read_with_mdx_extension(self): + result = _invoke("docs", "get-started/setup.mdx") assert result.exit_code == 0 - assert "# Setup" in result.stdout + assert "Setup" in result.stdout assert "nemo-setup" in result.stdout def test_docs_nonexistent(self): @@ -172,18 +115,13 @@ def test_docs_nonexistent(self): assert result.exit_code == 1 assert "not found" in result.output.lower() - def test_docs_hidden_topic_not_readable(self): - result = _invoke("docs", "customizer/about") - assert result.exit_code == 1 - assert "not found" in result.output.lower() - def test_docs_path_traversal_blocked(self, tmp_path: Path): result = _invoke("docs", "../../etc/passwd", env={"NMP_DOCS_ROOT": str(tmp_path)}) assert result.exit_code == 1 assert "invalid path" in result.output.lower() def test_docs_env_override(self, tmp_path: Path): - doc_file = tmp_path / "test-topic.md" + doc_file = tmp_path / "test-topic.mdx" doc_file.write_text("# Test Topic\nHello from test.") result = _invoke("docs", "test-topic", env={"NMP_DOCS_ROOT": str(tmp_path)}) diff --git a/script/generate_config_docs.py b/script/generate_config_docs.py index 49ff49fae3..b621cfe8b8 100644 --- a/script/generate_config_docs.py +++ b/script/generate_config_docs.py @@ -9,7 +9,7 @@ so workspace packages (nmp.common, nmp.core.*, etc.) are on the import path. Outputs: - - docs/set-up/config-reference.md Markdown with YAML sections and inline comments + - docs/set-up/config-reference.mdx Fern MDX with YAML sections and inline comments Usage (run from repository root): @@ -56,7 +56,6 @@ # Repo root (parent of script/) — used for default output paths REPO_ROOT = Path(__file__).resolve().parent.parent -PLATFORM_NAME_TOKEN = "{{platform_name}}" # All config classes to include in the reference, in display order. # Excluded services (e.g. hello_world) are omitted from this list. @@ -187,16 +186,11 @@ def _build_field_info_tree(model_class: type) -> dict[str, Any]: return out -def _format_doc_description(desc: str) -> str: - """Apply docs-only substitutions to generated config comments.""" - return desc.replace("NeMo Platform", PLATFORM_NAME_TOKEN) - - def _format_comment(desc: str, default: Any, possible: str | None) -> str: """Single-line YAML comment: description, default, possible values (no class names). Caller adds indent.""" parts: list[str] = [] if desc: - parts.append(_format_doc_description(desc)) + parts.append(desc) default_str = _scalar_default_str(default) if default is not None and not callable(default) else "" if default_str: parts.append(f"default: {default_str}") @@ -410,9 +404,12 @@ def generate_yaml(entries: list[tuple[str, Any]]) -> str: def generate_markdown(entries: list[tuple[str, Any]]) -> str: """Generate full markdown document: YAML-first with comments, no tables, no class names.""" lines: list[str] = [] + lines.append("---") + lines.append('title: "NeMo Platform configuration reference"') + lines.append('description: ""') + lines.append("---") lines.append("(platform-config-reference)=") lines.append("") - lines.append("# NeMo Platform configuration reference") lines.append("") lines.append( "This document describes the structure and defaults for the global config file for the NeMo Platform. " @@ -443,7 +440,7 @@ def main() -> int: parser.add_argument( "--markdown", type=Path, - default=REPO_ROOT / "docs" / "set-up" / "config-reference.md", + default=REPO_ROOT / "docs" / "set-up" / "config-reference.mdx", help="Output path for markdown file", ) parser.add_argument( diff --git a/sdk/python/nemo-platform/pyproject.toml b/sdk/python/nemo-platform/pyproject.toml index 95ae41c55d..6d81fa0806 100644 --- a/sdk/python/nemo-platform/pyproject.toml +++ b/sdk/python/nemo-platform/pyproject.toml @@ -104,7 +104,7 @@ packages = ["src/nemo_platform"] [tool.hatch.build.targets.wheel.force-include] "../../../docs" = "nemo_platform/cli/docs" -"../../../mkdocs.yml" = "nemo_platform/cli/mkdocs.yml" + [tool.hatch.build.targets.sdist] # Basically everything except hidden files/directories (such as .github, .devcontainers, .python-version, etc) include = [ diff --git a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/docs.py b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/docs.py index 6a62ec07f5..a7de5d5f51 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/cli/commands/docs.py +++ b/sdk/python/nemo-platform/src/nemo_platform/cli/commands/docs.py @@ -8,48 +8,12 @@ import os import posixpath import sys -from collections.abc import Sequence -from fnmatch import fnmatchcase from pathlib import Path from typing import Annotated import typer -import yaml - -class _MkDocsConfigLoader(yaml.SafeLoader): - """YAML loader that tolerates MkDocs-specific tags.""" - - -def _construct_env(loader: _MkDocsConfigLoader, node: yaml.Node) -> object: - if isinstance(node, yaml.SequenceNode): - values = loader.construct_sequence(node) - if len(values) == 1: - env_names = values - default = None - else: - env_names = values[:-1] - default = values[-1] if values else None - for env_name in env_names: - env_value = os.environ.get(str(env_name)) - if env_value is not None: - return env_value - return default - - env_name = loader.construct_scalar(node) - return os.environ.get(env_name, "") - - -def _construct_unknown(loader: _MkDocsConfigLoader, _tag_suffix: str, node: yaml.Node) -> object: - if isinstance(node, yaml.MappingNode): - return loader.construct_mapping(node) - if isinstance(node, yaml.SequenceNode): - return loader.construct_sequence(node) - return loader.construct_scalar(node) - - -_MkDocsConfigLoader.add_constructor("!ENV", _construct_env) -_MkDocsConfigLoader.add_multi_constructor("", _construct_unknown) +DOC_EXT = ".mdx" def _find_docs_root(module_file: Path | None = None) -> Path | None: @@ -65,116 +29,22 @@ def _find_docs_root(module_file: Path | None = None) -> Path | None: for parent in parents: docs_dir = parent / "docs" - mkdocs_config = parent / "mkdocs.yml" - if mkdocs_config.is_file() and docs_dir.is_dir(): - return docs_dir.resolve() - - for parent in parents: - docs_dir = parent / "docs" - if docs_dir.is_dir() and (docs_dir / "index.md").is_file(): + if docs_dir.is_dir() and any(docs_dir.rglob(f"*{DOC_EXT}")): return docs_dir.resolve() return None -def _normalize_path(path: str) -> str: - return posixpath.normpath(path.strip().replace("\\", "/")).lstrip("./") - - -def _normalize_pattern(pattern: object) -> str: - raw_pattern = str(pattern).strip().replace("\\", "/") - is_directory_pattern = raw_pattern.endswith("/") and not raw_pattern.endswith("/**") - normalized = _normalize_path(raw_pattern) - if is_directory_pattern and not normalized.endswith("/"): - return f"{normalized}/" - return normalized - - -def _is_enabled(value: object) -> bool: - if isinstance(value, str): - return value.strip().lower() not in {"", "0", "false", "no", "off"} - return bool(value) - - -def _load_mkdocs_config(docs_root: Path) -> dict[str, object]: - config_path = docs_root.parent / "mkdocs.yml" - if not config_path.is_file(): - return {} - - try: - loaded = yaml.load(config_path.read_text(encoding="utf-8"), Loader=_MkDocsConfigLoader) - except (OSError, yaml.YAMLError): - return {} - - return loaded if isinstance(loaded, dict) else {} - - -def _sequence_value(value: object) -> Sequence[object]: - if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): - return () - return value - - -def _exclude_patterns(config: dict[str, object]) -> tuple[str, ...]: - exclude_docs = config.get("exclude_docs", "") - if isinstance(exclude_docs, str): - return tuple( - _normalize_pattern(line) - for line in exclude_docs.splitlines() - if line.strip() and not line.lstrip().startswith("#") - ) - - return tuple(_normalize_pattern(pattern) for pattern in _sequence_value(exclude_docs) if str(pattern).strip()) - - -def _hidden_doc_patterns(config: dict[str, object]) -> tuple[str, ...]: - extra = config.get("extra", {}) - if not isinstance(extra, dict): - return () - - hidden_docs = extra.get("hidden_docs", {}) - if not isinstance(hidden_docs, dict) or not _is_enabled(hidden_docs.get("enabled")): - return () - - return tuple( - _normalize_pattern(pattern) for pattern in _sequence_value(hidden_docs.get("paths", ())) if str(pattern).strip() - ) - - -def _matches_pattern(path: str, pattern: str) -> bool: - if pattern.endswith("/"): - directory = pattern.rstrip("/") - return path == directory or path.startswith(f"{directory}/") - - if pattern.endswith("/**"): - directory = pattern[:-3] - return path == directory or path.startswith(f"{directory}/") - - return fnmatchcase(path, pattern) - - -def _matches_any(path: str, patterns: tuple[str, ...]) -> bool: - return any(_matches_pattern(path, pattern) for pattern in patterns) - - -def _is_visible_doc(rel_path: Path, exclude_patterns: tuple[str, ...], hidden_patterns: tuple[str, ...]) -> bool: - if any(part.startswith("_") or part.startswith(".") for part in rel_path.parts): - return False - - posix_rel_path = rel_path.as_posix() - return not _matches_any(posix_rel_path, exclude_patterns) and not _matches_any(posix_rel_path, hidden_patterns) +def _is_visible_doc(rel_path: Path) -> bool: + return not any(part.startswith("_") or part.startswith(".") for part in rel_path.parts) def _list_docs(docs_root: Path) -> list[str]: - """List doc topics that match the rendered MkDocs site.""" - config = _load_mkdocs_config(docs_root) - exclude = _exclude_patterns(config) - hidden = _hidden_doc_patterns(config) - + """List doc topics under the docs root.""" paths = [] - for md_file in sorted(docs_root.rglob("*.md")): - rel = md_file.relative_to(docs_root) - if not _is_visible_doc(rel, exclude, hidden): + for doc_file in sorted(docs_root.rglob(f"*{DOC_EXT}")): + rel = doc_file.relative_to(docs_root) + if not _is_visible_doc(rel): continue paths.append(rel.with_suffix("").as_posix()) return paths @@ -190,8 +60,10 @@ def _topic_from_user_path(path: str) -> str | None: return None topic = normalized.removeprefix("./") - if topic.endswith(".md"): - topic = topic[:-3] + for ext in (DOC_EXT, ".md"): + if topic.endswith(ext): + topic = topic[: -len(ext)] + break return topic or None @@ -205,7 +77,7 @@ def _resolve_doc_path(docs_root: Path, path: str) -> Path | None: if topic not in topics: return None - return (docs_root / f"{topic}.md").resolve() + return (docs_root / f"{topic}{DOC_EXT}").resolve() def docs_command( diff --git a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_docs.py b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_docs.py index b71dbb3bc9..e10931e894 100644 --- a/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_docs.py +++ b/sdk/python/nemo-platform/tests/vendored/nemo_platform_ext/cli/test_docs.py @@ -41,7 +41,7 @@ def test_docs_list(self): def test_docs_list_independent_of_cwd(self, tmp_path: Path, monkeypatch): unrelated_docs = tmp_path / "docs" unrelated_docs.mkdir() - (unrelated_docs / "jenkins-pipeline-guide.md").write_text("# Wrong docs root\n") + (unrelated_docs / "jenkins-pipeline-guide.mdx").write_text("# Wrong docs root\n") monkeypatch.delenv("NMP_DOCS_ROOT", raising=False) monkeypatch.chdir(tmp_path) @@ -52,19 +52,15 @@ def test_docs_list_independent_of_cwd(self, tmp_path: Path, monkeypatch): assert "get-started/setup" in result.stdout assert "jenkins-pipeline-guide" not in result.stdout - def test_docs_root_prefers_repo_docs_over_package_snapshot(self, tmp_path: Path, monkeypatch): + def test_docs_root_prefers_repo_docs_with_mdx(self, tmp_path: Path, monkeypatch): from nemo_platform.cli.commands import docs monkeypatch.delenv("NMP_DOCS_ROOT", raising=False) repo_docs = tmp_path / "docs" - package_docs = tmp_path / "src" / "nemo_platform_ext" / "cli" / "docs" commands_dir = tmp_path / "src" / "nemo_platform_ext" / "cli" / "commands" repo_docs.mkdir(parents=True) - package_docs.mkdir(parents=True) commands_dir.mkdir(parents=True) - (tmp_path / "mkdocs.yml").write_text("site_name: Test\n") - (repo_docs / "index.md").write_text("# Repo docs\n") - (package_docs / "index.md").write_text("# Package snapshot\n") + (repo_docs / "index.mdx").write_text("# Repo docs\n") result = docs._find_docs_root(commands_dir / "docs.py") @@ -78,77 +74,24 @@ def test_docs_root_uses_package_snapshot_without_repo_docs(self, tmp_path: Path, commands_dir = tmp_path / "site-packages" / "nemo_platform" / "cli" / "commands" package_docs.mkdir(parents=True) commands_dir.mkdir(parents=True) - (package_docs / "index.md").write_text("# Packaged docs\n") + (package_docs / "index.mdx").write_text("# Packaged docs\n") result = docs._find_docs_root(commands_dir / "docs.py") assert result == package_docs.resolve() - def test_packaged_docs_are_filtered_by_packaged_mkdocs_config(self, tmp_path: Path): + def test_list_docs_skips_underscore_and_hidden_paths(self, tmp_path: Path): from nemo_platform.cli.commands import docs - package_root = tmp_path / "site-packages" / "nemo_platform" / "cli" - package_docs = package_root / "docs" - package_docs.mkdir(parents=True) - (package_docs / "index.md").write_text("# Home\n") - (package_docs / "customizer").mkdir() - (package_docs / "customizer" / "about.md").write_text("# Hidden\n") - (package_docs / "template").mkdir() - (package_docs / "template" / "EULA.md").write_text("# Excluded\n") - (package_root / "mkdocs.yml").write_text( - """ -exclude_docs: | - template/ - -extra: - hidden_docs: - enabled: true - paths: - - customizer/** -""".lstrip() - ) - - assert docs._list_docs(package_docs) == ["index"] - - def test_single_item_env_sequence_is_not_treated_as_default(self, tmp_path: Path, monkeypatch): - from nemo_platform.cli.commands import docs - - monkeypatch.delenv("NMP_HIDE_TEST_DOCS", raising=False) - package_root = tmp_path / "site-packages" / "nemo_platform" / "cli" - package_docs = package_root / "docs" - package_docs.mkdir(parents=True) - (package_docs / "index.md").write_text("# Home\n") - (package_docs / "hidden").mkdir() - (package_docs / "hidden" / "topic.md").write_text("# Hidden\n") - (package_root / "mkdocs.yml").write_text( - """ -extra: - hidden_docs: - enabled: !ENV [NMP_HIDE_TEST_DOCS] - paths: - - hidden/** -""".lstrip() - ) - - assert docs._list_docs(package_docs) == ["hidden/topic", "index"] - - def test_docs_list_filters_unrendered_topics(self): - result = _invoke("docs", "--list") + (tmp_path / "index.mdx").write_text("# Home\n") + (tmp_path / "guide").mkdir() + (tmp_path / "guide" / "topic.mdx").write_text("# Topic\n") + (tmp_path / "_snippets").mkdir() + (tmp_path / "_snippets" / "fragment.mdx").write_text("snippet") + (tmp_path / ".hidden").mkdir() + (tmp_path / ".hidden" / "secret.mdx").write_text("secret") - assert result.exit_code == 0 - topics = _listed_topics(result.stdout) - assert "get-started/setup" in topics - assert "cli/configuration" in topics - assert "pysdk/client/index" in topics - assert "audit/index" not in topics - assert "auth/concepts" not in topics - assert "customizer/about" not in topics - assert "evaluator/metrics/job-management" not in topics - assert "helm/index" not in topics - assert "CONTRIBUTING" not in topics - assert "README" not in topics - assert "template/EULA" not in topics - assert "work/guardrails/README" not in topics + assert docs._list_docs(tmp_path) == ["guide/topic", "index"] def test_docs_no_args_shows_list(self): result = _invoke("docs") @@ -158,13 +101,13 @@ def test_docs_no_args_shows_list(self): def test_docs_read_setup(self): result = _invoke("docs", "get-started/setup") assert result.exit_code == 0 - assert "# Setup" in result.stdout + assert "Setup" in result.stdout assert "nemo-setup" in result.stdout - def test_docs_read_with_md_extension(self): - result = _invoke("docs", "get-started/setup.md") + def test_docs_read_with_mdx_extension(self): + result = _invoke("docs", "get-started/setup.mdx") assert result.exit_code == 0 - assert "# Setup" in result.stdout + assert "Setup" in result.stdout assert "nemo-setup" in result.stdout def test_docs_nonexistent(self): @@ -172,18 +115,13 @@ def test_docs_nonexistent(self): assert result.exit_code == 1 assert "not found" in result.output.lower() - def test_docs_hidden_topic_not_readable(self): - result = _invoke("docs", "customizer/about") - assert result.exit_code == 1 - assert "not found" in result.output.lower() - def test_docs_path_traversal_blocked(self, tmp_path: Path): result = _invoke("docs", "../../etc/passwd", env={"NMP_DOCS_ROOT": str(tmp_path)}) assert result.exit_code == 1 assert "invalid path" in result.output.lower() def test_docs_env_override(self, tmp_path: Path): - doc_file = tmp_path / "test-topic.md" + doc_file = tmp_path / "test-topic.mdx" doc_file.write_text("# Test Topic\nHello from test.") result = _invoke("docs", "test-topic", env={"NMP_DOCS_ROOT": str(tmp_path)}) diff --git a/services/core/auth/scripts/auth-tools.py b/services/core/auth/scripts/auth-tools.py index c5c1239bf6..f87a91801c 100755 --- a/services/core/auth/scripts/auth-tools.py +++ b/services/core/auth/scripts/auth-tools.py @@ -647,7 +647,7 @@ def check( console.print("\n[yellow]Tip: Add these permissions to the permissions section in static-authz.yaml.[/yellow]") # Check if permissions reference doc is up to date - docs_path = project_root / "docs" / "auth" / "authorization" / "permissions-reference.md" + docs_path = project_root / "docs" / "auth" / "authorization" / "permissions-reference.mdx" if docs_path.exists(): expected = _generate_permissions_reference(auth_config) actual = docs_path.read_text(encoding="utf-8") @@ -1166,7 +1166,7 @@ def _build_grouped_rows( rows.append(f"| `{perm_name}` | {description} | {role_marks} |") else: actions = [_perm_action(p) for p in group] - actions_str = " \\| ".join(actions) + actions_str = " \\\\| ".join(actions) display = f"`{prefix}.({actions_str})`" # Build a combined description from the common prefix desc_parts = prefix.split(".") @@ -1202,9 +1202,12 @@ def _generate_permissions_reference(auth_config: Dict) -> str: ordered_roles = ["Viewer", "Editor", "Admin"] lines: List[str] = [] + lines.append("---") + lines.append('title: "Permissions Reference"') + lines.append('description: ""') + lines.append("---") lines.append("(permissions-reference)=") lines.append("") - lines.append("# Permissions Reference") lines.append("") lines.append( "Complete reference of all permissions across the NeMo Platform APIs. " @@ -1217,8 +1220,11 @@ def _generate_permissions_reference(auth_config: Dict) -> str: "For the RBAC model, see [Authorization Concepts](../concepts.md)." ) lines.append("") - lines.append("!!! note\n PlatformAdmin is omitted — it bypasses permission checks entirely at the policy level.") + lines.append("") lines.append("") + lines.append("PlatformAdmin is omitted — it bypasses permission checks entirely at the policy level.") + lines.append("") + lines.append("") for area, perm_names in area_groups.items(): display_name = AREA_DISPLAY_NAMES.get(area, area.replace("-", " ").title()) @@ -1272,7 +1278,7 @@ def generate_docs( auth_path = project_root / auth_path if output_path is None: - output_path = project_root / "docs" / "auth" / "authorization" / "permissions-reference.md" + output_path = project_root / "docs" / "auth" / "authorization" / "permissions-reference.mdx" else: output_path = project_root / output_path diff --git a/tools/lint/lint-config-reference-docs.sh b/tools/lint/lint-config-reference-docs.sh index c9ea0c2f02..e6096ce9c8 100755 --- a/tools/lint/lint-config-reference-docs.sh +++ b/tools/lint/lint-config-reference-docs.sh @@ -2,7 +2,7 @@ set -euo pipefail # Verify config reference doc is up to date. uv run --frozen generate-config-docs -git diff --exit-code docs/set-up/config-reference.md || { - echo "Config reference doc is out of date. Run 'uv run generate-config-docs' and commit docs/set-up/config-reference.md" +git diff --exit-code docs/set-up/config-reference.mdx || { + echo "Config reference doc is out of date. Run 'uv run generate-config-docs' and commit docs/set-up/config-reference.mdx" exit 1 } diff --git a/tools/mcp-dev-tools/nmp_dev_mcp.py b/tools/mcp-dev-tools/nmp_dev_mcp.py index a6a91dd0ef..00e3fbbd57 100644 --- a/tools/mcp-dev-tools/nmp_dev_mcp.py +++ b/tools/mcp-dev-tools/nmp_dev_mcp.py @@ -790,39 +790,34 @@ async def make_target(target: str) -> dict[str, Any]: # === DOCS MAKE TARGET OPERATIONS === - @server.tool(description="Run a docs Makefile target from the docs/ directory") + @server.tool( + description="Run a Fern docs Makefile target (docs-*) from the repo root" + ) async def make_docs(target: str) -> dict[str, Any]: """ - Run a make target from the docs/ subdirectory (equivalent to `cd docs && make `). + Run a Fern docs make target from the repo root (equivalent to `make `). + + The docs site is built with Fern; targets live in the root Makefile. Args: target: Make target name. Allowed targets: - Build: html, html-fast, publish, clean, env - Serve: live, live-fast, test-serve - Quality: linkcheck, lint-python - Utility: replace-substitutions, sync, help + Setup: docs-deps, docs-login + Serve: docs + Quality: docs-check, docs-broken-links, docs-fix-links + Publish: docs-preview, docs-publish Returns: Dictionary with make command results """ allowed_docs_targets = { - # Build - "html", - "html-fast", - "publish", - "clean", - "env", - # Serve - "live", - "live-fast", - "test-serve", - # Quality - "linkcheck", - "lint-python", - # Utility - "replace-substitutions", - "sync", - "help", + "docs-deps", + "docs-login", + "docs", + "docs-check", + "docs-broken-links", + "docs-fix-links", + "docs-preview", + "docs-publish", } if target not in allowed_docs_targets: @@ -831,7 +826,7 @@ async def make_docs(target: str) -> dict[str, Any]: "error": f"Target '{target}' not in allowlist. Allowed: {sorted(allowed_docs_targets)}", } - return run_command(["make", "-C", "docs", target], timeout=600) + return run_command(["make", target], timeout=600) return server diff --git a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/post_generation_update.py b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/post_generation_update.py index 66ba3275fc..6841af78fe 100755 --- a/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/post_generation_update.py +++ b/tools/nemo-platform-sdk-tools/src/nemo_platform_sdk_tools/sdk/post_generation_update.py @@ -229,7 +229,6 @@ def update_pyproject_toml(sdk_info: SdkInfo) -> bool: wheel_target["packages"] = ["src/nemo_platform"] wheel_target["force-include"] = { "../../../docs": "nemo_platform/cli/docs", - "../../../mkdocs.yml": "nemo_platform/cli/mkdocs.yml", } # Remove legacy build hook config if present