From db6e2d1f65a9b1dfd8f1dd444e3ab77b670993c1 Mon Sep 17 00:00:00 2001 From: anubhutiv Date: Mon, 29 Jun 2026 01:23:40 -0700 Subject: [PATCH] feat(customizer): add docs and skills for customizer rl backend Signed-off-by: anubhutiv --- .../tutorials/dpo-customization-job.ipynb | 515 ++++++++++++++ .../skills/nemo-customizer/SKILL.md | 551 ++++----------- .../references/batch-sizing.md | 150 ++++ .../references/dataset-formats.md | 56 +- .../references/hf-conversion.md | 4 +- .../references/hyperparameters-automodel.md | 318 +++++++++ .../references/hyperparameters-rl.md | 83 +++ .../references/hyperparameters-unsloth.md | 285 ++++++++ .../references/hyperparameters.md | 649 +----------------- .../references/integrations-setup.md | 2 +- .../nemo-customizer/references/reporting.md | 257 +++++++ .../references/rl-kubernetes-runtime.md | 57 ++ .../references/troubleshooting.md | 8 +- .../skills/nemo-customizer/tests.json | 15 + .../fixtures/integrations_wandb_mlflow.json | 38 + .../nemo-rl/tests/fixtures/minimal_dpo.json | 15 + .../nemo-rl/tests/test_contract_job_inputs.py | 36 + 17 files changed, 2014 insertions(+), 1025 deletions(-) create mode 100644 docs/customizer/tutorials/dpo-customization-job.ipynb create mode 100644 plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/batch-sizing.md create mode 100644 plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters-automodel.md create mode 100644 plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters-rl.md create mode 100644 plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters-unsloth.md create mode 100644 plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/reporting.md create mode 100644 plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/rl-kubernetes-runtime.md create mode 100644 plugins/nemo-rl/tests/fixtures/integrations_wandb_mlflow.json create mode 100644 plugins/nemo-rl/tests/fixtures/minimal_dpo.json create mode 100644 plugins/nemo-rl/tests/test_contract_job_inputs.py diff --git a/docs/customizer/tutorials/dpo-customization-job.ipynb b/docs/customizer/tutorials/dpo-customization-job.ipynb new file mode 100644 index 0000000000..d9a56d4f64 --- /dev/null +++ b/docs/customizer/tutorials/dpo-customization-job.ipynb @@ -0,0 +1,515 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "# DPO Model Customization Job\n", + "\n", + "Learn how to use the NeMo Platform to align a model with **DPO** (Direct Preference Optimization) on a preference dataset. For each prompt, DPO trains on a *chosen* (preferred) and a *rejected* response so the model prefers the chosen style — no separate reward model required.\n", + "\n", + "This tutorial uses the `rl` customization backend (powered by [NVIDIA NeMo-RL](https://github.com/NVIDIA-NeMo/RL)), which runs DPO on a **Ray** cluster. Unlike the [SFT](./sft-customization-job) and [LoRA](./lora-customization-job) tutorials (Docker GPU jobs), `rl` requires a **Kubernetes-backed** NeMo Platform. DPO here is **full-weight** (no LoRA/adapter); the output is a full model entity.\n", + "\n", + "**Time to complete:** approximately 45-60 minutes. Job duration increases with model and dataset size." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "Before starting this tutorial, ensure you have:\n", + "\n", + "1. **Completed the [Quickstart](../../get-started/quickstart.md)** to install the NeMo Platform and Python SDK.\n", + "2. **Installed the Python SDK** (PyPI wrapper: `pip install \"nemo-platform[all]\"`; source checkout: run `make bootstrap` from the repository root).\n", + "3. **Installed the `datasets` package**: `pip install datasets`.\n", + "4. **A platform configured with `platform.runtime: kubernetes`.** The `rl` (DPO) backend provisions a Ray cluster and has **no local Docker fallback** — `submit` fails fast on a Docker-runtime platform. Multi-node jobs (`parallelism.num_nodes > 1`) additionally require the platform-side `NMP_RL_MULTINODE_SHARED_STORAGE_PATH`.\n", + "5. **A HuggingFace token** with access to the gated base model (this tutorial uses `meta-llama/Llama-3.2-1B-Instruct`). Export it as `HF_TOKEN`.\n", + "6. **At least one GPU with CUDA 12.8+** and a GPU execution profile (`nemo jobs list-execution-profiles`)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Quick Start\n", + "\n", + "### 1. Initialize the SDK\n", + "\n", + "The SDK needs your NeMo Platform server URL. By default `http://localhost:8080` is used; set `NMP_BASE_URL` to override:\n", + "\n", + "```sh\n", + "export NMP_BASE_URL=\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "import time\n", + "import uuid\n", + "from pathlib import Path\n", + "from nemo_platform import NeMoPlatform, ConflictError\n", + "from nemo_platform.types.secrets import PlatformSecretResponse\n", + "from nemo_platform.types.files import HuggingfaceStorageConfigParam\n", + "from nemo_rl_plugin.schema import RlJobInput\n", + "\n", + "\n", + "def max_wait_time_checker(seconds: int, label: str = \"\"):\n", + " \"\"\"Return a check() that raises TimeoutError once `seconds` have elapsed.\"\"\"\n", + " start = time.time()\n", + "\n", + " def check():\n", + " if time.time() - start > seconds:\n", + " raise TimeoutError(f\"{label} took longer than {seconds} seconds\")\n", + "\n", + " return check\n", + "\n", + "\n", + "NMP_BASE_URL = os.environ.get(\"NMP_BASE_URL\", \"http://localhost:8080\")\n", + "sdk = NeMoPlatform(base_url=NMP_BASE_URL, workspace=\"default\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Prepare the Preference Dataset\n", + "\n", + "DPO trains on **preference data**. The `rl` backend takes a **single** dataset fileset that holds both `training.jsonl` and `validation.jsonl`, and auto-detects the row schema from the first line. Three preference formats are supported (see the platform's `BinaryPreferenceDatasetItemSchema` / `HelpSteer3DatasetItemSchema` / `Tulu3PreferenceDatasetItemSchema`):" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Binary Preference Format\n", + "\n", + "Simple `prompt` / `chosen` / `rejected` (the `prompt` may be a string or a list of chat messages):\n", + "\n", + "```json\n", + "{\"prompt\": \"What is the capital of France?\", \"chosen\": \"The capital of France is Paris.\", \"rejected\": \"I'm not sure.\"}\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### HelpSteer3 Format (used here)\n", + "\n", + "A conversation `context` (string or chat messages), two candidate `response1` / `response2`, and a signed `overall_preference` in -3..3 — **negative** means response 1 is preferred, **positive** means response 2, **0** is a tie. This is the **raw** schema of `nvidia/HelpSteer3`, so no conversion is needed:\n", + "\n", + "```json\n", + "{\"context\": [{\"role\": \"user\", \"content\": \"Explain how to use git rebase\"}], \"response1\": \"...\", \"response2\": \"...\", \"overall_preference\": -2}\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Tulu3 Preference Format\n", + "\n", + "Full chat conversations for both the chosen and rejected branches (each a list of messages ending with the assistant turn):\n", + "\n", + "```json\n", + "{\"chosen\": [{\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"preferred\"}], \"rejected\": [{\"role\": \"user\", \"content\": \"...\"}, {\"role\": \"assistant\", \"content\": \"dispreferred\"}]}\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Download nvidia/HelpSteer3\n", + "\n", + "We use [nvidia/HelpSteer3](https://huggingface.co/datasets/nvidia/HelpSteer3) (the `preference` subset), NVIDIA's open preference dataset. It ships native `train` and `validation` splits and matches the HelpSteer3 schema above, so we upload the rows **as-is** — the platform's `HelpSteer3Dataset` loader handles the `overall_preference` semantics (including ties) at training time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import load_dataset, Dataset\n", + "\n", + "print(\"Loading dataset nvidia/HelpSteer3 (preference subset)\")\n", + "ds = load_dataset(\"nvidia/HelpSteer3\", \"preference\")\n", + "\n", + "# Small subsets keep the tutorial fast; larger sets train better but take longer.\n", + "training_size = 3000\n", + "validation_size = 300\n", + "DATASET_NAME = \"dpo-dataset\"\n", + "DATASET_PATH = Path(\"dpo-dataset\").absolute()\n", + "os.makedirs(DATASET_PATH, exist_ok=True)\n", + "\n", + "train_dataset = ds[\"train\"]\n", + "validation_dataset = ds[\"validation\"]\n", + "assert isinstance(train_dataset, Dataset) and isinstance(validation_dataset, Dataset)\n", + "\n", + "# Save raw HelpSteer3 rows directly — no conversion. The platform detects the\n", + "# HelpSteer3 schema from the row keys (context / response1 / response2 / overall_preference).\n", + "train_dataset.select(range(training_size)).to_json(f\"{DATASET_PATH}/training.jsonl\")\n", + "validation_dataset.select(range(validation_size)).to_json(f\"{DATASET_PATH}/validation.jsonl\")\n", + "\n", + "print(f\"Saved training.jsonl ({training_size} rows) and validation.jsonl ({validation_size} rows)\")\n", + "with open(f\"{DATASET_PATH}/training.jsonl\") as f:\n", + " sample = json.loads(f.readline())\n", + "print(\"Sample keys:\", sorted(sample.keys()))\n", + "print(\"overall_preference:\", sample[\"overall_preference\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Create FileSet and Upload Preference Data\n", + "\n", + "Upload both JSONL files to a single FileSet so the DPO job can read them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " sdk.files.filesets.create(workspace=\"default\", name=DATASET_NAME, description=\"DPO preference data\")\n", + " print(f\"Created fileset: {DATASET_NAME}\")\n", + "except ConflictError:\n", + " print(f\"Fileset '{DATASET_NAME}' already exists, continuing...\")\n", + "\n", + "sdk.files.upload(local_path=DATASET_PATH, remote_path=\"\", fileset=DATASET_NAME, workspace=\"default\")\n", + "\n", + "print(\"Preference data:\")\n", + "print(json.dumps([f.model_dump() for f in sdk.files.list(fileset=DATASET_NAME, workspace=\"default\").data], indent=2, default=str))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4. Secrets Setup\n", + "\n", + "The base model (`meta-llama/Llama-3.2-1B-Instruct`) is gated, so store your HuggingFace token as a platform secret named `hf-token` and reference it on the model fileset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "HF_TOKEN = os.getenv(\"HF_TOKEN\")\n", + "\n", + "def create_or_get_secret(name: str, value: str | None, label: str) -> PlatformSecretResponse | None:\n", + " if not value:\n", + " print(f\"{label} is not set - skipping secret (gated model downloads will fail without it)\")\n", + " return None\n", + " try:\n", + " secret = sdk.secrets.create(name=name, workspace=\"default\", value=value)\n", + " print(f\"Created secret: {name}\")\n", + " return secret\n", + " except ConflictError:\n", + " print(f\"Secret '{name}' already exists, continuing...\")\n", + " return sdk.secrets.retrieve(name=name, workspace=\"default\")\n", + "\n", + "\n", + "hf_secret = create_or_get_secret(\"hf-token\", HF_TOKEN, \"HF_TOKEN\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5. Create Base Model FileSet and Model Entity\n", + "\n", + "DPO starts from an instruction-tuned base model. The model entity's spec is inferred asynchronously after creation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "HF_REPO_ID = \"meta-llama/Llama-3.2-1B-Instruct\"\n", + "MODEL_NAME = \"llama-3-2-1b-instruct\"\n", + "\n", + "storage = HuggingfaceStorageConfigParam(type=\"huggingface\", repo_id=HF_REPO_ID, repo_type=\"model\")\n", + "if hf_secret:\n", + " storage[\"token_secret\"] = hf_secret.name\n", + "\n", + "try:\n", + " base_model_fs = sdk.files.filesets.create(\n", + " workspace=\"default\", name=MODEL_NAME, description=\"Llama 3.2 1B Instruct base model\", storage=storage\n", + " )\n", + " print(f\"Created base model fileset: {MODEL_NAME}\")\n", + "except ConflictError:\n", + " base_model_fs = sdk.files.filesets.retrieve(workspace=\"default\", name=MODEL_NAME)\n", + " print(\"Base model fileset already exists.\")\n", + "\n", + "try:\n", + " base_model = sdk.models.create(workspace=\"default\", name=MODEL_NAME, fileset=f\"default/{MODEL_NAME}\")\n", + "except ConflictError:\n", + " base_model = sdk.models.retrieve(workspace=\"default\", name=MODEL_NAME)\n", + "\n", + "print(f\"Base model fileset: fileset://default/{base_model.name}\")\n", + "\n", + "# Wait for the ModelSpec to be inferred from the checkpoint.\n", + "check = max_wait_time_checker(600, \"Model spec\")\n", + "while not base_model.spec:\n", + " check()\n", + " time.sleep(10)\n", + " base_model = sdk.models.retrieve(workspace=\"default\", name=MODEL_NAME)\n", + "print(\"Model spec ready\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 6. Create the DPO Customization Job\n", + "\n", + "Submit a DPO job to the `rl` backend with `RlJobInput`. Note the DPO-specific shape:\n", + "\n", + "- `model` is a string ref to the model entity; `dataset` is a **single** string ref to the preference fileset (holding both files).\n", + "- The training method is `{\"type\": \"dpo\", ...}` — full-weight, no `finetuning_type`/LoRA.\n", + "- `ref_policy_kl_penalty` is **β** (DPO paper): how strongly the policy stays tied to the reference model.\n", + "- `rl` auto-generates the job id (`rl-`); read it back from the response.\n", + "\n", + "Other configurable knobs: `optimizer_type`, `adam_eps`, `activation_checkpointing`, `keep_top_k`, `val_at_end`, `preference_loss_weight`, `sft_loss_weight`. Run `nemo customization rl explain` for the live schema." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "job_suffix = uuid.uuid4().hex[:8]\n", + "OUTPUT_NAME = f\"llama-3-2-1b-dpo-{job_suffix}\"\n", + "\n", + "spec = RlJobInput(\n", + " model=f\"default/{base_model.name}\",\n", + " dataset=f\"default/{DATASET_NAME}\",\n", + " training={\n", + " \"type\": \"dpo\",\n", + " \"epochs\": 1,\n", + " \"batch_size\": 16,\n", + " \"micro_batch_size\": 1,\n", + " \"learning_rate\": 5e-6,\n", + " \"max_seq_length\": 4096,\n", + " \"ref_policy_kl_penalty\": 0.1,\n", + " \"parallelism\": {\n", + " \"num_nodes\": 1,\n", + " \"num_gpus_per_node\": 1,\n", + " \"tensor_parallel_size\": 1,\n", + " \"pipeline_parallel_size\": 1,\n", + " },\n", + " },\n", + " output={\"name\": OUTPUT_NAME},\n", + ")\n", + "\n", + "# `rl` auto-generates the job id (rl-); do not pass name=.\n", + "job = sdk.customization.rl.jobs.create(spec=spec, workspace=\"default\")\n", + "print(f\"Job ID: {job.job.name}\")\n", + "print(f\"Output model: {OUTPUT_NAME}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 7. Track Training Progress\n", + "\n", + "The DPO job runs four steps: download -> **dpo-training** (Ray) -> upload -> model-entity. We poll the top-level job status and surface the training step's progress." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import clear_output\n", + "\n", + "check = max_wait_time_checker(7200, \"DPO job\")\n", + "while True:\n", + " check()\n", + " status = sdk.jobs.get_status(name=job.job.name, workspace=\"default\")\n", + " clear_output(wait=True)\n", + " print(f\"Job Status: {status.status}\")\n", + "\n", + " step = max_steps = phase = None\n", + " for job_step in status.steps or []:\n", + " if job_step.name == \"dpo-training\":\n", + " for task in job_step.tasks or []:\n", + " d = task.status_details or {}\n", + " step, max_steps, phase = d.get(\"step\"), d.get(\"max_steps\"), d.get(\"phase\")\n", + " break\n", + " break\n", + " if step is not None and max_steps:\n", + " print(f\"Training: Step {step}/{max_steps} ({100 * step / max_steps:.1f}%)\")\n", + " if phase:\n", + " print(f\"Phase: {phase}\")\n", + "\n", + " if status.status in (\"completed\", \"failed\", \"cancelled\", \"error\"):\n", + " print(f\"\\nJob finished: {status.status}\")\n", + " break\n", + " time.sleep(15)\n", + "\n", + "assert status.status == \"completed\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Interpreting DPO training metrics** (in `status_details.metrics`):\n", + "\n", + "- **`loss`** — the DPO loss; should trend down as the policy learns to separate chosen from rejected.\n", + "- **Reward margin** (chosen minus rejected reward) — should trend **up**: the model increasingly prefers chosen responses.\n", + "- **Validation `loss`** — watch for divergence from training loss (overfitting). Raise `ref_policy_kl_penalty` (β) or add `sft_loss_weight` if the policy drifts too far from the reference." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 8. Validate the Output Model\n", + "\n", + "DPO produces a **full-weight model entity** (not an adapter). Confirm it was registered." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_entity = sdk.models.retrieve(workspace=\"default\", name=OUTPUT_NAME)\n", + "print(model_entity.model_dump_json(indent=2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 9. Deploy and Evaluate (optional)\n", + "\n", + "The DPO output is a full model, so it deploys like any full-weight checkpoint (see the [Full SFT](./sft-customization-job) tutorial for details). We deploy with vLLM and send a chat completion." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "deploy_suffix = uuid.uuid4().hex[:8]\n", + "DEPLOYMENT_CONFIG_NAME = f\"dpo-deployment-cfg-{deploy_suffix}\"\n", + "DEPLOYMENT_NAME = f\"dpo-deployment-{deploy_suffix}\"\n", + "\n", + "deployment_config = sdk.inference.deployment_configs.create(\n", + " workspace=\"default\",\n", + " name=DEPLOYMENT_CONFIG_NAME,\n", + " engine=\"vllm\",\n", + " model_spec={\"model_namespace\": \"default\", \"model_name\": OUTPUT_NAME},\n", + " executor_config={\"gpu\": 1, \"image_name\": \"vllm/vllm-openai\", \"image_tag\": \"v0.22.1\"},\n", + ")\n", + "\n", + "deployment = sdk.inference.deployments.create(\n", + " workspace=\"default\", name=DEPLOYMENT_NAME, config=deployment_config.name\n", + ")\n", + "print(f\"Deployment name: {deployment.name}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "check = max_wait_time_checker(1800, \"Deployment\")\n", + "while True:\n", + " check()\n", + " deployment_status = sdk.inference.deployments.retrieve(name=deployment.name, workspace=\"default\")\n", + " clear_output(wait=True)\n", + " print(f\"Deployment status: {deployment_status.status}\")\n", + " if str(deployment_status.status).lower() in (\"ready\", \"running\", \"failed\", \"error\"):\n", + " break\n", + " time.sleep(15)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "messages = [\n", + " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", + " {\"role\": \"user\", \"content\": \"Write a short, friendly email to a colleague asking to reschedule our meeting to Thursday.\"},\n", + "]\n", + "\n", + "response = sdk.inference.gateway.provider.post(\n", + " \"v1/chat/completions\",\n", + " name=deployment.name,\n", + " workspace=\"default\",\n", + " body={\"model\": f\"default/{OUTPUT_NAME}\", \"messages\": messages, \"temperature\": 0.7, \"max_tokens\": 256},\n", + ")\n", + "print(\"Model output:\\n\")\n", + "print(response[\"choices\"][0][\"message\"][\"content\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "You aligned a base model with **DPO** on the NeMo Platform using the `rl` backend:\n", + "\n", + "- Uploaded a HelpSteer3 preference dataset **as-is** (the platform detects the schema natively).\n", + "- Submitted a full-weight DPO job that ran on a Ray cluster via the Kubernetes executor.\n", + "- Registered the output as a full model entity and (optionally) deployed it for inference.\n", + "\n", + "**Next steps:** tune the alignment strength with `ref_policy_kl_penalty` (β), add `sft_loss_weight` to anchor the policy to the chosen responses, enable `activation_checkpointing` for memory headroom, or scale up with `parallelism`. See the `nemo-customizer` skill's `references/hyperparameters.md` (section NeMo-RL (DPO)) for the full knob reference." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/SKILL.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/SKILL.md index 0aaf6ac106..053349bdb2 100644 --- a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/SKILL.md +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/SKILL.md @@ -1,11 +1,12 @@ --- name: nemo-customizer description: >- - Fine-tune models on NeMo Platform with `automodel` or `unsloth` (both `submit` → - Docker GPU jobs via the platform Jobs service): HF dataset conversion, filesets, - model entities, SFT/LoRA job JSON (hyperparameters, batch, schedule, optimizer), - and job polling. Use for train, fine-tune, customize, SFT, LoRA, learning rate, - epochs, or nemo customization. + Fine-tune models on NeMo Platform with `automodel`, `unsloth`, or `rl` (all + `submit`-only): HF dataset conversion, filesets, model entities, and job JSON + (hyperparameters, batch, schedule, optimizer) + job polling. `automodel`/`unsloth` + run SFT/LoRA as Docker GPU jobs; `rl` runs DPO (preference optimization) on a Ray + cluster (Kubernetes). Use for train, fine-tune, customize, SFT, LoRA, DPO, + preference optimization, learning rate, epochs, or nemo customization. triggers: - nemo-customizer - nemo customizer @@ -16,14 +17,21 @@ triggers: - customize a model - sft - lora + - dpo + - direct preference optimization + - preference optimization + - preference tuning - automodel - unsloth + - nemo-rl + - nemo rl - nemo customization - nemo-customization - customizer - customization training - automodel submit - unsloth submit + - rl submit not-for: - nemo-build-agent (agent scaffold/deploy, not weight training) - nemo-explore (agent design only) @@ -40,14 +48,17 @@ allowed-tools: [Bash, Read, Grep] # NeMo Customizer -End-to-end **SFT + LoRA** on NeMo Platform. Two backend plugins ship in this repo — both are **`submit`-only** (local `run` is hard-disabled on each): +End-to-end **SFT + LoRA** (automodel/unsloth) and **DPO** (rl) on NeMo Platform. Three backend plugins ship in this repo — all are **`submit`-only** (local `run` is hard-disabled on each): -| Backend | Verb | Where it runs | Pick when | -|---------|------|---------------|-----------| -| **`automodel`** (default) | `submit` | Platform **Docker GPU executor** (Jobs service schedules containers on the platform host's daemon) | General SFT/LoRA; multi-GPU (data/tensor parallel); distillation; full-weight SFT | -| **`unsloth`** | `submit` | Same — Docker GPU job with 4 steps (download → train → upload → model-entity) | User asks for Unsloth, or wants Unsloth's 4-bit LoRA path / optimizer defaults on a single GPU | +| Backend | Verb | Trains | Where it runs | Pick when | +|---------|------|--------|---------------|-----------| +| **`automodel`** (default) | `submit` | SFT / LoRA | Platform **Docker GPU executor** (Jobs service schedules containers on the platform host's daemon) | General SFT/LoRA; multi-GPU (data/tensor parallel); distillation; full-weight SFT | +| **`unsloth`** | `submit` | SFT / LoRA | Same — Docker GPU job with 4 steps (download → train → upload → model-entity) | User asks for Unsloth, or wants Unsloth's 4-bit LoRA path / optimizer defaults on a single GPU | +| **`rl`** | `submit` | **DPO** (preference) | Platform **Kubernetes executor** — provisions a **Ray** cluster; 4 steps (download → DPO train → upload → model-entity) | Preference optimization / DPO / RLHF-style alignment from a `{prompt, chosen, rejected}` dataset; full-weight only | -`nemo-customizer` is the router (`nemo customization …`); training backends are separate plugins (`nemo-automodel`, `nemo-unsloth`). `submit` posts to the platform API; the platform runs training in container steps — **not** in the CLI shell. Heavy ML deps live in container images only. +`nemo-customizer` is the router (`nemo customization …`); training backends are separate plugins (`nemo-automodel`, `nemo-unsloth`, `nemo-rl`). `submit` posts to the platform API; the platform runs training in container steps — **not** in the CLI shell. Heavy ML deps live in container images only. + +**Runtime split:** `automodel`/`unsloth` need `platform.runtime: docker`; `rl` needs `platform.runtime: kubernetes` (no local Docker fallback — it schedules a Ray cluster on the remote cluster). A given platform is usually one or the other — confirm with execution profiles before picking `rl`. Decision rule below in **Plugin pick**. Batch shell work; reuse resources with `--exist-ok`; skip CLI `--help` unless a command fails. @@ -101,12 +112,15 @@ Full create/update commands, fileset `token_secret`, license acceptance, and dow ## Plugin pick 1. Run `nemo jobs list-execution-profiles -f json` (login first only if auth is enabled — see **Authentication**; see `references/troubleshooting.md` for parsing). -2. If the user explicitly asked for Unsloth → **`unsloth`**. -3. Else if the user explicitly asked for Automodel → **`automodel`**. -4. Else if any profile has `provider: gpu` or `gpu_distributed` → **`automodel`** (default). -5. Else stop and tell the user GPU customization is unavailable (both backends need a GPU execution profile and `platform.runtime: docker` on the connected platform). +2. If the task is **DPO / preference optimization** (a `{prompt, chosen, rejected}` dataset, "align", "preference", "RLHF-style") **or** the user explicitly asked for NeMo-RL → **`rl`** (requires a GPU profile **and** `platform.runtime: kubernetes`). +3. Else if the user explicitly asked for Unsloth → **`unsloth`**. +4. Else if the user explicitly asked for Automodel → **`automodel`**. +5. Else if any profile has `provider: gpu` or `gpu_distributed` → **`automodel`** (default, SFT/LoRA). +6. Else stop and tell the user GPU customization is unavailable (all backends need a GPU execution profile; `automodel`/`unsloth` also need `platform.runtime: docker`, `rl` needs `platform.runtime: kubernetes`). + +**`rl` runtime gate:** `rl submit` fails fast unless the platform runs `platform.runtime: kubernetes` (`require_distributed_runtime`). rl job steps execute as **Kubernetes pods via the `kubernetes_job` execution backend** — the **`docker` job backend cannot run rl**. Before submitting rl, confirm with `nemo jobs list-execution-profiles -f json` that the `cpu`/`gpu` profiles report `backend: kubernetes_job` (or `volcano_job`). If they report `backend: docker`/`subprocess`, the platform is **not** configured for rl: stop and tell the user DPO needs a Kubernetes-runtime platform — do **not** start/reuse a docker-runtime platform, and do **not** fall back to automodel/unsloth (those are SFT/LoRA, not DPO). To stand up or configure one, see `references/rl-kubernetes-runtime.md`. -Training never runs inside the `nemo` CLI process. After `submit`, the platform's **local Docker executor** launches GPU container steps on the daemon attached to that platform host (often the same machine as `http://127.0.0.1:8080`, but always query the platform — not the agent's shell GPU or a separate `docker info` on another box). +For **`automodel`/`unsloth`**, training never runs inside the `nemo` CLI process. After `submit`, the platform's **local Docker executor** launches GPU container steps on the daemon attached to that platform host (often the same machine as `http://127.0.0.1:8080`, but always query the platform — not the agent's shell GPU or a separate `docker info` on another box). **`rl` does not use the Docker executor** — its steps run on the Kubernetes cluster the platform is configured against. ## Gotchas @@ -125,25 +139,36 @@ Training never runs inside the `nemo` CLI process. After `submit`, the platform' ``` Poll until healthy (`curl -sf http://127.0.0.1:8080/health/ready` or retry `nemo jobs list-execution-profiles -f json`), then continue the workflow. Do not start services without asking. -- **Both backends are `submit` only** — `nemo customization run …` hard-fails on automodel and unsloth with a pointer to `submit`. Do not improvise verbs or pass `--venv`. -- **Never set `max_steps` together with `epochs`** (both backends). `max_steps` is a global cap and stops mid-epoch. Test fixtures include `max_steps` for smoke tests — do not copy into production jobs. Unsloth's schema enforces this as a hard mutex; automodel allows both but the result is surprising. -- **Job done (both backends) = top-level `status`** in `completed` | `error` | `cancelled`. Steps can all be `completed` while the job is still `active` (upload, entity registration). `status_details.phase` may stay `training` with `progress_pct: 100` for a long time — keep polling. `poll_customization_job.sh` works for any job id (`automodel-…` or `unsloth-…`); it exits **1** on `error` or `cancelled`. + - ⚠️ **This default start is a DOCKER-runtime platform — valid for `automodel`/`unsloth` only.** It is **NOT** valid for **`rl`**: rl needs `platform.runtime: kubernetes` with a `kubernetes_job` execution backend. Starting this default and submitting rl will fail the runtime gate. For rl, configure/point at a Kubernetes-runtime platform instead — see `references/rl-kubernetes-runtime.md`. Never start or reuse a docker-runtime platform for rl. +- **All backends are `submit` only** — `nemo customization run …` hard-fails with a pointer to `submit` (automodel, unsloth, and rl each disable local `run`). Do not improvise verbs or pass `--venv`. +- **Never set `max_steps` together with `epochs`** (automodel + unsloth; rl has the same caveat — see **rl (DPO) gotchas**). `max_steps` is a global cap and stops mid-epoch. Test fixtures include `max_steps` for smoke tests — do not copy into production jobs. Unsloth's schema enforces this as a hard mutex; automodel allows both but the result is surprising. +- **Job done (all backends) = top-level `status`** in `completed` | `error` | `cancelled`. Steps can all be `completed` while the job is still `active` (upload, entity registration). `status_details.phase` may stay `training` with `progress_pct: 100` for a long time — keep polling. `poll_customization_job.sh` works for any job id (`automodel-…`, `unsloth-…`, or `rl-…`); it exits **1** on `error` or `cancelled`. - Model spec fills async: **submit without polling** `nemo models get` unless submit fails. - HF dataset id from the user → convert locally; do not ask for local paths first. - Dataset fileset name = HF dataset **name** only (`tau/commonsense_qa` → `commonsense_qa`), not the model name. - Prefer **CHAT** JSONL when the model has a chat template; details in `references/dataset-formats.md` (automodel auto-detects schema; unsloth needs `dataset.apply_chat_template: true` to consume `messages`). -- User asks to tune **batch or parallelism** (automodel) → **Batch sizing** / **Multi-GPU** below. Other fields (LR, epochs, LoRA rank, distillation) → `references/hyperparameters.md`. For unsloth, see **Batch sizing — unsloth** and the `Unsloth job JSON` section in `references/hyperparameters.md`. Run `nemo customization explain` for the live schema. -- Skill **defaults** (`micro_batch_size` 1, `global_batch_size` 4) are safe on unknown VRAM. When the user has **≥48 GB** on one GPU, use **Batch sizing** instead of defaults. Unsloth's analogues are `batch.per_device_train_batch_size` and `batch.gradient_accumulation_steps` (effective batch = product). +- User asks to tune **batch or parallelism** (automodel) → `references/batch-sizing.md`. Other fields (LR, epochs, LoRA rank, distillation) → `references/hyperparameters-automodel.md`. For unsloth batch sizing see `references/batch-sizing.md`; for unsloth fields see `references/hyperparameters-unsloth.md`. Run `nemo customization explain` for the live schema. +- Skill **defaults** (`micro_batch_size` 1, `global_batch_size` 4) are safe on unknown VRAM. When the user has **≥48 GB** on one GPU, use `references/batch-sizing.md` instead of defaults. Unsloth's analogues are `batch.per_device_train_batch_size` and `batch.gradient_accumulation_steps` (effective batch = product). - **Unsloth training is single-GPU per job** (inside the container). `hardware.gpus` sets `CUDA_VISIBLE_DEVICES` before `import torch` — **selection, not reservation**. No `parallelism`/TP/PP block in job JSON. Multi-GPU sharding → use automodel. Pass `--profile ` on `unsloth submit` when the default `gpu` profile is wrong (automodel sets `training.execution_profile` in JSON instead). -- **Unsloth validation defaults** — when `dataset.validation_path` is set and `schedule.eval_steps` is omitted, the trainer runs validation once per effective epoch automatically. Report final `metrics.val_loss` from job status (see **Report to user**). Set `eval_steps` explicitly to override cadence. +- **Unsloth validation defaults** — when `dataset.validation_path` is set and `schedule.eval_steps` is omitted, the trainer runs validation once per effective epoch automatically. Report final `metrics.val_loss` from job status (see `references/reporting.md`). Set `eval_steps` explicitly to override cadence. - **Do not use local `docker info`** to pick automodel vs unsloth. Run `nemo jobs list-execution-profiles -f json` against the user's platform (login first only if auth is enabled — see **Authentication**; see `references/troubleshooting.md`). Default output is a table — **`-f json` is required** for scripting; parse **stdout only** (do not pipe `2>&1` into `json.load`). - **Do not merge stderr into stdout when parsing JSON** — `submit`, `explain`, and `-f json` commands write **JSON on stdout**; harmless warnings like `Configuration file not found, using defaults` go to **stderr**. Piping with **`2>&1`** before `json.load` raises `JSONDecodeError` even when submit **succeeded** — a common cause of **duplicate jobs** when the agent re-submits after a parse error. Parse stdout only; redirect stderr if needed (`2>/dev/null`). See `references/troubleshooting.md` § **Parsing CLI JSON**. -- For submit/image/plugin errors (both backends), read `references/troubleshooting.md`. Unsloth needs the `nmp-unsloth-training` container image on the **platform host's** Docker daemon (see `docker/unsloth/README.md`). -- **Missing training image on a remote platform** — if the user gave a non-localhost `NMP_BASE_URL` and the job errors with `Failed to pull image`, `manifest unknown`, or missing `nmp-unsloth-training` / automodel training image: **do not** run `docker build`, `docker pull`, or `docker buildx bake` on the agent machine. Report with **Report to user** (use **Output adapter fileset (planned):** on error), then append on-target build steps from `references/troubleshooting.md` § **Missing training images**. +- For submit/image/plugin errors (all backends), read `references/troubleshooting.md`. Unsloth needs the `nmp-unsloth-training` container image on the **platform host's** Docker daemon (see `docker/unsloth/README.md`); rl needs the `nmp-rl-tasks` / `nmp-rl-training` images on the Kubernetes cluster (see **rl (DPO) gotchas** and `references/rl-kubernetes-runtime.md`). +- **Missing training image on a remote platform** — if the user gave a non-localhost `NMP_BASE_URL` and the job errors with `Failed to pull image`, `manifest unknown`, or missing `nmp-unsloth-training` / automodel training image: **do not** run `docker build`, `docker pull`, or `docker buildx bake` on the agent machine. Report with the template in `references/reporting.md` (use **Output adapter fileset (planned):** on error), then append on-target build steps from `references/troubleshooting.md` § **Missing training images**. - **Gated HuggingFace models** (Llama, Gemma, …) — confirm `hf-token` + fileset `token_secret` before submit; download fails with `Failed to access upstream storage` / 502 when missing. See **HuggingFace token (gated models)** and `references/troubleshooting.md` § **Gated HuggingFace models**. - **Post-training eval format** — use the same CHAT `messages` JSONL as training. **Do not** flatten rows to `prompt`/`expected` for the evaluator. Send `messages[:-1]` at inference (exclude final assistant label); score against `messages[-1].content`. See `references/post-training-eval.md` and `references/eval_helpers.py`. - **LoRA adapters load automatically for eval** — when a LoRA job completes (`save_method: lora`), the adapter is registered on the base model entity and hot-reloaded on any **READY** deployment with `lora_enabled: true`. **Do not** create or update deployments before LoRA eval. **Full SFT** (`finetuning_type: all_weights`) and **merged checkpoints** (`merged_16bit` / `merged_4bit`) register a new **model** entity at `output.name` — **deploy that entity for inference** before chat or eval; full weights are not hot-reloaded onto the base deployment. For LoRA eval, route through the **provider** gateway (`/provider//-/v1` with `model: default--`); the model-entity path (`/model//-/v1`) always hits the base model. See `references/post-training-eval.md` § **Request routing (base vs LoRA)**. +### rl (DPO) gotchas + +- **rl is DPO, not SFT** — it trains on **preference pairs** `{prompt, chosen, rejected}`, full-weight (no LoRA/adapter; `finetuning_type` is not user-set). Don't route SFT/LoRA work here, and don't route DPO to automodel/unsloth. +- **One preference fileset, two files** — `dataset` is a **single string** ref to a fileset that holds **both** `training.jsonl` and `validation.jsonl` (uploaded with `--remote-path`). Unlike automodel (`dataset.training`/`dataset.validation`) and unsloth (`dataset.path`/`validation_path`), there is no separate validation ref. See `references/dataset-formats.md` § NeMo-RL. +- **String refs** — `model` and `dataset` are plain strings (`"workspace/name"`), not objects. The training method goes under `training` with `type: "dpo"`. +- **Kubernetes job backend, not Docker** — rl steps run as Kubernetes pods via the `kubernetes_job` backend; the docker job backend cannot run rl. `rl submit` fails fast on a docker-runtime platform. The target cluster must have the **job-step images** (`nmp-rl-tasks`, `nmp-rl-training`), the **jobs-launcher** image (the per-step init container), and a **job-storage PVC**. Verify the platform with `nemo jobs list-execution-profiles -f json` (expect `backend: kubernetes_job`); to configure one, see `references/rl-kubernetes-runtime.md`. Multi-node (`parallelism.num_nodes > 1`) also needs the platform-side `NMP_RL_MULTINODE_SHARED_STORAGE_PATH` (shared FS for Ray coordination) or compile fails fast. +- **Job id prefix is `rl-`** and the platform auto-generates it — `rl submit` has **no `--name` flag** (the job JSON `name` is the *output* name, not the job id). Derive the job id from `nemo jobs list` (newest `rl-*`) for polling; `poll_customization_job.sh rl-` works. +- **DPO main knob is `ref_policy_kl_penalty`** (β). For OOM, enable `activation_checkpointing: true` first. Full DPO field reference: `references/hyperparameters-rl.md`. +- **`max_steps` + `epochs`** — same caveat as the other backends: `max_steps` caps mid-epoch; it's in the smoke fixture (`plugins/nemo-rl/tests/fixtures/minimal_dpo.json`) — omit for real runs. + ## Workflow Common steps then **branch by plugin pick**: @@ -163,15 +188,24 @@ Common steps then **branch by plugin pick**: - [ ] Write /tmp/job.json (batch sizing for ≥48 GB GPU; else Defaults table) - [ ] nemo customization automodel submit /tmp/job.json --workspace default - [ ] Poll until top-level terminal (`poll_customization_job.sh`; default 15s interval, or 30–60s manual polls) -- [ ] Report using output template below +- [ ] Report using the template in `references/reporting.md` - [ ] Optional: compare base vs adapter on validation — `references/eval_helpers.py …` (LoRA only; CHAT format; adapters hot-reload automatically; see `references/post-training-eval.md`) # unsloth branch (submit → Docker GPU job) - [ ] Write /tmp/job.json using the UnslothJobInput shape (see Fast path — unsloth) - [ ] nemo customization unsloth submit /tmp/job.json --workspace default [--profile ] - [ ] Poll until top-level terminal (`poll_customization_job.sh unsloth-`; default 15s interval) -- [ ] Report using output template below +- [ ] Report using the template in `references/reporting.md` - [ ] Optional: compare base vs adapter on validation — `references/eval_helpers.py …` (LoRA only; CHAT format; adapters hot-reload automatically; see `references/post-training-eval.md`) + +# rl branch (DPO; submit → Kubernetes/Ray job) — requires platform.runtime: kubernetes +- [ ] Verify execution backend: `nemo jobs list-execution-profiles -f json` shows cpu/gpu at `backend: kubernetes_job` (NOT docker/subprocess). If not → stop; do not start a docker platform; configure per references/rl-kubernetes-runtime.md +- [ ] Dataset is PREFERENCE data: upload training.jsonl + validation.jsonl ({prompt,chosen,rejected}) to ONE fileset +- [ ] Write /tmp/job.json using the RlJobInput shape (see Fast path — rl (DPO)) +- [ ] nemo customization rl submit /tmp/job.json --workspace default [--profile ] +- [ ] Derive job id (newest rl-* from `nemo jobs list` — submit has no --name flag) +- [ ] Poll until top-level terminal (`poll_customization_job.sh rl-`; default 15s interval) +- [ ] Report using the template in `references/reporting.md` ``` ## Fast path — automodel @@ -213,7 +247,7 @@ nemo models create "$MODEL_ENTITY" --workspace default --exist-ok \ For gated repos, add `"token_secret":"hf-token"` to the `--storage` JSON (after creating the secret). See troubleshooting § **Gated HuggingFace models**. -**3. Job JSON** — write `/tmp/job.json`. `model` is the **registered model entity** (`default/`), not an HF repo id or dataset fileset. Full hyperparameter reference: `references/hyperparameters.md`. +**3. Job JSON** — write `/tmp/job.json`. `model` is the **registered model entity** (`default/`), not an HF repo id or dataset fileset. Full hyperparameter reference: `references/hyperparameters-automodel.md`. ```json { @@ -253,7 +287,7 @@ Same substitutions as automodel. Steps 1 (dataset) and 2 (model entity) are iden **2. Model** — same as automodel Fast path step 2. -**3. Job JSON** — write `/tmp/job.json` using the **`UnslothJobInput`** shape (see `references/hyperparameters.md` → *Unsloth job JSON*). `model` is an **object** (not a string), `dataset.path` is a single fileset ref, `hardware.gpus` replaces the `parallelism` block (single GPU in the training container). `nemo customization unsloth explain` prints the live schema. +**3. Job JSON** — write `/tmp/job.json` using the **`UnslothJobInput`** shape (see `references/hyperparameters-unsloth.md`). `model` is an **object** (not a string), `dataset.path` is a single fileset ref, `hardware.gpus` replaces the `parallelism` block (single GPU in the training container). `nemo customization unsloth explain` prints the live schema. ```json { @@ -295,6 +329,52 @@ Read `` from the `"name"` field in submit stdout (JSON). **Do not use `2 If you try `nemo customization unsloth run …`, the CLI hard-fails with a pointer to `submit`. +## Fast path — rl (DPO) + +DPO on a Ray cluster — **Kubernetes runtime only**, full-weight. **Before anything else**, confirm the platform dispatches jobs to Kubernetes: `nemo jobs list-execution-profiles -f json` must show `cpu`/`gpu` at `backend: kubernetes_job` (not `docker`/`subprocess`). If it doesn't, stop — do not start/use a docker-runtime platform; configure a Kubernetes-runtime one per `references/rl-kubernetes-runtime.md`. Model-entity setup (step 2) is identical to automodel; the dataset is **preference data** and the job JSON is the `RlJobInput` shape. + +**1. Preference dataset** — rows are `{prompt, chosen, rejected}` (see `references/dataset-formats.md` § NeMo-RL). Upload **both** files to **one** fileset: + +```bash +DATASET= # e.g. dpo-data +nemo files filesets create "$DATASET" --workspace default --purpose dataset --exist-ok +nemo files upload /tmp/dpo-train.jsonl "$DATASET" --workspace default --remote-path training.jsonl +nemo files upload /tmp/dpo-val.jsonl "$DATASET" --workspace default --remote-path validation.jsonl +nemo files list "$DATASET" --workspace default +``` + +**2. Model** — same as automodel Fast path step 2 (HF weights fileset + model entity; gated repos need `token_secret`). + +**3. Job JSON** — write `/tmp/job.json`. `model` and `dataset` are **strings**; the method is under `training` with `type: "dpo"`. Full field reference: `references/hyperparameters-rl.md`. + +```json +{ + "model": "default/", + "dataset": "default/", + "training": { + "type": "dpo", + "epochs": 1, + "learning_rate": 5e-6, + "max_seq_length": 1024, + "batch_size": 32, + "micro_batch_size": 1, + "ref_policy_kl_penalty": 0.05, + "parallelism": { "num_nodes": 1, "num_gpus_per_node": 1 } + }, + "output": { "name": "" } +} +``` + +**4. Submit and poll** — `rl submit` has **no `--name` flag** (the platform auto-generates the `rl-` job id), so derive it after submit: + +```bash +nemo customization rl submit /tmp/job.json --workspace default # add --profile if the default gpu profile is wrong +JOB=$(nemo jobs list -f json | python3 -c "import sys,json;d=json.load(sys.stdin);items=d.get('data',d) if isinstance(d,dict) else d;rl=[j for j in items if str(j.get('name','')).startswith('rl-')];rl.sort(key=lambda j:j.get('created_at',''),reverse=True);print(rl[0]['name'] if rl else '')") +bash plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/scripts/poll_customization_job.sh "$JOB" +``` + +**Do not use `2>&1`** before `json.load` — warnings on stderr break parsing; see Gotchas. Or poll manually: `nemo jobs get-status rl-` every 30–60s. If submit fails on an unknown profile, re-list execution profiles and pass `--profile `. `nemo customization rl run …` is disabled (no local execution — it provisions a Ray cluster); `nemo customization rl explain` prints the live schema. + ## Defaults Shared: @@ -312,7 +392,7 @@ Automodel-specific: | Field | Value | |-------|-------| | Parallelism | 1 node, 1 GPU, TP=1 | -| Batch | `global_batch_size` 4, `micro_batch_size` 1 (unknown VRAM; see **Batch sizing** for ≥48 GB) | +| Batch | `global_batch_size` 4, `micro_batch_size` 1 (unknown VRAM; see `references/batch-sizing.md` for ≥48 GB) | | Optimizer | `learning_rate` 5e-5 | Unsloth-specific: @@ -321,157 +401,33 @@ Unsloth-specific: |-------|-------| | Hardware | `hardware.gpus` `"0"`, `hardware.precision` `bf16` (selection only, single GPU) | | Model load | `load_in_4bit: true`, `dtype: "auto"` | -| Batch | `batch.per_device_train_batch_size` 2, `batch.gradient_accumulation_steps` 4 (effective batch 8; see **Batch sizing — unsloth** for ≥48 GB ramp) | +| Batch | `batch.per_device_train_batch_size` 2, `batch.gradient_accumulation_steps` 4 (effective batch 8; see `references/batch-sizing.md` for ≥48 GB ramp) | | Optimizer | `learning_rate` 5e-5, `optim` `adamw_8bit` | | Output | `save_method: "lora"` (adapter-only) unless user asks for merged checkpoint | | Gradient checkpointing | `training.use_gradient_checkpointing: "unsloth"` | -## Batch sizing — automodel (≥48 GB VRAM) - -Tables, multi-GPU rules, and the tuning loop below are **automodel-specific** (fields `global_batch_size` / `micro_batch_size` / `tensor_parallel_size` / `num_gpus_per_node`). For unsloth see **Batch sizing — unsloth** further down. - -Assume **one GPU with at least 48 GB** (e.g. RTX 5880 / A6000 / L40), `parallelism` = 1 node × 1 GPU, `tensor_parallel_size` 1, bf16, `training_type` `sft`, LoRA **rank 16** unless the user asks otherwise. - -**How to size** - -1. Read **model size** from the entity (`nemo models get`) or HF card (parameter count). -2. Pick **`finetuning_type`**: `lora` (adapter only, default) vs `all_weights` (full SFT — much heavier). -3. Set **`max_seq_length`** (2048 is the skill default; shorter seq → more batch headroom). -4. Set **`micro_batch_size`** first (drives peak VRAM), then **`global_batch_size`** as a multiple of `micro_batch_size` (gradient accumulation when GBS > micro). - -**Constraint:** `global_batch_size` must be divisible by `micro_batch_size × data_parallel_size`, where `data_parallel_size = (num_nodes × num_gpus_per_node) / (tensor_parallel_size × pipeline_parallel_size × context_parallel_size)` (1 for a single-GPU job). - -### LoRA (`finetuning_type: lora`) — `max_seq_length` 2048 - -**VRAM does not scale linearly with `micro_batch_size`.** LoRA loads the full base weights once; activation memory grows slowly. On 48 GB, **`micro_batch_size` must decrease as model size grows** (smaller models always ≥ larger models in the table). Use **`global_batch_size` ≈ 4 × `micro_batch_size`**. - -**Default batch** — start here for a reliable full epoch. **High utilization** — optional; double from default (or ramp in steps) to reach **~35–40 GiB**. Halve both if OOM (exit **137**) or training crashes (exit **1**). - -| Model params | Default `micro` | Default GBS | `learning_rate` | High-util `micro` | High-util GBS | -|--------------|------------------:|------------:|----------------:|------------------:|--------------:| -| ≤4B | 32 | 128 | `1e-4` | 64 | 256 | -| 4B–8B | 24 | 96 | `8e-5` | 48 | 192 | -| 8B–14B | 16 | 64 | `8e-5` | 24 | 96 | -| >14B | 8 | 32 | `5e-5` | 16 | 64 | - -Validated (`commonsense_qa` @ 2048, 48 GB, one job per GPU): **Qwen3-1.7B** — `micro` 16 / GBS 64 ~8 min; defaults above leave headroom to ramp. **Qwen3-8B** — `micro` 2–4 ≈16–18.5 GiB (under-filled); **`micro` 16 / GBS 64** stable default (~153 steps/epoch); high-util **`micro` 24 / GBS 96** (32 / 128 hit ~40 GiB but failed mid-epoch with exit 1). - -### Multi-GPU (same node) - -Pick the path by whether the **base model fits in ~48 GB on one GPU** (LoRA or full SFT): - -| Situation | `tensor_parallel_size` | Goal | -|-----------|------------------------:|------| -| Model **fits** on one ≥48 GB GPU | **1** | **Data parallel** — more GPUs = faster training; keep `micro` per GPU, scale `global_batch_size` | -| Model **does not fit** on one ≥48 GB GPU | **> 1** (e.g. 2 on a 2-GPU node) | **Tensor parallel** — shard layers across GPUs so the model fits; lower `micro` / GBS vs single-GPU tables | - -**Data parallel (TP = 1)** — default for Qwen3-8B LoRA and similar on 48 GB cards: - -| Rule | Detail | -|------|--------| -| `micro_batch_size` | **Per GPU** — same as a stable single-GPU run | -| `global_batch_size` | ≈ **single-GPU GBS × `num_gpus_per_node`**; step count ≈ `samples / GBS` | -| Divisibility | `global_batch_size` ÷ **`micro_batch_size × num_gpus_per_node`** must be an integer | -| Scheduling | **One job** owns all GPUs; no overlapping 1-GPU and multi-GPU jobs | - -```json -"parallelism": { "num_nodes": 1, "num_gpus_per_node": 2, "tensor_parallel_size": 1 }, -"batch": { "global_batch_size": 128, "micro_batch_size": 16 } -``` - -**Tensor parallel (TP > 1)** — when weights + activations OOM on a single ≥48 GB GPU (large full SFT, very long `max_seq_length`, or models above the LoRA sizing table without fitting): - -- Set **`num_gpus_per_node`** and **`tensor_parallel_size`** so **`num_gpus_per_node` is divisible by `tensor_parallel_size`** (e.g. 2 GPUs → `tensor_parallel_size: 2`, or 4 GPUs → TP 2 or 4). -- **`data_parallel_size`** = `(num_nodes × num_gpus_per_node) / (tensor_parallel_size × pipeline_parallel_size × context_parallel_size)` — use this in the GBS divisibility rule instead of raw GPU count. -- Start with **lower `micro_batch_size`** than the single-GPU table; increase only if VRAM allows. MoE models: if `expert_parallel_size > 1`, **`tensor_parallel_size` must be 1**. - -```json -"parallelism": { "num_nodes": 1, "num_gpus_per_node": 2, "tensor_parallel_size": 2 }, -"batch": { "global_batch_size": 8, "micro_batch_size": 1 } -``` - -`execution_profile` is usually still **`"gpu"`** — confirm with `nemo jobs list-execution-profiles -f json`. - -**Example — Qwen3-8B LoRA, 2× 48 GB (fits one GPU):** single-GPU **micro 16 / GBS 64** → 2-GPU data parallel **micro 16 / GBS 128**, `learning_rate` `8e-5`. - -### Full-weight SFT (`finetuning_type: all_weights`) — `max_seq_length` 2048 - -| Model params | `micro_batch_size` | `global_batch_size` | `learning_rate` | -|--------------|-------------------:|--------------------:|----------------:| -| ≤2B | 2 | 8 | `2e-5` | -| 2B–4B | 1 | 4 | `1e-5` | -| 4B–8B | 1 | 2 | `5e-6` | -| >8B | 1 | 1 | lower LR or use TP / shorter seq | - -Output type is **model** (full checkpoint), not adapter. Expect much longer runs than LoRA at the same batch. **Inference:** deploy `default/` as a new model entity — full SFT does not hot-reload onto the base model's LoRA deployment. - -### `max_seq_length` scaling - -Scale **`micro_batch_size`** from the 2048 tables (round down, minimum 1): - -| `max_seq_length` | Multiply `micro_batch_size` by | -|------------------|-------------------------------:| -| 512 | 4× | -| 1024 | 2× | -| 2048 | 1× (tables above) | -| 4096 | 0.5× | - -Then set `global_batch_size` to a multiple of the new `micro_batch_size` (often keep the same ratio as the table, e.g. GBS = 4 × micro for LoRA). +rl-specific (DPO): -### LoRA rank - -Higher rank uses more VRAM. If OOM at rank 16, drop to rank 8 before lowering batch; if headroom remains, rank 32 is fine for training (deploy rank ≤32 on default NIM/vLLM). - -### Tuning loop - -| Symptom | Action | -|---------|--------| -| CUDA OOM | Halve `micro_batch_size`, then `global_batch_size`, then `max_seq_length` | -| Slow / low GPU memory use | Step up toward the **high-util** column (or double default `micro`+GBS); stop at ~35–40 GiB or when training fails, then use **default** for the retry | -| User wants max throughput | Raise `micro_batch_size` first; keep GBS ≈ 4× micro — avoid `micro_batch_size` 1 with huge GBS | - -Field glossary, distillation/KD, and schema pointers: `references/hyperparameters.md` (batch/multi-GPU → **this file**, not hyperparameters). - -## Batch sizing — unsloth (single GPU) - -Unsloth is single-GPU by design. The effective batch is the **product** of two fields, not a global/micro split: - -```text -effective_batch = batch.per_device_train_batch_size × batch.gradient_accumulation_steps -``` - -There is no `parallelism` block, no TP / PP / DP, no GBS divisibility math. Multi-GPU sharding → switch to automodel. - -**Field mapping from the automodel tables above:** - -| Automodel field | Unsloth analogue | Notes | -|-----------------|------------------|-------| -| `micro_batch_size` | `batch.per_device_train_batch_size` | Drives peak VRAM. | -| `global_batch_size` | `batch.per_device_train_batch_size × batch.gradient_accumulation_steps` | Set `gradient_accumulation_steps` so the product matches the GBS you'd pick on automodel. | -| `parallelism.num_gpus_per_node` | n/a — single GPU | Use `hardware.gpus: "0"` to pin to one GPU. | -| `tensor_parallel_size` | n/a | If the model doesn't fit on one GPU → use automodel. | - -**Starting points (LoRA, `max_seq_length` 2048, one ≥48 GB GPU):** - -| Model params | `per_device_train_batch_size` | `gradient_accumulation_steps` | Effective batch | `learning_rate` | -|--------------|------------------------------:|------------------------------:|----------------:|----------------:| -| ≤4B | 8 | 16 | 128 | `1e-4` | -| 4B–8B | 4 | 24 | 96 | `8e-5` | -| 8B–14B | 2 | 32 | 64 | `8e-5` | -| >14B | 1 | 32 | 32 | `5e-5` | - -`load_in_4bit: true` (default) keeps base weights in 4-bit, which is what makes the "smaller per-device batch on bigger models" rule milder than vanilla HF. If you raise `per_device_train_batch_size` and hit OOM (exit 137) or training crashes (exit 1), halve `per_device_train_batch_size` first and double `gradient_accumulation_steps` to keep the effective batch the same. +| Field | Value | +|-------|-------| +| Training | DPO, full-weight (`type: "dpo"`; no LoRA) | +| Model (if user gives none) | `Qwen/Qwen3-0.6B` | +| Dataset (if user gives none) | `nvidia/HelpSteer3` (preference subset; uploaded raw — see `references/dataset-formats.md` § NeMo-RL) | +| Schedule (if user gives none) | small demo run: `max_steps` 20 (completes fast, proves the pipeline). For a real run, set `epochs` and **omit** `max_steps`. | +| Parallelism | 1 node, 1 GPU (`parallelism.num_nodes`/`num_gpus_per_node`) | +| Batch | `batch_size` 32, `micro_batch_size` 1 | +| Optimizer | `learning_rate` 5e-6 (DPO uses a low LR), AdamW + cosine | +| DPO | `ref_policy_kl_penalty` (β) 0.05, `sft_loss_weight` 0.0 | +| Max sequence length | 1024 | +| Output | full-weight model entity (`output.name`); no adapter | -**Save method.** Default `output.save_method: "lora"` (adapter only — small, fast, hot-reloads on LoRA-enabled deployments). Use `"merged_16bit"` if the user wants a full-weight checkpoint to deploy as a standalone model entity; `"merged_4bit"` only when storage is tight (lossy). Merged methods require `training.finetuning_type: "lora"`. Merged and full SFT outputs must be **deployed for inference** — they do not hot-reload onto the base adapter deployment. +When the user asks for a DPO job **without specifics**, default to the above: a +20-step run of `Qwen/Qwen3-0.6B` on `nvidia/HelpSteer3` — small enough to finish +quickly and confirm the pipeline end-to-end. -**Tuning loop (unsloth):** +## Batch sizing -| Symptom | Action | -|---------|--------| -| CUDA OOM | Halve `per_device_train_batch_size` (keep effective batch via `gradient_accumulation_steps`); then lower `model.max_seq_length`; then drop `lora.rank` to 8 | -| Missing `nmp-unsloth-training` image | Build/pull the Unsloth container image — see `references/troubleshooting.md` and `docker/unsloth/README.md` | -| `Unsloth training requires platform.runtime: docker` | Platform not using the Docker executor | Start platform with `platform.runtime: docker` and a GPU execution profile; training runs in containers on that host's Docker daemon | -| Loss not moving | Raise `learning_rate` one step (e.g. `5e-5` → `1e-4`); confirm `apply_chat_template` matches the data shape; check the LoRA `target_modules` covers the right layers (defaults are Unsloth's 7-module set) | +`micro_batch_size` / `global_batch_size` (automodel) and `per_device_train_batch_size` × `gradient_accumulation_steps` (unsloth) on **≥48 GB GPUs**, multi-GPU (data vs tensor parallel), and OOM / throughput tuning live in **`references/batch-sizing.md`**. On unknown VRAM the **Defaults** above are safe — read batch-sizing before raising batch on a known ≥48 GB card. rl (DPO) batch knobs (`batch_size` / `micro_batch_size`) are in `references/hyperparameters-rl.md`. ## Worked example @@ -479,249 +435,46 @@ There is no `parallelism` block, no TP / PP / DP, no GBS divisibility math. Mult **Unsloth:** same model + dataset + entity + fileset, but `nemo customization unsloth submit /tmp/job.json -w default`. Job JSON ≤4B row: `batch.per_device_train_batch_size` 8, `batch.gradient_accumulation_steps` 16 (effective 128), `learning_rate` `1e-4`, `hardware.gpus` `"0"`, `output.save_method` `"lora"`. Poll `unsloth-` to completion. Reference fixture: `plugins/nemo-unsloth/tests/fixtures/minimal_unsloth_sft.json` (ignore `max_steps` for real runs). -## Report to user - -After polling reaches a **terminal** status (`completed`, `error`, or `cancelled`), report using this template for **both** backends. Fill fields from the job JSON and `nemo jobs get-status`. - -```markdown -## Fine-tune result - -- **Job:** -- **Backend:** -- **Model entity:** default/ -- **Dataset fileset:** default/ -- **Output adapter fileset:** -- **Status:** -- **Final train loss:** -- **Final validation loss:** -- **Notes:** -``` - -**Field guidance** - -| Field | Source | -|-------|--------| -| **Job** | Job id from submit or poll (`automodel-…` / `unsloth-…`) | -| **Backend** | Plugin used for submit | -| **Model entity** | `model` in job JSON (automodel: string ref; unsloth: `model.name`) | -| **Dataset fileset** | automodel: `dataset.training`; unsloth: `dataset.path` | -| **Output adapter fileset** | `output.name` from job JSON. Label **Output adapter fileset (planned):** when status is `error` or `cancelled` and no output was registered | -| **Status** | Top-level `status` from `nemo jobs get-status` — not step-level status | -| **Final train loss** | Last entry in `status_details.metrics.train_loss` (or nested under a step's `status_details.metrics`). Use the **last** `value` in the list — not `status_details.train_loss` alone (that is the most recent logged step, which may differ from epoch-average loss on some backends). Round to 3 decimal places. | -| **Final validation loss** | Last entry in `status_details.metrics.val_loss`. If the list is empty, report `n/a (no validation run)` and note whether validation data was configured. Automodel validates once per epoch by default. Unsloth validates once per epoch when `dataset.validation_path` is set and `schedule.eval_steps` is omitted (platform default: `max(1, effective_steps - 1)`). | -| **Notes** | See **Notes by status** below | - -**Metrics extraction** — after polling, always run `nemo jobs get-status ` and read `status_details.metrics` (both backends accumulate `train_loss` and `val_loss` time series there). Include both final losses in the report even when status is `error` if training completed before the failure (e.g. entity registration failed after upload). - -**Notes by status** - -| Status | Notes | -|--------|-------| -| `completed` | Brief success summary. LoRA (`save_method: lora`): adapter registered on base model entity. Full SFT / merged checkpoint: new model entity at `output.name`. When `metrics.train_loss` has ≥2 entries, add a loss-drop sentence: *Loss dropped from \ at step 1 to \ at step \; validation loss was \.* Append **Using the adapter** (LoRA) or **Using the fine-tuned model** (full SFT / merged) with discovered provider name and concrete gateway URLs (see below). | -| `error` | Quote `error_details.message` or the failing step; note setup that succeeded before the failure (auth, dataset upload, submit). | -| `cancelled` | Cancellation reason if available. | - -**Training configuration (always)** — append a `### Training configuration` table after the header block (before **Using the adapter** when `completed`). Fill rows from the submitted job JSON; omit rows whose fields were not set. Use backend-specific labels: - -| Setting | automodel source | unsloth source | -|---------|------------------|----------------| -| Training type | `training.training_type` | `training.training_type` | -| Finetuning type | `training.finetuning_type` | `training.finetuning_type` | -| LoRA rank / alpha | `training.lora.rank` / `training.lora.alpha` | same | -| Quantization | omit (full-precision / bf16 base weights) | `model.load_in_4bit` → `4-bit (load_in_4bit: true)` or omit when false | -| Max sequence length | `training.max_seq_length` | `model.max_seq_length` | -| Epochs | `schedule.epochs` | `schedule.epochs` | -| Batch | `micro_batch_size` / `global_batch_size` | `batch.per_device_train_batch_size` / `batch.gradient_accumulation_steps` | -| Effective batch size | `global_batch_size` | `per_device_train_batch_size × gradient_accumulation_steps` | -| Learning rate | `optimizer.learning_rate` | same | -| Optimizer | `optimizer` fields used (e.g. `weight_decay`, `warmup_steps`) | `optimizer.optim` (e.g. `adamw_8bit`) | -| Precision | `bf16` (default) | `hardware.precision` | -| GPU | `parallelism.num_gpus_per_node` (and `tensor_parallel_size` when >1) | `hardware.gpus` | -| Output save method | `output.type` (e.g. `adapter`) | `output.save_method` (e.g. `lora`) | - -**Automodel example:** - -```markdown -### Training configuration - -| Setting | Value | -|---------|-------| -| Training type | SFT | -| Finetuning type | LoRA | -| LoRA rank / alpha | 16 / 32 | -| Max sequence length | 2048 | -| Epochs | 1 | -| Micro batch size | 16 | -| Global batch size | 64 | -| Effective batch size | 64 | -| Learning rate | 1e-4 | -| Optimizer | weight_decay 0.01, warmup_steps 0 | -| Precision | bf16 | -| GPU | 1 (TP=1) | -| Output save method | adapter | -``` - -**Unsloth example:** - -```markdown -### Training configuration - -| Setting | Value | -|---------|-------| -| Training type | SFT | -| Finetuning type | LoRA | -| LoRA rank / alpha | 16 / 32 | -| Quantization | 4-bit (`load_in_4bit: true`) | -| Max sequence length | 2048 | -| Epochs | 1 | -| Per-device batch size | 8 | -| Gradient accumulation steps | 16 | -| Effective batch size | 128 | -| Learning rate | 1e-4 | -| Optimizer | adamw_8bit | -| Precision | bf16 | -| GPU | 0 | -| Output save method | lora | -``` - -**Using the output (`completed` only)** — after **Training configuration**, branch on output type: - -| Output | When | Report section | -|--------|------|----------------| -| LoRA adapter | `save_method: lora` (default) | **Using the adapter** — below | -| Full model | `finetuning_type: all_weights`, or `save_method: merged_16bit` / `merged_4bit` | **Using the fine-tuned model** — below | - -### Using the adapter (LoRA / `save_method: lora`) - -Run these discovery commands (parse stdout only; do not pipe `2>&1` into JSON parsers): - -1. `nemo models get --workspace default` — confirm `` appears under `adapters` with `enabled: true`. -2. `nemo inference providers list --workspace default -f json` — pick a **READY** provider whose `served_models` includes `default/` (base entity). Record its `name` as `` (often matches the deployment name). +**rl (DPO):** the no-details default — `Qwen/Qwen3-0.6B` + `nvidia/HelpSteer3` (preference subset, uploaded raw), output `qwen3-0.6b-dpo`. First confirm `kubernetes_job` backend (see **Plugin pick** → rl runtime gate). Upload `training.jsonl` + `validation.jsonl` to one fileset, register the model entity, then submit a **small 20-step demo** job: -On a deployment with `lora_enabled: true`, the adapter is **hot-reloaded automatically** — no new deployment, deployment update, or provider reconfiguration before inference or post-training eval. Append this section with **concrete URLs and provider name** from discovery: - -```markdown -### Using the adapter - -The adapter `` is registered on `default/`. Weights are hot-reloaded on LoRA-enabled deployments serving the **base** entity — no new deployment or provider update after training. - -#### Request routing (base vs LoRA) - -| Target | Gateway path | OpenAI base URL | Request `"model"` field | -|--------|--------------|-----------------|-------------------------| -| **Base** weights | model-entity | `$NMP_BASE_URL/apis/inference-gateway/v2/workspaces/default/model//-/v1` | `default/` | -| **LoRA adapter** | **provider** | `$NMP_BASE_URL/apis/inference-gateway/v2/workspaces/default/provider//-/v1` | `default--` | - -**Common mistake:** posting to the model-entity URL with `"model": "default--"` still runs the **base** model. Base-vs-adapter eval will look identical until LoRA requests use the **provider** URL above. See `references/post-training-eval.md` § **Request routing (base vs LoRA)**. - -#### Chat inference (CHAT-trained models) - -Match training context at inference — send **`messages[:-1]`** (all turns except the final assistant label). Single-turn rows are just the user message; multi-turn rows keep prior user/assistant history. - -| Setting | Value | Why | -|---------|-------|-----| -| `messages` | All turns except the final assistant label from the JSONL row | Same decode path as SFT | -| `max_tokens` | `64` for short assistant labels | Training targets are brief (e.g. MCQA choice text) | -| `temperature` | `0` | Reproducible eval / regression checks | -| `chat_template_kwargs.enable_thinking` | `false` for Qwen3 short-answer SFT | Thinking mode needs extra tokens and changes output shape vs training | - -#### Example — LoRA adapter via provider - -\`\`\`bash -export NMP_BASE_URL= # omit when using default localhost -nemo inference gateway provider post v1/chat/completions --workspace default \\ - --body '{ - "model": "default--", - "messages": [], - "max_tokens": 64, - "temperature": 0, - "chat_template_kwargs": {"enable_thinking": false} - }' -\`\`\` - -#### Example — base model via model-entity (comparison) - -\`\`\`bash -export NMP_BASE_URL= -nemo inference gateway model post v1/chat/completions --workspace default \\ - --body '{ - "model": "default/", - "messages": [], - "max_tokens": 64, - "temperature": 0, - "chat_template_kwargs": {"enable_thinking": false} - }' -\`\`\` - -#### Post-training eval (optional) - -Validation loss from training is **not** accuracy. To compare base vs adapter on the validation split with correct routing: - -\`\`\`bash -cd /path/to/nemo-platform -uv run python plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/eval_helpers.py \\ - --model-entity \\ - --adapter \\ - --provider \\ - --dataset-fileset \\ - --split validation.jsonl -\`\`\` - -Uses CHAT `messages` rows unchanged from the training fileset (`messages[:-1]` at inference). Repeat `--adapter` for multi-adapter compare. `--provider` is optional when a READY provider is auto-discovered. Set `NMP_BASE_URL` (or pass `--base-url`) when the platform is not localhost. LoRA only — full SFT / merged outputs need a deployed model entity (see **Using the fine-tuned model**). -``` - -### Using the fine-tuned model (full SFT / merged checkpoint) - -When `finetuning_type: all_weights` or `save_method` is `merged_16bit` / `merged_4bit`, the job registers a **model** entity at `output.name` with full fine-tuned weights. **Deploy that entity before inference or eval** — full checkpoints are not hot-reloaded onto the base model's LoRA deployment. - -1. `nemo models get --workspace default` — confirm the fine-tuned model entity exists. -2. Create or update an inference deployment / provider that serves `default/` (same workflow as deploying any model entity). -3. Append this section with the **READY** provider or deployment name and concrete gateway URL. - -```markdown -### Using the fine-tuned model - -Fine-tuned weights are on model entity `default/`. Unlike LoRA adapters, full checkpoints **require a new inference deployment** (or provider update) before chat or eval. - -| Target | Gateway path | OpenAI base URL | Request `"model"` field | -|--------|--------------|-----------------|-------------------------| -| Fine-tuned model | model-entity | `$NMP_BASE_URL/apis/inference-gateway/v2/workspaces/default/model//-/v1` | `default/` | - -Use the same chat settings as LoRA inference (`messages[:-1]`, `max_tokens`, `temperature`, `enable_thinking` as appropriate). Post-training eval: run generation eval against this model-entity URL (not `eval_helpers.py --adapter`, which is LoRA-specific). +```json +{ + "model": "default/qwen3-0.6b", + "dataset": "default/helpsteer3-dpo", + "training": { "type": "dpo", "max_steps": 20, "batch_size": 32, "micro_batch_size": 1, + "learning_rate": 5e-6, "max_seq_length": 1024, "ref_policy_kl_penalty": 0.05, + "parallelism": { "num_nodes": 1, "num_gpus_per_node": 1 } }, + "output": { "name": "qwen3-0.6b-dpo" } +} ``` -Use the user's platform URL in `NMP_BASE_URL` when they overrode it; omit the export line for default `http://127.0.0.1:8080`. Substitute ``, concrete URLs, and entity names with values from discovery — do not leave generic placeholders in the user-facing report. For **LoRA**, do **not** tell the user to update the deployment before calling the adapter — registration on the base model entity is sufficient. For **full SFT / merged**, tell the user they must deploy `` before inference. +`nemo customization rl submit /tmp/job.json -w default`, derive the `rl-` id (submit has no `--name`), poll to completion. Reference fixture: `plugins/nemo-rl/tests/fixtures/minimal_dpo.json`. For a real run, replace `max_steps: 20` with `epochs`. -**Save report to `/tmp`** — unless the user opts out, write the full Markdown report (header, **Training configuration**, **Using the adapter** when `completed`, and **Resources created** when a slug or new filesets were used) to `/tmp/fine-tune-result-.md`. Use the random slug from the run when one was assigned; otherwise use the job id suffix (e.g. `a925b07ff678`). - -**Error follow-ups** — when the failure has a known fix, append sections **below** the header block (do not replace the header). Examples: - -| Error type | Append | -|------------|--------| -| Missing training image + user-overridden `NMP_BASE_URL` | `references/troubleshooting.md` § **Missing training images** — on-target build steps, env vars, re-submit commands. **Do not** `docker build` locally for a remote platform. | -| Download fails / `Failed to access upstream storage` / 502 on gated HF model | `references/troubleshooting.md` § **Gated HuggingFace models** — create/update `hf-token`, add `token_secret` to fileset, confirm HF license, re-submit. | -| W&B not syncing / no `[launcher]` secret lines / `WandbCallback requires wandb` / wandb 401 | `references/troubleshooting.md` § **W&B / integrations not working** (jobs-launcher build, secret update, unsloth image). Setup: `references/integrations-setup.md`. | +## Report to user -For other terminal errors, keep the same header template; put remediation detail in **Notes** or a short **Next steps** section as appropriate. +After polling reaches a **terminal** status (`completed`, `error`, or `cancelled`), report using the template in **`references/reporting.md`** — one format for all backends. It covers the **Fine-tune result** header, the **Training configuration** table (with per-backend examples: automodel, unsloth, rl/DPO), and **Using the adapter** (automodel/unsloth LoRA) vs **Using the fine-tuned model** (full SFT / merged / rl DPO), plus metrics extraction, notes by status, `/tmp` report saving, and error follow-ups. ## Reference files | When | Read | |------|------| | HF conversion or MCQA shaping | `references/hf-conversion.md` | -| CHAT vs SFT vs CUSTOM (automodel); text vs messages (unsloth) | `references/dataset-formats.md` | -| Field glossary, distillation/KD, schema (both backends) | `references/hyperparameters.md` (not batch sizing) | -| Batch sizing (≥48 GB), OOM / throughput | **Batch sizing — automodel** / **Batch sizing — unsloth** above | -| Multi-GPU same node | **Multi-GPU (same node)** under automodel batch sizing (unsloth is single-GPU) | +| CHAT vs SFT vs CUSTOM (automodel); text vs messages (unsloth); preference triples (rl/DPO) | `references/dataset-formats.md` | +| Field glossary, full JSON template, distillation/KD, live-schema pointers (index routes per backend) | `references/hyperparameters.md` → `hyperparameters-automodel.md` / `hyperparameters-unsloth.md` / `hyperparameters-rl.md` | +| Batch sizing (≥48 GB), OOM / throughput (automodel + unsloth) | `references/batch-sizing.md` | +| Multi-GPU same node | `references/batch-sizing.md` § **Multi-GPU (same node)** (unsloth is single-GPU) | +| Reporting: result template, Training configuration, Using the adapter / fine-tuned model | `references/reporting.md` | | Backend choice, execution profiles, submit failure, container images, missing image on remote platform, gated HF auth / download 502, CLI, connection errors | `references/troubleshooting.md` (§ **Parsing CLI JSON** for `2>&1` / `json.load`; § **Gated HuggingFace models** for `hf-token`) | -| Live JSON schema | `uv run nemo customization automodel explain` / `uv run nemo customization unsloth explain` | +| rl (DPO) needs Kubernetes job execution — verifying / configuring `runtime: kubernetes` + `kubernetes_job` executors (local platform → remote cluster, launcher image, PVC, loopback) | `references/rl-kubernetes-runtime.md` | +| Live JSON schema | `uv run nemo customization automodel explain` / `uv run nemo customization unsloth explain` / `uv run nemo customization rl explain` | | Job JSON fixture (automodel, minimal) | `plugins/nemo-automodel/tests/fixtures/qwen3_0.6b_sft_lora.json` (ignore `max_steps` for real runs) | | Job JSON fixture (unsloth, minimal) | `plugins/nemo-unsloth/tests/fixtures/minimal_unsloth_sft.json` (ignore `max_steps` for real runs) | -| Job JSON fixture (integrations, both backends) | `plugins/nemo-automodel/tests/fixtures/integrations_wandb_mlflow.json`, `plugins/nemo-unsloth/tests/fixtures/integrations_wandb_mlflow.json` | +| Job JSON fixture (rl / DPO, minimal) | `plugins/nemo-rl/tests/fixtures/minimal_dpo.json` (ignore `max_steps` for real runs) | +| Job JSON fixture — integrations (W&B / MLflow) | automodel: `plugins/nemo-automodel/tests/fixtures/integrations_wandb_mlflow.json` · unsloth: `plugins/nemo-unsloth/tests/fixtures/integrations_wandb_mlflow.json` · rl: `plugins/nemo-rl/tests/fixtures/integrations_wandb_mlflow.json` | | Automodel compile-path contract configs | `services/automodel/tests/contract/input_configs/` → YAML in `output_configs/` (legacy `TrainingStepConfig` shape, not submit JSON) | -| W&B / MLflow field reference | `references/hyperparameters.md` § **Integrations (automodel + unsloth)** | +| W&B / MLflow field reference (all backends) | `references/hyperparameters.md` § **Integrations (all backends)** | | W&B secret + MLflow local server + jobs-launcher | `references/integrations-setup.md` | | Gated HF model auth (`hf-token`, fileset `token_secret`) | `references/troubleshooting.md` § **Gated HuggingFace models** | | Post-training eval (base vs LoRA, CHAT format parity) | `references/post-training-eval.md`, `references/eval_helpers.py` | -Related: `plugins/nemo-automodel/README.md`, `plugins/nemo-unsloth/README.md`, `plugins/nemo-customizer/docs/CUSTOMIZATION.md`, skills **`nemo-files`**, **`nemo-status`**, **`nemo-secrets`**. +Related: `plugins/nemo-automodel/README.md`, `plugins/nemo-unsloth/README.md`, `plugins/nemo-rl/README.md`, `docs/customizer/nemo-rl-dpo-plugin-design.md`, `plugins/nemo-customizer/docs/CUSTOMIZATION.md`, skills **`nemo-files`**, **`nemo-status`**, **`nemo-secrets`**. diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/batch-sizing.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/batch-sizing.md new file mode 100644 index 0000000000..4918cfbbb1 --- /dev/null +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/batch-sizing.md @@ -0,0 +1,150 @@ +# Batch sizing + +VRAM tables, multi-GPU rules, and throughput tuning for **automodel** and **unsloth** on ≥48 GB GPUs. Field glossary and full JSON templates live in `hyperparameters.md`; the skill workflow lives in `SKILL.md`. + +## Batch sizing — automodel (≥48 GB VRAM) + +Tables, multi-GPU rules, and the tuning loop below are **automodel-specific** (fields `global_batch_size` / `micro_batch_size` / `tensor_parallel_size` / `num_gpus_per_node`). For unsloth see **Batch sizing — unsloth** further down. + +Assume **one GPU with at least 48 GB** (e.g. RTX 5880 / A6000 / L40), `parallelism` = 1 node × 1 GPU, `tensor_parallel_size` 1, bf16, `training_type` `sft`, LoRA **rank 16** unless the user asks otherwise. + +**How to size** + +1. Read **model size** from the entity (`nemo models get`) or HF card (parameter count). +2. Pick **`finetuning_type`**: `lora` (adapter only, default) vs `all_weights` (full SFT — much heavier). +3. Set **`max_seq_length`** (2048 is the skill default; shorter seq → more batch headroom). +4. Set **`micro_batch_size`** first (drives peak VRAM), then **`global_batch_size`** as a multiple of `micro_batch_size` (gradient accumulation when GBS > micro). + +**Constraint:** `global_batch_size` must be divisible by `micro_batch_size × data_parallel_size`, where `data_parallel_size = (num_nodes × num_gpus_per_node) / (tensor_parallel_size × pipeline_parallel_size × context_parallel_size)` (1 for a single-GPU job). + +### LoRA (`finetuning_type: lora`) — `max_seq_length` 2048 + +**VRAM does not scale linearly with `micro_batch_size`.** LoRA loads the full base weights once; activation memory grows slowly. On 48 GB, **`micro_batch_size` must decrease as model size grows** (smaller models always ≥ larger models in the table). Use **`global_batch_size` ≈ 4 × `micro_batch_size`**. + +**Default batch** — start here for a reliable full epoch. **High utilization** — optional; double from default (or ramp in steps) to reach **~35–40 GiB**. Halve both if OOM (exit **137**) or training crashes (exit **1**). + +| Model params | Default `micro` | Default GBS | `learning_rate` | High-util `micro` | High-util GBS | +|--------------|------------------:|------------:|----------------:|------------------:|--------------:| +| ≤4B | 32 | 128 | `1e-4` | 64 | 256 | +| 4B–8B | 24 | 96 | `8e-5` | 48 | 192 | +| 8B–14B | 16 | 64 | `8e-5` | 24 | 96 | +| >14B | 8 | 32 | `5e-5` | 16 | 64 | + +Validated (`commonsense_qa` @ 2048, 48 GB, one job per GPU): **Qwen3-1.7B** — `micro` 16 / GBS 64 ~8 min; defaults above leave headroom to ramp. **Qwen3-8B** — `micro` 2–4 ≈16–18.5 GiB (under-filled); **`micro` 16 / GBS 64** stable default (~153 steps/epoch); high-util **`micro` 24 / GBS 96** (32 / 128 hit ~40 GiB but failed mid-epoch with exit 1). + +### Multi-GPU (same node) + +Pick the path by whether the **base model fits in ~48 GB on one GPU** (LoRA or full SFT): + +| Situation | `tensor_parallel_size` | Goal | +|-----------|------------------------:|------| +| Model **fits** on one ≥48 GB GPU | **1** | **Data parallel** — more GPUs = faster training; keep `micro` per GPU, scale `global_batch_size` | +| Model **does not fit** on one ≥48 GB GPU | **> 1** (e.g. 2 on a 2-GPU node) | **Tensor parallel** — shard layers across GPUs so the model fits; lower `micro` / GBS vs single-GPU tables | + +**Data parallel (TP = 1)** — default for Qwen3-8B LoRA and similar on 48 GB cards: + +| Rule | Detail | +|------|--------| +| `micro_batch_size` | **Per GPU** — same as a stable single-GPU run | +| `global_batch_size` | ≈ **single-GPU GBS × `num_gpus_per_node`**; step count ≈ `samples / GBS` | +| Divisibility | `global_batch_size` ÷ **`micro_batch_size × num_gpus_per_node`** must be an integer | +| Scheduling | **One job** owns all GPUs; no overlapping 1-GPU and multi-GPU jobs | + +```json +"parallelism": { "num_nodes": 1, "num_gpus_per_node": 2, "tensor_parallel_size": 1 }, +"batch": { "global_batch_size": 128, "micro_batch_size": 16 } +``` + +**Tensor parallel (TP > 1)** — when weights + activations OOM on a single ≥48 GB GPU (large full SFT, very long `max_seq_length`, or models above the LoRA sizing table without fitting): + +- Set **`num_gpus_per_node`** and **`tensor_parallel_size`** so **`num_gpus_per_node` is divisible by `tensor_parallel_size`** (e.g. 2 GPUs → `tensor_parallel_size: 2`, or 4 GPUs → TP 2 or 4). +- **`data_parallel_size`** = `(num_nodes × num_gpus_per_node) / (tensor_parallel_size × pipeline_parallel_size × context_parallel_size)` — use this in the GBS divisibility rule instead of raw GPU count. +- Start with **lower `micro_batch_size`** than the single-GPU table; increase only if VRAM allows. MoE models: if `expert_parallel_size > 1`, **`tensor_parallel_size` must be 1**. + +```json +"parallelism": { "num_nodes": 1, "num_gpus_per_node": 2, "tensor_parallel_size": 2 }, +"batch": { "global_batch_size": 8, "micro_batch_size": 1 } +``` + +`execution_profile` is usually still **`"gpu"`** — confirm with `nemo jobs list-execution-profiles -f json`. + +**Example — Qwen3-8B LoRA, 2× 48 GB (fits one GPU):** single-GPU **micro 16 / GBS 64** → 2-GPU data parallel **micro 16 / GBS 128**, `learning_rate` `8e-5`. + +### Full-weight SFT (`finetuning_type: all_weights`) — `max_seq_length` 2048 + +| Model params | `micro_batch_size` | `global_batch_size` | `learning_rate` | +|--------------|-------------------:|--------------------:|----------------:| +| ≤2B | 2 | 8 | `2e-5` | +| 2B–4B | 1 | 4 | `1e-5` | +| 4B–8B | 1 | 2 | `5e-6` | +| >8B | 1 | 1 | lower LR or use TP / shorter seq | + +Output type is **model** (full checkpoint), not adapter. Expect much longer runs than LoRA at the same batch. **Inference:** deploy `default/` as a new model entity — full SFT does not hot-reload onto the base model's LoRA deployment. + +### `max_seq_length` scaling + +Scale **`micro_batch_size`** from the 2048 tables (round down, minimum 1): + +| `max_seq_length` | Multiply `micro_batch_size` by | +|------------------|-------------------------------:| +| 512 | 4× | +| 1024 | 2× | +| 2048 | 1× (tables above) | +| 4096 | 0.5× | + +Then set `global_batch_size` to a multiple of the new `micro_batch_size` (often keep the same ratio as the table, e.g. GBS = 4 × micro for LoRA). + +### LoRA rank + +Higher rank uses more VRAM. If OOM at rank 16, drop to rank 8 before lowering batch; if headroom remains, rank 32 is fine for training (deploy rank ≤32 on default NIM/vLLM). + +### Tuning loop + +| Symptom | Action | +|---------|--------| +| CUDA OOM | Halve `micro_batch_size`, then `global_batch_size`, then `max_seq_length` | +| Slow / low GPU memory use | Step up toward the **high-util** column (or double default `micro`+GBS); stop at ~35–40 GiB or when training fails, then use **default** for the retry | +| User wants max throughput | Raise `micro_batch_size` first; keep GBS ≈ 4× micro — avoid `micro_batch_size` 1 with huge GBS | + +Field glossary, distillation/KD, and schema pointers: `references/hyperparameters.md` (batch/multi-GPU → **this file**, not hyperparameters). + +## Batch sizing — unsloth (single GPU) + +Unsloth is single-GPU by design. The effective batch is the **product** of two fields, not a global/micro split: + +```text +effective_batch = batch.per_device_train_batch_size × batch.gradient_accumulation_steps +``` + +There is no `parallelism` block, no TP / PP / DP, no GBS divisibility math. Multi-GPU sharding → switch to automodel. + +**Field mapping from the automodel tables above:** + +| Automodel field | Unsloth analogue | Notes | +|-----------------|------------------|-------| +| `micro_batch_size` | `batch.per_device_train_batch_size` | Drives peak VRAM. | +| `global_batch_size` | `batch.per_device_train_batch_size × batch.gradient_accumulation_steps` | Set `gradient_accumulation_steps` so the product matches the GBS you'd pick on automodel. | +| `parallelism.num_gpus_per_node` | n/a — single GPU | Use `hardware.gpus: "0"` to pin to one GPU. | +| `tensor_parallel_size` | n/a | If the model doesn't fit on one GPU → use automodel. | + +**Starting points (LoRA, `max_seq_length` 2048, one ≥48 GB GPU):** + +| Model params | `per_device_train_batch_size` | `gradient_accumulation_steps` | Effective batch | `learning_rate` | +|--------------|------------------------------:|------------------------------:|----------------:|----------------:| +| ≤4B | 8 | 16 | 128 | `1e-4` | +| 4B–8B | 4 | 24 | 96 | `8e-5` | +| 8B–14B | 2 | 32 | 64 | `8e-5` | +| >14B | 1 | 32 | 32 | `5e-5` | + +`load_in_4bit: true` (default) keeps base weights in 4-bit, which is what makes the "smaller per-device batch on bigger models" rule milder than vanilla HF. If you raise `per_device_train_batch_size` and hit OOM (exit 137) or training crashes (exit 1), halve `per_device_train_batch_size` first and double `gradient_accumulation_steps` to keep the effective batch the same. + +**Save method.** Default `output.save_method: "lora"` (adapter only — small, fast, hot-reloads on LoRA-enabled deployments). Use `"merged_16bit"` if the user wants a full-weight checkpoint to deploy as a standalone model entity; `"merged_4bit"` only when storage is tight (lossy). Merged methods require `training.finetuning_type: "lora"`. Merged and full SFT outputs must be **deployed for inference** — they do not hot-reload onto the base adapter deployment. + +**Tuning loop (unsloth):** + +| Symptom | Action | +|---------|--------| +| CUDA OOM | Halve `per_device_train_batch_size` (keep effective batch via `gradient_accumulation_steps`); then lower `model.max_seq_length`; then drop `lora.rank` to 8 | +| Missing `nmp-unsloth-training` image | Build/pull the Unsloth container image — see `references/troubleshooting.md` and `docker/unsloth/README.md` | +| `Unsloth training requires platform.runtime: docker` (platform not using the Docker executor) | Start platform with `platform.runtime: docker` and a GPU execution profile; training runs in containers on that host's Docker daemon | +| Loss not moving | Raise `learning_rate` one step (e.g. `5e-5` → `1e-4`); confirm `apply_chat_template` matches the data shape; check the LoRA `target_modules` covers the right layers (defaults are Unsloth's 7-module set) | diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/dataset-formats.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/dataset-formats.md index d1d026656f..1a8ea30069 100644 --- a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/dataset-formats.md +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/dataset-formats.md @@ -1,8 +1,11 @@ # Dataset formats -Both backends read JSONL from a platform fileset, but the **row shape and the job-JSON dataset block differ**. Pick the section that matches your plugin. +All three backends read JSONL from a platform fileset, but the **row shape and the job-JSON dataset block differ**. Pick the section that matches your plugin (automodel, unsloth, or rl/DPO). -Upload `train.jsonl` and optional `validation.jsonl` at the **fileset root**. For automodel use the same fileset for `dataset.training` and `dataset.validation`. For unsloth use `dataset.path` (and `dataset.validation_path`). +Upload the JSONL files at the **fileset root**, then reference the fileset from the job JSON `dataset` block. The filenames differ by backend, so keep the two contracts separate: + +- **SFT (automodel, unsloth):** upload `train.jsonl` and optional `validation.jsonl`. Automodel points `dataset.training` / `dataset.validation` at the fileset; unsloth uses `dataset.path` (and `dataset.validation_path`). +- **rl (DPO):** upload both `training.jsonl` **and** `validation.jsonl` to a single fileset, referenced by one `dataset` string (no separate validation ref) — see § NeMo-RL. ## Automodel @@ -63,6 +66,53 @@ Eval rows must use the **same CHAT `messages` shape** as training. Do not flatte |----------------|--------------|------------------------|------------------| | `messages` (single- or multi-turn) | Same fileset split (`validation.jsonl`) | `messages[:-1]` — exclude final assistant label — see `post-training-eval.md` | `{{ item.messages[-1].content }}` | -LoRA inference and eval use the **provider** gateway on the **base** entity (`/provider//-/v1`, `model: default--`). Base model uses the model-entity path. Full SFT / merged checkpoints use the **output** model entity's model-entity URL — deploy first. See `post-training-eval.md` and the **Using the adapter** / **Using the fine-tuned model** sections in `SKILL.md`. +LoRA inference and eval use the **provider** gateway on the **base** entity (`/provider//-/v1`, `model: default--`). Base model uses the model-entity path. Full SFT / merged checkpoints use the **output** model entity's model-entity URL — deploy first. See `post-training-eval.md` and the **Using the adapter** / **Using the fine-tuned model** sections in `reporting.md`. Shared helpers and compare CLI: `references/eval_helpers.py`. Full workflow: `references/post-training-eval.md`. +## NeMo-RL (DPO) — preference data + +DPO trains on **preference pairs**, not prompt→completion examples. The `rl` backend takes a **single** dataset fileset that must contain **both** `training.jsonl` **and** `validation.jsonl` at the fileset root (unlike automodel/unsloth, the dataset block in the job JSON is a single ref — there is no separate validation ref). + +The dataset-preparation step **auto-detects the row schema from the first line** and selects the matching NeMo-RL loader. **Three preference formats are supported** (platform schemas `BinaryPreferenceDatasetItemSchema` / `HelpSteer3DatasetItemSchema` / `Tulu3PreferenceDatasetItemSchema`): + +### Binary preference (`BinaryPreferenceDataset`) + +Simple `prompt` / `chosen` / `rejected` — the `prompt` may be a plain string **or** a list of chat messages: + +```json +{"prompt": "What is the capital of France?", "chosen": "The capital of France is Paris.", "rejected": "I'm not sure."} +``` + +| Key | Meaning | +|-----|---------| +| `prompt` | The input/context shown to the model (string or list of chat messages). | +| `chosen` | The preferred (higher-reward) response. | +| `rejected` | The dispreferred response. | + +### HelpSteer3 (`HelpSteer3`) + +A conversation `context` (string or chat messages), two candidate responses, and a signed `overall_preference` in **-3..3** — **negative** means `response1` is preferred, **positive** means `response2`, **0** is a tie. This is the **raw** schema of `nvidia/HelpSteer3` (the `preference` subset), so no conversion is needed: + +```json +{"context": [{"role": "user", "content": "Explain how to use git rebase"}], "response1": "...", "response2": "...", "overall_preference": -2} +``` + +### Tulu3 preference (`Tulu3Preference`) + +Full chat conversations for both branches — `chosen` and `rejected` are each a **list of messages** ending with the assistant turn: + +```json +{"chosen": [{"role": "user", "content": "..."}, {"role": "assistant", "content": "preferred"}], "rejected": [{"role": "user", "content": "..."}, {"role": "assistant", "content": "dispreferred"}]} +``` + +Whichever format you upload, the job JSON dataset block is just the single fileset ref: + +```json +"dataset": "default/" +``` + +**Notes** +- Upload both files to the **same** fileset (`--remote-path training.jsonl` and `--remote-path validation.jsonl`). +- `prompt` may be a plain string; the model's chat template is applied at training time (override with `training.chat_template` only when needed). +- DPO is **full-weight** — there is no LoRA/adapter dataset variant. The output is a full model checkpoint. + diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hf-conversion.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hf-conversion.md index 642be8b9ca..e9f045aa60 100644 --- a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hf-conversion.md +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hf-conversion.md @@ -55,7 +55,7 @@ Then upload (see main skill). Validate with `nemo files list --wo ## Mapping to job JSON -The same converted JSONL works for both backends, but the **dataset block in job JSON is shaped per backend**. +Only the **chat-template / `messages` (`to_chat`) output is backend-agnostic** — that JSONL feeds both SFT backends (automodel + unsloth). The `to_sft` (`prompt` / `completion`) shape is **automodel-only**; unsloth needs a `to_text` rendering instead (see below). Either way, the **dataset block in job JSON is shaped per backend**. (rl/DPO uses preference data, not this SFT output — see `dataset-formats.md` § NeMo-RL.) | Backend | Row format used | Dataset block in job JSON | |---------|----------------|---------------------------| @@ -73,4 +73,4 @@ def to_text(ex): return {"text": f"{user}\n{assistant}"} ``` -For the chat path (`has_chat` True), the `to_chat` JSONL works unchanged across both backends — only the job-JSON dataset block differs. +For the chat path (`has_chat` True), the `to_chat` JSONL works unchanged across both SFT backends (automodel + unsloth) — only the job-JSON dataset block differs. diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters-automodel.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters-automodel.md new file mode 100644 index 0000000000..d837802a05 --- /dev/null +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters-automodel.md @@ -0,0 +1,318 @@ + + +# Automodel job JSON + +Job JSON for `nemo customization automodel submit` uses **`AutomodelJobInput`** (`plugins/nemo-automodel/src/nemo_automodel_plugin/schema.py`). Only fields in that schema are accepted (`extra="forbid"`). + +**Schema dump:** + +```bash +nemo customization automodel explain +``` + +**Contract examples:** `services/automodel/tests/contract/input_configs/` (legacy shape; map `batch_size` → `global_batch_size` in submit JSON). + +## Job JSON layout + +| Section | Purpose | +|---------|---------| +| `model` | **Base model entity** ref (`default/`) — weights to fine-tune | +| `dataset` | **Dataset filesets** (`default/`); optional `prompt_template` for CUSTOM schema | +| `training` | Method, LoRA, `max_seq_length`, distillation/KD fields | +| `schedule` | Epochs, optional step cap, validation cadence, seed | +| `batch` | Global/micro batch, sequence packing | +| `optimizer` | LR, weight decay, warmup | +| `parallelism` | Nodes, GPUs, TP/PP/CP/EP | +| `output` | Output adapter/model fileset name | +| `integrations` | Optional W&B / MLflow | + +### `model` field (base model entity) + +`model` must name a **Models API entity** for the checkpoint being trained — not a dataset fileset, not an output adapter from a prior job, and not a raw Hugging Face repo id. + +| Valid | Invalid | +|-------|---------| +| `default/qwen3-1.7b` (entity from `nemo models create`) | `Qwen/Qwen3-1.7B` (HF id) | +| `default/llama-3.2-1b-instruct` | `default/commonsense_qa` (dataset fileset) | +| `other-ws/my-model` (qualified ref) | `qwen3-1.7b-commonsense-qa-lora` (output fileset only, unless registered as entity) | + +Register before submit (same as skill fast path): HF **model** fileset → `nemo models create …` with `"fileset":"default/"`. List: `nemo models list --workspace default`. + +Full template: + +```json +{ + "model": "default/", + "dataset": { + "training": "default/", + "validation": "default/", + "prompt_template": null + }, + "training": { + "training_type": "sft", + "finetuning_type": "lora", + "lora": { + "rank": 16, + "alpha": 32, + "dropout": 0.0, + "merge": false, + "target_modules": null, + "exclude_modules": null, + "use_triton": true + }, + "max_seq_length": 2048, + "precision": null, + "attn_implementation": "sdpa", + "execution_profile": null + }, + "schedule": { + "epochs": 1, + "max_steps": null, + "val_check_interval": null, + "seed": null + }, + "batch": { + "global_batch_size": 4, + "micro_batch_size": 1, + "sequence_packing": false, + "sequence_packing_max_samples": 1000 + }, + "optimizer": { + "learning_rate": 5e-5, + "min_learning_rate": null, + "weight_decay": 0.01, + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "adam_eps": 1e-8, + "optimizer": "Adam", + "lr_decay_style": "cosine", + "warmup_steps": 0 + }, + "parallelism": { + "num_nodes": 1, + "num_gpus_per_node": 1, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "context_parallel_size": 1, + "expert_parallel_size": null, + "sequence_parallel": false + }, + "output": { "name": "", "description": null }, + "integrations": null +} +``` + +--- + +## Field reference + +### Automodel `training` + +| Field | Default | Notes | +|-------|---------|-------| +| `training_type` | `sft` | `distillation` requires `teacher_model` (entity ref) | +| `finetuning_type` | `lora` | `all_weights` (full fine-tune), `lora_merged` (merge adapter into base) | +| `lora.rank` | `16` | Higher → more capacity, more VRAM. Typical training range 8–32; **cap at 32** if the adapter will be served with default NIM / vLLM (rank > 32 may not load) | +| `lora.alpha` | `32` | Scaling; common rule of thumb **alpha ≈ 2× rank** | +| `lora.dropout` | `0.0` | LoRA dropout (0.0–1.0) for regularization | +| `lora.merge` | `false` | If true with `lora_merged`, output is full weights not adapter | +| `lora.target_modules` | `null` | e.g. `["q_proj","v_proj"]`; null = platform default targets | +| `lora.exclude_modules` | `null` | Patterns to exclude from LoRA, e.g. `["*.out_proj"]` | +| `lora.use_triton` | `true` | Use the optimized Triton LoRA kernel | +| `max_seq_length` | `2048` | Truncate/pack to this length; lower if OOM | +| `precision` | `null` | `bf16` \| `fp16` \| `fp32` \| `fp8`; null auto-detects from the checkpoint | +| `attn_implementation` | `sdpa` | `sdpa` (PyTorch native) \| `flash_attention_2` \| `eager` | +| `teacher_model` | — | **Model entity ref** (not HF id). Required for distillation; see below | +| `distillation_ratio` | `0.5` | KD blend (0–1) | +| `distillation_temperature` | `1.0` | KD temperature | +| `teacher_precision` | `bf16` | `bf16` \| `fp16` \| `fp32` | +| `offload_teacher` | `false` | Offload teacher weights to CPU | + +LoRA block is auto-created when `finetuning_type` is `lora` or `lora_merged`. + +### Automodel `schedule` + +| Field | Default | Notes | +|-------|---------|-------| +| `epochs` | `1` | Must be **≥ 1**. Full passes over training set | +| `max_steps` | `null` | **Global step cap.** Omit for epoch-based runs | +| `val_check_interval` | `null` | `≤ 1.0` = fraction of epoch; `> 1` = every N steps | +| `seed` | `null` | Reproducibility | + +**Gotcha:** Do **not** set `max_steps` with `epochs` for normal training. `max_steps` stops early (e.g. `epochs: 1` + `max_steps: 100` ends at step 100). Use `max_steps` **alone** only for smoke tests. + +### Automodel `batch` + +| Field | Default | Notes | +|-------|---------|-------| +| `global_batch_size` | `8` (schema) | Effective batch across all GPUs; **≥48 GB LoRA tables → `batch-sizing.md`** | +| `micro_batch_size` | `1` (schema) | **Per GPU**; same SKILL tables for single- and multi-GPU (TP=1) | +| `sequence_packing` | `false` | Pack short sequences for throughput (needs compatible data) | +| `sequence_packing_max_samples` | `1000` | Samples analyzed to estimate the optimal pack size (only when packing) | + +**Validation:** `global_batch_size` must be divisible by `micro_batch_size × data_parallel_size`, where: + +`data_parallel_size = (num_nodes × num_gpus_per_node) / (tensor_parallel_size × pipeline_parallel_size × context_parallel_size)` + +Example: 1 node, 2 GPUs, TP=1 → DP=2 → GBS must be a multiple of `2 × micro_batch_size`. See **`batch-sizing.md` § Multi-GPU** for data parallel vs tensor parallel. + +### Automodel `optimizer` + +| Field | Default | Notes | +|-------|---------|-------| +| `learning_rate` | `5e-6` (schema) | Skill uses **5e-5** for small LoRA SFT; see tuning below | +| `min_learning_rate` | `null` | Floor for the cosine LR decay; null lets it decay toward 0 | +| `weight_decay` | `0.01` | L2-style regularization | +| `adam_beta1` | `0.9` | Adam optimizer beta1 | +| `adam_beta2` | `0.999` | Adam optimizer beta2 | +| `adam_eps` | `1e-8` | Adam/AdamW epsilon for numerical stability | +| `optimizer` | `Adam` | `Adam` \| `AdamW` | +| `lr_decay_style` | `cosine` | `cosine` \| `linear` \| `constant` | +| `warmup_steps` | `0` | Linear warmup; try ~10% of total steps for long runs | + +### `parallelism` + +| Field | Default | Notes | +|-------|---------|-------| +| `num_nodes` | `1` | Multi-node distributed jobs | +| `num_gpus_per_node` | `1` | GPUs per node | +| `tensor_parallel_size` | `1` | **> 1** when the model does not fit on one ≥48 GB GPU — see **`batch-sizing.md` § Multi-GPU** | +| `pipeline_parallel_size` | `1` | Pipeline stages | +| `context_parallel_size` | `1` | Long-context sharding | +| `expert_parallel_size` | `null` | MoE only; must divide `data_parallel_size × context_parallel_size` | +| `sequence_parallel` | `false` | Shard activations along the sequence dim (pairs with tensor parallelism) | + +**MoE:** If `expert_parallel_size > 1` and multiple GPUs, `tensor_parallel_size` must be **1**. + +### Automodel `integrations` (optional) + +See **Integrations (all backends)** in `hyperparameters.md`. + +--- + +## Tuning guide (when the user asks) + +Apply user overrides to `/tmp/job.json` before submit. For **batch / GPU count / parallelism**, follow **`batch-sizing.md`** (defaults table + § Batch sizing + § Multi-GPU). Below covers **non-batch** fields and defers VRAM/batch symptoms to the skill. + +| Symptom / goal | Try first | +|----------------|-----------| +| CUDA OOM | **`batch-sizing.md` tuning loop:** halve `micro_batch_size`, then `global_batch_size`, then `max_seq_length`; use TP > 1 only if the model does not fit one ≥48 GB GPU | +| Slow / low GPU use | **`batch-sizing.md`:** step toward high-util column or double `micro`+GBS until ~35–40 GiB; multi-GPU data parallel if model fits one GPU | +| Underfitting | More `epochs`, slightly higher `learning_rate`, higher LoRA `rank` (≤ 32 for NIM/vLLM deploy) | +| Overfitting | Fewer `epochs`, lower `learning_rate`, higher `weight_decay`, smaller `rank` | +| Quick smoke test | `max_steps` only (e.g. 10–50), **omit or ignore epoch goal**; or `epochs: 1` on tiny slice | +| Reproducibility | Set `schedule.seed` | + +### Automodel learning rate (LoRA SFT, starting points) + +| Model scale | Suggested `learning_rate` | +|-------------|---------------------------| +| ≤ 3B | `5e-5` – `1e-4` | +| 3B – 8B | `2e-5` – `5e-5` | +| > 8B | `1e-5` – `2e-5` | + +Schema default is `5e-6` (conservative). Fixtures: `qwen3_0.6b_sft_lora.json` uses `5e-5`; `minimal_sft_lora.json` uses `5e-6`. + +### Automodel LoRA rank / alpha + +**Deployment cap:** Default **NIM** and **vLLM** LoRA serving paths support rank **≤ 32**. Use `rank` 32 (not higher) when the fine-tuned adapter will be deployed for inference on those stacks unless the user confirms a higher rank is supported. + +| Use case | `rank` | `alpha` | +|----------|--------|---------| +| Default / balanced | 16 | 32 | +| Low VRAM / light touch | 8 | 16 | +| More capacity (inference-safe max) | 32 | 64 | + +### Epochs vs dataset size + +One epoch = one full pass over `train.jsonl`. Steps per epoch ≈ `train_samples / global_batch_size` (e.g. ~10k samples, GBS 64 → ~153 steps). Plan poll time from the **GBS you chose in `batch-sizing.md`**, not the unknown-VRAM default (GBS 4). + +--- + +## Presets (non-batch fields) + +Use **`batch-sizing.md` § Batch sizing** and **§ Multi-GPU** for `batch` and `parallelism` on ≥48 GB GPUs. Presets below only override schedule / training / optimizer. + +**Smoke test (step-capped)** + +```json +"schedule": { "epochs": 1, "max_steps": 50 } +``` + +**Higher-quality LoRA (more VRAM/time)** + +```json +"training": { "lora": { "rank": 32, "alpha": 64 }, "max_seq_length": 2048 }, +"schedule": { "epochs": 3 }, +"optimizer": { "learning_rate": 2e-5, "warmup_steps": 100 } +``` + +Pair with batch rows from **`batch-sizing.md`** (e.g. ≤4B default `micro` 32 / GBS 128, not `micro` 1 / GBS 4). + +--- + +## Distillation (`training_type: "distillation"`) + +Use only when the user requests KD/distillation. **`model`** is the **student** entity; **`teacher_model`** is a separate **teacher** entity in the same workspace (unless qualified as `other-ws/name`). + +### Teacher model entity + +`teacher_model` must be a registered **model entity ref**, same shape as `model`: + +| Form | Example | +|------|---------| +| Same workspace | `default/llama-3.2-3b-instruct` | +| Explicit workspace | `default/` | + +It is **not** a Hugging Face repo id. Register the teacher like the student before submit: + +```bash +TEACHER_WEIGHTS=llama-3.2-3b-instruct # fileset name +TEACHER_ENTITY=llama-3.2-3b-instruct # entity name +TEACHER_HF=meta-llama/Llama-3.2-3B-Instruct + +nemo files filesets create "$TEACHER_WEIGHTS" --workspace default --purpose model --exist-ok \ + --storage '{"type":"huggingface","repo_id":"'"$TEACHER_HF"'","repo_type":"model","revision":"main"}' + +nemo models create "$TEACHER_ENTITY" --workspace default --exist-ok \ + --input-data '{"name":"'"$TEACHER_ENTITY"'","fileset":"default/'"$TEACHER_WEIGHTS"'","custom_fields":{"hf_model_id":"'"$TEACHER_HF"'"}}' +``` + +Verify: `nemo models get --workspace default`. Reuse an existing entity with `nemo models list` when present. + +**Compatibility:** Student and teacher must share the **same vocabulary / tokenizer family** (compiler loads both for KD). Mismatched tokenizers fail at runtime. Prefer a larger instruct model as teacher and a smaller base/chat model as student in the same family when possible. + +**VRAM:** Set `offload_teacher: true` if the job OOMs loading student + teacher; `teacher_precision: "bf16"` is the default. + +### Job JSON + +```json +{ + "model": "default/", + "dataset": { "training": "default/" }, + "training": { + "training_type": "distillation", + "finetuning_type": "lora", + "teacher_model": "default/", + "distillation_ratio": 0.5, + "distillation_temperature": 1.0, + "teacher_precision": "bf16", + "offload_teacher": false, + "max_seq_length": 2048 + }, + "schedule": { "epochs": 1 }, + "batch": { "global_batch_size": 64, "micro_batch_size": 16 }, + "optimizer": { "learning_rate": 8e-5 }, + "parallelism": { "num_nodes": 1, "num_gpus_per_node": 1, "tensor_parallel_size": 1 }, + "output": { "name": "" } +} +``` + +(`batch` / `parallelism` example uses an 8B-scale row from **`batch-sizing.md`**; adjust for student size.) + +| Field | Meaning | +|-------|---------| +| `distillation_ratio` | Blend of KD vs CE loss (`0` = CE only, `1` = KD only) | +| `distillation_temperature` | Softmax temperature for teacher logits | +| `offload_teacher` | CPU-offload frozen teacher weights to save GPU memory | + diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters-rl.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters-rl.md new file mode 100644 index 0000000000..958b79694c --- /dev/null +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters-rl.md @@ -0,0 +1,83 @@ + + +# NeMo-RL (DPO) job JSON + +The `rl` backend (`nemo customization rl submit`) runs **DPO** (Direct Preference Optimization) on a Ray cluster — **Kubernetes runtime only**, full-weight (no LoRA). Schema: `RlJobInput` in `plugins/nemo-rl/src/nemo_rl_plugin/schema.py`. Run `nemo customization rl explain` for the live schema. + +## Job JSON layout + +```json +{ + "model": "default/", + "dataset": "default/", + "training": { + "type": "dpo", + "epochs": 1, + "learning_rate": 5e-6, + "max_seq_length": 1024, + "batch_size": 32, + "micro_batch_size": 1, + "ref_policy_kl_penalty": 0.05, + "parallelism": { "num_nodes": 1, "num_gpus_per_node": 1 } + }, + "output": { "name": "" } +} +``` + +- `model` is a **string** ref to a registered model entity (`"name"` or `"workspace/name"`) — not an object (unsloth) and not the HF id. +- `dataset` is a **single string** ref to a preference fileset containing `training.jsonl` + `validation.jsonl` (see `references/dataset-formats.md` § NeMo-RL). There is no separate validation ref. +- `output.name` is the full-weight model entity the job registers. DPO output type is always `model` (no adapter). + +## Field reference — `training` (DPOTraining) + +### General (shared training knobs) + +| Field | Default | Notes | +|-------|---------|-------| +| `learning_rate` | `1e-4` | Peak LR. DPO typically uses a **low** LR (e.g. `5e-6`–`1e-5`). | +| `min_learning_rate` | `null` | Floor for cosine decay. | +| `weight_decay` | `0.01` | | +| `adam_beta1` / `adam_beta2` | `0.9` / `0.999` | Adam betas. | +| `adam_eps` | `1e-5` | Adam epsilon (numerical stability). | +| `warmup_steps` | `0` | Linear warmup steps. | +| `optimizer_type` | `null` → `adamw_with_cosine_annealing` | One of `adamw_with_cosine_annealing`, `adam_with_cosine_annealing`, `adamw_with_flat_lr`, `adam_with_flat_lr` (optimizer × LR-scheduler). | +| `epochs` | `1` | Passes over the dataset. | +| `max_steps` | `null` | Global step cap. Caps the run at `min(max_steps, epochs × steps_per_epoch)`, so it's safe to combine with `epochs` to stop smoke jobs mid-epoch — omit for real runs. | +| `val_check_interval` | `null` | Float ≤ 1.0 = fraction of epoch; > 1.0 = step count. | +| `val_at_end` | `true` | Run a final validation pass after the last step. Keep enabled: it makes the final checkpoint carry validation metrics so best-checkpoint selection (`metric_name`/`keep_top_k`) works — otherwise NeMo-RL warns and falls back to the latest checkpoint. Set `false` only to skip the extra eval. | +| `keep_top_k` | `1` | Number of best checkpoints to retain (ranked by validation loss). | +| `batch_size` | `32` | Global batch (preference **pairs**) across all GPUs. | +| `micro_batch_size` | `1` | Per-GPU micro batch. | +| `max_seq_length` | `2048` | Max token sequence length. | +| `activation_checkpointing` | `false` | Recompute activations in the backward pass to cut memory — the first knob to enable for OOM / larger models / longer sequences. | +| `seed` | `null` → `42` | | +| `execution_profile` | `null` | GPU execution profile; falls back to the service default. | + +### DPO-specific + +| Field | Default | Notes | +|-------|---------|-------| +| `ref_policy_kl_penalty` | `0.05` | **β** in the DPO paper — strength of the KL penalty tying the policy to the reference model. Higher = stay closer to the reference. The main DPO knob. | +| `preference_loss_weight` | `1.0` | Weight on the preference (DPO) loss term. | +| `sft_loss_weight` | `0.0` | Weight on an auxiliary SFT regularization loss (`0` = pure DPO). Raise (e.g. `0.1`) to anchor the policy to the chosen responses. | +| `preference_average_log_probs` | `false` | Normalize preference log-probs by sequence length. | +| `sft_average_log_probs` | `false` | Normalize SFT-loss log-probs by sequence length. | +| `max_grad_norm` | `1.0` | Gradient clipping norm. | + +### `parallelism` + +Same block as automodel (`num_nodes`, `num_gpus_per_node`, `tensor_parallel_size`, `pipeline_parallel_size`, `context_parallel_size`, `sequence_parallel`). Divisibility rule (enforced by `RlJobOutput.validate_for_training`): `total_gpus = num_nodes × num_gpus_per_node` must be divisible by `tensor_parallel_size × pipeline_parallel_size × context_parallel_size`, and `batch_size` by `micro_batch_size × data_parallel_size`. **Multi-node (`num_nodes > 1`)** additionally requires the platform to set `NMP_RL_MULTINODE_SHARED_STORAGE_PATH` (shared filesystem for Ray's cross-node coordination); the compiler fails fast otherwise. + +## Integrations (W&B / MLflow) + +rl supports **W&B and MLflow** through the top-level `integrations` object (`integrations.wandb` / `integrations.mlflow`) — the same object shape used across all backends; full field reference in `hyperparameters.md` § **Integrations (all backends)**. rl specifics: the run name defaults to the **job id**, tags are auto-prefixed (`service:rl`, `framework:…`), and because rl runs on Kubernetes / Ray the `tracking_uri` / self-hosted W&B `base_url` must be reachable **from the cluster** (the local `docker0` recipe in `integrations-setup.md` is Docker-runtime only). + +## DPO tuning guide + +| Symptom | Action | +|---------|--------| +| Policy degenerates / drifts too far | Raise `ref_policy_kl_penalty` (β), e.g. `0.05` → `0.1`–`0.5`. | +| Barely changes from the reference | Lower β, or raise `learning_rate` one step (still keep it low for DPO). | +| Forgets base capabilities | Add SFT regularization: `sft_loss_weight` `0.1`–`0.5`. | +| CUDA OOM | Set `activation_checkpointing: true`; then lower `micro_batch_size`; then `max_seq_length`. | + diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters-unsloth.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters-unsloth.md new file mode 100644 index 0000000000..6c02c9d8d0 --- /dev/null +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters-unsloth.md @@ -0,0 +1,285 @@ + + +# Unsloth job JSON + +Job JSON for `nemo customization unsloth submit` uses **`UnslothJobInput`** (`plugins/nemo-unsloth/src/nemo_unsloth_plugin/schema.py`). Only fields in that schema are accepted (`extra="forbid"`). The canonical post-transform shape lives in `services/unsloth/src/nmp/unsloth/schemas.py` (`UnslothJobOutput`) and is what the training driver consumes in the GPU container. + +**Schema dump:** + +```bash +nemo customization unsloth explain +``` + +Unsloth is **submit-only, single-GPU inside the training container**. There is no `parallelism` block and no `training.execution_profile` in job JSON — pass `--profile` on `nemo customization unsloth submit` instead (default `gpu`). `hardware.gpus` sets `CUDA_VISIBLE_DEVICES` in the container before `import torch`. Multi-GPU sharding → use automodel. + +## Job JSON layout (unsloth) + +| Section | Purpose | +|---------|---------| +| `name` | Optional job name (auto-generated if omitted) | +| `model` | **Object** — base model entity ref + how to load it (4-bit, dtype, max_seq_length) | +| `dataset` | Single fileset ref (`path`) + optional `validation_path`; row shape selector (`text_field`, `apply_chat_template`, `packing`) | +| `training` | Method (`sft`), adapter shape (`lora`/`full`), LoRA hyperparams, gradient checkpointing | +| `schedule` | `epochs` xor `max_steps`; `warmup_steps` xor `warmup_ratio`; logging / save / eval cadence; LR scheduler | +| `batch` | `per_device_train_batch_size` × `gradient_accumulation_steps` = effective batch | +| `optimizer` | LR, weight decay, optimizer choice (`adamw_8bit` default) | +| `hardware` | GPU selection (`CUDA_VISIBLE_DEVICES`) + mixed precision (`bf16` / `fp16`) | +| `integrations` | Optional W&B / MLflow (same shape as automodel) | +| `output` | Output entity name, optional description, **`save_method`** (controls what's persisted) | + +Full template (every section, defaults inline): + +```json +{ + "name": "", + "model": { + "name": "default/", + "max_seq_length": 2048, + "load_in_4bit": true, + "load_in_8bit": false, + "dtype": "auto", + "trust_remote_code": false, + "device_map": null, + "rope_scaling": null + }, + "dataset": { + "path": "default/", + "validation_path": null, + "text_field": "text", + "apply_chat_template": true, + "packing": false + }, + "training": { + "training_type": "sft", + "finetuning_type": "lora", + "lora": { + "rank": 16, + "alpha": 16, + "dropout": 0.0, + "target_modules": ["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj"], + "bias": "none", + "use_rslora": false, + "random_state": 3407, + "use_dora": false, + "loftq_config": null, + "modules_to_save": null, + "layers_to_transform": null, + "layer_replication": null, + "init_lora_weights": true + }, + "use_gradient_checkpointing": "unsloth" + }, + "schedule": { + "epochs": 1, + "max_steps": null, + "warmup_steps": 0, + "warmup_ratio": null, + "lr_scheduler_type": "linear", + "lr_scheduler_kwargs": null, + "logging_steps": 1, + "save_steps": null, + "eval_steps": null, + "seed": 3407 + }, + "batch": { + "per_device_train_batch_size": 2, + "gradient_accumulation_steps": 4 + }, + "optimizer": { + "learning_rate": 5e-5, + "weight_decay": 0.0, + "optim": "adamw_8bit", + "adam_beta1": 0.9, + "adam_beta2": 0.999, + "adam_epsilon": 1e-8, + "max_grad_norm": 1.0, + "label_smoothing_factor": 0.0, + "neftune_noise_alpha": null + }, + "hardware": { + "gpus": "0", + "precision": "bf16" + }, + "integrations": null, + "output": { + "name": "", + "description": null, + "save_method": "lora" + } +} +``` + +## Field reference (unsloth) + +### `model` + +`model` is an **object** (not a string). `name` is the platform model entity ref. + +| Field | Default | Notes | +|-------|---------|-------| +| `name` | — | Model entity ref: `"name"` (uses job workspace) or `"workspace/name"`. Plugin resolves to a local path before training. | +| `max_seq_length` | `2048` | Truncate / pack to this length; lower if VRAM tight. | +| `load_in_4bit` | `true` | bitsandbytes 4-bit. Mutex with `load_in_8bit`. Default for Unsloth's headline path; required to fit larger models on small GPUs. | +| `load_in_8bit` | `false` | bitsandbytes 8-bit. Mutex with `load_in_4bit`. | +| `dtype` | `"auto"` | One of `"auto"`, `"bfloat16"`, `"float16"`, `"float32"`. | +| `trust_remote_code` | `false` | HF `trust_remote_code` flag for custom model code (required by some hybrid Mamba/MoE models, e.g. Nemotron-H). | +| `device_map` | `null` | Placement for `FastLanguageModel.from_pretrained`. `null` pins the whole model to the single visible GPU (`{"": 0}`) — the right default for this single-GPU backend. Leave unset unless experimenting; `"auto"`/`"balanced"`/`"sequential"` can spill layers to CPU on unified-memory hosts (GB10 / DGX Spark) and abort 4-bit loads. | +| `rope_scaling` | `null` | RoPE scaling for long-context extension, e.g. `{"type": "linear", "factor": 2.0}`. `null` uses the model's native context length. | + +**Mutex:** `load_in_4bit` xor `load_in_8bit`. Both quantization flags are also **incompatible with `training.finetuning_type: "all_weights"`** — full SFT must use a non-quantized base. + +> **Hybrid Mamba/MoE models (e.g. NVIDIA Nemotron-H `*-A3B`):** load in **16-bit** (`load_in_4bit: false`, `load_in_8bit: false`) — Unsloth's supported path for these. The 4-bit (bitsandbytes) path can hit a dtype mismatch inside the model's MoE expert accumulation. Keep `device_map` unset (single-GPU default) and set `trust_remote_code: true`. + +### `dataset` + +See `references/dataset-formats.md` § Unsloth for row-shape rules. + +| Field | Default | Notes | +|-------|---------|-------| +| `path` | — | Training fileset ref (`"name"` or `"workspace/name"`). | +| `validation_path` | `null` | Optional validation fileset ref. | +| `text_field` | `"text"` | Column SFTTrainer reads. In `apply_chat_template: true` mode, the rendered template string is written into this column. | +| `apply_chat_template` | `false` | Set `true` for rows with a `messages` array (preferred when the tokenizer has a chat template). | +| `packing` | `false` | trl.SFTTrainer packing for throughput on short rows. | + +### Unsloth `training` + +| Field | Default | Notes | +|-------|---------|-------| +| `training_type` | `"sft"` | Only `"sft"` is implemented today. | +| `finetuning_type` | `"lora"` | `"lora"` (adapter; default) or `"all_weights"` (full SFT — heavy, no quantization). | +| `lora` | auto-filled when `finetuning_type` is `lora` | See LoRA subsection below. | +| `use_gradient_checkpointing` | `"unsloth"` | `"unsloth"` (recommended), `"true"`, or `"false"`. Unsloth's variant is faster than HF's. | + +**LoRA block (`training.lora`):** + +| Field | Default | Notes | +|-------|---------|-------| +| `rank` | `16` | Higher → more capacity, more VRAM. Cap at 32 if the adapter will deploy via default NIM / vLLM. | +| `alpha` | `16` | LoRA scaling; common rule of thumb `alpha ≈ rank` or `2× rank`. | +| `dropout` | `0.0` | LoRA dropout (0.0–<1.0). | +| `target_modules` | Unsloth 7-module set: `q_proj`, `k_proj`, `v_proj`, `o_proj`, `gate_proj`, `up_proj`, `down_proj` | Full attention + MLP. Override with a subset like `["q_proj","v_proj"]` for a lighter touch. | +| `bias` | `"none"` | `"none"` / `"all"` / `"lora_only"`. | +| `use_rslora` | `false` | Rank-stabilized LoRA. | +| `random_state` | `3407` | Reproducibility seed for the LoRA init. | +| `use_dora` | `false` | DoRA (weight-decomposed LoRA). Better quality at low ranks; adds overhead. | +| `loftq_config` | `null` | LoftQ init config for quantized bases. `null` disables. | +| `modules_to_save` | `null` | Extra non-LoRA modules trained & saved in full, e.g. `["embed_tokens","lm_head"]` (vocab changes / continued pretraining). | +| `layers_to_transform` | `null` | Restrict LoRA to specific layer index(es). `null` = all layers. | +| `layer_replication` | `null` | Layer-replication ranges for stacking, e.g. `[[0,16],[8,24]]`. | +| `init_lora_weights` | `true` | Init scheme. `true` = PEFT default; `"gaussian"`/`"pissa"`/`"olora"`/`"loftq"` for advanced inits. | + +`lora` is auto-filled with these defaults when `finetuning_type: "lora"` and the user omits the block. Must be `null` / omitted when `finetuning_type: "all_weights"`. + +### Unsloth `schedule` + +| Field | Default | Notes | +|-------|---------|-------| +| `epochs` | `null` | Full passes. **`epochs` xor `max_steps`** — exactly one is required. | +| `max_steps` | `null` | Global step cap. Use alone for smoke tests; do not combine with `epochs`. | +| `warmup_steps` | `0` | Linear warmup. Mutex with `warmup_ratio`. | +| `warmup_ratio` | `null` | Fractional warmup over total steps. Mutex with `warmup_steps`. | +| `lr_scheduler_type` | `"linear"` | `"linear"`, `"cosine"`, `"constant"`, `"constant_with_warmup"`, `"cosine_with_restarts"`. | +| `lr_scheduler_kwargs` | `null` | Extra scheduler kwargs, e.g. `{"num_cycles": 3}` for `cosine_with_restarts`. `null` uses defaults. | +| `logging_steps` | `1` | Loss-log cadence. | +| `save_steps` | `null` | If set, save checkpoint every N steps. | +| `eval_steps` | `null` | If set with `validation_path`, eval every N steps. When `null` and `validation_path` is set, the training driver defaults to **one validation pass per effective epoch** at `max(1, effective_steps - 1)` (same effective-step cap as automodel's default `val_check_interval`). | +| `seed` | `3407` | Trainer seed (`TrainingArguments.seed`). | + +**Hard mutex enforced by the schema:** `epochs` xor `max_steps`; `warmup_steps` xor `warmup_ratio`. Validation errors surface at submit time. + +### Unsloth `batch` + +| Field | Default | Notes | +|-------|---------|-------| +| `per_device_train_batch_size` | `1` | Forwarded verbatim to `TrainingArguments`. Drives peak VRAM. | +| `gradient_accumulation_steps` | `1` | Multiplies effective batch without raising VRAM. | + +`effective_batch = per_device_train_batch_size × gradient_accumulation_steps`. No GBS divisibility math (single GPU). Starting points by model size are in `batch-sizing.md` § Batch sizing — unsloth. + +### Unsloth `optimizer` + +| Field | Default | Notes | +|-------|---------|-------| +| `learning_rate` | `2e-4` (schema default; skill uses `5e-5` for LoRA SFT) | See LR table below. | +| `weight_decay` | `0.0` | L2-style regularization. | +| `optim` | `"adamw_8bit"` | `"adamw_torch"`, `"adamw_torch_fused"` (Hopper+), `"adamw_8bit"`, `"paged_adamw_8bit"`, `"sgd"`. `adamw_8bit` has the smallest optimizer state and is Unsloth's notebook default. | +| `adam_beta1` | `0.9` | Adam/AdamW beta1. | +| `adam_beta2` | `0.999` | Adam/AdamW beta2. | +| `adam_epsilon` | `1e-8` | Adam/AdamW epsilon. | +| `max_grad_norm` | `1.0` | Gradient-clipping max norm (TRL default). | +| `label_smoothing_factor` | `0.0` | Label smoothing for the CE loss. `0.0` disables. | +| `neftune_noise_alpha` | `null` | NEFTune embedding-noise alpha (quality boost). `null` disables. | + +`warmup_steps` is on `schedule`, not on `optimizer` (different from the automodel schema). + +### `hardware` + +| Field | Default | Notes | +|-------|---------|-------| +| `gpus` | `null` | Comma-separated CUDA indices inside the training container: `"0"` (typical). Sets `CUDA_VISIBLE_DEVICES` **before** `import torch`. **Selection, not reservation.** Unsloth uses one GPU per training process. | +| `precision` | `"bf16"` | `"bf16"` (Ampere+) or `"fp16"`. | + +### Unsloth `integrations` + +See **Integrations (all backends)** in `hyperparameters.md`. + +### `output` + +| Field | Default | Notes | +|-------|---------|-------| +| `name` | auto-derived from `--` | The output model entity / fileset name. | +| `description` | `null` | Free-form description carried onto the entity and fileset. | +| `save_method` | `"lora"` | `"lora"` (adapter — hot-reloads on base LoRA deployment; no new inference deploy), `"merged_16bit"` (merged checkpoint — **deploy** `output.name` as model entity), `"merged_4bit"` (lossy, storage-tight; deploy like merged). `merged_*` requires `training.finetuning_type: "lora"`. | + +After `to_spec`, the canonical `OutputResponse` also carries `type` (`"adapter"` for `save_method: "lora"`, `"model"` otherwise) and `fileset` (defaults to `name`); both are derived — submitter doesn't set them. + +## Tuning guide (unsloth) + +VRAM / batch tuning is in **`batch-sizing.md` § Batch sizing — unsloth**. Below covers non-batch fields. + +### Unsloth learning rate (LoRA SFT, starting points) + +Same scale as automodel (the underlying optimizer math is the same): + +| Model scale | Suggested `learning_rate` | +|-------------|---------------------------| +| ≤ 3B | `5e-5` – `1e-4` | +| 3B – 8B | `2e-5` – `5e-5` | +| > 8B | `1e-5` – `2e-5` | + +Schema default is `2e-4` (Unsloth notebook default — works for small adapters with `adamw_8bit`). Skill defaults are conservative `5e-5`. + +### Unsloth LoRA rank / alpha + +| Use case | `rank` | `alpha` | +|----------|--------|---------| +| Default / balanced | 16 | 16 | +| Lighter touch | 8 | 16 | +| More capacity (inference-safe max on default NIM/vLLM) | 32 | 32 or 64 | + +Drop `rank` before lowering batch when OOM. Higher `alpha/rank` ratios amplify adapter influence; Unsloth's defaults keep `alpha == rank`. + +### Save-method picker + +| User wants | `save_method` | Inference after training | +|------------|---------------|--------------------------| +| Smallest artefact; hot-reload on base LoRA deployment | `lora` | No new deploy — adapter loads on existing `lora_enabled` deployment | +| Full-weight checkpoint as standalone model | `merged_16bit` | **Deploy** `output.name` as new model entity | +| Disk-tight merged checkpoint (lossy) | `merged_4bit` | **Deploy** `output.name` as new model entity | +| Full SFT (no LoRA) | `lora` is invalid; output is always a full model | **Deploy** `output.name` as new model entity | + +`merged_*` require `training.finetuning_type: "lora"`. The schema validator surfaces a clear error if violated. + +### Smoke test (unsloth) + +```json +"schedule": { "max_steps": 50 } +``` + +(omit `epochs`). + +### Distillation + +Not supported by unsloth today (`training_type` is `Literal["sft"]`). Use automodel for distillation. + diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters.md index 773f38e443..3ffa7c3b42 100644 --- a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters.md +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/hyperparameters.md @@ -1,21 +1,33 @@ # Hyperparameters -Two backend job schemas live in this skill. Pick by plugin: +Three backend job schemas live in this skill. Each backend has its own field reference file — **pick by plugin**: -| Plugin | Schema class | Schema dump | Section below | -|--------|--------------|-------------|---------------| -| `automodel` | `AutomodelJobInput` (`plugins/nemo-automodel/src/nemo_automodel_plugin/schema.py`) | `nemo customization automodel explain` | **Automodel job JSON** (below) | -| `unsloth` | `UnslothJobInput` (`plugins/nemo-unsloth/src/nemo_unsloth_plugin/schema.py`) | `nemo customization unsloth explain` | **Unsloth job JSON** (further down) | +| Plugin | Schema class | Schema dump | Field reference | +|--------|--------------|-------------|-----------------| +| `automodel` | `AutomodelJobInput` (`plugins/nemo-automodel/src/nemo_automodel_plugin/schema.py`) | `nemo customization automodel explain` | **`hyperparameters-automodel.md`** | +| `unsloth` | `UnslothJobInput` (`plugins/nemo-unsloth/src/nemo_unsloth_plugin/schema.py`) | `nemo customization unsloth explain` | **`hyperparameters-unsloth.md`** | +| `rl` (DPO) | `RlJobInput` (`plugins/nemo-rl/src/nemo_rl_plugin/schema.py`) | `nemo customization rl explain` | **`hyperparameters-rl.md`** | -Both schemas use `extra="forbid"` — unknown keys raise validation errors. Field names are **not** interchangeable across backends (e.g. automodel uses `micro_batch_size` / `global_batch_size` / `parallelism`; unsloth uses `per_device_train_batch_size` / `gradient_accumulation_steps` / `hardware`). Use the right schema for the chosen plugin. +All three schemas use `extra="forbid"` — unknown keys raise validation errors. Field names are **not** interchangeable across backends (e.g. automodel uses `micro_batch_size` / `global_batch_size` / `parallelism`; unsloth uses `per_device_train_batch_size` / `gradient_accumulation_steps` / `hardware`; rl uses `batch_size` / `micro_batch_size` under `training` and takes `model` / `dataset` as plain strings). Use the right schema for the chosen plugin. -**Batch sizing, 48 GB VRAM tables, multi-GPU (data parallel vs tensor parallel), and throughput tuning** live in **`SKILL.md`** (§ Batch sizing — automodel, § Batch sizing — unsloth, § Multi-GPU). This file is the **field glossary**, full JSON template per backend, distillation/KD, and schema pointers — not the place to pick batch sizes for production runs. +**Batch sizing, 48 GB VRAM tables, multi-GPU (data parallel vs tensor parallel), and throughput tuning** live in **`batch-sizing.md`** (automodel + unsloth). These per-backend files are the **field glossary**, full JSON template per backend, distillation/KD (automodel), and DPO knobs (rl) — not the place to pick batch sizes for production runs. + +## Table of contents + +| Read this file | For | +|----------------|-----| +| **`hyperparameters-automodel.md`** | Automodel job JSON layout, full template, `training` / `schedule` / `batch` / `optimizer` / `parallelism` field reference, LR & LoRA-rank tuning, presets, distillation/KD | +| **`hyperparameters-unsloth.md`** | Unsloth job JSON layout, full template, `model` / `dataset` / `training` / `schedule` / `batch` / `optimizer` / `hardware` / `output` field reference, LR & LoRA-rank tuning, save-method picker | +| **`hyperparameters-rl.md`** | NeMo-RL (DPO) job JSON layout, `training` (DPOTraining) field reference — shared knobs + DPO-specific (`ref_policy_kl_penalty` = β, `sft_loss_weight`), `parallelism`, DPO tuning guide | +| **`batch-sizing.md`** | ≥48 GB VRAM batch tables, multi-GPU (data vs tensor parallel), OOM / throughput tuning (automodel + unsloth) | +| **Integrations** (below) | W&B / MLflow `integrations` object — all three backends (automodel, unsloth, rl) | +| **Source of truth** (below) | Schema source files, compiler mappings, fixtures per backend | --- -## Integrations (automodel + unsloth) +## Integrations (all backends) -Both backends accept the same `integrations` object on job JSON (`IntegrationsSpec` in `nemo_platform_plugin.integrations`). A non-null backend block **requests** that integration; the training runtime **activates** it only when credentials/URIs are available (W&B needs `WANDB_API_KEY`, MLflow needs a tracking URI). Omit the field or set a backend to `null` to disable. There is no `enabled` flag and no `report_to` on input — `report_to` is derived at runtime from activated backends. The compiler logs a warning when W&B is requested without `api_key_secret` or MLflow without `tracking_uri`. +**All three backends** (automodel, unsloth, rl) accept the same `integrations` object on job JSON (`IntegrationsSpec` in `nemo_platform_plugin.integrations`) — **W&B** and **MLflow**. A non-null `wandb` / `mlflow` block **requests** that integration; the training runtime **activates** it only when credentials/URIs are available (W&B needs `WANDB_API_KEY`, MLflow needs a tracking URI). Omit the field or set a block to `null` to disable. There is no `enabled` flag and no `report_to` on input — `report_to` is derived at runtime from activated integrations. The compiler logs a warning when W&B is requested without `api_key_secret` or MLflow without `tracking_uri`. ```json "integrations": { @@ -53,614 +65,13 @@ Both backends accept the same `integrations` object on job JSON (`IntegrationsSp | `mlflow.name` | MLflow run name; defaults to job ID. Legacy `run_name` is accepted with a deprecation warning. | | `mlflow.tags` / `mlflow.description` | Optional run metadata. | -Set `"integrations": null` or omit the field when tracking is not needed. Contract examples: `plugins/nemo-automodel/tests/fixtures/integrations_wandb_mlflow.json`, `plugins/nemo-unsloth/tests/fixtures/integrations_wandb_mlflow.json`. - -**Local setup (MLflow server, `docker0` tracking URI, jobs-launcher, W&B secret):** `references/integrations-setup.md`. - -**Unsloth note:** HuggingFace `TrainingArguments.run_name` is shared by W&B and MLflow. When both backends are active, `wandb.name` wins if set; otherwise `mlflow.name` is used. If both names are set to different values, a runtime warning is logged and W&B's name is used. - ---- - -# Automodel job JSON - -Job JSON for `nemo customization automodel submit` uses **`AutomodelJobInput`** (`plugins/nemo-automodel/src/nemo_automodel_plugin/schema.py`). Only fields in that schema are accepted (`extra="forbid"`). - -**Schema dump:** - -```bash -nemo customization automodel explain -``` - -**Contract examples:** `services/automodel/tests/contract/input_configs/` (legacy shape; map `batch_size` → `global_batch_size` in submit JSON). - -## Job JSON layout - -| Section | Purpose | -|---------|---------| -| `model` | **Base model entity** ref (`default/`) — weights to fine-tune | -| `dataset` | **Dataset filesets** (`default/`); optional `prompt_template` for CUSTOM schema | -| `training` | Method, LoRA, `max_seq_length`, distillation/KD fields | -| `schedule` | Epochs, optional step cap, validation cadence, seed | -| `batch` | Global/micro batch, sequence packing | -| `optimizer` | LR, weight decay, warmup | -| `parallelism` | Nodes, GPUs, TP/PP/CP/EP | -| `output` | Output adapter/model fileset name | -| `integrations` | Optional W&B / MLflow | - -### `model` field (base model entity) - -`model` must name a **Models API entity** for the checkpoint being trained — not a dataset fileset, not an output adapter from a prior job, and not a raw Hugging Face repo id. - -| Valid | Invalid | -|-------|---------| -| `default/qwen3-1.7b` (entity from `nemo models create`) | `Qwen/Qwen3-1.7B` (HF id) | -| `default/llama-3.2-1b-instruct` | `default/commonsense_qa` (dataset fileset) | -| `other-ws/my-model` (qualified ref) | `qwen3-1.7b-commonsense-qa-lora` (output fileset only, unless registered as entity) | - -Register before submit (same as skill fast path): HF **model** fileset → `nemo models create …` with `"fileset":"default/"`. List: `nemo models list --workspace default`. - -Full template: - -```json -{ - "model": "default/", - "dataset": { - "training": "default/", - "validation": "default/", - "prompt_template": null - }, - "training": { - "training_type": "sft", - "finetuning_type": "lora", - "lora": { - "rank": 16, - "alpha": 32, - "dropout": 0.0, - "merge": false, - "target_modules": null, - "exclude_modules": null, - "use_triton": true - }, - "max_seq_length": 2048, - "precision": null, - "attn_implementation": "sdpa", - "execution_profile": null - }, - "schedule": { - "epochs": 1, - "max_steps": null, - "val_check_interval": null, - "seed": null - }, - "batch": { - "global_batch_size": 4, - "micro_batch_size": 1, - "sequence_packing": false, - "sequence_packing_max_samples": 1000 - }, - "optimizer": { - "learning_rate": 5e-5, - "min_learning_rate": null, - "weight_decay": 0.01, - "adam_beta1": 0.9, - "adam_beta2": 0.999, - "adam_eps": 1e-8, - "optimizer": "Adam", - "lr_decay_style": "cosine", - "warmup_steps": 0 - }, - "parallelism": { - "num_nodes": 1, - "num_gpus_per_node": 1, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "context_parallel_size": 1, - "expert_parallel_size": null, - "sequence_parallel": false - }, - "output": { "name": "", "description": null }, - "integrations": null -} -``` - ---- - -## Field reference - -### Automodel `training` - -| Field | Default | Notes | -|-------|---------|-------| -| `training_type` | `sft` | `distillation` requires `teacher_model` (entity ref) | -| `finetuning_type` | `lora` | `all_weights` (full fine-tune), `lora_merged` (merge adapter into base) | -| `lora.rank` | `16` | Higher → more capacity, more VRAM. Typical training range 8–32; **cap at 32** if the adapter will be served with default NIM / vLLM (rank > 32 may not load) | -| `lora.alpha` | `32` | Scaling; common rule of thumb **alpha ≈ 2× rank** | -| `lora.dropout` | `0.0` | LoRA dropout (0.0–1.0) for regularization | -| `lora.merge` | `false` | If true with `lora_merged`, output is full weights not adapter | -| `lora.target_modules` | `null` | e.g. `["q_proj","v_proj"]`; null = platform default targets | -| `lora.exclude_modules` | `null` | Patterns to exclude from LoRA, e.g. `["*.out_proj"]` | -| `lora.use_triton` | `true` | Use the optimized Triton LoRA kernel | -| `max_seq_length` | `2048` | Truncate/pack to this length; lower if OOM | -| `precision` | `null` | `bf16` \| `fp16` \| `fp32` \| `fp8`; null auto-detects from the checkpoint | -| `attn_implementation` | `sdpa` | `sdpa` (PyTorch native) \| `flash_attention_2` \| `eager` | -| `teacher_model` | — | **Model entity ref** (not HF id). Required for distillation; see below | -| `distillation_ratio` | `0.5` | KD blend (0–1) | -| `distillation_temperature` | `1.0` | KD temperature | -| `teacher_precision` | `bf16` | `bf16` \| `fp16` \| `fp32` | -| `offload_teacher` | `false` | Offload teacher weights to CPU | - -LoRA block is auto-created when `finetuning_type` is `lora` or `lora_merged`. - -### Automodel `schedule` - -| Field | Default | Notes | -|-------|---------|-------| -| `epochs` | `1` | Must be **≥ 1**. Full passes over training set | -| `max_steps` | `null` | **Global step cap.** Omit for epoch-based runs | -| `val_check_interval` | `null` | `≤ 1.0` = fraction of epoch; `> 1` = every N steps | -| `seed` | `null` | Reproducibility | - -**Gotcha:** Do **not** set `max_steps` with `epochs` for normal training. `max_steps` stops early (e.g. `epochs: 1` + `max_steps: 100` ends at step 100). Use `max_steps` **alone** only for smoke tests. - -### Automodel `batch` - -| Field | Default | Notes | -|-------|---------|-------| -| `global_batch_size` | `8` (schema) | Effective batch across all GPUs; **≥48 GB LoRA tables → `SKILL.md`** | -| `micro_batch_size` | `1` (schema) | **Per GPU**; same SKILL tables for single- and multi-GPU (TP=1) | -| `sequence_packing` | `false` | Pack short sequences for throughput (needs compatible data) | -| `sequence_packing_max_samples` | `1000` | Samples analyzed to estimate the optimal pack size (only when packing) | - -**Validation:** `global_batch_size` must be divisible by `micro_batch_size × data_parallel_size`, where: - -`data_parallel_size = (num_nodes × num_gpus_per_node) / (tensor_parallel_size × pipeline_parallel_size × context_parallel_size)` - -Example: 1 node, 2 GPUs, TP=1 → DP=2 → GBS must be a multiple of `2 × micro_batch_size`. See **`SKILL.md` § Multi-GPU** for data parallel vs tensor parallel. - -### Automodel `optimizer` - -| Field | Default | Notes | -|-------|---------|-------| -| `learning_rate` | `5e-6` (schema) | Skill uses **5e-5** for small LoRA SFT; see tuning below | -| `min_learning_rate` | `null` | Floor for the cosine LR decay; null lets it decay toward 0 | -| `weight_decay` | `0.01` | L2-style regularization | -| `adam_beta1` | `0.9` | Adam optimizer beta1 | -| `adam_beta2` | `0.999` | Adam optimizer beta2 | -| `adam_eps` | `1e-8` | Adam/AdamW epsilon for numerical stability | -| `optimizer` | `Adam` | `Adam` \| `AdamW` | -| `lr_decay_style` | `cosine` | `cosine` \| `linear` \| `constant` | -| `warmup_steps` | `0` | Linear warmup; try ~10% of total steps for long runs | - -### `parallelism` - -| Field | Default | Notes | -|-------|---------|-------| -| `num_nodes` | `1` | Multi-node distributed jobs | -| `num_gpus_per_node` | `1` | GPUs per node | -| `tensor_parallel_size` | `1` | **> 1** when the model does not fit on one ≥48 GB GPU — see **`SKILL.md` § Multi-GPU** | -| `pipeline_parallel_size` | `1` | Pipeline stages | -| `context_parallel_size` | `1` | Long-context sharding | -| `expert_parallel_size` | `null` | MoE only; must divide `data_parallel_size × context_parallel_size` | -| `sequence_parallel` | `false` | Shard activations along the sequence dim (pairs with tensor parallelism) | - -**MoE:** If `expert_parallel_size > 1` and multiple GPUs, `tensor_parallel_size` must be **1**. - -### Automodel `integrations` (optional) - -See **Integrations (automodel + unsloth)** above. - ---- - -## Tuning guide (when the user asks) - -Apply user overrides to `/tmp/job.json` before submit. For **batch / GPU count / parallelism**, follow **`SKILL.md`** (defaults table + § Batch sizing + § Multi-GPU). Below covers **non-batch** fields and defers VRAM/batch symptoms to the skill. - -| Symptom / goal | Try first | -|----------------|-----------| -| CUDA OOM | **`SKILL.md` tuning loop:** halve `micro_batch_size`, then `global_batch_size`, then `max_seq_length`; use TP > 1 only if the model does not fit one ≥48 GB GPU | -| Slow / low GPU use | **`SKILL.md`:** step toward high-util column or double `micro`+GBS until ~35–40 GiB; multi-GPU data parallel if model fits one GPU | -| Underfitting | More `epochs`, slightly higher `learning_rate`, higher LoRA `rank` (≤ 32 for NIM/vLLM deploy) | -| Overfitting | Fewer `epochs`, lower `learning_rate`, higher `weight_decay`, smaller `rank` | -| Quick smoke test | `max_steps` only (e.g. 10–50), **omit or ignore epoch goal**; or `epochs: 1` on tiny slice | -| Reproducibility | Set `schedule.seed` | - -### Automodel learning rate (LoRA SFT, starting points) - -| Model scale | Suggested `learning_rate` | -|-------------|---------------------------| -| ≤ 3B | `5e-5` – `1e-4` | -| 3B – 8B | `2e-5` – `5e-5` | -| > 8B | `1e-5` – `2e-5` | - -Schema default is `5e-6` (conservative). Fixtures: `qwen3_0.6b_sft_lora.json` uses `5e-5`; `minimal_sft_lora.json` uses `5e-6`. - -### Automodel LoRA rank / alpha - -**Deployment cap:** Default **NIM** and **vLLM** LoRA serving paths support rank **≤ 32**. Use `rank` 32 (not higher) when the fine-tuned adapter will be deployed for inference on those stacks unless the user confirms a higher rank is supported. - -| Use case | `rank` | `alpha` | -|----------|--------|---------| -| Default / balanced | 16 | 32 | -| Low VRAM / light touch | 8 | 16 | -| More capacity (inference-safe max) | 32 | 64 | - -### Epochs vs dataset size - -One epoch = one full pass over `train.jsonl`. Steps per epoch ≈ `train_samples / global_batch_size` (e.g. ~10k samples, GBS 64 → ~153 steps). Plan poll time from the **GBS you chose in `SKILL.md`**, not the unknown-VRAM default (GBS 4). - ---- - -## Presets (non-batch fields) - -Use **`SKILL.md` § Batch sizing** and **§ Multi-GPU** for `batch` and `parallelism` on ≥48 GB GPUs. Presets below only override schedule / training / optimizer. - -**Smoke test (step-capped)** - -```json -"schedule": { "epochs": 1, "max_steps": 50 } -``` - -**Higher-quality LoRA (more VRAM/time)** - -```json -"training": { "lora": { "rank": 32, "alpha": 64 }, "max_seq_length": 2048 }, -"schedule": { "epochs": 3 }, -"optimizer": { "learning_rate": 2e-5, "warmup_steps": 100 } -``` - -Pair with batch rows from **`SKILL.md`** (e.g. ≤4B default `micro` 32 / GBS 128, not `micro` 1 / GBS 4). - ---- - -## Distillation (`training_type: "distillation"`) - -Use only when the user requests KD/distillation. **`model`** is the **student** entity; **`teacher_model`** is a separate **teacher** entity in the same workspace (unless qualified as `other-ws/name`). - -### Teacher model entity - -`teacher_model` must be a registered **model entity ref**, same shape as `model`: - -| Form | Example | -|------|---------| -| Same workspace | `default/llama-3.2-3b-instruct` | -| Explicit workspace | `default/` | - -It is **not** a Hugging Face repo id. Register the teacher like the student before submit: +Set `"integrations": null` or omit the field when tracking is not needed. Fixtures per backend: automodel → `plugins/nemo-automodel/tests/fixtures/integrations_wandb_mlflow.json`; unsloth → `plugins/nemo-unsloth/tests/fixtures/integrations_wandb_mlflow.json`; rl → `plugins/nemo-rl/tests/fixtures/integrations_wandb_mlflow.json`. -```bash -TEACHER_WEIGHTS=llama-3.2-3b-instruct # fileset name -TEACHER_ENTITY=llama-3.2-3b-instruct # entity name -TEACHER_HF=meta-llama/Llama-3.2-3B-Instruct +**Local setup (MLflow server, `docker0` tracking URI, jobs-launcher, W&B secret) — Docker-runtime (automodel / unsloth):** `references/integrations-setup.md`. -nemo files filesets create "$TEACHER_WEIGHTS" --workspace default --purpose model --exist-ok \ - --storage '{"type":"huggingface","repo_id":"'"$TEACHER_HF"'","repo_type":"model","revision":"main"}' +**rl (DPO) note:** rl supports **W&B and MLflow** through this object exactly like automodel and unsloth. Two rl specifics: the run name defaults to the **job id** (stable across pause/resume) and NeMo-RL auto-adds tags (`service:rl`, `framework:…`, plus workspace / job / task / model); and because rl runs on **Kubernetes / Ray** (not the Docker executor), point `tracking_uri` and any self-hosted W&B `base_url` at an endpoint **reachable from the cluster** — the `docker0` local-MLflow recipe above is Docker-runtime only. (NeMo-RL's TensorBoard / SwanLab logger slots aren't exposed via `integrations`, same as the other backends — `IntegrationsSpec` carries only `wandb` + `mlflow`.) -nemo models create "$TEACHER_ENTITY" --workspace default --exist-ok \ - --input-data '{"name":"'"$TEACHER_ENTITY"'","fileset":"default/'"$TEACHER_WEIGHTS"'","custom_fields":{"hf_model_id":"'"$TEACHER_HF"'"}}' -``` - -Verify: `nemo models get --workspace default`. Reuse an existing entity with `nemo models list` when present. - -**Compatibility:** Student and teacher must share the **same vocabulary / tokenizer family** (compiler loads both for KD). Mismatched tokenizers fail at runtime. Prefer a larger instruct model as teacher and a smaller base/chat model as student in the same family when possible. - -**VRAM:** Set `offload_teacher: true` if the job OOMs loading student + teacher; `teacher_precision: "bf16"` is the default. - -### Job JSON - -```json -{ - "model": "default/", - "dataset": { "training": "default/" }, - "training": { - "training_type": "distillation", - "finetuning_type": "lora", - "teacher_model": "default/", - "distillation_ratio": 0.5, - "distillation_temperature": 1.0, - "teacher_precision": "bf16", - "offload_teacher": false, - "max_seq_length": 2048 - }, - "schedule": { "epochs": 1 }, - "batch": { "global_batch_size": 64, "micro_batch_size": 16 }, - "optimizer": { "learning_rate": 8e-5 }, - "parallelism": { "num_nodes": 1, "num_gpus_per_node": 1, "tensor_parallel_size": 1 }, - "output": { "name": "" } -} -``` - -(`batch` / `parallelism` example uses an 8B-scale row from **`SKILL.md`**; adjust for student size.) - -| Field | Meaning | -|-------|---------| -| `distillation_ratio` | Blend of KD vs CE loss (`0` = CE only, `1` = KD only) | -| `distillation_temperature` | Softmax temperature for teacher logits | -| `offload_teacher` | CPU-offload frozen teacher weights to save GPU memory | - ---- - -# Unsloth job JSON - -Job JSON for `nemo customization unsloth submit` uses **`UnslothJobInput`** (`plugins/nemo-unsloth/src/nemo_unsloth_plugin/schema.py`). Only fields in that schema are accepted (`extra="forbid"`). The canonical post-transform shape lives in `services/unsloth/src/nmp/unsloth/schemas.py` (`UnslothJobOutput`) and is what the training driver consumes in the GPU container. - -**Schema dump:** - -```bash -nemo customization unsloth explain -``` - -Unsloth is **submit-only, single-GPU inside the training container**. There is no `parallelism` block and no `training.execution_profile` in job JSON — pass `--profile` on `nemo customization unsloth submit` instead (default `gpu`). `hardware.gpus` sets `CUDA_VISIBLE_DEVICES` in the container before `import torch`. Multi-GPU sharding → use automodel. - -## Job JSON layout (unsloth) - -| Section | Purpose | -|---------|---------| -| `name` | Optional job name (auto-generated if omitted) | -| `model` | **Object** — base model entity ref + how to load it (4-bit, dtype, max_seq_length) | -| `dataset` | Single fileset ref (`path`) + optional `validation_path`; row shape selector (`text_field`, `apply_chat_template`, `packing`) | -| `training` | Method (`sft`), adapter shape (`lora`/`full`), LoRA hyperparams, gradient checkpointing | -| `schedule` | `epochs` xor `max_steps`; `warmup_steps` xor `warmup_ratio`; logging / save / eval cadence; LR scheduler | -| `batch` | `per_device_train_batch_size` × `gradient_accumulation_steps` = effective batch | -| `optimizer` | LR, weight decay, optimizer choice (`adamw_8bit` default) | -| `hardware` | GPU selection (`CUDA_VISIBLE_DEVICES`) + mixed precision (`bf16` / `fp16`) | -| `integrations` | Optional W&B / MLflow (same shape as automodel) | -| `output` | Output entity name, optional description, **`save_method`** (controls what's persisted) | - -Full template (every section, defaults inline): - -```json -{ - "name": "", - "model": { - "name": "default/", - "max_seq_length": 2048, - "load_in_4bit": true, - "load_in_8bit": false, - "dtype": "auto", - "trust_remote_code": false, - "device_map": null, - "rope_scaling": null - }, - "dataset": { - "path": "default/", - "validation_path": null, - "text_field": "text", - "apply_chat_template": true, - "packing": false - }, - "training": { - "training_type": "sft", - "finetuning_type": "lora", - "lora": { - "rank": 16, - "alpha": 16, - "dropout": 0.0, - "target_modules": ["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj"], - "bias": "none", - "use_rslora": false, - "random_state": 3407, - "use_dora": false, - "loftq_config": null, - "modules_to_save": null, - "layers_to_transform": null, - "layer_replication": null, - "init_lora_weights": true - }, - "use_gradient_checkpointing": "unsloth" - }, - "schedule": { - "epochs": 1, - "max_steps": null, - "warmup_steps": 0, - "warmup_ratio": null, - "lr_scheduler_type": "linear", - "lr_scheduler_kwargs": null, - "logging_steps": 1, - "save_steps": null, - "eval_steps": null, - "seed": 3407 - }, - "batch": { - "per_device_train_batch_size": 2, - "gradient_accumulation_steps": 4 - }, - "optimizer": { - "learning_rate": 5e-5, - "weight_decay": 0.0, - "optim": "adamw_8bit", - "adam_beta1": 0.9, - "adam_beta2": 0.999, - "adam_epsilon": 1e-8, - "max_grad_norm": 1.0, - "label_smoothing_factor": 0.0, - "neftune_noise_alpha": null - }, - "hardware": { - "gpus": "0", - "precision": "bf16" - }, - "integrations": null, - "output": { - "name": "", - "description": null, - "save_method": "lora" - } -} -``` - -## Field reference (unsloth) - -### `model` - -`model` is an **object** (not a string). `name` is the platform model entity ref. - -| Field | Default | Notes | -|-------|---------|-------| -| `name` | — | Model entity ref: `"name"` (uses job workspace) or `"workspace/name"`. Plugin resolves to a local path before training. | -| `max_seq_length` | `2048` | Truncate / pack to this length; lower if VRAM tight. | -| `load_in_4bit` | `true` | bitsandbytes 4-bit. Mutex with `load_in_8bit`. Default for Unsloth's headline path; required to fit larger models on small GPUs. | -| `load_in_8bit` | `false` | bitsandbytes 8-bit. Mutex with `load_in_4bit`. | -| `dtype` | `"auto"` | One of `"auto"`, `"bfloat16"`, `"float16"`, `"float32"`. | -| `trust_remote_code` | `false` | HF `trust_remote_code` flag for custom model code (required by some hybrid Mamba/MoE models, e.g. Nemotron-H). | -| `device_map` | `null` | Placement for `FastLanguageModel.from_pretrained`. `null` pins the whole model to the single visible GPU (`{"": 0}`) — the right default for this single-GPU backend. Leave unset unless experimenting; `"auto"`/`"balanced"`/`"sequential"` can spill layers to CPU on unified-memory hosts (GB10 / DGX Spark) and abort 4-bit loads. | -| `rope_scaling` | `null` | RoPE scaling for long-context extension, e.g. `{"type": "linear", "factor": 2.0}`. `null` uses the model's native context length. | - -**Mutex:** `load_in_4bit` xor `load_in_8bit`. Both quantization flags are also **incompatible with `training.finetuning_type: "all_weights"`** — full SFT must use a non-quantized base. - -> **Hybrid Mamba/MoE models (e.g. NVIDIA Nemotron-H `*-A3B`):** load in **16-bit** (`load_in_4bit: false`, `load_in_8bit: false`) — Unsloth's supported path for these. The 4-bit (bitsandbytes) path can hit a dtype mismatch inside the model's MoE expert accumulation. Keep `device_map` unset (single-GPU default) and set `trust_remote_code: true`. - -### `dataset` - -See `references/dataset-formats.md` § Unsloth for row-shape rules. - -| Field | Default | Notes | -|-------|---------|-------| -| `path` | — | Training fileset ref (`"name"` or `"workspace/name"`). | -| `validation_path` | `null` | Optional validation fileset ref. | -| `text_field` | `"text"` | Column SFTTrainer reads. In `apply_chat_template: true` mode, the rendered template string is written into this column. | -| `apply_chat_template` | `false` | Set `true` for rows with a `messages` array (preferred when the tokenizer has a chat template). | -| `packing` | `false` | trl.SFTTrainer packing for throughput on short rows. | - -### Unsloth `training` - -| Field | Default | Notes | -|-------|---------|-------| -| `training_type` | `"sft"` | Only `"sft"` is implemented today. | -| `finetuning_type` | `"lora"` | `"lora"` (adapter; default) or `"all_weights"` (full SFT — heavy, no quantization). | -| `lora` | auto-filled when `finetuning_type` is `lora` | See LoRA subsection below. | -| `use_gradient_checkpointing` | `"unsloth"` | `"unsloth"` (recommended), `"true"`, or `"false"`. Unsloth's variant is faster than HF's. | - -**LoRA block (`training.lora`):** - -| Field | Default | Notes | -|-------|---------|-------| -| `rank` | `16` | Higher → more capacity, more VRAM. Cap at 32 if the adapter will deploy via default NIM / vLLM. | -| `alpha` | `16` | LoRA scaling; common rule of thumb `alpha ≈ rank` or `2× rank`. | -| `dropout` | `0.0` | LoRA dropout (0.0–<1.0). | -| `target_modules` | Unsloth 7-module set: `q_proj`, `k_proj`, `v_proj`, `o_proj`, `gate_proj`, `up_proj`, `down_proj` | Full attention + MLP. Override with a subset like `["q_proj","v_proj"]` for a lighter touch. | -| `bias` | `"none"` | `"none"` / `"all"` / `"lora_only"`. | -| `use_rslora` | `false` | Rank-stabilized LoRA. | -| `random_state` | `3407` | Reproducibility seed for the LoRA init. | -| `use_dora` | `false` | DoRA (weight-decomposed LoRA). Better quality at low ranks; adds overhead. | -| `loftq_config` | `null` | LoftQ init config for quantized bases. `null` disables. | -| `modules_to_save` | `null` | Extra non-LoRA modules trained & saved in full, e.g. `["embed_tokens","lm_head"]` (vocab changes / continued pretraining). | -| `layers_to_transform` | `null` | Restrict LoRA to specific layer index(es). `null` = all layers. | -| `layer_replication` | `null` | Layer-replication ranges for stacking, e.g. `[[0,16],[8,24]]`. | -| `init_lora_weights` | `true` | Init scheme. `true` = PEFT default; `"gaussian"`/`"pissa"`/`"olora"`/`"loftq"` for advanced inits. | - -`lora` is auto-filled with these defaults when `finetuning_type: "lora"` and the user omits the block. Must be `null` / omitted when `finetuning_type: "all_weights"`. - -### Unsloth `schedule` - -| Field | Default | Notes | -|-------|---------|-------| -| `epochs` | `null` | Full passes. **`epochs` xor `max_steps`** — exactly one is required. | -| `max_steps` | `null` | Global step cap. Use alone for smoke tests; do not combine with `epochs`. | -| `warmup_steps` | `0` | Linear warmup. Mutex with `warmup_ratio`. | -| `warmup_ratio` | `null` | Fractional warmup over total steps. Mutex with `warmup_steps`. | -| `lr_scheduler_type` | `"linear"` | `"linear"`, `"cosine"`, `"constant"`, `"constant_with_warmup"`, `"cosine_with_restarts"`. | -| `lr_scheduler_kwargs` | `null` | Extra scheduler kwargs, e.g. `{"num_cycles": 3}` for `cosine_with_restarts`. `null` uses defaults. | -| `logging_steps` | `1` | Loss-log cadence. | -| `save_steps` | `null` | If set, save checkpoint every N steps. | -| `eval_steps` | `null` | If set with `validation_path`, eval every N steps. When `null` and `validation_path` is set, the training driver defaults to **one validation pass per effective epoch** at `max(1, effective_steps - 1)` (same effective-step cap as automodel's default `val_check_interval`). | -| `seed` | `3407` | Trainer seed (`TrainingArguments.seed`). | - -**Hard mutex enforced by the schema:** `epochs` xor `max_steps`; `warmup_steps` xor `warmup_ratio`. Validation errors surface at submit time. - -### Unsloth `batch` - -| Field | Default | Notes | -|-------|---------|-------| -| `per_device_train_batch_size` | `1` | Forwarded verbatim to `TrainingArguments`. Drives peak VRAM. | -| `gradient_accumulation_steps` | `1` | Multiplies effective batch without raising VRAM. | - -`effective_batch = per_device_train_batch_size × gradient_accumulation_steps`. No GBS divisibility math (single GPU). Starting points by model size are in `SKILL.md` § Batch sizing — unsloth. - -### Unsloth `optimizer` - -| Field | Default | Notes | -|-------|---------|-------| -| `learning_rate` | `2e-4` (schema default; skill uses `5e-5` for LoRA SFT) | See LR table below. | -| `weight_decay` | `0.0` | L2-style regularization. | -| `optim` | `"adamw_8bit"` | `"adamw_torch"`, `"adamw_torch_fused"` (Hopper+), `"adamw_8bit"`, `"paged_adamw_8bit"`, `"sgd"`. `adamw_8bit` has the smallest optimizer state and is Unsloth's notebook default. | -| `adam_beta1` | `0.9` | Adam/AdamW beta1. | -| `adam_beta2` | `0.999` | Adam/AdamW beta2. | -| `adam_epsilon` | `1e-8` | Adam/AdamW epsilon. | -| `max_grad_norm` | `1.0` | Gradient-clipping max norm (TRL default). | -| `label_smoothing_factor` | `0.0` | Label smoothing for the CE loss. `0.0` disables. | -| `neftune_noise_alpha` | `null` | NEFTune embedding-noise alpha (quality boost). `null` disables. | - -`warmup_steps` is on `schedule`, not on `optimizer` (different from the automodel schema). - -### `hardware` - -| Field | Default | Notes | -|-------|---------|-------| -| `gpus` | `null` | Comma-separated CUDA indices inside the training container: `"0"` (typical). Sets `CUDA_VISIBLE_DEVICES` **before** `import torch`. **Selection, not reservation.** Unsloth uses one GPU per training process. | -| `precision` | `"bf16"` | `"bf16"` (Ampere+) or `"fp16"`. | - -### Unsloth `integrations` - -See **Integrations (automodel + unsloth)** above. - -### `output` - -| Field | Default | Notes | -|-------|---------|-------| -| `name` | auto-derived from `--` | The output model entity / fileset name. | -| `description` | `null` | Free-form description carried onto the entity and fileset. | -| `save_method` | `"lora"` | `"lora"` (adapter — hot-reloads on base LoRA deployment; no new inference deploy), `"merged_16bit"` (merged checkpoint — **deploy** `output.name` as model entity), `"merged_4bit"` (lossy, storage-tight; deploy like merged). `merged_*` requires `training.finetuning_type: "lora"`. | - -After `to_spec`, the canonical `OutputResponse` also carries `type` (`"adapter"` for `save_method: "lora"`, `"model"` otherwise) and `fileset` (defaults to `name`); both are derived — submitter doesn't set them. - -## Tuning guide (unsloth) - -VRAM / batch tuning is in **`SKILL.md` § Batch sizing — unsloth**. Below covers non-batch fields. - -### Unsloth learning rate (LoRA SFT, starting points) - -Same scale as automodel (the underlying optimizer math is the same): - -| Model scale | Suggested `learning_rate` | -|-------------|---------------------------| -| ≤ 3B | `5e-5` – `1e-4` | -| 3B – 8B | `2e-5` – `5e-5` | -| > 8B | `1e-5` – `2e-5` | - -Schema default is `2e-4` (Unsloth notebook default — works for small adapters with `adamw_8bit`). Skill defaults are conservative `5e-5`. - -### Unsloth LoRA rank / alpha - -| Use case | `rank` | `alpha` | -|----------|--------|---------| -| Default / balanced | 16 | 16 | -| Lighter touch | 8 | 16 | -| More capacity (inference-safe max on default NIM/vLLM) | 32 | 32 or 64 | - -Drop `rank` before lowering batch when OOM. Higher `alpha/rank` ratios amplify adapter influence; Unsloth's defaults keep `alpha == rank`. - -### Save-method picker - -| User wants | `save_method` | Inference after training | -|------------|---------------|--------------------------| -| Smallest artefact; hot-reload on base LoRA deployment | `lora` | No new deploy — adapter loads on existing `lora_enabled` deployment | -| Full-weight checkpoint as standalone model | `merged_16bit` | **Deploy** `output.name` as new model entity | -| Disk-tight merged checkpoint (lossy) | `merged_4bit` | **Deploy** `output.name` as new model entity | -| Full SFT (no LoRA) | `lora` is invalid; output is always a full model | **Deploy** `output.name` as new model entity | - -`merged_*` require `training.finetuning_type: "lora"`. The schema validator surfaces a clear error if violated. - -### Smoke test (unsloth) - -```json -"schedule": { "max_steps": 50 } -``` - -(omit `epochs`). - -### Distillation - -Not supported by unsloth today (`training_type` is `Literal["sft"]`). Use automodel for distillation. +**Unsloth note:** HuggingFace `TrainingArguments.run_name` is shared by W&B and MLflow. When both backends are active, `wandb.name` wins if set; otherwise `mlflow.name` is used. If both names are set to different values, a runtime warning is logged and W&B's name is used. --- @@ -668,8 +79,8 @@ Not supported by unsloth today (`training_type` is `Literal["sft"]`). Use automo | Resource | Path | Use for | |----------|------|---------| -| **Batch / multi-GPU / 48 GB LoRA (automodel)** | `SKILL.md` § Batch sizing — automodel, § Multi-GPU | Choosing `micro`, GBS, LR, TP vs data parallel | -| **Batch (unsloth, single GPU)** | `SKILL.md` § Batch sizing — unsloth | `per_device_train_batch_size` × `gradient_accumulation_steps` starting points | +| **Batch / multi-GPU / 48 GB LoRA (automodel)** | `batch-sizing.md` § Batch sizing — automodel, § Multi-GPU | Choosing `micro`, GBS, LR, TP vs data parallel | +| **Batch (unsloth, single GPU)** | `batch-sizing.md` § Batch sizing — unsloth | `per_device_train_batch_size` × `gradient_accumulation_steps` starting points | | Submit schema (automodel) | `plugins/nemo-automodel/src/nemo_automodel_plugin/schema.py` | Allowed JSON fields | | Schema → compiler mapping (automodel) | `services/automodel/src/nmp/automodel/adapter.py` | `dataset.training` → compiler `dataset` string | | API field descriptions (automodel) | `services/automodel/src/nmp/automodel/api/v2/jobs/schemas.py` | Compiler-internal shape (not submit JSON) | @@ -680,3 +91,9 @@ Not supported by unsloth today (`training_type` is `Literal["sft"]`). Use automo | JSON example (unsloth) | `plugins/nemo-unsloth/tests/fixtures/minimal_unsloth_sft.json` | Smoke-test template (ignore `max_steps` for real runs) | | Full spec doc (automodel) | `plugins/nemo-automodel/SCOPE.md` (simplified JSON section) | Design notes | | Plugin README (unsloth) | `plugins/nemo-unsloth/README.md` | Submit-only CLI, 4-step container job, GPU selection | +| Submit schema (rl / DPO) | `plugins/nemo-rl/src/nemo_rl_plugin/schema.py` | Allowed JSON fields (`RlJobInput` / `DPOTraining`) | +| Canonical schema (rl / DPO) | `services/rl/src/nmp/rl/schemas.py` | Post-transform shape (`RlJobOutput`); divisibility validator | +| DPO config builder (rl) | `services/rl/src/nmp/rl/tasks/training/backends/nemo_rl/dpo_config.py` | Field → NeMo-RL YAML mapping | +| JSON fixture (rl / DPO) | `plugins/nemo-rl/tests/fixtures/minimal_dpo.json` | Minimal template (ignore `max_steps` for real runs) | +| Plugin README (rl / DPO) | `plugins/nemo-rl/README.md` | Submit-only CLI, Kubernetes/Ray runtime, constraints | +| Plugin design doc (rl / DPO) | `docs/customizer/nemo-rl-dpo-plugin-design.md` | Architecture, 4-step job, image split | diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/integrations-setup.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/integrations-setup.md index bbad950c37..6e2cda6411 100644 --- a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/integrations-setup.md +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/integrations-setup.md @@ -1,6 +1,6 @@ # Integrations setup (W&B + MLflow, local / Docker platform) -Use this when job JSON includes `integrations.wandb` and/or `integrations.mlflow` on a **local or single-node Docker** NeMo Platform (`platform.runtime: docker`). Field reference: `hyperparameters.md` § **Integrations (automodel + unsloth)**. +Use this when job JSON includes `integrations.wandb` and/or `integrations.mlflow` on a **local or single-node Docker** NeMo Platform (`platform.runtime: docker`) — i.e. **automodel / unsloth**. Field reference: `hyperparameters.md` § **Integrations (all backends)**. rl (DPO) accepts the same `integrations` block but runs on **Kubernetes / Ray**: reuse the field reference, but point `tracking_uri` / self-hosted W&B `base_url` at an endpoint reachable from the cluster (the `docker0` recipe below is Docker-runtime only). ## MLflow — local tracking server diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/reporting.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/reporting.md new file mode 100644 index 0000000000..f9cf8a7816 --- /dev/null +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/reporting.md @@ -0,0 +1,257 @@ +# Report to user + +After polling reaches a **terminal** status (`completed`, `error`, or `cancelled`), report using this template for **all** backends (automodel, unsloth, rl). Fill fields from the job JSON and `nemo jobs get-status`. + +## Result template + +```markdown +## Fine-tune result + +- **Job:** +- **Backend:** +- **Model entity:** default/ +- **Dataset fileset:** default/ +- **Output fileset:** +- **Status:** +- **Final train loss:** +- **Final validation loss:** +- **Notes:** +``` + +For **rl (DPO)** the output is a **full-weight model entity** (no adapter): label the line **Output model entity**, **skip Using the adapter**, and use **Using the fine-tuned model** (below) — confirm with `nemo models get --workspace default`. The DPO loss series lands in `status_details.metrics` like the other backends. + +## Field guidance + +| Field | Source | +|-------|--------| +| **Job** | Job id from submit or poll (`automodel-…` / `unsloth-…` / `rl-…`) | +| **Backend** | Plugin used for submit | +| **Model entity** | `model` in job JSON (automodel & rl: string ref; unsloth: `model.name`) | +| **Dataset fileset** | automodel: `dataset.training`; unsloth: `dataset.path`; rl: `dataset` (single preference-fileset string) | +| **Output adapter fileset** | `output.name` from job JSON. Label **Output adapter fileset (planned):** when status is `error` or `cancelled` and no output was registered | +| **Status** | Top-level `status` from `nemo jobs get-status` — not step-level status | +| **Final train loss** | Last entry in `status_details.metrics.train_loss` (or nested under a step's `status_details.metrics`). Use the **last** `value` in the list — not `status_details.train_loss` alone (that is the most recent logged step, which may differ from epoch-average loss on some backends). Round to 3 decimal places. | +| **Final validation loss** | Last entry in `status_details.metrics.val_loss`. If the list is empty, report `n/a (no validation run)` and note whether validation data was configured. Automodel validates once per epoch by default. Unsloth validates once per epoch when `dataset.validation_path` is set and `schedule.eval_steps` is omitted (platform default: `max(1, effective_steps - 1)`). | +| **Notes** | See **Notes by status** below | + +**Metrics extraction** — after polling, always run `nemo jobs get-status ` and read `status_details.metrics` (all backends accumulate `train_loss` and `val_loss` time series there). Include both final losses in the report even when status is `error` if training completed before the failure (e.g. entity registration failed after upload). + +## Notes by status + +| Status | Notes | +|--------|-------| +| `completed` | Brief success summary. LoRA (`save_method: lora`): adapter registered on base model entity. Full SFT / merged checkpoint: new model entity at `output.name`. When `metrics.train_loss` has ≥2 entries, add a loss-drop sentence: *Loss dropped from \ at step 1 to \ at step \; validation loss was \.* Append **Using the adapter** (LoRA) or **Using the fine-tuned model** (full SFT / merged) with discovered provider name and concrete gateway URLs (see below). | +| `error` | Quote `error_details.message` or the failing step; note setup that succeeded before the failure (auth, dataset upload, submit). | +| `cancelled` | Cancellation reason if available. | + +## Training configuration + +Append a `### Training configuration` table after the header block (before **Using the output** when `completed`). Fill rows from the submitted job JSON; omit rows whose fields were not set. Use backend-specific labels: + +| Setting | automodel source | unsloth source | +|---------|------------------|----------------| +| Training type | `training.training_type` | `training.training_type` | +| Finetuning type | `training.finetuning_type` | `training.finetuning_type` | +| LoRA rank / alpha | `training.lora.rank` / `training.lora.alpha` | same | +| Quantization | omit (full-precision / bf16 base weights) | `model.load_in_4bit` → `4-bit (load_in_4bit: true)` or omit when false | +| Max sequence length | `training.max_seq_length` | `model.max_seq_length` | +| Epochs | `schedule.epochs` | `schedule.epochs` | +| Batch | `micro_batch_size` / `global_batch_size` | `batch.per_device_train_batch_size` / `batch.gradient_accumulation_steps` | +| Effective batch size | `global_batch_size` | `per_device_train_batch_size × gradient_accumulation_steps` | +| Learning rate | `optimizer.learning_rate` | same | +| Optimizer | `optimizer` fields used (e.g. `weight_decay`, `warmup_steps`) | `optimizer.optim` (e.g. `adamw_8bit`) | +| Precision | `bf16` (default) | `hardware.precision` | +| GPU | `parallelism.num_gpus_per_node` (and `tensor_parallel_size` when >1) | `hardware.gpus` | +| Output save method | `output.type` (e.g. `adapter`) | `output.save_method` (e.g. `lora`) | + +The three examples below show the filled-in table per backend. + +## Automodel example + +```markdown +### Training configuration + +| Setting | Value | +|---------|-------| +| Training type | SFT | +| Finetuning type | LoRA | +| LoRA rank / alpha | 16 / 32 | +| Max sequence length | 2048 | +| Epochs | 1 | +| Micro batch size | 16 | +| Global batch size | 64 | +| Effective batch size | 64 | +| Learning rate | 1e-4 | +| Optimizer | weight_decay 0.01, warmup_steps 0 | +| Precision | bf16 | +| GPU | 1 (TP=1) | +| Output save method | adapter | +``` + +## Unsloth example + +```markdown +### Training configuration + +| Setting | Value | +|---------|-------| +| Training type | SFT | +| Finetuning type | LoRA | +| LoRA rank / alpha | 16 / 32 | +| Quantization | 4-bit (`load_in_4bit: true`) | +| Max sequence length | 2048 | +| Epochs | 1 | +| Per-device batch size | 8 | +| Gradient accumulation steps | 16 | +| Effective batch size | 128 | +| Learning rate | 1e-4 | +| Optimizer | adamw_8bit | +| Precision | bf16 | +| GPU | 0 | +| Output save method | lora | +``` + +## RL (DPO) example + +Map rows from the `training` (DPOTraining) block; there is no LoRA/save-method. Add DPO-specific rows (`ref_policy_kl_penalty` = β, `sft_loss_weight`): + +```markdown +### Training configuration + +| Setting | Value | +|---------|-------| +| Training type | DPO (full-weight) | +| Reference KL penalty (β) | 0.05 | +| SFT loss weight | 0.0 | +| Max sequence length | 1024 | +| Epochs | 1 | +| Micro batch size | 1 | +| Global batch size | 32 | +| Learning rate | 5e-6 | +| Optimizer | AdamW + cosine annealing | +| Precision | bf16 | +| GPU | 1 node × 1 GPU | +| Output | full-weight model entity | +``` + +## Using the output (`completed` only) + +After **Training configuration**, branch on output type: + +| Output | When | Report section | +|--------|------|----------------| +| LoRA adapter | `save_method: lora` (default) | **Using the adapter** — below | +| Full model | `finetuning_type: all_weights`, `save_method: merged_16bit` / `merged_4bit`, or **rl (DPO)** (always full-weight) | **Using the fine-tuned model** — below | + +### Using the adapter (LoRA / `save_method: lora`) + +**Automodel / unsloth LoRA only** — DPO (rl) never produces an adapter; for rl output use **Using the fine-tuned model** below. Run these discovery commands (parse stdout only; do not pipe `2>&1` into JSON parsers): + +1. `nemo models get --workspace default` — confirm `` appears under `adapters` with `enabled: true`. +2. `nemo inference providers list --workspace default -f json` — pick a **READY** provider whose `served_models` includes `default/` (base entity). Record its `name` as `` (often matches the deployment name). + +On a deployment with `lora_enabled: true`, the adapter is **hot-reloaded automatically** — no new deployment, deployment update, or provider reconfiguration before inference or post-training eval. Append this section with **concrete URLs and provider name** from discovery: + +```markdown +### Using the adapter + +The adapter `` is registered on `default/`. Weights are hot-reloaded on LoRA-enabled deployments serving the **base** entity — no new deployment or provider update after training. + +#### Request routing (base vs LoRA) + +| Target | Gateway path | OpenAI base URL | Request `"model"` field | +|--------|--------------|-----------------|-------------------------| +| **Base** weights | model-entity | `$NMP_BASE_URL/apis/inference-gateway/v2/workspaces/default/model//-/v1` | `default/` | +| **LoRA adapter** | **provider** | `$NMP_BASE_URL/apis/inference-gateway/v2/workspaces/default/provider//-/v1` | `default--` | + +**Common mistake:** posting to the model-entity URL with `"model": "default--"` still runs the **base** model. Base-vs-adapter eval will look identical until LoRA requests use the **provider** URL above. See `references/post-training-eval.md` § **Request routing (base vs LoRA)**. + +#### Chat inference (CHAT-trained models) + +Match training context at inference — send **`messages[:-1]`** (all turns except the final assistant label). Single-turn rows are just the user message; multi-turn rows keep prior user/assistant history. + +| Setting | Value | Why | +|---------|-------|-----| +| `messages` | All turns except the final assistant label from the JSONL row | Same decode path as SFT | +| `max_tokens` | `64` for short assistant labels | Training targets are brief (e.g. MCQA choice text) | +| `temperature` | `0` | Reproducible eval / regression checks | +| `chat_template_kwargs.enable_thinking` | `false` for Qwen3 short-answer SFT | Thinking mode needs extra tokens and changes output shape vs training | + +#### Example — LoRA adapter via provider + +\`\`\`bash +export NMP_BASE_URL= # omit when using default localhost +nemo inference gateway provider post v1/chat/completions --workspace default \\ + --body '{ + "model": "default--", + "messages": [], + "max_tokens": 64, + "temperature": 0, + "chat_template_kwargs": {"enable_thinking": false} + }' +\`\`\` + +#### Example — base model via model-entity (comparison) + +\`\`\`bash +export NMP_BASE_URL= +nemo inference gateway model post v1/chat/completions --workspace default \\ + --body '{ + "model": "default/", + "messages": [], + "max_tokens": 64, + "temperature": 0, + "chat_template_kwargs": {"enable_thinking": false} + }' +\`\`\` + +#### Post-training eval (optional) + +Validation loss from training is **not** accuracy. To compare base vs adapter on the validation split with correct routing: + +\`\`\`bash +cd /path/to/nemo-platform +uv run python plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/eval_helpers.py \\ + --model-entity \\ + --adapter \\ + --provider \\ + --dataset-fileset \\ + --split validation.jsonl +\`\`\` + +Uses CHAT `messages` rows unchanged from the training fileset (`messages[:-1]` at inference). Repeat `--adapter` for multi-adapter compare. `--provider` is optional when a READY provider is auto-discovered. Set `NMP_BASE_URL` (or pass `--base-url`) when the platform is not localhost. LoRA only — full SFT / merged outputs need a deployed model entity (see **Using the fine-tuned model**). +``` + +### Using the fine-tuned model (full SFT / merged checkpoint / DPO) + +When `finetuning_type: all_weights`, `save_method` is `merged_16bit` / `merged_4bit`, or the backend is **rl (DPO)**, the job registers a **model** entity at `output.name` with full fine-tuned weights. **Deploy that entity before inference or eval** — full checkpoints are not hot-reloaded onto the base model's LoRA deployment. + +1. `nemo models get --workspace default` — confirm the fine-tuned model entity exists. +2. Create or update an inference deployment / provider that serves `default/` (same workflow as deploying any model entity). +3. Append this section with the **READY** provider or deployment name and concrete gateway URL. + +```markdown +### Using the fine-tuned model + +Fine-tuned weights are on model entity `default/`. Unlike LoRA adapters, full checkpoints **require a new inference deployment** (or provider update) before chat or eval. + +| Target | Gateway path | OpenAI base URL | Request `"model"` field | +|--------|--------------|-----------------|-------------------------| +| Fine-tuned model | model-entity | `$NMP_BASE_URL/apis/inference-gateway/v2/workspaces/default/model//-/v1` | `default/` | + +Use the same chat settings as LoRA inference (`messages[:-1]`, `max_tokens`, `temperature`, `enable_thinking` as appropriate). Post-training eval: run generation eval against this model-entity URL (not `eval_helpers.py --adapter`, which is LoRA-specific). +``` + +Use the user's platform URL in `NMP_BASE_URL` when they overrode it; omit the export line for default `http://127.0.0.1:8080`. Substitute ``, concrete URLs, and entity names with values from discovery — do not leave generic placeholders in the user-facing report. For **LoRA**, do **not** tell the user to update the deployment before calling the adapter — registration on the base model entity is sufficient. For **full SFT / merged / DPO**, tell the user they must deploy `` before inference. + +**Save report to `/tmp`** — unless the user opts out, write the full Markdown report (header, **Training configuration**, **Using the adapter** or **Using the fine-tuned model** when `completed`, and **Resources created** when a slug or new filesets were used) to `/tmp/fine-tune-result-.md`. Use the random slug from the run when one was assigned; otherwise use the job id suffix (e.g. `a925b07ff678`). + +**Error follow-ups** — when the failure has a known fix, append sections **below** the header block (do not replace the header). Examples: + +| Error type | Append | +|------------|--------| +| Missing training image + user-overridden `NMP_BASE_URL` | `references/troubleshooting.md` § **Missing training images** — on-target build steps, env vars, re-submit commands. **Do not** `docker build` locally for a remote platform. | +| Download fails / `Failed to access upstream storage` / 502 on gated HF model | `references/troubleshooting.md` § **Gated HuggingFace models** — create/update `hf-token`, add `token_secret` to fileset, confirm HF license, re-submit. | +| W&B not syncing / no `[launcher]` secret lines / `WandbCallback requires wandb` / wandb 401 | `references/troubleshooting.md` § **W&B / integrations not working** (jobs-launcher build, secret update, unsloth image). Setup: `references/integrations-setup.md`. | + +For other terminal errors, keep the same header template; put remediation detail in **Notes** or a short **Next steps** section as appropriate. diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/rl-kubernetes-runtime.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/rl-kubernetes-runtime.md new file mode 100644 index 0000000000..68807ac106 --- /dev/null +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/rl-kubernetes-runtime.md @@ -0,0 +1,57 @@ +# rl backend — Kubernetes job execution requirement + +The `rl` (DPO) backend runs **each job step as a Kubernetes pod** via the +`kubernetes_job` execution backend. This is different from `automodel` / `unsloth`, +which use the **docker** job backend. So the platform you submit against must be +deployed/configured for Kubernetes job execution — the docker job backend cannot +run rl, and `rl submit` fails fast (`require_distributed_runtime`) on a +docker-runtime platform. + +Deployment model is the same as automodel/unsloth: **run the platform locally** +(`nemo services run` — never anything else). The only difference for rl is the +**execution backend** the local platform dispatches jobs to: `kubernetes_job` +(pointing at a Kubernetes GPU cluster via its kubeconfig) instead of `docker`. +What matters is that the platform's jobs backend is `kubernetes_job` and the job +pods can reach the platform's APIs. + +## Step 1 — verify the connected platform qualifies (always do this first) + +```bash +nemo jobs list-execution-profiles -f json +``` + +- `cpu` and `gpu` profiles report `backend: kubernetes_job` (or `volcano_job`) → + the platform is ready for rl. Proceed to submit. +- They report `backend: docker` / `subprocess` → the platform is **not** + configured for rl. Do **not** reuse it and do **not** fall back to + automodel/unsloth (those are SFT/LoRA, not DPO). Instead, run the local + platform configured for the `kubernetes_job` backend pointed at a Kubernetes + GPU cluster (see **Configuring the local platform for rl** below). If **no** + Kubernetes cluster is available to point at, stop and tell the user rl needs + one. + +## Configuring the local platform for rl + +When you start the platform locally (`nemo services run`) for an rl job, it must +be configured with all of: + +1. `platform.runtime: kubernetes`. +2. `jobs` `kubernetes_job` executors registered for **both** providers the + customizer stamps — `cpu` (download / upload / model-entity steps) and `gpu` + (DPO training) — at the resolved profile. +3. `platform.loopback_address` set to a platform address the **job pods can reach** + (the platform rewrites the `NMP_*_URL` it injects into pods to this, so the + download/upload steps can call the files/jobs APIs). +4. The target GPU cluster has, available as pullable/loaded images: the job-step + images (`nmp-rl-tasks`, `nmp-rl-training`), the **jobs-launcher** image (each + step runs a launcher init container), and a **job-storage PVC** the steps share. +5. Multi-node only (`parallelism.num_nodes > 1`): `NMP_RL_MULTINODE_SHARED_STORAGE_PATH` + (a shared filesystem for Ray's cross-node coordination). + +If a job pod shows `ErrImagePull` / `ImagePullBackOff` on the launcher init +container or a step image, that image isn't available in the cluster — surface it; +do not build/pull it as part of the customization workflow. + +Reference (local platform config): `docs/set-up/manage-jobs.mdx` (execution +backends — `kubernetes_job`), `docs/set-up/config-reference.mdx` +(`platform.runtime`, `loopback_address`, `kubernetes_job` executor config). diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/troubleshooting.md b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/troubleshooting.md index 88d8c47d11..ea63cf640b 100644 --- a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/troubleshooting.md +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/references/troubleshooting.md @@ -64,7 +64,7 @@ Each entry has `provider`, `profile` (name), and `backend` (e.g. `docker`, `kube | Response includes **`provider`: `gpu` or `gpu_distributed`** | **`automodel`** (default) | | No GPU profiles (only `subprocess` and/or CPU `provider`) | Report that GPU customization is unavailable | -Both backends are **`submit`-only**. After submit, the platform's **Docker executor** runs GPU container steps on the daemon attached to the connected platform host (`platform.runtime: docker`). Training does not run in the CLI shell — query execution profiles on the platform (`NMP_BASE_URL`), not GPU availability in the agent's terminal. +Automodel and unsloth are **`submit`-only**. After submit, the platform's **Docker executor** runs GPU container steps on the daemon attached to the connected platform host (`platform.runtime: docker`). (rl is also submit-only but runs on Kubernetes/Ray — see `rl-kubernetes-runtime.md`.) Training does not run in the CLI shell — query execution profiles on the platform (`NMP_BASE_URL`), not GPU availability in the agent's terminal. ### Pick execution profile @@ -181,7 +181,7 @@ Job errors like `Failed to pull image … nmp-unsloth-training:… Not Found`, ` | Situation | Action | |-----------|--------| -| **Remote platform** — user gave a host/URL (e.g. `10.0.0.51:8080`) or you set `NMP_BASE_URL` to something other than `http://127.0.0.1:8080` or `http://localhost:8080` | **Do not** run `docker build`, `docker pull`, or `docker buildx bake` on the agent machine — that only affects the agent's local daemon, not the remote platform. Tell the user they must build or load the image **on the target host** (the machine whose Docker daemon runs the GPU job steps). Report with **Report to user** in `SKILL.md`, then append **Report follow-up — missing image (remote platform)** below. Stop; do not retry submit until the user confirms the image is available on the target. | +| **Remote platform** — user gave a host/URL (e.g. `10.0.0.51:8080`) or you set `NMP_BASE_URL` to something other than `http://127.0.0.1:8080` or `http://localhost:8080` | **Do not** run `docker build`, `docker pull`, or `docker buildx bake` on the agent machine — that only affects the agent's local daemon, not the remote platform. Tell the user they must build or load the image **on the target host** (the machine whose Docker daemon runs the GPU job steps). Report with the template in `references/reporting.md`, then append **Report follow-up — missing image (remote platform)** below. Stop; do not retry submit until the user confirms the image is available on the target. | | **Local platform** — default URL only (`127.0.0.1:8080` / `localhost:8080`) | Build or pull on **that same host** where `nemo services run` and Docker share a daemon. See build commands below and `docker/unsloth/README.md` (unsloth) or automodel docker docs. Set env vars **before** starting/restarting the platform. | Image env vars are read when the platform starts (not per job): @@ -220,7 +220,7 @@ After the image is on the target, re-submit the same job JSON (use a fresh `outp ### Report follow-up — missing image (remote platform) -When submit or poll returns a missing-image error and the base URL is **user-overridden**, start with the **Report to user** template in `SKILL.md` (status `error`, **Output adapter fileset (planned):**, Notes quoting the pull error and naming the target host). Then append these sections: +When submit or poll returns a missing-image error and the base URL is **user-overridden**, start with the **Report to user** template in `references/reporting.md` (status `error`, **Output adapter fileset (planned):**, Notes quoting the pull error and naming the target host). Then append these sections: **What you need to do on the target host** — build or load the training image on the machine running the NeMo platform (where `docker info` works for the platform's daemon), set `NMP_UNSLOTH_TRAINING_IMAGE` or automodel image env vars, and restart platform services. Full steps: `docker/unsloth/README.md` (unsloth) or automodel docker docs. @@ -323,4 +323,4 @@ Unsloth: | Live schema | `nemo customization unsloth explain` | | Run (disabled) | `nemo customization unsloth run …` → hard-fails; use `submit` | -Both backends return a job id from `submit` — poll until top-level status is terminal (`completed`, `error`, or `cancelled`). +All backends return a job id from `submit` — poll until top-level status is terminal (`completed`, `error`, or `cancelled`). diff --git a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/tests.json b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/tests.json index 7663d036a4..49490836e7 100644 --- a/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/tests.json +++ b/plugins/nemo-customizer/src/nemo_customizer/skills/nemo-customizer/tests.json @@ -51,6 +51,21 @@ "prompt": "I want a quick LoRA adapter on Qwen with Unsloth's optimizer defaults, single GPU container job.", "expected_skill": "nemo-customizer" }, + { + "type": "explicit", + "prompt": "Use nemo customization rl submit to run a DPO job on NeMo Platform.", + "expected_skill": "nemo-customizer" + }, + { + "type": "implicit", + "prompt": "Align Qwen3-0.6B with DPO on a {prompt, chosen, rejected} preference dataset via nemo customization.", + "expected_skill": "nemo-customizer" + }, + { + "type": "implicit", + "prompt": "I want to run direct preference optimization on the platform's Ray cluster and register the trained model.", + "expected_skill": "nemo-customizer" + }, { "type": "contextual", "prompt": "NeMo Platform is running. Before any customization training, help me explore what my support agent should do.", diff --git a/plugins/nemo-rl/tests/fixtures/integrations_wandb_mlflow.json b/plugins/nemo-rl/tests/fixtures/integrations_wandb_mlflow.json new file mode 100644 index 0000000000..544966f5a2 --- /dev/null +++ b/plugins/nemo-rl/tests/fixtures/integrations_wandb_mlflow.json @@ -0,0 +1,38 @@ +{ + "name": "qwen3-dpo-integrations-smoke", + "model": "default/qwen3-0.6b", + "dataset": "default/helpsteer3-dpo", + "training": { + "type": "dpo", + "max_steps": 20, + "batch_size": 32, + "micro_batch_size": 1, + "learning_rate": 5e-6, + "max_seq_length": 1024, + "ref_policy_kl_penalty": 0.05, + "parallelism": { "num_nodes": 1, "num_gpus_per_node": 1 } + }, + "integrations": { + "wandb": { + "project": "my-project", + "name": "run-001", + "entity": "my-team", + "tags": ["sft", "llama"], + "notes": "Experiment notes", + "base_url": "https://wandb.internal", + "api_key_secret": "default/wandb-api-key" + }, + "mlflow": { + "experiment_name": "llama-finetuning", + "name": "run-001", + "tracking_uri": "http://mlflow:5000", + "tags": { + "team": "nlp" + }, + "description": "SFT experiment" + } + }, + "output": { + "name": "qwen3-0.6b-dpo" + } +} diff --git a/plugins/nemo-rl/tests/fixtures/minimal_dpo.json b/plugins/nemo-rl/tests/fixtures/minimal_dpo.json new file mode 100644 index 0000000000..4effe0b4bb --- /dev/null +++ b/plugins/nemo-rl/tests/fixtures/minimal_dpo.json @@ -0,0 +1,15 @@ +{ + "model": "default/qwen3-0.6b", + "dataset": "default/helpsteer3-dpo", + "training": { + "type": "dpo", + "max_steps": 20, + "batch_size": 32, + "micro_batch_size": 1, + "learning_rate": 5e-6, + "max_seq_length": 1024, + "ref_policy_kl_penalty": 0.05, + "parallelism": { "num_nodes": 1, "num_gpus_per_node": 1 } + }, + "output": { "name": "qwen3-0.6b-dpo" } +} diff --git a/plugins/nemo-rl/tests/test_contract_job_inputs.py b/plugins/nemo-rl/tests/test_contract_job_inputs.py new file mode 100644 index 0000000000..2f3ed08e53 --- /dev/null +++ b/plugins/nemo-rl/tests/test_contract_job_inputs.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Contract fixtures for submit-time RlJobInput JSON.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from nemo_rl_plugin.schema import RlJobInput + +FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures" + + +@pytest.mark.parametrize( + "fixture_name", + ["minimal_dpo.json", "integrations_wandb_mlflow.json"], +) +def test_contract_job_input_validates(fixture_name: str) -> None: + path = FIXTURES_DIR / fixture_name + spec = RlJobInput.model_validate(json.loads(path.read_text())) + assert spec.training.type == "dpo" + + if spec.integrations is None: + return + + assert spec.integrations.wandb is not None + assert spec.integrations.wandb.project == "my-project" + assert spec.integrations.wandb.name == "run-001" + assert spec.integrations.wandb.api_key_secret is not None + assert spec.integrations.wandb.api_key_secret.root == "default/wandb-api-key" + assert spec.integrations.mlflow is not None + assert spec.integrations.mlflow.tracking_uri == "http://mlflow:5000" + assert spec.integrations.mlflow.name == "run-001"