chore: nightly sync main into dev (22_04_2026)

.agents/skills

@@ -0,0 +1 @@
+../skills

.claude/skills

@@ -0,0 +1 @@
+../skills

.github/CODEOWNERS

@@ -1,4 +1,48 @@
-* @NVIDIA/core-nemo @NVIDIA/core-devtech
+megatron/core/ @NVIDIA/core-adlr @NVIDIA/core-nemo
+
+megatron/core/models/gpt/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/gpt
+
+megatron/core/models/multimodal/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/multi-modal
+
+megatron/core/models/mamba/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/hybrid-mamba
+megatron/core/ssm/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/hybrid-mamba
+
+megatron/core/models/hybrid/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/hybrid-model
+
+megatron/core/datasets/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/datasets
+
+megatron/core/tokenizers/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/tokenizers
+
+megatron/core/distributed/fsdp/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/megatron-fsdp
+
+megatron/core/transformer/fsdp_dtensor_checkpoint.py @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/megatron-fsdp
+
+megatron/core/dist_checkpointing/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/dist-checkpointing
+
+megatron/core/optimizer/distrib_optimizer/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/dist-optimizer
+
+megatron/core/inference/modelopt_support @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/quantization-and-inference
+
+megatron/core/datasets/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/datasets
+
+megatron/core/pipeline_parallel/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/pipeline-parallelism
+
+megatron/core/transformer/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/transformer
+
+megatron/core/transformer/moe/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/mixture-of-experts-adlr @NVIDIA/mixture-of-experts-devtech
+
+megatron/core/inference/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/inference
+
+megatron/core/parallel_state.py @NVIDIA/core-adlr @NVIDIA/core-nemo
+
+megatron/core/post_training/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/post-training
+
+megatron/post_training/ @NVIDIA/post-training
+
+megatron/core/transformer/cuda_graphs.py @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/cuda-graphs
+
+megatron/training/ @NVIDIA/training-adlr @NVIDIA/training-nemo
+megatron/training/arguments.py
 
 .gitlab/ @NVIDIA/ci
 .github/ @NVIDIA/ci

.github/actions/action.yml

@@ -83,7 +83,11 @@ runs:
           echo "apt attempt $i failed, retrying..."
           sleep 10
         done
-        curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh
+        for i in 1 2 3; do
+          curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh && break
+          echo "uv install attempt $i failed, retrying..."
+          sleep 10
+        done
 
     - name: Create run-script (unit test)
       shell: bash -x -e -u -o pipefail {0}
@@ -210,7 +214,7 @@ runs:
         LOG_BASE=$([[ "$IS_UNIT_TEST" == "true" ]] && echo "assets_dir/logs" || echo "assets_dir")
         LATEST_LOG=""
         if [[ -d "$LOG_BASE" ]]; then
-          LATEST_LOG=$(find "$LOG_BASE" -maxdepth 3 -name "*.log" ! -name "nccl_debug.log" -type f 2>/dev/null \
+          LATEST_LOG=$(find "$LOG_BASE" -name "*.log" ! -name "nccl_debug.log" -type f 2>/dev/null \
                          | xargs -r ls -t 2>/dev/null | head -1 || true)
         fi
         if [[ -n "$LATEST_LOG" ]]; then

.github/copy-pr-bot.yaml

@@ -1,4 +1,4 @@
 enabled: true
 auto_sync_draft: false
 auto_sync_ready: true
-trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "CarlosGomes98", "ChenhanYu", "Connor-XY", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JRD971000", "Phlip79", "QiZhangNV", "RPrenger", "ShriyaRishab", "Victarry", "WanZzzzzz", "Wohox", "ZhiyuLi-Nvidia", "ahmadki", "aklife97", "ananthsub", "asolergi-nv", "buptzyb", "chtruong814", "cjld", "cspades", "cuichenx", "deepakn94", "dimapihtar", "dingqingy-nv", "duncanriach", "erhoo82", "ericharper", "fanshiqing", "faradawn", "fitsumreda", "frsun-nvda", "gautham-kollu", "gdengk", "guihong-nv", "guyueh1", "hexinw-nvidia", "huvunvidia", "hxbai", "ilml", "jalbericiola", "janEbert", "jaredcasper", "jenchen13", "jiemingz", "jingqiny-99", "jkamalu", "jon-barker", "jstjohn", "kajalj22", "kanz-nv", "keshavb96", "kevalmorabia97", "ko3n1g", "ksivaman", "kunlunl", "kvareddy", "kwyss-nvidia", "layalir", "lhb8125", "lmcafee-nvidia", "maanug-nv", "mathemakitten", "matthieule", "mchrzanowski", "mehraakash", "minitu", "mkhona-nvidia", "nanz-nv", "parthmannan", "prajwal1210", "pthombre", "rhewett-nv", "rogerwaleffe", "sajadn", "sanandaraj5597", "sancha", "santhnm2", "sbak5", "shanmugamr1992", "sharathts", "sheliang-nv", "shengf-nv", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "sraman-rgb", "sudhakarsingh27", "tdene", "theothermike", "thomasdhc", "tomlifu", "trintamaki", "tylerpoon", "wdykas", "wplf", "wujingyue", "xiaoyao0115", "xuwchen", "yanring", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yueshen2016", "yuzhongw-nvidia", "zhongbozhu"]
+trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "CarlosGomes98", "ChenhanYu", "Connor-XY", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JRD971000", "Mellonta", "Phlip79", "QiZhangNV", "RPrenger", "ShriyaRishab", "Victarry", "WanZzzzzz", "Wohox", "YangFei1990", "ZhiyuLi-Nvidia", "ahmadki", "aklife97", "ananthsub", "aroshanghias-nvd", "asolergi-nv", "buptzyb", "chtruong814", "cjld", "cspades", "cuichenx", "deepakn94", "dimapihtar", "dingqingy-nv", "duncanriach", "erhoo82", "ericharper", "fanshiqing", "faradawn", "fitsumreda", "frsun-nvda", "gautham-kollu", "gdengk", "guihong-nv", "guyueh1", "hexinw-nvidia", "huvunvidia", "hxbai", "ilml", "jalbericiola", "janEbert", "jaredcasper", "jenchen13", "jiemingz", "jingqiny-99", "jkamalu", "jon-barker", "jstjohn", "kajalj22", "kanz-nv", "keshavb96", "kevalmorabia97", "ko3n1g", "ksivaman", "kunlunl", "kvareddy", "kwyss-nvidia", "layalir", "lhb8125", "lmcafee-nvidia", "maanug-nv", "mathemakitten", "matthieule", "mchrzanowski", "mehraakash", "minitu", "mkhona-nvidia", "nanz-nv", "parthmannan", "prajwal1210", "pthombre", "rhewett-nv", "rogerwaleffe", "sajadn", "sanandaraj5597", "sancha", "santhnm2", "sbak5", "shanmugamr1992", "sharathts", "sheliang-nv", "shengf-nv", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "sraman-rgb", "sudhakarsingh27", "tdene", "theothermike", "thomasdhc", "tomlifu", "trintamaki", "tylerpoon", "wdykas", "wplf", "wujingyue", "xiaoyao0115", "xuwchen", "yanring", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yueshen2016", "yuzhongw-nvidia", "zhongbozhu"]

.github/oncall_schedule.json

@@ -1,12 +1,4 @@
 [
-    {
-        "user": "ilml",
-        "date": "2026-04-08"
-    },
-    {
-        "user": "Phlip79",
-        "date": "2026-04-15"
-    },
     {
         "user": "asolergi-nv",
         "date": "2026-04-22"
@@ -50,5 +42,9 @@
     {
         "user": "maanug-nv",
         "date": "2026-07-01"
+    },
+    {
+        "user": "wujingyue",
+        "date": "2026-07-08"
     }
 ]

.github/workflows/claude_review.yml

@@ -5,8 +5,12 @@ on:
     types: [created]
 
 jobs:
-  review-on-comment:
-    name: Claude Review (comment trigger)
+  # ──────────────────────────────────────────────────────────────────
+  # Light review: quick pass for obvious bugs, typos, and test gaps
+  # Trigger: /claude review
+  # ──────────────────────────────────────────────────────────────────
+  light-review:
+    name: Claude Light Review
     if: |
       github.event_name == 'issue_comment' &&
       github.event.issue.pull_request &&
@@ -39,7 +43,7 @@ jobs:
             --method POST \
             -f content='eyes'
 
-      - name: Run Claude Code Review
+      - name: Run Claude Light Review
         uses: anthropics/claude-code-action@v1
         with:
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
@@ -77,3 +81,175 @@ jobs:
 
             It's perfectly acceptable to not have anything to comment on.
             If you do not have anything to comment on, approve the PR with: gh pr review $PR_NUMBER --repo $REPO --approve --body "LGTM"
+
+  # ──────────────────────────────────────────────────────────────────
+  # Strict review: comprehensive Megatron-LM focused analysis
+  # covering precision, parallelism correctness, performance,
+  # backward compatibility, and code quality
+  # Trigger: /claude strict-review
+  # ──────────────────────────────────────────────────────────────────
+  strict-review:
+    name: Claude Strict Review
+    if: |
+      github.event_name == 'issue_comment' &&
+      github.event.issue.pull_request &&
+      contains(github.event.comment.body, '/claude strict-review')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write
+      id-token: write
+    env:
+      GH_TOKEN: ${{ github.token }}
+      REPO: ${{ github.repository }}
+      PR_NUMBER: ${{ github.event.issue.number }}
+    steps:
+      - name: Get PR info
+        id: pr-info
+        run: |
+          PR_DATA=$(gh pr view $PR_NUMBER --repo $REPO --json headRefOid,baseRefName)
+          echo "sha=$(echo $PR_DATA | jq -r .headRefOid)" >> $GITHUB_OUTPUT
+          echo "base_ref=$(echo $PR_DATA | jq -r .baseRefName)" >> $GITHUB_OUTPUT
+
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 1
+          ref: ${{ steps.pr-info.outputs.sha }}
+
+      - name: Fetch base branch for diff analysis
+        run: git fetch origin ${{ steps.pr-info.outputs.base_ref }}
+
+      - name: React to trigger comment
+        run: |
+          gh api repos/$REPO/issues/comments/${{ github.event.comment.id }}/reactions \
+            --method POST \
+            -f content='eyes'
+
+      - name: Run Claude Strict Review
+        uses: anthropics/claude-code-action@v1
+        with:
+          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
+          trigger_phrase: "/claude strict-review"
+          show_full_output: true
+          claude_args: |
+            --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr review:*),Bash(git diff:*),Bash(git show:*),Bash(git log:*)"
+            --model "claude-opus-4-6"
+          prompt: |
+            REPO: ${{ env.REPO }}
+            PR NUMBER: ${{ env.PR_NUMBER }}
+            BASE REF: origin/${{ steps.pr-info.outputs.base_ref }}
+
+            You are performing a strict, comprehensive code review on a **Megatron-LM** Pull Request.
+            Megatron-LM is NVIDIA's large-scale distributed training framework for LLMs.
+            Review the diff with a focus on **implementation correctness**, **training performance**, and **backward compatibility**.
+
+            ## Review Procedure
+
+            1. Get PR metadata: `gh pr view $PR_NUMBER --repo $REPO --json title,body,baseRefName,headRefName,files,additions,deletions,changedFiles,author`
+            2. Get the full diff: `gh pr diff $PR_NUMBER --repo $REPO`
+               - For large PRs (>50 files), prioritize source code over config/lock/auto-generated files.
+            3. For each significant changed file, read the full file for surrounding context.
+            4. Trace data flow and dtype through computation paths to verify correctness.
+            5. For each newly introduced variable/argument/field, verify it has a meaningful runtime use path (see Mandatory Check below).
+            6. Post findings as inline comments with severity and category tags.
+
+            ## Critical Issues (Must Fix)
+
+            ### Implementation Correctness
+            - **dtype handling**: Verify operations use the correct dtype at each computation stage — explicit casts must be present at mixed-precision boundaries (e.g. fp16 compute → fp32 accumulation → fp16 output)
+            - **Loss scaling logic**: Verify DynamicLossScaler changes correctly detect inf/nan, adjust scale factor, and skip optimizer steps — incorrect logic causes training divergence or silent underflow
+            - **Reduction operations**: Verify reductions (sum, mean, allreduce) use correct dtype, reduction dimension, and normalization factor — wrong dimension or missing fp32 upcast produces silently wrong gradients
+            - **Normalization layers**: Verify LayerNorm/RMSNorm compute variance and mean on the correct dimension, with correct epsilon placement and upcast before rsqrt
+            - **Attention computation**: Verify QK^T scaling factor, softmax input dtype, causal mask application, and dropout placement match the intended algorithm
+            - **Residual connections**: Verify the correct tensor is added (pre-norm vs post-norm) with appropriate dtype for accumulation
+            - **Optimizer updates**: Verify state updates follow the correct formula — momentum/variance update order, bias correction, weight decay application
+            - **Gradient clipping**: Verify norm computation uses correct parameter set, norm type (L2 vs inf), and fp32 dtype
+            - **Embedding/output layer**: Verify weight tying is correctly wired, logit projection uses the right matrix, and output dtype matches expectation
+            - **MoE routing/aux loss**: Verify expert routing logic (top-k selection, capacity enforcement, token dropping) and auxiliary loss computation follow the intended algorithm
+
+            ### Correctness
+            - **Tensor parallel**: Incorrect scatter/gather or allreduce placement — silent wrong results across TP ranks
+            - **Pipeline parallel**: Wrong microbatch scheduling, missing send/recv synchronization, incorrect grad accumulation across pipeline stages
+            - **Sequence parallel**: Incorrect sequence dimension partitioning or missing allgather/reduce-scatter in SP regions
+            - **Context parallel**: Incorrect KV cache partitioning or ring attention implementation errors
+            - **Expert parallel**: Token routing/dispatch errors across EP ranks, incorrect capacity factor handling
+            - **Gradient accumulation**: Missing no_sync() context or incorrect division factor when accumulating across microbatches
+            - **Checkpoint save/load**: State dict key mismatch, missing optimizer states, incorrect RNG state restoration — causes silent divergence after resume
+            - **RNG state management**: Incorrect random seed handling across TP/PP/DP ranks, causing correlated dropout masks or data sampling
+
+            ## Important Issues (Should Fix)
+
+            ### Training Performance
+            - **Unnecessary CPU-GPU sync**: .item(), .cpu(), torch.cuda.synchronize(), Python-side tensor value checks in training loop — kills throughput
+            - **Redundant communication**: Allreduce/allgather that could be fused, overlapped with compute, or eliminated
+            - **Memory inefficiency**: Missing activation checkpointing on memory-heavy layers, unnecessary tensor clones or .contiguous() calls
+            - **Communication-computation overlap**: Missed opportunities to overlap allreduce with backward, or allgather with forward
+            - **Kernel launch overhead**: Python loops over small ops that should be fused into a single kernel
+            - **CUDA graph compatibility**: Dynamic shapes, Python-side conditionals on tensor values, host-device sync inside captured region
+
+            ### Backward Compatibility
+            - **Config/argument changes**: Renamed or removed arguments without deprecation path — breaks existing training scripts
+            - **Checkpoint format changes**: Modified state dict keys/structure without migration logic — makes existing checkpoints unloadable
+            - **Default value changes**: Changed defaults for training hyperparameters or parallelism settings — silently alters behavior for users relying on defaults
+            - **API contract changes**: Changed function signatures, return types, or side effects in megatron/core/ without backward-compat shim
+            - **Model architecture changes**: Altered layer ordering, initialization, or normalization placement — existing pretrained weights become incompatible
+
+            ### Mandatory Check: Unused New Variables / Arguments
+            - For each changed file, list newly added identifiers (function args, config fields, locals).
+            - Verify each has a meaningful read/use path — not just declaration/docstring or discard assignment (_ = new_arg).
+            - Use Grep to search for usage beyond declaration sites.
+            - Treat placeholder discard patterns as findings unless explicitly documented as temporary migration shim.
+            - If usage is intentionally deferred, flag and request explicit TODO + migration note.
+
+            ## Suggestions (Nice to Have)
+
+            ### Naming
+            - Name must describe what the thing *is*, not what it's *used for*
+            - No abbreviations in parallel/distributed code — use full names (token_dispatcher, routing_map, comm_manager, world_size)
+            - Naming consistency within scope for variables serving the same role
+
+            ### Function/Method Decomposition
+            - Functions over ~50 lines mixing data collection, reduction, computation, and I/O should be split
+            - Non-trivial logic blocks embedded in a method with different primary purpose should be extracted
+
+            ### Simplification
+            - Redundant operations (e.g. .reshape(()) on 0-dim tensor, two-step constructions where one suffices)
+            - Setup constant across training should not run on every forward pass — move to __init__
+            - Dead complexity that doesn't achieve its stated purpose
+            - Unnecessary intermediate aliases adding indirection with no abstraction value
+
+            ### Other
+            - Stale, imprecise, or misleading comments/docstrings — a wrong docstring is worse than none
+            - Missing shape/dtype assertions at parallelism boundaries
+
+            ## What NOT to Comment On
+            - Style/formatting issues (leave to linters)
+            - Test code that is reasonably clear
+            - Clearly intentional design decisions by the author
+            - Pure refactoring that preserves identical behavior (verify via diff)
+            - Findings invalidated by deeper analysis — drop them entirely rather than hedging
+
+            ## Comment Format
+
+            Prefix each comment with severity and category tag:
+            - `**[CRITICAL Implementation]**`, `**[CRITICAL Correctness]**`
+            - `**[IMPORTANT Performance]**`, `**[IMPORTANT Compatibility]**`
+            - `**[SUGGESTION Naming]**`, `**[SUGGESTION Simplification]**`
+
+            For each finding, explain: (1) what the issue is, (2) why it matters (impact/risk), (3) specific suggestion for fix.
+
+            Only use inline ```suggestion blocks for simple, self-contained line replacements (typos,
+            renames, single-line fixes). For structural changes that add, remove, or reorganize blocks
+            of code, use a top-level PR comment with a code block showing the proposed change instead.
+
+            ## Completion
+
+            After posting all inline comments, post a summary PR comment:
+            - List total findings by severity (CRITICAL: N, IMPORTANT: N, SUGGESTION: N)
+            - Highlight the most impactful findings
+            - Overall assessment of the PR's risk level
+
+            If no significant issues are found, approve the PR:
+            gh pr review $PR_NUMBER --repo $REPO --approve --body "Strict review passed — no significant issues found. LGTM"