feat(rccl): add GPT-OSS-120B on 2 nodes to the madengine gate

.github/workflows/therock-rccl-ci-linux.yml

@@ -274,6 +274,43 @@ jobs:
       notify_email: ${{ inputs.notify_email }}
       teams_webhook: ${{ inputs.notify_webhook }}
 
+  # Every other workload, on the weekly run only: test_scope == 'all' is what
+  # the Saturday cron in therock-rccl-ci.yml sets, and a dispatch can ask for
+  # it. The job above stays the nightly gate on its own.
+  #
+  # One matrix rather than a job per workload, so that max-parallel: 1 covers
+  # all of them: each wants an exclusive salloc on the same partition, and
+  # queue time counts against its own budget. `needs` the nightly job for
+  # ORDER ONLY -- `!cancelled()` keeps a failed nightly from skipping these.
+  # Nodes are per entry, not inputs.madengine_nodes: each workload's
+  # parallelism and memory headroom were validated at one scale.
+  therock-test-madengine-weekly:
+    name: "Test MADEngine workloads (weekly)"
+    if: ${{ !cancelled() && contains(fromJSON(needs.test-plan.outputs.suites || '[]'), 'madengine') && (needs.therock-build-linux.result == 'success' || needs.therock-build-linux.result == 'skipped') && inputs.amdgpu_families == 'gfx950-dcgpu' && inputs.test_scope == 'all' && (inputs.event_name == 'schedule' || inputs.event_name == 'workflow_dispatch') }}
+    needs: [test-plan, therock-build-linux, therock-test-madengine]
+    permissions:
+      contents: read
+      id-token: write
+    strategy:
+      # One workload failing says nothing about the others.
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        include:
+          - workload: gpt-oss-120b-training
+            nodes: '2'
+    uses: ./.github/workflows/therock-rccl-test-madengine.yml
+    secrets: inherit
+    with:
+      amdgpu_families: ${{ inputs.amdgpu_families }}
+      artifact_group: ${{ inputs.artifact_group }}
+      test_runs_on: ruby-linux-slurm-scale-runner
+      artifact_run_id: ${{ inputs.artifact_run_id || github.run_id }}
+      workload: ${{ matrix.workload }}
+      nodes: ${{ matrix.nodes }}
+      notify_email: ${{ inputs.notify_email }}
+      teams_webhook: ${{ inputs.notify_webhook }}
+
   therock-test-accl-profiler:
     name: "Test ACCL profiler decomposition (scheduled)"
     if: ${{ !cancelled() && contains(fromJSON(needs.test-plan.outputs.suites || '[]'), 'accl-profiler') }}

.github/workflows/therock-rccl-test-madengine.yml

@@ -12,6 +12,9 @@ on:
       artifact_run_id:
         type: string
       workload:
+        # A key of WORKLOAD_CONFIGS in test_madengine.py, which validates it.
+        # workflow_call inputs cannot be typed as choice, so the list of
+        # accepted values lives on the workflow_dispatch input below.
         type: string
         default: 'llama-3.1-70b-training'
       nodes:
@@ -37,7 +40,10 @@ on:
         type: string
       workload:
         description: 'MADEngine workload to run'
-        type: string
+        type: choice
+        options:
+          - llama-3.1-70b-training
+          - gpt-oss-120b-training
         default: 'llama-3.1-70b-training'
       nodes:
         description: 'Number of SLURM nodes to allocate'
@@ -69,8 +75,9 @@ jobs:
       RESULTS_DIR: "/apps/rccl-ci/madengine/perf"
       # run_attempt too: run_id is stable across re-runs, and the cleanup step
       # leaves the result files behind, so a re-run could parse a previous
-      # attempt's perf.csv.
-      WORK_DIR: "/apps/rccl-ci/madengine/workdir/${{ github.run_id }}-${{ github.run_attempt }}"
+      # attempt's perf.csv.  The workload is in the path because the scheduled
+      # caller runs several of them from one workflow run, which shares both.
+      WORK_DIR: "/apps/rccl-ci/madengine/workdir/${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.workload }}"
       # Workspace-local, as in the JAX and PyTorch test workflows: the fetched
       # ROCm tree is only read on the head node, and /apps/cvs_tests/dist_new
       # belongs to the CVS suite, which this job may neither traverse nor wipe.

projects/rccl/ci/scripts/test_madengine.py

@@ -80,7 +80,7 @@
 MADENGINE_REPO = "https://github.com/ROCm/madengine.git"
 MADENGINE_REF = "00ee3e80c0a308599633caf74e5d91a72164a2fe"  # develop, has #190 and #213
 MAD_REPO = "https://github.com/ROCm/MAD.git"
-MAD_REF = "7a9a25e742ef60e8deffb6e5fd9b559857d2018d"  # mad-rccl, 2026-10-09
+MAD_REF = "7a9a25e742ef60e8deffb6e5fd9b559857d2018d"  # mad-rccl, has MAD#271
 MAD_BRANCH = "mad-rccl"
 
 WORKLOAD_CONFIGS = {
@@ -111,25 +111,25 @@
             "--device=/dev/infiniband --cap-add IPC_LOCK "
             "--ulimit memlock=-1 -v /sys:/sys:ro -v /run/udev:/run/udev:ro",
     },
-    "llama-4-scout-training": {
+    "gpt-oss-120b-training": {
         "type": "training",
-        "model_repo": "primus_pyt_megatron_lm_train_llama-4-scout-17b-16e",
-        "model_repo_aliases": [
-            "primus_pyt_megatron_lm_train_llama-4-scout-17b-16e_overlay",
-            "primus_pyt_megatron_lm_train_llama-4-scout-17b-16e_scaleout",
-        ],
+        "model_repo": "primus_pyt_megatron_lm_train_gpt-oss-120b",
         "base_image": "rocm/primus:v26.4",
         "gpu_target": "gfx950",
         "metric_key": "tokens_per_second_per_gpu",
-        "multiple_results": "perf_primus-megatron-Llama-4-Scout-17B-16E.csv",
-        "precisions": ["BF16"],  # what MAD's run.sh trains on gfx950
-        "reference_values": {
-            "2N": 2734,
-            "4N": 2337,
+        "multiple_results": "perf_primus-megatron-GPT-OSS-120B.csv",
+        "precisions": ["BF16", "FP8"],  # what MAD's run.sh trains on gfx950
+        # The shipped config profiles iterations 6-7, which at 2 nodes cost
+        # 3.2x (BF16) and 3.7x (FP8) a steady iteration and leave the next two
+        # elevated. Primus averages from iteration 3, so 15 iterations give 13
+        # measured ones at ~65 s (BF16) and ~116 s (FP8) each.
+        "env_vars": {
+            "PRIMUS_TRAIN_ITERS": "15",
+            "PRIMUS_DISABLE_PROFILE": "1",
         },
         "slurm_partition": "meta64",
         "gpus_per_node": 8,
-        "time_limit": "03:00:00",  # one allocation, two runs, plus image staging
+        "time_limit": "04:00:00",  # one allocation, two runs, plus image staging
         "docker_mounts": {"/dev/infiniband": "/dev/infiniband"},
         "docker_run_options": "--privileged --group-add render --shm-size 64G "
             "--device=/dev/infiniband --cap-add IPC_LOCK "
@@ -336,6 +336,58 @@ def _split_rccl_version(version: str) -> tuple[str, str, bool]:
     return m.group(1), (m.group(2) or "").lower(), m.group(3) == "+"
 
 
+# RCCL prints this on EVERY call of a deprecated API name (src/misc/api_trace.cc),
+# at WARN, so NCCL_DEBUG=WARN keeps it. GPT-OSS-120B's MoE all-to-all makes it
+# 97% of each node log, which madengine keeps three copies of.
+_LOG_SPAM_RE = re.compile(rb"NCCL WARN Please note that (\w+) is deprecated")
+
+
+def squash_log_spam(run_dir: Path) -> int:
+    """Drop the repeats of _LOG_SPAM_RE lines from one phase's logs, in place.
+
+    The first occurrence of each stays where it was, and one line at the end
+    of the file says how many repeats went. Files with no repeats are left
+    untouched. Returns the number of bytes saved.
+    """
+    log_dir = run_dir / "slurm_output"
+    if not log_dir.is_dir():
+        return 0
+    saved = 0
+    for path in sorted(
+        p for p in log_dir.rglob("*")
+        if p.suffix in (".out", ".err", ".log") and p.is_file()
+    ):
+        dropped: dict[bytes, int] = {}
+        tmp = path.with_name(path.name + ".squash")
+        try:
+            before = path.stat().st_size
+            with path.open("rb") as src, tmp.open("wb") as dst:
+                for line in src:
+                    m = _LOG_SPAM_RE.search(line)
+                    if m:
+                        api = m.group(1)
+                        if api in dropped:
+                            dropped[api] += 1
+                            continue
+                        dropped[api] = 0
+                    dst.write(line)
+                for api, count in dropped.items():
+                    if count:
+                        dst.write(
+                            b"[rccl-ci] dropped %d repeats of 'NCCL WARN Please note "
+                            b"that %s is deprecated'\n" % (count, api)
+                        )
+            if not any(dropped.values()):
+                tmp.unlink()
+                continue
+            os.replace(tmp, path)
+            saved += before - path.stat().st_size
+        except OSError as exc:
+            log.warning("Could not squash %s: %s", path, exc)
+            tmp.unlink(missing_ok=True)
+    return saved
+
+
 def runtime_rccl_versions(work_dir: Path) -> list[str]:
     """Distinct ``RCCL version :`` banners across one phase's node logs."""
     log_dir = work_dir / "slurm_output"
@@ -595,7 +647,11 @@ def generate_manifest(
         if "," in ifname:
             nccl_env["NCCL_SOCKET_IFNAME"] = ifname.split(",")[0]
 
-    socket_ifname = nccl_env.get("NCCL_SOCKET_IFNAME", "")
+    # Last over everything below, socket interfaces included.
+    workload_env = workload_config.get("env_vars", {})
+    socket_ifname = workload_env.get(
+        "NCCL_SOCKET_IFNAME", nccl_env.get("NCCL_SOCKET_IFNAME", "")
+    )
 
     # HF_TOKEN is passed via MAD_SECRETS_HFTOKEN in the process environment
     # (set in the workflow). Do NOT write it into the manifest — the manifest
@@ -616,8 +672,9 @@ def generate_manifest(
         "NCCL_IB_DISABLE": "0",
         "NCCL_TIMEOUT": "900",
         "IBV_SHOW_WARNINGS": "1",
+        **workload_env,
     }
-    if socket_ifname:
+    if socket_ifname and "GLOO_SOCKET_IFNAME" not in workload_env:
         docker_env_vars["GLOO_SOCKET_IFNAME"] = socket_ifname
 
     docker_mounts = dict(workload_config.get("docker_mounts", {}))
@@ -1612,6 +1669,13 @@ def main() -> None:
     )
     exit_code = max(allocation["exit_codes"].values(), default=1)
 
+    # Step 5a: Shrink the node logs before anything reads or uploads them.
+    for phase, run_dir in phases:
+        saved = squash_log_spam(run_dir)
+        if saved:
+            log.info("%s: dropped %.1f MB of repeated RCCL warnings from the node logs",
+                     phase, saved / 1e6)
+
     # Step 5b: Verify RCCL replacement. Only the candidate is meant to carry the
     # CI library — the baseline runs the image's bundled copy by design.
     rccl_verification_failed = False

projects/rccl/ci/scripts/tests/test_log_squash.py

@@ -0,0 +1,74 @@
+"""Tests for squash_log_spam() in test_madengine.py.
+
+Run from projects/rccl/ci/scripts:  python3 -m unittest discover -s tests
+"""
+
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, os.fspath(Path(__file__).resolve().parent.parent))
+import test_madengine as tm
+
+SPAM = (b"[2026-10-12 21:11:21] node:985:985 [0] api_trace.cc:569 NCCL WARN Please note "
+        b"that ncclAllToAllv is deprecated, please use ncclAlltoAllv instead\n")
+SPAM2 = SPAM.replace(b"ncclAllToAllv", b"ncclAllToAll").replace(b"ncclAlltoAllv", b"ncclAlltoAll")
+BANNER = b"RCCL version : 2.32.3-develop:2b88119\n"
+REAL_WARN = b"node:985:985 [0] NCCL WARN NET/IB : Got async event : port error\n"
+
+
+class SquashLogSpam(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory()
+        self.run_dir = Path(self._tmp.name)
+        self.logs = self.run_dir / "slurm_output"
+        (self.logs / "model" / "node_0").mkdir(parents=True)
+
+    def tearDown(self):
+        self._tmp.cleanup()
+
+    def test_keeps_first_and_counts_the_rest(self):
+        log = self.logs / "madengine-x_node_0.out"
+        log.write_bytes(BANNER + SPAM * 5 + REAL_WARN + SPAM * 3 + b"tok/s 1500\n")
+        saved = tm.squash_log_spam(self.run_dir)
+        self.assertEqual(
+            log.read_bytes(),
+            BANNER + SPAM + REAL_WARN + b"tok/s 1500\n"
+            + b"[rccl-ci] dropped 7 repeats of 'NCCL WARN Please note that "
+              b"ncclAllToAllv is deprecated'\n",
+        )
+        self.assertEqual(saved, 7 * len(SPAM) - len(log.read_bytes().splitlines(True)[-1]))
+        self.assertEqual(tm.runtime_rccl_versions(self.run_dir), ["2.32.3-develop:2b88119"])
+
+    def test_each_api_counted_separately(self):
+        log = self.logs / "model" / "node_0" / "stdout.out"
+        log.write_bytes(SPAM + SPAM2 + SPAM + SPAM2 + SPAM2)
+        tm.squash_log_spam(self.run_dir)
+        self.assertEqual(
+            log.read_bytes(),
+            SPAM + SPAM2
+            + b"[rccl-ci] dropped 1 repeats of 'NCCL WARN Please note that "
+              b"ncclAllToAllv is deprecated'\n"
+            + b"[rccl-ci] dropped 2 repeats of 'NCCL WARN Please note that "
+              b"ncclAllToAll is deprecated'\n",
+        )
+
+    def test_file_without_repeats_is_untouched(self):
+        log = self.logs / "madengine-x_node_1.out"
+        body = BANNER + SPAM + REAL_WARN
+        log.write_bytes(body)
+        mtime = log.stat().st_mtime_ns
+        self.assertEqual(tm.squash_log_spam(self.run_dir), 0)
+        self.assertEqual(log.read_bytes(), body)
+        self.assertEqual(log.stat().st_mtime_ns, mtime)
+        self.assertEqual(sorted(p.name for p in self.logs.iterdir() if p.is_file()),
+                         ["madengine-x_node_1.out"])
+
+    def test_missing_slurm_output(self):
+        self.assertEqual(tm.squash_log_spam(self.run_dir / "nope"), 0)
+
+
+if __name__ == "__main__":
+    unittest.main()