diff --git a/.github/workflows/therock-rccl-ci-linux.yml b/.github/workflows/therock-rccl-ci-linux.yml index b28f2aa157ba..07ce91119210 100644 --- a/.github/workflows/therock-rccl-ci-linux.yml +++ b/.github/workflows/therock-rccl-ci-linux.yml @@ -274,6 +274,43 @@ jobs: notify_email: ${{ inputs.notify_email }} teams_webhook: ${{ inputs.notify_webhook }} + # Every other workload, on the weekly run only: test_scope == 'all' is what + # the Saturday cron in therock-rccl-ci.yml sets, and a dispatch can ask for + # it. The job above stays the nightly gate on its own. + # + # One matrix rather than a job per workload, so that max-parallel: 1 covers + # all of them: each wants an exclusive salloc on the same partition, and + # queue time counts against its own budget. `needs` the nightly job for + # ORDER ONLY -- `!cancelled()` keeps a failed nightly from skipping these. + # Nodes are per entry, not inputs.madengine_nodes: each workload's + # parallelism and memory headroom were validated at one scale. + therock-test-madengine-weekly: + name: "Test MADEngine workloads (weekly)" + if: ${{ !cancelled() && contains(fromJSON(needs.test-plan.outputs.suites || '[]'), 'madengine') && (needs.therock-build-linux.result == 'success' || needs.therock-build-linux.result == 'skipped') && inputs.amdgpu_families == 'gfx950-dcgpu' && inputs.test_scope == 'all' && (inputs.event_name == 'schedule' || inputs.event_name == 'workflow_dispatch') }} + needs: [test-plan, therock-build-linux, therock-test-madengine] + permissions: + contents: read + id-token: write + strategy: + # One workload failing says nothing about the others. + fail-fast: false + max-parallel: 1 + matrix: + include: + - workload: gpt-oss-120b-training + nodes: '2' + uses: ./.github/workflows/therock-rccl-test-madengine.yml + secrets: inherit + with: + amdgpu_families: ${{ inputs.amdgpu_families }} + artifact_group: ${{ inputs.artifact_group }} + test_runs_on: ruby-linux-slurm-scale-runner + artifact_run_id: ${{ inputs.artifact_run_id || github.run_id }} + workload: ${{ matrix.workload }} + nodes: ${{ matrix.nodes }} + notify_email: ${{ inputs.notify_email }} + teams_webhook: ${{ inputs.notify_webhook }} + therock-test-accl-profiler: name: "Test ACCL profiler decomposition (scheduled)" if: ${{ !cancelled() && contains(fromJSON(needs.test-plan.outputs.suites || '[]'), 'accl-profiler') }} diff --git a/.github/workflows/therock-rccl-test-madengine.yml b/.github/workflows/therock-rccl-test-madengine.yml index b4f090672351..d67c1e910cc4 100644 --- a/.github/workflows/therock-rccl-test-madengine.yml +++ b/.github/workflows/therock-rccl-test-madengine.yml @@ -12,6 +12,9 @@ on: artifact_run_id: type: string workload: + # A key of WORKLOAD_CONFIGS in test_madengine.py, which validates it. + # workflow_call inputs cannot be typed as choice, so the list of + # accepted values lives on the workflow_dispatch input below. type: string default: 'llama-3.1-70b-training' nodes: @@ -37,7 +40,10 @@ on: type: string workload: description: 'MADEngine workload to run' - type: string + type: choice + options: + - llama-3.1-70b-training + - gpt-oss-120b-training default: 'llama-3.1-70b-training' nodes: description: 'Number of SLURM nodes to allocate' @@ -69,8 +75,9 @@ jobs: RESULTS_DIR: "/apps/rccl-ci/madengine/perf" # run_attempt too: run_id is stable across re-runs, and the cleanup step # leaves the result files behind, so a re-run could parse a previous - # attempt's perf.csv. - WORK_DIR: "/apps/rccl-ci/madengine/workdir/${{ github.run_id }}-${{ github.run_attempt }}" + # attempt's perf.csv. The workload is in the path because the scheduled + # caller runs several of them from one workflow run, which shares both. + WORK_DIR: "/apps/rccl-ci/madengine/workdir/${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.workload }}" # Workspace-local, as in the JAX and PyTorch test workflows: the fetched # ROCm tree is only read on the head node, and /apps/cvs_tests/dist_new # belongs to the CVS suite, which this job may neither traverse nor wipe. diff --git a/projects/rccl/ci/scripts/test_madengine.py b/projects/rccl/ci/scripts/test_madengine.py index 38aecd2585e3..c2e052bcb285 100644 --- a/projects/rccl/ci/scripts/test_madengine.py +++ b/projects/rccl/ci/scripts/test_madengine.py @@ -80,7 +80,7 @@ MADENGINE_REPO = "https://github.com/ROCm/madengine.git" MADENGINE_REF = "00ee3e80c0a308599633caf74e5d91a72164a2fe" # develop, has #190 and #213 MAD_REPO = "https://github.com/ROCm/MAD.git" -MAD_REF = "7a9a25e742ef60e8deffb6e5fd9b559857d2018d" # mad-rccl, 2026-10-09 +MAD_REF = "7a9a25e742ef60e8deffb6e5fd9b559857d2018d" # mad-rccl, has MAD#271 MAD_BRANCH = "mad-rccl" WORKLOAD_CONFIGS = { @@ -111,25 +111,25 @@ "--device=/dev/infiniband --cap-add IPC_LOCK " "--ulimit memlock=-1 -v /sys:/sys:ro -v /run/udev:/run/udev:ro", }, - "llama-4-scout-training": { + "gpt-oss-120b-training": { "type": "training", - "model_repo": "primus_pyt_megatron_lm_train_llama-4-scout-17b-16e", - "model_repo_aliases": [ - "primus_pyt_megatron_lm_train_llama-4-scout-17b-16e_overlay", - "primus_pyt_megatron_lm_train_llama-4-scout-17b-16e_scaleout", - ], + "model_repo": "primus_pyt_megatron_lm_train_gpt-oss-120b", "base_image": "rocm/primus:v26.4", "gpu_target": "gfx950", "metric_key": "tokens_per_second_per_gpu", - "multiple_results": "perf_primus-megatron-Llama-4-Scout-17B-16E.csv", - "precisions": ["BF16"], # what MAD's run.sh trains on gfx950 - "reference_values": { - "2N": 2734, - "4N": 2337, + "multiple_results": "perf_primus-megatron-GPT-OSS-120B.csv", + "precisions": ["BF16", "FP8"], # what MAD's run.sh trains on gfx950 + # The shipped config profiles iterations 6-7, which at 2 nodes cost + # 3.2x (BF16) and 3.7x (FP8) a steady iteration and leave the next two + # elevated. Primus averages from iteration 3, so 15 iterations give 13 + # measured ones at ~65 s (BF16) and ~116 s (FP8) each. + "env_vars": { + "PRIMUS_TRAIN_ITERS": "15", + "PRIMUS_DISABLE_PROFILE": "1", }, "slurm_partition": "meta64", "gpus_per_node": 8, - "time_limit": "03:00:00", # one allocation, two runs, plus image staging + "time_limit": "04:00:00", # one allocation, two runs, plus image staging "docker_mounts": {"/dev/infiniband": "/dev/infiniband"}, "docker_run_options": "--privileged --group-add render --shm-size 64G " "--device=/dev/infiniband --cap-add IPC_LOCK " @@ -336,6 +336,58 @@ def _split_rccl_version(version: str) -> tuple[str, str, bool]: return m.group(1), (m.group(2) or "").lower(), m.group(3) == "+" +# RCCL prints this on EVERY call of a deprecated API name (src/misc/api_trace.cc), +# at WARN, so NCCL_DEBUG=WARN keeps it. GPT-OSS-120B's MoE all-to-all makes it +# 97% of each node log, which madengine keeps three copies of. +_LOG_SPAM_RE = re.compile(rb"NCCL WARN Please note that (\w+) is deprecated") + + +def squash_log_spam(run_dir: Path) -> int: + """Drop the repeats of _LOG_SPAM_RE lines from one phase's logs, in place. + + The first occurrence of each stays where it was, and one line at the end + of the file says how many repeats went. Files with no repeats are left + untouched. Returns the number of bytes saved. + """ + log_dir = run_dir / "slurm_output" + if not log_dir.is_dir(): + return 0 + saved = 0 + for path in sorted( + p for p in log_dir.rglob("*") + if p.suffix in (".out", ".err", ".log") and p.is_file() + ): + dropped: dict[bytes, int] = {} + tmp = path.with_name(path.name + ".squash") + try: + before = path.stat().st_size + with path.open("rb") as src, tmp.open("wb") as dst: + for line in src: + m = _LOG_SPAM_RE.search(line) + if m: + api = m.group(1) + if api in dropped: + dropped[api] += 1 + continue + dropped[api] = 0 + dst.write(line) + for api, count in dropped.items(): + if count: + dst.write( + b"[rccl-ci] dropped %d repeats of 'NCCL WARN Please note " + b"that %s is deprecated'\n" % (count, api) + ) + if not any(dropped.values()): + tmp.unlink() + continue + os.replace(tmp, path) + saved += before - path.stat().st_size + except OSError as exc: + log.warning("Could not squash %s: %s", path, exc) + tmp.unlink(missing_ok=True) + return saved + + def runtime_rccl_versions(work_dir: Path) -> list[str]: """Distinct ``RCCL version :`` banners across one phase's node logs.""" log_dir = work_dir / "slurm_output" @@ -595,7 +647,11 @@ def generate_manifest( if "," in ifname: nccl_env["NCCL_SOCKET_IFNAME"] = ifname.split(",")[0] - socket_ifname = nccl_env.get("NCCL_SOCKET_IFNAME", "") + # Last over everything below, socket interfaces included. + workload_env = workload_config.get("env_vars", {}) + socket_ifname = workload_env.get( + "NCCL_SOCKET_IFNAME", nccl_env.get("NCCL_SOCKET_IFNAME", "") + ) # HF_TOKEN is passed via MAD_SECRETS_HFTOKEN in the process environment # (set in the workflow). Do NOT write it into the manifest — the manifest @@ -616,8 +672,9 @@ def generate_manifest( "NCCL_IB_DISABLE": "0", "NCCL_TIMEOUT": "900", "IBV_SHOW_WARNINGS": "1", + **workload_env, } - if socket_ifname: + if socket_ifname and "GLOO_SOCKET_IFNAME" not in workload_env: docker_env_vars["GLOO_SOCKET_IFNAME"] = socket_ifname docker_mounts = dict(workload_config.get("docker_mounts", {})) @@ -1612,6 +1669,13 @@ def main() -> None: ) exit_code = max(allocation["exit_codes"].values(), default=1) + # Step 5a: Shrink the node logs before anything reads or uploads them. + for phase, run_dir in phases: + saved = squash_log_spam(run_dir) + if saved: + log.info("%s: dropped %.1f MB of repeated RCCL warnings from the node logs", + phase, saved / 1e6) + # Step 5b: Verify RCCL replacement. Only the candidate is meant to carry the # CI library — the baseline runs the image's bundled copy by design. rccl_verification_failed = False diff --git a/projects/rccl/ci/scripts/tests/test_log_squash.py b/projects/rccl/ci/scripts/tests/test_log_squash.py new file mode 100644 index 000000000000..c2cfe5f29869 --- /dev/null +++ b/projects/rccl/ci/scripts/tests/test_log_squash.py @@ -0,0 +1,74 @@ +"""Tests for squash_log_spam() in test_madengine.py. + +Run from projects/rccl/ci/scripts: python3 -m unittest discover -s tests +""" + +import os +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, os.fspath(Path(__file__).resolve().parent.parent)) +import test_madengine as tm + +SPAM = (b"[2026-10-12 21:11:21] node:985:985 [0] api_trace.cc:569 NCCL WARN Please note " + b"that ncclAllToAllv is deprecated, please use ncclAlltoAllv instead\n") +SPAM2 = SPAM.replace(b"ncclAllToAllv", b"ncclAllToAll").replace(b"ncclAlltoAllv", b"ncclAlltoAll") +BANNER = b"RCCL version : 2.32.3-develop:2b88119\n" +REAL_WARN = b"node:985:985 [0] NCCL WARN NET/IB : Got async event : port error\n" + + +class SquashLogSpam(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.run_dir = Path(self._tmp.name) + self.logs = self.run_dir / "slurm_output" + (self.logs / "model" / "node_0").mkdir(parents=True) + + def tearDown(self): + self._tmp.cleanup() + + def test_keeps_first_and_counts_the_rest(self): + log = self.logs / "madengine-x_node_0.out" + log.write_bytes(BANNER + SPAM * 5 + REAL_WARN + SPAM * 3 + b"tok/s 1500\n") + saved = tm.squash_log_spam(self.run_dir) + self.assertEqual( + log.read_bytes(), + BANNER + SPAM + REAL_WARN + b"tok/s 1500\n" + + b"[rccl-ci] dropped 7 repeats of 'NCCL WARN Please note that " + b"ncclAllToAllv is deprecated'\n", + ) + self.assertEqual(saved, 7 * len(SPAM) - len(log.read_bytes().splitlines(True)[-1])) + self.assertEqual(tm.runtime_rccl_versions(self.run_dir), ["2.32.3-develop:2b88119"]) + + def test_each_api_counted_separately(self): + log = self.logs / "model" / "node_0" / "stdout.out" + log.write_bytes(SPAM + SPAM2 + SPAM + SPAM2 + SPAM2) + tm.squash_log_spam(self.run_dir) + self.assertEqual( + log.read_bytes(), + SPAM + SPAM2 + + b"[rccl-ci] dropped 1 repeats of 'NCCL WARN Please note that " + b"ncclAllToAllv is deprecated'\n" + + b"[rccl-ci] dropped 2 repeats of 'NCCL WARN Please note that " + b"ncclAllToAll is deprecated'\n", + ) + + def test_file_without_repeats_is_untouched(self): + log = self.logs / "madengine-x_node_1.out" + body = BANNER + SPAM + REAL_WARN + log.write_bytes(body) + mtime = log.stat().st_mtime_ns + self.assertEqual(tm.squash_log_spam(self.run_dir), 0) + self.assertEqual(log.read_bytes(), body) + self.assertEqual(log.stat().st_mtime_ns, mtime) + self.assertEqual(sorted(p.name for p in self.logs.iterdir() if p.is_file()), + ["madengine-x_node_1.out"]) + + def test_missing_slurm_output(self): + self.assertEqual(tm.squash_log_spam(self.run_dir / "nope"), 0) + + +if __name__ == "__main__": + unittest.main()