test(rccl): guard Makefile dep tracking stays on -MM

projects/rccl/src/test_makefile_depflags.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""Guard that Makefile header-dep tracking uses -MM (NCCL PR #1806).
+
+NCCL 2.30.7 switched include tracking from -M to -MM so system headers are
+omitted from generated .d files. src/device/Makefile (and the diagnostics
+copy) then realpath-filters those paths down to the tree; listing libc / HIP
+/ compiler headers made that loop excessively slow on some filesystems.
+
+This is a source scan, not a compile. Nothing in RCCL CI runs ctest, so the
+copy that gates is test/host/run_host_tests.sh `guards` (folded into `run`).
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+# Longest-match first so -MMD is not read as -MM or -M, and -MD / -MM are
+# not read as -M. -march and friends fail the trailing boundary.
+DEP_FLAG_RE = re.compile(r"(?<!\S)(-MMD|-MM|-MD|-MF|-MG|-MP|-MT|-MQ|-M)(?=\s|$)")
+
+# -MM / -MMD omit system headers. -M / -MD include them (the #1806 bug).
+USER_ONLY = frozenset({"-MM", "-MMD"})
+SYSTEM_INCLUSIVE = frozenset({"-M", "-MD"})
+
+# Files that carry the NCCL #1806 recipes (plus the RCCL diagnostics copy).
+REQUIRED = (
+    ("Makefile", ("d_tmp",), 2),
+    (os.path.join("device", "Makefile"), ("DEPENDS.cu", "DEPENDS.cc"), 2),
+    (os.path.join("diagnostics", "device", "Makefile"), ("DEPENDS.cu",), 1),
+)
+
+
+def _join_continued_lines(text):
+    """Collapse makefile / recipe backslash continuations into logical lines."""
+    logical = []
+    buf = []
+    for raw in text.splitlines():
+        stripped = raw.rstrip()
+        if stripped.endswith("\\"):
+            buf.append(stripped[:-1])
+            continue
+        buf.append(stripped)
+        logical.append("".join(buf))
+        buf = []
+    if buf:
+        logical.append("".join(buf))
+    return logical
+
+
+def _dep_flags(line):
+    return tuple(DEP_FLAG_RE.findall(line))
+
+
+def _is_depgen_line(line):
+    """True for a compiler dep-tracking recipe, not a later sed of .d.tmp."""
+    if not line or line.lstrip().startswith("#"):
+        return False
+    flags = _dep_flags(line)
+    if not flags:
+        return False
+    return (
+        ".d.tmp" in line
+        or re.search(r"DEPENDS\.(cu|cc)\s*=", line) is not None
+    )
+
+
+def depgen_sites(text):
+    """Return (lineno, line, flags) for each header-dep recipe in `text`."""
+    sites = []
+    for i, line in enumerate(_join_continued_lines(text), start=1):
+        if _is_depgen_line(line):
+            sites.append((i, line, _dep_flags(line)))
+    return sites
+
+
+def site_errors(flags):
+    """Human-readable problems for one recipe's -M* flags, or []."""
+    have = set(flags)
+    errors = []
+    bad = have & SYSTEM_INCLUSIVE
+    if bad:
+        errors.append("uses system-header flag %s (want -MM or -MMD)" %
+                      ", ".join(sorted(bad)))
+    if not (have & USER_ONLY):
+        errors.append("missing -MM / -MMD")
+    return errors
+
+
+class TestMakefileDepFlags(unittest.TestCase):
+    def test_parser_accepts_dash_mm(self):
+        sites = depgen_sites(
+            "\t@$(CXX) -I. $(CXXFLAGS) -MM $< > $(@:%.o=%.d.tmp)\n"
+        )
+        self.assertEqual(len(sites), 1)
+        self.assertEqual(sites[0][2], ("-MM",))
+        self.assertEqual(site_errors(sites[0][2]), [])
+
+    def test_parser_rejects_dash_m(self):
+        sites = depgen_sites(
+            "\t@$(CXX) -I. $(CXXFLAGS) -M $< > $(@:%.o=%.d.tmp)\n"
+        )
+        self.assertEqual(len(sites), 1)
+        self.assertEqual(sites[0][2], ("-M",))
+        self.assertTrue(site_errors(sites[0][2]))
+
+    def test_parser_rejects_dash_md(self):
+        sites = depgen_sites("DEPENDS.cu = $(NVCC) $(NVCUFLAGS) -MD -dc $1\n")
+        self.assertEqual(len(sites), 1)
+        self.assertTrue(site_errors(sites[0][2]))
+
+    def test_parser_ignores_sed_of_d_tmp(self):
+        text = (
+            "\t@sed \"0,/^.*:/s//x:/\" $(@:%.o=%.d.tmp) > $(@:%.o=%.d)\n"
+            "\t$(CXX) -c $< -o $@\n"
+        )
+        self.assertEqual(depgen_sites(text), [])
+
+    def test_parser_does_not_treat_march_as_dep_flag(self):
+        sites = depgen_sites(
+            "\t$(CXX) -march=native -MM $< > foo.d.tmp\n"
+        )
+        self.assertEqual(len(sites), 1)
+        self.assertEqual(sites[0][2], ("-MM",))
+
+    def test_src_makefiles_use_user_only_deps(self):
+        failures = []
+        for rel, kinds, min_sites in REQUIRED:
+            path = os.path.join(HERE, rel)
+            self.assertTrue(os.path.isfile(path), "missing %s" % path)
+            with open(path) as f:
+                text = f.read()
+            sites = depgen_sites(text)
+            if len(sites) < min_sites:
+                failures.append(
+                    "%s: expected at least %d depgen site(s), found %d"
+                    % (rel, min_sites, len(sites))
+                )
+            for kind in kinds:
+                if kind == "d_tmp":
+                    if not any(".d.tmp" in line for _, line, _ in sites):
+                        failures.append("%s: no .d.tmp -MM recipe" % rel)
+                    continue
+                if not any(kind in line for _, line, _ in sites):
+                    failures.append("%s: no %s recipe" % (rel, kind))
+            for lineno, line, flags in sites:
+                errs = site_errors(flags)
+                if errs:
+                    failures.append(
+                        "%s:%d: %s\n    %s"
+                        % (rel, lineno, "; ".join(errs), line.strip())
+                    )
+        self.assertFalse(failures, "NCCL PR #1806 -MM guard failed:\n" +
+                         "\n".join(failures))
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

projects/rccl/test/CMakeLists.txt

@@ -39,6 +39,15 @@ if(BUILD_TESTS)
             "${PROJECT_SOURCE_DIR}/src/device/ce_reduce/test_generate_ce_reduce.py"
   )
 
+  # CPU-only Makefile scan: NCCL PR #1806 (-M -> -MM) must stay in force.
+  # Registered here for local `ctest` only; CI gates via the `guards` phase of
+  # test/host/run_host_tests.sh.
+  add_test(
+    NAME rccl-makefile-depflags
+    COMMAND "${Python3_EXECUTABLE}"
+            "${PROJECT_SOURCE_DIR}/src/test_makefile_depflags.py"
+  )
+
   # The kernel-count leak guards and the __hip_atomic_* poison probe
   # (src/include/test_poison_hip_atomics.py) are intentionally NOT registered as
   # ctest cases here: nothing in RCCL CI runs `ctest`, so an add_test() would

projects/rccl/test/host/run_host_tests.sh

@@ -23,7 +23,8 @@
 #   configure       configure test/host
 #   build           build all host binaries (default target)
 #   guards          device-table unittest and kernel-count pytest plus
-#                   src/include/test_poison_hip_atomics.py
+#                   src/include/test_poison_hip_atomics.py and
+#                   src/test_makefile_depflags.py
 #   run             run the suite (timestamped log + JUnit XML). Always emits
 #                   llvm source-based coverage profiles (*.profraw) into
 #                   <BUILD_DIR>/coverage (requires the host tests to be built
@@ -224,6 +225,15 @@ do_poison_hip_atomics() {
   python3 "$RCCL_ROOT/src/include/test_poison_hip_atomics.py"
 }
 
+# NCCL PR #1806: Makefile header tracking must keep using -MM so system
+# headers never enter the device DEPENDS realpath loop. Plain unittest,
+# python3 only. Also registered with add_test() in test/CMakeLists.txt;
+# CI still gates via this phase, not ctest.
+do_makefile_depflags() {
+  echo "==> Makefile -MM depflags (unittest: src/test_makefile_depflags.py)"
+  python3 "$RCCL_ROOT/src/test_makefile_depflags.py" -v
+}
+
 # Run the device-table generator guard. It is plain unittest and needs only
 # python3. The suite is also registered with add_test() in test/CMakeLists.txt,
 # but nothing in RCCL CI runs `ctest`, so that registration never gates. Running
@@ -247,13 +257,15 @@ do_kernel_count_guards() {
   "$venv/bin/python" -m pytest "$gd/tests" -v
 }
 
-# All CPU-only guards: the device-table unittest, the kernel-count pytest suite,
-# then the __hip_atomic_* poison compile probe. Collected with `|| rc=1` rather
-# than run back to back so that under `set -e` (line 53) an early failure still
-# leaves the later guards running and reported, instead of aborting the phase at
-# the first one. Same idiom as do_host_tests above.
+# All CPU-only guards: makefile -MM scan, the device-table unittest, the
+# kernel-count pytest suite, then the __hip_atomic_* poison compile probe.
+# Collected with `|| rc=1` rather than run back to back so that under `set -e`
+# (line 53) an early failure still leaves the later guards running and
+# reported, instead of aborting the phase at the first one. Same idiom as
+# do_host_tests above.
 do_guards() {
   local rc=0
+  do_makefile_depflags || rc=1
   do_device_table_guards || rc=1
   do_kernel_count_guards || rc=1
   do_poison_hip_atomics || rc=1