Activate the static consumer-graph parser for test selection

.github/workflows/test_artifacts.yml

@@ -221,7 +221,7 @@ jobs:
       - name: "Determine test dependencies"
         id: test_deps
         # Reads the committed consumer graph directly (no fetch/configure); kept
-        # in sync by test_consumer_graph_drift.yml.
+        # in sync by the consumer-graph parser test in the Test build_tools job.
         shell: bash
         env:
           CHANGED_PROJECTS: ${{ inputs.changed_projects }}

CMakeLists.txt

@@ -559,6 +559,3 @@ therock_subproject_merge_compile_commands()
 
 # Write artifact→subprojects manifest for Python scripts to use
 therock_write_subproject_manifest()
-
-include(therock_emit_consumer_graph)
-therock_emit_consumer_graph("${CMAKE_BINARY_DIR}/therock_consumer_graph.json")

build_tools/_therock_utils/cmake_consumer_graph.py

@@ -106,6 +106,7 @@
     "DEFAULT_GPU_TARGETS",
     "FPRINT_FILE_GLOBS",
     "INSTALL_OPTIONAL_COMPONENTS",
+    "HARDENING_DEFERRED",
 }
 _DECLARATION_KEYWORDS = (
     _DECLARATION_FLAGS | _DECLARATION_ONE_VALUE_ARGS | _DECLARATION_MULTI_VALUE_ARGS
@@ -934,14 +935,17 @@ def _resolve_source_dir_section(
         return {value for value in expansion.values if value}
 
 
-def _declaration_sections(tokens: list[Token]) -> dict[str, list[Token]]:
-    """Group a declaration's argument tokens by their keyword (BUILD_DEPS, ...)."""
+def _declaration_sections(
+    tokens: list[Token],
+    keywords: frozenset[str] | set[str] = _DECLARATION_KEYWORDS,
+) -> dict[str, list[Token]]:
+    """Group an argument list's tokens by their keyword (BUILD_DEPS, ...)."""
     sections: dict[str, list[Token]] = {}
     current_keyword: str | None = None
     for token in tokens:
         if token.kind == "COMMENT":
             continue
-        if token.value in _DECLARATION_KEYWORDS:
+        if token.value in keywords:
             current_keyword = token.value
             sections.setdefault(current_keyword, [])
         elif current_keyword is not None:

build_tools/generate_consumer_graph.py

@@ -2,17 +2,22 @@
 # Copyright Advanced Micro Devices, Inc.
 # SPDX-License-Identifier: MIT
 
-"""Generate test_tools/therock_consumer_graph.json from the super-project CMake.
+"""Generate the committed consumer-graph metadata from the super-project CMake.
 
 Statically parses the tracked CMake files (no configure, no submodules) and writes
-the reverse-dependency consumer graph. See cmake_consumer_graph.py for the full
+two committed files:
+
+* ``test_tools/therock_consumer_graph.json`` — the reverse-dependency consumer graph.
+* ``test_tools/therock_subtree_map.json`` — each external source subtree mapped to the
+  graph key(s) built from it.
+
+This parser generates both files. See cmake_consumer_graph.py for the full
 description and limitations.
 
-``--check`` analyzes in memory and diffs the result against the committed graph
-without writing, exiting non-zero on any difference. While the committed graph is
-still emitted by CMake, this parser is a conservative *superset* of it, so
-``--check`` reports that expected superset delta; it is intentionally not wired
-into any CI gate.
+``--check`` analyzes in memory and diffs the result against ALL committed files
+without writing, exiting non-zero on any difference. It is the drift gate: the
+``Test build_tools`` pytest step enforces the same invariant, so a stale committed
+file fails CI.
 """
 
 import argparse
@@ -29,14 +34,17 @@
 THIS_SCRIPT_DIR = Path(__file__).resolve().parent
 THEROCK_DIR = THIS_SCRIPT_DIR.parent
 CONSUMER_GRAPH_PATH = THEROCK_DIR / "test_tools" / "therock_consumer_graph.json"
+SUBTREE_MAP_PATH = THEROCK_DIR / "test_tools" / "therock_subtree_map.json"
+# Informational artifact→source-subtree map; committed and drift-gated but not yet
+# consumed by selection (see module docstring).
 
 
 def _write_json(path: Path, data: object) -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
     path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
 
 
-def _print_delta(comparison) -> None:
+def _print_graph_delta(comparison) -> None:
     """Print a concise summary of how the generated graph differs from committed."""
     for label, items in (
         ("generated-only nodes", comparison.generated_only_nodes),
@@ -49,27 +57,52 @@ def _print_delta(comparison) -> None:
             print(f"    {item}")
 
 
+def _map_delta(generated: dict, committed: dict, noun: str) -> list[str]:
+    """Return human-readable difference lines between two ``str -> list`` maps."""
+    lines: list[str] = []
+    generated_keys, committed_keys = set(generated), set(committed)
+    for key in sorted(generated_keys - committed_keys):
+        lines.append(f"    generated-only {noun}: {key} -> {generated[key]}")
+    for key in sorted(committed_keys - generated_keys):
+        lines.append(f"    committed-only {noun}: {key} -> {committed[key]}")
+    for key in sorted(generated_keys & committed_keys):
+        if generated[key] != committed[key]:
+            lines.append(
+                f"    {noun} {key}: generated {generated[key]} != committed {committed[key]}"
+            )
+    return lines
+
+
+def _read_committed_map(path: Path) -> dict:
+    """Committed JSON map, or empty if missing so --check reports full drift."""
+    if not path.exists():
+        return {}
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(
-        description="Generate test_tools/therock_consumer_graph.json from the super-project CMake."
+        description="Generate the committed consumer-graph metadata from the super-project CMake."
     )
     parser.add_argument(
         "--check",
         action="store_true",
         help=(
-            "Diff the generated graph against the committed one without writing; "
-            "exit non-zero on any difference. The parser is currently a superset "
-            "of the committed graph, so this reports that expected delta and is not "
-            "wired into a CI gate."
+            "Diff the generated graph and subtree map against the "
+            "committed files without writing; exit non-zero on any difference. This "
+            "is the drift gate."
         ),
     )
     args = parser.parse_args(argv)
     result = RepositoryAnalyzer(THEROCK_DIR).analyze()
     graph = result.build_consumer_graph()
+    subtree_map = result.build_subtree_map()
 
     if args.check:
-        committed = load_consumer_graph(CONSUMER_GRAPH_PATH)
-        comparison = compare_graphs(graph, committed)
+        differs = False
+
+        committed_graph = load_consumer_graph(CONSUMER_GRAPH_PATH)
+        comparison = compare_graphs(graph, committed_graph)
         if (
             comparison.generated_only_nodes
             or comparison.reference_only_nodes
@@ -80,15 +113,30 @@ def main(argv: list[str] | None = None) -> int:
                 "Generated consumer graph differs from "
                 f"{CONSUMER_GRAPH_PATH.relative_to(THEROCK_DIR).as_posix()}:"
             )
-            _print_delta(comparison)
+            _print_graph_delta(comparison)
+            differs = True
+
+        committed_subtree = _read_committed_map(SUBTREE_MAP_PATH)
+        subtree_delta = _map_delta(subtree_map, committed_subtree, "subtree")
+        if subtree_delta:
+            print(
+                "Generated subtree map differs from "
+                f"{SUBTREE_MAP_PATH.relative_to(THEROCK_DIR).as_posix()}:"
+            )
+            print("\n".join(subtree_delta))
+            differs = True
+
+        if differs:
             return 1
-        print("Generated consumer graph matches the committed graph.")
+        print("Generated consumer graph and subtree map match the committed files.")
         return 0
 
     _write_json(CONSUMER_GRAPH_PATH, graph)
+    _write_json(SUBTREE_MAP_PATH, subtree_map)
     print(
-        f"Wrote consumer graph for {len(graph)} subprojects to "
-        f"{CONSUMER_GRAPH_PATH.relative_to(THEROCK_DIR).as_posix()}"
+        f"Wrote consumer graph ({len(graph)} subprojects) and subtree map "
+        f"({len(subtree_map)} subtrees) to "
+        f"{CONSUMER_GRAPH_PATH.parent.relative_to(THEROCK_DIR).as_posix()}/"
     )
     return 0
 

build_tools/github_actions/configure_external_repo_ci.py

@@ -101,8 +101,9 @@
 # to one of these paths yields no matched subtree -> empty changed_projects ->
 # TheRock falls back to building and testing everything. Each entry MUST have a
 # corresponding mapping in TheRock (build-topology alias + the test selector's
-# _EXTERNAL_SUBTREE_ALIASES), so keep this list in lock-step with TheRock when
-# adding directories.
+# generated subtree map or _SUBTREE_ALIAS_OVERRIDES); the test selector hard-fails
+# on an unmapped shared/*, dnn-providers/*, or emulation/* path, so keep this list
+# in lock-step with TheRock when adding directories.
 CI_RELEVANT_NON_SUBTREE_PREFIXES = {
     # hipCCL is a monorepo-native project (projects/hipccl/hipccl3, the unified
     # rocPRIM+hipCUB+rocThrust superbuild); it has no upstream repo to subtree-

build_tools/github_actions/configure_multi_arch_ci.py

@@ -168,7 +168,10 @@ def _resolve_skipped_stages(build_stages: list[str]) -> list[str]:
 
 # Maps build stages to the test labels that can run with artifacts from those
 # stages. Used to auto-filter tests when build_stages narrows the build graph.
-# Test labels not listed here require a full build.
+# Test labels not listed here require a full build. Most labels have a
+# fetch_test_configurations.test_matrix entry; kfdtest and composable-kernel are
+# real artifacts tested through other paths and have none (see
+# label_consistency_test, which guards against a genuinely unknown label).
 STAGE_TO_TEST_LABELS: dict[str, list[str]] = {
     "compiler-runtime": ["kfdtest"],
     "runtime-tests": ["hip-tests", "rocrtst"],