From c0a9d58619f496a58dbf150f932d42f58f37390f Mon Sep 17 00:00:00 2001 From: Sahil Faizal Date: Fri, 2 Oct 2026 17:53:59 -0700 Subject: [PATCH 1/8] ci: enable host-ASan tests for library and system bumps --- .../github_actions/amdgpu_family_matrix.py | 2 + build_tools/github_actions/bump_automation.py | 8 ++-- .../github_actions/configure_multi_arch_ci.py | 21 ++++++--- .../fetch_test_configurations.py | 9 ++++ .../tests/configure_multi_arch_ci_test.py | 45 +++++++++++++++++++ .../tests/fetch_test_configurations_test.py | 37 ++++++++++++++- docs/development/ci_behavior_manipulation.md | 8 ++++ 7 files changed, 120 insertions(+), 10 deletions(-) diff --git a/build_tools/github_actions/amdgpu_family_matrix.py b/build_tools/github_actions/amdgpu_family_matrix.py index 6eb594bbcc5..527b1663cbe 100644 --- a/build_tools/github_actions/amdgpu_family_matrix.py +++ b/build_tools/github_actions/amdgpu_family_matrix.py @@ -261,6 +261,7 @@ def select_build_runner(platform: str, build_variant: str, size: str = "large") ], # TODO(#3433): Remove sandbox label once ASAN tests are passing "test-runs-on-sandbox": "linux-gfx942-1gpu-asan-sandbox-rocm", + "test-runs-on-host-asan": "linux-gfx942-8gpu-asan-sandbox-rocm", "test-runs-on-multi-gpu": "linux-gfx942-8gpu-ossci-rocm", "test-runs-on-multi-gpu-labels": [ {"label": "linux-gfx942-8gpu-ossci-rocm", "count": 10}, @@ -457,6 +458,7 @@ def select_build_runner(platform: str, build_variant: str, size: str = "large") "linux": { "test-runs-on": "linux-gfx950-1gpu-ccs-ossci-rocm", "test-runs-on-sandbox": "linux-gfx950-1gpu-asan-sandbox-rocm", + "test-runs-on-host-asan": "linux-gfx950-8gpu-asan-sandbox-rocm", "test-runs-on-multi-gpu": "linux-gfx950-8gpu-ccs-ossci-rocm", "family": "gfx950-dcgpu", "fetch-gfx-targets": ["gfx950"], diff --git a/build_tools/github_actions/bump_automation.py b/build_tools/github_actions/bump_automation.py index e1fad181050..0a565584661 100644 --- a/build_tools/github_actions/bump_automation.py +++ b/build_tools/github_actions/bump_automation.py @@ -61,10 +61,10 @@ "bot_author": "systems-assistant[bot]", # Changes to rocm-systems should run the full matrix of CI jobs: # * Build for all gfx archs - # * Build for all variants (asan) + # * Build release and host-ASan variants # * All builds and tests (including downstream rocm-libraries jobs) # * gfx950-dcgpu/gfx125X-dcgpu tests (limited hardware, label-gated) - "labels": [*COMMON_CI_LABELS, "ci:asan", "gfx950-dcgpu", "gfx125X-dcgpu"], + "labels": [*COMMON_CI_LABELS, "ci:host-asan", "gfx950-dcgpu", "gfx125X-dcgpu"], }, "rocm-libraries": { "repo": "ROCm/rocm-libraries", @@ -79,10 +79,10 @@ "bot_author": "assistant-librarian[bot]", # Changes to rocm-libraries should run the full matrix of CI jobs: # * Build for all gfx archs - # * Build for all variants (asan) + # * Build release and host-ASan variants # * All rocm-libraries tests # * gfx950-dcgpu/gfx125X-dcgpu tests (limited hardware, label-gated) - "labels": [*COMMON_CI_LABELS, "ci:asan", "gfx950-dcgpu", "gfx125X-dcgpu"], + "labels": [*COMMON_CI_LABELS, "ci:host-asan", "gfx950-dcgpu", "gfx125X-dcgpu"], }, "debug-tools/rocgdb/source": { "repo": "ROCm/rocgdb", diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index e59995cc043..c6ad043d26c 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -1382,16 +1382,27 @@ def _expand_build_config_for_platform( # TODO(#3433): Remove once ASAN tests pass and test_rocm.action is plumbed. if build_variant.startswith("host-asan"): - # Run host-asan tests only on nightly (schedule or workflow_dispatch) - # due to limited ASAN runner capacity and stability concerns. - if not (ci_inputs.is_schedule or ci_inputs.is_workflow_dispatch): + # Library and system submodule bumps exercise the host-ASan sandbox + # in addition to scheduled and manually dispatched runs. + has_host_asan_bump = bool( + {"rocm-libraries", "rocm-systems"} + & set(git_context.changed_files or []) + & set(git_context.submodule_paths or []) + ) + if not ( + ci_inputs.is_schedule + or ci_inputs.is_workflow_dispatch + or has_host_asan_bump + ): test_runs_on = "" print( - f" {family_name}: host-asan tests only run on nightly, " + f" {family_name}: host-asan tests require nightly or a library/system bump, " f"disabling tests" ) elif "test-runs-on-sandbox" in platform_info: - test_runs_on = platform_info["test-runs-on-sandbox"] + test_runs_on = platform_info.get( + "test-runs-on-host-asan", platform_info["test-runs-on-sandbox"] + ) print( f" {family_name}: using host-asan sandbox runner: {test_runs_on}" ) diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index 6a7ac0f69b0..905ab148d8b 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -1225,6 +1225,15 @@ def run(): "test-runs-on-multi-gpu", "" ) test_runs_on_sandbox = platform_info.get("test-runs-on-sandbox", "") + if build_variant.startswith("host-asan"): + test_runs_on_sandbox = platform_info.get( + "test-runs-on-host-asan", test_runs_on_sandbox + ) + if platform_info.get("test-runs-on-host-asan"): + # The host-ASan sandbox has eight GPUs and can also run + # multi-GPU components without using the production pool. + test_runs_on_multi_gpu_default = test_runs_on_sandbox + test_runs_on_multi_gpu_labels = None logging.info(f"Selecting projects: {projects_to_test}") diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index 07cc05379b4..2d48d6e006f 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -20,6 +20,7 @@ sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) import configure_multi_arch_ci as cm +from bump_automation import SUBMODULE_CONFIG from amdgpu_family_matrix import get_all_families_for_trigger_types from configure_multi_arch_ci_summary import format_summary from workflow_utils import WORKFLOWS_DIR @@ -1774,6 +1775,50 @@ def test_asan_runner_selection(self): entry = result.linux.per_family_info[0] self.assertEqual(entry["test-runs-on"], "") + def test_host_asan_submodule_bump_runners(self): + targets = cm.TargetSelection(linux_families=["gfx94x", "gfx950"]) + for path in ( + "rocm-libraries", + "rocm-systems", + "compiler/amd-llvm", + "README.md", + ): + for event in ("pull_request", "push"): + for variant in ("asan", "host-asan-debug"): + with self.subTest(path=path, event=event, variant=variant): + result = cm.expand_build_configs( + ci_inputs=self._inputs( + event_name=event, + build_variant=variant, + pr_labels=SUBMODULE_CONFIG.get(path, {}).get( + "labels", ["ci:host-asan"] + ), + ), + git_context=cm.GitContext( + changed_files=[path], + submodule_paths=[ + "rocm-libraries", + "rocm-systems", + "compiler/amd-llvm", + ], + ), + targets=targets, + jobs=_jobs(), + ) + runners = [ + entry["test-runs-on"] + for entry in result.linux.per_family_info + ] + expected = ( + [ + "linux-gfx942-8gpu-asan-sandbox-rocm", + "linux-gfx950-8gpu-asan-sandbox-rocm", + ] + if path in ("rocm-libraries", "rocm-systems") + else ["", ""] + ) + self.assertEqual(runners, expected) + def test_asan_debug_uses_sandbox_runner(self): """asan-debug variant uses sandbox runner like asan.""" targets = cm.TargetSelection(linux_families=["gfx94x"]) diff --git a/build_tools/github_actions/tests/fetch_test_configurations_test.py b/build_tools/github_actions/tests/fetch_test_configurations_test.py index d34ae067357..cd4946cc630 100644 --- a/build_tools/github_actions/tests/fetch_test_configurations_test.py +++ b/build_tools/github_actions/tests/fetch_test_configurations_test.py @@ -727,6 +727,7 @@ def fake_get_all_families(_): "linux": { "test-runs-on": "linux-gfx942-prod", "test-runs-on-sandbox": "linux-sandbox-runner", + "test-runs-on-host-asan": "linux-host-asan-runner", } } } @@ -739,7 +740,41 @@ def fake_get_all_families(_): components = self._get_components() hipblas = next(j for j in components if j["job_name"] == "hipblas") - self.assertEqual(hipblas["test_runner"], "linux-sandbox-runner") + self.assertEqual( + hipblas["test_runner"], + ( + "linux-host-asan-runner" + if build_variant.startswith("host-asan") + else "linux-sandbox-runner" + ), + ) + + def test_host_asan_multi_gpu_uses_sandbox_runner(self): + os.environ["BUILD_VARIANT"] = "host-asan" + os.environ["PROJECTS_TO_TEST"] = "rccl" + os.environ["TEST_LABELS"] = json.dumps(["ci:run-multi-gpu"]) + + def fake_get_all_families(_): + return { + "gfx94x": { + "linux": { + "test-runs-on-host-asan": "linux-gfx942-8gpu-asan-sandbox-rocm", + "test-runs-on-multi-gpu": "production-default", + "test-runs-on-multi-gpu-labels": [ + {"label": "production-weighted", "count": 1} + ], + } + } + } + + fetch_test_configurations.get_all_families_for_trigger_types = ( + fake_get_all_families + ) + fetch_test_configurations.run() + rccl = next(j for j in self._get_components() if j["job_name"] == "rccl") + self.assertEqual( + rccl["multi_gpu_runner"], "linux-gfx942-8gpu-asan-sandbox-rocm" + ) def test_release_build_uses_count_runner(self): """Release builds should use count-based runner labels, not sandbox.""" diff --git a/docs/development/ci_behavior_manipulation.md b/docs/development/ci_behavior_manipulation.md index 87c81e810a2..a7b2f6f5ff8 100644 --- a/docs/development/ci_behavior_manipulation.md +++ b/docs/development/ci_behavior_manipulation.md @@ -2,6 +2,14 @@ **Multi-Arch CI** ([`multi_arch_ci.yml`](https://github.com/ROCm/TheRock/actions/workflows/multi_arch_ci.yml)) is configured by [`configure_multi_arch_ci.py`](../../build_tools/github_actions/configure_multi_arch_ci.py) and reads GPU family definitions from [`amdgpu_family_matrix.py`](../../build_tools/github_actions/amdgpu_family_matrix.py). +Host-ASan tests run for `rocm-libraries` and `rocm-systems` submodule bumps +when ASAN CI is enabled (the bump automation adds `ci:host-asan`). For `gfx94x` +and `gfx950`, host-ASan sanity checks and component tests use +`linux-gfx942-8gpu-asan-sandbox-rocm` and +`linux-gfx950-8gpu-asan-sandbox-rocm`, respectively. Scheduled and manually +dispatched host-ASan tests use the same pools. Other PRs retain the existing +host-ASan test restriction, and full ASan retains its nightly/manual restriction. + ## Trigger behavior The CI pipelines test a growing set of GPU targets depending on trigger type/frequency: From 4cec0daee473857f350932fc85c10efc9bdb6269 Mon Sep 17 00:00:00 2001 From: Sahil Faizal Date: Fri, 2 Oct 2026 18:04:18 -0700 Subject: [PATCH 2/8] ci: test LLVM submodule bumps with host-ASan debug --- .../github_actions/configure_multi_arch_ci.py | 19 ++++++++-- .../tests/configure_multi_arch_ci_test.py | 35 ++++++++++++++++++- docs/development/ci_behavior_manipulation.md | 7 +++- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index c6ad043d26c..933397e782e 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -505,6 +505,14 @@ def has_submodule_changes(self) -> bool | None: return None return bool(set(self.submodule_paths) & set(self.changed_files)) + @property + def has_llvm_submodule_changes(self) -> bool: + return bool( + {"compiler/amd-llvm"} + & set(self.changed_files or []) + & set(self.submodule_paths or []) + ) + def log(self) -> None: """Log git context for CI diagnostics.""" if self.changed_files is None: @@ -1382,10 +1390,10 @@ def _expand_build_config_for_platform( # TODO(#3433): Remove once ASAN tests pass and test_rocm.action is plumbed. if build_variant.startswith("host-asan"): - # Library and system submodule bumps exercise the host-ASan sandbox + # Library, system, and LLVM bumps exercise the host-ASan sandbox # in addition to scheduled and manually dispatched runs. has_host_asan_bump = bool( - {"rocm-libraries", "rocm-systems"} + {"rocm-libraries", "rocm-systems", "compiler/amd-llvm"} & set(git_context.changed_files or []) & set(git_context.submodule_paths or []) ) @@ -1396,7 +1404,7 @@ def _expand_build_config_for_platform( ): test_runs_on = "" print( - f" {family_name}: host-asan tests require nightly or a library/system bump, " + f" {family_name}: host-asan tests require nightly or a library/system/LLVM bump, " f"disabling tests" ) elif "test-runs-on-sandbox" in platform_info: @@ -1687,6 +1695,11 @@ def expand_build_configs( build_variant = "host-asan" print(" Using host-asan variant (push/pull_request default)") + # LLVM bump diagnostics need source locations from the debug preset. + if build_variant == "host-asan" and git_context.has_llvm_submodule_changes: + build_variant = "host-asan-debug" + print(" Using host-asan-debug variant for LLVM submodule bump") + linux_config: BuildConfig | None = None windows_config: BuildConfig | None = None diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index 2d48d6e006f..67b57cb9cfa 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -1781,6 +1781,7 @@ def test_host_asan_submodule_bump_runners(self): "rocm-libraries", "rocm-systems", "compiler/amd-llvm", + "compiler/hipify", "README.md", ): for event in ("pull_request", "push"): @@ -1800,6 +1801,7 @@ def test_host_asan_submodule_bump_runners(self): "rocm-libraries", "rocm-systems", "compiler/amd-llvm", + "compiler/hipify", ], ), targets=targets, @@ -1814,10 +1816,41 @@ def test_host_asan_submodule_bump_runners(self): "linux-gfx942-8gpu-asan-sandbox-rocm", "linux-gfx950-8gpu-asan-sandbox-rocm", ] - if path in ("rocm-libraries", "rocm-systems") + if path + in ("rocm-libraries", "rocm-systems", "compiler/amd-llvm") else ["", ""] ) self.assertEqual(runners, expected) + expected_variant = ( + "host-asan-debug" + if path == "compiler/amd-llvm" + or variant == "host-asan-debug" + else "host-asan" + ) + self.assertEqual( + result.linux.build_variant_label, expected_variant + ) + self.assertEqual( + result.linux.build_variant_cmake_preset, + f"linux-release-{expected_variant}", + ) + + def test_llvm_bump_preserves_explicit_full_asan(self): + result = cm.expand_build_configs( + ci_inputs=self._inputs( + event_name="pull_request", build_variant="asan", pr_labels=["ci:asan"] + ), + git_context=cm.GitContext( + changed_files=["compiler/amd-llvm"], + submodule_paths=["compiler/amd-llvm"], + ), + targets=cm.TargetSelection(linux_families=["gfx94x", "gfx950"]), + jobs=_jobs(), + ) + self.assertEqual(result.linux.build_variant_label, "asan") + self.assertEqual( + [e["test-runs-on"] for e in result.linux.per_family_info], ["", ""] + ) def test_asan_debug_uses_sandbox_runner(self): """asan-debug variant uses sandbox runner like asan.""" diff --git a/docs/development/ci_behavior_manipulation.md b/docs/development/ci_behavior_manipulation.md index a7b2f6f5ff8..5fd75da6680 100644 --- a/docs/development/ci_behavior_manipulation.md +++ b/docs/development/ci_behavior_manipulation.md @@ -7,7 +7,12 @@ when ASAN CI is enabled (the bump automation adds `ci:host-asan`). For `gfx94x` and `gfx950`, host-ASan sanity checks and component tests use `linux-gfx942-8gpu-asan-sandbox-rocm` and `linux-gfx950-8gpu-asan-sandbox-rocm`, respectively. Scheduled and manually -dispatched host-ASan tests use the same pools. Other PRs retain the existing +dispatched host-ASan tests use the same pools. LLVM submodule bump PRs +(`compiler/amd-llvm`) can opt in with `ci:host-asan`; they select +`host-asan-debug` (`linux-release-host-asan-debug`) and enable sandbox tests. +Add `ci:gfx950-dcgpu` to include gfx950 alongside the default gfx94x coverage, +or use `ci:run-all-archs`. An explicit `ci:asan` label still selects full ASan. +The separate Compiler Daily Bump workflow is unchanged. Other PRs retain the existing host-ASan test restriction, and full ASan retains its nightly/manual restriction. ## Trigger behavior From 0ed53fb4cdc6cdeb34a01d49d1c2a54426ff3278 Mon Sep 17 00:00:00 2001 From: Sahil Faizal Date: Tue, 6 Oct 2026 21:21:37 -0700 Subject: [PATCH 3/8] ci: automatically validate LLVM bumps with host-ASan debug --- build_tools/github_actions/bump_automation.py | 2 +- .../github_actions/configure_multi_arch_ci.py | 25 +-- .../fetch_test_configurations.py | 16 +- .../tests/configure_multi_arch_ci_test.py | 195 +++++++++++------- docs/development/ci_behavior_manipulation.md | 8 +- 5 files changed, 144 insertions(+), 102 deletions(-) diff --git a/build_tools/github_actions/bump_automation.py b/build_tools/github_actions/bump_automation.py index 0a565584661..3371baf1614 100644 --- a/build_tools/github_actions/bump_automation.py +++ b/build_tools/github_actions/bump_automation.py @@ -725,7 +725,7 @@ def create_therock_bump(submodule: str, token: str) -> None: ) try: - # Add CI labels to the PR (run-all-archs + asan for full coverage) + # Add the submodule-specific CI labels to the PR gh_api( token, f"repos/{THEROCK_REPO}/issues/{pr['number']}/labels", diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 933397e782e..87ff2b2a26b 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -507,11 +507,9 @@ def has_submodule_changes(self) -> bool | None: @property def has_llvm_submodule_changes(self) -> bool: - return bool( - {"compiler/amd-llvm"} - & set(self.changed_files or []) - & set(self.submodule_paths or []) - ) + return "compiler/amd-llvm" in ( + self.changed_files or [] + ) and "compiler/amd-llvm" in (self.submodule_paths or []) def log(self) -> None: """Log git context for CI diagnostics.""" @@ -794,10 +792,8 @@ def should_skip_ci( print(" Skipping: 'ci:skip' PR label") return True - # Skip ASAN on PRs unless an enabling label is present. - # This avoids running expensive ASAN builds on every PR. - # Labels that enable ASAN CI: - # - ci:asan / ci:host-asan: explicit opt-in for ASAN testing + # LLVM pointer bumps run host-ASan debug automatically. Other PRs require + # ci:asan or ci:host-asan to opt in to expensive sanitizer builds. has_asan_label = ( "ci:asan" in ci_inputs.pr_labels or "ci:host-asan" in ci_inputs.pr_labels ) @@ -805,6 +801,7 @@ def should_skip_ci( ci_inputs.is_pull_request and ci_inputs.build_variant == "asan" and not has_asan_label + and not git_context.has_llvm_submodule_changes ): print( " Skipping: ASAN PR without enabling label (add 'ci:asan' or 'ci:host-asan' to enable)" @@ -1349,6 +1346,11 @@ def _expand_build_config_for_platform( test_runner_kernel = label.split(":")[1] break + has_host_asan_bump = bool( + {"rocm-libraries", "rocm-systems", "compiler/amd-llvm"} + & set(git_context.changed_files or []) + & set(git_context.submodule_paths or []) + ) per_family_info: list[dict] = [] for family_name in families: # select_targets already validates family names and filters by @@ -1392,11 +1394,6 @@ def _expand_build_config_for_platform( if build_variant.startswith("host-asan"): # Library, system, and LLVM bumps exercise the host-ASan sandbox # in addition to scheduled and manually dispatched runs. - has_host_asan_bump = bool( - {"rocm-libraries", "rocm-systems", "compiler/amd-llvm"} - & set(git_context.changed_files or []) - & set(git_context.submodule_paths or []) - ) if not ( ci_inputs.is_schedule or ci_inputs.is_workflow_dispatch diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index 905ab148d8b..8d4aac894af 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -1225,15 +1225,13 @@ def run(): "test-runs-on-multi-gpu", "" ) test_runs_on_sandbox = platform_info.get("test-runs-on-sandbox", "") - if build_variant.startswith("host-asan"): - test_runs_on_sandbox = platform_info.get( - "test-runs-on-host-asan", test_runs_on_sandbox - ) - if platform_info.get("test-runs-on-host-asan"): - # The host-ASan sandbox has eight GPUs and can also run - # multi-GPU components without using the production pool. - test_runs_on_multi_gpu_default = test_runs_on_sandbox - test_runs_on_multi_gpu_labels = None + if build_variant.startswith("host-asan") and platform_info.get( + "test-runs-on-host-asan" + ): + # The host-ASan sandbox also runs multi-GPU tests. + test_runs_on_sandbox = platform_info["test-runs-on-host-asan"] + test_runs_on_multi_gpu_default = test_runs_on_sandbox + test_runs_on_multi_gpu_labels = None logging.info(f"Selecting projects: {projects_to_test}") diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index 67b57cb9cfa..8176cbb6bb9 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -331,9 +331,16 @@ def _inputs(self, **kwargs): def test_skip_ci_label(self): """PR with ci:skip label skips CI regardless of changed files.""" - inputs = self._inputs(pr_labels=["ci:skip"]) - git = cm.GitContext(changed_files=["CMakeLists.txt"]) - self.assertTrue(cm.should_skip_ci(inputs, git)) + for variant, path in ( + ("release", "CMakeLists.txt"), + ("asan", "compiler/amd-llvm"), + ): + with self.subTest(variant=variant, path=path): + inputs = self._inputs(build_variant=variant, pr_labels=["ci:skip"]) + git = cm.GitContext( + changed_files=[path], submodule_paths=["compiler/amd-llvm"] + ) + self.assertTrue(cm.should_skip_ci(inputs, git)) def test_pr_without_skip_label_proceeds(self): """PR without ci:skip label proceeds to path filtering.""" @@ -380,6 +387,24 @@ def test_asan_pr_with_ci_asan_label_runs(self): ) self.assertFalse(cm.should_skip_ci(inputs, git)) + def test_llvm_source_change_without_pointer_bump_requires_label(self): + """Automatic LLVM ASan CI requires a changed submodule pointer.""" + inputs = self._inputs(build_variant="asan") + for changed_files, submodule_paths in ( + ( + ["compiler/amd-llvm/lib/Target/AMDGPU/example.cpp"], + ["compiler/amd-llvm"], + ), + (["compiler/amd-llvm"], []), + ): + with self.subTest( + changed_files=changed_files, submodule_paths=submodule_paths + ): + git = cm.GitContext( + changed_files=changed_files, submodule_paths=submodule_paths + ) + self.assertTrue(cm.should_skip_ci(inputs, git)) + def test_asan_non_pr_runs(self): """ASAN on schedule/push runs regardless of labels.""" inputs = self._inputs(event_name="schedule", build_variant="asan") @@ -1777,80 +1802,60 @@ def test_asan_runner_selection(self): def test_host_asan_submodule_bump_runners(self): targets = cm.TargetSelection(linux_families=["gfx94x", "gfx950"]) - for path in ( - "rocm-libraries", - "rocm-systems", - "compiler/amd-llvm", - "compiler/hipify", - "README.md", - ): - for event in ("pull_request", "push"): - for variant in ("asan", "host-asan-debug"): - with self.subTest(path=path, event=event, variant=variant): - result = cm.expand_build_configs( - ci_inputs=self._inputs( - event_name=event, - build_variant=variant, - pr_labels=SUBMODULE_CONFIG.get(path, {}).get( - "labels", ["ci:host-asan"] - ), - ), - git_context=cm.GitContext( - changed_files=[path], - submodule_paths=[ - "rocm-libraries", - "rocm-systems", - "compiler/amd-llvm", - "compiler/hipify", - ], - ), - targets=targets, - jobs=_jobs(), - ) - runners = [ - entry["test-runs-on"] - for entry in result.linux.per_family_info - ] - expected = ( - [ - "linux-gfx942-8gpu-asan-sandbox-rocm", - "linux-gfx950-8gpu-asan-sandbox-rocm", - ] - if path - in ("rocm-libraries", "rocm-systems", "compiler/amd-llvm") - else ["", ""] - ) - self.assertEqual(runners, expected) - expected_variant = ( - "host-asan-debug" - if path == "compiler/amd-llvm" - or variant == "host-asan-debug" - else "host-asan" - ) - self.assertEqual( - result.linux.build_variant_label, expected_variant - ) - self.assertEqual( - result.linux.build_variant_cmake_preset, - f"linux-release-{expected_variant}", - ) - - def test_llvm_bump_preserves_explicit_full_asan(self): - result = cm.expand_build_configs( - ci_inputs=self._inputs( - event_name="pull_request", build_variant="asan", pr_labels=["ci:asan"] - ), - git_context=cm.GitContext( - changed_files=["compiler/amd-llvm"], - submodule_paths=["compiler/amd-llvm"], + sandbox_runners = [ + "linux-gfx942-8gpu-asan-sandbox-rocm", + "linux-gfx950-8gpu-asan-sandbox-rocm", + ] + disabled_runners = ["", ""] + cases = ( + ("rocm-libraries", "asan", "host-asan", sandbox_runners), + ("rocm-libraries", "host-asan-debug", "host-asan-debug", sandbox_runners), + ("rocm-systems", "asan", "host-asan", sandbox_runners), + ("rocm-systems", "host-asan-debug", "host-asan-debug", sandbox_runners), + ("compiler/amd-llvm", "asan", "host-asan-debug", sandbox_runners), + ( + "compiler/amd-llvm", + "host-asan-debug", + "host-asan-debug", + sandbox_runners, ), - targets=cm.TargetSelection(linux_families=["gfx94x", "gfx950"]), - jobs=_jobs(), - ) - self.assertEqual(result.linux.build_variant_label, "asan") - self.assertEqual( - [e["test-runs-on"] for e in result.linux.per_family_info], ["", ""] + ("compiler/hipify", "asan", "host-asan", disabled_runners), + ("compiler/hipify", "host-asan-debug", "host-asan-debug", disabled_runners), + ("README.md", "asan", "host-asan", disabled_runners), + ("README.md", "host-asan-debug", "host-asan-debug", disabled_runners), ) + for path, variant, expected_variant, expected_runners in cases: + for event in ("pull_request", "push"): + with self.subTest(path=path, event=event, variant=variant): + result = cm.expand_build_configs( + ci_inputs=self._inputs( + event_name=event, + build_variant=variant, + pr_labels=SUBMODULE_CONFIG.get(path, {}).get( + "labels", ["ci:host-asan"] + ), + ), + git_context=cm.GitContext( + changed_files=[path], + submodule_paths=[ + "rocm-libraries", + "rocm-systems", + "compiler/amd-llvm", + "compiler/hipify", + ], + ), + targets=targets, + jobs=_jobs(), + ) + runners = [ + entry["test-runs-on"] for entry in result.linux.per_family_info + ] + self.assertEqual(runners, expected_runners) + self.assertEqual(result.linux.build_variant_label, expected_variant) + self.assertEqual( + result.linux.build_variant_cmake_preset, + f"linux-release-{expected_variant}", + ) def test_asan_debug_uses_sandbox_runner(self): """asan-debug variant uses sandbox runner like asan.""" @@ -2011,6 +2016,46 @@ def test_skipped_ci(self): class TestConfigurePipeline(unittest.TestCase): """Test the full pipeline via configure().""" + def test_llvm_bump_asan_label_selection(self): + """LLVM bumps run automatically; explicit full-ASan labels take precedence.""" + sandbox_runners = [ + "linux-gfx942-8gpu-asan-sandbox-rocm", + "linux-gfx950-8gpu-asan-sandbox-rocm", + ] + cases = ( + ([], "host-asan-debug", sandbox_runners), + (["ci:host-asan"], "host-asan-debug", sandbox_runners), + (["ci:asan"], "asan", ["", ""]), + (["ci:asan", "ci:host-asan"], "asan", ["", ""]), + ) + git = cm.GitContext( + changed_files=["compiler/amd-llvm"], + submodule_paths=["compiler/amd-llvm"], + ) + for labels, expected_variant, expected_runners in cases: + with self.subTest(labels=labels): + inputs = cm.CIInputs( + run_id="12345", + event_name="pull_request", + commit_ref="feature", + base_ref="HEAD^", + build_variant="asan", + pr_labels=labels, + linux_amdgpu_families=["gfx94x", "gfx950"], + ) + result = cm.configure(inputs, git) + self.assertTrue(result.is_ci_enabled) + linux = result.builds.linux + self.assertEqual(linux.build_variant_label, expected_variant) + self.assertEqual( + linux.build_variant_cmake_preset, + f"linux-release-{expected_variant}", + ) + self.assertEqual( + [entry["test-runs-on"] for entry in linux.per_family_info], + expected_runners, + ) + def test_skipped_outputs(self): """CIOutputs.skipped produces empty, disabled outputs.""" outputs = cm.CIOutputs.skipped() diff --git a/docs/development/ci_behavior_manipulation.md b/docs/development/ci_behavior_manipulation.md index 5fd75da6680..82582bb731f 100644 --- a/docs/development/ci_behavior_manipulation.md +++ b/docs/development/ci_behavior_manipulation.md @@ -8,10 +8,12 @@ and `gfx950`, host-ASan sanity checks and component tests use `linux-gfx942-8gpu-asan-sandbox-rocm` and `linux-gfx950-8gpu-asan-sandbox-rocm`, respectively. Scheduled and manually dispatched host-ASan tests use the same pools. LLVM submodule bump PRs -(`compiler/amd-llvm`) can opt in with `ci:host-asan`; they select -`host-asan-debug` (`linux-release-host-asan-debug`) and enable sandbox tests. +(`compiler/amd-llvm`) automatically run `host-asan-debug` +(`linux-release-host-asan-debug`) and enable sandbox tests without a CI label. Add `ci:gfx950-dcgpu` to include gfx950 alongside the default gfx94x coverage, -or use `ci:run-all-archs`. An explicit `ci:asan` label still selects full ASan. +or use `ci:run-all-archs`. The `ci:host-asan` label selects host-ASan (debug +for LLVM bumps); `ci:asan` selects full ASan and takes precedence if both labels +are present. The `ci:skip` label still skips CI. The separate Compiler Daily Bump workflow is unchanged. Other PRs retain the existing host-ASan test restriction, and full ASan retains its nightly/manual restriction. From 49231cafb2f1aa2bcfd04eb08b77a365a713e02e Mon Sep 17 00:00:00 2001 From: Sahil Faizal Date: Wed, 7 Oct 2026 20:29:22 -0700 Subject: [PATCH 4/8] ci: use full ASan debug builds for LLVM bumps --- .../github_actions/configure_multi_arch_ci.py | 20 ++++++++++++------- .../tests/configure_multi_arch_ci_test.py | 10 +++++----- docs/development/ci_behavior_manipulation.md | 15 ++++++++------ 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 144587bd900..a0852588971 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -1835,22 +1835,28 @@ def expand_build_configs( # ASAN variant selection: # 1. ci:asan label -> asan (explicit full ASAN, highest priority) # 2. ci:host-asan label -> host-asan (explicit) - # 3. push/pull_request events -> host-asan (default for pre/postsubmit) - # 4. schedule/workflow_dispatch -> asan (nightly/manual get full ASAN) + # 3. LLVM bumps -> asan-debug (device instrumentation and debug info) + # 4. push/pull_request events -> host-asan (default for pre/postsubmit) + # 5. schedule/workflow_dispatch -> asan (nightly/manual get full ASAN) if build_variant == "asan": if "ci:asan" in ci_inputs.pr_labels: print(" Using full asan variant (ci:asan label)") elif "ci:host-asan" in ci_inputs.pr_labels: build_variant = "host-asan" print(" Using host-asan variant (ci:host-asan label)") - elif ci_inputs.is_push or ci_inputs.is_pull_request: + elif ( + ci_inputs.is_push or ci_inputs.is_pull_request + ) and not git_context.has_llvm_submodule_changes: build_variant = "host-asan" print(" Using host-asan variant (push/pull_request default)") - # LLVM bump diagnostics need source locations from the debug preset. - if build_variant == "host-asan" and git_context.has_llvm_submodule_changes: - build_variant = "host-asan-debug" - print(" Using host-asan-debug variant for LLVM submodule bump") + # Exercise debug-info generation with the bumped LLVM compiler. + if ( + build_variant in ("asan", "host-asan") + and git_context.has_llvm_submodule_changes + ): + build_variant += "-debug" + print(f" Using {build_variant} variant for LLVM submodule bump") linux_config: BuildConfig | None = None windows_config: BuildConfig | None = None diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index 5ab4dec45aa..aae9c9620b8 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -2319,15 +2319,15 @@ class TestConfigurePipeline(unittest.TestCase): def test_llvm_bump_asan_label_selection(self): """LLVM bumps run automatically; explicit full-ASan labels take precedence.""" - sandbox_runners = [ + host_sandbox_runners = [ "linux-gfx942-8gpu-asan-sandbox-rocm", "linux-gfx950-8gpu-asan-sandbox-rocm", ] cases = ( - ([], "host-asan-debug", sandbox_runners), - (["ci:host-asan"], "host-asan-debug", sandbox_runners), - (["ci:asan"], "asan", ["", ""]), - (["ci:asan", "ci:host-asan"], "asan", ["", ""]), + ([], "asan-debug", ["", ""]), + (["ci:host-asan"], "host-asan-debug", host_sandbox_runners), + (["ci:asan"], "asan-debug", ["", ""]), + (["ci:asan", "ci:host-asan"], "asan-debug", ["", ""]), ) git = cm.GitContext( changed_files=["compiler/amd-llvm"], diff --git a/docs/development/ci_behavior_manipulation.md b/docs/development/ci_behavior_manipulation.md index db89091aee1..0f966fd6e97 100644 --- a/docs/development/ci_behavior_manipulation.md +++ b/docs/development/ci_behavior_manipulation.md @@ -8,14 +8,17 @@ and `gfx950`, host-ASan sanity checks and component tests use `linux-gfx942-8gpu-asan-sandbox-rocm` and `linux-gfx950-8gpu-asan-sandbox-rocm`, respectively. Scheduled and manually dispatched host-ASan tests use the same pools. LLVM submodule bump PRs -(`compiler/amd-llvm`) automatically run `host-asan-debug` -(`linux-release-host-asan-debug`) and enable sandbox tests without a CI label. +(`compiler/amd-llvm`) automatically run `asan-debug` +(`linux-release-asan-debug`) builds without a CI label. Full-ASan GPU tests remain +limited to scheduled and manually dispatched runs. +This exercises both device instrumentation and debug-info generation with the +new compiler. Add `ci:gfx950-dcgpu` to include gfx950 alongside the default gfx94x coverage, or use `ci:run-all-archs`. The `ci:host-asan` label selects host-ASan (debug -for LLVM bumps); `ci:asan` selects full ASan and takes precedence if both labels +for LLVM bumps); `ci:asan` selects full ASan (also debug for LLVM bumps) and takes precedence if both labels are present. The `ci:skip` label still skips CI. The separate Compiler Daily Bump workflow is unchanged. Other PRs retain the existing -host-ASan test restriction, and full ASan retains its nightly/manual restriction. +host-ASan test restriction. ## Trigger behavior @@ -38,8 +41,8 @@ The following labels may be added to a pull request to modify CI behavior: | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ci:skip` | Skip all builds and tests | | `ci:run-all-archs` | Build and test all possible architectures | -| `ci:asan` | Enable ASAN CI builds and tests. ASAN CI is skipped by default on PRs unless this label is present. | -| `ci:host-asan` | Alias for `ci:asan`. Enable ASAN CI builds and tests. | +| `ci:asan` | Select full ASan builds (`asan-debug` for LLVM bumps). GPU tests remain limited to nightly/manual runs. Takes precedence over `ci:host-asan`. | +| `ci:host-asan` | Select host-ASan builds (`host-asan-debug` for LLVM bumps). GPU tests follow the host-ASan policy described above. | | `ci:run-multi-gpu` | Opt-in to running multi-GPU tests. | | `ci:platform:linux` | Run CI only on Linux (skip Windows builds/tests) | | `ci:platform:windows` | Run CI only on Windows (skip Linux builds/tests) | From 7dc63de1bcb5a5e79e4422aaeb983ecbf1d53e25 Mon Sep 17 00:00:00 2001 From: Sahil Faizal Date: Wed, 7 Oct 2026 20:59:37 -0700 Subject: [PATCH 5/8] ci: keep sanitizer submodule bumps build-only --- .../github_actions/amdgpu_family_matrix.py | 2 - .../github_actions/configure_multi_arch_ci.py | 23 +++------- .../fetch_test_configurations.py | 7 --- .../tests/configure_multi_arch_ci_test.py | 45 +++++-------------- .../tests/fetch_test_configurations_test.py | 37 +-------------- docs/development/ci_behavior_manipulation.md | 30 +++++-------- 6 files changed, 31 insertions(+), 113 deletions(-) diff --git a/build_tools/github_actions/amdgpu_family_matrix.py b/build_tools/github_actions/amdgpu_family_matrix.py index 1b05bbeab9a..5311ab97c66 100644 --- a/build_tools/github_actions/amdgpu_family_matrix.py +++ b/build_tools/github_actions/amdgpu_family_matrix.py @@ -265,7 +265,6 @@ def select_build_runner(platform: str, build_variant: str, size: str = "large") ], # TODO(#3433): Remove sandbox label once ASAN tests are passing "test-runs-on-sandbox": "linux-gfx942-1gpu-asan-sandbox-rocm", - "test-runs-on-host-asan": "linux-gfx942-8gpu-asan-sandbox-rocm", "test-runs-on-multi-gpu": "linux-gfx942-8gpu-ossci-rocm", "test-runs-on-multi-gpu-labels": [ {"label": "linux-gfx942-8gpu-ossci-rocm", "count": 10}, @@ -462,7 +461,6 @@ def select_build_runner(platform: str, build_variant: str, size: str = "large") "linux": { "test-runs-on": "linux-gfx950-1gpu-ccs-ossci-rocm", "test-runs-on-sandbox": "linux-gfx950-1gpu-asan-sandbox-rocm", - "test-runs-on-host-asan": "linux-gfx950-8gpu-asan-sandbox-rocm", "test-runs-on-multi-gpu": "linux-gfx950-8gpu-ccs-ossci-rocm", "family": "gfx950-dcgpu", "fetch-gfx-targets": ["gfx950"], diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index a0852588971..21e6bfe70e7 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -815,7 +815,7 @@ def should_skip_ci( print(" Skipping: 'ci:skip' PR label") return True - # LLVM pointer bumps run host-ASan debug automatically. Other PRs require + # LLVM pointer bumps run ASan debug builds automatically. Other PRs require # ci:asan or ci:host-asan to opt in to expensive sanitizer builds. has_asan_label = ( "ci:asan" in ci_inputs.pr_labels or "ci:host-asan" in ci_inputs.pr_labels @@ -1483,11 +1483,6 @@ def _expand_build_config_for_platform( test_runner_kernel = label.split(":")[1] break - has_host_asan_bump = bool( - {"rocm-libraries", "rocm-systems", "compiler/amd-llvm"} - & set(git_context.changed_files or []) - & set(git_context.submodule_paths or []) - ) per_family_info: list[dict] = [] for family_name in families: # select_targets already validates family names and filters by @@ -1529,22 +1524,16 @@ def _expand_build_config_for_platform( # TODO(#3433): Remove once ASAN tests pass and test_rocm.action is plumbed. if build_variant.startswith("host-asan"): - # Library, system, and LLVM bumps exercise the host-ASan sandbox - # in addition to scheduled and manually dispatched runs. - if not ( - ci_inputs.is_schedule - or ci_inputs.is_workflow_dispatch - or has_host_asan_bump - ): + # Run host-asan tests only on nightly (schedule or workflow_dispatch) + # due to limited ASAN runner capacity and stability concerns. + if not (ci_inputs.is_schedule or ci_inputs.is_workflow_dispatch): test_runs_on = "" print( - f" {family_name}: host-asan tests require nightly or a library/system/LLVM bump, " + f" {family_name}: host-asan tests only run on nightly, " f"disabling tests" ) elif "test-runs-on-sandbox" in platform_info: - test_runs_on = platform_info.get( - "test-runs-on-host-asan", platform_info["test-runs-on-sandbox"] - ) + test_runs_on = platform_info["test-runs-on-sandbox"] print( f" {family_name}: using host-asan sandbox runner: {test_runs_on}" ) diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index c1d00d28e76..cc9cf7575cb 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -1268,13 +1268,6 @@ def run(): "test-runs-on-multi-gpu", "" ) test_runs_on_sandbox = platform_info.get("test-runs-on-sandbox", "") - if build_variant.startswith("host-asan") and platform_info.get( - "test-runs-on-host-asan" - ): - # The host-ASan sandbox also runs multi-GPU tests. - test_runs_on_sandbox = platform_info["test-runs-on-host-asan"] - test_runs_on_multi_gpu_default = test_runs_on_sandbox - test_runs_on_multi_gpu_labels = None logging.info(f"Selecting projects: {projects_to_test}") diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index aae9c9620b8..526a4dedab5 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -1960,31 +1960,14 @@ def test_asan_runner_selection(self): entry = result.linux.per_family_info[0] self.assertEqual(entry["test-runs-on"], "") - def test_host_asan_submodule_bump_runners(self): + def test_host_asan_submodule_bumps_are_build_only(self): targets = cm.TargetSelection(linux_families=["gfx94x", "gfx950"]) - sandbox_runners = [ - "linux-gfx942-8gpu-asan-sandbox-rocm", - "linux-gfx950-8gpu-asan-sandbox-rocm", - ] - disabled_runners = ["", ""] cases = ( - ("rocm-libraries", "asan", "host-asan", sandbox_runners), - ("rocm-libraries", "host-asan-debug", "host-asan-debug", sandbox_runners), - ("rocm-systems", "asan", "host-asan", sandbox_runners), - ("rocm-systems", "host-asan-debug", "host-asan-debug", sandbox_runners), - ("compiler/amd-llvm", "asan", "host-asan-debug", sandbox_runners), - ( - "compiler/amd-llvm", - "host-asan-debug", - "host-asan-debug", - sandbox_runners, - ), - ("compiler/hipify", "asan", "host-asan", disabled_runners), - ("compiler/hipify", "host-asan-debug", "host-asan-debug", disabled_runners), - ("README.md", "asan", "host-asan", disabled_runners), - ("README.md", "host-asan-debug", "host-asan-debug", disabled_runners), + ("rocm-libraries", "asan", "host-asan"), + ("rocm-systems", "asan", "host-asan"), + ("compiler/amd-llvm", "asan", "host-asan-debug"), ) - for path, variant, expected_variant, expected_runners in cases: + for path, variant, expected_variant in cases: for event in ("pull_request", "push"): with self.subTest(path=path, event=event, variant=variant): result = cm.expand_build_configs( @@ -2010,7 +1993,7 @@ def test_host_asan_submodule_bump_runners(self): runners = [ entry["test-runs-on"] for entry in result.linux.per_family_info ] - self.assertEqual(runners, expected_runners) + self.assertEqual(runners, ["", ""]) self.assertEqual(result.linux.build_variant_label, expected_variant) self.assertEqual( result.linux.build_variant_cmake_preset, @@ -2319,21 +2302,17 @@ class TestConfigurePipeline(unittest.TestCase): def test_llvm_bump_asan_label_selection(self): """LLVM bumps run automatically; explicit full-ASan labels take precedence.""" - host_sandbox_runners = [ - "linux-gfx942-8gpu-asan-sandbox-rocm", - "linux-gfx950-8gpu-asan-sandbox-rocm", - ] cases = ( - ([], "asan-debug", ["", ""]), - (["ci:host-asan"], "host-asan-debug", host_sandbox_runners), - (["ci:asan"], "asan-debug", ["", ""]), - (["ci:asan", "ci:host-asan"], "asan-debug", ["", ""]), + ([], "asan-debug"), + (["ci:host-asan"], "host-asan-debug"), + (["ci:asan"], "asan-debug"), + (["ci:asan", "ci:host-asan"], "asan-debug"), ) git = cm.GitContext( changed_files=["compiler/amd-llvm"], submodule_paths=["compiler/amd-llvm"], ) - for labels, expected_variant, expected_runners in cases: + for labels, expected_variant in cases: with self.subTest(labels=labels): inputs = cm.CIInputs( run_id="12345", @@ -2354,7 +2333,7 @@ def test_llvm_bump_asan_label_selection(self): ) self.assertEqual( [entry["test-runs-on"] for entry in linux.per_family_info], - expected_runners, + ["", ""], ) def test_skipped_outputs(self): diff --git a/build_tools/github_actions/tests/fetch_test_configurations_test.py b/build_tools/github_actions/tests/fetch_test_configurations_test.py index 378f52859c5..ba015d4a5f6 100644 --- a/build_tools/github_actions/tests/fetch_test_configurations_test.py +++ b/build_tools/github_actions/tests/fetch_test_configurations_test.py @@ -761,7 +761,6 @@ def fake_get_all_families(_): "linux": { "test-runs-on": "linux-gfx942-prod", "test-runs-on-sandbox": "linux-sandbox-runner", - "test-runs-on-host-asan": "linux-host-asan-runner", } } } @@ -774,41 +773,7 @@ def fake_get_all_families(_): components = self._get_components() hipblas = next(j for j in components if j["job_name"] == "hipblas") - self.assertEqual( - hipblas["test_runner"], - ( - "linux-host-asan-runner" - if build_variant.startswith("host-asan") - else "linux-sandbox-runner" - ), - ) - - def test_host_asan_multi_gpu_uses_sandbox_runner(self): - os.environ["BUILD_VARIANT"] = "host-asan" - os.environ["PROJECTS_TO_TEST"] = "rccl" - os.environ["TEST_LABELS"] = json.dumps(["ci:run-multi-gpu"]) - - def fake_get_all_families(_): - return { - "gfx94x": { - "linux": { - "test-runs-on-host-asan": "linux-gfx942-8gpu-asan-sandbox-rocm", - "test-runs-on-multi-gpu": "production-default", - "test-runs-on-multi-gpu-labels": [ - {"label": "production-weighted", "count": 1} - ], - } - } - } - - fetch_test_configurations.get_all_families_for_trigger_types = ( - fake_get_all_families - ) - fetch_test_configurations.run() - rccl = next(j for j in self._get_components() if j["job_name"] == "rccl") - self.assertEqual( - rccl["multi_gpu_runner"], "linux-gfx942-8gpu-asan-sandbox-rocm" - ) + self.assertEqual(hipblas["test_runner"], "linux-sandbox-runner") def test_release_build_uses_count_runner(self): """Release builds should use count-based runner labels, not sandbox.""" diff --git a/docs/development/ci_behavior_manipulation.md b/docs/development/ci_behavior_manipulation.md index 0f966fd6e97..04460b281f8 100644 --- a/docs/development/ci_behavior_manipulation.md +++ b/docs/development/ci_behavior_manipulation.md @@ -2,23 +2,17 @@ **Multi-Arch CI** ([`multi_arch_ci.yml`](https://github.com/ROCm/TheRock/actions/workflows/multi_arch_ci.yml)) is configured by [`configure_multi_arch_ci.py`](../../build_tools/github_actions/configure_multi_arch_ci.py) and reads GPU family definitions from [`amdgpu_family_matrix.py`](../../build_tools/github_actions/amdgpu_family_matrix.py). -Host-ASan tests run for `rocm-libraries` and `rocm-systems` submodule bumps -when ASAN CI is enabled (the bump automation adds `ci:host-asan`). For `gfx94x` -and `gfx950`, host-ASan sanity checks and component tests use -`linux-gfx942-8gpu-asan-sandbox-rocm` and -`linux-gfx950-8gpu-asan-sandbox-rocm`, respectively. Scheduled and manually -dispatched host-ASan tests use the same pools. LLVM submodule bump PRs -(`compiler/amd-llvm`) automatically run `asan-debug` -(`linux-release-asan-debug`) builds without a CI label. Full-ASan GPU tests remain -limited to scheduled and manually dispatched runs. -This exercises both device instrumentation and debug-info generation with the -new compiler. -Add `ci:gfx950-dcgpu` to include gfx950 alongside the default gfx94x coverage, -or use `ci:run-all-archs`. The `ci:host-asan` label selects host-ASan (debug -for LLVM bumps); `ci:asan` selects full ASan (also debug for LLVM bumps) and takes precedence if both labels -are present. The `ci:skip` label still skips CI. -The separate Compiler Daily Bump workflow is unchanged. Other PRs retain the existing -host-ASan test restriction. +Library and system submodule bump PRs automatically receive `ci:host-asan` +and run host-ASan builds. LLVM pointer bumps (`compiler/amd-llvm`) automatically +run `asan-debug` (`linux-release-asan-debug`) builds to exercise device +instrumentation and debug-info generation with the new compiler. + +Sanitizer bump CI is build-only: GPU sanity checks and component tests remain +limited to scheduled and manually dispatched runs. The `ci:host-asan` label +selects host-ASan (`host-asan-debug` for LLVM bumps); `ci:asan` selects full ASan +(`asan-debug` for LLVM bumps) and takes precedence if both labels are present. +The `ci:skip` label still skips CI. The separate Compiler Daily Bump workflow +is unchanged. ## Trigger behavior @@ -42,7 +36,7 @@ The following labels may be added to a pull request to modify CI behavior: | `ci:skip` | Skip all builds and tests | | `ci:run-all-archs` | Build and test all possible architectures | | `ci:asan` | Select full ASan builds (`asan-debug` for LLVM bumps). GPU tests remain limited to nightly/manual runs. Takes precedence over `ci:host-asan`. | -| `ci:host-asan` | Select host-ASan builds (`host-asan-debug` for LLVM bumps). GPU tests follow the host-ASan policy described above. | +| `ci:host-asan` | Select host-ASan builds (`host-asan-debug` for LLVM bumps). GPU tests remain limited to nightly/manual runs. | | `ci:run-multi-gpu` | Opt-in to running multi-GPU tests. | | `ci:platform:linux` | Run CI only on Linux (skip Windows builds/tests) | | `ci:platform:windows` | Run CI only on Windows (skip Linux builds/tests) | From be9bbc193f6843f4571a636a22ce8f551edf5ac5 Mon Sep 17 00:00:00 2001 From: Sahil Faizal Date: Wed, 7 Oct 2026 21:11:28 -0700 Subject: [PATCH 6/8] test: consolidate sanitizer bump runner coverage --- .../tests/configure_multi_arch_ci_test.py | 45 +++++-------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index 526a4dedab5..43546942022 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -1950,42 +1950,25 @@ def test_asan_runner_selection(self): entry = result.linux.per_family_info[0] self.assertIn("sandbox", entry["test-runs-on"]) - # PR: disables tests (empty runner) - result = cm.expand_build_configs( - ci_inputs=self._inputs(event_name="pull_request", build_variant="asan"), - git_context=cm.GitContext(), - targets=targets, - jobs=_jobs(), - ) - entry = result.linux.per_family_info[0] - self.assertEqual(entry["test-runs-on"], "") - - def test_host_asan_submodule_bumps_are_build_only(self): - targets = cm.TargetSelection(linux_families=["gfx94x", "gfx950"]) + # PR/push, including submodule bumps: build only. cases = ( - ("rocm-libraries", "asan", "host-asan"), - ("rocm-systems", "asan", "host-asan"), - ("compiler/amd-llvm", "asan", "host-asan-debug"), + (None, "host-asan"), + ("rocm-libraries", "host-asan"), + ("rocm-systems", "host-asan"), + ("compiler/amd-llvm", "asan-debug"), ) - for path, variant, expected_variant in cases: + for path, expected_variant in cases: for event in ("pull_request", "push"): - with self.subTest(path=path, event=event, variant=variant): + with self.subTest(path=path, event=event): result = cm.expand_build_configs( ci_inputs=self._inputs( event_name=event, - build_variant=variant, - pr_labels=SUBMODULE_CONFIG.get(path, {}).get( - "labels", ["ci:host-asan"] - ), + build_variant="asan", + pr_labels=SUBMODULE_CONFIG.get(path, {}).get("labels", []), ), git_context=cm.GitContext( - changed_files=[path], - submodule_paths=[ - "rocm-libraries", - "rocm-systems", - "compiler/amd-llvm", - "compiler/hipify", - ], + changed_files=[path] if path else [], + submodule_paths=[path] if path else [], ), targets=targets, jobs=_jobs(), @@ -1993,12 +1976,8 @@ def test_host_asan_submodule_bumps_are_build_only(self): runners = [ entry["test-runs-on"] for entry in result.linux.per_family_info ] - self.assertEqual(runners, ["", ""]) + self.assertEqual(runners, [""]) self.assertEqual(result.linux.build_variant_label, expected_variant) - self.assertEqual( - result.linux.build_variant_cmake_preset, - f"linux-release-{expected_variant}", - ) def test_asan_debug_uses_sandbox_runner(self): """asan-debug variant uses sandbox runner like asan.""" From a6efb4c38db870540315ca4a40a54fadb4b81be7 Mon Sep 17 00:00:00 2001 From: Sahil Faizal Date: Thu, 8 Oct 2026 11:04:19 -0700 Subject: [PATCH 7/8] ci: select ASan debug explicitly with a PR label --- .../github_actions/configure_multi_arch_ci.py | 39 ++--- .../tests/configure_multi_arch_ci_test.py | 165 +++++++----------- docs/development/ci_behavior_manipulation.md | 17 +- 3 files changed, 89 insertions(+), 132 deletions(-) diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 21e6bfe70e7..600e94cffaf 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -508,12 +508,6 @@ def has_submodule_changes(self) -> bool | None: return None return bool(set(self.submodule_paths) & set(self.changed_files)) - @property - def has_llvm_submodule_changes(self) -> bool: - return "compiler/amd-llvm" in ( - self.changed_files or [] - ) and "compiler/amd-llvm" in (self.submodule_paths or []) - def log(self) -> None: """Log git context for CI diagnostics.""" if self.changed_files is None: @@ -815,19 +809,19 @@ def should_skip_ci( print(" Skipping: 'ci:skip' PR label") return True - # LLVM pointer bumps run ASan debug builds automatically. Other PRs require - # ci:asan or ci:host-asan to opt in to expensive sanitizer builds. + # PRs require an explicit label to opt in to expensive sanitizer builds. has_asan_label = ( - "ci:asan" in ci_inputs.pr_labels or "ci:host-asan" in ci_inputs.pr_labels + "ci:asan-debug" in ci_inputs.pr_labels + or "ci:asan" in ci_inputs.pr_labels + or "ci:host-asan" in ci_inputs.pr_labels ) if ( ci_inputs.is_pull_request and ci_inputs.build_variant == "asan" and not has_asan_label - and not git_context.has_llvm_submodule_changes ): print( - " Skipping: ASAN PR without enabling label (add 'ci:asan' or 'ci:host-asan' to enable)" + " Skipping: ASAN PR without enabling label (add 'ci:asan-debug', 'ci:asan' or 'ci:host-asan' to enable)" ) return True @@ -1822,31 +1816,24 @@ def expand_build_configs( all_families = _apply_external_family_overrides(all_families) build_variant = ci_inputs.build_variant # ASAN variant selection: - # 1. ci:asan label -> asan (explicit full ASAN, highest priority) - # 2. ci:host-asan label -> host-asan (explicit) - # 3. LLVM bumps -> asan-debug (device instrumentation and debug info) + # 1. ci:asan-debug label -> asan-debug (full ASAN with debug info) + # 2. ci:asan label -> asan (explicit full ASAN) + # 3. ci:host-asan label -> host-asan (explicit) # 4. push/pull_request events -> host-asan (default for pre/postsubmit) # 5. schedule/workflow_dispatch -> asan (nightly/manual get full ASAN) if build_variant == "asan": - if "ci:asan" in ci_inputs.pr_labels: + if "ci:asan-debug" in ci_inputs.pr_labels: + build_variant = "asan-debug" + print(" Using full asan-debug variant (ci:asan-debug label)") + elif "ci:asan" in ci_inputs.pr_labels: print(" Using full asan variant (ci:asan label)") elif "ci:host-asan" in ci_inputs.pr_labels: build_variant = "host-asan" print(" Using host-asan variant (ci:host-asan label)") - elif ( - ci_inputs.is_push or ci_inputs.is_pull_request - ) and not git_context.has_llvm_submodule_changes: + elif ci_inputs.is_push or ci_inputs.is_pull_request: build_variant = "host-asan" print(" Using host-asan variant (push/pull_request default)") - # Exercise debug-info generation with the bumped LLVM compiler. - if ( - build_variant in ("asan", "host-asan") - and git_context.has_llvm_submodule_changes - ): - build_variant += "-debug" - print(f" Using {build_variant} variant for LLVM submodule bump") - linux_config: BuildConfig | None = None windows_config: BuildConfig | None = None diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index 43546942022..31ea5513f21 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -20,7 +20,6 @@ sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) import configure_multi_arch_ci as cm -from bump_automation import SUBMODULE_CONFIG from amdgpu_family_matrix import get_all_families_for_trigger_types from configure_multi_arch_ci_summary import format_summary from workflow_utils import WORKFLOWS_DIR @@ -331,16 +330,9 @@ def _inputs(self, **kwargs): def test_skip_ci_label(self): """PR with ci:skip label skips CI regardless of changed files.""" - for variant, path in ( - ("release", "CMakeLists.txt"), - ("asan", "compiler/amd-llvm"), - ): - with self.subTest(variant=variant, path=path): - inputs = self._inputs(build_variant=variant, pr_labels=["ci:skip"]) - git = cm.GitContext( - changed_files=[path], submodule_paths=["compiler/amd-llvm"] - ) - self.assertTrue(cm.should_skip_ci(inputs, git)) + inputs = self._inputs(pr_labels=["ci:skip"]) + git = cm.GitContext(changed_files=["CMakeLists.txt"]) + self.assertTrue(cm.should_skip_ci(inputs, git)) def test_pr_without_skip_label_proceeds(self): """PR without ci:skip label proceeds to path filtering.""" @@ -374,37 +366,26 @@ def test_none_changed_files_skips_path_filter(self, mock_filter): def test_asan_pr_without_label_skips(self): """ASAN PR without enabling label skips CI.""" inputs = self._inputs(build_variant="asan", pr_labels=[]) - git = cm.GitContext( - changed_files=["CMakeLists.txt", "build_tools/script.py"], - ) - self.assertTrue(cm.should_skip_ci(inputs, git)) - - def test_asan_pr_with_ci_asan_label_runs(self): - """ASAN PR with ci:asan label runs CI.""" - inputs = self._inputs(build_variant="asan", pr_labels=["ci:asan"]) - git = cm.GitContext( - changed_files=["CMakeLists.txt", "build_tools/script.py"], - ) - self.assertFalse(cm.should_skip_ci(inputs, git)) - - def test_llvm_source_change_without_pointer_bump_requires_label(self): - """Automatic LLVM ASan CI requires a changed submodule pointer.""" - inputs = self._inputs(build_variant="asan") - for changed_files, submodule_paths in ( - ( - ["compiler/amd-llvm/lib/Target/AMDGPU/example.cpp"], - ["compiler/amd-llvm"], - ), - (["compiler/amd-llvm"], []), + for changed_files in ( + ["CMakeLists.txt", "build_tools/script.py"], + ["compiler/amd-llvm"], ): - with self.subTest( - changed_files=changed_files, submodule_paths=submodule_paths - ): + with self.subTest(changed_files=changed_files): git = cm.GitContext( - changed_files=changed_files, submodule_paths=submodule_paths + changed_files=changed_files, submodule_paths=["compiler/amd-llvm"] ) self.assertTrue(cm.should_skip_ci(inputs, git)) + def test_asan_pr_with_enabling_label_runs(self): + """Each sanitizer label opts a PR into ASAN CI.""" + git = cm.GitContext( + changed_files=["CMakeLists.txt", "build_tools/script.py"], + ) + for label in ("ci:asan-debug", "ci:asan", "ci:host-asan"): + with self.subTest(label=label): + inputs = self._inputs(build_variant="asan", pr_labels=[label]) + self.assertFalse(cm.should_skip_ci(inputs, git)) + def test_asan_non_pr_runs(self): """ASAN on schedule/push runs regardless of labels.""" inputs = self._inputs(event_name="schedule", build_variant="asan") @@ -1950,34 +1931,15 @@ def test_asan_runner_selection(self): entry = result.linux.per_family_info[0] self.assertIn("sandbox", entry["test-runs-on"]) - # PR/push, including submodule bumps: build only. - cases = ( - (None, "host-asan"), - ("rocm-libraries", "host-asan"), - ("rocm-systems", "host-asan"), - ("compiler/amd-llvm", "asan-debug"), - ) - for path, expected_variant in cases: - for event in ("pull_request", "push"): - with self.subTest(path=path, event=event): - result = cm.expand_build_configs( - ci_inputs=self._inputs( - event_name=event, - build_variant="asan", - pr_labels=SUBMODULE_CONFIG.get(path, {}).get("labels", []), - ), - git_context=cm.GitContext( - changed_files=[path] if path else [], - submodule_paths=[path] if path else [], - ), - targets=targets, - jobs=_jobs(), - ) - runners = [ - entry["test-runs-on"] for entry in result.linux.per_family_info - ] - self.assertEqual(runners, [""]) - self.assertEqual(result.linux.build_variant_label, expected_variant) + # PR: disables tests (empty runner) + result = cm.expand_build_configs( + ci_inputs=self._inputs(event_name="pull_request", build_variant="asan"), + git_context=cm.GitContext(), + targets=targets, + jobs=_jobs(), + ) + entry = result.linux.per_family_info[0] + self.assertEqual(entry["test-runs-on"], "") def test_asan_debug_uses_sandbox_runner(self): """asan-debug variant uses sandbox runner like asan.""" @@ -2279,41 +2241,46 @@ def test_skipped_ci(self): class TestConfigurePipeline(unittest.TestCase): """Test the full pipeline via configure().""" - def test_llvm_bump_asan_label_selection(self): - """LLVM bumps run automatically; explicit full-ASan labels take precedence.""" + def test_asan_label_selection(self): + """Labels select sanitizer builds independently of submodule changes.""" cases = ( - ([], "asan-debug"), - (["ci:host-asan"], "host-asan-debug"), - (["ci:asan"], "asan-debug"), - (["ci:asan", "ci:host-asan"], "asan-debug"), - ) - git = cm.GitContext( - changed_files=["compiler/amd-llvm"], - submodule_paths=["compiler/amd-llvm"], - ) - for labels, expected_variant in cases: - with self.subTest(labels=labels): - inputs = cm.CIInputs( - run_id="12345", - event_name="pull_request", - commit_ref="feature", - base_ref="HEAD^", - build_variant="asan", - pr_labels=labels, - linux_amdgpu_families=["gfx94x", "gfx950"], - ) - result = cm.configure(inputs, git) - self.assertTrue(result.is_ci_enabled) - linux = result.builds.linux - self.assertEqual(linux.build_variant_label, expected_variant) - self.assertEqual( - linux.build_variant_cmake_preset, - f"linux-release-{expected_variant}", - ) - self.assertEqual( - [entry["test-runs-on"] for entry in linux.per_family_info], - ["", ""], - ) + ([], None), + (["ci:host-asan"], "host-asan"), + (["ci:asan"], "asan"), + (["ci:asan-debug"], "asan-debug"), + (["ci:asan", "ci:host-asan"], "asan"), + (["ci:asan-debug", "ci:asan", "ci:host-asan"], "asan-debug"), + (["ci:asan-debug", "ci:skip"], None), + ) + for path in ("CMakeLists.txt", "compiler/amd-llvm"): + git = cm.GitContext( + changed_files=[path], submodule_paths=["compiler/amd-llvm"] + ) + for labels, expected_variant in cases: + with self.subTest(path=path, labels=labels): + inputs = cm.CIInputs( + run_id="12345", + event_name="pull_request", + commit_ref="feature", + base_ref="HEAD^", + build_variant="asan", + pr_labels=labels, + linux_amdgpu_families=["gfx94x", "gfx950"], + ) + result = cm.configure(inputs, git) + self.assertEqual(result.is_ci_enabled, expected_variant is not None) + if expected_variant is None: + continue + linux = result.builds.linux + self.assertEqual(linux.build_variant_label, expected_variant) + self.assertEqual( + linux.build_variant_cmake_preset, + f"linux-release-{expected_variant}", + ) + self.assertEqual( + [entry["test-runs-on"] for entry in linux.per_family_info], + ["", ""], + ) def test_skipped_outputs(self): """CIOutputs.skipped produces empty, disabled outputs.""" diff --git a/docs/development/ci_behavior_manipulation.md b/docs/development/ci_behavior_manipulation.md index 04460b281f8..b5db536d447 100644 --- a/docs/development/ci_behavior_manipulation.md +++ b/docs/development/ci_behavior_manipulation.md @@ -3,14 +3,16 @@ **Multi-Arch CI** ([`multi_arch_ci.yml`](https://github.com/ROCm/TheRock/actions/workflows/multi_arch_ci.yml)) is configured by [`configure_multi_arch_ci.py`](../../build_tools/github_actions/configure_multi_arch_ci.py) and reads GPU family definitions from [`amdgpu_family_matrix.py`](../../build_tools/github_actions/amdgpu_family_matrix.py). Library and system submodule bump PRs automatically receive `ci:host-asan` -and run host-ASan builds. LLVM pointer bumps (`compiler/amd-llvm`) automatically -run `asan-debug` (`linux-release-asan-debug`) builds to exercise device -instrumentation and debug-info generation with the new compiler. +and run host-ASan builds. Add `ci:asan-debug` to LLVM promotion PRs to run +`asan-debug` (`linux-release-asan-debug`) builds and exercise device +instrumentation and debug-info generation with the new compiler. Submodule +changes alone do not enable sanitizer CI. Sanitizer bump CI is build-only: GPU sanity checks and component tests remain limited to scheduled and manually dispatched runs. The `ci:host-asan` label -selects host-ASan (`host-asan-debug` for LLVM bumps); `ci:asan` selects full ASan -(`asan-debug` for LLVM bumps) and takes precedence if both labels are present. +selects host-ASan; `ci:asan` selects full ASan; `ci:asan-debug` selects full ASan +with debug information. When multiple labels are present, `ci:asan-debug` takes +precedence over `ci:asan`, which takes precedence over `ci:host-asan`. The `ci:skip` label still skips CI. The separate Compiler Daily Bump workflow is unchanged. @@ -35,8 +37,9 @@ The following labels may be added to a pull request to modify CI behavior: | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ci:skip` | Skip all builds and tests | | `ci:run-all-archs` | Build and test all possible architectures | -| `ci:asan` | Select full ASan builds (`asan-debug` for LLVM bumps). GPU tests remain limited to nightly/manual runs. Takes precedence over `ci:host-asan`. | -| `ci:host-asan` | Select host-ASan builds (`host-asan-debug` for LLVM bumps). GPU tests remain limited to nightly/manual runs. | +| `ci:asan-debug` | Select full ASan builds with debug information. GPU tests remain limited to nightly/manual runs. Takes precedence over `ci:asan` and `ci:host-asan`. | +| `ci:asan` | Select full ASan builds. GPU tests remain limited to nightly/manual runs. Takes precedence over `ci:host-asan`. | +| `ci:host-asan` | Select host-ASan builds. GPU tests remain limited to nightly/manual runs. | | `ci:run-multi-gpu` | Opt-in to running multi-GPU tests. | | `ci:platform:linux` | Run CI only on Linux (skip Windows builds/tests) | | `ci:platform:windows` | Run CI only on Windows (skip Linux builds/tests) | From 0eeb302ea84d5140ccf1ed34ac554306b4fed005 Mon Sep 17 00:00:00 2001 From: Sahil Faizal Date: Tue, 13 Oct 2026 20:33:48 -0700 Subject: [PATCH 8/8] ci: address sanitizer label review feedback --- .../github_actions/configure_ci_labels.py | 2 + .../github_actions/configure_multi_arch_ci.py | 19 +++-- .../tests/configure_multi_arch_ci_test.py | 74 +++++++++---------- docs/development/ci_behavior_manipulation.md | 37 ++++++---- 4 files changed, 67 insertions(+), 65 deletions(-) diff --git a/build_tools/github_actions/configure_ci_labels.py b/build_tools/github_actions/configure_ci_labels.py index bf1fb6f77ef..7be874ae392 100755 --- a/build_tools/github_actions/configure_ci_labels.py +++ b/build_tools/github_actions/configure_ci_labels.py @@ -62,7 +62,9 @@ class Label: Label("ci:run-non-multi-arch", COLOR_CI_GENERAL, "Opt-in to running non-multi-arch CI on a pull request"), Label("ci:build-jax", COLOR_CI_GENERAL, "Enable Jax Build"), Label("ci:asan", COLOR_CI_GENERAL, "Opt-in to building ASAN"), + Label("ci:asan-debug", COLOR_CI_GENERAL, "Opt-in to building ASAN with debug info"), Label("ci:host-asan", COLOR_CI_GENERAL, "Opt-in to running multi-arch host-asan CI on a pull request"), + Label("ci:host-asan-debug", COLOR_CI_GENERAL, "Opt-in to running multi-arch host-asan CI with debug info on a pull request"), Label("ci:run-multi-gpu", COLOR_CI_GENERAL, "Opt-in to running multi-GPU tests"), # ci:platform: labels (platform selection) Label("ci:platform:linux", COLOR_CI_PLATFORM, "Run CI only on Linux (skip Windows builds/tests)"), diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 600e94cffaf..39e54feb20b 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -810,10 +810,9 @@ def should_skip_ci( return True # PRs require an explicit label to opt in to expensive sanitizer builds. - has_asan_label = ( - "ci:asan-debug" in ci_inputs.pr_labels - or "ci:asan" in ci_inputs.pr_labels - or "ci:host-asan" in ci_inputs.pr_labels + has_asan_label = any( + label in ci_inputs.pr_labels + for label in ["ci:asan-debug", "ci:asan", "ci:host-asan-debug", "ci:host-asan"] ) if ( ci_inputs.is_pull_request @@ -821,7 +820,7 @@ def should_skip_ci( and not has_asan_label ): print( - " Skipping: ASAN PR without enabling label (add 'ci:asan-debug', 'ci:asan' or 'ci:host-asan' to enable)" + " Skipping: ASAN PR without enabling label (add 'ci:asan-debug', 'ci:asan', 'ci:host-asan-debug' or 'ci:host-asan' to enable)" ) return True @@ -1818,15 +1817,19 @@ def expand_build_configs( # ASAN variant selection: # 1. ci:asan-debug label -> asan-debug (full ASAN with debug info) # 2. ci:asan label -> asan (explicit full ASAN) - # 3. ci:host-asan label -> host-asan (explicit) - # 4. push/pull_request events -> host-asan (default for pre/postsubmit) - # 5. schedule/workflow_dispatch -> asan (nightly/manual get full ASAN) + # 3. ci:host-asan-debug label -> host-asan-debug (host-only ASAN with debug info) + # 4. ci:host-asan label -> host-asan (explicit) + # 5. push/pull_request events -> host-asan (default for pre/postsubmit) + # 6. schedule/workflow_dispatch -> asan (nightly/manual get full ASAN) if build_variant == "asan": if "ci:asan-debug" in ci_inputs.pr_labels: build_variant = "asan-debug" print(" Using full asan-debug variant (ci:asan-debug label)") elif "ci:asan" in ci_inputs.pr_labels: print(" Using full asan variant (ci:asan label)") + elif "ci:host-asan-debug" in ci_inputs.pr_labels: + build_variant = "host-asan-debug" + print(" Using host-asan-debug variant (ci:host-asan-debug label)") elif "ci:host-asan" in ci_inputs.pr_labels: build_variant = "host-asan" print(" Using host-asan variant (ci:host-asan label)") diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index 31ea5513f21..3ff0c480dfb 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -1838,6 +1838,39 @@ def test_variant_filters_by_trigger(self): build_variant_cmake_preset = result.linux.build_variant_cmake_preset self.assertEqual(build_variant_cmake_preset, expected_variant) + def test_asan_label_selection(self): + """PR labels select the sanitizer variant in priority order.""" + cases = ( + (["ci:host-asan"], "host-asan"), + (["ci:asan"], "asan"), + (["ci:asan-debug"], "asan-debug"), + (["ci:asan", "ci:host-asan"], "asan"), + (["ci:asan-debug", "ci:asan", "ci:host-asan"], "asan-debug"), + ) + targets = cm.TargetSelection(linux_families=["gfx94x", "gfx950"]) + for labels, expected_variant in cases: + with self.subTest(labels=labels): + result = cm.expand_build_configs( + ci_inputs=self._inputs( + event_name="pull_request", + build_variant="asan", + pr_labels=labels, + ), + git_context=cm.GitContext(), + targets=targets, + jobs=_jobs(), + ) + linux = result.linux + self.assertEqual(linux.build_variant_label, expected_variant) + self.assertEqual( + linux.build_variant_cmake_preset, + f"linux-release-{expected_variant}", + ) + self.assertEqual( + [entry["test-runs-on"] for entry in linux.per_family_info], + ["", ""], + ) + def test_push_asan_excludes_families_without_host_asan_support(self): """Push ASAN: gfx110x only supports release, so it must be excluded. @@ -2241,47 +2274,6 @@ def test_skipped_ci(self): class TestConfigurePipeline(unittest.TestCase): """Test the full pipeline via configure().""" - def test_asan_label_selection(self): - """Labels select sanitizer builds independently of submodule changes.""" - cases = ( - ([], None), - (["ci:host-asan"], "host-asan"), - (["ci:asan"], "asan"), - (["ci:asan-debug"], "asan-debug"), - (["ci:asan", "ci:host-asan"], "asan"), - (["ci:asan-debug", "ci:asan", "ci:host-asan"], "asan-debug"), - (["ci:asan-debug", "ci:skip"], None), - ) - for path in ("CMakeLists.txt", "compiler/amd-llvm"): - git = cm.GitContext( - changed_files=[path], submodule_paths=["compiler/amd-llvm"] - ) - for labels, expected_variant in cases: - with self.subTest(path=path, labels=labels): - inputs = cm.CIInputs( - run_id="12345", - event_name="pull_request", - commit_ref="feature", - base_ref="HEAD^", - build_variant="asan", - pr_labels=labels, - linux_amdgpu_families=["gfx94x", "gfx950"], - ) - result = cm.configure(inputs, git) - self.assertEqual(result.is_ci_enabled, expected_variant is not None) - if expected_variant is None: - continue - linux = result.builds.linux - self.assertEqual(linux.build_variant_label, expected_variant) - self.assertEqual( - linux.build_variant_cmake_preset, - f"linux-release-{expected_variant}", - ) - self.assertEqual( - [entry["test-runs-on"] for entry in linux.per_family_info], - ["", ""], - ) - def test_skipped_outputs(self): """CIOutputs.skipped produces empty, disabled outputs.""" outputs = cm.CIOutputs.skipped() diff --git a/docs/development/ci_behavior_manipulation.md b/docs/development/ci_behavior_manipulation.md index b5db536d447..2bddb8e8565 100644 --- a/docs/development/ci_behavior_manipulation.md +++ b/docs/development/ci_behavior_manipulation.md @@ -2,20 +2,6 @@ **Multi-Arch CI** ([`multi_arch_ci.yml`](https://github.com/ROCm/TheRock/actions/workflows/multi_arch_ci.yml)) is configured by [`configure_multi_arch_ci.py`](../../build_tools/github_actions/configure_multi_arch_ci.py) and reads GPU family definitions from [`amdgpu_family_matrix.py`](../../build_tools/github_actions/amdgpu_family_matrix.py). -Library and system submodule bump PRs automatically receive `ci:host-asan` -and run host-ASan builds. Add `ci:asan-debug` to LLVM promotion PRs to run -`asan-debug` (`linux-release-asan-debug`) builds and exercise device -instrumentation and debug-info generation with the new compiler. Submodule -changes alone do not enable sanitizer CI. - -Sanitizer bump CI is build-only: GPU sanity checks and component tests remain -limited to scheduled and manually dispatched runs. The `ci:host-asan` label -selects host-ASan; `ci:asan` selects full ASan; `ci:asan-debug` selects full ASan -with debug information. When multiple labels are present, `ci:asan-debug` takes -precedence over `ci:asan`, which takes precedence over `ci:host-asan`. -The `ci:skip` label still skips CI. The separate Compiler Daily Bump workflow -is unchanged. - ## Trigger behavior The CI pipelines test a growing set of GPU targets depending on trigger type/frequency: @@ -37,8 +23,9 @@ The following labels may be added to a pull request to modify CI behavior: | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ci:skip` | Skip all builds and tests | | `ci:run-all-archs` | Build and test all possible architectures | -| `ci:asan-debug` | Select full ASan builds with debug information. GPU tests remain limited to nightly/manual runs. Takes precedence over `ci:asan` and `ci:host-asan`. | -| `ci:asan` | Select full ASan builds. GPU tests remain limited to nightly/manual runs. Takes precedence over `ci:host-asan`. | +| `ci:asan-debug` | Select full ASan builds with debug information. GPU tests remain limited to nightly/manual runs. Takes precedence over all other sanitizer labels. | +| `ci:asan` | Select full ASan builds. GPU tests remain limited to nightly/manual runs. Takes precedence over host-ASan labels. | +| `ci:host-asan-debug` | Select host-ASan builds with debug information. GPU tests remain limited to nightly/manual runs. Takes precedence over `ci:host-asan`. | | `ci:host-asan` | Select host-ASan builds. GPU tests remain limited to nightly/manual runs. | | `ci:run-multi-gpu` | Opt-in to running multi-GPU tests. | | `ci:platform:linux` | Run CI only on Linux (skip Windows builds/tests) | @@ -90,6 +77,24 @@ standard `ci:gfx...` labels. > for that architecture. If no test runner exists for the family, the label is > ignored with a warning. +#### Automated submodule update pull requests + +The [submodule bump automation](../../build_tools/github_actions/bump_automation.py) +adds `ci:run-all-archs` to its pull requests to cover all GPU families. +`rocm-systems` and `rocm-libraries` updates also receive `ci:host-asan` for +host-ASan build coverage, plus `gfx950-dcgpu` and `gfx125X-dcgpu` to enable +label-gated tests on limited hardware. `rocgdb` updates receive `test:rocgdb`; +`mesa-fork` updates receive `test:rocdecode` and `test:rocjpeg` to restrict tests +to the affected projects. + +Other pull requests can use the same sanitizer labels to opt in, or remove all +sanitizer labels to opt out. Submodule changes alone do not enable sanitizer CI. +For LLVM promotion pull requests, optionally add `ci:asan-debug` to exercise +device instrumentation and debug-info generation with the new compiler. + +Sanitizer pull request CI is build-only: GPU sanity checks and component tests +run only on scheduled or manually dispatched runs. `ci:skip` still skips CI. + ### Push CI runs on pushes to `main` if modified files pass the filters in