diff --git a/build_tools/github_actions/bump_automation.py b/build_tools/github_actions/bump_automation.py index 855d90d3f93..6f47e4e9b9f 100644 --- a/build_tools/github_actions/bump_automation.py +++ b/build_tools/github_actions/bump_automation.py @@ -46,10 +46,10 @@ "bot_author": "systems-assistant[bot]", # Changes to rocm-systems should run the full matrix of CI jobs: # * Build for all gfx archs - # * Build for all variants (asan) + # * Build release and host-ASan variants # * All builds and tests (including downstream rocm-libraries jobs) # * gfx950-dcgpu/gfx125X-dcgpu tests (limited hardware, label-gated) - "labels": [*COMMON_CI_LABELS, "ci:asan", "gfx950-dcgpu", "gfx125X-dcgpu"], + "labels": [*COMMON_CI_LABELS, "ci:host-asan", "gfx950-dcgpu", "gfx125X-dcgpu"], }, "rocm-libraries": { "repo": "ROCm/rocm-libraries", @@ -65,10 +65,10 @@ "bot_author": "assistant-librarian[bot]", # Changes to rocm-libraries should run the full matrix of CI jobs: # * Build for all gfx archs - # * Build for all variants (asan) + # * Build release and host-ASan variants # * All rocm-libraries tests # * gfx950-dcgpu/gfx125X-dcgpu tests (limited hardware, label-gated) - "labels": [*COMMON_CI_LABELS, "ci:asan", "gfx950-dcgpu", "gfx125X-dcgpu"], + "labels": [*COMMON_CI_LABELS, "ci:host-asan", "gfx950-dcgpu", "gfx125X-dcgpu"], }, "debug-tools/rocgdb/source": { "repo": "ROCm/rocgdb", @@ -755,7 +755,7 @@ def create_therock_bump(submodule: str, token: str) -> None: ) try: - # Add CI labels to the PR (run-all-archs + asan for full coverage) + # Add the submodule-specific CI labels to the PR gh_api( token, f"repos/{THEROCK_REPO}/issues/{pr['number']}/labels", diff --git a/build_tools/github_actions/configure_ci_labels.py b/build_tools/github_actions/configure_ci_labels.py index e25dc2e6957..aab755a7384 100755 --- a/build_tools/github_actions/configure_ci_labels.py +++ b/build_tools/github_actions/configure_ci_labels.py @@ -62,7 +62,9 @@ class Label: Label("ci:run-non-multi-arch", COLOR_CI_GENERAL, "Opt-in to running non-multi-arch CI on a pull request"), Label("ci:build-jax", COLOR_CI_GENERAL, "Enable Jax Build"), Label("ci:asan", COLOR_CI_GENERAL, "Opt-in to building ASAN"), + Label("ci:asan-debug", COLOR_CI_GENERAL, "Opt-in to building ASAN with debug info"), Label("ci:host-asan", COLOR_CI_GENERAL, "Opt-in to running multi-arch host-asan CI on a pull request"), + Label("ci:host-asan-debug", COLOR_CI_GENERAL, "Opt-in to running multi-arch host-asan CI with debug info on a pull request"), Label("ci:run-multi-gpu", COLOR_CI_GENERAL, "Opt-in to running multi-GPU tests"), Label("ci:cpu-test-only", COLOR_CI_GENERAL, "Run only CPU tests (skip GPU tests)"), # ci:platform: labels (platform selection) diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index e5b2905b540..c1c7e759351 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -804,7 +804,7 @@ def should_skip_ci( - 'ci:skip' PR label - Only skippable files changed (docs, .md, etc.) - No files changed - - ASAN builds on PRs without ci:asan or ci:host-asan labels + - ASAN builds on PRs without an enabling sanitizer label For external repo builds, path filtering uses changed_files and skip_ci_patterns from the external_repo JSON (both must be provided). @@ -858,8 +858,9 @@ def should_skip_ci( return True # 3. ASAN skip - only evaluated if path filtering didn't skip CI - has_asan_label = ( - "ci:asan" in ci_inputs.pr_labels or "ci:host-asan" in ci_inputs.pr_labels + has_asan_label = any( + label in ci_inputs.pr_labels + for label in ["ci:asan-debug", "ci:asan", "ci:host-asan-debug", "ci:host-asan"] ) if ( ci_inputs.is_pull_request @@ -867,7 +868,7 @@ def should_skip_ci( and not has_asan_label ): print( - " Skipping: ASAN PR without enabling label (add 'ci:asan' or 'ci:host-asan' to enable)" + " Skipping: ASAN PR without enabling label (add 'ci:asan-debug', 'ci:asan', 'ci:host-asan-debug' or 'ci:host-asan' to enable)" ) return True @@ -1853,13 +1854,21 @@ def expand_build_configs( all_families = _apply_external_family_overrides(all_families) build_variant = ci_inputs.build_variant # ASAN variant selection: - # 1. ci:asan label -> asan (explicit full ASAN, highest priority) - # 2. ci:host-asan label -> host-asan (explicit) - # 3. push/pull_request events -> host-asan (default for pre/postsubmit) - # 4. schedule/workflow_dispatch -> asan (nightly/manual get full ASAN) + # 1. ci:asan-debug label -> asan-debug (full ASAN with debug info) + # 2. ci:asan label -> asan (explicit full ASAN) + # 3. ci:host-asan-debug label -> host-asan-debug (host-only ASAN with debug info) + # 4. ci:host-asan label -> host-asan (explicit) + # 5. push/pull_request events -> host-asan (default for pre/postsubmit) + # 6. schedule/workflow_dispatch -> asan (nightly/manual get full ASAN) if build_variant == "asan": - if "ci:asan" in ci_inputs.pr_labels: + if "ci:asan-debug" in ci_inputs.pr_labels: + build_variant = "asan-debug" + print(" Using full asan-debug variant (ci:asan-debug label)") + elif "ci:asan" in ci_inputs.pr_labels: print(" Using full asan variant (ci:asan label)") + elif "ci:host-asan-debug" in ci_inputs.pr_labels: + build_variant = "host-asan-debug" + print(" Using host-asan-debug variant (ci:host-asan-debug label)") elif "ci:host-asan" in ci_inputs.pr_labels: build_variant = "host-asan" print(" Using host-asan variant (ci:host-asan label)") diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index a8cacc0618f..f4e12438f2a 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -366,18 +366,25 @@ def test_none_changed_files_skips_path_filter(self, mock_filter): def test_asan_pr_without_label_skips(self): """ASAN PR without enabling label skips CI.""" inputs = self._inputs(build_variant="asan", pr_labels=[]) - git = cm.GitContext( - changed_files=["CMakeLists.txt", "build_tools/script.py"], - ) - self.assertTrue(cm.should_skip_ci(inputs, git)) + for changed_files in ( + ["CMakeLists.txt", "build_tools/script.py"], + ["compiler/amd-llvm"], + ): + with self.subTest(changed_files=changed_files): + git = cm.GitContext( + changed_files=changed_files, submodule_paths=["compiler/amd-llvm"] + ) + self.assertTrue(cm.should_skip_ci(inputs, git)) - def test_asan_pr_with_ci_asan_label_runs(self): - """ASAN PR with ci:asan label runs CI.""" - inputs = self._inputs(build_variant="asan", pr_labels=["ci:asan"]) + def test_asan_pr_with_enabling_label_runs(self): + """Each sanitizer label opts a PR into ASAN CI.""" git = cm.GitContext( changed_files=["CMakeLists.txt", "build_tools/script.py"], ) - self.assertFalse(cm.should_skip_ci(inputs, git)) + for label in ("ci:asan-debug", "ci:asan", "ci:host-asan"): + with self.subTest(label=label): + inputs = self._inputs(build_variant="asan", pr_labels=[label]) + self.assertFalse(cm.should_skip_ci(inputs, git)) def test_asan_non_pr_runs(self): """ASAN on schedule/push runs regardless of labels.""" @@ -1832,6 +1839,39 @@ def test_variant_filters_by_trigger(self): build_variant_cmake_preset = result.linux.build_variant_cmake_preset self.assertEqual(build_variant_cmake_preset, expected_variant) + def test_asan_label_selection(self): + """PR labels select the sanitizer variant in priority order.""" + cases = ( + (["ci:host-asan"], "host-asan"), + (["ci:asan"], "asan"), + (["ci:asan-debug"], "asan-debug"), + (["ci:asan", "ci:host-asan"], "asan"), + (["ci:asan-debug", "ci:asan", "ci:host-asan"], "asan-debug"), + ) + targets = cm.TargetSelection(linux_families=["gfx94x", "gfx950"]) + for labels, expected_variant in cases: + with self.subTest(labels=labels): + result = cm.expand_build_configs( + ci_inputs=self._inputs( + event_name="pull_request", + build_variant="asan", + pr_labels=labels, + ), + git_context=cm.GitContext(), + targets=targets, + jobs=_jobs(), + ) + linux = result.linux + self.assertEqual(linux.build_variant_label, expected_variant) + self.assertEqual( + linux.build_variant_cmake_preset, + f"linux-release-{expected_variant}", + ) + self.assertEqual( + [entry["test-runs-on"] for entry in linux.per_family_info], + ["", ""], + ) + def test_push_asan_excludes_families_without_host_asan_support(self): """Push ASAN: gfx110x only supports release, so it must be excluded. diff --git a/docs/development/ci_behavior_manipulation.md b/docs/development/ci_behavior_manipulation.md index 4b528f659f0..3eb0ea6e490 100644 --- a/docs/development/ci_behavior_manipulation.md +++ b/docs/development/ci_behavior_manipulation.md @@ -23,8 +23,10 @@ The following labels may be added to a pull request to modify CI behavior: | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ci:skip` | Skip all builds and tests | | `ci:run-all-archs` | Build and test all possible architectures | -| `ci:asan` | Enable ASAN CI builds and tests. ASAN CI is skipped by default on PRs unless this label is present. | -| `ci:host-asan` | Alias for `ci:asan`. Enable ASAN CI builds and tests. | +| `ci:asan-debug` | Select full ASan builds with debug information. GPU tests remain limited to nightly/manual runs. Takes precedence over all other sanitizer labels. | +| `ci:asan` | Select full ASan builds. GPU tests remain limited to nightly/manual runs. Takes precedence over host-ASan labels. | +| `ci:host-asan-debug` | Select host-ASan builds with debug information. GPU tests remain limited to nightly/manual runs. Takes precedence over `ci:host-asan`. | +| `ci:host-asan` | Select host-ASan builds. GPU tests remain limited to nightly/manual runs. | | `ci:run-multi-gpu` | Opt-in to running multi-GPU tests. | | `ci:cpu-test-only` | Run only CPU tests (skip GPU tests). Useful for changes that don't require GPU testing. | | `ci:platform:linux` | Run CI only on Linux (skip Windows builds/tests) | @@ -76,6 +78,25 @@ standard `ci:gfx...` labels. > for that architecture. If no test runner exists for the family, the label is > ignored with a warning. +#### Automated submodule update pull requests + +The [submodule bump automation](../../build_tools/github_actions/bump_automation.py) +adds `ci:run-all-archs` to its pull requests to cover all GPU families. +`rocm-systems` and `rocm-libraries` updates also receive `ci:host-asan` for +host-ASan build coverage, plus `gfx950-dcgpu` and `gfx125X-dcgpu` to enable +label-gated tests on limited hardware. `rocgdb` updates receive `test:rocgdb`; +`mesa-fork` updates receive `test:rocdecode` and `test:rocjpeg` to restrict tests +to the affected projects. + +Other pull requests can use the same sanitizer labels to opt in, or remove all +sanitizer labels to opt out. Submodule changes alone do not enable sanitizer CI. +For LLVM promotion pull requests, optionally add `ci:asan-debug` to exercise +device instrumentation and debug-info generation with the new compiler. + +Sanitizer labels do not override GPU test gating. GPU sanity checks and +component tests remain subject to trigger and runner restrictions. CPU tests +may still run. `ci:skip` still skips CI. + ### Push CI runs on pushes to `main` if modified files pass the filters in