diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index aec89e7408b..6e53961ed26 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -205,6 +205,16 @@ def _family_matches( # component whose suite is too slow for the quick sanity tier opts out of it with: # "test_types": ["standard", "comprehensive", "full"] # +# A component may mark itself advisory (non-blocking) on specific GPU families via +# "expect_failure", a map keyed by platform whose value is a list of family entries, +# matched exactly like include_family/exclude_family (so either the family group +# string or an individual gfx target works). It is resolved to a single boolean per +# job before the matrix is emitted, which test_component.yml maps to +# continue-on-error plus an "(xfail)" job-name suffix. The job is still scheduled and +# still reports; it just cannot fail the run. Use it to land coverage for an arch +# that is still being qualified without blocking everyone else. For example: +# "expect_failure": {"linux": ["gfx950"]} +# # A component's "skip_stages" lists build stages whose absence excludes the test. # If any listed stage is in SKIP_STAGES, the component is skipped. Reused stages # still supply artifacts and do not exclude tests. Omit the field for no stage gate. @@ -710,8 +720,10 @@ def _family_matches( # The runner ships in the MIOpen dist (share/miopen/bin/run_dbsync_rocjitsu.py, pulled via # --miopen; defined in rocm-libraries projects/miopen/test/gtest/dbsync/): it resolves arch + CU # list from AMDGPU_FAMILIES, sparse-builds the pinned rocjitsu KMD, and runs StaticFDBSync once - # per CU with a CU-corrected config. include_family restricts it to gfx942, whose FAMILY_MAP - # entry covers both CU variants -- MI300X (304 CU) and MI300A (228 CU) -- in a single job. + # per CU with a CU-corrected config. include_family restricts it to the arches rocjitsu has a + # KMD config for: gfx942, whose FAMILY_MAP entry covers both CU variants -- MI300X (304 CU) and + # MI300A (228 CU) -- in a single job, and gfx950 (MI355X, 256 CU; advisory, see + # expect_failure below). # linux_cpu_runner: no scarce GPU test runner needed; uses the default no_rocm Ubuntu container # (the runner sudo-apt-installs cmake/build-essential/libdrm-dev to build rocjitsu). "miopen-dbsync": { @@ -721,15 +733,20 @@ def _family_matches( # job entirely on the `quick` tier -- no job is scheduled, so no artifact fetch # or rocjitsu build is paid for on quick (the runner script also self-skips on # TEST_TYPE=quick as a backstop). Runs serially (MIOPEN_DBSYNC_MAX_THREADS=1) - # under rocjitsu; full set (gfx942 304+228) + artifact fetch + rocjitsu build - # measures ~15 min, so 30 gives margin and fails a hung interposer faster. + # under rocjitsu; full set (gfx942 304+228, gfx950 256) + artifact fetch + rocjitsu + # build measures ~15 min, so 30 gives margin and fails a hung interposer faster. "timeout_minutes": 30, "test_script": "python ./build/share/miopen/bin/run_dbsync_rocjitsu.py", "platform": ["linux"], "linux_cpu_runner": True, "test_types": ["standard", "comprehensive", "full"], "include_family": { - "linux": ["gfx942"], + "linux": ["gfx942", "gfx950"], + }, + # TODO(ALMIOPEN-2674): drop this once gfx950 dbsync has been green for a + # few consecutive standard/comprehensive/full runs. + "expect_failure": { + "linux": ["gfx950"], }, "total_shards_dict": { "linux": 1, @@ -1439,6 +1456,17 @@ def run(): job_config_data = {**_common_settings, **test_matrix[key]} job_config_data["test_type"] = test_type + # A component declares the families it is advisory on via + # "expect_failure" (see the key reference above). Resolve that + # declaration down to the single boolean the workflows read -- + # test_component.yml compares it with `== true` to set + # continue-on-error, so the emitted value must never stay a map. + _xfail = job_config_data.get("expect_failure") + if _xfail: + job_config_data["expect_failure"] = _family_matches( + _xfail.get(platform, []), amdgpu_families, _family_gfx_targets + ) + # tensilelite: append the tensilelite/tests C++ gtest suite (run via # ctest -L , driven by the shared test_runner.py) after # the existing pytest stage, for every tier except quick -- that diff --git a/build_tools/github_actions/tests/fetch_test_configurations_test.py b/build_tools/github_actions/tests/fetch_test_configurations_test.py index 9c62efbec88..e969b344c0a 100644 --- a/build_tools/github_actions/tests/fetch_test_configurations_test.py +++ b/build_tools/github_actions/tests/fetch_test_configurations_test.py @@ -465,6 +465,124 @@ def test_miopen_dbsync_excluded_on_quick(self): os.environ["TEST_TYPE"] = "quick" self.assertNotIn("miopen-dbsync", self._selected_names()) + def _dbsync_job(self, families, gated_gpu, gfx_target): + """Run selection for miopen-dbsync on `families`; return its job or None.""" + os.environ["PROJECTS_TO_TEST"] = "miopen-dbsync" + os.environ["AMDGPU_FAMILIES"] = families + if gated_gpu: + # configure_multi_arch_ci blanked the GPU runner (e.g. tests_on_trigger). + os.environ["TEST_RUNS_ON"] = "" + target = families.split("-")[0].lower() + fetch_test_configurations.get_all_families_for_trigger_types = lambda _: { + target: { + "linux": { + "test-runs-on": "" if gated_gpu else "some-gpu-runner", + "fetch-gfx-targets": [gfx_target], + } + } + } + fetch_test_configurations.run() + jobs = [j for j in self._get_components() if j["job_name"] == "miopen-dbsync"] + return jobs[0] if jobs else None + + def test_miopen_dbsync_include_family_covers_gfx942_and_gfx950(self): + config = fetch_test_configurations.test_matrix["miopen-dbsync"] + self.assertEqual(config["include_family"]["linux"], ["gfx942", "gfx950"]) + + def test_miopen_dbsync_declares_gfx950_advisory(self): + # The matrix entry itself says gfx950 is advisory -- no conditional elsewhere. + config = fetch_test_configurations.test_matrix["miopen-dbsync"] + self.assertEqual(config["expect_failure"]["linux"], ["gfx950"]) + + def test_miopen_dbsync_gfx950_scheduled_on_cpu_runner_when_gpu_gated(self): + job = self._dbsync_job("gfx950-dcgpu", True, "gfx950") + self.assertIsNotNone(job) + self.assertTrue(job["linux_cpu_runner"]) + self.assertTrue(job["test_runner"]) + + def test_miopen_dbsync_gfx950_is_advisory(self): + job = self._dbsync_job("gfx950-dcgpu", True, "gfx950") + self.assertTrue(job["expect_failure"]) + + def test_miopen_dbsync_gfx942_stays_blocking(self): + job = self._dbsync_job("gfx94X-dcgpu", False, "gfx942") + self.assertIsNotNone(job) + self.assertFalse(job["expect_failure"]) + + def test_miopen_dbsync_excluded_on_other_families(self): + self.assertIsNone(self._dbsync_job("gfx120X-all", False, "gfx1200")) + + def test_gpu_components_not_scheduled_on_gated_gfx950(self): + # The CPU lane must not leak GPU-requiring components onto gated gfx950. + os.environ["PROJECTS_TO_TEST"] = "miopen-dbsync,rocblas" + os.environ["AMDGPU_FAMILIES"] = "gfx950-dcgpu" + os.environ["TEST_RUNS_ON"] = "" + fetch_test_configurations.get_all_families_for_trigger_types = lambda _: { + "gfx950": {"linux": {"test-runs-on": "", "fetch-gfx-targets": ["gfx950"]}} + } + names = self._selected_names() + self.assertIn("miopen-dbsync", names) + self.assertNotIn("rocblas", names) + + # ----------------------- + # expect_failure (per-family advisory jobs) + # ----------------------- + + def _job_by_name(self, name): + """Run selection and return the emitted job dict for `name`, or None.""" + fetch_test_configurations.run() + jobs = [j for j in self._get_components() if j["job_name"] == name] + return jobs[0] if jobs else None + + def test_expect_failure_matches_by_gfx_target(self): + # Default AMDGPU_FAMILIES is the gfx94X-dcgpu group; "gfx942" is one of its + # fetch-gfx-targets, so the target form matches -- same as include_family. + self._inject_job("xf-target", expect_failure={"linux": ["gfx942"]}) + self.assertIs(self._job_by_name("xf-target")["expect_failure"], True) + + def test_expect_failure_matches_by_group_string(self): + self._inject_job("xf-group", expect_failure={"linux": ["gfx94X-dcgpu"]}) + self.assertIs(self._job_by_name("xf-group")["expect_failure"], True) + + def test_expect_failure_does_not_match_other_family(self): + # The job is still scheduled; it is just blocking rather than advisory. + self._inject_job("xf-other", expect_failure={"linux": ["gfx1100"]}) + job = self._job_by_name("xf-other") + self.assertIsNotNone(job) + self.assertIs(job["expect_failure"], False) + + def test_expect_failure_is_scoped_per_platform(self): + # A windows-only declaration does not make the linux job advisory. + self._inject_job("xf-otherplat", expect_failure={"windows": ["gfx942"]}) + self.assertIs(self._job_by_name("xf-otherplat")["expect_failure"], False) + + def test_expect_failure_omitted_leaves_job_blocking(self): + # Components that never opt in do not carry the key at all. + self._inject_job("xf-absent") + self.assertNotIn("expect_failure", self._job_by_name("xf-absent")) + + def test_expect_failure_is_always_emitted_as_a_bool(self): + # test_component.yml compares with `== true`, so the declaration map must + # never survive into the emitted component JSON. + for name, decl in [ + ("xf-bool-hit", {"linux": ["gfx942"]}), + ("xf-bool-miss", {"linux": ["gfx1100"]}), + ]: + with self.subTest(job=name): + self._inject_job(name, expect_failure=decl) + self.assertIsInstance(self._job_by_name(name)["expect_failure"], bool) + + def test_expect_failure_does_not_mutate_test_matrix(self): + # The boolean is written to the merged copy, so the declaration survives + # for the next run rather than being consumed by the first one. + self._inject_job("xf-repeat", expect_failure={"linux": ["gfx942"]}) + self.assertIs(self._job_by_name("xf-repeat")["expect_failure"], True) + self.assertIs(self._job_by_name("xf-repeat")["expect_failure"], True) + self.assertEqual( + fetch_test_configurations.test_matrix["xf-repeat"]["expect_failure"], + {"linux": ["gfx942"]}, + ) + # ----------------------- # Multi-GPU logic (RCCL) # -----------------------