Changes to integrate Test Filter Standardization for miopen in TheRock

.github/workflows/test_component.yml

@@ -140,6 +140,7 @@ jobs:
           SHARD_INDEX: ${{ matrix.shard }}
           TOTAL_SHARDS: ${{ fromJSON(inputs.component).total_shards }}
           TEST_TYPE: ${{ fromJSON(inputs.component).test_type }}
+          TEST_COMPONENT: ${{ fromJSON(inputs.component).job_name }}
           ROCM_KPACK_DEBUG: "1"
         run: |
           ${{ fromJSON(inputs.component).test_script }}

build_tools/github_actions/fetch_test_configurations.py

@@ -282,7 +282,7 @@ def _get_script_path(script_name: str) -> str:
         "job_name": "miopen",
         "fetch_artifact_args": "--blas --miopen --tests",
         "timeout_minutes": 60,
-        "test_script": f"python {_get_script_path('test_miopen.py')}",
+        "test_script": f"python {_get_script_path('test_runner.py')}",
         "platform": ["linux", "windows"],
         "total_shards_dict": {
             "linux": 4,

build_tools/github_actions/github_actions_utils.py

@@ -695,6 +695,40 @@ def get_first_gpu_architecture(env=None, therock_bin_dir: str | None = None) ->
     raise RuntimeError("No GPU architecture found in rocminfo output")
 
 
+def find_matching_gpu_arch(gpu_arch: str, available_gpu_archs: set[str]) -> str | None:
+    """
+    Find the most specific GPU architecture in the set that matches the given GPU.
+
+    Tries in order from most specific to least specific:
+    # Example:
+    # find_matching_gpu_arch('gfx1151', {'gfx1151', 'gfx115X', 'gfx11X'}) gives 'gfx1151'
+    # find_matching_gpu_arch('gfx1151', {'gfx1150', 'gfx94X', 'gfx11X'}) gives 'gfx11X'
+    - Wildcard matches (gfx115X, gfx11X, etc.)
+
+    Returns the matching architecture string or None if no match found.
+    """
+    # First, try exact match
+    if gpu_arch in available_gpu_archs:
+        return gpu_arch
+
+    # Generate possible wildcard patterns from most specific to least specific
+    # For gfx1151: try gfx115X, gfx11X
+    possible_patterns = []
+    arch_str = gpu_arch
+
+    # Generate patterns by replacing characters with X from right to left
+    for i in range(len(arch_str) - 1, 1, -1):
+        pattern = arch_str[:i] + "X"
+        possible_patterns.append(pattern)
+
+    # Try each pattern
+    for pattern in possible_patterns:
+        if pattern in available_gpu_archs:
+            return pattern
+
+    return None
+
+
 def is_asan():
     """Using artifact_group, determines if this is an asan build"""
     ARTIFACT_GROUP = os.getenv("ARTIFACT_GROUP")

build_tools/github_actions/test_executable_scripts/README_test_runner.md

@@ -0,0 +1,86 @@
+# Test Runner
+
+`test_runner.py` is a **generic test runner** used by GitHub Actions to run component tests (e.g. MIOpen). It relies on a standardized CTest naming and labeling scheme so that only tests valid for the current scenario are run.
+
+This document describes how the mechanism works, what the component must provide, and how it ties into CI.
+
+______________________________________________________________________
+
+## Overview
+
+1. The script is invoked by the **Test component** workflow with env vars set (component name, GPU arch, test type, sharding).
+1. The script expects the component under test to have a ctest based interface, with labels corresponding to test categories like quick, standard, comprehensive and full(which can be run for scenarios like smoke test, pre-commit etc)
+1. The component can opt to always exclude some tests or based on condition like OS and GPU where the test is getting run
+1. The script discovers which **GPU-specific test suites** exist by running `ctest --print-labels` and parsing labels of the form `ex_gpu_{gpu_arch}`.
+1. It builds and runs a **ctest** command with the right labels and options (category, parallelism, sharding, etc.).
+
+No component-specific logic is required beyond using the expected test names and CTest labels.
+
+______________________________________________________________________
+
+## Test Naming and Labels (Component Contract)
+
+For this runner to work, the component’s CTest configuration must follow this convention.
+
+### CTest labels (discovery)
+
+The script runs `ctest --print-labels --test-dir <component>` and collects every label that starts with **`ex_gpu_`**. The suffix is treated as the GPU architecture (e.g. `ex_gpu_gfx110X` → `gfx110X`, `ex_gpu_gfx950` → `gfx950`). Only labels whose suffix starts with `gfx` are used.
+
+### CTest labels (selection)
+
+- **Category:** Tests are selected by **label** for the run (e.g. “run all tests with label `quick`” or `standard`). The script uses `-L <category>`.
+- **GPU-specific suites:** For a given `gpu_arch`, the script looks for a label `ex_gpu_{gpu_arch}` (e.g. `ex_gpu_gfx1150`, `ex_gpu_gfx11X`). It uses `-L ex_gpu_<arch>` when it finds a matching label, or `-LE ex_gpu` to exclude all GPU-specific tests when no match is found or when no GPU is specified.
+
+So the component must assign GPU-specific tests the label `ex_gpu_{gpu_arch}` (and typically a category label like `quick` or `standard`).
+
+______________________________________________________________________
+
+## Execution Flow
+
+1. **Resolve component directory**
+   Map `TEST_COMPONENT` (e.g. `miopen`) to the test directory name (e.g. `MIOpen`) via `COMPONENT_DIR_MAPPING`. Fail if the test directory does not exist under `THEROCK_BIN_DIR`.
+1. **Discover GPU suites**
+   Run `ctest --print-labels --test-dir {THEROCK_BIN_DIR}/{TEST_COMPONENT}`. Collect every label that starts with `ex_gpu_` and whose suffix starts with `gfx` (e.g. `ex_gpu_gfx110X` → `gfx110X`). This yields the set of **available GPU architectures** for which a label exists.
+1. **Choose category**
+   From `TEST_TYPE`: `smoke` → `quick`, else → `standard`.
+1. **Resolve GPU arch**
+   Parse `AMDGPU_FAMILIES` for the first `gfx...` token (e.g. `gfx1151`). If missing or generic, the script will exclude all GPU-specific tests (`-LE ex_gpu`).
+1. **Match GPU to suite**
+   Using `find_matching_gpu_arch()` (from `github_actions_utils`):
+
+- Prefer **exact** match in the discovered set (e.g. `gfx1151`).
+- Else try **wildcard** patterns from most to least specific (e.g. for `gfx1151`: `gfx115X`, then `gfx11X`).
+- If a match is found, add `-L ex_gpu_{matching_arch}`; otherwise add `-LE ex_gpu`.
+
+6. **Build ctest command**
+
+- `ctest -L <category>` (and optionally `-L ex_gpu_<arch>` or `-LE ex_gpu`).
+- Common options: `--output-on-failure`, `--parallel <N>`, `--test-dir`, `-V`, `--tests-information <SHARD_INDEX>,<TOTAL_SHARDS>`.
+- Parallelism: default 8; can be adjusted according to `AMDGPU_FAMILIES`
+
+7. **Run ctest**
+   Execute the command in `THEROCK_DIR` with the environment that includes `ROCM_PATH`, `GTEST_SHARD_INDEX`, and `GTEST_TOTAL_SHARDS`.
+
+______________________________________________________________________
+
+## CI Integration
+
+- **Workflow:** `.github/workflows/test_component.yml` runs the test step with env vars such as `TEST_COMPONENT`, `TEST_TYPE`, `AMDGPU_FAMILIES`, `SHARD_INDEX`, `TOTAL_SHARDS`.
+- **Test script:** For components that use this mechanism, the test script is set to `python .../test_runner.py` in `build_tools/github_actions/fetch_test_configurations.py` (e.g. MIOpen).
+- **Sharding:** The workflow matrix uses `shard_arr` from the same config; the script passes sharding to ctest via `--tests-information` and to GTest via `GTEST_SHARD_INDEX` / `GTEST_TOTAL_SHARDS`.
+
+______________________________________________________________________
+
+## Adding a New Component to This Runner
+
+1. **In the component (e.g. CMake/CTest):**
+
+- Name GPU-specific tests `{target}-{category}-{gpu_arch}-suite`.
+- Assign labels `ex_gpu_{gpu_arch}` and the category label (`quick` / `standard` or equivalent).
+
+2. **In TheRock:**
+
+- In `test_filters.py`, add the job name → directory mapping in `COMPONENT_DIR_MAPPING`.
+- In `fetch_test_configurations.py`, set the component’s `test_script` to `python .../test_filters.py` and set `job_name` (and shards, timeout, etc.) as needed.
+
+After that, the generic flow (discovery → match GPU → run ctest with the right labels) applies without further changes to `test_filters.py` for that component.

build_tools/github_actions/test_executable_scripts/test_runner.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+"""
+This is a generic test runner that can test multiple components.
+This works on components in rocm-libraries/rocm-systems which use test_categories.yml for test categorization.
+
+Environment variables used:
+- TEST_COMPONENT: Job name of the component to test (e.g., "miopen", "rocrand", "hiprand")
+  This is automatically set by the GitHub Actions workflow from the job_name field.
+  The script maps these job names to actual test directory names (e.g., "miopen" -> "MIOpen")
+  Defaults to "miopen" if not set.
+- TEST_TYPE: "smoke" runs tests with "quick" category, otherwise runs "standard" category
+- AMDGPU_FAMILIES: Parsed to extract GPU architecture (e.g., "gfx1151")
+
+The script discovers GPU-specific labels via ctest --print-labels and runs the appropriate tests for the current GPU architecture.
+"""
+
+import sys
+import subprocess
+import re
+import os
+
+import logging
+import shlex
+from pathlib import Path
+
+sys.path.insert(0, os.fspath(Path(__file__).resolve().parent.parent))
+from github_actions_utils import find_matching_gpu_arch
+
+THEROCK_BIN_DIR = os.getenv("THEROCK_BIN_DIR")
+SCRIPT_DIR = Path(__file__).resolve().parent
+THEROCK_DIR = SCRIPT_DIR.parent.parent.parent
+TEST_TYPE = os.getenv("TEST_TYPE", "smoke")
+AMDGPU_FAMILIES = os.getenv("AMDGPU_FAMILIES")
+
+# Map job names to actual test directory names
+# The job names come from TEST_COMPONENT env var (set by GitHub Actions workflow)
+# and need to be mapped to the actual directory names in THEROCK_BIN_DIR
+COMPONENT_DIR_MAPPING = {
+    "miopen": "MIOpen",
+    "rocrand": "rocRAND",
+    "hiprand": "hipRAND",
+    "rocthrust": "rocthrust",
+    "rocprim": "rocprim",
+    "rocwmma": "rocwmma",
+    "hipcub": "hipcub",
+    "hipdnn": "hipdnn",
+    "hipdnn-samples": "hipdnn_samples",
+    "miopen_plugin": "miopen_legacy_plugin",
+    # Add more mappings as needed
+}
+
+# Get the test component from environment (required - no default)
+test_component_job_name = os.getenv("TEST_COMPONENT")
+if not test_component_job_name:
+    print(
+        "ERROR: TEST_COMPONENT environment variable is required but not set.",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+TEST_COMPONENT = COMPONENT_DIR_MAPPING.get(
+    test_component_job_name, test_component_job_name
+)
+
+# GTest sharding
+SHARD_INDEX = os.getenv("SHARD_INDEX", 1)
+TOTAL_SHARDS = os.getenv("TOTAL_SHARDS", 1)
+
+# CTest parallel jobs (use fewer in less capable platforms)
+ctest_parallel_count = 8
+if AMDGPU_FAMILIES and "gfx1152" in AMDGPU_FAMILIES:
+    ctest_parallel_count = 4
+elif AMDGPU_FAMILIES and "gfx1153" in AMDGPU_FAMILIES:
+    ctest_parallel_count = 4
+environ_vars = os.environ.copy()
+# Set the GTEST env vars for Gtest based tests
+# Set ROCM_PATH for tests that rely on it
+environ_vars["GTEST_SHARD_INDEX"] = str(int(SHARD_INDEX) - 1)
+environ_vars["GTEST_TOTAL_SHARDS"] = str(TOTAL_SHARDS)
+ROCM_PATH = Path(THEROCK_BIN_DIR).resolve().parent
+environ_vars["ROCM_PATH"] = str(ROCM_PATH)
+
+logging.basicConfig(level=logging.INFO)
+##############################################
+
+
+def get_available_gpu_suite_tests():
+    """
+    Get all available GPU architecture labels from ctest --print-labels.
+
+    Parses labels of the form ex_gpu_{gpu_arch} (e.g. ex_gpu_gfx110X, ex_gpu_gfx950).
+    Returns a set of gpu_arch strings (e.g., 'gfx110X', 'gfx115X', 'gfx950').
+    """
+    test_dir = Path(THEROCK_BIN_DIR) / TEST_COMPONENT
+    if not test_dir.exists() or not test_dir.is_dir():
+        print(f"Error: Test directory does not exist: {test_dir}", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        # Ensure the component has at least one test
+        list_result = subprocess.run(
+            ["ctest", "-N", "--test-dir", str(test_dir)],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        total_tests = sum(
+            1
+            for line in list_result.stdout.splitlines()
+            if re.search(r"Test\s+#\d+:", line)
+        )
+        if total_tests == 0:
+            print(
+                f"Error: No tests found in {test_dir}. Cannot run test suite.",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        result = subprocess.run(
+            ["ctest", "--print-labels", "--test-dir", str(test_dir)],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+
+        gpu_archs = set()
+        prefix = "ex_gpu_"
+        for line in result.stdout.splitlines():
+            label = line.strip()
+            if label.startswith(prefix):
+                gpu_arch = label[len(prefix) :]
+                if gpu_arch.startswith("gfx"):
+                    gpu_archs.add(gpu_arch)
+
+        return gpu_archs
+    except subprocess.CalledProcessError as e:
+        print(f"Error running ctest --print-labels: {e}", file=sys.stderr)
+        sys.exit(1)
+    except FileNotFoundError:
+        print(
+            "Error: ctest not found. Make sure CMake/CTest is installed.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+
+def build_ctest_command(category, gpu_arch, available_gpu_archs):
+    """
+    Build the appropriate ctest command based on the category and GPU architecture.
+
+    Returns a list of command arguments suitable for subprocess.run()
+    """
+    cmd = ["ctest", "-L", category]
+
+    # Add common ctest parameters
+    cmd.extend(
+        [
+            "--output-on-failure",
+            "--parallel",
+            f"{ctest_parallel_count}",
+            "--test-dir",
+            f"{THEROCK_BIN_DIR}/{TEST_COMPONENT}",
+            "-V",  # Always run in verbose mode
+            # Shards the tests by running a specific set of tests based on starting test (shard_index) and stride (total_shards)
+            "--tests-information",
+            f"{SHARD_INDEX},,{TOTAL_SHARDS}",
+        ]
+    )
+
+    if gpu_arch.lower() in ["generic", "none", ""]:
+        # For generic/unspecified GPU, exclude all GPU-specific suite tests
+        cmd.extend(["-LE", "ex_gpu"])
+        return cmd
+
+    # Find the appropriate GPU suite
+    matching_arch = find_matching_gpu_arch(gpu_arch, available_gpu_archs)
+
+    if matching_arch:
+        # Run the specific GPU suite using the ex_gpu label
+        gpu_label = f"ex_gpu_{matching_arch}"
+        cmd.extend(["-L", gpu_label])
+        print(f"# Using GPU suite label: {gpu_label}")
+    else:
+        # No specific GPU suite found, run standard tests excluding all GPU-specific ones
+        cmd.extend(["-LE", "ex_gpu"])
+        print(f"# No GPU suite found for {gpu_arch}, excluding all ex_gpu tests")
+
+    return cmd
+
+
+def main():
+    # Use only two categories for now - quick and standard - depending on TEST_TYPE.
+    if TEST_TYPE and TEST_TYPE.lower() == "smoke":
+        category = "quick"
+    else:
+        category = "standard"
+
+    # Use AMDGPU_FAMILIES from environment variable, extract gfx<xxx> part
+    gpu_arch = ""
+    if AMDGPU_FAMILIES:
+        # Extract gfx<xxx> pattern from AMDGPU_FAMILIES string
+        # Pattern matches: gfx followed by alphanumeric characters (e.g., gfx1151, gfx950, gfx11X)
+        match = re.search(r"gfx[0-9a-zA-Z]+", AMDGPU_FAMILIES)
+        if match:
+            gpu_arch = match.group(0)
+        else:
+            print(
+                f"# Warning: Could not extract GPU architecture from AMDGPU_FAMILIES='{AMDGPU_FAMILIES}', using default '{gpu_arch}'"
+            )
+
+    print(
+        f"# TEST_COMPONENT: {test_component_job_name} -> Test Directory: {TEST_COMPONENT}"
+    )
+    print(f"# TEST_TYPE: {TEST_TYPE} -> Category: {category}")
+    print(f"# AMDGPU_FAMILIES: {AMDGPU_FAMILIES} -> GPU Architecture: {gpu_arch}")
+    print()
+
+    # Get available GPU suite tests from ctest
+    print("# Discovering available GPU suite tests...")
+    available_gpu_archs = get_available_gpu_suite_tests()
+
+    if available_gpu_archs:
+        print(f"# Found {len(available_gpu_archs)} GPU suite test(s)")
+        print(f"# Available GPU architectures: {sorted(available_gpu_archs)}")
+    else:
+        print("# Warning: No GPU specific test suites available")
+    print()
+
+    # Build the ctest command
+    cmd = build_ctest_command(category, gpu_arch, available_gpu_archs)
+
+    print(f"# Running: {' '.join(cmd)}")
+    print()
+
+    # Execute the command
+    try:
+        logging.info(f"++ Exec [{THEROCK_DIR}]$ {shlex.join(cmd)}")
+        result = subprocess.run(cmd, cwd=THEROCK_DIR, env=environ_vars, check=False)
+        return result.returncode
+    except Exception as e:
+        print(f"Error running ctest: {e}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())