From 218823315f01b0e24691d97dc397756fa0696d1b Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 6 Jan 2026 13:12:31 +0000 Subject: [PATCH 01/28] Restructure benchmarks to unified test framework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renamed benchmarks/ to test_framework/ with support for benchmark, and functional tests. - Framework restructure (benchmarks → test_framework) - Test type organization (benchmark, functional) - Shared utilities moved to framework level - benchmark_client.py → test_client.py rename - benchmark_test_matrix.py → benchmark_matrix.py rename - Functional test base class (functional_base.py) - Updated documentation (main + benchmark READMEs) - Updated all imports Signed-off-by: Lenine Ajagappane --- .../github_actions/benchmarks/README.md | 259 --------------- .../github_actions/benchmarks/__init__.py | 1 - .../fetch_test_configurations.py | 2 +- .../github_actions/test_framework/README.md | 297 +++++++++++++++++ .../github_actions/test_framework/__init__.py | 12 + .../test_framework/benchmark/README.md | 226 +++++++++++++ .../benchmark/benchmark_matrix.py} | 2 +- .../benchmark}/configs/hipblaslt.json | 0 .../benchmark}/configs/rocfft.json | 0 .../benchmark}/scripts/benchmark_base.py | 4 +- .../scripts/test_hipblaslt_benchmark.py | 0 .../scripts/test_rocfft_benchmark.py | 0 .../scripts/test_rocrand_benchmark.py | 0 .../scripts/test_rocsolver_benchmark.py | 0 .../configs/config.yml | 0 .../configs/miopen_driver_conv.json | 49 +++ .../functional/functional_matrix.py | 32 ++ .../functional/scripts/__init__.py | 2 + .../functional/scripts/functional_base.py | 304 ++++++++++++++++++ .../scripts/test_miopendriver_conv.py | 240 ++++++++++++++ .../performance/performance_matrix.py | 26 ++ .../performance/scripts/__init__.py | 0 .../utils/README.md | 6 +- .../utils/__init__.py | 6 +- .../utils/config/__init__.py | 0 .../utils/config/config_helper.py | 0 .../utils/config/config_parser.py | 0 .../utils/config/config_validator.py | 0 .../utils/constants.py | 0 .../utils/exceptions.py | 0 .../utils/logger.py | 0 .../utils/results/__init__.py | 0 .../utils/results/results_api.py | 0 .../utils/results/results_handler.py | 0 .../utils/results/schemas/payload_schema.json | 0 .../utils/system/__init__.py | 0 .../utils/system/hardware.py | 0 .../utils/system/platform.py | 0 .../utils/system/rocm_detector.py | 0 .../utils/system/system_detector.py | 0 .../utils/test_client.py} | 14 +- .../github_actions/tests/configure_ci_test.py | 2 +- 42 files changed, 1209 insertions(+), 275 deletions(-) delete mode 100644 build_tools/github_actions/benchmarks/README.md delete mode 100644 build_tools/github_actions/benchmarks/__init__.py create mode 100644 build_tools/github_actions/test_framework/README.md create mode 100644 build_tools/github_actions/test_framework/__init__.py create mode 100644 build_tools/github_actions/test_framework/benchmark/README.md rename build_tools/github_actions/{benchmarks/benchmark_test_matrix.py => test_framework/benchmark/benchmark_matrix.py} (96%) rename build_tools/github_actions/{benchmarks => test_framework/benchmark}/configs/hipblaslt.json (100%) rename build_tools/github_actions/{benchmarks => test_framework/benchmark}/configs/rocfft.json (100%) rename build_tools/github_actions/{benchmarks => test_framework/benchmark}/scripts/benchmark_base.py (98%) rename build_tools/github_actions/{benchmarks => test_framework/benchmark}/scripts/test_hipblaslt_benchmark.py (100%) rename build_tools/github_actions/{benchmarks => test_framework/benchmark}/scripts/test_rocfft_benchmark.py (100%) rename build_tools/github_actions/{benchmarks => test_framework/benchmark}/scripts/test_rocrand_benchmark.py (100%) rename build_tools/github_actions/{benchmarks => test_framework/benchmark}/scripts/test_rocsolver_benchmark.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/configs/config.yml (100%) create mode 100644 build_tools/github_actions/test_framework/functional/configs/miopen_driver_conv.json create mode 100644 build_tools/github_actions/test_framework/functional/functional_matrix.py create mode 100644 build_tools/github_actions/test_framework/functional/scripts/__init__.py create mode 100644 build_tools/github_actions/test_framework/functional/scripts/functional_base.py create mode 100644 build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py create mode 100644 build_tools/github_actions/test_framework/performance/performance_matrix.py create mode 100644 build_tools/github_actions/test_framework/performance/scripts/__init__.py rename build_tools/github_actions/{benchmarks => test_framework}/utils/README.md (95%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/__init__.py (86%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/config/__init__.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/config/config_helper.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/config/config_parser.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/config/config_validator.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/constants.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/exceptions.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/logger.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/results/__init__.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/results/results_api.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/results/results_handler.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/results/schemas/payload_schema.json (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/system/__init__.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/system/hardware.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/system/platform.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/system/rocm_detector.py (100%) rename build_tools/github_actions/{benchmarks => test_framework}/utils/system/system_detector.py (100%) rename build_tools/github_actions/{benchmarks/utils/benchmark_client.py => test_framework/utils/test_client.py} (94%) diff --git a/build_tools/github_actions/benchmarks/README.md b/build_tools/github_actions/benchmarks/README.md deleted file mode 100644 index 9bcbf21f589..00000000000 --- a/build_tools/github_actions/benchmarks/README.md +++ /dev/null @@ -1,259 +0,0 @@ -# Benchmark Testing Framework - -Automated benchmark testing framework for ROCm libraries with system detection, results collection, and performance tracking. - -## Table of Contents - -- [Features](#features) -- [Quick Start](#quick-start) -- [Project Structure](#project-structure) -- [CI/CD Integration](#cicd-integration) -- [Architecture](#architecture) -- [Adding New Benchmarks](#adding-new-benchmarks) - -## Features - -- **Automated Benchmark Execution** - ROCfft, ROCrand, ROCsolver, hipBLASLt -- **System Auto-Detection** - Hardware, OS, GPU, and ROCm version detection -- **Results Management** - Local storage (JSON) and API upload with retry logic -- **Performance Tracking** - LKG (Last Known Good) comparison -- **Comprehensive Logging** - File rotation and configurable log levels -- **Modular Architecture** - Extensible design for adding new benchmarks -- **CI/CD Integration** - Parallel execution with regular tests in nightly CI - -## Quick Start - -### Available Benchmarks - -- `benchmarks/scripts/test_hipblaslt_benchmark.py` - hipBLASLt benchmark suite -- `benchmarks/scripts/test_rocsolver_benchmark.py` - ROCsolver benchmark suite -- `benchmarks/scripts/test_rocrand_benchmark.py` - ROCrand benchmark suite -- `benchmarks/scripts/test_rocfft_benchmark.py` - ROCfft benchmark suite - -## Project Structure - -``` -build_tools/github_actions/ -├── benchmarks/ # All benchmark-related code -│ ├── scripts/ # Benchmark test implementations -│ │ ├── benchmark_base.py # Base class for all benchmarks -│ │ ├── test_hipblaslt_benchmark.py -│ │ ├── test_rocsolver_benchmark.py -│ │ ├── test_rocrand_benchmark.py -│ │ └── test_rocfft_benchmark.py -│ │ -│ ├── configs/ # Benchmark configs -│ │ ├── config.yml # Framework configuration -│ │ ├── hipblaslt.json # hipBLASLt benchmark config -│ │ └── rocfft.json # ROCfft benchmark config -│ │ -│ ├── utils/ # Benchmark utilities -│ │ ├── benchmark_client.py # Main client API -│ │ ├── logger.py # Logging utilities -│ │ ├── config/ # Configuration management -│ │ ├── system/ # System detection (GPU, ROCm, OS) -│ │ └── results/ # Results API client & schemas -│ │ -│ ├── benchmark_test_matrix.py # Benchmark matrix definitions -│ └── README.md # This file -│ -├── test_executable_scripts/ # Regular functional tests -│ -├── configure_ci.py # CI workflow orchestration -├── fetch_test_configurations.py # Test matrix builder -└── github_actions_utils.py # GitHub Actions utilities -``` - -## CI/CD Integration - -### When Benchmark Tests Run - -Benchmark tests run **only on nightly CI builds** to save time and resources on pull request validation: - -| Workflow Trigger | Benchmark Tests | Regular Tests | -| -------------------------- | ------------------------------ | ---------------------- | -| **Pull Request (PR)** | Skipped | Run (smoke: 1 shard) | -| **Nightly CI (scheduled)** | Run (in parallel, always full) | Run (full: all shards) | -| **Push to main** | Skipped | Run (smoke: 1 shard) | -| **Manual workflow** | Optional | Optional | - -**Note:** Benchmarks always run with `total_shards=1` and do not use `test_type` or `test_labels` filtering. - -### Parallel Execution Architecture - -Benchmarks run **in parallel** with regular tests for faster CI execution: - -``` -ci_nightly.yml → ci_linux.yml - │ - ├─ build_artifacts (30 min) - │ - ├─ test_artifacts (45 min) ────┐ - │ └─ Regular tests │ Run in - │ (rocblas, hipblas, ...) │ PARALLEL - │ │ - └─ test_benchmarks (60 min) ────┘ - └─ Benchmark tests - (hipblaslt_bench, rocfft_bench, ...) - -``` - -### Available Benchmark Tests in CI - -The following benchmark tests are defined in `benchmarks/benchmark_test_matrix.py`: - -| Test Name | Library | Platform | Timeout | Shards | -| ----------------- | --------- | -------- | ------- | ------ | -| `hipblaslt_bench` | hipBLASLt | Linux | 60 min | 1 | -| `rocsolver_bench` | ROCsolver | Linux | 60 min | 1 | -| `rocrand_bench` | ROCrand | Linux | 60 min | 1 | -| `rocfft_bench` | ROCfft | Linux | 60 min | 1 | - -### Implementation Details - -1. **Nightly Trigger:** `configure_ci.py` adds benchmark test names to test labels -1. **Parallel Jobs:** `ci_linux.yml` spawns two parallel jobs: - - `test_artifacts` → Regular tests via `test_artifacts.yml` - - `test_benchmarks` → Benchmarks via `test_benchmarks.yml` -1. **Matrix Generation:** `fetch_test_configurations.py` uses `IS_BENCHMARK_WORKFLOW=true` flag to select only benchmarks from `benchmark_test_matrix.py` -1. **Dedicated Runners:** Benchmarks can use dedicated GPU runners specified by `benchmark-runs-on` in `amdgpu_family_matrix.py` - -## Architecture - -### Workflow Integration - -``` -.github/workflows/ci_nightly.yml - └─ calls → ci_linux.yml - ├─ job: build_artifacts - ├─ job: test_artifacts (parallel) - └─ job: test_benchmarks (parallel) ← NEW - └─ calls → test_benchmarks.yml - ├─ configure_benchmark_matrix - │ └─ fetch_test_configurations.py - │ (IS_BENCHMARK_WORKFLOW=true) - └─ run_benchmarks - └─ test_component.yml (matrix) -``` - -### Benchmark Script Execution Flow - -``` -1. Initialize BenchmarkClient - ↓ Auto-detect system (GPU, OS, ROCm version) - ↓ Load configuration from config.yml - -2. Run Benchmarks - ↓ Execute benchmark binary - ↓ Capture output to log file - -3. Parse Results - ↓ Extract metrics from log file - ↓ Structure data according to schema - -4. Upload Results - ↓ Submit to API (with retry) - ↓ Save JSON locally - -5. Compare with LKG - ↓ Fetch last known good results - ↓ Calculate performance delta - -6. Report Results - ↓ Display formatted table - ↓ Append to GitHub Actions step summary - ↓ Return exit code (0=success, 1=failure) -``` - -## Adding New Benchmarks - -To add a new benchmark test to the nightly CI: - -### 1. Create Benchmark Script - -Create `benchmarks/scripts/test_your_benchmark.py`. Reference existing benchmarks like `test_rocfft_benchmark.py` as a template. - -Key components: - -- Inherit from `BenchmarkBase` class -- Implement `run_benchmarks()` - executes binary and logs output -- Implement `parse_results()` - parses logs and returns structured data -- Results are automatically uploaded to API via base class - -Example: - -```python -import sys -from pathlib import Path -from typing import Dict, List, Tuple, Any -from prettytable import PrettyTable - -sys.path.insert(0, str(Path(__file__).parent.parent)) # For utils -sys.path.insert(0, str(Path(__file__).parent)) # For benchmark_base -from benchmark_base import BenchmarkBase, run_benchmark_main -from utils.logger import log - - -class YourBenchmark(BenchmarkBase): - def __init__(self): - super().__init__(benchmark_name="your_lib", display_name="YourLib") - self.log_file = self.script_dir / "your_lib_bench.log" - - def run_benchmarks(self) -> None: - """Execute benchmark binary and log output.""" - # Load config if needed - config_file = self.script_dir.parent / "configs" / "your_lib.json" - - # Your benchmark execution logic here - pass - - def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable]: - """Parse log file and return (test_results, table).""" - # Your parsing logic here - # Use self.create_test_result() to build result dictionaries - pass - - -if __name__ == "__main__": - run_benchmark_main(YourBenchmark()) # Handles sys.exit() internally -``` - -### 2. Add to Benchmark Test Matrix - -Edit `benchmarks/benchmark_test_matrix.py`: - -```python -"your_benchmark": { - "job_name": "your_benchmark", - "fetch_artifact_args": "--your-lib --tests", - "timeout_minutes": 60, - "test_script": f"python {_get_benchmark_script_path('test_your_benchmark.py')}", - "platform": ["linux"], - "total_shards": 1, -}, -``` - -The benchmark will automatically be included in nightly CI runs: - -- `configure_ci.py` adds benchmark names to test labels -- `ci_linux.yml` spawns `test_benchmarks` job -- `test_benchmarks.yml` calls `fetch_test_configurations.py` with `IS_BENCHMARK_WORKFLOW=true` -- Only benchmarks from `benchmark_test_matrix.py` are executed - -### 3. Test Locally - -```bash -# Set environment variables -export THEROCK_BIN_DIR=/path/to/build/bin -export ARTIFACT_RUN_ID=local-test -export AMDGPU_FAMILIES=gfx950-dcgpu - -# Run the benchmark -python3 build_tools/github_actions/benchmarks/scripts/test_your_benchmark.py -``` - -## Related Documentation - -- [Utils Module Documentation](utils/README.md) - Utility modules reference -- [CI Nightly Workflow](https://github.com/ROCm/TheRock/actions/workflows/ci_nightly.yml) - GitHub Actions -- [Test Benchmarks Workflow](../../.github/workflows/test_benchmarks.yml) - Benchmark execution workflow diff --git a/build_tools/github_actions/benchmarks/__init__.py b/build_tools/github_actions/benchmarks/__init__.py deleted file mode 100644 index 1efc188e614..00000000000 --- a/build_tools/github_actions/benchmarks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Benchmark testing framework for ROCm libraries.""" diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index 4e03e580ba4..cf435980d73 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -11,7 +11,7 @@ from pathlib import Path from github_actions_utils import * -from benchmarks.benchmark_test_matrix import benchmark_matrix +from test_framework.benchmark.benchmark_matrix import benchmark_matrix logging.basicConfig(level=logging.INFO) diff --git a/build_tools/github_actions/test_framework/README.md b/build_tools/github_actions/test_framework/README.md new file mode 100644 index 00000000000..117c98a844e --- /dev/null +++ b/build_tools/github_actions/test_framework/README.md @@ -0,0 +1,297 @@ +# TheRock Test Framework + +Unified testing framework for TheRock ROCm distribution, supporting benchmark, performance, and functional testing with automated execution, system detection, and results management. + +## Table of Contents + +- [Overview](#overview) +- [Test Types](#test-types) +- [Quick Start](#quick-start) +- [Project Structure](#project-structure) +- [CI/CD Integration](#cicd-integration) +- [Architecture](#architecture) +- [Adding Tests](#adding-tests) + +## Overview + +The test framework provides a unified infrastructure for multiple test types: + +### Features + +- **Multi-Type Testing** - Benchmark, performance, and functional tests +- **Shared Infrastructure** - Common utilities, configuration, and results management +- **System Auto-Detection** - Hardware, OS, GPU, and ROCm version detection +- **Results Management** - Local storage (JSON) and API upload with retry logic +- **Comprehensive Logging** - File rotation and configurable log levels +- **Error Handling** - Custom exceptions with clear, actionable messages +- **Modular Architecture** - Extensible design for adding new test types +- **CI/CD Integration** - Parallel execution in nightly CI + +## Test Types + +### 1. Benchmark Tests (`benchmark/`) + +**Purpose:** Detect performance regressions by comparing against Last Known Good (LKG) baselines. + +- **Result:** PASS/FAIL/UNKNOWN +- **Comparison:** Current vs baseline (with tolerance) +- **Frequency:** Every nightly CI +- **Use Case:** Automated CI gates to prevent regressions +- **Example:** "GEMM performance is 5% slower than baseline" → FAIL + +**See:** [benchmark/README.md](benchmark/README.md) + +### 2. Performance Tests (`performance/`) + +**Purpose:** Comprehensive performance characterization with detailed metrics and analysis. + +- **Result:** Detailed metrics, scaling curves, bottleneck analysis +- **Comparison:** Current vs hardware specs/targets +- **Frequency:** Weekly, before releases, on-demand +- **Use Case:** Performance optimization, hardware characterization +- **Example:** "Achieved 42.5 TFLOPS (94% of peak)" + scaling analysis + +**Status:** Framework ready, tests to be implemented + +### 3. Functional Tests (`functional/`) + +**Purpose:** Validate correctness, API contracts, and expected behavior. + +- **Result:** PASS/FAIL with detailed error messages +- **Comparison:** Output vs expected results +- **Frequency:** Every PR, nightly CI +- **Use Case:** Correctness validation, API testing +- **Example:** "Matrix multiplication result matches expected output" → PASS + +**Status:** Framework ready, tests to be implemented + +## Quick Start + +### Running Tests + +```bash +# Set environment variables +export THEROCK_BIN_DIR=/path/to/therock/build/bin +export ARTIFACT_RUN_ID=local-test +export AMDGPU_FAMILIES=gfx950-dcgpu + +# Run benchmark test +python test_framework/benchmark/scripts/test_hipblaslt_benchmark.py + +# Run performance test (when implemented) +python test_framework/performance/scripts/test_rocblas_performance.py + +# Run functional test (when implemented) +python test_framework/functional/scripts/test_rocblas_functional.py +``` + +### Available Tests + +**Benchmark Tests:** +- `test_hipblaslt_benchmark.py` - hipBLASLt GEMM benchmarks +- `test_rocfft_benchmark.py` - rocFFT transform benchmarks +- `test_rocrand_benchmark.py` - rocRAND generation benchmarks +- `test_rocsolver_benchmark.py` - ROCsolver linear algebra benchmarks + +**Performance Tests:** (to be implemented) + +**Functional Tests:** (to be implemented) + +## Project Structure + +``` +test_framework/ +├── __init__.py +├── README.md # This file +│ +├── configs/ # SHARED configuration +│ └── config.yml # Framework config (logging, API, execution) +│ +├── benchmark/ # Benchmark tests (LKG comparison) +│ ├── scripts/ # Test implementations +│ │ ├── benchmark_base.py # Base class with LKG logic +│ │ └── test_*_benchmark.py # Individual benchmark tests +│ ├── configs/ # Test-specific configurations +│ │ ├── hipblaslt.json +│ │ └── rocfft.json +│ ├── benchmark_matrix.py # CI test matrix +│ └── README.md # Benchmark-specific docs +│ +├── performance/ # Performance characterization tests +│ ├── scripts/ # Test implementations (to be added) +│ ├── configs/ # Test-specific configurations +│ ├── performance_matrix.py # CI test matrix +│ └── README.md # (to be created) +│ +├── functional/ # Functional/correctness tests +│ ├── scripts/ # Test implementations (to be added) +│ ├── configs/ # Test-specific configurations +│ ├── functional_matrix.py # CI test matrix +│ └── README.md # (to be created)│ +│ +└── utils/ # SHARED utilities for all test types + ├── exceptions.py # Custom exception classes + │ ├── BenchmarkExecutionError # Execution/parsing failures + │ ├── BenchmarkResultError # Result validation failures + │ └── FrameworkException # Base exception + │ + ├── logger.py # Logging utilities + ├── test_client.py # Test execution client + ├── constants.py # Global constants + │ + ├── config/ # Configuration parsers + │ ├── config_parser.py + │ ├── config_validator.py + │ └── config_helper.py + │ + ├── results/ # Results handling & LKG + │ ├── results_api.py # API for storing/retrieving results + │ └── results_handler.py # Process and format results + │ + └── system/ # Hardware & ROCm detection + ├── hardware.py # GPU detection and capabilities + ├── platform.py # Platform-specific utilities + └── rocm_detector.py # ROCm version detection +``` + +## CI/CD Integration + +### Test Execution Schedule + +Benchmark tests run **only on nightly CI builds** to save time and resources on pull request validation: + +| Workflow Trigger | Benchmark/Functional Tests | Regular Tests | +| -------------------------- | ------------------------------ | ---------------------- | +| **Pull Request (PR)** | Skipped | Run (smoke: 1 shard) | +| **Nightly CI (scheduled)** | Run (in parallel, always full) | Run (full: all shards) | +| **Push to main** | Skipped | Run (smoke: 1 shard) | +| **Manual workflow** | Optional | Optional | + +**Note:** Benchmarks always run with `total_shards=1` and do not use `test_type` or `test_labels` filtering. + +### Parallel Execution Architecture + +Benchmarks run **in parallel** with regular tests for faster CI execution: + +``` +ci_nightly.yml → ci_linux.yml + │ + ├─ build_artifacts (30 min) + │ + ├─ test_artifacts (45 min) ────┐ + │ └─ Regular tests │ Run in + │ (rocblas, hipblas, ...) │ PARALLEL + │ │ + └─ test_benchmarks (60 min) ────┘ + └─ Benchmark tests + (hipblaslt_bench, rocfft_bench, ...) + +``` + + +### Workflow Integration Details + +``` +.github/workflows/ci_nightly.yml + └─ calls → ci_linux.yml + ├─ job: build_artifacts + │ └─ Builds TheRock binaries + │ + ├─ job: test_artifacts (parallel) + │ └─ calls → test_artifacts.yml + │ └─ Functional tests matrix + │ + └─ job: test_benchmarks (parallel) + └─ calls → test_benchmarks.yml + ├─ configure_benchmark_matrix + │ └─ fetch_test_configurations.py + │ (IS_BENCHMARK_WORKFLOW=true) + └─ run_benchmarks + └─ test_component.yml (matrix) +``` + +## Architecture + +### Execution Flow (Common Pattern) + +``` +┌─────────────────────────────────────────┐ +│ 1. Initialize Test Runner │ +│ - Auto-detect system (GPU, ROCm) │ +│ - Load configuration │ +│ - Setup logging │ +└──────────────┬──────────────────────────┘ + ↓ +┌─────────────────────────────────────────┐ +│ 2. Execute Tests │ +│ - Run test binaries/scripts │ +│ - Capture output to log files │ +│ - Handle errors gracefully │ +└──────────────┬──────────────────────────┘ + ↓ +┌─────────────────────────────────────────┐ +│ 3. Parse Results │ +│ - Extract metrics from logs │ +│ - Structure data according to schema │ +│ - Validate results │ +└──────────────┬──────────────────────────┘ + ↓ +┌─────────────────────────────────────────┐ +│ 4. Process Results (Type-Specific) │ +│ Benchmark: Compare with LKG baseline │ +│ Performance: Analyze metrics/trends │ +│ Functional: Validate correctness │ +└──────────────┬──────────────────────────┘ + ↓ +┌─────────────────────────────────────────┐ +│ 5. Report Results │ +│ - Display formatted output │ +│ - Upload to results API │ +│ - Append to GitHub Actions summary │ +│ - Return exit code │ +└─────────────────────────────────────────┘ +``` + +## Adding Tests + +### Adding a Benchmark Test + +See detailed guide in [benchmark/README.md](benchmark/README.md#adding-a-new-benchmark) + +Quick steps: +1. Create `benchmark/scripts/test_yourlib_benchmark.py` +2. Inherit from `BenchmarkBase`, implement `run_benchmarks()` and `parse_results()` +3. Add to `benchmark/benchmark_matrix.py` +4. Test locally + +### Adding a Performance Test + +1. Create `performance/scripts/test_yourlib_performance.py` +2. Inherit from `PerformanceRunner` (to be implemented) +3. Implement metrics collection and analysis +4. Add to `performance/performance_matrix.py` + +### Adding a Functional Test + +1. Create `functional/scripts/test_yourlib_functional.py` +2. Inherit from `FunctionalRunner` (to be implemented) +3. Implement validation logic +4. Add to `functional/functional_matrix.py` + + +### Getting Help + +- **Framework issues:** See this README +- **Benchmark-specific:** See [benchmark/README.md](benchmark/README.md) +- **Performance tests:** See `performance/README.md` (to be created) +- **Functional tests:** See `functional/README.md` (to be created) +- **Utilities:** See [utils/README.md](utils/README.md) + + +## Related Files + +- `configure_ci.py` - CI workflow orchestration +- `fetch_test_configurations.py` - Test matrix builder +- `github_actions_utils.py` - GitHub Actions utilities +- `.github/workflows/test_benchmarks.yml` - Benchmark execution workflow +- `.github/workflows/ci_nightly.yml` - Nightly CI orchestration diff --git a/build_tools/github_actions/test_framework/__init__.py b/build_tools/github_actions/test_framework/__init__.py new file mode 100644 index 00000000000..60a5207df86 --- /dev/null +++ b/build_tools/github_actions/test_framework/__init__.py @@ -0,0 +1,12 @@ +""" +TheRock Test Framework + +Unified framework for benchmark, performance, and functional testing. + +Test Types: +- benchmark: Regression detection via LKG comparison (PASS/FAIL gates) +- performance: Detailed performance characterization and analysis +- functional: Correctness validation and API testing +""" + +__version__ = "1.0.0" diff --git a/build_tools/github_actions/test_framework/benchmark/README.md b/build_tools/github_actions/test_framework/benchmark/README.md new file mode 100644 index 00000000000..0441262b464 --- /dev/null +++ b/build_tools/github_actions/test_framework/benchmark/README.md @@ -0,0 +1,226 @@ +# Benchmark Tests + +Regression detection tests that compare current performance against Last Known Good (LKG) baselines. + +## Purpose + +Benchmark tests detect **performance regressions** by comparing test results against established baselines: +- **Result:** PASS (within tolerance) / FAIL (regression detected) / UNKNOWN (no baseline) +- **Frequency:** Every nightly CI +- **Use Case:** Automated CI gates to prevent performance degradation + +## Quick Start + +### Available Benchmark Tests + +| Test Script | Library | Description | +|-------------|---------|-------------| +| `test_hipblaslt_benchmark.py` | hipBLASLt | Matrix multiplication benchmarks (GEMM) | +| `test_rocfft_benchmark.py` | rocFFT | FFT benchmarks (1D, 2D, 3D transforms) | +| `test_rocrand_benchmark.py` | rocRAND | Random number generation benchmarks | +| `test_rocsolver_benchmark.py` | ROCsolver | Dense linear algebra benchmarks | + +### Running Locally + +```bash +# Set required environment variables +export THEROCK_BIN_DIR=/path/to/therock/build/bin +export ARTIFACT_RUN_ID=local-test +export AMDGPU_FAMILIES=gfx950-dcgpu + +# Run a benchmark +cd build_tools/github_actions/test_framework/benchmark/scripts +python test_hipblaslt_benchmark.py +``` + +## Directory Structure + +``` +benchmark/ +├── scripts/ # Benchmark implementations +│ ├── benchmark_base.py # Base class (LKG comparison logic) +│ ├── test_hipblaslt_benchmark.py +│ ├── test_rocfft_benchmark.py +│ ├── test_rocrand_benchmark.py +│ └── test_rocsolver_benchmark.py +│ +├── configs/ # Benchmark-specific test configurations +│ ├── hipblaslt.json # Test parameters (matrix sizes, precisions) +│ └── rocfft.json # Test parameters (FFT sizes, dimensions) +│ +├── benchmark_matrix.py # CI test matrix definitions +└── README.md # This file +``` + +## Benchmark Test Matrix + +Tests defined in `benchmark_matrix.py` for nightly CI: + +| Test Name | Library | Platform | Timeout | Artifacts Needed | +|-----------|---------|----------|---------|------------------| +| `hipblaslt_bench` | hipBLASLt | Linux | 60 min | `--blas --tests` | +| `rocfft_bench` | rocFFT | Linux | 60 min | `--fft --rand --tests` | +| `rocrand_bench` | rocRAND | Linux | 60 min | `--rand --tests` | +| `rocsolver_bench` | ROCsolver | Linux | 60 min | `--blas --tests` | + +## How Benchmark Tests Work + +### 1. Execution Flow + +``` +┌─────────────────────────────────────────┐ +│ 1. Initialize Benchmark │ +│ - Auto-detect GPU, ROCm version │ +│ - Load configuration │ +└──────────────┬──────────────────────────┘ + ↓ +┌─────────────────────────────────────────┐ +│ 2. Run Benchmark Binary │ +│ - Execute (e.g., rocblas-bench) │ +│ - Capture output to log file │ +└──────────────┬──────────────────────────┘ + ↓ +┌─────────────────────────────────────────┐ +│ 3. Parse Results │ +│ - Extract metrics from logs │ +│ - Structure as test results │ +└──────────────┬──────────────────────────┘ + ↓ +┌─────────────────────────────────────────┐ +│ 4. Compare with LKG Baseline │ +│ - Fetch last known good results │ +│ - Calculate performance delta │ +│ - Determine: PASS/FAIL/UNKNOWN │ +└──────────────┬──────────────────────────┘ + ↓ +┌─────────────────────────────────────────┐ +│ 5. Report Results │ +│ - Display formatted table │ +│ - Upload to results API │ +│ - Return exit code (0=pass, 1=fail) │ +└─────────────────────────────────────────┘ +``` + +### 2. LKG Comparison + +```python +# Pseudocode +current_score = 42.5 # TFLOPS +lkg_baseline = 45.0 # TFLOPS from last successful run +tolerance = 0.05 # 5% tolerance + +if current_score < lkg_baseline * (1 - tolerance): + result = "FAIL" # Performance regression! +elif lkg_baseline is None: + result = "UNKNOWN" # No baseline data yet +else: + result = "PASS" # Performance acceptable +``` + +### 3. Result Statuses + +| Status | Meaning | Action | +|--------|---------|--------| +| **PASS** | Performance within tolerance of baseline | ✅ CI passes | +| **FAIL** | Performance degraded beyond tolerance | ❌ CI fails, blocks merge | +| **UNKNOWN** | No baseline data available (new test) | ⚠️ CI passes, baseline established | + +## Adding a New Benchmark + +### Step 1: Create Benchmark Script + +Create `scripts/test_yourlib_benchmark.py`: + +```python +#!/usr/bin/env python3 +import sys +from pathlib import Path +from typing import Dict, List, Tuple, Any +from prettytable import PrettyTable + +# Add parent directories to path for imports +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # For utils +sys.path.insert(0, str(Path(__file__).parent)) # For benchmark_base + +from benchmark_base import BenchmarkBase, run_benchmark_main +from utils.logger import log + + +class YourLibBenchmark(BenchmarkBase): + """Benchmark tests for YourLib library.""" + + def __init__(self): + super().__init__( + benchmark_name="yourlib", + display_name="YourLib" + ) + self.benchmark_bin = "yourlib-bench" + self.log_file = self.script_dir / "yourlib_bench.log" + + def run_benchmarks(self) -> None: + """Execute benchmark binary and log output.""" + # Your benchmark execution logic + # Use self.run_command() to execute binaries + pass + + def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable]: + """Parse log file and return (test_results, results_table).""" + test_results = [] + table = PrettyTable() + + # Your parsing logic + # Use self.create_test_result() to build result dictionaries + + return test_results, table + + +if __name__ == "__main__": + run_benchmark_main(YourLibBenchmark()) +``` + +### Step 2: Add to Benchmark Matrix + +Edit `benchmark_matrix.py`: + +```python +"yourlib_bench": { + "job_name": "yourlib_bench", + "fetch_artifact_args": "--yourlib --tests", + "timeout_minutes": 60, + "test_script": f"python {_get_benchmark_script_path('test_yourlib_benchmark.py')}", + "platform": ["linux"], + "total_shards": 1, +}, +``` + + +### Step 3: Test Locally + +```bash +export THEROCK_BIN_DIR=/path/to/build/bin +export ARTIFACT_RUN_ID=local-test +export AMDGPU_FAMILIES=gfx950-dcgpu + +python scripts/test_yourlib_benchmark.py +``` + +## CI Integration + +Benchmark tests automatically run in nightly CI: + +1. **Trigger:** Nightly scheduled run +2. **Execution:** Parallel with regular tests +3. **Matrix:** Generated from `benchmark_matrix.py` +4. **Runners:** Dedicated GPU runners (if configured) +5. **Results:** Uploaded to results API, compared with LKG + +See [main test framework README](../README.md) for full CI architecture. + + +## Related Documentation + +- [Test Framework README](../README.md) - Main framework documentation +- [Shared Utils](../utils/README.md) - Utility modules reference +- [Performance Tests](../performance/) - Performance characterization tests +- [Functional Tests](../functional/) - Correctness validation tests + diff --git a/build_tools/github_actions/benchmarks/benchmark_test_matrix.py b/build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py similarity index 96% rename from build_tools/github_actions/benchmarks/benchmark_test_matrix.py rename to build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py index 453718ea7b2..92ce200dba0 100644 --- a/build_tools/github_actions/benchmarks/benchmark_test_matrix.py +++ b/build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py @@ -8,7 +8,7 @@ from pathlib import Path # Note: these paths are relative to the repository root. -SCRIPT_DIR = Path("build_tools") / "github_actions" / "benchmarks" / "scripts" +SCRIPT_DIR = Path("build_tools") / "github_actions" / "test_framework" / "benchmark" / "scripts" def _get_benchmark_script_path(script_name: str) -> str: diff --git a/build_tools/github_actions/benchmarks/configs/hipblaslt.json b/build_tools/github_actions/test_framework/benchmark/configs/hipblaslt.json similarity index 100% rename from build_tools/github_actions/benchmarks/configs/hipblaslt.json rename to build_tools/github_actions/test_framework/benchmark/configs/hipblaslt.json diff --git a/build_tools/github_actions/benchmarks/configs/rocfft.json b/build_tools/github_actions/test_framework/benchmark/configs/rocfft.json similarity index 100% rename from build_tools/github_actions/benchmarks/configs/rocfft.json rename to build_tools/github_actions/test_framework/benchmark/configs/rocfft.json diff --git a/build_tools/github_actions/benchmarks/scripts/benchmark_base.py b/build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py similarity index 98% rename from build_tools/github_actions/benchmarks/scripts/benchmark_base.py rename to build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py index c8100263447..7666d7577bc 100644 --- a/build_tools/github_actions/benchmarks/scripts/benchmark_base.py +++ b/build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py @@ -9,7 +9,7 @@ # Add parent directory to path for utils import sys.path.insert(0, str(Path(__file__).parent.parent)) # benchmarks/ sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # github_actions/ -from utils import BenchmarkClient +from utils import TestClient from utils.logger import log from github_actions_utils import gha_append_step_summary @@ -197,7 +197,7 @@ def run(self) -> int: log.info(f"Initializing {self.display_name} Benchmark Test") # Initialize benchmark client and print system info - self.client = BenchmarkClient(auto_detect=True) + self.client = TestClient(auto_detect=True) self.client.print_system_summary() # Run benchmarks (implemented by child class) diff --git a/build_tools/github_actions/benchmarks/scripts/test_hipblaslt_benchmark.py b/build_tools/github_actions/test_framework/benchmark/scripts/test_hipblaslt_benchmark.py similarity index 100% rename from build_tools/github_actions/benchmarks/scripts/test_hipblaslt_benchmark.py rename to build_tools/github_actions/test_framework/benchmark/scripts/test_hipblaslt_benchmark.py diff --git a/build_tools/github_actions/benchmarks/scripts/test_rocfft_benchmark.py b/build_tools/github_actions/test_framework/benchmark/scripts/test_rocfft_benchmark.py similarity index 100% rename from build_tools/github_actions/benchmarks/scripts/test_rocfft_benchmark.py rename to build_tools/github_actions/test_framework/benchmark/scripts/test_rocfft_benchmark.py diff --git a/build_tools/github_actions/benchmarks/scripts/test_rocrand_benchmark.py b/build_tools/github_actions/test_framework/benchmark/scripts/test_rocrand_benchmark.py similarity index 100% rename from build_tools/github_actions/benchmarks/scripts/test_rocrand_benchmark.py rename to build_tools/github_actions/test_framework/benchmark/scripts/test_rocrand_benchmark.py diff --git a/build_tools/github_actions/benchmarks/scripts/test_rocsolver_benchmark.py b/build_tools/github_actions/test_framework/benchmark/scripts/test_rocsolver_benchmark.py similarity index 100% rename from build_tools/github_actions/benchmarks/scripts/test_rocsolver_benchmark.py rename to build_tools/github_actions/test_framework/benchmark/scripts/test_rocsolver_benchmark.py diff --git a/build_tools/github_actions/benchmarks/configs/config.yml b/build_tools/github_actions/test_framework/configs/config.yml similarity index 100% rename from build_tools/github_actions/benchmarks/configs/config.yml rename to build_tools/github_actions/test_framework/configs/config.yml diff --git a/build_tools/github_actions/test_framework/functional/configs/miopen_driver_conv.json b/build_tools/github_actions/test_framework/functional/configs/miopen_driver_conv.json new file mode 100644 index 00000000000..f34200a76d4 --- /dev/null +++ b/build_tools/github_actions/test_framework/functional/configs/miopen_driver_conv.json @@ -0,0 +1,49 @@ +{ + "description": "MIOpen Driver Convolution Test Configurations", + "test_suites": { + "Forward_Conv_1": { + "algorithm": "ConvAsmImplicitGemmV4R1DynamicFwd", + "commands": [ + "conv -n 16 -c 16 -H 56 -W 56 -k 64 -y 1 -x 1 -p 0 -q 0 -u 1 -v 1 -l 1 -j 1 -F 1 -t 1", + "conv -n 16 -c 64 -H 34 -W 34 -k 64 -y 3 -x 3 -p 0 -q 0 -u 1 -v 1 -l 1 -j 1 -F 1 -t 1", + "conv -n 32 -c 32 -H 17 -W 17 -k 32 -y 1 -x 7 -p 0 -q 3 -u 1 -v 1 -l 1 -j 1 -F 1 -t 1", + "conv -n 64 -c 256 -H 34 -W 34 -k 256 -y 1 -x 1 -p 0 -q 0 -u 1 -v 1 -l 1 -j 1 -F 1 -t 1", + "conv -n 128 -c 128 -H 35 -W 35 -k 128 -y 3 -x 3 -p 0 -q 0 -u 2 -v 2 -l 1 -j 1 -F 1 -t 1", + "conv -n 128 -c 48 -H 7 -W 7 -k 128 -y 5 -x 5 -p 2 -q 2 -u 1 -v 1 -l 1 -j 1 -F 1 -t 1" + ] + }, + "Forward_Conv_2": { + "algorithm": "ConvAsmImplicitGemmV4R1DynamicFwd_1x1", + "commands": [ + "conv -n 16 -c 16 -H 56 -W 56 -k 64 -y 1 -x 1 -p 0 -q 0 -u 1 -v 1 -l 1 -j 1 -F 1 -t 1", + "conv -n 128 -c 256 -H 28 -W 28 -k 128 -y 1 -x 1 -p 0 -q 0 -u 1 -v 1 -l 1 -j 1 -F 1 -t 1", + "conv -n 64 -c 1536 -H 8 -W 8 -k 256 -y 1 -x 1 -p 0 -q 3 -u 1 -v 1 -l 1 -j 1 -F 1 -t 1" + ] + }, + "Backward_Conv_1": { + "algorithm": "ConvAsmImplicitGemmV4R1DynamicBwd", + "commands": [ + "conv -n 64 -c 64 -H 28 -W 28 -k 16 -y 1 -x 1 -p 0 -q 0 -u 1 -v 1 -l 1 -j 1 -F 2 -t 1", + "conv -n 64 -c 64 -H 56 -W 56 -k 256 -y 1 -x 1 -p 0 -q 0 -u 1 -v 1 -l 1 -j 1 -F 2 -t 1", + "conv -n 16 -c 128 -H 36 -W 36 -k 32 -y 1 -x 1 -p 0 -q 0 -u 1 -v 1 -l 1 -j 1 -F 2 -t 1", + "conv -n 32 -c 128 -H 34 -W 34 -k 64 -y 3 -x 3 -p 0 -q 0 -u 1 -v 1 -l 1 -j 1 -F 2 -t 1", + "conv -n 128 -c 128 -H 35 -W 35 -k 128 -y 3 -x 3 -p 1 -q 1 -u 1 -v 1 -l 1 -j 1 -F 2 -t 1" + ] + }, + "Backward_Conv_2": { + "algorithm": "ConvAsmImplicitGemmV4R1DynamicWrw", + "commands": [ + "conv -n 64 -c 64 -H 28 -W 28 -k 32 -y 1 -x 1 -p 0 -q 0 -u 1 -v 1 -l 1 -j 1 -F 4 -t 1", + "conv -n 32 -c 128 -H 34 -W 34 -k 64 -y 3 -x 3 -p 0 -q 0 -u 1 -v 1 -l 1 -j 1 -F 4 -t 1", + "conv -n 128 -c 128 -H 35 -W 35 -k 128 -y 3 -x 3 -p 1 -q 1 -u 1 -v 1 -l 1 -j 1 -F 4 -t 1", + "conv -n 128 -c 256 -H 56 -W 56 -k 64 -y 1 -x 1 -p 0 -q 0 -u 1 -v 1 -l 1 -j 1 -F 4 -t 1" + ] + } + }, + "gpu_specific_flags": { + "gfx906": { + "backward_flags": "-V 0" + } + } +} + diff --git a/build_tools/github_actions/test_framework/functional/functional_matrix.py b/build_tools/github_actions/test_framework/functional/functional_matrix.py new file mode 100644 index 00000000000..1accac66bb3 --- /dev/null +++ b/build_tools/github_actions/test_framework/functional/functional_matrix.py @@ -0,0 +1,32 @@ +""" +Functional test matrix definitions. + +This module contains the functional_matrix dictionary which defines all functional tests. +""" + +from pathlib import Path + +# Note: these paths are relative to the repository root. +SCRIPT_DIR = Path("build_tools") / "github_actions" / "test_framework" / "functional" / "scripts" + + +def _get_script_path(script_name: str) -> str: + platform_path = SCRIPT_DIR / script_name + # Convert to posix (using `/` instead of `\\`) so test workflows can use + # 'bash' as the shell on Linux and Windows. + posix_path = platform_path.as_posix() + return str(posix_path) + + +functional_matrix = { + "test_miopendriver_conv": { + "job_name": "test_miopendriver_conv", + "fetch_artifact_args": "--miopen --tests", + "timeout_minutes": 30, + "test_script": f"python {SCRIPT_DIR}/test_miopendriver_conv.py", + "test_script": f"python {_get_script_path('test_rocrand_benchmark.py')}", + # TODO(lajagapp): Add windows support + "platform": ["linux"], + "total_shards": 1 + }, +} diff --git a/build_tools/github_actions/test_framework/functional/scripts/__init__.py b/build_tools/github_actions/test_framework/functional/scripts/__init__.py new file mode 100644 index 00000000000..a845c298ffc --- /dev/null +++ b/build_tools/github_actions/test_framework/functional/scripts/__init__.py @@ -0,0 +1,2 @@ +"""Functional test scripts.""" + diff --git a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py b/build_tools/github_actions/test_framework/functional/scripts/functional_base.py new file mode 100644 index 00000000000..5196d766403 --- /dev/null +++ b/build_tools/github_actions/test_framework/functional/scripts/functional_base.py @@ -0,0 +1,304 @@ +"""Base class for functional tests with common functionality.""" + +import json +import os +import sys +from pathlib import Path +from typing import Dict, List, Tuple, Any, Optional +from prettytable import PrettyTable + +# Add parent directory to path for utils import +sys.path.insert(0, str(Path(__file__).parent.parent)) # functional/ +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # github_actions/ +from utils import TestClient +from utils.logger import log +from utils.exceptions import TestExecutionError +from github_actions_utils import gha_append_step_summary + + +class FunctionalBase: + """Base class providing common functional test logic. + + Child classes must implement run_tests() and parse_results(). + + Unlike benchmarks (which measure performance), functional tests verify + correctness and produce pass/fail results without performance metrics. + """ + + def __init__(self, test_name: str, display_name: str = None): + """Initialize functional test. + + Args: + test_name: Internal test name (e.g., 'miopen_driver_conv') + display_name: Display name for reports (e.g., 'MIOpen Driver Convolution') + """ + self.test_name = test_name + self.display_name = display_name or test_name.replace('_', ' ').title() + + # Environment variables + self.therock_bin_dir = os.getenv("THEROCK_BIN_DIR") + self.artifact_run_id = os.getenv("ARTIFACT_RUN_ID") + self.amdgpu_families = os.getenv("AMDGPU_FAMILIES") + self.script_dir = Path(__file__).resolve().parent + self.therock_dir = self.script_dir.parent.parent.parent.parent.parent + + # Initialize test client (will be set in run()) + self.client = None + + def load_config(self, config_filename: str) -> Dict[str, Any]: + """Load test configuration from JSON file. + + Args: + config_filename: Name of JSON config file (e.g., 'miopen_driver_conv.json') + + Returns: + Parsed JSON configuration dictionary + + Raises: + TestExecutionError: If config file not found or invalid JSON + """ + config_file = self.script_dir.parent / "configs" / config_filename + + try: + with open(config_file, "r") as f: + return json.load(f) + except FileNotFoundError: + raise TestExecutionError( + f"Configuration file not found: {config_file}", + action=f"Ensure {config_filename} exists in configs/ directory" + ) + except json.JSONDecodeError as e: + raise TestExecutionError( + f"Invalid JSON in configuration file: {e}", + action=f"Check JSON syntax in {config_filename}" + ) + + def create_test_result( + self, + test_name: str, + subtest_name: str, + status: str, + **kwargs, + ) -> Dict[str, Any]: + """Create a standardized functional test result dictionary. + + Args: + test_name: Test name + subtest_name: Specific test/suite identifier + status: Test status ('PASS' or 'FAIL') + **kwargs: Additional test-specific parameters (pass_count, fail_count, etc.) + + Returns: + Dict[str, Any]: Test result dictionary + """ + test_config = { + "test_name": test_name, + "sub_test_name": subtest_name, + "python_version": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", + "environment_dependencies": [], + } + + # Add any additional kwargs to test_config + for key, value in kwargs.items(): + test_config[key] = value + + return { + "test_name": test_name, + "subtest": subtest_name, + "status": status, + "test_config": test_config, + **kwargs, # Include pass_count, fail_count, etc. at top level too + } + + def calculate_statistics( + self, test_results: List[Dict[str, Any]] + ) -> Dict[str, Any]: + """Calculate test statistics from results. + + Args: + test_results: List of test result dictionaries with 'status' key + + Returns: + Dictionary with: + - passed: Number of passed tests + - failed: Number of failed tests + - total: Total number of tests + - overall_status: 'PASS' if no failures, else 'FAIL' + """ + passed = sum(1 for r in test_results if r.get("status") == "PASS") + failed = sum(1 for r in test_results if r.get("status") == "FAIL") + overall_status = "PASS" if failed == 0 else "FAIL" + + return { + "passed": passed, + "failed": failed, + "total": len(test_results), + "overall_status": overall_status, + } + + def upload_results( + self, test_results: List[Dict[str, Any]], stats: Dict[str, Any] + ) -> bool: + """Upload results to API and save locally. + + Args: + test_results: List of test result dictionaries + stats: Test statistics dictionary + + Returns: + True if upload successful, False otherwise + """ + log.info("Uploading Results to API") + success = self.client.upload_results( + test_name=f"{self.test_name}_functional", + test_results=test_results, + test_status=stats["overall_status"], + test_metadata={ + "artifact_run_id": self.artifact_run_id, + "amdgpu_families": self.amdgpu_families, + "test_name": self.test_name, + "total_tests": stats["total"], + "passed_tests": stats["passed"], + "failed_tests": stats["failed"], + }, + save_local=True, + output_dir=str(self.script_dir / "results"), + ) + + if success: + log.info("✓ Results uploaded successfully") + else: + log.info("⚠ Results saved locally only (API upload disabled or failed)") + + return success + + def write_step_summary( + self, stats: Dict[str, Any], table: PrettyTable + ) -> None: + """Write results to GitHub Actions step summary. + + Args: + stats: Test statistics dictionary + table: PrettyTable with test results + """ + status_emoji = "✅" if stats["overall_status"] == "PASS" else "❌" + + gha_append_step_summary( + f"## {status_emoji} {self.display_name} - Functional Test Results\n\n" + f"**Status:** {stats['overall_status']} | " + f"**Passed:** {stats['passed']}/{stats['total']} | " + f"**Failed:** {stats['failed']}/{stats['total']}\n\n" + f"
\n" + f"View detailed results ({stats['total']} tests)\n\n" + f"```\n{table}\n```\n\n" + f"
" + ) + + def run_tests(self) -> None: + """Run functional tests and save output to log file. + + Must be implemented by child classes. + + Raises: + NotImplementedError: If not overridden by child class + """ + raise NotImplementedError("Child class must implement run_tests()") + + def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable]: + """Parse test results from log file. + + Must be implemented by child classes. + + Returns: + tuple: (test_results list, PrettyTable object) + + Raises: + NotImplementedError: If not overridden by child class + """ + raise NotImplementedError("Child class must implement parse_results()") + + def run(self) -> int: + """Execute functional test workflow and return exit code (0=PASS, 1=FAIL). + + Returns: + 0 if all tests passed, 1 if any test failed + """ + log.info("="*80) + log.info(f"{self.display_name} - Starting Functional Test") + log.info("="*80) + + # Initialize test client and print system info + self.client = TestClient(auto_detect=True) + self.client.print_system_summary() + + # Run tests (implemented by child class) + self.run_tests() + + # Parse results (implemented by child class) + test_results, table = self.parse_results() + + if not test_results: + raise TestExecutionError( + "No test results generated", + action="Check if tests executed properly and log file contains output" + ) + + # Calculate statistics + stats = self.calculate_statistics(test_results) + log.info(f"\nTest Summary: {stats['passed']} passed, {stats['failed']} failed") + + # Display results + log.info("\n" + "="*80) + log.info("TEST RESULTS") + log.info("="*80) + log.info(f"\n{table}") + log.info(f"\nFinal Status: {stats['overall_status']}") + log.info("="*80) + + # Upload results (optional, may not be available in all environments) + try: + self.upload_results(test_results, stats) + except Exception as e: + log.warning(f"Could not upload results: {e}") + + # Write to GitHub Actions step summary + try: + self.write_step_summary(stats, table) + except Exception as e: + log.warning(f"Could not write GitHub Actions summary: {e}") + + # Return 0 only if PASS, otherwise return 1 + return 0 if stats["overall_status"] == "PASS" else 1 + + +def run_functional_test_main(test_instance): + """Run functional test with standard error handling. + + Args: + test_instance: Instance of FunctionalBase (or child class) + + Exits with: + 0 on success (all tests passed) + 1 on failure (any test failed or execution error) + + Raises: + KeyboardInterrupt: If execution is interrupted by user + """ + try: + exit_code = test_instance.run() + sys.exit(exit_code) + + except KeyboardInterrupt: + log.warning("\nExecution interrupted by user") + sys.exit(1) + + except TestExecutionError as e: + log.error(f"Test Execution Error: {e}") + sys.exit(1) + + except Exception as e: + log.error(f"Unexpected error: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + diff --git a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py new file mode 100644 index 00000000000..6e6e86a97a9 --- /dev/null +++ b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py @@ -0,0 +1,240 @@ +""" +MIOpen Driver Convolution Functional Test + +Tests MIOpenDriver convolution operations (Forward and Backward) to ensure +correct functionality across different GPU architectures. +""" + +import json +import os +import re +import shlex +import subprocess +import sys +from pathlib import Path +from typing import Dict, List, Tuple, Any +from prettytable import PrettyTable + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # For utils +sys.path.insert(0, str(Path(__file__).parent)) # For functional_base +from functional_base import FunctionalBase, run_functional_test_main +from utils.logger import log +from utils.exceptions import TestExecutionError + + +class MIOpenDriverConvTest(FunctionalBase): + """MIOpen Driver convolution functional test.""" + + def __init__(self): + super().__init__( + test_name="miopen_driver_conv", + display_name="MIOpen Driver Convolution" + ) + + self.log_file = self.script_dir / "miopendriver_conv.log" + + # Load test configurations from JSON + config = self.load_config("miopen_driver_conv.json") + + # Parse test suites + test_suites = config.get("test_suites", {}) + self.tests_cmd = {} + self.envs = {} + self.tests_list = [] + + for suite_name, suite_config in test_suites.items(): + self.tests_list.append(suite_name) + self.tests_cmd[suite_name] = suite_config.get("commands", []) + self.envs[suite_name] = suite_config.get("algorithm", "") + + # Load GPU-specific flags + self.gpu_specific_flags = config.get("gpu_specific_flags", {}) + + def get_gpu_id(self) -> str: + """Detect GPU ID using rocminfo.""" + try: + result = subprocess.run( + ["rocminfo"], + capture_output=True, + text=True, + check=True + ) + # Extract GPU name (e.g., gfx906, gfx90a, gfx942) + match = re.search(r'Name:\s+(gfx\w+)', result.stdout) + if match: + return match.group(1) + except (subprocess.CalledProcessError, FileNotFoundError): + log.warning("Could not detect GPU ID, assuming default") + + return "unknown" + + def run_tests(self) -> None: + """Run MIOpen driver convolution tests and save output to log file.""" + log.info(f"Running {self.display_name} Tests") + + gpu_id = self.get_gpu_id() + log.info(f"Detected GPU: {gpu_id}") + + miopen_driver = f"{self.therock_bin_dir}/MIOpenDriver" + if not Path(miopen_driver).exists(): + raise TestExecutionError( + f"MIOpenDriver not found at {miopen_driver}", + action="Ensure MIOpen is installed correctly" + ) + + with open(self.log_file, "w+") as f: + for test_suite in self.tests_list: + log.info(f"Running test suite: {test_suite}") + f.write(f"\n{'='*80}\n") + f.write(f"Test Suite: {test_suite}\n") + f.write(f"{'='*80}\n\n") + + # Set environment variable for specific algorithm + env = os.environ.copy() + env['MIOPEN_FIND_ENFORCE'] = self.envs[test_suite] + + for cmd_str in self.tests_cmd[test_suite]: + # Build full command with MIOpenDriver path + full_cmd = f"{miopen_driver} {cmd_str}" + + # Add GPU-specific flags if needed + if 'Backward_Conv' in test_suite and gpu_id in self.gpu_specific_flags: + backward_flags = self.gpu_specific_flags[gpu_id].get("backward_flags", "") + if backward_flags: + full_cmd = f"{full_cmd} {backward_flags}" + + cmd = shlex.split(full_cmd) + + log.info(f"++ Exec [{self.therock_dir}]$ {shlex.join(cmd)}") + f.write(f"\nCommand: {shlex.join(cmd)}\n") + f.write(f"-" * 80 + "\n") + + try: + process = subprocess.Popen( + cmd, + cwd=self.therock_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env=env + ) + + for line in process.stdout: + log.info(line.strip()) + f.write(line) + + process.wait() + f.write(f"\nReturn code: {process.returncode}\n\n") + + except Exception as e: + log.error(f"Error running command: {e}") + f.write(f"ERROR: {e}\n\n") + + log.info("Test execution complete") + + def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable]: + """Parse test results from log file. + + Returns: + tuple: (test_results list, PrettyTable object) + """ + log.info("Parsing Results") + + # Setup table + field_names = [ + "TestSuite", + "CommandIndex", + "Status", + "PassCount", + "FailCount", + "TotalCommands" + ] + table = PrettyTable(field_names) + + test_results = [] + + try: + with open(self.log_file, "r") as f: + content = f.read() + + # Parse results for each test suite + for test_suite in self.tests_list: + # Find the section for this test suite + pattern = re.compile( + rf"Test Suite: {test_suite}.*?(?=Test Suite:|$)", + re.DOTALL + ) + match = pattern.search(content) + + if not match: + log.warning(f"Could not find results for {test_suite}") + continue + + suite_content = match.group(0) + + # Count commands and their results + total_commands = len(self.tests_cmd[test_suite]) + pass_count = 0 + fail_count = 0 + + # Look for "PASSED" or return code 0 indicators + # MIOpenDriver typically outputs success indicators + for i, cmd_str in enumerate(self.tests_cmd[test_suite], 1): + # Build the command pattern to search for + miopen_driver = f"{self.therock_bin_dir}/MIOpenDriver" + full_cmd = f"{miopen_driver} {cmd_str}" + + # Simple heuristic: if command appears and no error follows, assume pass + # This is a simplified parser - adjust based on actual MIOpenDriver output + if f"Command: {full_cmd}" in suite_content or cmd_str in suite_content: + # Check for return code 0 or success indicators + if "Return code: 0" in suite_content or "PASSED" in suite_content: + pass_count += 1 + status = "PASS" + else: + fail_count += 1 + status = "FAIL" + else: + fail_count += 1 + status = "FAIL" + + # Determine overall suite status + overall_status = "PASS" if fail_count == 0 else "FAIL" + + # Add to table + table.add_row([ + test_suite, + f"1-{total_commands}", + overall_status, + pass_count, + fail_count, + total_commands + ]) + + # Add to results + test_results.append({ + "test_name": self.test_name, + "subtest_name": test_suite, + "status": overall_status, + "pass_count": pass_count, + "fail_count": fail_count, + "total_commands": total_commands + }) + + except FileNotFoundError: + raise TestExecutionError( + f"Log file not found: {self.log_file}", + action="Ensure tests were executed successfully" + ) + except OSError as e: + raise TestExecutionError( + f"Error reading log file: {e}", + action="Check file permissions and disk space" + ) + + return test_results, table + + +if __name__ == "__main__": + run_functional_test_main(MIOpenDriverConvTest()) diff --git a/build_tools/github_actions/test_framework/performance/performance_matrix.py b/build_tools/github_actions/test_framework/performance/performance_matrix.py new file mode 100644 index 00000000000..0c8a514c49a --- /dev/null +++ b/build_tools/github_actions/test_framework/performance/performance_matrix.py @@ -0,0 +1,26 @@ +""" +Performance test matrix definitions. + +Performance tests characterize system performance with detailed metrics, +scaling analysis, and bottleneck identification. +""" + +from pathlib import Path + +SCRIPT_DIR = Path("build_tools/github_actions/test_framework/performance/scripts") + +performance_matrix = { + # Example performance test (uncomment and customize): + # "rocblas_performance": { + # "job_name": "rocBLAS Performance Characterization", + # "fetch_artifact_args": "--blas", + # "timeout_minutes": 90, + # "test_script": f"python {SCRIPT_DIR}/test_rocblas_performance.py", + # "platform": ["linux"], + # "metrics": ["tflops", "latency", "bandwidth"], + # "targets": { + # "tflops_min": 40.0, + # "latency_max_ms": 2.0 + # } + # }, +} diff --git a/build_tools/github_actions/test_framework/performance/scripts/__init__.py b/build_tools/github_actions/test_framework/performance/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/build_tools/github_actions/benchmarks/utils/README.md b/build_tools/github_actions/test_framework/utils/README.md similarity index 95% rename from build_tools/github_actions/benchmarks/utils/README.md rename to build_tools/github_actions/test_framework/utils/README.md index 65ac0fe85c0..80e85fe07f1 100644 --- a/build_tools/github_actions/benchmarks/utils/README.md +++ b/build_tools/github_actions/test_framework/utils/README.md @@ -7,7 +7,7 @@ Utility modules organized into logical subdirectories for maintainability and sc ``` benchmarks/utils/ ├── __init__.py # Public exports -├── benchmark_client.py # Main BenchmarkClient API +├── test_client.py # Main TestClient API ├── constants.py # Framework constants ├── exceptions.py # Custom exceptions ├── logger.py # Logging configuration @@ -50,7 +50,7 @@ from utils.constants import Constants from utils.exceptions import ConfigurationError # Main API classes -from utils.benchmark_client import BenchmarkClient +from utils.test_client import TestClient from utils.system.system_detector import SystemDetector from utils.config.config_helper import ConfigHelper from utils.results.results_handler import ResultsHandler @@ -81,7 +81,7 @@ from utils.results import ResultsHandler, ResultsAPI - **constants.py** - Framework constants and defaults - **exceptions.py** - Custom exception classes - **logger.py** - Logging configuration -- **benchmark_client.py** - Main BenchmarkClient API +- **test_client.py** - Main TestClient API ### Config diff --git a/build_tools/github_actions/benchmarks/utils/__init__.py b/build_tools/github_actions/test_framework/utils/__init__.py similarity index 86% rename from build_tools/github_actions/benchmarks/utils/__init__.py rename to build_tools/github_actions/test_framework/utils/__init__.py index 6fc647d44bb..fdd41c61802 100644 --- a/build_tools/github_actions/benchmarks/utils/__init__.py +++ b/build_tools/github_actions/test_framework/utils/__init__.py @@ -1,6 +1,6 @@ """Utils package for test execution with system detection and result reporting. -Provides BenchmarkClient API for collecting system info (OS, hardware, ROCm) and +Provides TestClient API for collecting system info (OS, hardware, ROCm) and uploading test results to API endpoints. Organization: @@ -17,7 +17,7 @@ "constants", "exceptions", # Main API - "BenchmarkClient", + "TestClient", # Commonly used exports "SystemContext", "SystemDetector", @@ -32,7 +32,7 @@ ] # Import main API -from .benchmark_client import BenchmarkClient +from .test_client import TestClient # Export commonly used classes from subdirectories from .system import ( diff --git a/build_tools/github_actions/benchmarks/utils/config/__init__.py b/build_tools/github_actions/test_framework/utils/config/__init__.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/config/__init__.py rename to build_tools/github_actions/test_framework/utils/config/__init__.py diff --git a/build_tools/github_actions/benchmarks/utils/config/config_helper.py b/build_tools/github_actions/test_framework/utils/config/config_helper.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/config/config_helper.py rename to build_tools/github_actions/test_framework/utils/config/config_helper.py diff --git a/build_tools/github_actions/benchmarks/utils/config/config_parser.py b/build_tools/github_actions/test_framework/utils/config/config_parser.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/config/config_parser.py rename to build_tools/github_actions/test_framework/utils/config/config_parser.py diff --git a/build_tools/github_actions/benchmarks/utils/config/config_validator.py b/build_tools/github_actions/test_framework/utils/config/config_validator.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/config/config_validator.py rename to build_tools/github_actions/test_framework/utils/config/config_validator.py diff --git a/build_tools/github_actions/benchmarks/utils/constants.py b/build_tools/github_actions/test_framework/utils/constants.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/constants.py rename to build_tools/github_actions/test_framework/utils/constants.py diff --git a/build_tools/github_actions/benchmarks/utils/exceptions.py b/build_tools/github_actions/test_framework/utils/exceptions.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/exceptions.py rename to build_tools/github_actions/test_framework/utils/exceptions.py diff --git a/build_tools/github_actions/benchmarks/utils/logger.py b/build_tools/github_actions/test_framework/utils/logger.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/logger.py rename to build_tools/github_actions/test_framework/utils/logger.py diff --git a/build_tools/github_actions/benchmarks/utils/results/__init__.py b/build_tools/github_actions/test_framework/utils/results/__init__.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/results/__init__.py rename to build_tools/github_actions/test_framework/utils/results/__init__.py diff --git a/build_tools/github_actions/benchmarks/utils/results/results_api.py b/build_tools/github_actions/test_framework/utils/results/results_api.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/results/results_api.py rename to build_tools/github_actions/test_framework/utils/results/results_api.py diff --git a/build_tools/github_actions/benchmarks/utils/results/results_handler.py b/build_tools/github_actions/test_framework/utils/results/results_handler.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/results/results_handler.py rename to build_tools/github_actions/test_framework/utils/results/results_handler.py diff --git a/build_tools/github_actions/benchmarks/utils/results/schemas/payload_schema.json b/build_tools/github_actions/test_framework/utils/results/schemas/payload_schema.json similarity index 100% rename from build_tools/github_actions/benchmarks/utils/results/schemas/payload_schema.json rename to build_tools/github_actions/test_framework/utils/results/schemas/payload_schema.json diff --git a/build_tools/github_actions/benchmarks/utils/system/__init__.py b/build_tools/github_actions/test_framework/utils/system/__init__.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/system/__init__.py rename to build_tools/github_actions/test_framework/utils/system/__init__.py diff --git a/build_tools/github_actions/benchmarks/utils/system/hardware.py b/build_tools/github_actions/test_framework/utils/system/hardware.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/system/hardware.py rename to build_tools/github_actions/test_framework/utils/system/hardware.py diff --git a/build_tools/github_actions/benchmarks/utils/system/platform.py b/build_tools/github_actions/test_framework/utils/system/platform.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/system/platform.py rename to build_tools/github_actions/test_framework/utils/system/platform.py diff --git a/build_tools/github_actions/benchmarks/utils/system/rocm_detector.py b/build_tools/github_actions/test_framework/utils/system/rocm_detector.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/system/rocm_detector.py rename to build_tools/github_actions/test_framework/utils/system/rocm_detector.py diff --git a/build_tools/github_actions/benchmarks/utils/system/system_detector.py b/build_tools/github_actions/test_framework/utils/system/system_detector.py similarity index 100% rename from build_tools/github_actions/benchmarks/utils/system/system_detector.py rename to build_tools/github_actions/test_framework/utils/system/system_detector.py diff --git a/build_tools/github_actions/benchmarks/utils/benchmark_client.py b/build_tools/github_actions/test_framework/utils/test_client.py similarity index 94% rename from build_tools/github_actions/benchmarks/utils/benchmark_client.py rename to build_tools/github_actions/test_framework/utils/test_client.py index 4ced8c57202..88c923e6061 100644 --- a/build_tools/github_actions/benchmarks/utils/benchmark_client.py +++ b/build_tools/github_actions/test_framework/utils/test_client.py @@ -1,7 +1,7 @@ -"""Benchmark Client for system detection and result reporting. +"""Test Client for system detection and result reporting. Unified interface for collecting system information (OS, hardware, ROCm) and -uploading benchmark results to API or local storage. +uploading test results to API or local storage. """ import time @@ -19,8 +19,14 @@ from .results import ResultsHandler -class BenchmarkClient: +class TestClient: """Client for system detection, result collection, and API upload. + + Used by all test types (benchmark, performance, functional) for: + - System information detection (OS, hardware, ROCm) + - Test result collection and formatting + - API upload and local storage + - LKG (Last Known Good) comparison Attributes: config: Configuration dictionary @@ -29,7 +35,7 @@ class BenchmarkClient: system_context: SystemContext with detected info (or None if not detected) Example: - >>> client = BenchmarkClient() + >>> client = TestClient() >>> results = [{"test_name": "fft_1024", "score": 1234.5, "unit": "GFLOPS"}] >>> client.upload_results("rocfft_benchmark", results) """ diff --git a/build_tools/github_actions/tests/configure_ci_test.py b/build_tools/github_actions/tests/configure_ci_test.py index c69876c4d89..3cb1dc848db 100644 --- a/build_tools/github_actions/tests/configure_ci_test.py +++ b/build_tools/github_actions/tests/configure_ci_test.py @@ -6,7 +6,7 @@ sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) import configure_ci -from benchmarks.benchmark_test_matrix import benchmark_matrix +from test_framework.benchmark.benchmark_matrix import benchmark_matrix therock_test_runner_dict = { "gfx110x": { From 5adda5a50a643f9cbf2ae12049af6985c13e0daf Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 6 Jan 2026 13:57:25 +0000 Subject: [PATCH 02/28] Add functional test framework with MIOpen driver test Created FunctionalBase class and MIOpen driver convolution test. Features: two-table display, 4 status types, JSON configs, accurate per-command parsing with 'Verifies OK' detection, GitHub Actions integration. Consistent with benchmark test patterns. Signed-off-by: Lenine Ajagappane --- .../functional/scripts/functional_base.py | 155 ++++++++++-------- .../scripts/test_miopendriver_conv.py | 123 ++++++-------- 2 files changed, 141 insertions(+), 137 deletions(-) diff --git a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py b/build_tools/github_actions/test_framework/functional/scripts/functional_base.py index 5196d766403..8792d348658 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py +++ b/build_tools/github_actions/test_framework/functional/scripts/functional_base.py @@ -2,9 +2,11 @@ import json import os +import subprocess +import re import sys from pathlib import Path -from typing import Dict, List, Tuple, Any, Optional +from typing import Dict, List, Tuple, Any from prettytable import PrettyTable # Add parent directory to path for utils import @@ -85,7 +87,7 @@ def create_test_result( Args: test_name: Test name subtest_name: Specific test/suite identifier - status: Test status ('PASS' or 'FAIL') + status: Test status ('PASS', 'FAIL', 'ERROR', 'SKIP') **kwargs: Additional test-specific parameters (pass_count, fail_count, etc.) Returns: @@ -119,23 +121,60 @@ def calculate_statistics( test_results: List of test result dictionaries with 'status' key Returns: - Dictionary with: - - passed: Number of passed tests - - failed: Number of failed tests - - total: Total number of tests - - overall_status: 'PASS' if no failures, else 'FAIL' + Dictionary with detailed statistics including all status types """ passed = sum(1 for r in test_results if r.get("status") == "PASS") failed = sum(1 for r in test_results if r.get("status") == "FAIL") - overall_status = "PASS" if failed == 0 else "FAIL" + error = sum(1 for r in test_results if r.get("status") == "ERROR") + skipped = sum(1 for r in test_results if r.get("status") == "SKIP") + + # Overall status: PASS only if no failures/errors + overall_status = "PASS" if (failed == 0 and error == 0) else "FAIL" return { "passed": passed, "failed": failed, + "error": error, + "skipped": skipped, "total": len(test_results), "overall_status": overall_status, } + def create_summary_table( + self, stats: Dict[str, Any], num_suites: int + ) -> PrettyTable: + """Create overall summary table with all statistics. + + Args: + stats: Test statistics dictionary + num_suites: Number of test suites + + Returns: + PrettyTable with summary statistics + """ + summary_table = PrettyTable() + summary_table.field_names = [ + "Total TestSuites", + "Total TestCases", + "Passed", + "Failed", + "Errored", + "Skipped", + "Final Result" + ] + + summary_table.add_row([ + num_suites, + stats["total"], + stats["passed"], + stats["failed"], + stats["error"], + stats["skipped"], + stats["overall_status"] + ]) + + return summary_table + def upload_results( self, test_results: List[Dict[str, Any]], stats: Dict[str, Any] ) -> bool: @@ -160,6 +199,8 @@ def upload_results( "total_tests": stats["total"], "passed_tests": stats["passed"], "failed_tests": stats["failed"], + "error_tests": stats["error"], + "skipped_tests": stats["skipped"], }, save_local=True, output_dir=str(self.script_dir / "results"), @@ -173,49 +214,41 @@ def upload_results( return success def write_step_summary( - self, stats: Dict[str, Any], table: PrettyTable + self, stats: Dict[str, Any], detailed_table: PrettyTable, summary_table: PrettyTable ) -> None: """Write results to GitHub Actions step summary. Args: stats: Test statistics dictionary - table: PrettyTable with test results + detailed_table: PrettyTable with detailed test results + summary_table: PrettyTable with summary statistics """ - status_emoji = "✅" if stats["overall_status"] == "PASS" else "❌" - gha_append_step_summary( - f"## {status_emoji} {self.display_name} - Functional Test Results\n\n" - f"**Status:** {stats['overall_status']} | " - f"**Passed:** {stats['passed']}/{stats['total']} | " - f"**Failed:** {stats['failed']}/{stats['total']}\n\n" + f"## {self.display_name} - Functional Test Results\n\n" + f"### Detailed Results\n\n" + f"```\n{detailed_table}\n```\n\n" f"
\n" - f"View detailed results ({stats['total']} tests)\n\n" - f"```\n{table}\n```\n\n" + f"View summary ({stats['total']} tests)\n\n" + f"```\n{summary_table}\n```\n\n" f"
" ) - def run_tests(self) -> None: - """Run functional tests and save output to log file. - - Must be implemented by child classes. - - Raises: - NotImplementedError: If not overridden by child class - """ - raise NotImplementedError("Child class must implement run_tests()") - - def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable]: - """Parse test results from log file. - - Must be implemented by child classes. - - Returns: - tuple: (test_results list, PrettyTable object) - - Raises: - NotImplementedError: If not overridden by child class - """ - raise NotImplementedError("Child class must implement parse_results()") + def get_gpu_id(self) -> str: + """Detect GPU ID using rocminfo.""" + try: + result = subprocess.run( + ["rocminfo"], + capture_output=True, + text=True, + check=True + ) + # Extract GPU name (e.g., gfx906, gfx90a, gfx942) + match = re.search(r'Name:\s+(gfx\w+)', result.stdout) + if match: + return match.group(1) + except (subprocess.CalledProcessError, FileNotFoundError): + log.warning("Could not detect GPU ID, assuming default") + return "unknown" def run(self) -> int: """Execute functional test workflow and return exit code (0=PASS, 1=FAIL). @@ -223,9 +256,7 @@ def run(self) -> int: Returns: 0 if all tests passed, 1 if any test failed """ - log.info("="*80) log.info(f"{self.display_name} - Starting Functional Test") - log.info("="*80) # Initialize test client and print system info self.client = TestClient(auto_detect=True) @@ -235,7 +266,7 @@ def run(self) -> int: self.run_tests() # Parse results (implemented by child class) - test_results, table = self.parse_results() + test_results, detailed_table, num_suites = self.parse_results() if not test_results: raise TestExecutionError( @@ -245,15 +276,17 @@ def run(self) -> int: # Calculate statistics stats = self.calculate_statistics(test_results) - log.info(f"\nTest Summary: {stats['passed']} passed, {stats['failed']} failed") + + # Create summary table + summary_table = self.create_summary_table(stats, num_suites) # Display results - log.info("\n" + "="*80) - log.info("TEST RESULTS") - log.info("="*80) - log.info(f"\n{table}") + log.info("DETAILED RESULTS") + log.info(f"\n{detailed_table}") + + log.info("\nSUMMARY") + log.info(f"\n{summary_table}") log.info(f"\nFinal Status: {stats['overall_status']}") - log.info("="*80) # Upload results (optional, may not be available in all environments) try: @@ -263,7 +296,7 @@ def run(self) -> int: # Write to GitHub Actions step summary try: - self.write_step_summary(stats, table) + self.write_step_summary(stats, detailed_table, summary_table) except Exception as e: log.warning(f"Could not write GitHub Actions summary: {e}") @@ -274,31 +307,19 @@ def run(self) -> int: def run_functional_test_main(test_instance): """Run functional test with standard error handling. - Args: - test_instance: Instance of FunctionalBase (or child class) - - Exits with: - 0 on success (all tests passed) - 1 on failure (any test failed or execution error) - Raises: KeyboardInterrupt: If execution is interrupted by user + Exception: If test execution fails """ try: exit_code = test_instance.run() - sys.exit(exit_code) - + if exit_code != 0: + raise RuntimeError(f"Test failed with exit code {exit_code}") except KeyboardInterrupt: log.warning("\nExecution interrupted by user") - sys.exit(1) - - except TestExecutionError as e: - log.error(f"Test Execution Error: {e}") - sys.exit(1) - + raise except Exception as e: log.error(f"Unexpected error: {e}") import traceback traceback.print_exc() - sys.exit(1) - + raise diff --git a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py index 6e6e86a97a9..0f2132fd510 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py +++ b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py @@ -5,7 +5,6 @@ correct functionality across different GPU architectures. """ -import json import os import re import shlex @@ -50,24 +49,6 @@ def __init__(self): # Load GPU-specific flags self.gpu_specific_flags = config.get("gpu_specific_flags", {}) - def get_gpu_id(self) -> str: - """Detect GPU ID using rocminfo.""" - try: - result = subprocess.run( - ["rocminfo"], - capture_output=True, - text=True, - check=True - ) - # Extract GPU name (e.g., gfx906, gfx90a, gfx942) - match = re.search(r'Name:\s+(gfx\w+)', result.stdout) - if match: - return match.group(1) - except (subprocess.CalledProcessError, FileNotFoundError): - log.warning("Could not detect GPU ID, assuming default") - - return "unknown" - def run_tests(self) -> None: """Run MIOpen driver convolution tests and save output to log file.""" log.info(f"Running {self.display_name} Tests") @@ -133,24 +114,21 @@ def run_tests(self) -> None: log.info("Test execution complete") - def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable]: + def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: """Parse test results from log file. Returns: - tuple: (test_results list, PrettyTable object) + tuple: (test_results list, detailed PrettyTable, number of test suites) """ log.info("Parsing Results") - # Setup table - field_names = [ + # Setup detailed table - show each individual test case + detailed_table = PrettyTable() + detailed_table.field_names = [ "TestSuite", - "CommandIndex", - "Status", - "PassCount", - "FailCount", - "TotalCommands" + "TestCase", + "Status" ] - table = PrettyTable(field_names) test_results = [] @@ -173,54 +151,58 @@ def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable]: suite_content = match.group(0) - # Count commands and their results - total_commands = len(self.tests_cmd[test_suite]) - pass_count = 0 - fail_count = 0 - - # Look for "PASSED" or return code 0 indicators - # MIOpenDriver typically outputs success indicators + # Parse each command in the suite for i, cmd_str in enumerate(self.tests_cmd[test_suite], 1): # Build the command pattern to search for miopen_driver = f"{self.therock_bin_dir}/MIOpenDriver" full_cmd = f"{miopen_driver} {cmd_str}" + test_case_name = f"{test_suite}_case{i}" - # Simple heuristic: if command appears and no error follows, assume pass - # This is a simplified parser - adjust based on actual MIOpenDriver output - if f"Command: {full_cmd}" in suite_content or cmd_str in suite_content: - # Check for return code 0 or success indicators - if "Return code: 0" in suite_content or "PASSED" in suite_content: - pass_count += 1 - status = "PASS" + # Find this specific command in the suite content + cmd_pattern = re.escape(f"Command: {full_cmd}") + cmd_match = re.search(cmd_pattern, suite_content) + + if cmd_match: + # Extract content after this command until next command or end + start_pos = cmd_match.end() + next_cmd_match = re.search(r"\nCommand:", suite_content[start_pos:]) + if next_cmd_match: + cmd_section = suite_content[start_pos:start_pos + next_cmd_match.start()] else: - fail_count += 1 + cmd_section = suite_content[start_pos:] + + # Check return code in THIS command's section only + return_code_match = re.search(r"Return code:\s*(\d+)", cmd_section) + if return_code_match: + return_code = int(return_code_match.group(1)) + status = "PASS" if return_code == 0 else "FAIL" + elif "PASSED" in cmd_section: + status = "PASS" + elif "FAILED" in cmd_section or "ERROR" in cmd_section: status = "FAIL" + else: + status = "FAIL" # Unknown result, assume fail else: - fail_count += 1 - status = "FAIL" - - # Determine overall suite status - overall_status = "PASS" if fail_count == 0 else "FAIL" - - # Add to table - table.add_row([ - test_suite, - f"1-{total_commands}", - overall_status, - pass_count, - fail_count, - total_commands - ]) - - # Add to results - test_results.append({ - "test_name": self.test_name, - "subtest_name": test_suite, - "status": overall_status, - "pass_count": pass_count, - "fail_count": fail_count, - "total_commands": total_commands - }) + status = "FAIL" # Command not found in log + + # Add each individual test case to detailed table + detailed_table.add_row([ + test_suite, + test_case_name, + status + ]) + + # Add each test case to results list using helper + test_results.append( + self.create_test_result( + test_name=self.test_name, + subtest_name=test_case_name, + status=status, + suite=test_suite, + command_index=i, + command=cmd_str + ) + ) except FileNotFoundError: raise TestExecutionError( @@ -233,7 +215,8 @@ def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable]: action="Check file permissions and disk space" ) - return test_results, table + num_suites = len(self.tests_list) + return test_results, detailed_table, num_suites if __name__ == "__main__": From 64d9a409b86daf8c15ad77f1d39680b1d1ad96db Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 6 Jan 2026 15:24:34 +0000 Subject: [PATCH 03/28] Fix import errors Signed-off-by: Lenine Ajagappane --- .../test_framework/benchmark/scripts/benchmark_base.py | 4 ++-- .../benchmark/scripts/test_hipblaslt_benchmark.py | 2 +- .../test_framework/benchmark/scripts/test_rocfft_benchmark.py | 2 +- .../benchmark/scripts/test_rocrand_benchmark.py | 2 +- .../benchmark/scripts/test_rocsolver_benchmark.py | 2 +- .../test_framework/functional/scripts/__init__.py | 1 + .../test_framework/functional/scripts/functional_base.py | 4 ++-- 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py b/build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py index 7666d7577bc..be9d7098444 100644 --- a/build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py +++ b/build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py @@ -7,8 +7,8 @@ from prettytable import PrettyTable # Add parent directory to path for utils import -sys.path.insert(0, str(Path(__file__).parent.parent)) # benchmarks/ -sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # github_actions/ +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # test_framework/ +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) # github_actions/ from utils import TestClient from utils.logger import log from github_actions_utils import gha_append_step_summary diff --git a/build_tools/github_actions/test_framework/benchmark/scripts/test_hipblaslt_benchmark.py b/build_tools/github_actions/test_framework/benchmark/scripts/test_hipblaslt_benchmark.py index 32ce14d05a4..92095dcc6fd 100644 --- a/build_tools/github_actions/test_framework/benchmark/scripts/test_hipblaslt_benchmark.py +++ b/build_tools/github_actions/test_framework/benchmark/scripts/test_hipblaslt_benchmark.py @@ -13,7 +13,7 @@ from typing import Dict, List, Tuple, Any from prettytable import PrettyTable -sys.path.insert(0, str(Path(__file__).parent.parent)) # For utils +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # For utils sys.path.insert(0, str(Path(__file__).parent)) # For benchmark_base from benchmark_base import BenchmarkBase, run_benchmark_main from utils.logger import log diff --git a/build_tools/github_actions/test_framework/benchmark/scripts/test_rocfft_benchmark.py b/build_tools/github_actions/test_framework/benchmark/scripts/test_rocfft_benchmark.py index b1ab081d01e..0fb5af12007 100644 --- a/build_tools/github_actions/test_framework/benchmark/scripts/test_rocfft_benchmark.py +++ b/build_tools/github_actions/test_framework/benchmark/scripts/test_rocfft_benchmark.py @@ -13,7 +13,7 @@ from typing import Dict, List, Tuple, Any from prettytable import PrettyTable -sys.path.insert(0, str(Path(__file__).parent.parent)) # For utils +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # For utils sys.path.insert(0, str(Path(__file__).parent)) # For benchmark_base from benchmark_base import BenchmarkBase, run_benchmark_main from utils.logger import log diff --git a/build_tools/github_actions/test_framework/benchmark/scripts/test_rocrand_benchmark.py b/build_tools/github_actions/test_framework/benchmark/scripts/test_rocrand_benchmark.py index d244597c6c7..1c971bd37c2 100644 --- a/build_tools/github_actions/test_framework/benchmark/scripts/test_rocrand_benchmark.py +++ b/build_tools/github_actions/test_framework/benchmark/scripts/test_rocrand_benchmark.py @@ -14,7 +14,7 @@ from typing import Dict, List, Tuple, Any from prettytable import PrettyTable -sys.path.insert(0, str(Path(__file__).parent.parent)) # For utils +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # For utils sys.path.insert(0, str(Path(__file__).parent)) # For benchmark_base from benchmark_base import BenchmarkBase, run_benchmark_main from utils.logger import log diff --git a/build_tools/github_actions/test_framework/benchmark/scripts/test_rocsolver_benchmark.py b/build_tools/github_actions/test_framework/benchmark/scripts/test_rocsolver_benchmark.py index 7f3870d57c8..6959e2fb297 100644 --- a/build_tools/github_actions/test_framework/benchmark/scripts/test_rocsolver_benchmark.py +++ b/build_tools/github_actions/test_framework/benchmark/scripts/test_rocsolver_benchmark.py @@ -13,7 +13,7 @@ from typing import Dict, List, Tuple, Any from prettytable import PrettyTable -sys.path.insert(0, str(Path(__file__).parent.parent)) # For utils +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # For utils sys.path.insert(0, str(Path(__file__).parent)) # For benchmark_base from benchmark_base import BenchmarkBase, run_benchmark_main from utils.logger import log diff --git a/build_tools/github_actions/test_framework/functional/scripts/__init__.py b/build_tools/github_actions/test_framework/functional/scripts/__init__.py index a845c298ffc..cbb899c7a7e 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/__init__.py +++ b/build_tools/github_actions/test_framework/functional/scripts/__init__.py @@ -1,2 +1,3 @@ """Functional test scripts.""" + diff --git a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py b/build_tools/github_actions/test_framework/functional/scripts/functional_base.py index 8792d348658..5901e0e9dbd 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py +++ b/build_tools/github_actions/test_framework/functional/scripts/functional_base.py @@ -10,8 +10,8 @@ from prettytable import PrettyTable # Add parent directory to path for utils import -sys.path.insert(0, str(Path(__file__).parent.parent)) # functional/ -sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # github_actions/ +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # test_framework/ +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) # github_actions/ from utils import TestClient from utils.logger import log from utils.exceptions import TestExecutionError From 7db32928b0813f79d22350a54c3da82144af4425 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 6 Jan 2026 18:49:33 +0000 Subject: [PATCH 04/28] Fix pre-commit errors Signed-off-by: Lenine Ajagappane --- .../github_actions/test_framework/README.md | 23 +++-- .../test_framework/benchmark/README.md | 67 +++++++-------- .../benchmark/benchmark_matrix.py | 4 +- .../configs/miopen_driver_conv.json | 1 - .../functional/functional_matrix.py | 6 +- .../functional/scripts/__init__.py | 2 - .../functional/scripts/functional_base.py | 84 +++++++++---------- .../scripts/test_miopendriver_conv.py | 69 +++++++-------- .../test_framework/utils/test_client.py | 2 +- 9 files changed, 122 insertions(+), 136 deletions(-) diff --git a/build_tools/github_actions/test_framework/README.md b/build_tools/github_actions/test_framework/README.md index 117c98a844e..61878b4ffb3 100644 --- a/build_tools/github_actions/test_framework/README.md +++ b/build_tools/github_actions/test_framework/README.md @@ -88,6 +88,7 @@ python test_framework/functional/scripts/test_rocblas_functional.py ### Available Tests **Benchmark Tests:** + - `test_hipblaslt_benchmark.py` - hipBLASLt GEMM benchmarks - `test_rocfft_benchmark.py` - rocFFT transform benchmarks - `test_rocrand_benchmark.py` - rocRAND generation benchmarks @@ -188,7 +189,6 @@ ci_nightly.yml → ci_linux.yml ``` - ### Workflow Integration Details ``` @@ -259,25 +259,25 @@ ci_nightly.yml → ci_linux.yml See detailed guide in [benchmark/README.md](benchmark/README.md#adding-a-new-benchmark) Quick steps: + 1. Create `benchmark/scripts/test_yourlib_benchmark.py` -2. Inherit from `BenchmarkBase`, implement `run_benchmarks()` and `parse_results()` -3. Add to `benchmark/benchmark_matrix.py` -4. Test locally +1. Inherit from `BenchmarkBase`, implement `run_benchmarks()` and `parse_results()` +1. Add to `benchmark/benchmark_matrix.py` +1. Test locally ### Adding a Performance Test 1. Create `performance/scripts/test_yourlib_performance.py` -2. Inherit from `PerformanceRunner` (to be implemented) -3. Implement metrics collection and analysis -4. Add to `performance/performance_matrix.py` +1. Inherit from `PerformanceRunner` (to be implemented) +1. Implement metrics collection and analysis +1. Add to `performance/performance_matrix.py` ### Adding a Functional Test 1. Create `functional/scripts/test_yourlib_functional.py` -2. Inherit from `FunctionalRunner` (to be implemented) -3. Implement validation logic -4. Add to `functional/functional_matrix.py` - +1. Inherit from `FunctionalRunner` (to be implemented) +1. Implement validation logic +1. Add to `functional/functional_matrix.py` ### Getting Help @@ -287,7 +287,6 @@ Quick steps: - **Functional tests:** See `functional/README.md` (to be created) - **Utilities:** See [utils/README.md](utils/README.md) - ## Related Files - `configure_ci.py` - CI workflow orchestration diff --git a/build_tools/github_actions/test_framework/benchmark/README.md b/build_tools/github_actions/test_framework/benchmark/README.md index 0441262b464..118f9c82351 100644 --- a/build_tools/github_actions/test_framework/benchmark/README.md +++ b/build_tools/github_actions/test_framework/benchmark/README.md @@ -5,6 +5,7 @@ Regression detection tests that compare current performance against Last Known G ## Purpose Benchmark tests detect **performance regressions** by comparing test results against established baselines: + - **Result:** PASS (within tolerance) / FAIL (regression detected) / UNKNOWN (no baseline) - **Frequency:** Every nightly CI - **Use Case:** Automated CI gates to prevent performance degradation @@ -13,12 +14,12 @@ Benchmark tests detect **performance regressions** by comparing test results aga ### Available Benchmark Tests -| Test Script | Library | Description | -|-------------|---------|-------------| +| Test Script | Library | Description | +| ----------------------------- | --------- | --------------------------------------- | | `test_hipblaslt_benchmark.py` | hipBLASLt | Matrix multiplication benchmarks (GEMM) | -| `test_rocfft_benchmark.py` | rocFFT | FFT benchmarks (1D, 2D, 3D transforms) | -| `test_rocrand_benchmark.py` | rocRAND | Random number generation benchmarks | -| `test_rocsolver_benchmark.py` | ROCsolver | Dense linear algebra benchmarks | +| `test_rocfft_benchmark.py` | rocFFT | FFT benchmarks (1D, 2D, 3D transforms) | +| `test_rocrand_benchmark.py` | rocRAND | Random number generation benchmarks | +| `test_rocsolver_benchmark.py` | ROCsolver | Dense linear algebra benchmarks | ### Running Locally @@ -56,12 +57,12 @@ benchmark/ Tests defined in `benchmark_matrix.py` for nightly CI: -| Test Name | Library | Platform | Timeout | Artifacts Needed | -|-----------|---------|----------|---------|------------------| -| `hipblaslt_bench` | hipBLASLt | Linux | 60 min | `--blas --tests` | -| `rocfft_bench` | rocFFT | Linux | 60 min | `--fft --rand --tests` | -| `rocrand_bench` | rocRAND | Linux | 60 min | `--rand --tests` | -| `rocsolver_bench` | ROCsolver | Linux | 60 min | `--blas --tests` | +| Test Name | Library | Platform | Timeout | Artifacts Needed | +| ----------------- | --------- | -------- | ------- | ---------------------- | +| `hipblaslt_bench` | hipBLASLt | Linux | 60 min | `--blas --tests` | +| `rocfft_bench` | rocFFT | Linux | 60 min | `--fft --rand --tests` | +| `rocrand_bench` | rocRAND | Linux | 60 min | `--rand --tests` | +| `rocsolver_bench` | ROCsolver | Linux | 60 min | `--blas --tests` | ## How Benchmark Tests Work @@ -106,24 +107,24 @@ Tests defined in `benchmark_matrix.py` for nightly CI: ```python # Pseudocode current_score = 42.5 # TFLOPS -lkg_baseline = 45.0 # TFLOPS from last successful run -tolerance = 0.05 # 5% tolerance +lkg_baseline = 45.0 # TFLOPS from last successful run +tolerance = 0.05 # 5% tolerance if current_score < lkg_baseline * (1 - tolerance): - result = "FAIL" # Performance regression! + result = "FAIL" # Performance regression! elif lkg_baseline is None: result = "UNKNOWN" # No baseline data yet else: - result = "PASS" # Performance acceptable + result = "PASS" # Performance acceptable ``` ### 3. Result Statuses -| Status | Meaning | Action | -|--------|---------|--------| -| **PASS** | Performance within tolerance of baseline | ✅ CI passes | -| **FAIL** | Performance degraded beyond tolerance | ❌ CI fails, blocks merge | -| **UNKNOWN** | No baseline data available (new test) | ⚠️ CI passes, baseline established | +| Status | Meaning | Action | +| ----------- | ---------------------------------------- | ---------------------------------- | +| **PASS** | Performance within tolerance of baseline | ✅ CI passes | +| **FAIL** | Performance degraded beyond tolerance | ❌ CI fails, blocks merge | +| **UNKNOWN** | No baseline data available (new test) | ⚠️ CI passes, baseline established | ## Adding a New Benchmark @@ -148,29 +149,26 @@ from utils.logger import log class YourLibBenchmark(BenchmarkBase): """Benchmark tests for YourLib library.""" - + def __init__(self): - super().__init__( - benchmark_name="yourlib", - display_name="YourLib" - ) + super().__init__(benchmark_name="yourlib", display_name="YourLib") self.benchmark_bin = "yourlib-bench" self.log_file = self.script_dir / "yourlib_bench.log" - + def run_benchmarks(self) -> None: """Execute benchmark binary and log output.""" # Your benchmark execution logic # Use self.run_command() to execute binaries pass - + def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable]: """Parse log file and return (test_results, results_table).""" test_results = [] table = PrettyTable() - + # Your parsing logic # Use self.create_test_result() to build result dictionaries - + return test_results, table @@ -193,7 +191,6 @@ Edit `benchmark_matrix.py`: }, ``` - ### Step 3: Test Locally ```bash @@ -209,18 +206,16 @@ python scripts/test_yourlib_benchmark.py Benchmark tests automatically run in nightly CI: 1. **Trigger:** Nightly scheduled run -2. **Execution:** Parallel with regular tests -3. **Matrix:** Generated from `benchmark_matrix.py` -4. **Runners:** Dedicated GPU runners (if configured) -5. **Results:** Uploaded to results API, compared with LKG +1. **Execution:** Parallel with regular tests +1. **Matrix:** Generated from `benchmark_matrix.py` +1. **Runners:** Dedicated GPU runners (if configured) +1. **Results:** Uploaded to results API, compared with LKG See [main test framework README](../README.md) for full CI architecture. - ## Related Documentation - [Test Framework README](../README.md) - Main framework documentation - [Shared Utils](../utils/README.md) - Utility modules reference - [Performance Tests](../performance/) - Performance characterization tests - [Functional Tests](../functional/) - Correctness validation tests - diff --git a/build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py b/build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py index 92ce200dba0..b14b880a408 100644 --- a/build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py +++ b/build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py @@ -8,7 +8,9 @@ from pathlib import Path # Note: these paths are relative to the repository root. -SCRIPT_DIR = Path("build_tools") / "github_actions" / "test_framework" / "benchmark" / "scripts" +SCRIPT_DIR = ( + Path("build_tools") / "github_actions" / "test_framework" / "benchmark" / "scripts" +) def _get_benchmark_script_path(script_name: str) -> str: diff --git a/build_tools/github_actions/test_framework/functional/configs/miopen_driver_conv.json b/build_tools/github_actions/test_framework/functional/configs/miopen_driver_conv.json index f34200a76d4..8a4dbfd15b4 100644 --- a/build_tools/github_actions/test_framework/functional/configs/miopen_driver_conv.json +++ b/build_tools/github_actions/test_framework/functional/configs/miopen_driver_conv.json @@ -46,4 +46,3 @@ } } } - diff --git a/build_tools/github_actions/test_framework/functional/functional_matrix.py b/build_tools/github_actions/test_framework/functional/functional_matrix.py index 1accac66bb3..ab99e007e1b 100644 --- a/build_tools/github_actions/test_framework/functional/functional_matrix.py +++ b/build_tools/github_actions/test_framework/functional/functional_matrix.py @@ -7,7 +7,9 @@ from pathlib import Path # Note: these paths are relative to the repository root. -SCRIPT_DIR = Path("build_tools") / "github_actions" / "test_framework" / "functional" / "scripts" +SCRIPT_DIR = ( + Path("build_tools") / "github_actions" / "test_framework" / "functional" / "scripts" +) def _get_script_path(script_name: str) -> str: @@ -27,6 +29,6 @@ def _get_script_path(script_name: str) -> str: "test_script": f"python {_get_script_path('test_rocrand_benchmark.py')}", # TODO(lajagapp): Add windows support "platform": ["linux"], - "total_shards": 1 + "total_shards": 1, }, } diff --git a/build_tools/github_actions/test_framework/functional/scripts/__init__.py b/build_tools/github_actions/test_framework/functional/scripts/__init__.py index cbb899c7a7e..96bd1ebf8e0 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/__init__.py +++ b/build_tools/github_actions/test_framework/functional/scripts/__init__.py @@ -1,3 +1 @@ """Functional test scripts.""" - - diff --git a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py b/build_tools/github_actions/test_framework/functional/scripts/functional_base.py index 5901e0e9dbd..11b51e1ba04 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py +++ b/build_tools/github_actions/test_framework/functional/scripts/functional_base.py @@ -22,7 +22,7 @@ class FunctionalBase: """Base class providing common functional test logic. Child classes must implement run_tests() and parse_results(). - + Unlike benchmarks (which measure performance), functional tests verify correctness and produce pass/fail results without performance metrics. """ @@ -35,7 +35,7 @@ def __init__(self, test_name: str, display_name: str = None): display_name: Display name for reports (e.g., 'MIOpen Driver Convolution') """ self.test_name = test_name - self.display_name = display_name or test_name.replace('_', ' ').title() + self.display_name = display_name or test_name.replace("_", " ").title() # Environment variables self.therock_bin_dir = os.getenv("THEROCK_BIN_DIR") @@ -67,12 +67,12 @@ def load_config(self, config_filename: str) -> Dict[str, Any]: except FileNotFoundError: raise TestExecutionError( f"Configuration file not found: {config_file}", - action=f"Ensure {config_filename} exists in configs/ directory" + action=f"Ensure {config_filename} exists in configs/ directory", ) except json.JSONDecodeError as e: raise TestExecutionError( f"Invalid JSON in configuration file: {e}", - action=f"Check JSON syntax in {config_filename}" + action=f"Check JSON syntax in {config_filename}", ) def create_test_result( @@ -127,7 +127,7 @@ def calculate_statistics( failed = sum(1 for r in test_results if r.get("status") == "FAIL") error = sum(1 for r in test_results if r.get("status") == "ERROR") skipped = sum(1 for r in test_results if r.get("status") == "SKIP") - + # Overall status: PASS only if no failures/errors overall_status = "PASS" if (failed == 0 and error == 0) else "FAIL" @@ -144,11 +144,11 @@ def create_summary_table( self, stats: Dict[str, Any], num_suites: int ) -> PrettyTable: """Create overall summary table with all statistics. - + Args: stats: Test statistics dictionary num_suites: Number of test suites - + Returns: PrettyTable with summary statistics """ @@ -160,30 +160,36 @@ def create_summary_table( "Failed", "Errored", "Skipped", - "Final Result" + "Final Result", ] - - summary_table.add_row([ - num_suites, - stats["total"], - stats["passed"], - stats["failed"], - stats["error"], - stats["skipped"], - stats["overall_status"] - ]) - + + # Set consistent column width for uniform appearance + for field in summary_table.field_names: + summary_table.min_width[field] = 17 + + summary_table.add_row( + [ + num_suites, + stats["total"], + stats["passed"], + stats["failed"], + stats["error"], + stats["skipped"], + stats["overall_status"], + ] + ) + return summary_table def upload_results( self, test_results: List[Dict[str, Any]], stats: Dict[str, Any] ) -> bool: """Upload results to API and save locally. - + Args: test_results: List of test result dictionaries stats: Test statistics dictionary - + Returns: True if upload successful, False otherwise """ @@ -207,43 +213,34 @@ def upload_results( ) if success: - log.info("✓ Results uploaded successfully") + log.info("Results uploaded successfully") else: - log.info("⚠ Results saved locally only (API upload disabled or failed)") + log.info("Results saved locally only (API upload disabled or failed)") return success def write_step_summary( - self, stats: Dict[str, Any], detailed_table: PrettyTable, summary_table: PrettyTable + self, stats: Dict[str, Any], summary_table: PrettyTable ) -> None: """Write results to GitHub Actions step summary. - + Args: stats: Test statistics dictionary - detailed_table: PrettyTable with detailed test results summary_table: PrettyTable with summary statistics """ gha_append_step_summary( f"## {self.display_name} - Functional Test Results\n\n" - f"### Detailed Results\n\n" - f"```\n{detailed_table}\n```\n\n" - f"
\n" - f"View summary ({stats['total']} tests)\n\n" - f"```\n{summary_table}\n```\n\n" - f"
" + f"```\n{summary_table}\n```\n" ) def get_gpu_id(self) -> str: """Detect GPU ID using rocminfo.""" try: result = subprocess.run( - ["rocminfo"], - capture_output=True, - text=True, - check=True + ["rocminfo"], capture_output=True, text=True, check=True ) - # Extract GPU name (e.g., gfx906, gfx90a, gfx942) - match = re.search(r'Name:\s+(gfx\w+)', result.stdout) + # Extract GPU name (e.g., gfx90a, gfx942) + match = re.search(r"Name:\s+(gfx\w+)", result.stdout) if match: return match.group(1) except (subprocess.CalledProcessError, FileNotFoundError): @@ -252,7 +249,7 @@ def get_gpu_id(self) -> str: def run(self) -> int: """Execute functional test workflow and return exit code (0=PASS, 1=FAIL). - + Returns: 0 if all tests passed, 1 if any test failed """ @@ -271,19 +268,19 @@ def run(self) -> int: if not test_results: raise TestExecutionError( "No test results generated", - action="Check if tests executed properly and log file contains output" + action="Check if tests executed properly and log file contains output", ) # Calculate statistics stats = self.calculate_statistics(test_results) - + # Create summary table summary_table = self.create_summary_table(stats, num_suites) # Display results log.info("DETAILED RESULTS") log.info(f"\n{detailed_table}") - + log.info("\nSUMMARY") log.info(f"\n{summary_table}") log.info(f"\nFinal Status: {stats['overall_status']}") @@ -296,7 +293,7 @@ def run(self) -> int: # Write to GitHub Actions step summary try: - self.write_step_summary(stats, detailed_table, summary_table) + self.write_step_summary(stats, summary_table) except Exception as e: log.warning(f"Could not write GitHub Actions summary: {e}") @@ -321,5 +318,6 @@ def run_functional_test_main(test_instance): except Exception as e: log.error(f"Unexpected error: {e}") import traceback + traceback.print_exc() raise diff --git a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py index 0f2132fd510..7b551d7404e 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py +++ b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py @@ -5,7 +5,6 @@ correct functionality across different GPU architectures. """ -import os import re import shlex import subprocess @@ -26,10 +25,9 @@ class MIOpenDriverConvTest(FunctionalBase): def __init__(self): super().__init__( - test_name="miopen_driver_conv", - display_name="MIOpen Driver Convolution" + test_name="miopen_driver_conv", display_name="MIOpen Driver Convolution" ) - + self.log_file = self.script_dir / "miopendriver_conv.log" # Load test configurations from JSON @@ -38,13 +36,11 @@ def __init__(self): # Parse test suites test_suites = config.get("test_suites", {}) self.tests_cmd = {} - self.envs = {} self.tests_list = [] for suite_name, suite_config in test_suites.items(): self.tests_list.append(suite_name) self.tests_cmd[suite_name] = suite_config.get("commands", []) - self.envs[suite_name] = suite_config.get("algorithm", "") # Load GPU-specific flags self.gpu_specific_flags = config.get("gpu_specific_flags", {}) @@ -60,7 +56,7 @@ def run_tests(self) -> None: if not Path(miopen_driver).exists(): raise TestExecutionError( f"MIOpenDriver not found at {miopen_driver}", - action="Ensure MIOpen is installed correctly" + action="Ensure MIOpen is installed correctly", ) with open(self.log_file, "w+") as f: @@ -70,17 +66,18 @@ def run_tests(self) -> None: f.write(f"Test Suite: {test_suite}\n") f.write(f"{'='*80}\n\n") - # Set environment variable for specific algorithm - env = os.environ.copy() - env['MIOPEN_FIND_ENFORCE'] = self.envs[test_suite] - for cmd_str in self.tests_cmd[test_suite]: # Build full command with MIOpenDriver path full_cmd = f"{miopen_driver} {cmd_str}" - + # Add GPU-specific flags if needed - if 'Backward_Conv' in test_suite and gpu_id in self.gpu_specific_flags: - backward_flags = self.gpu_specific_flags[gpu_id].get("backward_flags", "") + if ( + "Backward_Conv" in test_suite + and gpu_id in self.gpu_specific_flags + ): + backward_flags = self.gpu_specific_flags[gpu_id].get( + "backward_flags", "" + ) if backward_flags: full_cmd = f"{full_cmd} {backward_flags}" @@ -98,7 +95,6 @@ def run_tests(self) -> None: stderr=subprocess.STDOUT, text=True, bufsize=1, - env=env ) for line in process.stdout: @@ -124,11 +120,7 @@ def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: # Setup detailed table - show each individual test case detailed_table = PrettyTable() - detailed_table.field_names = [ - "TestSuite", - "TestCase", - "Status" - ] + detailed_table.field_names = ["TestSuite", "TestCase", "Status"] test_results = [] @@ -140,39 +132,44 @@ def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: for test_suite in self.tests_list: # Find the section for this test suite pattern = re.compile( - rf"Test Suite: {test_suite}.*?(?=Test Suite:|$)", - re.DOTALL + rf"Test Suite: {test_suite}.*?(?=Test Suite:|$)", re.DOTALL ) match = pattern.search(content) - + if not match: log.warning(f"Could not find results for {test_suite}") continue suite_content = match.group(0) - + # Parse each command in the suite for i, cmd_str in enumerate(self.tests_cmd[test_suite], 1): # Build the command pattern to search for miopen_driver = f"{self.therock_bin_dir}/MIOpenDriver" full_cmd = f"{miopen_driver} {cmd_str}" test_case_name = f"{test_suite}_case{i}" - + # Find this specific command in the suite content cmd_pattern = re.escape(f"Command: {full_cmd}") cmd_match = re.search(cmd_pattern, suite_content) - + if cmd_match: # Extract content after this command until next command or end start_pos = cmd_match.end() - next_cmd_match = re.search(r"\nCommand:", suite_content[start_pos:]) + next_cmd_match = re.search( + r"\nCommand:", suite_content[start_pos:] + ) if next_cmd_match: - cmd_section = suite_content[start_pos:start_pos + next_cmd_match.start()] + cmd_section = suite_content[ + start_pos : start_pos + next_cmd_match.start() + ] else: cmd_section = suite_content[start_pos:] - + # Check return code in THIS command's section only - return_code_match = re.search(r"Return code:\s*(\d+)", cmd_section) + return_code_match = re.search( + r"Return code:\s*(\d+)", cmd_section + ) if return_code_match: return_code = int(return_code_match.group(1)) status = "PASS" if return_code == 0 else "FAIL" @@ -186,11 +183,7 @@ def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: status = "FAIL" # Command not found in log # Add each individual test case to detailed table - detailed_table.add_row([ - test_suite, - test_case_name, - status - ]) + detailed_table.add_row([test_suite, test_case_name, status]) # Add each test case to results list using helper test_results.append( @@ -200,19 +193,19 @@ def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: status=status, suite=test_suite, command_index=i, - command=cmd_str + command=cmd_str, ) ) except FileNotFoundError: raise TestExecutionError( f"Log file not found: {self.log_file}", - action="Ensure tests were executed successfully" + action="Ensure tests were executed successfully", ) except OSError as e: raise TestExecutionError( f"Error reading log file: {e}", - action="Check file permissions and disk space" + action="Check file permissions and disk space", ) num_suites = len(self.tests_list) diff --git a/build_tools/github_actions/test_framework/utils/test_client.py b/build_tools/github_actions/test_framework/utils/test_client.py index 88c923e6061..760fe6dc7a5 100644 --- a/build_tools/github_actions/test_framework/utils/test_client.py +++ b/build_tools/github_actions/test_framework/utils/test_client.py @@ -21,7 +21,7 @@ class TestClient: """Client for system detection, result collection, and API upload. - + Used by all test types (benchmark, performance, functional) for: - System information detection (OS, hardware, ROCm) - Test result collection and formatting From 8013e71431023547be6b9c0d0c92b2dd097991e0 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 6 Jan 2026 19:34:54 +0000 Subject: [PATCH 05/28] Update overall README.md Signed-off-by: Lenine Ajagappane --- .../github_actions/test_framework/README.md | 69 ++---- .../test_framework/benchmark/README.md | 3 +- .../test_framework/functional/README.md | 219 ++++++++++++++++++ .../performance/performance_matrix.py | 26 --- .../performance/scripts/__init__.py | 0 .../test_framework/utils/README.md | 17 +- 6 files changed, 244 insertions(+), 90 deletions(-) create mode 100644 build_tools/github_actions/test_framework/functional/README.md delete mode 100644 build_tools/github_actions/test_framework/performance/performance_matrix.py delete mode 100644 build_tools/github_actions/test_framework/performance/scripts/__init__.py diff --git a/build_tools/github_actions/test_framework/README.md b/build_tools/github_actions/test_framework/README.md index 61878b4ffb3..725374d0ed3 100644 --- a/build_tools/github_actions/test_framework/README.md +++ b/build_tools/github_actions/test_framework/README.md @@ -1,6 +1,6 @@ # TheRock Test Framework -Unified testing framework for TheRock ROCm distribution, supporting benchmark, performance, and functional testing with automated execution, system detection, and results management. +Unified testing framework for TheRock ROCm distribution, supporting benchmark and functional testing with automated execution, system detection, and results management. ## Table of Contents @@ -18,7 +18,7 @@ The test framework provides a unified infrastructure for multiple test types: ### Features -- **Multi-Type Testing** - Benchmark, performance, and functional tests +- **Multi-Type Testing** - Benchmark and functional tests - **Shared Infrastructure** - Common utilities, configuration, and results management - **System Auto-Detection** - Hardware, OS, GPU, and ROCm version detection - **Results Management** - Local storage (JSON) and API upload with retry logic @@ -41,29 +41,17 @@ The test framework provides a unified infrastructure for multiple test types: **See:** [benchmark/README.md](benchmark/README.md) -### 2. Performance Tests (`performance/`) - -**Purpose:** Comprehensive performance characterization with detailed metrics and analysis. - -- **Result:** Detailed metrics, scaling curves, bottleneck analysis -- **Comparison:** Current vs hardware specs/targets -- **Frequency:** Weekly, before releases, on-demand -- **Use Case:** Performance optimization, hardware characterization -- **Example:** "Achieved 42.5 TFLOPS (94% of peak)" + scaling analysis - -**Status:** Framework ready, tests to be implemented - -### 3. Functional Tests (`functional/`) +### 2. Functional Tests (`functional/`) **Purpose:** Validate correctness, API contracts, and expected behavior. - **Result:** PASS/FAIL with detailed error messages - **Comparison:** Output vs expected results -- **Frequency:** Every PR, nightly CI +- **Frequency:** Every nightly CI - **Use Case:** Correctness validation, API testing - **Example:** "Matrix multiplication result matches expected output" → PASS -**Status:** Framework ready, tests to be implemented +**See:** [functional/README.md](functional/README.md) ## Quick Start @@ -78,11 +66,8 @@ export AMDGPU_FAMILIES=gfx950-dcgpu # Run benchmark test python test_framework/benchmark/scripts/test_hipblaslt_benchmark.py -# Run performance test (when implemented) -python test_framework/performance/scripts/test_rocblas_performance.py - -# Run functional test (when implemented) -python test_framework/functional/scripts/test_rocblas_functional.py +# Run functional test +python test_framework/functional/scripts/test_miopendriver_conv.py ``` ### Available Tests @@ -94,9 +79,9 @@ python test_framework/functional/scripts/test_rocblas_functional.py - `test_rocrand_benchmark.py` - rocRAND generation benchmarks - `test_rocsolver_benchmark.py` - ROCsolver linear algebra benchmarks -**Performance Tests:** (to be implemented) +**Functional Tests:** -**Functional Tests:** (to be implemented) +- `test_miopendriver_conv.py` - MIOpen driver convolution functional tests ## Project Structure @@ -118,17 +103,14 @@ test_framework/ │ ├── benchmark_matrix.py # CI test matrix │ └── README.md # Benchmark-specific docs │ -├── performance/ # Performance characterization tests -│ ├── scripts/ # Test implementations (to be added) -│ ├── configs/ # Test-specific configurations -│ ├── performance_matrix.py # CI test matrix -│ └── README.md # (to be created) -│ ├── functional/ # Functional/correctness tests -│ ├── scripts/ # Test implementations (to be added) +│ ├── scripts/ # Test implementations +│ │ ├── functional_base.py # Base class for functional tests +│ │ └── test_miopendriver_conv.py # MIOpen convolution tests │ ├── configs/ # Test-specific configurations +│ │ └── miopen_driver_conv.json │ ├── functional_matrix.py # CI test matrix -│ └── README.md # (to be created)│ +│ └── README.md # (to be created) │ └── utils/ # SHARED utilities for all test types ├── exceptions.py # Custom exception classes @@ -239,7 +221,6 @@ ci_nightly.yml → ci_linux.yml ┌─────────────────────────────────────────┐ │ 4. Process Results (Type-Specific) │ │ Benchmark: Compare with LKG baseline │ -│ Performance: Analyze metrics/trends │ │ Functional: Validate correctness │ └──────────────┬──────────────────────────┘ ↓ @@ -258,33 +239,15 @@ ci_nightly.yml → ci_linux.yml See detailed guide in [benchmark/README.md](benchmark/README.md#adding-a-new-benchmark) -Quick steps: - -1. Create `benchmark/scripts/test_yourlib_benchmark.py` -1. Inherit from `BenchmarkBase`, implement `run_benchmarks()` and `parse_results()` -1. Add to `benchmark/benchmark_matrix.py` -1. Test locally - -### Adding a Performance Test - -1. Create `performance/scripts/test_yourlib_performance.py` -1. Inherit from `PerformanceRunner` (to be implemented) -1. Implement metrics collection and analysis -1. Add to `performance/performance_matrix.py` - ### Adding a Functional Test -1. Create `functional/scripts/test_yourlib_functional.py` -1. Inherit from `FunctionalRunner` (to be implemented) -1. Implement validation logic -1. Add to `functional/functional_matrix.py` +See detailed guide in [functional/README.md](functional/README.md#adding-a-new-benchmark) ### Getting Help - **Framework issues:** See this README - **Benchmark-specific:** See [benchmark/README.md](benchmark/README.md) -- **Performance tests:** See `performance/README.md` (to be created) -- **Functional tests:** See `functional/README.md` (to be created) +- **Functional tests:** See [functional/README.md](functional/README.md) - **Utilities:** See [utils/README.md](utils/README.md) ## Related Files diff --git a/build_tools/github_actions/test_framework/benchmark/README.md b/build_tools/github_actions/test_framework/benchmark/README.md index 118f9c82351..a3c7f17d6ee 100644 --- a/build_tools/github_actions/test_framework/benchmark/README.md +++ b/build_tools/github_actions/test_framework/benchmark/README.md @@ -217,5 +217,4 @@ See [main test framework README](../README.md) for full CI architecture. - [Test Framework README](../README.md) - Main framework documentation - [Shared Utils](../utils/README.md) - Utility modules reference -- [Performance Tests](../performance/) - Performance characterization tests -- [Functional Tests](../functional/) - Correctness validation tests +- [Functional Tests](../functional/README.md) - Correctness validation tests diff --git a/build_tools/github_actions/test_framework/functional/README.md b/build_tools/github_actions/test_framework/functional/README.md new file mode 100644 index 00000000000..f50ffeb3ee7 --- /dev/null +++ b/build_tools/github_actions/test_framework/functional/README.md @@ -0,0 +1,219 @@ +# Functional Tests + +Correctness validation tests that verify expected behavior without performance measurements. + +## Purpose + +Functional tests validate **correctness and behavior** by checking that outputs match expectations: + +- **Result:** PASS (correct behavior) / FAIL (incorrect behavior) / ERROR (execution failure) / SKIP (prerequisites not met) +- **Frequency:** Every nightly CI +- **Use Case:** Automated correctness validation, API contract verification + +## Quick Start + +### Available Functional Tests + +| Test Script | Library | Description | +| -------------------------- | ------- | ---------------------------------------------- | +| `test_miopendriver_conv.py` | MIOpen | Convolution forward/backward correctness tests | + +### Running Locally + +```bash +# Set required environment variables +export THEROCK_BIN_DIR=/path/to/therock/build/bin +export ARTIFACT_RUN_ID=local-test +export AMDGPU_FAMILIES=gfx94x-dcgpu + +# Run a functional test +python build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py +``` + +## Directory Structure + +``` +functional/ +├── scripts/ # Functional test implementations +│ ├── functional_base.py # Base class (common test logic) +│ └── test_miopendriver_conv.py +│ +├── configs/ # Test-specific configurations +│ └── miopen_driver_conv.json # MIOpen test parameters +│ +├── functional_matrix.py # CI test matrix definitions +└── README.md # This file +``` + +## Functional Test Matrix + +Tests defined in `functional_matrix.py` for CI: + +| Test Name | Library | Platform | Timeout | Artifacts Needed | +| ------------------- | ------- | -------- | ------- | ---------------- | +| `miopen_driver_conv` | MIOpen | Linux | 30 min | `--miopen --tests` | + +## How Functional Tests Work + +### 1. Execution Flow + +``` +┌─────────────────────────────────────────┐ +│ 1. Initialize Test │ +│ - Auto-detect GPU, ROCm version │ +│ - Load configuration from JSON │ +└──────────────┬──────────────────────────┘ + ↓ +┌─────────────────────────────────────────┐ +│ 2. Run Test Commands │ +│ - Execute test binary/driver │ +│ - Capture output to log file │ +└──────────────┬──────────────────────────┘ + ↓ +┌─────────────────────────────────────────┐ +│ 3. Parse Results │ +│ - Extract pass/fail from logs │ +│ - Generate detailed per-case results │ +│ - Calculate statistics │ +└──────────────┬──────────────────────────┘ + ↓ +┌─────────────────────────────────────────┐ +│ 4. Report Results │ +│ - Display formatted tables │ +│ - Upload to results API │ +│ - Generate GitHub Actions summary │ +└─────────────────────────────────────────┘ +``` + +### 2. Result Tables + +Functional tests generate two tables: + +**Detailed Table:** One row per test case +``` ++--------------+--------------------+--------+ +| TestSuite | TestCase | Status | ++--------------+--------------------+--------+ +| Forward_Conv | Forward_Conv_case1 | PASS | +| Forward_Conv | Forward_Conv_case2 | PASS | ++--------------+--------------------+--------+ +``` + +**Summary Table:** Overall statistics +``` ++-------------------+-------------------+--------+--------+---------+---------+-------------------+ +| Total TestSuites | Total TestCases | Passed | Failed | Errored | Skipped | Final Result | ++-------------------+-------------------+--------+--------+---------+---------+-------------------+ +| 2 | 18 | 18 | 0 | 0 | 0 | PASS | ++-------------------+-------------------+--------+--------+---------+---------+-------------------+ +``` + +## Adding New Functional Tests + +### Step 1: Create Test Script + +Create `test_yourtest.py` in `scripts/`: + +```python +"""YourTest Functional Test""" + +import sys +from pathlib import Path +from typing import Dict, List, Tuple, Any +from prettytable import PrettyTable + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # test_framework/ +sys.path.insert(0, str(Path(__file__).parent)) # For functional_base +from functional_base import FunctionalBase, run_functional_test_main +from utils.logger import log +from utils.exceptions import TestExecutionError + + +class YourTest(FunctionalBase): + """YourTest functional test.""" + + def __init__(self): + super().__init__( + test_name="yourtest_functional", + display_name="YourTest Functional Test" + ) + self.log_file = self.script_dir / "yourtest_functional.log" + + # Load test configuration from JSON + config = self.load_config("yourtest_functional.json") + self.test_cases = config.get("test_cases", []) + + def run_tests(self) -> None: + """Run functional tests and save output to log file.""" + log.info(f"Running {self.display_name}") + + with open(self.log_file, "w+") as f: + for test_case in self.test_cases: + cmd = test_case["command"] + # Execute command, capture output + # Write to log file + pass + + def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: + """Parse log file and return (detailed_table, num_suites).""" + log.info("Parsing Results") + + detailed_table = PrettyTable() + detailed_table.field_names = ["TestCase", "Status"] + + test_results = [] + + # Parse log file for each test case + # Use self.create_test_result() to create result dictionaries + + return test_results, detailed_table, num_suites + + +if __name__ == "__main__": + run_functional_test_main(YourTestFunctionalTest()) +``` + + +### Step 2: Add to Functional Matrix + +Edit `functional_matrix.py`: + +```python +"yourtest_name": { + "job_name": "yourtest_name", + "fetch_artifact_args": "--yourtest --tests", + "timeout_minutes": 30, + "test_script": f"python {_get_script_path('test_yourtest.py')}", + "platform": ["linux"], + "total_shards": 1, +}, +``` + +### Step 3: Test Locally + +```bash +export THEROCK_BIN_DIR=/path/to/build/bin +export ARTIFACT_RUN_ID=local-test +export AMDGPU_FAMILIES=gfx94x-dcgpu + +python scripts/test_yourtest.py +``` + +## CI Integration + +Functional tests run in CI workflows: + +1. **Trigger:** Nightly scheduled run +1. **Execution:** Parallel with other tests +1. **Matrix:** Generated from `functional_matrix.py` +1. **Runners:** Standard test runners +1. **Results:** Uploaded to results API, displayed in GHA summary + +See [main test framework README](../README.md) for full CI architecture. + +## Related Documentation + +- [Test Framework README](../README.md) - Main framework documentation +- [Shared Utils](../utils/README.md) - Utility modules reference +- [Benchmark Tests](../benchmark/README.md) - Performance regression tests + diff --git a/build_tools/github_actions/test_framework/performance/performance_matrix.py b/build_tools/github_actions/test_framework/performance/performance_matrix.py deleted file mode 100644 index 0c8a514c49a..00000000000 --- a/build_tools/github_actions/test_framework/performance/performance_matrix.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Performance test matrix definitions. - -Performance tests characterize system performance with detailed metrics, -scaling analysis, and bottleneck identification. -""" - -from pathlib import Path - -SCRIPT_DIR = Path("build_tools/github_actions/test_framework/performance/scripts") - -performance_matrix = { - # Example performance test (uncomment and customize): - # "rocblas_performance": { - # "job_name": "rocBLAS Performance Characterization", - # "fetch_artifact_args": "--blas", - # "timeout_minutes": 90, - # "test_script": f"python {SCRIPT_DIR}/test_rocblas_performance.py", - # "platform": ["linux"], - # "metrics": ["tflops", "latency", "bandwidth"], - # "targets": { - # "tflops_min": 40.0, - # "latency_max_ms": 2.0 - # } - # }, -} diff --git a/build_tools/github_actions/test_framework/performance/scripts/__init__.py b/build_tools/github_actions/test_framework/performance/scripts/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/build_tools/github_actions/test_framework/utils/README.md b/build_tools/github_actions/test_framework/utils/README.md index 80e85fe07f1..123f4f99018 100644 --- a/build_tools/github_actions/test_framework/utils/README.md +++ b/build_tools/github_actions/test_framework/utils/README.md @@ -5,7 +5,7 @@ Utility modules organized into logical subdirectories for maintainability and sc ## Structure ``` -benchmarks/utils/ +test_framework/utils/ ├── __init__.py # Public exports ├── test_client.py # Main TestClient API ├── constants.py # Framework constants @@ -36,13 +36,15 @@ benchmarks/utils/ ## Usage -### From Benchmark Scripts +### From Test Scripts -Benchmark scripts add `benchmarks/` to `sys.path`, then import: +Test scripts (benchmark/functional) add `test_framework/` to `sys.path`, then import: ```python -# Import path setup (already done in benchmark_base.py) -sys.path.insert(0, str(Path(__file__).parent.parent)) # Adds benchmarks/ to path +# Import path setup (already done in base classes) +sys.path.insert( + 0, str(Path(__file__).parent.parent.parent) +) # Adds test_framework/ to path # Core utilities from utils.logger import log @@ -127,10 +129,7 @@ Test results formatting, saving, and API submission. # Run from project root cd /path/to/TheRock -# Run a benchmark test (imports are handled internally) -python build_tools/github_actions/benchmarks/scripts/test_rocfft_benchmark.py - # Verify utils imports work -cd build_tools/github_actions/benchmarks +cd build_tools/github_actions/test_framework python -c "from utils.logger import log; print('✓ Utils imports working')" ``` From fb3521babbb79707e36ee1a4e9d81f2583a0f9a5 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Wed, 7 Jan 2026 15:47:40 +0000 Subject: [PATCH 06/28] Merge main into users/lajagapp/add-miopen-driver-test Signed-off-by: Lenine Ajagappane --- .github/workflows/ci_nightly.yml | 2 + .github/workflows/ci_windows.yml | 29 ++++ .github/workflows/test_sanity_check.yml | 1 + .../github_actions/amdgpu_family_matrix.py | 12 +- .../github_actions/test_framework/__init__.py | 1 - .../test_framework/benchmark/README.md | 32 ++++- .../benchmark/benchmark_matrix.py | 14 +- .../benchmark/scripts/benchmark_base.py | 4 +- .../scripts/test_rocrand_benchmark.py | 8 +- .../test_framework/utils/README.md | 2 +- .../utils/config/config_validator.py | 12 +- .../utils/results/results_api.py | 12 +- .../utils/results/results_handler.py | 6 +- .../test_framework/utils/system/hardware.py | 135 +++++++++++++++++- .../utils/system/system_detector.py | 12 +- 15 files changed, 226 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci_nightly.yml b/.github/workflows/ci_nightly.yml index ad410526b4d..fcd9af7d46e 100644 --- a/.github/workflows/ci_nightly.yml +++ b/.github/workflows/ci_nightly.yml @@ -102,10 +102,12 @@ jobs: matrix: variant: ${{ fromJSON(needs.setup.outputs.windows_variants) }} uses: ./.github/workflows/ci_windows.yml + secrets: inherit with: amdgpu_families: ${{ matrix.variant.family }} artifact_group: ${{ matrix.variant.artifact_group }} test_runs_on: ${{ matrix.variant.test-runs-on }} + benchmark_runs_on: ${{ matrix.variant.benchmark-runs-on }} build_variant_label: ${{ matrix.variant.build_variant_label }} build_variant_suffix: ${{ matrix.variant.build_variant_suffix }} build_variant_cmake_preset: ${{ matrix.variant.build_variant_cmake_preset }} diff --git a/.github/workflows/ci_windows.yml b/.github/workflows/ci_windows.yml index 536463a2c4e..064ab33c57c 100644 --- a/.github/workflows/ci_windows.yml +++ b/.github/workflows/ci_windows.yml @@ -19,6 +19,9 @@ on: type: string test_runs_on: type: string + benchmark_runs_on: + type: string + default: "" expect_failure: type: boolean use_prebuilt_artifacts: @@ -85,6 +88,32 @@ jobs: test_labels: ${{ inputs.test_labels }} sanity_check_only_for_family: ${{ inputs.sanity_check_only_for_family == true }} + test_windows_benchmarks: + needs: [build_windows_artifacts] + name: Test Windows Benchmarks + # Run benchmarks if: + # - Build succeeded (or using prebuilt artifacts) + # - Not expecting failure + # - Benchmark runner is available (benchmark_runs_on is set) + if: >- + ${{ + !failure() && + !cancelled() && + ( + inputs.use_prebuilt_artifacts == 'false' || + inputs.use_prebuilt_artifacts == 'true' + ) && + inputs.expect_failure == false && + inputs.benchmark_runs_on != '' + }} + uses: ./.github/workflows/test_benchmarks.yml + secrets: inherit + with: + artifact_group: ${{ inputs.artifact_group }} + amdgpu_families: ${{ inputs.amdgpu_families }} + test_runs_on: ${{ inputs.benchmark_runs_on }} + artifact_run_id: ${{ inputs.artifact_run_id }} + build_windows_python_packages: needs: [build_windows_artifacts] name: Build Python diff --git a/.github/workflows/test_sanity_check.yml b/.github/workflows/test_sanity_check.yml index 372fbc30846..5934e5cf7cd 100644 --- a/.github/workflows/test_sanity_check.yml +++ b/.github/workflows/test_sanity_check.yml @@ -60,6 +60,7 @@ jobs: ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" OUTPUT_ARTIFACTS_DIR: ${{ github.workspace }}/build THEROCK_BIN_DIR: ${{ github.workspace }}/build/bin + AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} steps: - name: "Fetch 'build_tools' from repository" if: ${{ runner.os == 'Windows' }} diff --git a/build_tools/github_actions/amdgpu_family_matrix.py b/build_tools/github_actions/amdgpu_family_matrix.py index d7f315ce7c3..e5f0b169855 100644 --- a/build_tools/github_actions/amdgpu_family_matrix.py +++ b/build_tools/github_actions/amdgpu_family_matrix.py @@ -49,7 +49,7 @@ "gfx94x": { "linux": { "test-runs-on": "linux-mi325-1gpu-ossci-rocm-frac", - # TODO: Add new benchmark-runs-on runner for benchmarks + # TODO(#2754): Add new benchmark-runs-on runner for benchmarks "benchmark-runs-on": "linux-mi325-1gpu-ossci-rocm-frac", "family": "gfx94X-dcgpu", "build_variants": ["release", "asan"], @@ -75,9 +75,7 @@ }, "gfx1151": { "linux": { - # TODO(#2741): Re-enable machine once `amdsmi` test timeout is fixed - # Label is "linux-gfx1151-gpu-rocm" - "test-runs-on": "", + "test-runs-on": "linux-gfx1151-gpu-rocm", "family": "gfx1151", "bypass_tests_for_releases": True, "build_variants": ["release"], @@ -85,6 +83,8 @@ }, "windows": { "test-runs-on": "windows-gfx1151-gpu-rocm", + # TODO(#2754): Add new benchmark-runs-on runner for benchmarks + "benchmark-runs-on": "windows-gfx1151-gpu-rocm", "family": "gfx1151", "build_variants": ["release"], }, @@ -175,9 +175,7 @@ }, "gfx1150": { "linux": { - # TODO(#2614): Re-enable machine once it is stable - # Label is "linux-gfx1150-gpu-rocm" - "test-runs-on": "", + "test-runs-on": "linux-gfx1150-gpu-rocm", "family": "gfx1150", "build_variants": ["release"], "sanity_check_only_for_family": True, diff --git a/build_tools/github_actions/test_framework/__init__.py b/build_tools/github_actions/test_framework/__init__.py index 60a5207df86..71ccce9e6c1 100644 --- a/build_tools/github_actions/test_framework/__init__.py +++ b/build_tools/github_actions/test_framework/__init__.py @@ -5,7 +5,6 @@ Test Types: - benchmark: Regression detection via LKG comparison (PASS/FAIL gates) -- performance: Detailed performance characterization and analysis - functional: Correctness validation and API testing """ diff --git a/build_tools/github_actions/test_framework/benchmark/README.md b/build_tools/github_actions/test_framework/benchmark/README.md index a3c7f17d6ee..d78733d298e 100644 --- a/build_tools/github_actions/test_framework/benchmark/README.md +++ b/build_tools/github_actions/test_framework/benchmark/README.md @@ -57,12 +57,28 @@ benchmark/ Tests defined in `benchmark_matrix.py` for nightly CI: -| Test Name | Library | Platform | Timeout | Artifacts Needed | -| ----------------- | --------- | -------- | ------- | ---------------------- | -| `hipblaslt_bench` | hipBLASLt | Linux | 60 min | `--blas --tests` | -| `rocfft_bench` | rocFFT | Linux | 60 min | `--fft --rand --tests` | -| `rocrand_bench` | rocRAND | Linux | 60 min | `--rand --tests` | -| `rocsolver_bench` | ROCsolver | Linux | 60 min | `--blas --tests` | +| Test Name | Library | Platform | Timeout | Artifacts Needed | +| ----------------- | --------- | --------------- | ------- | ---------------------- | +| `hipblaslt_bench` | hipBLASLt | Linux, Windows | 60 min | `--blas --tests` | +| `rocfft_bench` | rocFFT | Linux, Windows | 60 min | `--fft --rand --tests` | +| `rocrand_bench` | rocRAND | Linux, Windows | 60 min | `--rand --tests` | +| `rocsolver_bench` | ROCsolver | Linux, Windows | 60 min | `--blas --tests` | + +**GPU Family Support:** + +| GPU Family | Platform | Architecture | Benchmark Supported | Benchmark CI Status | +| ---------- | -------- | --------------------- | ------------------- | -------------------- | +| `gfx94x` | Linux | MI300X/MI325X (CDNA3) | Yes | Enabled (nightly CI) | +| `gfx1151` | Windows | RDNA 3.5 | Yes | Enabled (nightly CI) | +| `gfx950` | Linux | MI355X (CDNA4) | Yes | Not enabled | +| `gfx110x` | Windows | RDNA 2 | Yes | Not enabled | +| `gfx110x` | Linux | RDNA 2 | Yes | Not enabled | +| `gfx120x` | Linux | RDNA 3 | Yes | Not enabled | +| `gfx120x` | Windows | RDNA 3 | Yes | Not enabled | +| `gfx90x` | Linux | MI200 (CDNA2) | Yes | Not enabled | +| `gfx1151` | Linux | RDNA 3.5 | Yes | Not enabled | + +> **Note:** All benchmarks are **architecture-agnostic** and support any ROCm-compatible GPU. The table above lists GPU families actively used in CI testing. To add support for additional GPU families, update [`amdgpu_family_matrix.py`](../amdgpu_family_matrix.py) with appropriate `benchmark-runs-on` runners. ## How Benchmark Tests Work @@ -186,8 +202,10 @@ Edit `benchmark_matrix.py`: "fetch_artifact_args": "--yourlib --tests", "timeout_minutes": 60, "test_script": f"python {_get_benchmark_script_path('test_yourlib_benchmark.py')}", - "platform": ["linux"], + "platform": ["linux", "windows"], # Supported platforms "total_shards": 1, + # TODO: Remove xfail once dedicated performance servers are added + "expect_failure": True, }, ``` diff --git a/build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py b/build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py index b14b880a408..a912f477fac 100644 --- a/build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py +++ b/build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py @@ -28,8 +28,7 @@ def _get_benchmark_script_path(script_name: str) -> str: "fetch_artifact_args": "--blas --tests", "timeout_minutes": 60, "test_script": f"python {_get_benchmark_script_path('test_hipblaslt_benchmark.py')}", - # TODO(lajagapp): Add windows support (https://github.com/ROCm/TheRock/issues/2478) - "platform": ["linux"], + "platform": ["linux", "windows"], "total_shards": 1, # TODO: Remove xfail once dedicated performance servers are added in "benchmark-runs-on" "expect_failure": True, @@ -40,8 +39,7 @@ def _get_benchmark_script_path(script_name: str) -> str: "fetch_artifact_args": "--blas --tests", "timeout_minutes": 60, "test_script": f"python {_get_benchmark_script_path('test_rocsolver_benchmark.py')}", - # TODO(lajagapp): Add windows support (https://github.com/ROCm/TheRock/issues/2478) - "platform": ["linux"], + "platform": ["linux", "windows"], "total_shards": 1, # TODO: Remove xfail once dedicated performance servers are added in "benchmark-runs-on" "expect_failure": True, @@ -50,10 +48,9 @@ def _get_benchmark_script_path(script_name: str) -> str: "rocrand_bench": { "job_name": "rocrand_bench", "fetch_artifact_args": "--rand --tests", - "timeout_minutes": 60, + "timeout_minutes": 90, "test_script": f"python {_get_benchmark_script_path('test_rocrand_benchmark.py')}", - # TODO(lajagapp): Add windows support (https://github.com/ROCm/TheRock/issues/2478) - "platform": ["linux"], + "platform": ["linux", "windows"], "total_shards": 1, # TODO: Remove xfail once dedicated performance servers are added in "benchmark-runs-on" "expect_failure": True, @@ -64,8 +61,7 @@ def _get_benchmark_script_path(script_name: str) -> str: "fetch_artifact_args": "--fft --rand --tests", "timeout_minutes": 60, "test_script": f"python {_get_benchmark_script_path('test_rocfft_benchmark.py')}", - # TODO(lajagapp): Add windows support (https://github.com/ROCm/TheRock/issues/2478) - "platform": ["linux"], + "platform": ["linux", "windows"], "total_shards": 1, # TODO: Remove xfail once dedicated performance servers are added in "benchmark-runs-on" "expect_failure": True, diff --git a/build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py b/build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py index be9d7098444..fde9d3b10e6 100644 --- a/build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py +++ b/build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py @@ -142,9 +142,9 @@ def upload_results( ) if success: - log.info("✓ Results uploaded successfully") + log.info("Results uploaded successfully") else: - log.info("⚠ Results saved locally only (API upload disabled or failed)") + log.info("Results saved locally only (API upload disabled or failed)") return success diff --git a/build_tools/github_actions/test_framework/benchmark/scripts/test_rocrand_benchmark.py b/build_tools/github_actions/test_framework/benchmark/scripts/test_rocrand_benchmark.py index 1c971bd37c2..ba60696619f 100644 --- a/build_tools/github_actions/test_framework/benchmark/scripts/test_rocrand_benchmark.py +++ b/build_tools/github_actions/test_framework/benchmark/scripts/test_rocrand_benchmark.py @@ -44,16 +44,10 @@ def run_benchmarks(self) -> None: bench_type = match.group(1) log_file = self.script_dir / f"{bench_type}_bench.log" - # Check if binary exists - bench_path = Path(self.therock_bin_dir) / bench_bin - if not bench_path.exists(): - log.error(f"Benchmark binary not found: {bench_bin}") - continue - # Run benchmark with open(log_file, "w+") as f: cmd = [ - str(bench_path), + f"{self.therock_bin_dir}/{bench_bin}", "--trials", str(NUM_TRIALS), "--benchmark_color=false", diff --git a/build_tools/github_actions/test_framework/utils/README.md b/build_tools/github_actions/test_framework/utils/README.md index 123f4f99018..55e5bd148da 100644 --- a/build_tools/github_actions/test_framework/utils/README.md +++ b/build_tools/github_actions/test_framework/utils/README.md @@ -131,5 +131,5 @@ cd /path/to/TheRock # Verify utils imports work cd build_tools/github_actions/test_framework -python -c "from utils.logger import log; print('✓ Utils imports working')" +python -c "from utils.logger import log; print('Utils imports working')" ``` diff --git a/build_tools/github_actions/test_framework/utils/config/config_validator.py b/build_tools/github_actions/test_framework/utils/config/config_validator.py index 07a752b87be..a7fd93cb41a 100644 --- a/build_tools/github_actions/test_framework/utils/config/config_validator.py +++ b/build_tools/github_actions/test_framework/utils/config/config_validator.py @@ -181,19 +181,19 @@ def print_properties(props: Dict, indent: int = 0): print(f"{prefix}{key} ({prop_type})") if desc: - print(f"{prefix} → {desc}") + print(f"{prefix} - {desc}") if "enum" in value: - print(f"{prefix} → Options: {', '.join(value['enum'])}") + print(f"{prefix} - Options: {', '.join(value['enum'])}") if "minimum" in value or "maximum" in value: - min_val = value.get("minimum", "-∞") - max_val = value.get("maximum", "∞") - print(f"{prefix} → Range: {min_val} to {max_val}") + min_val = value.get("minimum", "-inf") + max_val = value.get("maximum", "inf") + print(f"{prefix} - Range: {min_val} to {max_val}") if "properties" in value: if required: - print(f"{prefix} → Required fields: {', '.join(required)}") + print(f"{prefix} - Required fields: {', '.join(required)}") print_properties(value["properties"], indent + 1) print() diff --git a/build_tools/github_actions/test_framework/utils/results/results_api.py b/build_tools/github_actions/test_framework/utils/results/results_api.py index db2dfa67f67..a77900890a7 100644 --- a/build_tools/github_actions/test_framework/utils/results/results_api.py +++ b/build_tools/github_actions/test_framework/utils/results/results_api.py @@ -88,7 +88,7 @@ def _try_submit( response.raise_for_status() # If we reach here, request was successful - log.info(f"✓ Results submitted successfully to {url_type} API") + log.info(f"Results submitted successfully to {url_type} API") try: response_data = response.json() log.debug(f"API Response: {json.dumps(response_data, indent=2)}") @@ -98,14 +98,14 @@ def _try_submit( except requests.exceptions.Timeout as e: url_type = "fallback" if is_fallback else "primary" - log.warning(f"✗ {url_type.capitalize()} API Request Timed Out: {e}") + log.warning(f"{url_type.capitalize()} API Request Timed Out: {e}") if not is_fallback: log.debug(" Will try fallback URL if configured") return False except requests.exceptions.ConnectionError as e: url_type = "fallback" if is_fallback else "primary" - log.warning(f"✗ {url_type.capitalize()} API Connection Failed: {e}") + log.warning(f"{url_type.capitalize()} API Connection Failed: {e}") if not is_fallback: log.debug(" Will try fallback URL if configured") return False @@ -117,18 +117,18 @@ def _try_submit( e.response.text[:200] if e.response and e.response.text else str(e) ) log.warning( - f"✗ {url_type.capitalize()} API Error ({status_code}): {error_msg}" + f"{url_type.capitalize()} API Error ({status_code}): {error_msg}" ) return False except json.JSONDecodeError as e: url_type = "fallback" if is_fallback else "primary" - log.warning(f"✗ {url_type.capitalize()} API Invalid JSON Response: {e}") + log.warning(f"{url_type.capitalize()} API Invalid JSON Response: {e}") return False except Exception as e: url_type = "fallback" if is_fallback else "primary" - log.warning(f"✗ {url_type.capitalize()} API Unexpected Error: {e}") + log.warning(f"{url_type.capitalize()} API Unexpected Error: {e}") return False diff --git a/build_tools/github_actions/test_framework/utils/results/results_handler.py b/build_tools/github_actions/test_framework/utils/results/results_handler.py index bac7bbe08cf..d49ddd616fd 100644 --- a/build_tools/github_actions/test_framework/utils/results/results_handler.py +++ b/build_tools/github_actions/test_framework/utils/results/results_handler.py @@ -125,7 +125,7 @@ def save_local_results( with open(output_file, "w") as f: json.dump(results_data, f, indent=2) - log.info(f"✓ Results saved: {output_file}") + log.info(f"Results saved: {output_file}") return output_file except Exception as e: log.error(f"Failed to save local results: {e}") @@ -219,10 +219,10 @@ def upload_to_api( success = api_client.submit_results(payload) if success: - log.info("✓ Results submitted to API successfully") + log.info("Results submitted to API successfully") return True else: - log.warning("⚠ Failed to submit results to API") + log.warning("Failed to submit results to API") return False except Exception as e: diff --git a/build_tools/github_actions/test_framework/utils/system/hardware.py b/build_tools/github_actions/test_framework/utils/system/hardware.py index 769960eab8b..4d48367d94a 100644 --- a/build_tools/github_actions/test_framework/utils/system/hardware.py +++ b/build_tools/github_actions/test_framework/utils/system/hardware.py @@ -3,6 +3,8 @@ import subprocess import re import os +import platform +import multiprocessing from typing import List, Optional, Dict from dataclasses import dataclass, field @@ -112,7 +114,18 @@ def detect_all(self): self.detect_gpu() def detect_cpu(self) -> CpuInfo: - """Detect CPU information from /proc/cpuinfo and lscpu. + """Detect CPU information (cross-platform). + + Returns: + CpuInfo object + """ + if platform.system() == "Windows": + return self._detect_cpu_windows() + else: + return self._detect_cpu_linux() + + def _detect_cpu_linux(self) -> CpuInfo: + """Detect CPU information from /proc/cpuinfo and lscpu (Linux). Returns: CpuInfo object @@ -239,6 +252,126 @@ def detect_cpu(self) -> CpuInfo: return self.cpu_info + def _detect_cpu_windows(self) -> CpuInfo: + """Detect CPU information using Windows APIs and wmic (Windows). + + Returns: + CpuInfo object + """ + try: + # Get CPU cores using multiprocessing + cores = multiprocessing.cpu_count() + + # Get CPU info using wmic + model_name = "Unknown" + clock_speed_mhz = 0 + sockets = 1 + l1_cache_kb = 0 + l2_cache_kb = 0 + l3_cache_kb = 0 + ram_size_gb = 0 + + try: + # Get CPU model name + cpu_output = subprocess.check_output( + ["wmic", "cpu", "get", "Name"], text=True, stderr=subprocess.DEVNULL + ).strip() + lines = [ + line.strip() for line in cpu_output.split("\n") if line.strip() + ] + if len(lines) > 1: + model_name = lines[1] # First line is header "Name" + + # Get CPU clock speed (in MHz) + speed_output = subprocess.check_output( + ["wmic", "cpu", "get", "MaxClockSpeed"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + lines = [ + line.strip() for line in speed_output.split("\n") if line.strip() + ] + if len(lines) > 1 and lines[1].isdigit(): + clock_speed_mhz = int(lines[1]) + + # Get number of physical CPUs (sockets) + socket_output = subprocess.check_output( + ["wmic", "cpu", "get", "NumberOfCores"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + lines = [ + line.strip() for line in socket_output.split("\n") if line.strip() + ] + if len(lines) > 1 and lines[1].isdigit(): + physical_cores = int(lines[1]) + # If we have hyperthreading, sockets = logical cores / (physical cores * 2) + # Otherwise sockets = logical cores / physical cores + if physical_cores > 0: + sockets = max(1, cores // physical_cores) + + # Get L2 cache size + cache_output = subprocess.check_output( + ["wmic", "cpu", "get", "L2CacheSize"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + lines = [ + line.strip() for line in cache_output.split("\n") if line.strip() + ] + if len(lines) > 1 and lines[1].isdigit(): + l2_cache_kb = int(lines[1]) + + # Get L3 cache size + cache_output = subprocess.check_output( + ["wmic", "cpu", "get", "L3CacheSize"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + lines = [ + line.strip() for line in cache_output.split("\n") if line.strip() + ] + if len(lines) > 1 and lines[1].isdigit(): + l3_cache_kb = int(lines[1]) + + # Get total RAM size (in KB, convert to GB) + mem_output = subprocess.check_output( + ["wmic", "computersystem", "get", "TotalPhysicalMemory"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + lines = [ + line.strip() for line in mem_output.split("\n") if line.strip() + ] + if len(lines) > 1 and lines[1].isdigit(): + ram_size_gb = int(lines[1]) // (1024 * 1024 * 1024) + + except Exception: + # If wmic fails, use basic detection + pass + + self.cpu_info = CpuInfo( + model_name=model_name, + cores=cores, + sockets=sockets, + ram_size_gb=ram_size_gb, + numa_nodes=1, # NUMA detection on Windows is complex, default to 1 + clock_speed_mhz=clock_speed_mhz, + l1_cache_kb=l1_cache_kb, + l2_cache_kb=l2_cache_kb, + l3_cache_kb=l3_cache_kb, + ) + + except Exception: + # Fallback to minimal detection with at least the core count + try: + cores = multiprocessing.cpu_count() + self.cpu_info = CpuInfo(cores=cores, sockets=1) + except Exception: + self.cpu_info = CpuInfo() + + return self.cpu_info + def detect_gpu(self) -> List[GpuInfo]: """Detect GPU information using lspci and ROCm tools. diff --git a/build_tools/github_actions/test_framework/utils/system/system_detector.py b/build_tools/github_actions/test_framework/utils/system/system_detector.py index 43f04869120..419ec6faf88 100644 --- a/build_tools/github_actions/test_framework/utils/system/system_detector.py +++ b/build_tools/github_actions/test_framework/utils/system/system_detector.py @@ -179,7 +179,7 @@ def detect_all(self, verbose: bool = True) -> SystemContext: context = self.build_system_context() if verbose: - log.info("✓ System detection complete") + log.info("System detection complete") return context @@ -298,21 +298,21 @@ def log_system_info(self, context: SystemContext): Args: context: SystemContext with detected information """ - log.info(f"✓ Platform detected: {context.os_name} {context.os_version}") + log.info(f"Platform detected: {context.os_name} {context.os_version}") log.info(f" Kernel: {context.kernel}") log.info(f" Hostname: {context.hostname}") - log.info(f"✓ CPU detected: {context.cpu_model}") + log.info(f"CPU detected: {context.cpu_model}") log.info(f" Cores: {context.cpu_cores}, Sockets: {context.cpu_sockets}") if context.gpu_count > 0: - log.info(f"✓ GPU detected: {context.gpu_count} GPU(s)") + log.info(f"GPU detected: {context.gpu_count} GPU(s)") log.info( f" GPU 0: {context.gpu_name} (Device ID: {context.gpu_device_id})" ) else: - log.warning("⚠ No GPU detected") + log.warning("No GPU detected") - log.info(f"✓ ROCm detected: {context.rocm_version}") + log.info(f"ROCm detected: {context.rocm_version}") log.info(f" Install type: {context.rocm_install_type}") log.info(f" Build type: {context.rocm_build_type}") From 398c9b7a70d1df84b11d22064005cb1e8976ae26 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Wed, 7 Jan 2026 15:50:19 +0000 Subject: [PATCH 07/28] Fix pre-commit errors Signed-off-by: Lenine Ajagappane --- .../test_framework/benchmark/README.md | 12 ++++----- .../test_framework/functional/README.md | 27 +++++++++---------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/build_tools/github_actions/test_framework/benchmark/README.md b/build_tools/github_actions/test_framework/benchmark/README.md index d78733d298e..01e48b00a5b 100644 --- a/build_tools/github_actions/test_framework/benchmark/README.md +++ b/build_tools/github_actions/test_framework/benchmark/README.md @@ -57,12 +57,12 @@ benchmark/ Tests defined in `benchmark_matrix.py` for nightly CI: -| Test Name | Library | Platform | Timeout | Artifacts Needed | -| ----------------- | --------- | --------------- | ------- | ---------------------- | -| `hipblaslt_bench` | hipBLASLt | Linux, Windows | 60 min | `--blas --tests` | -| `rocfft_bench` | rocFFT | Linux, Windows | 60 min | `--fft --rand --tests` | -| `rocrand_bench` | rocRAND | Linux, Windows | 60 min | `--rand --tests` | -| `rocsolver_bench` | ROCsolver | Linux, Windows | 60 min | `--blas --tests` | +| Test Name | Library | Platform | Timeout | Artifacts Needed | +| ----------------- | --------- | -------------- | ------- | ---------------------- | +| `hipblaslt_bench` | hipBLASLt | Linux, Windows | 60 min | `--blas --tests` | +| `rocfft_bench` | rocFFT | Linux, Windows | 60 min | `--fft --rand --tests` | +| `rocrand_bench` | rocRAND | Linux, Windows | 60 min | `--rand --tests` | +| `rocsolver_bench` | ROCsolver | Linux, Windows | 60 min | `--blas --tests` | **GPU Family Support:** diff --git a/build_tools/github_actions/test_framework/functional/README.md b/build_tools/github_actions/test_framework/functional/README.md index f50ffeb3ee7..069ef3c8d08 100644 --- a/build_tools/github_actions/test_framework/functional/README.md +++ b/build_tools/github_actions/test_framework/functional/README.md @@ -14,8 +14,8 @@ Functional tests validate **correctness and behavior** by checking that outputs ### Available Functional Tests -| Test Script | Library | Description | -| -------------------------- | ------- | ---------------------------------------------- | +| Test Script | Library | Description | +| --------------------------- | ------- | ---------------------------------------------- | | `test_miopendriver_conv.py` | MIOpen | Convolution forward/backward correctness tests | ### Running Locally @@ -49,8 +49,8 @@ functional/ Tests defined in `functional_matrix.py` for CI: -| Test Name | Library | Platform | Timeout | Artifacts Needed | -| ------------------- | ------- | -------- | ------- | ---------------- | +| Test Name | Library | Platform | Timeout | Artifacts Needed | +| -------------------- | ------- | -------- | ------- | ------------------ | | `miopen_driver_conv` | MIOpen | Linux | 30 min | `--miopen --tests` | ## How Functional Tests Work @@ -90,6 +90,7 @@ Tests defined in `functional_matrix.py` for CI: Functional tests generate two tables: **Detailed Table:** One row per test case + ``` +--------------+--------------------+--------+ | TestSuite | TestCase | Status | @@ -100,6 +101,7 @@ Functional tests generate two tables: ``` **Summary Table:** Overall statistics + ``` +-------------------+-------------------+--------+--------+---------+---------+-------------------+ | Total TestSuites | Total TestCases | Passed | Failed | Errored | Skipped | Final Result | @@ -134,11 +136,10 @@ class YourTest(FunctionalBase): def __init__(self): super().__init__( - test_name="yourtest_functional", - display_name="YourTest Functional Test" + test_name="yourtest_functional", display_name="YourTest Functional Test" ) self.log_file = self.script_dir / "yourtest_functional.log" - + # Load test configuration from JSON config = self.load_config("yourtest_functional.json") self.test_cases = config.get("test_cases", []) @@ -146,7 +147,7 @@ class YourTest(FunctionalBase): def run_tests(self) -> None: """Run functional tests and save output to log file.""" log.info(f"Running {self.display_name}") - + with open(self.log_file, "w+") as f: for test_case in self.test_cases: cmd = test_case["command"] @@ -157,15 +158,15 @@ class YourTest(FunctionalBase): def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: """Parse log file and return (detailed_table, num_suites).""" log.info("Parsing Results") - + detailed_table = PrettyTable() detailed_table.field_names = ["TestCase", "Status"] - + test_results = [] - + # Parse log file for each test case # Use self.create_test_result() to create result dictionaries - + return test_results, detailed_table, num_suites @@ -173,7 +174,6 @@ if __name__ == "__main__": run_functional_test_main(YourTestFunctionalTest()) ``` - ### Step 2: Add to Functional Matrix Edit `functional_matrix.py`: @@ -216,4 +216,3 @@ See [main test framework README](../README.md) for full CI architecture. - [Test Framework README](../README.md) - Main framework documentation - [Shared Utils](../utils/README.md) - Utility modules reference - [Benchmark Tests](../benchmark/README.md) - Performance regression tests - From a56f9c41d7bebf0c82967a99eafd876ba02deaa4 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 13 Jan 2026 03:14:25 +0000 Subject: [PATCH 08/28] Refactor test framework READMEs to eliminate duplication - Consolidate common setup and architecture in main README - Move implementation details to benchmark/functional sub-READMEs Signed-off-by: Lenine Ajagappane --- .../github_actions/test_framework/README.md | 202 +++++------------- .../test_framework/benchmark/README.md | 130 +++-------- .../test_framework/functional/README.md | 99 ++------- .../scripts/test_miopendriver_conv.py | 2 +- 4 files changed, 103 insertions(+), 330 deletions(-) diff --git a/build_tools/github_actions/test_framework/README.md b/build_tools/github_actions/test_framework/README.md index 725374d0ed3..c00d9162c3c 100644 --- a/build_tools/github_actions/test_framework/README.md +++ b/build_tools/github_actions/test_framework/README.md @@ -5,20 +5,22 @@ Unified testing framework for TheRock ROCm distribution, supporting benchmark an ## Table of Contents - [Overview](#overview) -- [Test Types](#test-types) - [Quick Start](#quick-start) - [Project Structure](#project-structure) - [CI/CD Integration](#cicd-integration) - [Architecture](#architecture) -- [Adding Tests](#adding-tests) ## Overview -The test framework provides a unified infrastructure for multiple test types: +The test framework provides infrastructure for two test types: -### Features +| Test Type | Purpose | Result Types | When to Use | +| ----------------------------- | -------------------------------- | -------------------- | -------------------------------------------- | +| **[Benchmark](benchmark/)** | Performance regression detection | PASS/FAIL/UNKNOWN | Prevent performance degradation (nightly CI) | +| **[Functional](functional/)** | Correctness validation | PASS/FAIL/ERROR/SKIP | Verify expected behavior (nightly CI) | + +### Key Features -- **Multi-Type Testing** - Benchmark and functional tests - **Shared Infrastructure** - Common utilities, configuration, and results management - **System Auto-Detection** - Hardware, OS, GPU, and ROCm version detection - **Results Management** - Local storage (JSON) and API upload with retry logic @@ -27,61 +29,24 @@ The test framework provides a unified infrastructure for multiple test types: - **Modular Architecture** - Extensible design for adding new test types - **CI/CD Integration** - Parallel execution in nightly CI -## Test Types - -### 1. Benchmark Tests (`benchmark/`) - -**Purpose:** Detect performance regressions by comparing against Last Known Good (LKG) baselines. - -- **Result:** PASS/FAIL/UNKNOWN -- **Comparison:** Current vs baseline (with tolerance) -- **Frequency:** Every nightly CI -- **Use Case:** Automated CI gates to prevent regressions -- **Example:** "GEMM performance is 5% slower than baseline" → FAIL - -**See:** [benchmark/README.md](benchmark/README.md) - -### 2. Functional Tests (`functional/`) - -**Purpose:** Validate correctness, API contracts, and expected behavior. - -- **Result:** PASS/FAIL with detailed error messages -- **Comparison:** Output vs expected results -- **Frequency:** Every nightly CI -- **Use Case:** Correctness validation, API testing -- **Example:** "Matrix multiplication result matches expected output" → PASS - -**See:** [functional/README.md](functional/README.md) - ## Quick Start -### Running Tests - -```bash -# Set environment variables -export THEROCK_BIN_DIR=/path/to/therock/build/bin -export ARTIFACT_RUN_ID=local-test -export AMDGPU_FAMILIES=gfx950-dcgpu +### Environment Setup -# Run benchmark test -python test_framework/benchmark/scripts/test_hipblaslt_benchmark.py +All tests require these environment variables: -# Run functional test -python test_framework/functional/scripts/test_miopendriver_conv.py +```bash +export THEROCK_BIN_DIR=/path/to/therock/build/bin # Path to TheRock binaries +export ARTIFACT_RUN_ID=local-test # Unique identifier for this test run +export AMDGPU_FAMILIES=gfx950-dcgpu # Target GPU family ``` -### Available Tests - -**Benchmark Tests:** - -- `test_hipblaslt_benchmark.py` - hipBLASLt GEMM benchmarks -- `test_rocfft_benchmark.py` - rocFFT transform benchmarks -- `test_rocrand_benchmark.py` - rocRAND generation benchmarks -- `test_rocsolver_benchmark.py` - ROCsolver linear algebra benchmarks +### Running Tests -**Functional Tests:** +See test-specific READMEs for detailed instructions and examples: -- `test_miopendriver_conv.py` - MIOpen driver convolution functional tests +- **[Benchmark Tests](benchmark/README.md)** - Performance regression testing +- **[Functional Tests](functional/README.md)** - Correctness validation testing ## Project Structure @@ -110,7 +75,7 @@ test_framework/ │ ├── configs/ # Test-specific configurations │ │ └── miopen_driver_conv.json │ ├── functional_matrix.py # CI test matrix -│ └── README.md # (to be created) +│ └── README.md # Functional-specific docs │ └── utils/ # SHARED utilities for all test types ├── exceptions.py # Custom exception classes @@ -141,119 +106,50 @@ test_framework/ ### Test Execution Schedule -Benchmark tests run **only on nightly CI builds** to save time and resources on pull request validation: - -| Workflow Trigger | Benchmark/Functional Tests | Regular Tests | -| -------------------------- | ------------------------------ | ---------------------- | -| **Pull Request (PR)** | Skipped | Run (smoke: 1 shard) | -| **Nightly CI (scheduled)** | Run (in parallel, always full) | Run (full: all shards) | -| **Push to main** | Skipped | Run (smoke: 1 shard) | -| **Manual workflow** | Optional | Optional | - -**Note:** Benchmarks always run with `total_shards=1` and do not use `test_type` or `test_labels` filtering. +| Workflow Trigger | Benchmark Tests | Functional Tests | +| -------------------------- | --------------- | ---------------- | +| **Pull Request (PR)** | Skipped | Optional | +| **Nightly CI (scheduled)** | Run (parallel) | Run (parallel) | +| **Push to main** | Skipped | Optional | ### Parallel Execution Architecture -Benchmarks run **in parallel** with regular tests for faster CI execution: +Tests run in **parallel** for faster CI execution: ``` -ci_nightly.yml → ci_linux.yml - │ - ├─ build_artifacts (30 min) - │ - ├─ test_artifacts (45 min) ────┐ - │ └─ Regular tests │ Run in - │ (rocblas, hipblas, ...) │ PARALLEL - │ │ - └─ test_benchmarks (60 min) ────┘ - └─ Benchmark tests - (hipblaslt_bench, rocfft_bench, ...) - +ci_nightly.yml + └─ ci_linux.yml + ├─ build_artifacts + │ + ├─ test_artifacts ────────┐ Run in parallel + │ └─ Functional tests │ after build + │ │ + └─ test_benchmarks ────────┘ + └─ Benchmark tests ``` -### Workflow Integration Details +**Workflow Files:** -``` -.github/workflows/ci_nightly.yml - └─ calls → ci_linux.yml - ├─ job: build_artifacts - │ └─ Builds TheRock binaries - │ - ├─ job: test_artifacts (parallel) - │ └─ calls → test_artifacts.yml - │ └─ Functional tests matrix - │ - └─ job: test_benchmarks (parallel) - └─ calls → test_benchmarks.yml - ├─ configure_benchmark_matrix - │ └─ fetch_test_configurations.py - │ (IS_BENCHMARK_WORKFLOW=true) - └─ run_benchmarks - └─ test_component.yml (matrix) -``` +- `.github/workflows/ci_nightly.yml` - Nightly CI orchestration +- `.github/workflows/test_benchmarks.yml` - Benchmark execution +- `.github/workflows/test_artifacts.yml` - Functional test execution ## Architecture -### Execution Flow (Common Pattern) +### Common Test Execution Flow -``` -┌─────────────────────────────────────────┐ -│ 1. Initialize Test Runner │ -│ - Auto-detect system (GPU, ROCm) │ -│ - Load configuration │ -│ - Setup logging │ -└──────────────┬──────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ 2. Execute Tests │ -│ - Run test binaries/scripts │ -│ - Capture output to log files │ -│ - Handle errors gracefully │ -└──────────────┬──────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ 3. Parse Results │ -│ - Extract metrics from logs │ -│ - Structure data according to schema │ -│ - Validate results │ -└──────────────┬──────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ 4. Process Results (Type-Specific) │ -│ Benchmark: Compare with LKG baseline │ -│ Functional: Validate correctness │ -└──────────────┬──────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ 5. Report Results │ -│ - Display formatted output │ -│ - Upload to results API │ -│ - Append to GitHub Actions summary │ -│ - Return exit code │ -└─────────────────────────────────────────┘ -``` - -## Adding Tests - -### Adding a Benchmark Test - -See detailed guide in [benchmark/README.md](benchmark/README.md#adding-a-new-benchmark) +All tests follow this pattern: -### Adding a Functional Test +1. **Initialize** - Auto-detect system (GPU, ROCm), load configuration, setup logging +1. **Execute** - Run test binaries/scripts, capture output to log files +1. **Parse** - Extract metrics/results from logs, structure data +1. **Process** - Type-specific validation (LKG comparison or correctness check) +1. **Report** - Display results, upload to API, update GitHub Actions summary -See detailed guide in [functional/README.md](functional/README.md#adding-a-new-benchmark) +### Implementation Details -### Getting Help +See test-specific READMEs for detailed implementation guides: -- **Framework issues:** See this README -- **Benchmark-specific:** See [benchmark/README.md](benchmark/README.md) -- **Functional tests:** See [functional/README.md](functional/README.md) -- **Utilities:** See [utils/README.md](utils/README.md) - -## Related Files - -- `configure_ci.py` - CI workflow orchestration -- `fetch_test_configurations.py` - Test matrix builder -- `github_actions_utils.py` - GitHub Actions utilities -- `.github/workflows/test_benchmarks.yml` - Benchmark execution workflow -- `.github/workflows/ci_nightly.yml` - Nightly CI orchestration +- **[Benchmark Tests](benchmark/README.md)** - LKG comparison logic and adding new benchmarks +- **[Functional Tests](functional/README.md)** - Result validation and adding new tests +- **[Shared Utils](utils/README.md)** - Common utilities, exceptions, and helpers diff --git a/build_tools/github_actions/test_framework/benchmark/README.md b/build_tools/github_actions/test_framework/benchmark/README.md index 01e48b00a5b..5971ce2dc1a 100644 --- a/build_tools/github_actions/test_framework/benchmark/README.md +++ b/build_tools/github_actions/test_framework/benchmark/README.md @@ -1,18 +1,19 @@ # Benchmark Tests -Regression detection tests that compare current performance against Last Known Good (LKG) baselines. +Performance regression detection tests that compare current results against Last Known Good (LKG) baselines. -## Purpose +> **Prerequisites:** See [Test Framework Overview](../README.md) for environment setup and general architecture. -Benchmark tests detect **performance regressions** by comparing test results against established baselines: +## Overview -- **Result:** PASS (within tolerance) / FAIL (regression detected) / UNKNOWN (no baseline) -- **Frequency:** Every nightly CI -- **Use Case:** Automated CI gates to prevent performance degradation +Benchmark tests detect **performance regressions** by comparing against baselines: -## Quick Start +- **Result Types:** PASS (within tolerance) / FAIL (regression) / UNKNOWN (no baseline) +- **Comparison:** Current performance vs. LKG baseline with configurable tolerance +- **CI Execution:** Nightly only (not on PRs to save resources) +- **Exit Code:** Non-zero if any test FAILS -### Available Benchmark Tests +## Available Tests | Test Script | Library | Description | | ----------------------------- | --------- | --------------------------------------- | @@ -21,48 +22,24 @@ Benchmark tests detect **performance regressions** by comparing test results aga | `test_rocrand_benchmark.py` | rocRAND | Random number generation benchmarks | | `test_rocsolver_benchmark.py` | ROCsolver | Dense linear algebra benchmarks | -### Running Locally +## Quick Start ```bash -# Set required environment variables -export THEROCK_BIN_DIR=/path/to/therock/build/bin -export ARTIFACT_RUN_ID=local-test -export AMDGPU_FAMILIES=gfx950-dcgpu - -# Run a benchmark +# Run a benchmark (environment variables from main README required) cd build_tools/github_actions/test_framework/benchmark/scripts python test_hipblaslt_benchmark.py ``` -## Directory Structure - -``` -benchmark/ -├── scripts/ # Benchmark implementations -│ ├── benchmark_base.py # Base class (LKG comparison logic) -│ ├── test_hipblaslt_benchmark.py -│ ├── test_rocfft_benchmark.py -│ ├── test_rocrand_benchmark.py -│ └── test_rocsolver_benchmark.py -│ -├── configs/ # Benchmark-specific test configurations -│ ├── hipblaslt.json # Test parameters (matrix sizes, precisions) -│ └── rocfft.json # Test parameters (FFT sizes, dimensions) -│ -├── benchmark_matrix.py # CI test matrix definitions -└── README.md # This file -``` +## CI Test Matrix -## Benchmark Test Matrix +Tests defined in `benchmark_matrix.py`: -Tests defined in `benchmark_matrix.py` for nightly CI: - -| Test Name | Library | Platform | Timeout | Artifacts Needed | -| ----------------- | --------- | -------------- | ------- | ---------------------- | -| `hipblaslt_bench` | hipBLASLt | Linux, Windows | 60 min | `--blas --tests` | -| `rocfft_bench` | rocFFT | Linux, Windows | 60 min | `--fft --rand --tests` | -| `rocrand_bench` | rocRAND | Linux, Windows | 60 min | `--rand --tests` | -| `rocsolver_bench` | ROCsolver | Linux, Windows | 60 min | `--blas --tests` | +| Test Name | Library | Platform | Timeout | Artifacts Needed | CI Status | +| ----------------- | --------- | -------------- | ------- | ---------------------- | ----------------- | +| `hipblaslt_bench` | hipBLASLt | Linux, Windows | 60 min | `--blas --tests` | Enabled (nightly) | +| `rocfft_bench` | rocFFT | Linux, Windows | 60 min | `--fft --rand --tests` | Enabled (nightly) | +| `rocrand_bench` | rocRAND | Linux, Windows | 60 min | `--rand --tests` | Enabled (nightly) | +| `rocsolver_bench` | ROCsolver | Linux, Windows | 60 min | `--blas --tests` | Enabled (nightly) | **GPU Family Support:** @@ -82,43 +59,7 @@ Tests defined in `benchmark_matrix.py` for nightly CI: ## How Benchmark Tests Work -### 1. Execution Flow - -``` -┌─────────────────────────────────────────┐ -│ 1. Initialize Benchmark │ -│ - Auto-detect GPU, ROCm version │ -│ - Load configuration │ -└──────────────┬──────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ 2. Run Benchmark Binary │ -│ - Execute (e.g., rocblas-bench) │ -│ - Capture output to log file │ -└──────────────┬──────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ 3. Parse Results │ -│ - Extract metrics from logs │ -│ - Structure as test results │ -└──────────────┬──────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ 4. Compare with LKG Baseline │ -│ - Fetch last known good results │ -│ - Calculate performance delta │ -│ - Determine: PASS/FAIL/UNKNOWN │ -└──────────────┬──────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ 5. Report Results │ -│ - Display formatted table │ -│ - Upload to results API │ -│ - Return exit code (0=pass, 1=fail) │ -└─────────────────────────────────────────┘ -``` - -### 2. LKG Comparison +### LKG (Last Known Good) Comparison ```python # Pseudocode @@ -134,13 +75,13 @@ else: result = "PASS" # Performance acceptable ``` -### 3. Result Statuses +### Result Statuses -| Status | Meaning | Action | -| ----------- | ---------------------------------------- | ---------------------------------- | -| **PASS** | Performance within tolerance of baseline | ✅ CI passes | -| **FAIL** | Performance degraded beyond tolerance | ❌ CI fails, blocks merge | -| **UNKNOWN** | No baseline data available (new test) | ⚠️ CI passes, baseline established | +| Status | Meaning | Action | +| ----------- | ---------------------------------------- | ------------------------------- | +| **PASS** | Performance within tolerance of baseline | CI passes | +| **FAIL** | Performance degraded beyond tolerance | CI fails, blocks merge | +| **UNKNOWN** | No baseline data available (new test) | CI passes, baseline established | ## Adding a New Benchmark @@ -219,20 +160,13 @@ export AMDGPU_FAMILIES=gfx950-dcgpu python scripts/test_yourlib_benchmark.py ``` -## CI Integration - -Benchmark tests automatically run in nightly CI: - -1. **Trigger:** Nightly scheduled run -1. **Execution:** Parallel with regular tests -1. **Matrix:** Generated from `benchmark_matrix.py` -1. **Runners:** Dedicated GPU runners (if configured) -1. **Results:** Uploaded to results API, compared with LKG +## Configuration -See [main test framework README](../README.md) for full CI architecture. +- **Test Matrix:** `benchmark_matrix.py` - CI test definitions +- **Test Parameters:** `configs/*.json` - Benchmark-specific parameters (sizes, precisions, etc.) +- **Performance Tolerance:** Default 5% degradation threshold (configurable per test) -## Related Documentation +## See Also -- [Test Framework README](../README.md) - Main framework documentation -- [Shared Utils](../utils/README.md) - Utility modules reference +- [Test Framework Overview](../README.md) - Environment setup, CI/CD architecture - [Functional Tests](../functional/README.md) - Correctness validation tests diff --git a/build_tools/github_actions/test_framework/functional/README.md b/build_tools/github_actions/test_framework/functional/README.md index 069ef3c8d08..08f53e636a3 100644 --- a/build_tools/github_actions/test_framework/functional/README.md +++ b/build_tools/github_actions/test_framework/functional/README.md @@ -2,90 +2,41 @@ Correctness validation tests that verify expected behavior without performance measurements. -## Purpose +> **Prerequisites:** See [Test Framework Overview](../README.md) for environment setup and general architecture. -Functional tests validate **correctness and behavior** by checking that outputs match expectations: +## Overview -- **Result:** PASS (correct behavior) / FAIL (incorrect behavior) / ERROR (execution failure) / SKIP (prerequisites not met) -- **Frequency:** Every nightly CI -- **Use Case:** Automated correctness validation, API contract verification +Functional tests validate **correctness and behavior**: -## Quick Start +- **Result Types:** PASS / FAIL / ERROR / SKIP +- **Validation:** Output correctness, API contracts, expected behavior +- **CI Execution:** Nightly CI and optionally on PRs +- **Exit Code:** Non-zero if any test FAILS or has ERRORs -### Available Functional Tests +## Available Tests | Test Script | Library | Description | | --------------------------- | ------- | ---------------------------------------------- | | `test_miopendriver_conv.py` | MIOpen | Convolution forward/backward correctness tests | -### Running Locally +## Quick Start ```bash -# Set required environment variables -export THEROCK_BIN_DIR=/path/to/therock/build/bin -export ARTIFACT_RUN_ID=local-test -export AMDGPU_FAMILIES=gfx94x-dcgpu - -# Run a functional test +# Run a functional test (environment variables from main README required) python build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py ``` -## Directory Structure - -``` -functional/ -├── scripts/ # Functional test implementations -│ ├── functional_base.py # Base class (common test logic) -│ └── test_miopendriver_conv.py -│ -├── configs/ # Test-specific configurations -│ └── miopen_driver_conv.json # MIOpen test parameters -│ -├── functional_matrix.py # CI test matrix definitions -└── README.md # This file -``` +## CI Test Matrix -## Functional Test Matrix +Tests defined in `functional_matrix.py`: -Tests defined in `functional_matrix.py` for CI: - -| Test Name | Library | Platform | Timeout | Artifacts Needed | -| -------------------- | ------- | -------- | ------- | ------------------ | -| `miopen_driver_conv` | MIOpen | Linux | 30 min | `--miopen --tests` | +| Test Name | Library | Platform | Timeout | Artifacts Needed | CI Status | +| -------------------- | ------- | -------- | ------- | ------------------ | ----------------- | +| `miopen_driver_conv` | MIOpen | Linux | 30 min | `--miopen --tests` | Enabled (nightly) | ## How Functional Tests Work -### 1. Execution Flow - -``` -┌─────────────────────────────────────────┐ -│ 1. Initialize Test │ -│ - Auto-detect GPU, ROCm version │ -│ - Load configuration from JSON │ -└──────────────┬──────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ 2. Run Test Commands │ -│ - Execute test binary/driver │ -│ - Capture output to log file │ -└──────────────┬──────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ 3. Parse Results │ -│ - Extract pass/fail from logs │ -│ - Generate detailed per-case results │ -│ - Calculate statistics │ -└──────────────┬──────────────────────────┘ - ↓ -┌─────────────────────────────────────────┐ -│ 4. Report Results │ -│ - Display formatted tables │ -│ - Upload to results API │ -│ - Generate GitHub Actions summary │ -└─────────────────────────────────────────┘ -``` - -### 2. Result Tables +### Result Tables Functional tests generate two tables: @@ -199,20 +150,12 @@ export AMDGPU_FAMILIES=gfx94x-dcgpu python scripts/test_yourtest.py ``` -## CI Integration - -Functional tests run in CI workflows: - -1. **Trigger:** Nightly scheduled run -1. **Execution:** Parallel with other tests -1. **Matrix:** Generated from `functional_matrix.py` -1. **Runners:** Standard test runners -1. **Results:** Uploaded to results API, displayed in GHA summary +## Configuration -See [main test framework README](../README.md) for full CI architecture. +- **Test Matrix:** `functional_matrix.py` - CI test definitions +- **Test Parameters:** `configs/*.json` - Test-specific parameters and configurations -## Related Documentation +## See Also -- [Test Framework README](../README.md) - Main framework documentation -- [Shared Utils](../utils/README.md) - Utility modules reference +- [Test Framework Overview](../README.md) - Environment setup, CI/CD architecture - [Benchmark Tests](../benchmark/README.md) - Performance regression tests diff --git a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py index 7b551d7404e..76795bc4711 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py +++ b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py @@ -108,7 +108,7 @@ def run_tests(self) -> None: log.error(f"Error running command: {e}") f.write(f"ERROR: {e}\n\n") - log.info("Test execution complete") + log.info("MIOpenDriver convolution test execution complete") def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: """Parse test results from log file. From 48d75c72e28d29e996d8c1c0a2b0acb9fe31c02f Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 13 Jan 2026 03:22:03 +0000 Subject: [PATCH 09/28] Rename test matrix files for clarity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - benchmark_matrix.py → benchmark_test_matrix.py - functional_matrix.py → functional_test_matrix.py - Update all imports and documentation references Signed-off-by: Lenine Ajagappane --- build_tools/github_actions/fetch_test_configurations.py | 2 +- build_tools/github_actions/test_framework/README.md | 4 ++-- .../github_actions/test_framework/benchmark/README.md | 6 +++--- .../{benchmark_matrix.py => benchmark_test_matrix.py} | 0 .../github_actions/test_framework/functional/README.md | 6 +++--- .../{functional_matrix.py => functional_test_matrix.py} | 0 build_tools/github_actions/tests/configure_ci_test.py | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) rename build_tools/github_actions/test_framework/benchmark/{benchmark_matrix.py => benchmark_test_matrix.py} (100%) rename build_tools/github_actions/test_framework/functional/{functional_matrix.py => functional_test_matrix.py} (100%) diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index 7ff49b27595..06b16f09453 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -11,7 +11,7 @@ from pathlib import Path from github_actions_utils import * -from test_framework.benchmark.benchmark_matrix import benchmark_matrix +from test_framework.benchmark.benchmark_test_matrix import benchmark_matrix logging.basicConfig(level=logging.INFO) diff --git a/build_tools/github_actions/test_framework/README.md b/build_tools/github_actions/test_framework/README.md index c00d9162c3c..0afa2e38b79 100644 --- a/build_tools/github_actions/test_framework/README.md +++ b/build_tools/github_actions/test_framework/README.md @@ -65,7 +65,7 @@ test_framework/ │ ├── configs/ # Test-specific configurations │ │ ├── hipblaslt.json │ │ └── rocfft.json -│ ├── benchmark_matrix.py # CI test matrix +│ ├── benchmark_test_matrix.py # Benchmark test matrix │ └── README.md # Benchmark-specific docs │ ├── functional/ # Functional/correctness tests @@ -74,7 +74,7 @@ test_framework/ │ │ └── test_miopendriver_conv.py # MIOpen convolution tests │ ├── configs/ # Test-specific configurations │ │ └── miopen_driver_conv.json -│ ├── functional_matrix.py # CI test matrix +│ ├── functional_test_matrix.py # Functional test matrix │ └── README.md # Functional-specific docs │ └── utils/ # SHARED utilities for all test types diff --git a/build_tools/github_actions/test_framework/benchmark/README.md b/build_tools/github_actions/test_framework/benchmark/README.md index 5971ce2dc1a..3782281b691 100644 --- a/build_tools/github_actions/test_framework/benchmark/README.md +++ b/build_tools/github_actions/test_framework/benchmark/README.md @@ -32,7 +32,7 @@ python test_hipblaslt_benchmark.py ## CI Test Matrix -Tests defined in `benchmark_matrix.py`: +Tests defined in `benchmark_test_matrix.py`: | Test Name | Library | Platform | Timeout | Artifacts Needed | CI Status | | ----------------- | --------- | -------------- | ------- | ---------------------- | ----------------- | @@ -135,7 +135,7 @@ if __name__ == "__main__": ### Step 2: Add to Benchmark Matrix -Edit `benchmark_matrix.py`: +Edit `benchmark_test_matrix.py`: ```python "yourlib_bench": { @@ -162,7 +162,7 @@ python scripts/test_yourlib_benchmark.py ## Configuration -- **Test Matrix:** `benchmark_matrix.py` - CI test definitions +- **Test Matrix:** `benchmark_test_matrix.py` - CI test definitions - **Test Parameters:** `configs/*.json` - Benchmark-specific parameters (sizes, precisions, etc.) - **Performance Tolerance:** Default 5% degradation threshold (configurable per test) diff --git a/build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py b/build_tools/github_actions/test_framework/benchmark/benchmark_test_matrix.py similarity index 100% rename from build_tools/github_actions/test_framework/benchmark/benchmark_matrix.py rename to build_tools/github_actions/test_framework/benchmark/benchmark_test_matrix.py diff --git a/build_tools/github_actions/test_framework/functional/README.md b/build_tools/github_actions/test_framework/functional/README.md index 08f53e636a3..6017a371ee9 100644 --- a/build_tools/github_actions/test_framework/functional/README.md +++ b/build_tools/github_actions/test_framework/functional/README.md @@ -28,7 +28,7 @@ python build_tools/github_actions/test_framework/functional/scripts/test_miopend ## CI Test Matrix -Tests defined in `functional_matrix.py`: +Tests defined in `functional_test_matrix.py`: | Test Name | Library | Platform | Timeout | Artifacts Needed | CI Status | | -------------------- | ------- | -------- | ------- | ------------------ | ----------------- | @@ -127,7 +127,7 @@ if __name__ == "__main__": ### Step 2: Add to Functional Matrix -Edit `functional_matrix.py`: +Edit `functional_test_matrix.py`: ```python "yourtest_name": { @@ -152,7 +152,7 @@ python scripts/test_yourtest.py ## Configuration -- **Test Matrix:** `functional_matrix.py` - CI test definitions +- **Test Matrix:** `functional_test_matrix.py` - CI test definitions - **Test Parameters:** `configs/*.json` - Test-specific parameters and configurations ## See Also diff --git a/build_tools/github_actions/test_framework/functional/functional_matrix.py b/build_tools/github_actions/test_framework/functional/functional_test_matrix.py similarity index 100% rename from build_tools/github_actions/test_framework/functional/functional_matrix.py rename to build_tools/github_actions/test_framework/functional/functional_test_matrix.py diff --git a/build_tools/github_actions/tests/configure_ci_test.py b/build_tools/github_actions/tests/configure_ci_test.py index 34620199536..41ad7acf8da 100644 --- a/build_tools/github_actions/tests/configure_ci_test.py +++ b/build_tools/github_actions/tests/configure_ci_test.py @@ -6,7 +6,7 @@ sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) import configure_ci -from test_framework.benchmark.benchmark_matrix import benchmark_matrix +from test_framework.benchmark.benchmark_test_matrix import benchmark_matrix therock_test_runner_dict = { "gfx110x": { From 049af2e2b5a01578452fe28b8394c41cc3ad63b4 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 13 Jan 2026 03:31:17 +0000 Subject: [PATCH 10/28] Update functional_base.py - Move get_gpu_architecture() to hardware.py - Simplify display_name default Signed-off-by: Lenine Ajagappane --- .../functional/functional_test_matrix.py | 1 - .../functional/scripts/__init__.py | 1 - .../functional/scripts/functional_base.py | 16 +------ .../scripts/test_miopendriver_conv.py | 11 +++-- .../test_framework/utils/system/hardware.py | 47 +++++++++++++++++++ 5 files changed, 55 insertions(+), 21 deletions(-) delete mode 100644 build_tools/github_actions/test_framework/functional/scripts/__init__.py diff --git a/build_tools/github_actions/test_framework/functional/functional_test_matrix.py b/build_tools/github_actions/test_framework/functional/functional_test_matrix.py index ab99e007e1b..973e20b3532 100644 --- a/build_tools/github_actions/test_framework/functional/functional_test_matrix.py +++ b/build_tools/github_actions/test_framework/functional/functional_test_matrix.py @@ -25,7 +25,6 @@ def _get_script_path(script_name: str) -> str: "job_name": "test_miopendriver_conv", "fetch_artifact_args": "--miopen --tests", "timeout_minutes": 30, - "test_script": f"python {SCRIPT_DIR}/test_miopendriver_conv.py", "test_script": f"python {_get_script_path('test_rocrand_benchmark.py')}", # TODO(lajagapp): Add windows support "platform": ["linux"], diff --git a/build_tools/github_actions/test_framework/functional/scripts/__init__.py b/build_tools/github_actions/test_framework/functional/scripts/__init__.py deleted file mode 100644 index 96bd1ebf8e0..00000000000 --- a/build_tools/github_actions/test_framework/functional/scripts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Functional test scripts.""" diff --git a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py b/build_tools/github_actions/test_framework/functional/scripts/functional_base.py index 11b51e1ba04..a33bf2d1345 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py +++ b/build_tools/github_actions/test_framework/functional/scripts/functional_base.py @@ -35,7 +35,7 @@ def __init__(self, test_name: str, display_name: str = None): display_name: Display name for reports (e.g., 'MIOpen Driver Convolution') """ self.test_name = test_name - self.display_name = display_name or test_name.replace("_", " ").title() + self.display_name = display_name or test_name # Environment variables self.therock_bin_dir = os.getenv("THEROCK_BIN_DIR") @@ -233,20 +233,6 @@ def write_step_summary( f"```\n{summary_table}\n```\n" ) - def get_gpu_id(self) -> str: - """Detect GPU ID using rocminfo.""" - try: - result = subprocess.run( - ["rocminfo"], capture_output=True, text=True, check=True - ) - # Extract GPU name (e.g., gfx90a, gfx942) - match = re.search(r"Name:\s+(gfx\w+)", result.stdout) - if match: - return match.group(1) - except (subprocess.CalledProcessError, FileNotFoundError): - log.warning("Could not detect GPU ID, assuming default") - return "unknown" - def run(self) -> int: """Execute functional test workflow and return exit code (0=PASS, 1=FAIL). diff --git a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py index 76795bc4711..a72f0c70cd3 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py +++ b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py @@ -17,6 +17,7 @@ sys.path.insert(0, str(Path(__file__).parent)) # For functional_base from functional_base import FunctionalBase, run_functional_test_main from utils.logger import log +from utils.system.hardware import HardwareDetector from utils.exceptions import TestExecutionError @@ -49,8 +50,10 @@ def run_tests(self) -> None: """Run MIOpen driver convolution tests and save output to log file.""" log.info(f"Running {self.display_name} Tests") - gpu_id = self.get_gpu_id() - log.info(f"Detected GPU: {gpu_id}") + # Detect GPU architecture using HardwareDetector + detector = HardwareDetector() + gfx_id = detector.get_gpu_architecture() + log.info(f"Detected GPU: {gfx_id}") miopen_driver = f"{self.therock_bin_dir}/MIOpenDriver" if not Path(miopen_driver).exists(): @@ -73,9 +76,9 @@ def run_tests(self) -> None: # Add GPU-specific flags if needed if ( "Backward_Conv" in test_suite - and gpu_id in self.gpu_specific_flags + and gfx_id in self.gpu_specific_flags ): - backward_flags = self.gpu_specific_flags[gpu_id].get( + backward_flags = self.gpu_specific_flags[gfx_id].get( "backward_flags", "" ) if backward_flags: diff --git a/build_tools/github_actions/test_framework/utils/system/hardware.py b/build_tools/github_actions/test_framework/utils/system/hardware.py index 4d48367d94a..1a0dd6a9afa 100644 --- a/build_tools/github_actions/test_framework/utils/system/hardware.py +++ b/build_tools/github_actions/test_framework/utils/system/hardware.py @@ -79,6 +79,9 @@ class GpuInfo: partition_mode: str = "Unknown" xgmi_type: str = "Unknown" host_driver: str = "Unknown" + target_graphics_version: str = ( + "Unknown" # Target GPU architecture (e.g., gfx950, gfx942) + ) firmwares: List[Dict[str, str]] = field(default_factory=list) def __str__(self): @@ -522,6 +525,50 @@ def detect_gpu(self) -> List[GpuInfo]: return self.gpu_list + def get_gpu_architecture(self) -> str: + """Detect GPU architecture using rocminfo. + Returns: + str: GPU architecture ID (e.g., 'gfx908', 'gfx90a', 'gfx942') + Raises: + ValueError: If GPU architecture cannot be detected + """ + import logging + + logger = logging.getLogger(__name__) + logger.debug("Detecting GPU architecture...") + + therock_bin_dir = os.getenv("THEROCK_BIN_DIR") + if therock_bin_dir: + rocminfo_path = os.path.join(therock_bin_dir, "rocminfo") + else: + rocminfo_path = "rocminfo" + + try: + result = subprocess.run( + [rocminfo_path], + capture_output=True, + text=True, + check=True, + timeout=10, + ) + + for line in result.stdout.splitlines(): + if "Name:" in line and "gfx" in line: + # Extract gfx ID + for part in line.split(): + if part.startswith("gfx"): + gfx_id = part.strip() + logger.info(f"Detected GPU architecture: {gfx_id}") + return gfx_id + + except subprocess.CalledProcessError as e: + logger.error(f"rocminfo failed with return code {e.returncode}") + raise ValueError(f"Could not detect GPU architecture: {e}") + except FileNotFoundError: + raise ValueError(f"rocminfo not found at {rocminfo_path}") + + raise ValueError("Could not detect GPU architecture from rocminfo output") + def _enhance_gpu_with_rocm(self): """Enhance GPU info with ROCm tools (amd-smi or rocm-smi) for VRAM, clocks, and firmware.""" import logging From c87ecac97fcbe860ab6daf67df918450d81fea76 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 13 Jan 2026 15:13:57 +0000 Subject: [PATCH 11/28] Refactor test_miopendriver_conv.py with JSON result parsing Signed-off-by: Lenine Ajagappane --- .../scripts/test_miopendriver_conv.py | 177 ++++++++++-------- 1 file changed, 103 insertions(+), 74 deletions(-) diff --git a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py index a72f0c70cd3..016be87210d 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py +++ b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py @@ -5,7 +5,7 @@ correct functionality across different GPU architectures. """ -import re +import json import shlex import subprocess import sys @@ -30,6 +30,7 @@ def __init__(self): ) self.log_file = self.script_dir / "miopendriver_conv.log" + self.results_json = self.script_dir / "miopendriver_conv_results.json" # Load test configurations from JSON config = self.load_config("miopen_driver_conv.json") @@ -47,7 +48,7 @@ def __init__(self): self.gpu_specific_flags = config.get("gpu_specific_flags", {}) def run_tests(self) -> None: - """Run MIOpen driver convolution tests and save output to log file.""" + """Run MIOpen driver convolution tests and save output to log file and results to JSON.""" log.info(f"Running {self.display_name} Tests") # Detect GPU architecture using HardwareDetector @@ -55,13 +56,22 @@ def run_tests(self) -> None: gfx_id = detector.get_gpu_architecture() log.info(f"Detected GPU: {gfx_id}") - miopen_driver = f"{self.therock_bin_dir}/MIOpenDriver" - if not Path(miopen_driver).exists(): + # Use Path object consistently + miopen_driver = Path(self.therock_bin_dir) / "MIOpenDriver" + if not miopen_driver.exists(): raise TestExecutionError( f"MIOpenDriver not found at {miopen_driver}", action="Ensure MIOpen is installed correctly", ) + # Calculate total number of tests for progress indicator + total_tests = sum(len(self.tests_cmd[suite]) for suite in self.tests_list) + current_test = 0 + log.info(f"Total tests to run: {total_tests}") + + # Store results as we execute + all_results = [] + with open(self.log_file, "w+") as f: for test_suite in self.tests_list: log.info(f"Running test suite: {test_suite}") @@ -69,7 +79,9 @@ def run_tests(self) -> None: f.write(f"Test Suite: {test_suite}\n") f.write(f"{'='*80}\n\n") - for cmd_str in self.tests_cmd[test_suite]: + for i, cmd_str in enumerate(self.tests_cmd[test_suite], 1): + current_test += 1 + # Build full command with MIOpenDriver path full_cmd = f"{miopen_driver} {cmd_str}" @@ -86,10 +98,16 @@ def run_tests(self) -> None: cmd = shlex.split(full_cmd) + # Progress indicator + test_case_name = f"{test_suite}_case{i}" + log.info(f"[{current_test}/{total_tests}] Running {test_case_name}") log.info(f"++ Exec [{self.therock_dir}]$ {shlex.join(cmd)}") f.write(f"\nCommand: {shlex.join(cmd)}\n") f.write(f"-" * 80 + "\n") + return_code = None + error_message = None + try: process = subprocess.Popen( cmd, @@ -105,16 +123,44 @@ def run_tests(self) -> None: f.write(line) process.wait() - f.write(f"\nReturn code: {process.returncode}\n\n") + return_code = process.returncode + f.write(f"\nReturn code: {return_code}\n\n") except Exception as e: log.error(f"Error running command: {e}") f.write(f"ERROR: {e}\n\n") + error_message = str(e) + return_code = -1 + + # Determine status based on return code + status = "PASS" if return_code == 0 else "FAIL" + + # Store result immediately + result = { + "test_suite": test_suite, + "test_case": test_case_name, + "command": cmd_str, + "command_index": i, + "return_code": return_code, + "status": status, + } + if error_message: + result["error"] = error_message + + all_results.append(result) + log.info( + f"[{current_test}/{total_tests}] {test_case_name}: {status}" + ) + # Write all results to JSON file + with open(self.results_json, "w") as f: + json.dump(all_results, f, indent=2) + + log.info(f"Results saved to {self.results_json}") log.info("MIOpenDriver convolution test execution complete") def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: - """Parse test results from log file. + """Parse test results from JSON file. Returns: tuple: (test_results list, detailed PrettyTable, number of test suites) @@ -128,89 +174,72 @@ def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: test_results = [] try: - with open(self.log_file, "r") as f: - content = f.read() - - # Parse results for each test suite - for test_suite in self.tests_list: - # Find the section for this test suite - pattern = re.compile( - rf"Test Suite: {test_suite}.*?(?=Test Suite:|$)", re.DOTALL + # Read results from JSON file + with open(self.results_json, "r") as f: + json_results = json.load(f) + + if not isinstance(json_results, list): + raise TestExecutionError( + "Results JSON is not a list", + action="Check results file format", ) - match = pattern.search(content) - if not match: - log.warning(f"Could not find results for {test_suite}") + # Process each result with safe key access + for idx, result in enumerate(json_results): + if not isinstance(result, dict): + log.warning(f"Result {idx} is not a dictionary, skipping") continue - suite_content = match.group(0) - - # Parse each command in the suite - for i, cmd_str in enumerate(self.tests_cmd[test_suite], 1): - # Build the command pattern to search for - miopen_driver = f"{self.therock_bin_dir}/MIOpenDriver" - full_cmd = f"{miopen_driver} {cmd_str}" - test_case_name = f"{test_suite}_case{i}" - - # Find this specific command in the suite content - cmd_pattern = re.escape(f"Command: {full_cmd}") - cmd_match = re.search(cmd_pattern, suite_content) + # Use .get() with defaults to handle missing keys gracefully + test_suite = result.get("test_suite", "unknown_suite") + test_case = result.get("test_case", f"unknown_case_{idx}") + status = result.get("status", "FAIL") # Default to FAIL if unknown + command = result.get("command", "") + command_index = result.get("command_index", idx + 1) + + # Log warning if critical keys are missing + if "test_suite" not in result or "test_case" not in result: + log.warning( + f"Result {idx} missing critical keys (test_suite/test_case)" + ) - if cmd_match: - # Extract content after this command until next command or end - start_pos = cmd_match.end() - next_cmd_match = re.search( - r"\nCommand:", suite_content[start_pos:] - ) - if next_cmd_match: - cmd_section = suite_content[ - start_pos : start_pos + next_cmd_match.start() - ] - else: - cmd_section = suite_content[start_pos:] - - # Check return code in THIS command's section only - return_code_match = re.search( - r"Return code:\s*(\d+)", cmd_section - ) - if return_code_match: - return_code = int(return_code_match.group(1)) - status = "PASS" if return_code == 0 else "FAIL" - elif "PASSED" in cmd_section: - status = "PASS" - elif "FAILED" in cmd_section or "ERROR" in cmd_section: - status = "FAIL" - else: - status = "FAIL" # Unknown result, assume fail - else: - status = "FAIL" # Command not found in log - - # Add each individual test case to detailed table - detailed_table.add_row([test_suite, test_case_name, status]) - - # Add each test case to results list using helper - test_results.append( - self.create_test_result( - test_name=self.test_name, - subtest_name=test_case_name, - status=status, - suite=test_suite, - command_index=i, - command=cmd_str, - ) + # Add to detailed table + detailed_table.add_row([test_suite, test_case, status]) + + # Add to results list using helper + test_results.append( + self.create_test_result( + test_name=self.test_name, + subtest_name=test_case, + status=status, + suite=test_suite, + command_index=command_index, + command=command, ) + ) except FileNotFoundError: raise TestExecutionError( - f"Log file not found: {self.log_file}", + f"Results JSON file not found: {self.results_json}", action="Ensure tests were executed successfully", ) + except json.JSONDecodeError as e: + raise TestExecutionError( + f"Error parsing results JSON: {e}", + action="Check if results file is valid JSON", + ) except OSError as e: raise TestExecutionError( - f"Error reading log file: {e}", + f"Error reading results file: {e}", action="Check file permissions and disk space", ) + if not test_results: + raise TestExecutionError( + "No valid test results found in JSON file", + action="Check if tests executed successfully and results were saved", + ) + num_suites = len(self.tests_list) return test_results, detailed_table, num_suites From a9d7c26a2c37bd62785d863e047a23adf949eb08 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 13 Jan 2026 17:06:05 +0000 Subject: [PATCH 12/28] Simplify exception handling and enhance functional tests - Add TestResultError for result failures vs execution errors - Remove sys.exit() - let exceptions propagate naturally - Add progress indicators and better error messages - Consistent exception pattern across all test files Signed-off-by: Lenine Ajagappane --- .../functional/scripts/functional_base.py | 63 ++++++++++--------- .../scripts/test_miopendriver_conv.py | 23 ++++--- .../test_framework/utils/exceptions.py | 6 ++ 3 files changed, 50 insertions(+), 42 deletions(-) diff --git a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py b/build_tools/github_actions/test_framework/functional/scripts/functional_base.py index a33bf2d1345..b62fa4b1c4d 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py +++ b/build_tools/github_actions/test_framework/functional/scripts/functional_base.py @@ -14,7 +14,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) # github_actions/ from utils import TestClient from utils.logger import log -from utils.exceptions import TestExecutionError +from utils.exceptions import TestExecutionError, TestResultError from github_actions_utils import gha_append_step_summary @@ -66,13 +66,13 @@ def load_config(self, config_filename: str) -> Dict[str, Any]: return json.load(f) except FileNotFoundError: raise TestExecutionError( - f"Configuration file not found: {config_file}", - action=f"Ensure {config_filename} exists in configs/ directory", + f"Configuration file not found: {config_file}\n" + f"Ensure {config_filename} exists in configs/ directory" ) except json.JSONDecodeError as e: raise TestExecutionError( - f"Invalid JSON in configuration file: {e}", - action=f"Check JSON syntax in {config_filename}", + f"Invalid JSON in configuration file: {e}\n" + f"Check JSON syntax in {config_filename}" ) def create_test_result( @@ -233,11 +233,16 @@ def write_step_summary( f"```\n{summary_table}\n```\n" ) - def run(self) -> int: - """Execute functional test workflow and return exit code (0=PASS, 1=FAIL). + def run(self) -> None: + """Execute functional test workflow. - Returns: - 0 if all tests passed, 1 if any test failed + Raises: + TestExecutionError: If test execution encounters errors (missing files, etc.) + TestResultError: If tests run successfully but results show failures + + Note: + On success, returns normally (exit code 0) + On failure, raises exception (exit code 1) """ log.info(f"{self.display_name} - Starting Functional Test") @@ -251,10 +256,11 @@ def run(self) -> int: # Parse results (implemented by child class) test_results, detailed_table, num_suites = self.parse_results() + # Validate test results structure if not test_results: raise TestExecutionError( - "No test results generated", - action="Check if tests executed properly and log file contains output", + "No test results generated - parse_results() returned empty list\n" + "Check if tests executed successfully and results were saved to file" ) # Calculate statistics @@ -283,27 +289,24 @@ def run(self) -> int: except Exception as e: log.warning(f"Could not write GitHub Actions summary: {e}") - # Return 0 only if PASS, otherwise return 1 - return 0 if stats["overall_status"] == "PASS" else 1 + # Raise exception if tests failed + if stats["overall_status"] != "PASS": + failed = stats["failed"] + errored = stats["error"] + total = stats["total"] + raise TestResultError( + f"Test suite completed with failures: " + f"{failed} failed, {errored} errors out of {total} total tests" + ) def run_functional_test_main(test_instance): - """Run functional test with standard error handling. + """Run functional test. - Raises: - KeyboardInterrupt: If execution is interrupted by user - Exception: If test execution fails + Raises exceptions on failure, returns normally on success. + This is the Pythonic way - let Python set exit codes automatically: + - Success: Returns normally → exit code 0 + - Execution Error: Raises TestExecutionError → exit code 1 + - Result Failure: Raises TestResultError → exit code 1 """ - try: - exit_code = test_instance.run() - if exit_code != 0: - raise RuntimeError(f"Test failed with exit code {exit_code}") - except KeyboardInterrupt: - log.warning("\nExecution interrupted by user") - raise - except Exception as e: - log.error(f"Unexpected error: {e}") - import traceback - - traceback.print_exc() - raise + test_instance.run() diff --git a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py index 016be87210d..941aec5b1f1 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py +++ b/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py @@ -60,8 +60,8 @@ def run_tests(self) -> None: miopen_driver = Path(self.therock_bin_dir) / "MIOpenDriver" if not miopen_driver.exists(): raise TestExecutionError( - f"MIOpenDriver not found at {miopen_driver}", - action="Ensure MIOpen is installed correctly", + f"MIOpenDriver not found at {miopen_driver}\n" + f"Ensure MIOpen is installed correctly" ) # Calculate total number of tests for progress indicator @@ -180,8 +180,7 @@ def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: if not isinstance(json_results, list): raise TestExecutionError( - "Results JSON is not a list", - action="Check results file format", + "Results JSON is not a list\n" "Check results file format" ) # Process each result with safe key access @@ -220,24 +219,24 @@ def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: except FileNotFoundError: raise TestExecutionError( - f"Results JSON file not found: {self.results_json}", - action="Ensure tests were executed successfully", + f"Results JSON file not found: {self.results_json}\n" + f"Ensure tests were executed successfully" ) except json.JSONDecodeError as e: raise TestExecutionError( - f"Error parsing results JSON: {e}", - action="Check if results file is valid JSON", + f"Error parsing results JSON: {e}\n" + f"Check if results file is valid JSON" ) except OSError as e: raise TestExecutionError( - f"Error reading results file: {e}", - action="Check file permissions and disk space", + f"Error reading results file: {e}\n" + f"Check file permissions and disk space" ) if not test_results: raise TestExecutionError( - "No valid test results found in JSON file", - action="Check if tests executed successfully and results were saved", + "No valid test results found in JSON file\n" + "Check if tests executed successfully and results were saved" ) num_suites = len(self.tests_list) diff --git a/build_tools/github_actions/test_framework/utils/exceptions.py b/build_tools/github_actions/test_framework/utils/exceptions.py index b54c811d755..827faa2d85f 100644 --- a/build_tools/github_actions/test_framework/utils/exceptions.py +++ b/build_tools/github_actions/test_framework/utils/exceptions.py @@ -39,6 +39,12 @@ class TestExecutionError(FrameworkException): pass +class TestResultError(FrameworkException): + """Test result failures (tests ran successfully but results show failures).""" + + pass + + class ValidationError(FrameworkException): """Data or input validation failures.""" From 22d338e367d62dc1d4dafe144800481330738e73 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 13 Jan 2026 17:35:54 +0000 Subject: [PATCH 13/28] Rename test_framework to extended_tests Renamed top-level folder from test_framework to extended_tests for better clarity. Signed-off-by: Lenine Ajagappane --- .../{test_framework => extended_tests}/README.md | 4 ++-- .../{test_framework => extended_tests}/__init__.py | 0 .../benchmark/README.md | 6 +++--- .../benchmark/benchmark_test_matrix.py | 2 +- .../benchmark/configs/hipblaslt.json | 0 .../benchmark/configs/rocfft.json | 0 .../benchmark/scripts/benchmark_base.py | 2 +- .../benchmark/scripts/test_hipblaslt_benchmark.py | 0 .../benchmark/scripts/test_rocfft_benchmark.py | 0 .../benchmark/scripts/test_rocrand_benchmark.py | 0 .../benchmark/scripts/test_rocsolver_benchmark.py | 0 .../{test_framework => extended_tests}/configs/config.yml | 0 .../functional/README.md | 8 ++++---- .../functional/configs/miopen_driver_conv.json | 0 .../functional/functional_test_matrix.py | 2 +- .../functional/scripts/functional_base.py | 2 +- .../functional/scripts/test_miopendriver_conv.py | 0 .../{test_framework => extended_tests}/utils/README.md | 8 ++++---- .../{test_framework => extended_tests}/utils/__init__.py | 0 .../utils/config/__init__.py | 0 .../utils/config/config_helper.py | 0 .../utils/config/config_parser.py | 0 .../utils/config/config_validator.py | 0 .../{test_framework => extended_tests}/utils/constants.py | 0 .../utils/exceptions.py | 0 .../{test_framework => extended_tests}/utils/logger.py | 0 .../utils/results/__init__.py | 0 .../utils/results/results_api.py | 0 .../utils/results/results_handler.py | 0 .../utils/results/schemas/payload_schema.json | 0 .../utils/system/__init__.py | 0 .../utils/system/hardware.py | 0 .../utils/system/platform.py | 0 .../utils/system/rocm_detector.py | 0 .../utils/system/system_detector.py | 0 .../utils/test_client.py | 0 build_tools/github_actions/fetch_test_configurations.py | 2 +- build_tools/github_actions/tests/configure_ci_test.py | 2 +- 38 files changed, 19 insertions(+), 19 deletions(-) rename build_tools/github_actions/{test_framework => extended_tests}/README.md (99%) rename build_tools/github_actions/{test_framework => extended_tests}/__init__.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/benchmark/README.md (96%) rename build_tools/github_actions/{test_framework => extended_tests}/benchmark/benchmark_test_matrix.py (97%) rename build_tools/github_actions/{test_framework => extended_tests}/benchmark/configs/hipblaslt.json (100%) rename build_tools/github_actions/{test_framework => extended_tests}/benchmark/configs/rocfft.json (100%) rename build_tools/github_actions/{test_framework => extended_tests}/benchmark/scripts/benchmark_base.py (99%) rename build_tools/github_actions/{test_framework => extended_tests}/benchmark/scripts/test_hipblaslt_benchmark.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/benchmark/scripts/test_rocfft_benchmark.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/benchmark/scripts/test_rocrand_benchmark.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/benchmark/scripts/test_rocsolver_benchmark.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/configs/config.yml (100%) rename build_tools/github_actions/{test_framework => extended_tests}/functional/README.md (94%) rename build_tools/github_actions/{test_framework => extended_tests}/functional/configs/miopen_driver_conv.json (100%) rename build_tools/github_actions/{test_framework => extended_tests}/functional/functional_test_matrix.py (93%) rename build_tools/github_actions/{test_framework => extended_tests}/functional/scripts/functional_base.py (99%) rename build_tools/github_actions/{test_framework => extended_tests}/functional/scripts/test_miopendriver_conv.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/README.md (95%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/__init__.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/config/__init__.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/config/config_helper.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/config/config_parser.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/config/config_validator.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/constants.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/exceptions.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/logger.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/results/__init__.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/results/results_api.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/results/results_handler.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/results/schemas/payload_schema.json (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/system/__init__.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/system/hardware.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/system/platform.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/system/rocm_detector.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/system/system_detector.py (100%) rename build_tools/github_actions/{test_framework => extended_tests}/utils/test_client.py (100%) diff --git a/build_tools/github_actions/test_framework/README.md b/build_tools/github_actions/extended_tests/README.md similarity index 99% rename from build_tools/github_actions/test_framework/README.md rename to build_tools/github_actions/extended_tests/README.md index 0afa2e38b79..619a4cd7e5f 100644 --- a/build_tools/github_actions/test_framework/README.md +++ b/build_tools/github_actions/extended_tests/README.md @@ -1,4 +1,4 @@ -# TheRock Test Framework +# TheRock Extended Tests Framework Unified testing framework for TheRock ROCm distribution, supporting benchmark and functional testing with automated execution, system detection, and results management. @@ -51,7 +51,7 @@ See test-specific READMEs for detailed instructions and examples: ## Project Structure ``` -test_framework/ +extended_tests/ ├── __init__.py ├── README.md # This file │ diff --git a/build_tools/github_actions/test_framework/__init__.py b/build_tools/github_actions/extended_tests/__init__.py similarity index 100% rename from build_tools/github_actions/test_framework/__init__.py rename to build_tools/github_actions/extended_tests/__init__.py diff --git a/build_tools/github_actions/test_framework/benchmark/README.md b/build_tools/github_actions/extended_tests/benchmark/README.md similarity index 96% rename from build_tools/github_actions/test_framework/benchmark/README.md rename to build_tools/github_actions/extended_tests/benchmark/README.md index 3782281b691..e8562ecc5a1 100644 --- a/build_tools/github_actions/test_framework/benchmark/README.md +++ b/build_tools/github_actions/extended_tests/benchmark/README.md @@ -2,7 +2,7 @@ Performance regression detection tests that compare current results against Last Known Good (LKG) baselines. -> **Prerequisites:** See [Test Framework Overview](../README.md) for environment setup and general architecture. +> **Prerequisites:** See [Extended Tests Framework Overview](../README.md) for environment setup and general architecture. ## Overview @@ -26,7 +26,7 @@ Benchmark tests detect **performance regressions** by comparing against baseline ```bash # Run a benchmark (environment variables from main README required) -cd build_tools/github_actions/test_framework/benchmark/scripts +cd build_tools/github_actions/extended_tests/benchmark/scripts python test_hipblaslt_benchmark.py ``` @@ -168,5 +168,5 @@ python scripts/test_yourlib_benchmark.py ## See Also -- [Test Framework Overview](../README.md) - Environment setup, CI/CD architecture +- [Extended Tests Framework Overview](../README.md) - Environment setup, CI/CD architecture - [Functional Tests](../functional/README.md) - Correctness validation tests diff --git a/build_tools/github_actions/test_framework/benchmark/benchmark_test_matrix.py b/build_tools/github_actions/extended_tests/benchmark/benchmark_test_matrix.py similarity index 97% rename from build_tools/github_actions/test_framework/benchmark/benchmark_test_matrix.py rename to build_tools/github_actions/extended_tests/benchmark/benchmark_test_matrix.py index a912f477fac..18b225ba631 100644 --- a/build_tools/github_actions/test_framework/benchmark/benchmark_test_matrix.py +++ b/build_tools/github_actions/extended_tests/benchmark/benchmark_test_matrix.py @@ -9,7 +9,7 @@ # Note: these paths are relative to the repository root. SCRIPT_DIR = ( - Path("build_tools") / "github_actions" / "test_framework" / "benchmark" / "scripts" + Path("build_tools") / "github_actions" / "extended_tests" / "benchmark" / "scripts" ) diff --git a/build_tools/github_actions/test_framework/benchmark/configs/hipblaslt.json b/build_tools/github_actions/extended_tests/benchmark/configs/hipblaslt.json similarity index 100% rename from build_tools/github_actions/test_framework/benchmark/configs/hipblaslt.json rename to build_tools/github_actions/extended_tests/benchmark/configs/hipblaslt.json diff --git a/build_tools/github_actions/test_framework/benchmark/configs/rocfft.json b/build_tools/github_actions/extended_tests/benchmark/configs/rocfft.json similarity index 100% rename from build_tools/github_actions/test_framework/benchmark/configs/rocfft.json rename to build_tools/github_actions/extended_tests/benchmark/configs/rocfft.json diff --git a/build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py b/build_tools/github_actions/extended_tests/benchmark/scripts/benchmark_base.py similarity index 99% rename from build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py rename to build_tools/github_actions/extended_tests/benchmark/scripts/benchmark_base.py index fde9d3b10e6..859c8396cd1 100644 --- a/build_tools/github_actions/test_framework/benchmark/scripts/benchmark_base.py +++ b/build_tools/github_actions/extended_tests/benchmark/scripts/benchmark_base.py @@ -7,7 +7,7 @@ from prettytable import PrettyTable # Add parent directory to path for utils import -sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # test_framework/ +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # extended_tests/ sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) # github_actions/ from utils import TestClient from utils.logger import log diff --git a/build_tools/github_actions/test_framework/benchmark/scripts/test_hipblaslt_benchmark.py b/build_tools/github_actions/extended_tests/benchmark/scripts/test_hipblaslt_benchmark.py similarity index 100% rename from build_tools/github_actions/test_framework/benchmark/scripts/test_hipblaslt_benchmark.py rename to build_tools/github_actions/extended_tests/benchmark/scripts/test_hipblaslt_benchmark.py diff --git a/build_tools/github_actions/test_framework/benchmark/scripts/test_rocfft_benchmark.py b/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocfft_benchmark.py similarity index 100% rename from build_tools/github_actions/test_framework/benchmark/scripts/test_rocfft_benchmark.py rename to build_tools/github_actions/extended_tests/benchmark/scripts/test_rocfft_benchmark.py diff --git a/build_tools/github_actions/test_framework/benchmark/scripts/test_rocrand_benchmark.py b/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocrand_benchmark.py similarity index 100% rename from build_tools/github_actions/test_framework/benchmark/scripts/test_rocrand_benchmark.py rename to build_tools/github_actions/extended_tests/benchmark/scripts/test_rocrand_benchmark.py diff --git a/build_tools/github_actions/test_framework/benchmark/scripts/test_rocsolver_benchmark.py b/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocsolver_benchmark.py similarity index 100% rename from build_tools/github_actions/test_framework/benchmark/scripts/test_rocsolver_benchmark.py rename to build_tools/github_actions/extended_tests/benchmark/scripts/test_rocsolver_benchmark.py diff --git a/build_tools/github_actions/test_framework/configs/config.yml b/build_tools/github_actions/extended_tests/configs/config.yml similarity index 100% rename from build_tools/github_actions/test_framework/configs/config.yml rename to build_tools/github_actions/extended_tests/configs/config.yml diff --git a/build_tools/github_actions/test_framework/functional/README.md b/build_tools/github_actions/extended_tests/functional/README.md similarity index 94% rename from build_tools/github_actions/test_framework/functional/README.md rename to build_tools/github_actions/extended_tests/functional/README.md index 6017a371ee9..9b29583f1d6 100644 --- a/build_tools/github_actions/test_framework/functional/README.md +++ b/build_tools/github_actions/extended_tests/functional/README.md @@ -2,7 +2,7 @@ Correctness validation tests that verify expected behavior without performance measurements. -> **Prerequisites:** See [Test Framework Overview](../README.md) for environment setup and general architecture. +> **Prerequisites:** See [Extended Tests Framework Overview](../README.md) for environment setup and general architecture. ## Overview @@ -23,7 +23,7 @@ Functional tests validate **correctness and behavior**: ```bash # Run a functional test (environment variables from main README required) -python build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py +python build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py ``` ## CI Test Matrix @@ -75,7 +75,7 @@ from pathlib import Path from typing import Dict, List, Tuple, Any from prettytable import PrettyTable -sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # test_framework/ +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # extended_tests/ sys.path.insert(0, str(Path(__file__).parent)) # For functional_base from functional_base import FunctionalBase, run_functional_test_main from utils.logger import log @@ -157,5 +157,5 @@ python scripts/test_yourtest.py ## See Also -- [Test Framework Overview](../README.md) - Environment setup, CI/CD architecture +- [Extended Tests Framework Overview](../README.md) - Environment setup, CI/CD architecture - [Benchmark Tests](../benchmark/README.md) - Performance regression tests diff --git a/build_tools/github_actions/test_framework/functional/configs/miopen_driver_conv.json b/build_tools/github_actions/extended_tests/functional/configs/miopen_driver_conv.json similarity index 100% rename from build_tools/github_actions/test_framework/functional/configs/miopen_driver_conv.json rename to build_tools/github_actions/extended_tests/functional/configs/miopen_driver_conv.json diff --git a/build_tools/github_actions/test_framework/functional/functional_test_matrix.py b/build_tools/github_actions/extended_tests/functional/functional_test_matrix.py similarity index 93% rename from build_tools/github_actions/test_framework/functional/functional_test_matrix.py rename to build_tools/github_actions/extended_tests/functional/functional_test_matrix.py index 973e20b3532..ddaabb21ecb 100644 --- a/build_tools/github_actions/test_framework/functional/functional_test_matrix.py +++ b/build_tools/github_actions/extended_tests/functional/functional_test_matrix.py @@ -8,7 +8,7 @@ # Note: these paths are relative to the repository root. SCRIPT_DIR = ( - Path("build_tools") / "github_actions" / "test_framework" / "functional" / "scripts" + Path("build_tools") / "github_actions" / "extended_tests" / "functional" / "scripts" ) diff --git a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py b/build_tools/github_actions/extended_tests/functional/scripts/functional_base.py similarity index 99% rename from build_tools/github_actions/test_framework/functional/scripts/functional_base.py rename to build_tools/github_actions/extended_tests/functional/scripts/functional_base.py index b62fa4b1c4d..017a4c55a68 100644 --- a/build_tools/github_actions/test_framework/functional/scripts/functional_base.py +++ b/build_tools/github_actions/extended_tests/functional/scripts/functional_base.py @@ -10,7 +10,7 @@ from prettytable import PrettyTable # Add parent directory to path for utils import -sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # test_framework/ +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # extended_tests/ sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent)) # github_actions/ from utils import TestClient from utils.logger import log diff --git a/build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py b/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py similarity index 100% rename from build_tools/github_actions/test_framework/functional/scripts/test_miopendriver_conv.py rename to build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py diff --git a/build_tools/github_actions/test_framework/utils/README.md b/build_tools/github_actions/extended_tests/utils/README.md similarity index 95% rename from build_tools/github_actions/test_framework/utils/README.md rename to build_tools/github_actions/extended_tests/utils/README.md index 55e5bd148da..83d3cf04c30 100644 --- a/build_tools/github_actions/test_framework/utils/README.md +++ b/build_tools/github_actions/extended_tests/utils/README.md @@ -5,7 +5,7 @@ Utility modules organized into logical subdirectories for maintainability and sc ## Structure ``` -test_framework/utils/ +extended_tests/utils/ ├── __init__.py # Public exports ├── test_client.py # Main TestClient API ├── constants.py # Framework constants @@ -38,13 +38,13 @@ test_framework/utils/ ### From Test Scripts -Test scripts (benchmark/functional) add `test_framework/` to `sys.path`, then import: +Test scripts (benchmark/functional) add `extended_tests/` to `sys.path`, then import: ```python # Import path setup (already done in base classes) sys.path.insert( 0, str(Path(__file__).parent.parent.parent) -) # Adds test_framework/ to path +) # Adds extended_tests/ to path # Core utilities from utils.logger import log @@ -130,6 +130,6 @@ Test results formatting, saving, and API submission. cd /path/to/TheRock # Verify utils imports work -cd build_tools/github_actions/test_framework +cd build_tools/github_actions/extended_tests python -c "from utils.logger import log; print('Utils imports working')" ``` diff --git a/build_tools/github_actions/test_framework/utils/__init__.py b/build_tools/github_actions/extended_tests/utils/__init__.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/__init__.py rename to build_tools/github_actions/extended_tests/utils/__init__.py diff --git a/build_tools/github_actions/test_framework/utils/config/__init__.py b/build_tools/github_actions/extended_tests/utils/config/__init__.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/config/__init__.py rename to build_tools/github_actions/extended_tests/utils/config/__init__.py diff --git a/build_tools/github_actions/test_framework/utils/config/config_helper.py b/build_tools/github_actions/extended_tests/utils/config/config_helper.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/config/config_helper.py rename to build_tools/github_actions/extended_tests/utils/config/config_helper.py diff --git a/build_tools/github_actions/test_framework/utils/config/config_parser.py b/build_tools/github_actions/extended_tests/utils/config/config_parser.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/config/config_parser.py rename to build_tools/github_actions/extended_tests/utils/config/config_parser.py diff --git a/build_tools/github_actions/test_framework/utils/config/config_validator.py b/build_tools/github_actions/extended_tests/utils/config/config_validator.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/config/config_validator.py rename to build_tools/github_actions/extended_tests/utils/config/config_validator.py diff --git a/build_tools/github_actions/test_framework/utils/constants.py b/build_tools/github_actions/extended_tests/utils/constants.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/constants.py rename to build_tools/github_actions/extended_tests/utils/constants.py diff --git a/build_tools/github_actions/test_framework/utils/exceptions.py b/build_tools/github_actions/extended_tests/utils/exceptions.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/exceptions.py rename to build_tools/github_actions/extended_tests/utils/exceptions.py diff --git a/build_tools/github_actions/test_framework/utils/logger.py b/build_tools/github_actions/extended_tests/utils/logger.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/logger.py rename to build_tools/github_actions/extended_tests/utils/logger.py diff --git a/build_tools/github_actions/test_framework/utils/results/__init__.py b/build_tools/github_actions/extended_tests/utils/results/__init__.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/results/__init__.py rename to build_tools/github_actions/extended_tests/utils/results/__init__.py diff --git a/build_tools/github_actions/test_framework/utils/results/results_api.py b/build_tools/github_actions/extended_tests/utils/results/results_api.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/results/results_api.py rename to build_tools/github_actions/extended_tests/utils/results/results_api.py diff --git a/build_tools/github_actions/test_framework/utils/results/results_handler.py b/build_tools/github_actions/extended_tests/utils/results/results_handler.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/results/results_handler.py rename to build_tools/github_actions/extended_tests/utils/results/results_handler.py diff --git a/build_tools/github_actions/test_framework/utils/results/schemas/payload_schema.json b/build_tools/github_actions/extended_tests/utils/results/schemas/payload_schema.json similarity index 100% rename from build_tools/github_actions/test_framework/utils/results/schemas/payload_schema.json rename to build_tools/github_actions/extended_tests/utils/results/schemas/payload_schema.json diff --git a/build_tools/github_actions/test_framework/utils/system/__init__.py b/build_tools/github_actions/extended_tests/utils/system/__init__.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/system/__init__.py rename to build_tools/github_actions/extended_tests/utils/system/__init__.py diff --git a/build_tools/github_actions/test_framework/utils/system/hardware.py b/build_tools/github_actions/extended_tests/utils/system/hardware.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/system/hardware.py rename to build_tools/github_actions/extended_tests/utils/system/hardware.py diff --git a/build_tools/github_actions/test_framework/utils/system/platform.py b/build_tools/github_actions/extended_tests/utils/system/platform.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/system/platform.py rename to build_tools/github_actions/extended_tests/utils/system/platform.py diff --git a/build_tools/github_actions/test_framework/utils/system/rocm_detector.py b/build_tools/github_actions/extended_tests/utils/system/rocm_detector.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/system/rocm_detector.py rename to build_tools/github_actions/extended_tests/utils/system/rocm_detector.py diff --git a/build_tools/github_actions/test_framework/utils/system/system_detector.py b/build_tools/github_actions/extended_tests/utils/system/system_detector.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/system/system_detector.py rename to build_tools/github_actions/extended_tests/utils/system/system_detector.py diff --git a/build_tools/github_actions/test_framework/utils/test_client.py b/build_tools/github_actions/extended_tests/utils/test_client.py similarity index 100% rename from build_tools/github_actions/test_framework/utils/test_client.py rename to build_tools/github_actions/extended_tests/utils/test_client.py diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index c2e1bd56898..bdf13f39749 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -11,7 +11,7 @@ from pathlib import Path from github_actions_utils import * -from test_framework.benchmark.benchmark_test_matrix import benchmark_matrix +from extended_tests.benchmark.benchmark_test_matrix import benchmark_matrix from amdgpu_family_matrix import get_all_families_for_trigger_types logging.basicConfig(level=logging.INFO) diff --git a/build_tools/github_actions/tests/configure_ci_test.py b/build_tools/github_actions/tests/configure_ci_test.py index 41ad7acf8da..f5101f1feab 100644 --- a/build_tools/github_actions/tests/configure_ci_test.py +++ b/build_tools/github_actions/tests/configure_ci_test.py @@ -6,7 +6,7 @@ sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) import configure_ci -from test_framework.benchmark.benchmark_test_matrix import benchmark_matrix +from extended_tests.benchmark.benchmark_test_matrix import benchmark_matrix therock_test_runner_dict = { "gfx110x": { From 23d0b0ff8081e6dc9efccd187bd4e1157c185afd Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 13 Jan 2026 17:40:27 +0000 Subject: [PATCH 14/28] Update test framework name Signed-off-by: Lenine Ajagappane --- build_tools/github_actions/extended_tests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/github_actions/extended_tests/__init__.py b/build_tools/github_actions/extended_tests/__init__.py index 71ccce9e6c1..ee8538cc322 100644 --- a/build_tools/github_actions/extended_tests/__init__.py +++ b/build_tools/github_actions/extended_tests/__init__.py @@ -1,7 +1,7 @@ """ -TheRock Test Framework +TheRock Extended Tests Framework -Unified framework for benchmark, performance, and functional testing. +Unified framework for benchmark and functional testing. Test Types: - benchmark: Regression detection via LKG comparison (PASS/FAIL gates) From 0112ef5a488d13052e96355312768cb0b0a8a56e Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Thu, 15 Jan 2026 18:45:57 +0000 Subject: [PATCH 15/28] Update therock_dir path in benchmark scripts Signed-off-by: Lenine Ajagappane --- .../extended_tests/benchmark/scripts/benchmark_base.py | 1 + .../extended_tests/benchmark/scripts/test_hipblaslt_benchmark.py | 1 - .../extended_tests/benchmark/scripts/test_rocfft_benchmark.py | 1 - .../extended_tests/benchmark/scripts/test_rocrand_benchmark.py | 1 - .../extended_tests/benchmark/scripts/test_rocsolver_benchmark.py | 1 - 5 files changed, 1 insertion(+), 4 deletions(-) diff --git a/build_tools/github_actions/extended_tests/benchmark/scripts/benchmark_base.py b/build_tools/github_actions/extended_tests/benchmark/scripts/benchmark_base.py index 859c8396cd1..f92227fc4b6 100644 --- a/build_tools/github_actions/extended_tests/benchmark/scripts/benchmark_base.py +++ b/build_tools/github_actions/extended_tests/benchmark/scripts/benchmark_base.py @@ -35,6 +35,7 @@ def __init__(self, benchmark_name: str, display_name: str = None): self.artifact_run_id = os.getenv("ARTIFACT_RUN_ID") self.amdgpu_families = os.getenv("AMDGPU_FAMILIES") self.script_dir = Path(__file__).resolve().parent + self.therock_dir = self.script_dir.parent.parent.parent.parent.parent # Initialize test client (will be set in run()) self.client = None diff --git a/build_tools/github_actions/extended_tests/benchmark/scripts/test_hipblaslt_benchmark.py b/build_tools/github_actions/extended_tests/benchmark/scripts/test_hipblaslt_benchmark.py index 92095dcc6fd..2b7a5d4c03f 100644 --- a/build_tools/github_actions/extended_tests/benchmark/scripts/test_hipblaslt_benchmark.py +++ b/build_tools/github_actions/extended_tests/benchmark/scripts/test_hipblaslt_benchmark.py @@ -25,7 +25,6 @@ class HipblasltBenchmark(BenchmarkBase): def __init__(self): super().__init__(benchmark_name="hipblaslt", display_name="hipBLASLt") self.log_file = self.script_dir / "hipblaslt_bench.log" - self.therock_dir = self.script_dir.parent.parent.parent.parent def run_benchmarks(self) -> None: """Run hipBLASLt benchmarks and save output to log file.""" diff --git a/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocfft_benchmark.py b/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocfft_benchmark.py index 0fb5af12007..5fd6164ec66 100644 --- a/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocfft_benchmark.py +++ b/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocfft_benchmark.py @@ -25,7 +25,6 @@ class ROCfftBenchmark(BenchmarkBase): def __init__(self): super().__init__(benchmark_name="rocfft", display_name="ROCfft") self.log_file = self.script_dir / "rocfft_bench.log" - self.therock_dir = self.script_dir.parent.parent.parent.parent def run_benchmarks(self) -> None: """Run ROCfft benchmarks and save output to log file.""" diff --git a/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocrand_benchmark.py b/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocrand_benchmark.py index ba60696619f..15f562be237 100644 --- a/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocrand_benchmark.py +++ b/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocrand_benchmark.py @@ -25,7 +25,6 @@ class ROCrandBenchmark(BenchmarkBase): def __init__(self): super().__init__(benchmark_name="rocrand", display_name="ROCrand") - self.therock_dir = self.script_dir.parent.parent.parent.parent self.bench_bins = ["benchmark_rocrand_host_api", "benchmark_rocrand_device_api"] def run_benchmarks(self) -> None: diff --git a/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocsolver_benchmark.py b/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocsolver_benchmark.py index 6959e2fb297..e7154e6b539 100644 --- a/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocsolver_benchmark.py +++ b/build_tools/github_actions/extended_tests/benchmark/scripts/test_rocsolver_benchmark.py @@ -25,7 +25,6 @@ class ROCsolverBenchmark(BenchmarkBase): def __init__(self): super().__init__(benchmark_name="rocsolver", display_name="ROCsolver") self.log_file = self.script_dir / "rocsolver_bench.log" - self.therock_dir = self.script_dir.parent.parent.parent.parent def run_benchmarks(self) -> None: """Run ROCsolver benchmarks and save output to log file.""" From 1f1451bc302a9ce54a91f8a5826e73260dce36e1 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Fri, 16 Jan 2026 11:50:24 +0000 Subject: [PATCH 16/28] Add extended functional tests to nightly CI - Add extended functional test workflow running on test_runs_on machines - Integrate into ci_linux.yml and ci_windows.yml alongside benchmarks Extended functional tests now run in parallel with benchmarks and regular tests during nightly CI. Signed-off-by: Lenine Ajagappane --- .github/workflows/ci_linux.yml | 26 ++++++ .github/workflows/ci_windows.yml | 26 ++++++ .../workflows/test_extended_functional.yml | 87 +++++++++++++++++++ .../functional/functional_test_matrix.py | 2 +- .../fetch_test_configurations.py | 7 ++ 5 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test_extended_functional.yml diff --git a/.github/workflows/ci_linux.yml b/.github/workflows/ci_linux.yml index c5a381f61ff..bd45871e8e7 100644 --- a/.github/workflows/ci_linux.yml +++ b/.github/workflows/ci_linux.yml @@ -114,6 +114,32 @@ jobs: test_runs_on: ${{ inputs.benchmark_runs_on }} artifact_run_id: ${{ inputs.artifact_run_id }} + test_linux_extended_functional: + needs: [build_portable_linux_artifacts] + name: Test Linux Extended Functional + # Run extended functional tests if: + # - Build succeeded (or using prebuilt artifacts) + # - Not expecting failure + # - Test runner is available (test_runs_on is set) + if: >- + ${{ + !failure() && + !cancelled() && + ( + inputs.use_prebuilt_artifacts == 'false' || + inputs.use_prebuilt_artifacts == 'true' + ) && + inputs.expect_failure == false && + inputs.test_runs_on != '' + }} + uses: ./.github/workflows/test_extended_functional.yml + secrets: inherit + with: + artifact_group: ${{ inputs.artifact_group }} + amdgpu_families: ${{ inputs.amdgpu_families }} + test_runs_on: ${{ inputs.test_runs_on }} + artifact_run_id: ${{ inputs.artifact_run_id }} + build_portable_linux_python_packages: needs: [build_portable_linux_artifacts] name: Build Python diff --git a/.github/workflows/ci_windows.yml b/.github/workflows/ci_windows.yml index 064ab33c57c..cfb3d1730e9 100644 --- a/.github/workflows/ci_windows.yml +++ b/.github/workflows/ci_windows.yml @@ -114,6 +114,32 @@ jobs: test_runs_on: ${{ inputs.benchmark_runs_on }} artifact_run_id: ${{ inputs.artifact_run_id }} + test_windows_extended_functional: + needs: [build_windows_artifacts] + name: Test Windows Extended Functional + # Run extended functional tests if: + # - Build succeeded (or using prebuilt artifacts) + # - Not expecting failure + # - Test runner is available (test_runs_on is set) + if: >- + ${{ + !failure() && + !cancelled() && + ( + inputs.use_prebuilt_artifacts == 'false' || + inputs.use_prebuilt_artifacts == 'true' + ) && + inputs.expect_failure == false && + inputs.test_runs_on != '' + }} + uses: ./.github/workflows/test_extended_functional.yml + secrets: inherit + with: + artifact_group: ${{ inputs.artifact_group }} + amdgpu_families: ${{ inputs.amdgpu_families }} + test_runs_on: ${{ inputs.test_runs_on }} + artifact_run_id: ${{ inputs.artifact_run_id }} + build_windows_python_packages: needs: [build_windows_artifacts] name: Build Python diff --git a/.github/workflows/test_extended_functional.yml b/.github/workflows/test_extended_functional.yml new file mode 100644 index 00000000000..33de001a627 --- /dev/null +++ b/.github/workflows/test_extended_functional.yml @@ -0,0 +1,87 @@ +name: Test Extended Functional + +on: + workflow_dispatch: + inputs: + artifact_group: + type: string + artifact_run_id: + type: string + default: "" + amdgpu_families: + type: string + test_runs_on: + type: string + workflow_call: + inputs: + artifact_group: + type: string + artifact_run_id: + type: string + default: "" + amdgpu_families: + type: string + test_runs_on: + type: string + +permissions: + contents: read + +jobs: + configure_extended_functional_matrix: + name: "Configure extended functional matrix" + # if there is a test machine available + if: ${{ inputs.test_runs_on != '' }} + runs-on: ${{ inputs.test_runs_on }} + outputs: + components: ${{ steps.configure.outputs.components }} + platform: ${{ steps.configure.outputs.platform }} + steps: + - name: "Fetch 'build_tools' from repository" + if: ${{ runner.os == 'Windows' }} + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + sparse-checkout: build_tools + path: "prejob" + + # Checkout failure is possible on Windows, as it's the first job on a GPU test runner. + # Post-job cleanup isn't necessary since no executables are launched in this job. + - name: Pre-job cleanup processes on Windows + if: ${{ runner.os == 'Windows' }} + shell: powershell + run: . '${{ github.workspace }}\prejob\build_tools\github_actions\cleanup_processes.ps1' + + - name: "Checking out repository" + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + + - name: Setting up Python + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: 3.12 + + - name: "Configuring extended functional options" + id: configure + env: + ARTIFACT_GROUP: ${{ inputs.artifact_group }} + AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} + IS_EXTENDED_FUNCTIONAL_WORKFLOW: "true" + run: python ./build_tools/github_actions/fetch_test_configurations.py + + run_extended_functional: + name: 'Extended Functional ${{ matrix.components.job_name }}' + needs: [configure_extended_functional_matrix] + # skip extended functional tests if no extended functional matrix to run + if: ${{ needs.configure_extended_functional_matrix.outputs.components != '[]' }} + strategy: + fail-fast: false + matrix: + components: ${{ fromJSON(needs.configure_extended_functional_matrix.outputs.components) }} + uses: './.github/workflows/test_component.yml' + secrets: inherit + with: + artifact_run_id: ${{ inputs.artifact_run_id }} + artifact_group: ${{ inputs.artifact_group }} + amdgpu_families: ${{ inputs.amdgpu_families }} + test_runs_on: ${{ inputs.test_runs_on }} + platform: ${{ needs.configure_extended_functional_matrix.outputs.platform }} + component: ${{ toJSON(matrix.components) }} diff --git a/build_tools/github_actions/extended_tests/functional/functional_test_matrix.py b/build_tools/github_actions/extended_tests/functional/functional_test_matrix.py index ddaabb21ecb..0746b90a645 100644 --- a/build_tools/github_actions/extended_tests/functional/functional_test_matrix.py +++ b/build_tools/github_actions/extended_tests/functional/functional_test_matrix.py @@ -25,7 +25,7 @@ def _get_script_path(script_name: str) -> str: "job_name": "test_miopendriver_conv", "fetch_artifact_args": "--miopen --tests", "timeout_minutes": 30, - "test_script": f"python {_get_script_path('test_rocrand_benchmark.py')}", + "test_script": f"python {_get_script_path('test_miopendriver_conv.py')}", # TODO(lajagapp): Add windows support "platform": ["linux"], "total_shards": 1, diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index 0ce81d1d1b9..ac65bb60596 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -12,6 +12,7 @@ from github_actions_utils import * from extended_tests.benchmark.benchmark_test_matrix import benchmark_matrix +from extended_tests.functional.functional_test_matrix import functional_matrix from amdgpu_family_matrix import get_all_families_for_trigger_types logging.basicConfig(level=logging.INFO) @@ -286,6 +287,7 @@ def run(): test_type = os.getenv("TEST_TYPE", "full") test_labels = json.loads(os.getenv("TEST_LABELS", "[]")) is_benchmark_workflow = str2bool(os.getenv("IS_BENCHMARK_WORKFLOW", "false")) + is_extended_functional_workflow = str2bool(os.getenv("IS_EXTENDED_FUNCTIONAL_WORKFLOW", "false")) logging.info(f"Selecting projects: {project_to_test}") @@ -295,6 +297,11 @@ def run(): # Benchmarks don't use test_type/test_labels (all have total_shards=1, no filtering) logging.info("Using benchmark_matrix only (benchmark tests)") selected_matrix = benchmark_matrix.copy() + elif is_extended_functional_workflow: + # For extended functional workflow, use ONLY functional_matrix + # Extended functional tests don't use test_type/test_labels (all have total_shards=1, no filtering) + logging.info("Using functional_matrix only (extended functional tests)") + selected_matrix = functional_matrix.copy() else: # For regular workflow, use ONLY test_matrix logging.info("Using test_matrix only (regular tests)") From d75236884840944e4fca1297b27d7a899c1b8d3c Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Fri, 16 Jan 2026 11:54:50 +0000 Subject: [PATCH 17/28] Fix pre-commit errors Signed-off-by: Lenine Ajagappane --- build_tools/github_actions/fetch_test_configurations.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index ac65bb60596..0f8e70cc3a1 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -287,7 +287,9 @@ def run(): test_type = os.getenv("TEST_TYPE", "full") test_labels = json.loads(os.getenv("TEST_LABELS", "[]")) is_benchmark_workflow = str2bool(os.getenv("IS_BENCHMARK_WORKFLOW", "false")) - is_extended_functional_workflow = str2bool(os.getenv("IS_EXTENDED_FUNCTIONAL_WORKFLOW", "false")) + is_extended_functional_workflow = str2bool( + os.getenv("IS_EXTENDED_FUNCTIONAL_WORKFLOW", "false") + ) logging.info(f"Selecting projects: {project_to_test}") From 245c6b02a19396f0284326f54de7d5093ca653b2 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Fri, 16 Jan 2026 18:23:44 +0000 Subject: [PATCH 18/28] Restrict extended functional tests to nightly runs only Extended functional tests were running on all PRs. Changed to use a boolean flag `run_extended_functional` (default: false) that is only set to true in ci_nightly.yml, similar to how benchmarks are controlled. Changes: - Added `run_extended_functional` input to ci_linux.yml and ci_windows.yml - Updated test conditions to check flag instead of test_type - ci_nightly.yml sets run_extended_functional: true - Updated documentation to reflect nightly-only execution Signed-off-by: Lenine Ajagappane --- .github/workflows/ci_linux.yml | 7 ++++- .github/workflows/ci_nightly.yml | 2 ++ .github/workflows/ci_windows.yml | 7 ++++- .../github_actions/extended_tests/README.md | 31 +++++++++++++------ .../extended_tests/functional/README.md | 2 +- 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci_linux.yml b/.github/workflows/ci_linux.yml index bd45871e8e7..136ae2c8ce8 100644 --- a/.github/workflows/ci_linux.yml +++ b/.github/workflows/ci_linux.yml @@ -22,6 +22,9 @@ on: benchmark_runs_on: type: string default: "" + run_extended_functional: + type: boolean + default: false expect_failure: type: boolean use_prebuilt_artifacts: @@ -121,6 +124,7 @@ jobs: # - Build succeeded (or using prebuilt artifacts) # - Not expecting failure # - Test runner is available (test_runs_on is set) + # - Extended functional flag is enabled (only set to true in ci_nightly.yml) if: >- ${{ !failure() && @@ -130,7 +134,8 @@ jobs: inputs.use_prebuilt_artifacts == 'true' ) && inputs.expect_failure == false && - inputs.test_runs_on != '' + inputs.test_runs_on != '' && + inputs.run_extended_functional == true }} uses: ./.github/workflows/test_extended_functional.yml secrets: inherit diff --git a/.github/workflows/ci_nightly.yml b/.github/workflows/ci_nightly.yml index fcd9af7d46e..ff7c512f889 100644 --- a/.github/workflows/ci_nightly.yml +++ b/.github/workflows/ci_nightly.yml @@ -75,6 +75,7 @@ jobs: artifact_group: ${{ matrix.variant.artifact_group }} test_runs_on: ${{ matrix.variant.test-runs-on }} benchmark_runs_on: ${{ matrix.variant.benchmark-runs-on }} + run_extended_functional: true build_variant_label: ${{ matrix.variant.build_variant_label }} build_variant_suffix: ${{ matrix.variant.build_variant_suffix }} build_variant_cmake_preset: ${{ matrix.variant.build_variant_cmake_preset }} @@ -108,6 +109,7 @@ jobs: artifact_group: ${{ matrix.variant.artifact_group }} test_runs_on: ${{ matrix.variant.test-runs-on }} benchmark_runs_on: ${{ matrix.variant.benchmark-runs-on }} + run_extended_functional: true build_variant_label: ${{ matrix.variant.build_variant_label }} build_variant_suffix: ${{ matrix.variant.build_variant_suffix }} build_variant_cmake_preset: ${{ matrix.variant.build_variant_cmake_preset }} diff --git a/.github/workflows/ci_windows.yml b/.github/workflows/ci_windows.yml index cfb3d1730e9..2656844bdbf 100644 --- a/.github/workflows/ci_windows.yml +++ b/.github/workflows/ci_windows.yml @@ -22,6 +22,9 @@ on: benchmark_runs_on: type: string default: "" + run_extended_functional: + type: boolean + default: false expect_failure: type: boolean use_prebuilt_artifacts: @@ -121,6 +124,7 @@ jobs: # - Build succeeded (or using prebuilt artifacts) # - Not expecting failure # - Test runner is available (test_runs_on is set) + # - Extended functional flag is enabled (only set to true in ci_nightly.yml) if: >- ${{ !failure() && @@ -130,7 +134,8 @@ jobs: inputs.use_prebuilt_artifacts == 'true' ) && inputs.expect_failure == false && - inputs.test_runs_on != '' + inputs.test_runs_on != '' && + inputs.run_extended_functional == true }} uses: ./.github/workflows/test_extended_functional.yml secrets: inherit diff --git a/build_tools/github_actions/extended_tests/README.md b/build_tools/github_actions/extended_tests/README.md index 619a4cd7e5f..5fca0a63625 100644 --- a/build_tools/github_actions/extended_tests/README.md +++ b/build_tools/github_actions/extended_tests/README.md @@ -108,9 +108,9 @@ extended_tests/ | Workflow Trigger | Benchmark Tests | Functional Tests | | -------------------------- | --------------- | ---------------- | -| **Pull Request (PR)** | Skipped | Optional | +| **Pull Request (PR)** | Skipped | Skipped | | **Nightly CI (scheduled)** | Run (parallel) | Run (parallel) | -| **Push to main** | Skipped | Optional | +| **Push to main** | Skipped | Skipped | ### Parallel Execution Architecture @@ -118,21 +118,32 @@ Tests run in **parallel** for faster CI execution: ``` ci_nightly.yml - └─ ci_linux.yml + └─ ci_linux.yml / ci_windows.yml ├─ build_artifacts │ - ├─ test_artifacts ────────┐ Run in parallel - │ └─ Functional tests │ after build - │ │ - └─ test_benchmarks ────────┘ - └─ Benchmark tests + ├─ test_artifacts ────────────────────┐ + │ └─ Component tests (smoke/full) │ Run in parallel + │ │ after build + ├─ test_benchmarks ───────────────────┤ + │ └─ Benchmark tests │ + │ │ + └─ test_extended_functional ──────────┘ + └─ Extended functional tests ``` **Workflow Files:** - `.github/workflows/ci_nightly.yml` - Nightly CI orchestration -- `.github/workflows/test_benchmarks.yml` - Benchmark execution -- `.github/workflows/test_artifacts.yml` - Functional test execution +- `.github/workflows/ci_linux.yml` / `ci_windows.yml` - Platform-specific CI logic +- `.github/workflows/test_benchmarks.yml` - Benchmark test execution (uses `benchmark_runs_on`) +- `.github/workflows/test_extended_functional.yml` - Extended functional test execution (uses `test_runs_on`) +- `.github/workflows/test_artifacts.yml` - Component test execution (uses `test_runs_on`) + +**Key Differences:** + +- **Component Tests**: Run on all PRs (smoke) and nightly (full), use regular runners +- **Benchmark Tests**: Run only on nightly, use dedicated performance runners (`benchmark_runs_on`) +- **Extended Functional Tests**: Run only on nightly, use regular runners (`test_runs_on`), controlled by `run_extended_functional` flag ## Architecture diff --git a/build_tools/github_actions/extended_tests/functional/README.md b/build_tools/github_actions/extended_tests/functional/README.md index 9b29583f1d6..565523df43d 100644 --- a/build_tools/github_actions/extended_tests/functional/README.md +++ b/build_tools/github_actions/extended_tests/functional/README.md @@ -10,7 +10,7 @@ Functional tests validate **correctness and behavior**: - **Result Types:** PASS / FAIL / ERROR / SKIP - **Validation:** Output correctness, API contracts, expected behavior -- **CI Execution:** Nightly CI and optionally on PRs +- **CI Execution:** Nightly CI only (controlled by `run_extended_functional` flag) - **Exit Code:** Non-zero if any test FAILS or has ERRORs ## Available Tests From 7181402fdbee9c4b71ec39fce7f5ec8bd33dff25 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Mon, 19 Jan 2026 18:52:50 +0000 Subject: [PATCH 19/28] Refactor: Improve naming consistency for extended functional tests - Rename workflow file and parameters with _tests suffix for clarity - Simplify internal job names by removing redundant "extended_functional" prefix - Add input parameter descriptions with examples Signed-off-by: Lenine Ajagappane --- .github/workflows/ci_linux.yml | 10 +++--- .github/workflows/ci_nightly.yml | 4 +-- .github/workflows/ci_windows.yml | 10 +++--- ...yml => test_extended_functional_tests.yml} | 32 ++++++++++++------- .../benchmark/scripts/benchmark_base.py | 1 + .../extended_tests/utils/README.md | 5 ++- .../extended_tests/utils/system/hardware.py | 5 ++- 7 files changed, 37 insertions(+), 30 deletions(-) rename .github/workflows/{test_extended_functional.yml => test_extended_functional_tests.yml} (63%) diff --git a/.github/workflows/ci_linux.yml b/.github/workflows/ci_linux.yml index 136ae2c8ce8..cc015c21083 100644 --- a/.github/workflows/ci_linux.yml +++ b/.github/workflows/ci_linux.yml @@ -22,7 +22,7 @@ on: benchmark_runs_on: type: string default: "" - run_extended_functional: + run_extended_functional_tests: type: boolean default: false expect_failure: @@ -117,9 +117,9 @@ jobs: test_runs_on: ${{ inputs.benchmark_runs_on }} artifact_run_id: ${{ inputs.artifact_run_id }} - test_linux_extended_functional: + run_linux_extended_functional_tests: needs: [build_portable_linux_artifacts] - name: Test Linux Extended Functional + name: Run Extended Functional Tests # Run extended functional tests if: # - Build succeeded (or using prebuilt artifacts) # - Not expecting failure @@ -135,9 +135,9 @@ jobs: ) && inputs.expect_failure == false && inputs.test_runs_on != '' && - inputs.run_extended_functional == true + inputs.run_extended_functional_tests == true }} - uses: ./.github/workflows/test_extended_functional.yml + uses: ./.github/workflows/test_extended_functional_tests.yml secrets: inherit with: artifact_group: ${{ inputs.artifact_group }} diff --git a/.github/workflows/ci_nightly.yml b/.github/workflows/ci_nightly.yml index ff7c512f889..2fa02cb9972 100644 --- a/.github/workflows/ci_nightly.yml +++ b/.github/workflows/ci_nightly.yml @@ -75,7 +75,7 @@ jobs: artifact_group: ${{ matrix.variant.artifact_group }} test_runs_on: ${{ matrix.variant.test-runs-on }} benchmark_runs_on: ${{ matrix.variant.benchmark-runs-on }} - run_extended_functional: true + run_extended_functional_tests: true build_variant_label: ${{ matrix.variant.build_variant_label }} build_variant_suffix: ${{ matrix.variant.build_variant_suffix }} build_variant_cmake_preset: ${{ matrix.variant.build_variant_cmake_preset }} @@ -109,7 +109,7 @@ jobs: artifact_group: ${{ matrix.variant.artifact_group }} test_runs_on: ${{ matrix.variant.test-runs-on }} benchmark_runs_on: ${{ matrix.variant.benchmark-runs-on }} - run_extended_functional: true + run_extended_functional_tests: true build_variant_label: ${{ matrix.variant.build_variant_label }} build_variant_suffix: ${{ matrix.variant.build_variant_suffix }} build_variant_cmake_preset: ${{ matrix.variant.build_variant_cmake_preset }} diff --git a/.github/workflows/ci_windows.yml b/.github/workflows/ci_windows.yml index 2656844bdbf..3d15d4c2d2b 100644 --- a/.github/workflows/ci_windows.yml +++ b/.github/workflows/ci_windows.yml @@ -22,7 +22,7 @@ on: benchmark_runs_on: type: string default: "" - run_extended_functional: + run_extended_functional_tests: type: boolean default: false expect_failure: @@ -117,9 +117,9 @@ jobs: test_runs_on: ${{ inputs.benchmark_runs_on }} artifact_run_id: ${{ inputs.artifact_run_id }} - test_windows_extended_functional: + run_windows_extended_functional_tests: needs: [build_windows_artifacts] - name: Test Windows Extended Functional + name: Run Extended Functional Tests # Run extended functional tests if: # - Build succeeded (or using prebuilt artifacts) # - Not expecting failure @@ -135,9 +135,9 @@ jobs: ) && inputs.expect_failure == false && inputs.test_runs_on != '' && - inputs.run_extended_functional == true + inputs.run_extended_functional_tests == true }} - uses: ./.github/workflows/test_extended_functional.yml + uses: ./.github/workflows/test_extended_functional_tests.yml secrets: inherit with: artifact_group: ${{ inputs.artifact_group }} diff --git a/.github/workflows/test_extended_functional.yml b/.github/workflows/test_extended_functional_tests.yml similarity index 63% rename from .github/workflows/test_extended_functional.yml rename to .github/workflows/test_extended_functional_tests.yml index 33de001a627..87b719ab641 100644 --- a/.github/workflows/test_extended_functional.yml +++ b/.github/workflows/test_extended_functional_tests.yml @@ -1,35 +1,43 @@ -name: Test Extended Functional +name: Run Extended Functional Tests on: workflow_dispatch: inputs: artifact_group: + description: 'Build artifact group to test (e.g., gfx94X-dcgpu, gfx101X-dgpu, gfx1151, gfx120X-all)' type: string artifact_run_id: + description: 'GitHub Actions run ID where artifacts were built (empty to use current run)' type: string default: "" amdgpu_families: + description: 'Comma-separated list of AMD GPU families to test (e.g., gfx908, gfx90a, gfx942)' type: string test_runs_on: + description: 'GitHub Actions runner label for the test machine' type: string workflow_call: inputs: artifact_group: + description: 'Build artifact group to test (e.g., gfx94X-dcgpu, gfx101X-dgpu, gfx1151, gfx120X-all)' type: string artifact_run_id: + description: 'GitHub Actions run ID where artifacts were built (empty to use current run)' type: string default: "" amdgpu_families: + description: 'Comma-separated list of AMD GPU families to test (e.g., gfx908, gfx90a, gfx942)' type: string test_runs_on: + description: 'GitHub Actions runner label for the test machine' type: string permissions: contents: read jobs: - configure_extended_functional_matrix: - name: "Configure extended functional matrix" + configure_test_matrix: + name: "Configure extended functional test matrix" # if there is a test machine available if: ${{ inputs.test_runs_on != '' }} runs-on: ${{ inputs.test_runs_on }} @@ -59,7 +67,7 @@ jobs: with: python-version: 3.12 - - name: "Configuring extended functional options" + - name: "Configuring test options" id: configure env: ARTIFACT_GROUP: ${{ inputs.artifact_group }} @@ -67,21 +75,21 @@ jobs: IS_EXTENDED_FUNCTIONAL_WORKFLOW: "true" run: python ./build_tools/github_actions/fetch_test_configurations.py - run_extended_functional: - name: 'Extended Functional ${{ matrix.components.job_name }}' - needs: [configure_extended_functional_matrix] - # skip extended functional tests if no extended functional matrix to run - if: ${{ needs.configure_extended_functional_matrix.outputs.components != '[]' }} + run_tests: + name: 'Test ${{ matrix.components.job_name }}' + needs: [configure_test_matrix] + # skip tests if no test matrix to run + if: ${{ needs.configure_test_matrix.outputs.components != '[]' }} strategy: fail-fast: false matrix: - components: ${{ fromJSON(needs.configure_extended_functional_matrix.outputs.components) }} - uses: './.github/workflows/test_component.yml' + components: ${{ fromJSON(needs.configure_test_matrix.outputs.components) }} + uses: ./.github/workflows/test_component.yml secrets: inherit with: artifact_run_id: ${{ inputs.artifact_run_id }} artifact_group: ${{ inputs.artifact_group }} amdgpu_families: ${{ inputs.amdgpu_families }} test_runs_on: ${{ inputs.test_runs_on }} - platform: ${{ needs.configure_extended_functional_matrix.outputs.platform }} + platform: ${{ needs.configure_test_matrix.outputs.platform }} component: ${{ toJSON(matrix.components) }} diff --git a/build_tools/github_actions/extended_tests/benchmark/scripts/benchmark_base.py b/build_tools/github_actions/extended_tests/benchmark/scripts/benchmark_base.py index 8b5ee7790e5..0bd1ca4157e 100644 --- a/build_tools/github_actions/extended_tests/benchmark/scripts/benchmark_base.py +++ b/build_tools/github_actions/extended_tests/benchmark/scripts/benchmark_base.py @@ -39,6 +39,7 @@ def __init__(self, benchmark_name: str, display_name: str = None): self.artifact_run_id = os.getenv("ARTIFACT_RUN_ID") self.amdgpu_families = os.getenv("AMDGPU_FAMILIES") self.script_dir = Path(__file__).resolve().parent + # Navigate from benchmark/scripts/ up to TheRock root directory self.therock_dir = self.script_dir.parent.parent.parent.parent.parent # Initialize test client (will be set in run()) diff --git a/build_tools/github_actions/extended_tests/utils/README.md b/build_tools/github_actions/extended_tests/utils/README.md index 83d3cf04c30..3121a2449a6 100644 --- a/build_tools/github_actions/extended_tests/utils/README.md +++ b/build_tools/github_actions/extended_tests/utils/README.md @@ -42,9 +42,8 @@ Test scripts (benchmark/functional) add `extended_tests/` to `sys.path`, then im ```python # Import path setup (already done in base classes) -sys.path.insert( - 0, str(Path(__file__).parent.parent.parent) -) # Adds extended_tests/ to path +# Adds extended_tests/ to path +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # Core utilities from utils.logger import log diff --git a/build_tools/github_actions/extended_tests/utils/system/hardware.py b/build_tools/github_actions/extended_tests/utils/system/hardware.py index e63f851b3dc..be374900b43 100644 --- a/build_tools/github_actions/extended_tests/utils/system/hardware.py +++ b/build_tools/github_actions/extended_tests/utils/system/hardware.py @@ -82,9 +82,8 @@ class GpuInfo: partition_mode: str = "Unknown" xgmi_type: str = "Unknown" host_driver: str = "Unknown" - target_graphics_version: str = ( - "Unknown" # Target GPU architecture (e.g., gfx950, gfx942) - ) + # Target GPU architecture (e.g., gfx950, gfx942) + target_graphics_version: str = "Unknown" firmwares: List[Dict[str, str]] = field(default_factory=list) def __str__(self): From 82579d5e5af0b63723c509e87e86b6076e2911a4 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Mon, 19 Jan 2026 19:22:35 +0000 Subject: [PATCH 20/28] Docs: Improve extended tests documentation and environment setup guidance Enhanced README documentation for extended tests with clearer environment variable setup instructions for local testing. Expanded functional_base.py docstring with usage examples and framework clarifications. Signed-off-by: Lenine Ajagappane --- .../github_actions/extended_tests/README.md | 13 +++++++++---- .../extended_tests/functional/README.md | 2 ++ .../functional/scripts/functional_base.py | 17 ++++++++++++++++- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/build_tools/github_actions/extended_tests/README.md b/build_tools/github_actions/extended_tests/README.md index 5fca0a63625..51962a7dfa3 100644 --- a/build_tools/github_actions/extended_tests/README.md +++ b/build_tools/github_actions/extended_tests/README.md @@ -33,12 +33,17 @@ The test framework provides infrastructure for two test types: ### Environment Setup -All tests require these environment variables: +All tests require these environment variables. **Note:** These are automatically configured in CI runs. For local testing, adjust values based on your setup: ```bash -export THEROCK_BIN_DIR=/path/to/therock/build/bin # Path to TheRock binaries -export ARTIFACT_RUN_ID=local-test # Unique identifier for this test run -export AMDGPU_FAMILIES=gfx950-dcgpu # Target GPU family +# Required: Update to your actual TheRock build directory +export THEROCK_BIN_DIR=/path/to/therock/build/bin + +# Optional: Unique identifier for this test run (default: local-test) +export ARTIFACT_RUN_ID=local-test + +# Required: Update to match your GPU family (e.g., gfx908, gfx90a, gfx942, gfx950-dcgpu) +export AMDGPU_FAMILIES=gfx950-dcgpu ``` ### Running Tests diff --git a/build_tools/github_actions/extended_tests/functional/README.md b/build_tools/github_actions/extended_tests/functional/README.md index 565523df43d..2d9118c793b 100644 --- a/build_tools/github_actions/extended_tests/functional/README.md +++ b/build_tools/github_actions/extended_tests/functional/README.md @@ -142,6 +142,8 @@ Edit `functional_test_matrix.py`: ### Step 3: Test Locally +For example, run test for gfx94x-dcgpu: + ```bash export THEROCK_BIN_DIR=/path/to/build/bin export ARTIFACT_RUN_ID=local-test diff --git a/build_tools/github_actions/extended_tests/functional/scripts/functional_base.py b/build_tools/github_actions/extended_tests/functional/scripts/functional_base.py index 017a4c55a68..0786ceff854 100644 --- a/build_tools/github_actions/extended_tests/functional/scripts/functional_base.py +++ b/build_tools/github_actions/extended_tests/functional/scripts/functional_base.py @@ -1,4 +1,19 @@ -"""Base class for functional tests with common functionality.""" +""" +Base class for functional tests with common functionality. + +Provides test execution, result parsing, logging, and GitHub Actions integration +for functional correctness tests. Unlike benchmarks which measure performance, +functional tests verify correctness with pass/fail results. + +Usage: + Subclass and implement run_tests() and parse_results(): + + class MyTest(FunctionalBase): + def run_tests(self): ... + def parse_results(self, log_file): ... + +Required environment variables: THEROCK_BIN_DIR, ARTIFACT_RUN_ID, AMDGPU_FAMILIES +""" import json import os From c1839fad7de28b043b0a54f66af8b28b5c8a31f8 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Mon, 19 Jan 2026 20:13:11 +0000 Subject: [PATCH 21/28] Refactor: Improve naming and documentation for extended test framework - Improve GPU architecture detection logic and multi-GPU handling - Add module docstrings and clarify baseline selection Signed-off-by: Lenine Ajagappane --- .../github_actions/extended_tests/README.md | 3 ++ .../extended_tests/benchmark/README.md | 36 +++++-------------- .../extended_tests/functional/README.md | 5 ++- .../scripts/test_miopendriver_conv.py | 1 + .../extended_tests/utils/system/hardware.py | 28 ++++++++++----- 5 files changed, 35 insertions(+), 38 deletions(-) diff --git a/build_tools/github_actions/extended_tests/README.md b/build_tools/github_actions/extended_tests/README.md index 51962a7dfa3..80699c62847 100644 --- a/build_tools/github_actions/extended_tests/README.md +++ b/build_tools/github_actions/extended_tests/README.md @@ -44,6 +44,9 @@ export ARTIFACT_RUN_ID=local-test # Required: Update to match your GPU family (e.g., gfx908, gfx90a, gfx942, gfx950-dcgpu) export AMDGPU_FAMILIES=gfx950-dcgpu + +# Optional: Control GPU visibility on multi-GPU nodes (e.g., ROCR_VISIBLE_DEVICES=0) +# export ROCR_VISIBLE_DEVICES=0 ``` ### Running Tests diff --git a/build_tools/github_actions/extended_tests/benchmark/README.md b/build_tools/github_actions/extended_tests/benchmark/README.md index 157ae34e86e..417b85607f0 100644 --- a/build_tools/github_actions/extended_tests/benchmark/README.md +++ b/build_tools/github_actions/extended_tests/benchmark/README.md @@ -10,6 +10,7 @@ Benchmark tests detect **performance regressions** by comparing against baseline - **Result Types:** PASS (within tolerance) / FAIL (regression) / UNKNOWN (no baseline) - **Comparison:** Current performance vs. LKG baseline with configurable tolerance +- **Baseline Selection:** LKG baselines are retrieved from the benchmark database API, currently filtered by ROCm version, test name and test configuration (GPU architecture, hostname). The baseline represents the last known good performance metrics for each test configuration. - **CI Execution:** Nightly only (not on PRs to save resources) - **Exit Code:** Non-zero if any test FAILS @@ -32,34 +33,13 @@ cd build_tools/github_actions/extended_tests/benchmark/scripts python test_hipblaslt_benchmark.py ``` -## CI Test Matrix - -Tests defined in `benchmark_test_matrix.py`: - -| Test Name | Library | Platform | Timeout | Artifacts Needed | CI Status | -| ----------------- | --------- | -------------- | ------- | ---------------------- | ----------------- | -| `hipblaslt_bench` | hipBLASLt | Linux, Windows | 60 min | `--blas --tests` | Enabled (nightly) | -| `rocfft_bench` | rocFFT | Linux, Windows | 60 min | `--fft --rand --tests` | Enabled (nightly) | -| `rocrand_bench` | rocRAND | Linux, Windows | 60 min | `--rand --tests` | Enabled (nightly) | -| `rocsolver_bench` | ROCsolver | Linux, Windows | 60 min | `--blas --tests` | Enabled (nightly) | -| `rccl_bench` | RCCL | Linux | 90 min | `--rccl --tests` | Enabled (nightly) | -| `rocblas_bench` | rocBLAS | Linux, Windows | 90 min | `--blas --tests` | Enabled (nightly) | - -**GPU Family Support:** - -| GPU Family | Platform | Architecture | Benchmark Supported | Benchmark CI Status | -| ---------- | -------- | --------------------- | ------------------- | -------------------- | -| `gfx94x` | Linux | MI300X/MI325X (CDNA3) | Yes | Enabled (nightly CI) | -| `gfx1151` | Windows | RDNA 3.5 | Yes | Enabled (nightly CI) | -| `gfx950` | Linux | MI355X (CDNA4) | Yes | Not enabled | -| `gfx110x` | Windows | RDNA 2 | Yes | Not enabled | -| `gfx110x` | Linux | RDNA 2 | Yes | Not enabled | -| `gfx120x` | Linux | RDNA 3 | Yes | Not enabled | -| `gfx120x` | Windows | RDNA 3 | Yes | Not enabled | -| `gfx90x` | Linux | MI200 (CDNA2) | Yes | Not enabled | -| `gfx1151` | Linux | RDNA 3.5 | Yes | Not enabled | - -> **Note:** All benchmarks are **architecture-agnostic** and support any ROCm-compatible GPU. The table above lists GPU families actively used in CI testing. To add support for additional GPU families, update [`amdgpu_family_matrix.py`](../amdgpu_family_matrix.py) with appropriate `benchmark-runs-on` runners. +## CI Configuration + +- **Test Matrix:** See [`benchmark_test_matrix.py`](benchmark_test_matrix.py) for complete test definitions (platforms, timeouts, artifacts needed) +- **GPU Family Support:** See [`amdgpu_family_matrix.py`](../amdgpu_family_matrix.py) for CI-enabled GPU families and runner configurations +- **Execution:** All benchmarks run in nightly CI builds only (not on PRs to save resources) +- **Architecture Support:** Benchmarks are architecture-agnostic and support any ROCm-compatible GPU +- **Architecture Exclusions:** If a specific GPU architecture is not supported for a test, use the `exclude_family` field in the test matrix to skip that architecture/platform combination (see `fetch_test_configurations.py` for filtering logic) ## How Benchmark Tests Work diff --git a/build_tools/github_actions/extended_tests/functional/README.md b/build_tools/github_actions/extended_tests/functional/README.md index 2d9118c793b..126b0dc8115 100644 --- a/build_tools/github_actions/extended_tests/functional/README.md +++ b/build_tools/github_actions/extended_tests/functional/README.md @@ -111,12 +111,15 @@ class YourTest(FunctionalBase): log.info("Parsing Results") detailed_table = PrettyTable() + # Field names are flexible - customize based on your test output + # Common patterns: ["TestCase", "Status"] or ["TestSuite", "TestCase", "Status"] detailed_table.field_names = ["TestCase", "Status"] test_results = [] # Parse log file for each test case - # Use self.create_test_result() to create result dictionaries + # MANDATORY: Use self.create_test_result(test_name, subtest_name, status, **kwargs) + # to create standardized result dictionaries for database upload return test_results, detailed_table, num_suites diff --git a/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py b/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py index 941aec5b1f1..f37857e1001 100644 --- a/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py +++ b/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py @@ -52,6 +52,7 @@ def run_tests(self) -> None: log.info(f"Running {self.display_name} Tests") # Detect GPU architecture using HardwareDetector + # Returns first discrete GPU. Use ROCR_VISIBLE_DEVICES to control which GPU if needed. detector = HardwareDetector() gfx_id = detector.get_gpu_architecture() log.info(f"Detected GPU: {gfx_id}") diff --git a/build_tools/github_actions/extended_tests/utils/system/hardware.py b/build_tools/github_actions/extended_tests/utils/system/hardware.py index be374900b43..30a4b2fea07 100644 --- a/build_tools/github_actions/extended_tests/utils/system/hardware.py +++ b/build_tools/github_actions/extended_tests/utils/system/hardware.py @@ -555,10 +555,19 @@ def _detect_gpu_with_lspci(self) -> bool: def get_gpu_architecture(self) -> str: """Detect GPU architecture using rocminfo. + Returns: str: GPU architecture ID (e.g., 'gfx908', 'gfx90a', 'gfx942') + Raises: ValueError: If GPU architecture cannot be detected + + Note: + Returns the first discrete AMD GPU architecture found. Integrated/CPU GPUs + are automatically filtered out (gfx000x). On multi-GPU nodes with homogeneous + GPUs (typical case), all GPUs have the same architecture. For heterogeneous + setups or to target a specific GPU, use ROCR_VISIBLE_DEVICES or + HIP_VISIBLE_DEVICES environment variables to control GPU visibility. """ import logging @@ -581,13 +590,16 @@ def get_gpu_architecture(self) -> str: ) for line in result.stdout.splitlines(): - if "Name:" in line and "gfx" in line: - # Extract gfx ID - for part in line.split(): - if part.startswith("gfx"): - gfx_id = part.strip() - logger.info(f"Detected GPU architecture: {gfx_id}") - return gfx_id + # Look for lines starting with "Name:" that contain "gfx" + if line.strip().startswith("Name:") and "gfx" in line: + # Extract gfx ID from the end of the line + arch = line.split(":")[-1].strip() + # Filter for discrete GPU architectures (skip CPU/integrated) + if arch.startswith("gfx") and not arch.startswith("gfx000"): + logger.info(f"Detected GPU architecture: {arch}") + return arch + + raise ValueError("No discrete AMD GPUs found in rocminfo output") except subprocess.CalledProcessError as e: logger.error(f"rocminfo failed with return code {e.returncode}") @@ -595,8 +607,6 @@ def get_gpu_architecture(self) -> str: except FileNotFoundError: raise ValueError(f"rocminfo not found at {rocminfo_path}") - raise ValueError("Could not detect GPU architecture from rocminfo output") - def _detect_gpu_with_rocminfo(self) -> bool: """Detect GPUs using rocminfo (secondary method, ROCm runtime-based). From 6564b8e597727db3769687838febea47771dd920 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Mon, 19 Jan 2026 20:34:39 +0000 Subject: [PATCH 22/28] Update functional/README.md Signed-off-by: Lenine Ajagappane --- .../extended_tests/functional/README.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/build_tools/github_actions/extended_tests/functional/README.md b/build_tools/github_actions/extended_tests/functional/README.md index 126b0dc8115..4db1e4dd7da 100644 --- a/build_tools/github_actions/extended_tests/functional/README.md +++ b/build_tools/github_actions/extended_tests/functional/README.md @@ -26,13 +26,12 @@ Functional tests validate **correctness and behavior**: python build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py ``` -## CI Test Matrix +## CI Configuration -Tests defined in `functional_test_matrix.py`: - -| Test Name | Library | Platform | Timeout | Artifacts Needed | CI Status | -| -------------------- | ------- | -------- | ------- | ------------------ | ----------------- | -| `miopen_driver_conv` | MIOpen | Linux | 30 min | `--miopen --tests` | Enabled (nightly) | +- **Test Matrix:** See [`functional_test_matrix.py`](functional_test_matrix.py) for complete test definitions (platforms, timeouts, artifacts needed) +- **Execution:** All functional tests run in nightly CI builds only +- **Architecture Support:** Tests are architecture-agnostic and support any ROCm-compatible GPU +- **Architecture Exclusions:** If a specific GPU architecture is not supported for a test, use the `exclude_family` field in the test matrix to skip that architecture/platform combination (see `fetch_test_configurations.py` for filtering logic) ## How Functional Tests Work @@ -75,8 +74,10 @@ from pathlib import Path from typing import Dict, List, Tuple, Any from prettytable import PrettyTable +# Add parent directories to path for imports sys.path.insert(0, str(Path(__file__).parent.parent.parent)) # extended_tests/ sys.path.insert(0, str(Path(__file__).parent)) # For functional_base + from functional_base import FunctionalBase, run_functional_test_main from utils.logger import log from utils.exceptions import TestExecutionError @@ -128,6 +129,12 @@ if __name__ == "__main__": run_functional_test_main(YourTestFunctionalTest()) ``` +**Required Methods:** +- `run_tests()` → Execute tests and save results +- `parse_results()` → Returns `(test_results_list, display_table, num_suites)` + - Must use `self.create_test_result(test_name, subtest_name, status, **kwargs)` + - Status must be: `"PASS"`, `"FAIL"`, `"ERROR"`, or `"SKIP"` + ### Step 2: Add to Functional Matrix Edit `functional_test_matrix.py`: From 6590d7e9aa5dded14795b7314c1e77c5e84a0c3b Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 20 Jan 2026 16:17:24 +0000 Subject: [PATCH 23/28] Update README with test_extended_functional_tests.yml Signed-off-by: Lenine Ajagappane --- .github/workflows/test_extended_functional_tests.yml | 2 +- build_tools/github_actions/extended_tests/README.md | 6 +++--- .../github_actions/extended_tests/functional/README.md | 2 +- build_tools/github_actions/fetch_test_configurations.py | 6 +++--- rocm-libraries | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_extended_functional_tests.yml b/.github/workflows/test_extended_functional_tests.yml index 87b719ab641..d71e63820ca 100644 --- a/.github/workflows/test_extended_functional_tests.yml +++ b/.github/workflows/test_extended_functional_tests.yml @@ -72,7 +72,7 @@ jobs: env: ARTIFACT_GROUP: ${{ inputs.artifact_group }} AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} - IS_EXTENDED_FUNCTIONAL_WORKFLOW: "true" + IS_EXTENDED_FUNCTIONAL_TESTS: "true" run: python ./build_tools/github_actions/fetch_test_configurations.py run_tests: diff --git a/build_tools/github_actions/extended_tests/README.md b/build_tools/github_actions/extended_tests/README.md index 80699c62847..74d32fe0e7c 100644 --- a/build_tools/github_actions/extended_tests/README.md +++ b/build_tools/github_actions/extended_tests/README.md @@ -135,7 +135,7 @@ ci_nightly.yml ├─ test_benchmarks ───────────────────┤ │ └─ Benchmark tests │ │ │ - └─ test_extended_functional ──────────┘ + └─ test_extended_functional_tests ──────┘ └─ Extended functional tests ``` @@ -144,14 +144,14 @@ ci_nightly.yml - `.github/workflows/ci_nightly.yml` - Nightly CI orchestration - `.github/workflows/ci_linux.yml` / `ci_windows.yml` - Platform-specific CI logic - `.github/workflows/test_benchmarks.yml` - Benchmark test execution (uses `benchmark_runs_on`) -- `.github/workflows/test_extended_functional.yml` - Extended functional test execution (uses `test_runs_on`) +- `.github/workflows/test_extended_functional_tests.yml` - Extended functional test execution (uses `test_runs_on`) - `.github/workflows/test_artifacts.yml` - Component test execution (uses `test_runs_on`) **Key Differences:** - **Component Tests**: Run on all PRs (smoke) and nightly (full), use regular runners - **Benchmark Tests**: Run only on nightly, use dedicated performance runners (`benchmark_runs_on`) -- **Extended Functional Tests**: Run only on nightly, use regular runners (`test_runs_on`), controlled by `run_extended_functional` flag +- **Extended Functional Tests**: Run only on nightly, use regular runners (`test_runs_on`), controlled by `run_extended_functional_tests` flag ## Architecture diff --git a/build_tools/github_actions/extended_tests/functional/README.md b/build_tools/github_actions/extended_tests/functional/README.md index 4db1e4dd7da..c4af3663707 100644 --- a/build_tools/github_actions/extended_tests/functional/README.md +++ b/build_tools/github_actions/extended_tests/functional/README.md @@ -10,7 +10,7 @@ Functional tests validate **correctness and behavior**: - **Result Types:** PASS / FAIL / ERROR / SKIP - **Validation:** Output correctness, API contracts, expected behavior -- **CI Execution:** Nightly CI only (controlled by `run_extended_functional` flag) +- **CI Execution:** Nightly CI only (controlled by `run_extended_functional_tests` flag) - **Exit Code:** Non-zero if any test FAILS or has ERRORs ## Available Tests diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index 0f8e70cc3a1..0cbf1a18635 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -287,8 +287,8 @@ def run(): test_type = os.getenv("TEST_TYPE", "full") test_labels = json.loads(os.getenv("TEST_LABELS", "[]")) is_benchmark_workflow = str2bool(os.getenv("IS_BENCHMARK_WORKFLOW", "false")) - is_extended_functional_workflow = str2bool( - os.getenv("IS_EXTENDED_FUNCTIONAL_WORKFLOW", "false") + is_extended_functional_tests = str2bool( + os.getenv("IS_EXTENDED_FUNCTIONAL_TESTS", "false") ) logging.info(f"Selecting projects: {project_to_test}") @@ -299,7 +299,7 @@ def run(): # Benchmarks don't use test_type/test_labels (all have total_shards=1, no filtering) logging.info("Using benchmark_matrix only (benchmark tests)") selected_matrix = benchmark_matrix.copy() - elif is_extended_functional_workflow: + elif is_extended_functional_tests: # For extended functional workflow, use ONLY functional_matrix # Extended functional tests don't use test_type/test_labels (all have total_shards=1, no filtering) logging.info("Using functional_matrix only (extended functional tests)") diff --git a/rocm-libraries b/rocm-libraries index 37865a97a6c..34459f66eaf 160000 --- a/rocm-libraries +++ b/rocm-libraries @@ -1 +1 @@ -Subproject commit 37865a97a6c70054a0a65ee710b3ddf7d6fc5678 +Subproject commit 34459f66eaf5ba4b8235adce2ad24926cef8bcbe From 52cae6273cc00de4b060b4a6a9af5ff47bafe024 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Tue, 20 Jan 2026 16:31:31 +0000 Subject: [PATCH 24/28] Fix pre-commit errors Signed-off-by: Lenine Ajagappane --- build_tools/github_actions/extended_tests/functional/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/build_tools/github_actions/extended_tests/functional/README.md b/build_tools/github_actions/extended_tests/functional/README.md index c4af3663707..9ed14bb7450 100644 --- a/build_tools/github_actions/extended_tests/functional/README.md +++ b/build_tools/github_actions/extended_tests/functional/README.md @@ -130,6 +130,7 @@ if __name__ == "__main__": ``` **Required Methods:** + - `run_tests()` → Execute tests and save results - `parse_results()` → Returns `(test_results_list, display_table, num_suites)` - Must use `self.create_test_result(test_name, subtest_name, status, **kwargs)` From fece86049720eaec361be1c7caa9f6255fbfd685 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Mon, 26 Jan 2026 17:47:51 +0000 Subject: [PATCH 25/28] Restore rocm-libraries submodule to match main --- rocm-libraries | 1 + 1 file changed, 1 insertion(+) create mode 160000 rocm-libraries diff --git a/rocm-libraries b/rocm-libraries new file mode 160000 index 00000000000..c1da468ec3e --- /dev/null +++ b/rocm-libraries @@ -0,0 +1 @@ +Subproject commit c1da468ec3ea32d0b9d11a3c644068608ffebe79 From 49a64c000d01c6ba65aeab71b6d6ce8cc6785a85 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Mon, 2 Feb 2026 20:15:55 +0000 Subject: [PATCH 26/28] Improve functional test logging and remove redundancy - Remove write_step_summary wrapper and redundant log file writing - Use display_name consistently in logging Signed-off-by: Lenine Ajagappane --- .github/workflows/ci_linux.yml | 2 +- .github/workflows/ci_windows.yml | 2 +- .../test_extended_functional_tests.yml | 4 +- .../functional/functional_test_matrix.py | 2 +- .../functional/scripts/functional_base.py | 23 +-- .../scripts/test_miopendriver_conv.py | 158 ++++++++---------- 6 files changed, 84 insertions(+), 107 deletions(-) diff --git a/.github/workflows/ci_linux.yml b/.github/workflows/ci_linux.yml index cc015c21083..e0bb3ac3e9a 100644 --- a/.github/workflows/ci_linux.yml +++ b/.github/workflows/ci_linux.yml @@ -119,7 +119,7 @@ jobs: run_linux_extended_functional_tests: needs: [build_portable_linux_artifacts] - name: Run Extended Functional Tests + name: Run Linux Extended Functional Tests # Run extended functional tests if: # - Build succeeded (or using prebuilt artifacts) # - Not expecting failure diff --git a/.github/workflows/ci_windows.yml b/.github/workflows/ci_windows.yml index 3d15d4c2d2b..f273fdac49e 100644 --- a/.github/workflows/ci_windows.yml +++ b/.github/workflows/ci_windows.yml @@ -119,7 +119,7 @@ jobs: run_windows_extended_functional_tests: needs: [build_windows_artifacts] - name: Run Extended Functional Tests + name: Run Windows Extended Functional Tests # Run extended functional tests if: # - Build succeeded (or using prebuilt artifacts) # - Not expecting failure diff --git a/.github/workflows/test_extended_functional_tests.yml b/.github/workflows/test_extended_functional_tests.yml index d71e63820ca..9af85ff9813 100644 --- a/.github/workflows/test_extended_functional_tests.yml +++ b/.github/workflows/test_extended_functional_tests.yml @@ -75,8 +75,8 @@ jobs: IS_EXTENDED_FUNCTIONAL_TESTS: "true" run: python ./build_tools/github_actions/fetch_test_configurations.py - run_tests: - name: 'Test ${{ matrix.components.job_name }}' + run_extended_functional_tests: + name: 'Run Extended Functional Tests ${{ matrix.components.job_name }}' needs: [configure_test_matrix] # skip tests if no test matrix to run if: ${{ needs.configure_test_matrix.outputs.components != '[]' }} diff --git a/build_tools/github_actions/extended_tests/functional/functional_test_matrix.py b/build_tools/github_actions/extended_tests/functional/functional_test_matrix.py index 0746b90a645..79e928fef09 100644 --- a/build_tools/github_actions/extended_tests/functional/functional_test_matrix.py +++ b/build_tools/github_actions/extended_tests/functional/functional_test_matrix.py @@ -26,7 +26,7 @@ def _get_script_path(script_name: str) -> str: "fetch_artifact_args": "--miopen --tests", "timeout_minutes": 30, "test_script": f"python {_get_script_path('test_miopendriver_conv.py')}", - # TODO(lajagapp): Add windows support + # TODO(lajagapp): Add windows support (https://github.com/ROCm/TheRock/issues/3207) "platform": ["linux"], "total_shards": 1, }, diff --git a/build_tools/github_actions/extended_tests/functional/scripts/functional_base.py b/build_tools/github_actions/extended_tests/functional/scripts/functional_base.py index 0786ceff854..0c764045268 100644 --- a/build_tools/github_actions/extended_tests/functional/scripts/functional_base.py +++ b/build_tools/github_actions/extended_tests/functional/scripts/functional_base.py @@ -208,7 +208,7 @@ def upload_results( Returns: True if upload successful, False otherwise """ - log.info("Uploading Results to API") + log.info("Uploading Functional Tests Results to API") success = self.client.upload_results( test_name=f"{self.test_name}_functional", test_results=test_results, @@ -234,20 +234,6 @@ def upload_results( return success - def write_step_summary( - self, stats: Dict[str, Any], summary_table: PrettyTable - ) -> None: - """Write results to GitHub Actions step summary. - - Args: - stats: Test statistics dictionary - summary_table: PrettyTable with summary statistics - """ - gha_append_step_summary( - f"## {self.display_name} - Functional Test Results\n\n" - f"```\n{summary_table}\n```\n" - ) - def run(self) -> None: """Execute functional test workflow. @@ -300,9 +286,12 @@ def run(self) -> None: # Write to GitHub Actions step summary try: - self.write_step_summary(stats, summary_table) + gha_append_step_summary( + f"## {self.display_name} - Functional Test Results\n\n" + f"```\n{summary_table}\n```\n" + ) except Exception as e: - log.warning(f"Could not write GitHub Actions summary: {e}") + log.error(f"Could not write GitHub Actions summary: {e}") # Raise exception if tests failed if stats["overall_status"] != "PASS": diff --git a/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py b/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py index f37857e1001..6171cfeb739 100644 --- a/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py +++ b/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py @@ -29,7 +29,6 @@ def __init__(self): test_name="miopen_driver_conv", display_name="MIOpen Driver Convolution" ) - self.log_file = self.script_dir / "miopendriver_conv.log" self.results_json = self.script_dir / "miopendriver_conv_results.json" # Load test configurations from JSON @@ -48,7 +47,7 @@ def __init__(self): self.gpu_specific_flags = config.get("gpu_specific_flags", {}) def run_tests(self) -> None: - """Run MIOpen driver convolution tests and save output to log file and results to JSON.""" + """Run MIOpen driver convolution tests and save results to JSON.""" log.info(f"Running {self.display_name} Tests") # Detect GPU architecture using HardwareDetector @@ -57,7 +56,6 @@ def run_tests(self) -> None: gfx_id = detector.get_gpu_architecture() log.info(f"Detected GPU: {gfx_id}") - # Use Path object consistently miopen_driver = Path(self.therock_bin_dir) / "MIOpenDriver" if not miopen_driver.exists(): raise TestExecutionError( @@ -68,97 +66,87 @@ def run_tests(self) -> None: # Calculate total number of tests for progress indicator total_tests = sum(len(self.tests_cmd[suite]) for suite in self.tests_list) current_test = 0 - log.info(f"Total tests to run: {total_tests}") + log.info(f"Total {self.display_name} tests to run: {total_tests}") # Store results as we execute all_results = [] - with open(self.log_file, "w+") as f: - for test_suite in self.tests_list: - log.info(f"Running test suite: {test_suite}") - f.write(f"\n{'='*80}\n") - f.write(f"Test Suite: {test_suite}\n") - f.write(f"{'='*80}\n\n") - - for i, cmd_str in enumerate(self.tests_cmd[test_suite], 1): - current_test += 1 - - # Build full command with MIOpenDriver path - full_cmd = f"{miopen_driver} {cmd_str}" - - # Add GPU-specific flags if needed - if ( - "Backward_Conv" in test_suite - and gfx_id in self.gpu_specific_flags - ): - backward_flags = self.gpu_specific_flags[gfx_id].get( - "backward_flags", "" - ) - if backward_flags: - full_cmd = f"{full_cmd} {backward_flags}" - - cmd = shlex.split(full_cmd) - - # Progress indicator - test_case_name = f"{test_suite}_case{i}" - log.info(f"[{current_test}/{total_tests}] Running {test_case_name}") - log.info(f"++ Exec [{self.therock_dir}]$ {shlex.join(cmd)}") - f.write(f"\nCommand: {shlex.join(cmd)}\n") - f.write(f"-" * 80 + "\n") - - return_code = None - error_message = None - - try: - process = subprocess.Popen( - cmd, - cwd=self.therock_dir, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - ) - - for line in process.stdout: - log.info(line.strip()) - f.write(line) - - process.wait() - return_code = process.returncode - f.write(f"\nReturn code: {return_code}\n\n") - - except Exception as e: - log.error(f"Error running command: {e}") - f.write(f"ERROR: {e}\n\n") - error_message = str(e) - return_code = -1 - - # Determine status based on return code - status = "PASS" if return_code == 0 else "FAIL" - - # Store result immediately - result = { - "test_suite": test_suite, - "test_case": test_case_name, - "command": cmd_str, - "command_index": i, - "return_code": return_code, - "status": status, - } - if error_message: - result["error"] = error_message - - all_results.append(result) - log.info( - f"[{current_test}/{total_tests}] {test_case_name}: {status}" + for test_suite in self.tests_list: + log.info(f"Running test suite: {test_suite}") + + for i, cmd_str in enumerate(self.tests_cmd[test_suite], 1): + current_test += 1 + + # Build full command with MIOpenDriver path + full_cmd = f"{miopen_driver} {cmd_str}" + + # Add GPU-specific flags if needed + if ( + "Backward_Conv" in test_suite + and gfx_id in self.gpu_specific_flags + ): + backward_flags = self.gpu_specific_flags[gfx_id].get( + "backward_flags", "" ) + full_cmd = f"{full_cmd} {backward_flags}" + + cmd = shlex.split(full_cmd) + + # Progress indicator + test_case_name = f"{test_suite}_case{i}" + log.info(f"[{current_test}/{total_tests}] Running {test_case_name}") + log.info(f"++ Exec [{self.therock_dir}]$ {shlex.join(cmd)}") + + return_code = None + error_message = None + + try: + process = subprocess.Popen( + cmd, + cwd=self.therock_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + + for line in process.stdout: + log.info(line.strip()) + + process.wait() + return_code = process.returncode + + except Exception as e: + log.error(f"Error running command: {e}") + error_message = str(e) + return_code = -1 + + # Determine status based on return code + status = "PASS" if return_code == 0 else "FAIL" + + # Store result immediately + result = { + "test_suite": test_suite, + "test_case": test_case_name, + "command": cmd_str, + "command_index": i, + "return_code": return_code, + "status": status, + } + if error_message: + result["error"] = error_message + + all_results.append(result) + log.info( + f"[{current_test}/{total_tests}] {test_case_name}: {status}" + ) # Write all results to JSON file with open(self.results_json, "w") as f: json.dump(all_results, f, indent=2) - log.info(f"Results saved to {self.results_json}") - log.info("MIOpenDriver convolution test execution complete") + log.info(f"{self.display_name} results saved to {self.results_json}") + log.info(f"{self.display_name} test execution complete") def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: """Parse test results from JSON file. @@ -166,7 +154,7 @@ def parse_results(self) -> Tuple[List[Dict[str, Any]], PrettyTable, int]: Returns: tuple: (test_results list, detailed PrettyTable, number of test suites) """ - log.info("Parsing Results") + log.info(f"Parsing {self.display_name} Results") # Setup detailed table - show each individual test case detailed_table = PrettyTable() From 7329da880501d482691cde96d7c36b5fa866046b Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Mon, 2 Feb 2026 20:43:08 +0000 Subject: [PATCH 27/28] Rename functional test workflow for clarity - Rename test_extended_functional_tests.yml to test_functional_tests.yml - Update all variables and references in workflow files and documentation Signed-off-by: Lenine Ajagappane --- .github/workflows/ci_linux.yml | 14 +++++++------- .github/workflows/ci_nightly.yml | 4 ++-- .github/workflows/ci_windows.yml | 14 +++++++------- ...ctional_tests.yml => test_functional_tests.yml} | 10 +++++----- .../github_actions/extended_tests/README.md | 8 ++++---- .../extended_tests/functional/README.md | 2 +- .../github_actions/fetch_test_configurations.py | 12 ++++++------ 7 files changed, 32 insertions(+), 32 deletions(-) rename .github/workflows/{test_extended_functional_tests.yml => test_functional_tests.yml} (93%) diff --git a/.github/workflows/ci_linux.yml b/.github/workflows/ci_linux.yml index e0bb3ac3e9a..5399207dbd0 100644 --- a/.github/workflows/ci_linux.yml +++ b/.github/workflows/ci_linux.yml @@ -22,7 +22,7 @@ on: benchmark_runs_on: type: string default: "" - run_extended_functional_tests: + run_functional_tests: type: boolean default: false expect_failure: @@ -117,14 +117,14 @@ jobs: test_runs_on: ${{ inputs.benchmark_runs_on }} artifact_run_id: ${{ inputs.artifact_run_id }} - run_linux_extended_functional_tests: + run_linux_functional_tests: needs: [build_portable_linux_artifacts] - name: Run Linux Extended Functional Tests - # Run extended functional tests if: + name: Run Linux Functional Tests + # Run functional tests if: # - Build succeeded (or using prebuilt artifacts) # - Not expecting failure # - Test runner is available (test_runs_on is set) - # - Extended functional flag is enabled (only set to true in ci_nightly.yml) + # - Functional flag is enabled (only set to true in ci_nightly.yml) if: >- ${{ !failure() && @@ -135,9 +135,9 @@ jobs: ) && inputs.expect_failure == false && inputs.test_runs_on != '' && - inputs.run_extended_functional_tests == true + inputs.run_functional_tests == true }} - uses: ./.github/workflows/test_extended_functional_tests.yml + uses: ./.github/workflows/test_functional_tests.yml secrets: inherit with: artifact_group: ${{ inputs.artifact_group }} diff --git a/.github/workflows/ci_nightly.yml b/.github/workflows/ci_nightly.yml index 2fa02cb9972..b27938b00a6 100644 --- a/.github/workflows/ci_nightly.yml +++ b/.github/workflows/ci_nightly.yml @@ -75,7 +75,7 @@ jobs: artifact_group: ${{ matrix.variant.artifact_group }} test_runs_on: ${{ matrix.variant.test-runs-on }} benchmark_runs_on: ${{ matrix.variant.benchmark-runs-on }} - run_extended_functional_tests: true + run_functional_tests: true build_variant_label: ${{ matrix.variant.build_variant_label }} build_variant_suffix: ${{ matrix.variant.build_variant_suffix }} build_variant_cmake_preset: ${{ matrix.variant.build_variant_cmake_preset }} @@ -109,7 +109,7 @@ jobs: artifact_group: ${{ matrix.variant.artifact_group }} test_runs_on: ${{ matrix.variant.test-runs-on }} benchmark_runs_on: ${{ matrix.variant.benchmark-runs-on }} - run_extended_functional_tests: true + run_functional_tests: true build_variant_label: ${{ matrix.variant.build_variant_label }} build_variant_suffix: ${{ matrix.variant.build_variant_suffix }} build_variant_cmake_preset: ${{ matrix.variant.build_variant_cmake_preset }} diff --git a/.github/workflows/ci_windows.yml b/.github/workflows/ci_windows.yml index f273fdac49e..2b0c6589c06 100644 --- a/.github/workflows/ci_windows.yml +++ b/.github/workflows/ci_windows.yml @@ -22,7 +22,7 @@ on: benchmark_runs_on: type: string default: "" - run_extended_functional_tests: + run_functional_tests: type: boolean default: false expect_failure: @@ -117,14 +117,14 @@ jobs: test_runs_on: ${{ inputs.benchmark_runs_on }} artifact_run_id: ${{ inputs.artifact_run_id }} - run_windows_extended_functional_tests: + run_windows_functional_tests: needs: [build_windows_artifacts] - name: Run Windows Extended Functional Tests - # Run extended functional tests if: + name: Run Windows Functional Tests + # Run functional tests if: # - Build succeeded (or using prebuilt artifacts) # - Not expecting failure # - Test runner is available (test_runs_on is set) - # - Extended functional flag is enabled (only set to true in ci_nightly.yml) + # - Functional flag is enabled (only set to true in ci_nightly.yml) if: >- ${{ !failure() && @@ -135,9 +135,9 @@ jobs: ) && inputs.expect_failure == false && inputs.test_runs_on != '' && - inputs.run_extended_functional_tests == true + inputs.run_functional_tests == true }} - uses: ./.github/workflows/test_extended_functional_tests.yml + uses: ./.github/workflows/test_functional_tests.yml secrets: inherit with: artifact_group: ${{ inputs.artifact_group }} diff --git a/.github/workflows/test_extended_functional_tests.yml b/.github/workflows/test_functional_tests.yml similarity index 93% rename from .github/workflows/test_extended_functional_tests.yml rename to .github/workflows/test_functional_tests.yml index 9af85ff9813..da556a4447b 100644 --- a/.github/workflows/test_extended_functional_tests.yml +++ b/.github/workflows/test_functional_tests.yml @@ -1,4 +1,4 @@ -name: Run Extended Functional Tests +name: Run Functional Tests on: workflow_dispatch: @@ -37,7 +37,7 @@ permissions: jobs: configure_test_matrix: - name: "Configure extended functional test matrix" + name: "Configure functional test matrix" # if there is a test machine available if: ${{ inputs.test_runs_on != '' }} runs-on: ${{ inputs.test_runs_on }} @@ -72,11 +72,11 @@ jobs: env: ARTIFACT_GROUP: ${{ inputs.artifact_group }} AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} - IS_EXTENDED_FUNCTIONAL_TESTS: "true" + IS_FUNCTIONAL_TESTS: "true" run: python ./build_tools/github_actions/fetch_test_configurations.py - run_extended_functional_tests: - name: 'Run Extended Functional Tests ${{ matrix.components.job_name }}' + run_functional_tests: + name: 'Run Functional Tests ${{ matrix.components.job_name }}' needs: [configure_test_matrix] # skip tests if no test matrix to run if: ${{ needs.configure_test_matrix.outputs.components != '[]' }} diff --git a/build_tools/github_actions/extended_tests/README.md b/build_tools/github_actions/extended_tests/README.md index 74d32fe0e7c..514bd3e7201 100644 --- a/build_tools/github_actions/extended_tests/README.md +++ b/build_tools/github_actions/extended_tests/README.md @@ -135,8 +135,8 @@ ci_nightly.yml ├─ test_benchmarks ───────────────────┤ │ └─ Benchmark tests │ │ │ - └─ test_extended_functional_tests ──────┘ - └─ Extended functional tests + └─ test_functional_tests ─────────────┘ + └─ Functional tests ``` **Workflow Files:** @@ -144,14 +144,14 @@ ci_nightly.yml - `.github/workflows/ci_nightly.yml` - Nightly CI orchestration - `.github/workflows/ci_linux.yml` / `ci_windows.yml` - Platform-specific CI logic - `.github/workflows/test_benchmarks.yml` - Benchmark test execution (uses `benchmark_runs_on`) -- `.github/workflows/test_extended_functional_tests.yml` - Extended functional test execution (uses `test_runs_on`) +- `.github/workflows/test_functional_tests.yml` - Functional test execution (uses `test_runs_on`) - `.github/workflows/test_artifacts.yml` - Component test execution (uses `test_runs_on`) **Key Differences:** - **Component Tests**: Run on all PRs (smoke) and nightly (full), use regular runners - **Benchmark Tests**: Run only on nightly, use dedicated performance runners (`benchmark_runs_on`) -- **Extended Functional Tests**: Run only on nightly, use regular runners (`test_runs_on`), controlled by `run_extended_functional_tests` flag +- **Functional Tests**: Run only on nightly, use regular runners (`test_runs_on`), controlled by `run_functional_tests` flag ## Architecture diff --git a/build_tools/github_actions/extended_tests/functional/README.md b/build_tools/github_actions/extended_tests/functional/README.md index 9ed14bb7450..afe720ef2c7 100644 --- a/build_tools/github_actions/extended_tests/functional/README.md +++ b/build_tools/github_actions/extended_tests/functional/README.md @@ -10,7 +10,7 @@ Functional tests validate **correctness and behavior**: - **Result Types:** PASS / FAIL / ERROR / SKIP - **Validation:** Output correctness, API contracts, expected behavior -- **CI Execution:** Nightly CI only (controlled by `run_extended_functional_tests` flag) +- **CI Execution:** Nightly CI only (controlled by `run_functional_tests` flag) - **Exit Code:** Non-zero if any test FAILS or has ERRORs ## Available Tests diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index f81a2a9e839..5489b256487 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -304,8 +304,8 @@ def run(): test_type = os.getenv("TEST_TYPE", "full") test_labels = json.loads(os.getenv("TEST_LABELS") or "[]") is_benchmark_workflow = str2bool(os.getenv("IS_BENCHMARK_WORKFLOW", "false")) - is_extended_functional_tests = str2bool( - os.getenv("IS_EXTENDED_FUNCTIONAL_TESTS", "false") + is_functional_tests = str2bool( + os.getenv("IS_FUNCTIONAL_TESTS", "false") ) logging.info(f"Selecting projects: {projects_to_test}") @@ -316,10 +316,10 @@ def run(): # Benchmarks don't use test_type/test_labels (all have total_shards=1, no filtering) logging.info("Using benchmark_matrix only (benchmark tests)") selected_matrix = benchmark_matrix.copy() - elif is_extended_functional_tests: - # For extended functional workflow, use ONLY functional_matrix - # Extended functional tests don't use test_type/test_labels (all have total_shards=1, no filtering) - logging.info("Using functional_matrix only (extended functional tests)") + elif is_functional_tests: + # For functional workflow, use ONLY functional_matrix + # Functional tests don't use test_type/test_labels (all have total_shards=1, no filtering) + logging.info("Using functional_matrix only (functional tests)") selected_matrix = functional_matrix.copy() else: # For regular workflow, use ONLY test_matrix From f58d124cad941fe436167805d79f7c65c700acd2 Mon Sep 17 00:00:00 2001 From: Lenine Ajagappane Date: Mon, 2 Feb 2026 20:48:11 +0000 Subject: [PATCH 28/28] Fix pre-commit errors Signed-off-by: Lenine Ajagappane --- .../functional/scripts/test_miopendriver_conv.py | 9 ++------- build_tools/github_actions/fetch_test_configurations.py | 4 +--- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py b/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py index 6171cfeb739..1c006f7c49a 100644 --- a/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py +++ b/build_tools/github_actions/extended_tests/functional/scripts/test_miopendriver_conv.py @@ -81,10 +81,7 @@ def run_tests(self) -> None: full_cmd = f"{miopen_driver} {cmd_str}" # Add GPU-specific flags if needed - if ( - "Backward_Conv" in test_suite - and gfx_id in self.gpu_specific_flags - ): + if "Backward_Conv" in test_suite and gfx_id in self.gpu_specific_flags: backward_flags = self.gpu_specific_flags[gfx_id].get( "backward_flags", "" ) @@ -137,9 +134,7 @@ def run_tests(self) -> None: result["error"] = error_message all_results.append(result) - log.info( - f"[{current_test}/{total_tests}] {test_case_name}: {status}" - ) + log.info(f"[{current_test}/{total_tests}] {test_case_name}: {status}") # Write all results to JSON file with open(self.results_json, "w") as f: diff --git a/build_tools/github_actions/fetch_test_configurations.py b/build_tools/github_actions/fetch_test_configurations.py index 5489b256487..45d1a3129fb 100644 --- a/build_tools/github_actions/fetch_test_configurations.py +++ b/build_tools/github_actions/fetch_test_configurations.py @@ -304,9 +304,7 @@ def run(): test_type = os.getenv("TEST_TYPE", "full") test_labels = json.loads(os.getenv("TEST_LABELS") or "[]") is_benchmark_workflow = str2bool(os.getenv("IS_BENCHMARK_WORKFLOW", "false")) - is_functional_tests = str2bool( - os.getenv("IS_FUNCTIONAL_TESTS", "false") - ) + is_functional_tests = str2bool(os.getenv("IS_FUNCTIONAL_TESTS", "false")) logging.info(f"Selecting projects: {projects_to_test}")