From 874c3f5b7d0324d198c36b5e61898b28aeecb238 Mon Sep 17 00:00:00 2001 From: Tony Davis Date: Thu, 15 Jan 2026 17:18:17 -0500 Subject: [PATCH] Add optional enhanced metadata fields to test categories - Add documentation for optional fields (notes, source_coverage, api_coverage, feature_tags, dependencies) - Add clarifying comment to parser (no logic changes - YAML naturally ignores unknown fields) - Enhance level1 category as example, keep level2-level6 minimal to show backward compatibility All fields are optional and backward compatible. Teams can adopt incrementally. Enables richer test documentation and future AI-assisted test selection. --- .../clients/gtest/test_categories.yaml | 39 ++++++++- shared/ctest/README.md | 82 ++++++++++++++++--- shared/ctest/parse_test_categories.py | 76 +++++++++-------- 3 files changed, 150 insertions(+), 47 deletions(-) diff --git a/projects/hipblas/clients/gtest/test_categories.yaml b/projects/hipblas/clients/gtest/test_categories.yaml index 358e4a44c16c..3e3f5907d4c2 100644 --- a/projects/hipblas/clients/gtest/test_categories.yaml +++ b/projects/hipblas/clients/gtest/test_categories.yaml @@ -4,13 +4,50 @@ test_categories: level1: description: "Auxiliary functions (set/get operations, utility functions)" + + # Optional: Context for developers and AI tools + notes: | + Fast-running sanity checks for auxiliary functions (set/get operations, + mode handling, matrix/vector transfers). + + Run these when: + - Auxiliary functions change (library/src/auxiliary.cpp, library/src/handle.cpp) + - API initialization/cleanup changes + - Basic data movement functionality changes + + These are foundational tests that should pass before running more complex test suites. + + # Optional: Map to source code (enables test-to-code lookup) + source_coverage: + - "library/src/auxiliary.cpp" + - "library/src/handle.cpp" + + # Optional: API functions tested (enables API coverage analysis) + api_coverage: + - "hipblasCreate" + - "hipblasDestroy" + - "hipblasSetPointerMode" + - "hipblasGetPointerMode" + - "hipblasSetVector" + - "hipblasGetVector" + - "hipblasSetMatrix" + - "hipblasGetMatrix" + + # Optional: Semantic tags for classification + feature_tags: + - "initialization" + - "auxiliary" + - "fast" + - "foundational" + + # Standard fields (unchanged from base PR) test_patterns: - "*auxiliary*" - "*set_get*" - "*auxil*" test_files: - "auxil/auxiliary_gtest.cpp" - - "auxil/set_get_mode_gtest.cpp" + - "auxil/set_get_mode_gtest.cpp" - "auxil/set_get_matrix_vector_gtest.cpp" labels: - "auxil" diff --git a/shared/ctest/README.md b/shared/ctest/README.md index b15609b732de..0a4e1b2da342 100644 --- a/shared/ctest/README.md +++ b/shared/ctest/README.md @@ -62,25 +62,25 @@ CMake module providing: flowchart TD A[Project CMakeLists.txt] -->|include| B[TestCategories.cmake] A -->|call| C[apply_test_category_labels
target, yaml, workdir] - + C -->|execute_process| E[parse_test_categories.py] - + E -->|read| F[test_categories.yaml] E -->|apply| H[Exclusion Rules] E -->|generate| I[CMake Code] - + I -->|write to| J[Generated CMake
build/test_categories.cmake] - + J -->|add_test, set_tests_properties| K[Test Registration] - + K --> M[CTest Execution] - - + + M -->|ctest -L category_name| N[Run by Category] M -->|ctest -L label| O[Run by Label] M -->|ctest --timeout 600| P[Run with Timeout] - + style A fill:#e1f5ff style E fill:#fff4e1 style J fill:#e8f5e9 @@ -109,6 +109,68 @@ execution_settings: category_name: 600 ``` +### **Enhanced Structure (Optional Fields)** + +All fields below are **optional** and can be added incrementally. Teams can use them for richer test documentation and enable future capabilities like AI-assisted test selection: + +```yaml +test_categories: + category_name: + # Required fields (same as base) + description: "Human-readable description" + test_patterns: ["*pattern1*", "*pattern2*"] + labels: ["label1", "label2"] + + # Optional enhancement fields - add only if useful for your project + notes: | + Human-readable context about when to run these tests. + Can include historical context, gotchas, or guidance for developers and AI tools. + source_coverage: + - "library/src/file.cpp" + - "library/src/module.cpp:function_name" + api_coverage: + - "apiFunction1" + - "apiFunction2" + feature_tags: + - "performance-critical" + - "numerical-stability" + dependencies: + - "other_category" # Run this category after dependencies complete + + # Standard fields (from base) + exclude: ["*always_exclude*"] + exclude_windows: ["*linux_only*"] + exclude_linux: ["*windows_only*"] + +# Optional: Top-level context for AI/LLM tools +llm_context: + code_to_test_mapping_guidelines: | + Guidance for AI tools on how to map code changes to test categories. + Projects can use this for AI-assisted test selection. + +execution_settings: + default_timeout: 300 + category_timeouts: + category_name: 600 +``` + +**Optional Field Descriptions:** + +| Field | Purpose | Example Use | +|-------|---------|-------------| +| `notes` | Free-form text for context and documentation | "Run when epilogue changes. See bug #8765 for history" | +| `source_coverage` | Source files/functions tested by this category | `["library/src/gemm.cpp:matmul_kernel"]` | +| `api_coverage` | API functions tested by this category | `["hipblasLtMatmul", "hipblasLtMatmulAlgo"]` | +| `feature_tags` | Semantic tags for classification and filtering | `["performance-critical", "mixed-precision"]` | +| `dependencies` | Test categories that should run first | `["auxiliary"]` - run auxiliary tests before this category | +| `llm_context` | Top-level guidance for AI-assisted workflows | Instructions for AI tools on test selection logic | + +**Key Points:** +- All enhancement fields are **optional** - teams can ignore them entirely ✅ +- Projects can adopt incrementally: start with just `notes`, add more later ✅ +- Parser gracefully ignores unknown fields - no code changes needed ✅ +- Enables richer test documentation and future AI-assisted workflows ✅ + ### **Exclusion Hierarchy** The parser applies exclusions in this order: @@ -142,7 +204,7 @@ include(${ROCM_LIBRARIES_ROOT}/shared/ctest/TestCategories.cmake) if(BUILD_TESTING) enable_testing() - + # Apply test categorization apply_test_category_labels( myproject-test # Test executable name @@ -178,5 +240,3 @@ Projects currently using this architecture: - **hipblas** - [test_categories.yaml](../../projects/hipblas/clients/gtest/test_categories.yaml) | [CMakeLists.txt](../../projects/hipblas/clients/gtest/CMakeLists.txt) - **rocfft** - [test_categories.yaml](../../projects/rocfft/clients/tests/test_categories.yaml) | [CMakeLists.txt](../../projects/rocfft/clients/tests/CMakeLists.txt) - - diff --git a/shared/ctest/parse_test_categories.py b/shared/ctest/parse_test_categories.py index 422179d9fb4b..2861c8fe2455 100644 --- a/shared/ctest/parse_test_categories.py +++ b/shared/ctest/parse_test_categories.py @@ -4,74 +4,80 @@ import platform import argparse + def main(): parser = argparse.ArgumentParser( - description='Parse test_categories.yaml and generate CMake test definitions' + description="Parse test_categories.yaml and generate CMake test definitions" + ) + parser.add_argument("yaml_file", help="Path to the test_categories.yaml file") + parser.add_argument( + "target_name", help="Name of the test target (e.g., hipblas-test)" ) - parser.add_argument('yaml_file', help='Path to the test_categories.yaml file') - parser.add_argument('target_name', help='Name of the test target (e.g., hipblas-test)') - parser.add_argument('working_dir', help='Working directory for running tests') - + parser.add_argument("working_dir", help="Working directory for running tests") + args = parser.parse_args() - + yaml_file = args.yaml_file target_name = args.target_name working_dir = args.working_dir try: - with open(yaml_file, 'r') as f: + with open(yaml_file, "r") as f: config = yaml.safe_load(f) except Exception as e: - print(f'Error loading YAML: {e}', file=sys.stderr) + print(f"Error loading YAML: {e}", file=sys.stderr) sys.exit(1) - categories = config.get('test_categories', {}) - timeouts = config.get('execution_settings', {}).get('category_timeouts', {}) + categories = config.get("test_categories", {}) + timeouts = config.get("execution_settings", {}).get("category_timeouts", {}) # Detect OS - is_windows = platform.system() == 'Windows' - is_linux = platform.system() == 'Linux' + is_windows = platform.system() == "Windows" + is_linux = platform.system() == "Linux" - print('# Generated CMake code for test categories') - print(f'# Detected OS: {platform.system()}') + print("# Generated CMake code for test categories") + print(f"# Detected OS: {platform.system()}") + # Parse each test category + # Note: Optional enhancement fields (notes, source_coverage, api_coverage, feature_tags, + # dependencies, llm_context) are preserved in YAML but not processed by this parser. + # They're available for documentation and future tooling (e.g., AI-assisted test selection). for category_name, category_info in categories.items(): - patterns = category_info.get('test_patterns', []) - labels = category_info.get('labels', []) - exclude = category_info.get('exclude', []) + patterns = category_info.get("test_patterns", []) + labels = category_info.get("labels", []) + exclude = category_info.get("exclude", []) if exclude == None: exclude = [] - + # Add OS-specific exclusions if is_windows: - exclude_windows = category_info.get('exclude_windows', []) + exclude_windows = category_info.get("exclude_windows", []) if exclude_windows: exclude.extend(exclude_windows) - + if is_linux: - exclude_linux = category_info.get('exclude_linux', []) + exclude_linux = category_info.get("exclude_linux", []) if exclude_linux: exclude.extend(exclude_linux) - + timeout = timeouts.get(category_name, 300) - print(f'# Category: {category_name}') + print(f"# Category: {category_name}") print(f'# Description: {category_info.get("description", "")}') - patterns[:]=[x for x in patterns if x not in exclude] - + patterns[:] = [x for x in patterns if x not in exclude] - pattern_string="" + pattern_string = "" for pattern in patterns: - pattern_string = pattern_string+':'+pattern - pattern_string='"'+pattern_string[1:]+'"' + pattern_string = pattern_string + ":" + pattern + pattern_string = '"' + pattern_string[1:] + '"' label_string = "" for label in labels: - label_string = label_string+';'+label - label_string = '"'+label_string[1:]+'"' + label_string = label_string + ";" + label + label_string = '"' + label_string[1:] + '"' print("add_test(") - print(f' NAME {target_name}-{category_name}-suite') - print(f' COMMAND {target_name} --gtest_filter={pattern_string}') - print( f" WORKING_DIRECTORY {working_dir}") + print(f" NAME {target_name}-{category_name}-suite") + print(f" COMMAND {target_name} --gtest_filter={pattern_string}") + print(f" WORKING_DIRECTORY {working_dir}") print(")") print(f"set_tests_properties({target_name}-{category_name}-suite PROPERTIES") @@ -79,7 +85,7 @@ def main(): print(f" TIMEOUT {timeout}") print(")") print() -if __name__ == '__main__': - main() +if __name__ == "__main__": + main()