Add optional enhanced metadata fields to test categories

projects/hipblas/clients/gtest/test_categories.yaml

@@ -4,13 +4,50 @@
 test_categories:
   level1:
     description: "Auxiliary functions (set/get operations, utility functions)"
+
+    # Optional: Context for developers and AI tools
+    notes: |
+      Fast-running sanity checks for auxiliary functions (set/get operations,
+      mode handling, matrix/vector transfers).
+
+      Run these when:
+      - Auxiliary functions change (library/src/auxiliary.cpp, library/src/handle.cpp)
+      - API initialization/cleanup changes
+      - Basic data movement functionality changes
+
+      These are foundational tests that should pass before running more complex test suites.
+
+    # Optional: Map to source code (enables test-to-code lookup)
+    source_coverage:
+      - "library/src/auxiliary.cpp"
+      - "library/src/handle.cpp"
+
+    # Optional: API functions tested (enables API coverage analysis)
+    api_coverage:
+      - "hipblasCreate"
+      - "hipblasDestroy"
+      - "hipblasSetPointerMode"
+      - "hipblasGetPointerMode"
+      - "hipblasSetVector"
+      - "hipblasGetVector"
+      - "hipblasSetMatrix"
+      - "hipblasGetMatrix"
+
+    # Optional: Semantic tags for classification
+    feature_tags:
+      - "initialization"
+      - "auxiliary"
+      - "fast"
+      - "foundational"
+
+    # Standard fields (unchanged from base PR)
     test_patterns:
       - "*auxiliary*"
       - "*set_get*"
       - "*auxil*"
     test_files:
       - "auxil/auxiliary_gtest.cpp"
-      - "auxil/set_get_mode_gtest.cpp" 
+      - "auxil/set_get_mode_gtest.cpp"
       - "auxil/set_get_matrix_vector_gtest.cpp"
     labels:
       - "auxil"

shared/ctest/README.md

@@ -62,25 +62,25 @@ CMake module providing:
 flowchart TD
     A[Project CMakeLists.txt] -->|include| B[TestCategories.cmake]
     A -->|call| C[apply_test_category_labels<br/>target, yaml, workdir]
-    
+
     C -->|execute_process| E[parse_test_categories.py]
-    
+
     E -->|read| F[test_categories.yaml]
 
     E -->|apply| H[Exclusion Rules]
     E -->|generate| I[CMake Code]
-    
+
     I -->|write to| J[Generated CMake<br/>build/test_categories.cmake]
-    
+
     J -->|add_test, set_tests_properties| K[Test Registration]
-        
+
     K --> M[CTest Execution]
-    
-    
+
+
     M -->|ctest -L category_name| N[Run by Category]
     M -->|ctest -L label| O[Run by Label]
     M -->|ctest --timeout 600| P[Run with Timeout]
-    
+
     style A fill:#e1f5ff
     style E fill:#fff4e1
     style J fill:#e8f5e9
@@ -109,6 +109,68 @@ execution_settings:
     category_name: 600
 ```
 
+### **Enhanced Structure (Optional Fields)**
+
+All fields below are **optional** and can be added incrementally. Teams can use them for richer test documentation and enable future capabilities like AI-assisted test selection:
+
+```yaml
+test_categories:
+  category_name:
+    # Required fields (same as base)
+    description: "Human-readable description"
+    test_patterns: ["*pattern1*", "*pattern2*"]
+    labels: ["label1", "label2"]
+
+    # Optional enhancement fields - add only if useful for your project
+    notes: |
+      Human-readable context about when to run these tests.
+      Can include historical context, gotchas, or guidance for developers and AI tools.
+    source_coverage:
+      - "library/src/file.cpp"
+      - "library/src/module.cpp:function_name"
+    api_coverage:
+      - "apiFunction1"
+      - "apiFunction2"
+    feature_tags:
+      - "performance-critical"
+      - "numerical-stability"
+    dependencies:
+      - "other_category"  # Run this category after dependencies complete
+
+    # Standard fields (from base)
+    exclude: ["*always_exclude*"]
+    exclude_windows: ["*linux_only*"]
+    exclude_linux: ["*windows_only*"]
+
+# Optional: Top-level context for AI/LLM tools
+llm_context:
+  code_to_test_mapping_guidelines: |
+    Guidance for AI tools on how to map code changes to test categories.
+    Projects can use this for AI-assisted test selection.
+
+execution_settings:
+  default_timeout: 300
+  category_timeouts:
+    category_name: 600
+```
+
+**Optional Field Descriptions:**
+
+| Field | Purpose | Example Use |
+|-------|---------|-------------|
+| `notes` | Free-form text for context and documentation | "Run when epilogue changes. See bug #8765 for history" |
+| `source_coverage` | Source files/functions tested by this category | `["library/src/gemm.cpp:matmul_kernel"]` |
+| `api_coverage` | API functions tested by this category | `["hipblasLtMatmul", "hipblasLtMatmulAlgo"]` |
+| `feature_tags` | Semantic tags for classification and filtering | `["performance-critical", "mixed-precision"]` |
+| `dependencies` | Test categories that should run first | `["auxiliary"]` - run auxiliary tests before this category |
+| `llm_context` | Top-level guidance for AI-assisted workflows | Instructions for AI tools on test selection logic |
+
+**Key Points:**
+- All enhancement fields are **optional** - teams can ignore them entirely ✅
+- Projects can adopt incrementally: start with just `notes`, add more later ✅
+- Parser gracefully ignores unknown fields - no code changes needed ✅
+- Enables richer test documentation and future AI-assisted workflows ✅
+
 ### **Exclusion Hierarchy**
 
 The parser applies exclusions in this order:
@@ -142,7 +204,7 @@ include(${ROCM_LIBRARIES_ROOT}/shared/ctest/TestCategories.cmake)
 
 if(BUILD_TESTING)
     enable_testing()
-    
+
     # Apply test categorization
     apply_test_category_labels(
         myproject-test                               # Test executable name
@@ -178,5 +240,3 @@ Projects currently using this architecture:
 
 - **hipblas** - [test_categories.yaml](../../projects/hipblas/clients/gtest/test_categories.yaml) | [CMakeLists.txt](../../projects/hipblas/clients/gtest/CMakeLists.txt)
 - **rocfft** - [test_categories.yaml](../../projects/rocfft/clients/tests/test_categories.yaml) | [CMakeLists.txt](../../projects/rocfft/clients/tests/CMakeLists.txt)
-
-

shared/ctest/parse_test_categories.py

@@ -4,82 +4,88 @@
 import platform
 import argparse
 
+
 def main():
     parser = argparse.ArgumentParser(
-        description='Parse test_categories.yaml and generate CMake test definitions'
+        description="Parse test_categories.yaml and generate CMake test definitions"
+    )
+    parser.add_argument("yaml_file", help="Path to the test_categories.yaml file")
+    parser.add_argument(
+        "target_name", help="Name of the test target (e.g., hipblas-test)"
     )
-    parser.add_argument('yaml_file', help='Path to the test_categories.yaml file')
-    parser.add_argument('target_name', help='Name of the test target (e.g., hipblas-test)')
-    parser.add_argument('working_dir', help='Working directory for running tests')
-
+    parser.add_argument("working_dir", help="Working directory for running tests")
+
     args = parser.parse_args()
-    
+
     yaml_file = args.yaml_file
     target_name = args.target_name
     working_dir = args.working_dir
 
     try:
-        with open(yaml_file, 'r') as f:
+        with open(yaml_file, "r") as f:
             config = yaml.safe_load(f)
     except Exception as e:
-        print(f'Error loading YAML: {e}', file=sys.stderr)
+        print(f"Error loading YAML: {e}", file=sys.stderr)
         sys.exit(1)
 
-    categories = config.get('test_categories', {})
-    timeouts = config.get('execution_settings', {}).get('category_timeouts', {})
+    categories = config.get("test_categories", {})
+    timeouts = config.get("execution_settings", {}).get("category_timeouts", {})
 
     # Detect OS
-    is_windows = platform.system() == 'Windows'
-    is_linux = platform.system() == 'Linux'
+    is_windows = platform.system() == "Windows"
+    is_linux = platform.system() == "Linux"
 
-    print('# Generated CMake code for test categories')
-    print(f'# Detected OS: {platform.system()}')
+    print("# Generated CMake code for test categories")
+    print(f"# Detected OS: {platform.system()}")
 
+    # Parse each test category
+    # Note: Optional enhancement fields (notes, source_coverage, api_coverage, feature_tags,
+    # dependencies, llm_context) are preserved in YAML but not processed by this parser.
+    # They're available for documentation and future tooling (e.g., AI-assisted test selection).
     for category_name, category_info in categories.items():
-        patterns = category_info.get('test_patterns', [])
-        labels = category_info.get('labels', [])
-        exclude = category_info.get('exclude', [])
+        patterns = category_info.get("test_patterns", [])
+        labels = category_info.get("labels", [])
+        exclude = category_info.get("exclude", [])
         if exclude == None:
             exclude = []
-        
+
         # Add OS-specific exclusions
         if is_windows:
-            exclude_windows = category_info.get('exclude_windows', [])
+            exclude_windows = category_info.get("exclude_windows", [])
             if exclude_windows:
                 exclude.extend(exclude_windows)
-        
+
         if is_linux:
-            exclude_linux = category_info.get('exclude_linux', [])
+            exclude_linux = category_info.get("exclude_linux", [])
             if exclude_linux:
                 exclude.extend(exclude_linux)
-        
+
         timeout = timeouts.get(category_name, 300)
-        print(f'# Category: {category_name}')
+        print(f"# Category: {category_name}")
         print(f'# Description: {category_info.get("description", "")}')
-        patterns[:]=[x for x in patterns if x not in exclude]
-
+        patterns[:] = [x for x in patterns if x not in exclude]
 
-        pattern_string=""
+        pattern_string = ""
         for pattern in patterns:
-          pattern_string = pattern_string+':'+pattern
-        pattern_string='"'+pattern_string[1:]+'"'
+            pattern_string = pattern_string + ":" + pattern
+        pattern_string = '"' + pattern_string[1:] + '"'
 
         label_string = ""
         for label in labels:
-          label_string = label_string+';'+label
-        label_string = '"'+label_string[1:]+'"'
+            label_string = label_string + ";" + label
+        label_string = '"' + label_string[1:] + '"'
         print("add_test(")
-        print(f'  NAME {target_name}-{category_name}-suite')
-        print(f'  COMMAND {target_name} --gtest_filter={pattern_string}')
-        print( f"  WORKING_DIRECTORY {working_dir}")
+        print(f"  NAME {target_name}-{category_name}-suite")
+        print(f"  COMMAND {target_name} --gtest_filter={pattern_string}")
+        print(f"  WORKING_DIRECTORY {working_dir}")
         print(")")
 
         print(f"set_tests_properties({target_name}-{category_name}-suite PROPERTIES")
         print(f"  LABELS {label_string}")
         print(f"  TIMEOUT {timeout}")
         print(")")
         print()
-if __name__ == '__main__':
-    main()
 
 
+if __name__ == "__main__":
+    main()