diff --git a/scripts/xfails_tracker.py b/scripts/xfails_tracker.py new file mode 100644 index 0000000000..02da8ff737 --- /dev/null +++ b/scripts/xfails_tracker.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +""" +XFails Tracker - Report Generator for pytest.mark.xfail markers + +This script scans the test suite for xfail markers and generates a report +showing the total number of xfails and their reasons. + +Usage: + python scripts/xfails_tracker.py +""" + +import ast +import sys +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import List, Dict, Optional + + +@dataclass +class XFailInfo: + """Information about a single xfail marker.""" + + file_path: str + line_number: int + test_name: str + reason: str + condition: Optional[str] + strict: Optional[bool] + xfail_type: str # 'decorator', 'parameter', or 'runtime' + + def __str__(self): + return f"{self.file_path}:{self.line_number} - {self.test_name}" + + +class XFailCollector(ast.NodeVisitor): + """AST visitor to collect xfail markers from test files.""" + + def __init__(self, file_path: str): + self.file_path = file_path + self.xfails: List[XFailInfo] = [] + self.current_function_name: Optional[str] = None + self.in_test_function: bool = False + + def visit_FunctionDef(self, node: ast.FunctionDef): + """Visit function definitions to check for xfail decorators.""" + # Save the current function name for context + old_function_name = self.current_function_name + old_in_test_function = self.in_test_function + + self.current_function_name = node.name + # Check if this is a test function (starts with "test_" or has @pytest marks) + self.in_test_function = node.name.startswith("test_") or any( + self._has_pytest_mark(dec) for dec in node.decorator_list + ) + + # Check decorators for xfail markers + for decorator in node.decorator_list: + self._check_decorator(decorator, node.name, node.lineno) + + # Continue visiting child nodes (to find parametrize xfails and runtime calls) + self.generic_visit(node) + + # Restore previous function name and test status + self.current_function_name = old_function_name + self.in_test_function = old_in_test_function + + def visit_Call(self, node: ast.Call): + """Visit function calls to find pytest.param with xfail marks and runtime xfails.""" + # Check if this is a pytest.param call + if self._is_pytest_param_call(node): + # Look for marks keyword argument + for keyword in node.keywords: + if keyword.arg == "marks": + self._check_marks_argument(keyword.value, node.lineno) + + # Check if this is a runtime pytest.xfail() call + # Note: We check for xfails even outside test functions, as they can be in helper functions + elif self._is_pytest_xfail_call(node): + self._extract_runtime_xfail(node) + + self.generic_visit(node) + + def _is_pytest_param_call(self, node: ast.Call) -> bool: + """Check if a call is pytest.param.""" + if isinstance(node.func, ast.Attribute): + if node.func.attr == "param" and isinstance(node.func.value, ast.Name): + if node.func.value.id == "pytest": + return True + return False + + def _is_pytest_xfail_call(self, node: ast.Call) -> bool: + """Check if a call is pytest.xfail().""" + if isinstance(node.func, ast.Attribute): + if node.func.attr == "xfail" and isinstance(node.func.value, ast.Name): + if node.func.value.id == "pytest": + return True + return False + + def _has_pytest_mark(self, node) -> bool: + """Check if a decorator is a pytest mark.""" + if isinstance(node, ast.Attribute): + if node.attr in ["parametrize", "xfail", "skip"] and isinstance( + node.value, ast.Attribute + ): + if node.value.attr == "mark" and isinstance(node.value.value, ast.Name): + if node.value.value.id == "pytest": + return True + elif isinstance(node, ast.Call): + return self._has_pytest_mark(node.func) + return False + + def _check_decorator(self, decorator, test_name: str, lineno: int): + """Check if a decorator is an xfail marker.""" + # Handle @pytest.mark.xfail(...) + if isinstance(decorator, ast.Call): + if self._is_xfail_marker(decorator.func): + self._extract_xfail_info(decorator, test_name, lineno, "decorator") + # Handle @pytest.mark.xfail (without parentheses) + elif self._is_xfail_marker(decorator): + self.xfails.append( + XFailInfo( + file_path=self.file_path, + line_number=lineno, + test_name=test_name, + reason="No reason provided", + condition=None, + strict=None, + xfail_type="decorator", + ) + ) + + def _check_marks_argument(self, marks_node, lineno: int): + """Check the marks argument in pytest.param for xfail markers.""" + # marks can be a single marker or a list of markers + if isinstance(marks_node, ast.Call) and self._is_xfail_marker(marks_node.func): + test_name = self.current_function_name or "unknown" + self._extract_xfail_info(marks_node, test_name, lineno, "parameter") + elif isinstance(marks_node, (ast.List, ast.Tuple)): + for mark in marks_node.elts: + if isinstance(mark, ast.Call) and self._is_xfail_marker(mark.func): + test_name = self.current_function_name or "unknown" + self._extract_xfail_info(mark, test_name, lineno, "parameter") + + def _is_xfail_marker(self, node) -> bool: + """Check if a node represents pytest.mark.xfail.""" + if isinstance(node, ast.Attribute): + if node.attr == "xfail" and isinstance(node.value, ast.Attribute): + if ( + node.value.attr == "mark" + and isinstance(node.value.value, ast.Name) + and node.value.value.id == "pytest" + ): + return True + return False + + def _extract_xfail_info( + self, call_node: ast.Call, test_name: str, lineno: int, xfail_type: str + ): + """Extract information from an xfail marker call.""" + reason = "No reason provided" + condition = None + strict = None + + # Check positional arguments (first arg might be condition) + if call_node.args: + condition = self._ast_to_string(call_node.args[0]) + + # Check keyword arguments + for keyword in call_node.keywords: + if keyword.arg == "reason": + reason = self._get_string_value(keyword.value) + elif keyword.arg == "strict": + strict = self._get_bool_value(keyword.value) + elif keyword.arg is None: # **kwargs + pass + + self.xfails.append( + XFailInfo( + file_path=self.file_path, + line_number=lineno, + test_name=test_name, + reason=reason, + condition=condition, + strict=strict, + xfail_type=xfail_type, + ) + ) + + def _get_string_value(self, node) -> str: + """Extract string value from an AST node.""" + if isinstance(node, ast.Constant): + return str(node.value) + elif isinstance(node, ast.JoinedStr): # f-string + return self._ast_to_string(node) + else: + return self._ast_to_string(node) + + def _get_bool_value(self, node) -> Optional[bool]: + """Extract boolean value from an AST node.""" + if isinstance(node, ast.Constant): + if isinstance(node.value, bool): + return node.value + return None + + def _ast_to_string(self, node) -> str: + """Convert an AST node to a string representation.""" + try: + return ast.unparse(node) + except AttributeError: + # Python < 3.9 doesn't have ast.unparse + return ast.dump(node) + + def _extract_runtime_xfail(self, call_node: ast.Call): + """Extract information from a runtime pytest.xfail() call.""" + test_name = self.current_function_name or "unknown" + reason = "No reason provided" + condition = ( + None # Runtime xfails are already conditional (inside if statements) + ) + + # The first positional argument is typically the reason + if call_node.args: + reason = self._get_string_value(call_node.args[0]) + + # Check keyword arguments + for keyword in call_node.keywords: + if keyword.arg == "reason": + reason = self._get_string_value(keyword.value) + + self.xfails.append( + XFailInfo( + file_path=self.file_path, + line_number=call_node.lineno, + test_name=test_name, + reason=reason, + condition=condition, + strict=None, + xfail_type="runtime", + ) + ) + + +def find_test_files(tests_dir: Path) -> List[Path]: + """Find all Python test files in the tests directory.""" + test_files = [] + # Find test files + for pattern in ["test_*.py", "*_test.py"]: + test_files.extend(tests_dir.rglob(pattern)) + # Also find utility files that might contain xfail calls (like utils.py, conftest.py) + for pattern in ["utils.py", "conftest.py", "helpers.py"]: + test_files.extend(tests_dir.rglob(pattern)) + return sorted(set(test_files)) # Remove duplicates + + +def collect_xfails(test_files: List[Path]) -> List[XFailInfo]: + """Collect all xfail markers from test files.""" + all_xfails = [] + + for test_file in test_files: + try: + with open(test_file, "r", encoding="utf-8") as f: + content = f.read() + + tree = ast.parse(content, filename=str(test_file)) + collector = XFailCollector(str(test_file)) + collector.visit(tree) + all_xfails.extend(collector.xfails) + + except SyntaxError as e: + print(f"Warning: Syntax error in {test_file}: {e}", file=sys.stderr) + except Exception as e: + print(f"Warning: Error processing {test_file}: {e}", file=sys.stderr) + + return all_xfails + + +def group_xfails_by_reason(xfails: List[XFailInfo]) -> Dict[str, List[XFailInfo]]: + """Group xfails by their reason.""" + grouped = defaultdict(list) + for xfail in xfails: + grouped[xfail.reason].append(xfail) + return dict(grouped) + + +def format_table(xfails: List[XFailInfo], workspace_root: Path) -> str: + """Format xfails as a table.""" + if not xfails: + return "No xfails found in the test suite! 🎉" + + # Group by reason + grouped = group_xfails_by_reason(xfails) + + # Build the report + lines = [] + lines.append("=" * 100) + lines.append("XFAILS REPORT") + lines.append("=" * 100) + lines.append(f"\nTotal xfails: {len(xfails)}") + lines.append(f"Unique reasons: {len(grouped)}") + lines.append("\n") + + # Summary table + lines.append("-" * 100) + lines.append(f"{'Reason':<60} {'Count':>10} {'Type':>15}") + lines.append("-" * 100) + + # Sort by count (descending) + sorted_reasons = sorted(grouped.items(), key=lambda x: len(x[1]), reverse=True) + + for reason, xfails_list in sorted_reasons: + count = len(xfails_list) + # Get the most common type for this reason + types = [x.xfail_type for x in xfails_list] + type_str = max(set(types), key=types.count) + # Truncate long reasons + display_reason = reason if len(reason) <= 58 else reason[:55] + "..." + lines.append(f"{display_reason:<60} {count:>10} {type_str:>15}") + + lines.append("-" * 100) + lines.append("\n") + + # Detailed breakdown + lines.append("=" * 100) + lines.append("DETAILED BREAKDOWN BY REASON") + lines.append("=" * 100) + + for reason, xfails_list in sorted_reasons: + lines.append(f"\n[{len(xfails_list)} xfails] {reason}") + lines.append("-" * 100) + + for xfail in xfails_list: + # Make path relative to workspace root + try: + rel_path = Path(xfail.file_path).relative_to(workspace_root) + except ValueError: + rel_path = Path(xfail.file_path) + + location = f"{rel_path}:{xfail.line_number}" + lines.append(f" • {location}") + lines.append(f" Test: {xfail.test_name}") + lines.append(f" Type: {xfail.xfail_type}") + if xfail.condition: + lines.append(f" Condition: {xfail.condition}") + if xfail.strict is not None: + lines.append(f" Strict: {xfail.strict}") + lines.append("") + + lines.append("=" * 100) + + return "\n".join(lines) + + +def main(): + # Determine workspace root and tests directory + workspace_root = Path(__file__).parent.parent + tests_dir = workspace_root / "tests" + + if not tests_dir.exists(): + print(f"Error: Tests directory not found: {tests_dir}", file=sys.stderr) + sys.exit(1) + + # Find and process test files + print(f"Scanning for test files in {tests_dir}...", file=sys.stderr) + test_files = find_test_files(tests_dir) + print(f"Found {len(test_files)} test files", file=sys.stderr) + + print("Collecting xfail markers...", file=sys.stderr) + xfails = collect_xfails(test_files) + + # Format and print output as table + output = format_table(xfails, workspace_root) + print(output) + + +if __name__ == "__main__": + main()