diff --git a/packages/testing/src/execution_testing/cli/hasher.py b/packages/testing/src/execution_testing/cli/hasher.py index ecb49665ac..5b13e229b1 100644 --- a/packages/testing/src/execution_testing/cli/hasher.py +++ b/packages/testing/src/execution_testing/cli/hasher.py @@ -2,12 +2,15 @@ import hashlib import json +import sys from dataclasses import dataclass, field from enum import IntEnum, auto from pathlib import Path -from typing import Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, TypeVar import click +from rich.console import Console +from rich.markup import escape as rich_escape class HashableItemType(IntEnum): @@ -42,26 +45,43 @@ def hash(self) -> bytes: all_hash_bytes += item_hash_bytes return hashlib.sha256(all_hash_bytes).digest() - def print( + def format_lines( self, *, name: str, level: int = 0, print_type: Optional[HashableItemType] = None, - ) -> None: - """Print the hash of the item and sub-items.""" + max_depth: Optional[int] = None, + ) -> List[str]: + """Return the hash lines for the item and sub-items.""" + lines: List[str] = [] next_level = level print_name = name + if level == 0 and self.parents: separator = "::" if self.type == HashableItemType.TEST else "/" print_name = f"{'/'.join(self.parents)}{separator}{name}" + if print_type is None or self.type >= print_type: next_level += 1 - print(f"{' ' * level}{print_name}: 0x{self.hash().hex()}") + lines.append(f"{' ' * level}{print_name}: 0x{self.hash().hex()}") + + # Stop recursion if we've reached max_depth + if max_depth is not None and next_level > max_depth: + return lines if self.items is not None: for key, item in sorted(self.items.items()): - item.print(name=key, level=next_level, print_type=print_type) + lines.extend( + item.format_lines( + name=key, + level=next_level, + print_type=print_type, + max_depth=max_depth, + ) + ) + + return lines @classmethod def from_json_file( @@ -126,34 +146,247 @@ def from_folder( return cls(type=HashableItemType.FOLDER, items=items, parents=parents) -@click.command() +def render_hash_report( + folder: Path, + *, + files: bool, + tests: bool, + root: bool, + name_override: Optional[str] = None, + max_depth: Optional[int] = None, +) -> List[str]: + """Return canonical output lines for a folder.""" + item = HashableItem.from_folder(folder_path=folder) + if root: + return [f"0x{item.hash().hex()}"] + print_type: Optional[HashableItemType] = None + if files: + print_type = HashableItemType.FILE + elif tests: + print_type = HashableItemType.TEST + name = name_override if name_override is not None else folder.name + return item.format_lines( + name=name, print_type=print_type, max_depth=max_depth + ) + + +def collect_hashes( + item: HashableItem, + *, + path: str = "", + print_type: Optional[HashableItemType] = None, + max_depth: Optional[int] = None, + depth: int = 0, +) -> Dict[str, str]: + """Collect hashes from item tree as {path: hash_hex}.""" + result: Dict[str, str] = {} + + if print_type is None or item.type >= print_type: + if path: + result[path] = f"0x{item.hash().hex()}" + depth += 1 + if max_depth is not None and depth > max_depth: + return result + + if item.items: + for name, child in sorted(item.items.items()): + child_path = f"{path}/{name}" if path else name + result.update( + collect_hashes( + child, + path=child_path, + print_type=print_type, + max_depth=max_depth, + depth=depth, + ) + ) + + return result + + +def display_diff( + left: Dict[str, str], + right: Dict[str, str], + *, + left_label: str, + right_label: str, +) -> None: + """Render diff showing only changed hashes.""" + differences: List[tuple[str, str, str]] = [] + + for path in left: + right_hash = right.get(path, "") + if left[path] != right_hash: + differences.append((path, left[path], right_hash)) + + for path in right: + if path not in left: + differences.append((path, "", right[path])) + + if not differences: + return + + console = Console() + console.print("── Fixture Hash Differences ──", style="bold") + console.print(f"[dim]--- {left_label}[/dim]") + console.print(f"[dim]+++ {right_label}[/dim]") + console.print() + + for path, left_hash, right_hash in differences: + depth = path.count("/") + indent = " " * (depth + 1) + console.print(f"{indent}[bold]{rich_escape(path)}[/bold]") + console.print(f"{indent} [red]- {left_hash}[/red]") + console.print(f"{indent} [green]+ {right_hash}[/green]") + console.print() + + +class DefaultGroup(click.Group): + """Click group with a default command fallback.""" + + def __init__( + self, *args: Any, default_cmd_name: str = "hash", **kwargs: Any + ): + super().__init__(*args, **kwargs) + self.default_cmd_name = default_cmd_name + + def resolve_command( + self, ctx: click.Context, args: List[str] + ) -> tuple[Optional[str], Optional[click.Command], List[str]]: + """Resolve command, inserting default if no subcommand given.""" + first_arg_idx = next( + (i for i, a in enumerate(args) if not a.startswith("-")), None + ) + if ( + first_arg_idx is not None + and args[first_arg_idx] not in self.commands + ): + args = list(args) + args.insert(first_arg_idx, self.default_cmd_name) + return super().resolve_command(ctx, args) + + +F = TypeVar("F", bound=Callable[..., None]) + + +def hash_options(func: F) -> F: + """Decorator for common hash options.""" + func = click.option( + "--root", "-r", is_flag=True, help="Only print hash of root folder" + )(func) + func = click.option( + "--tests", "-t", is_flag=True, help="Print hash of tests" + )(func) + func = click.option( + "--files", "-f", is_flag=True, help="Print hash of files" + )(func) + return func + + +@click.group( + cls=DefaultGroup, + default_cmd_name="hash", + context_settings={"help_option_names": ["-h", "--help"]}, +) +def hasher() -> None: + """Hash folders of JSON fixtures and compare them.""" + pass + + +@hasher.command(name="hash") @click.argument( "folder_path_str", type=click.Path( exists=True, file_okay=False, dir_okay=True, readable=True ), ) -@click.option("--files", "-f", is_flag=True, help="Print hash of files") -@click.option("--tests", "-t", is_flag=True, help="Print hash of tests") +@hash_options +def hash_cmd( + folder_path_str: str, files: bool, tests: bool, root: bool +) -> None: + """Hash folders of JSON fixtures and print their hashes.""" + lines = render_hash_report( + Path(folder_path_str), files=files, tests=tests, root=root + ) + for line in lines: + print(line) + + +@hasher.command(name="compare") +@click.argument( + "left_folder", + type=click.Path( + exists=True, file_okay=False, dir_okay=True, readable=True + ), +) +@click.argument( + "right_folder", + type=click.Path( + exists=True, file_okay=False, dir_okay=True, readable=True + ), +) @click.option( - "--root", "-r", is_flag=True, help="Only print hash of root folder" + "--depth", + "-d", + type=int, + default=None, + help="Limit to N levels (0=root, 1=folders, 2=files, 3=tests).", ) -def main(folder_path_str: str, files: bool, tests: bool, root: bool) -> None: - """Hash folders of JSON fixtures and print their hashes.""" - folder_path: Path = Path(folder_path_str) - item = HashableItem.from_folder(folder_path=folder_path) +@hash_options +def compare_cmd( + left_folder: str, + right_folder: str, + files: bool, + tests: bool, + root: bool, + depth: Optional[int], +) -> None: + """Compare two fixture directories and show differences.""" + try: + left_item = HashableItem.from_folder(folder_path=Path(left_folder)) + right_item = HashableItem.from_folder(folder_path=Path(right_folder)) - if root: - print(f"0x{item.hash().hex()}") - return + if root: + if left_item.hash() == right_item.hash(): + sys.exit(0) + left_hashes = {"root": f"0x{left_item.hash().hex()}"} + right_hashes = {"root": f"0x{right_item.hash().hex()}"} + else: + print_type: Optional[HashableItemType] = None + if files: + print_type = HashableItemType.FILE + elif tests: + print_type = HashableItemType.TEST + + left_hashes = collect_hashes( + left_item, print_type=print_type, max_depth=depth + ) + right_hashes = collect_hashes( + right_item, print_type=print_type, max_depth=depth + ) + + if left_hashes == right_hashes: + sys.exit(0) + + display_diff( + left_hashes, + right_hashes, + left_label=left_folder, + right_label=right_folder, + ) + sys.exit(1) + except PermissionError as e: + click.echo(f"Error: Permission denied - {e}", err=True) + sys.exit(2) + except (json.JSONDecodeError, KeyError, TypeError) as e: + click.echo(f"Error: Invalid fixture format - {e}", err=True) + sys.exit(2) + except Exception as e: + click.echo(f"Error: {e}", err=True) + sys.exit(2) - print_type: Optional[HashableItemType] = None - if files: - print_type = HashableItemType.FILE - elif tests: - print_type = HashableItemType.TEST - item.print(name=folder_path.name, print_type=print_type) +main = hasher # Entry point alias if __name__ == "__main__": diff --git a/packages/testing/src/execution_testing/cli/tests/test_hasher.py b/packages/testing/src/execution_testing/cli/tests/test_hasher.py new file mode 100644 index 0000000000..b80bdc1e30 --- /dev/null +++ b/packages/testing/src/execution_testing/cli/tests/test_hasher.py @@ -0,0 +1,342 @@ +"""Tests for the hasher CLI tool.""" + +import json +from pathlib import Path + +from click.testing import CliRunner + +from execution_testing.cli.hasher import hasher + + +def create_fixture(path: Path, test_name: str, hash_value: str) -> None: + """Create a test fixture JSON file.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({test_name: {"_info": {"hash": hash_value}}})) + + +class TestCompareIdenticalDirectories: + """Test comparing identical directories.""" + + def test_compare_identical_directories(self, tmp_path: Path) -> None: + """Same content in both dirs should exit 0 with no output.""" + dir_a = tmp_path / "dir_a" / "state_tests" + dir_b = tmp_path / "dir_b" / "state_tests" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + create_fixture(dir_b / "test.json", "test1", "0xabc123") + + runner = CliRunner() + result = runner.invoke( + hasher, ["compare", str(dir_a.parent), str(dir_b.parent)] + ) + assert result.exit_code == 0 + assert result.output == "" + + +class TestCompareDifferentDirectories: + """Test comparing different directories.""" + + def test_compare_different_directories(self, tmp_path: Path) -> None: + """Different hashes should exit 1 with diff in stdout.""" + dir_a = tmp_path / "dir_a" / "state_tests" + dir_b = tmp_path / "dir_b" / "state_tests" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + create_fixture(dir_b / "test.json", "test1", "0xdef456") + + runner = CliRunner() + result = runner.invoke( + hasher, ["compare", str(dir_a.parent), str(dir_b.parent)] + ) + assert result.exit_code == 1 + assert "Fixture Hash Differences" in result.output + # Verify the new format shows the path and both hashes + assert "test1" in result.output + assert "0xabc123" in result.output + assert "0xdef456" in result.output + + +class TestCompareMissingDirectory: + """Test comparing when a directory doesn't exist.""" + + def test_compare_missing_directory(self, tmp_path: Path) -> None: + """One path doesn't exist should exit 2 with error in stderr.""" + dir_a = tmp_path / "dir_a" / "state_tests" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + + runner = CliRunner() + result = runner.invoke( + hasher, + ["compare", str(dir_a.parent), str(tmp_path / "nonexistent")], + ) + assert result.exit_code == 2 + + +class TestCompareFlagParity: + """Test that flags work consistently between hash and compare commands.""" + + def test_compare_flag_parity_files(self, tmp_path: Path) -> None: + """Hasher -f X vs hasher compare -f X X should exit 0.""" + dir_a = tmp_path / "dir_a" / "state_tests" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + + runner = CliRunner() + # Compare same directory with -f flag + result = runner.invoke( + hasher, ["compare", "-f", str(dir_a.parent), str(dir_a.parent)] + ) + assert result.exit_code == 0 + + def test_compare_flag_parity_tests(self, tmp_path: Path) -> None: + """Hasher -t X vs hasher compare -t X X should exit 0.""" + dir_a = tmp_path / "dir_a" / "state_tests" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + + runner = CliRunner() + # Compare same directory with -t flag + result = runner.invoke( + hasher, ["compare", "-t", str(dir_a.parent), str(dir_a.parent)] + ) + assert result.exit_code == 0 + + def test_compare_flag_parity_root(self, tmp_path: Path) -> None: + """Hasher -r X vs hasher compare -r X X should exit 0.""" + dir_a = tmp_path / "dir_a" / "state_tests" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + + runner = CliRunner() + # Compare same directory with -r flag + result = runner.invoke( + hasher, ["compare", "-r", str(dir_a.parent), str(dir_a.parent)] + ) + assert result.exit_code == 0 + + +class TestBackwardsCompatibility: + """Test backwards compatibility with existing hasher FOLDER syntax.""" + + def test_backwards_compat(self, tmp_path: Path) -> None: + """Hasher FOLDER without subcommand should work as before.""" + dir_a = tmp_path / "dir_a" / "state_tests" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + + runner = CliRunner() + # Old syntax without subcommand + result = runner.invoke(hasher, [str(dir_a.parent)]) + assert result.exit_code == 0 + assert "0x" in result.output + + def test_explicit_hash_subcommand(self, tmp_path: Path) -> None: + """Hasher hash FOLDER should work.""" + dir_a = tmp_path / "dir_a" / "state_tests" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + + runner = CliRunner() + # Explicit hash subcommand + result = runner.invoke(hasher, ["hash", str(dir_a.parent)]) + assert result.exit_code == 0 + assert "0x" in result.output + + def test_hash_output_matches_between_syntaxes( + self, tmp_path: Path + ) -> None: + """Both syntaxes should produce identical output.""" + dir_a = tmp_path / "dir_a" / "state_tests" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + + runner = CliRunner() + # Old syntax + result_old = runner.invoke(hasher, [str(dir_a.parent)]) + # New syntax + result_new = runner.invoke(hasher, ["hash", str(dir_a.parent)]) + + assert result_old.exit_code == result_new.exit_code + assert result_old.output == result_new.output + + +class TestCompareEmptyDirectories: + """Test comparing empty directories.""" + + def test_compare_empty_directories(self, tmp_path: Path) -> None: + """Both dirs empty should exit 0.""" + dir_a = tmp_path / "dir_a" + dir_b = tmp_path / "dir_b" + dir_a.mkdir(parents=True) + dir_b.mkdir(parents=True) + + runner = CliRunner() + result = runner.invoke(hasher, ["compare", str(dir_a), str(dir_b)]) + assert result.exit_code == 0 + + +class TestErrorToStderr: + """Test that errors go to stderr.""" + + def test_error_to_stderr(self, tmp_path: Path) -> None: + """Invalid fixture JSON should produce error message.""" + dir_a = tmp_path / "dir_a" + dir_a.mkdir(parents=True) + (dir_a / "invalid.json").write_text("not valid json") + + runner = CliRunner() + result = runner.invoke(hasher, ["compare", str(dir_a), str(dir_a)]) + assert result.exit_code == 2 + assert "Error" in result.output + + +class TestHashCommandFlags: + """Test hash command with various flags.""" + + def test_hash_with_files_flag(self, tmp_path: Path) -> None: + """Hasher hash -f FOLDER should work.""" + dir_a = tmp_path / "dir_a" / "state_tests" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + + runner = CliRunner() + result = runner.invoke(hasher, ["hash", "-f", str(dir_a.parent)]) + assert result.exit_code == 0 + assert "test.json" in result.output + + def test_hash_with_tests_flag(self, tmp_path: Path) -> None: + """Hasher hash -t FOLDER should work.""" + dir_a = tmp_path / "dir_a" / "state_tests" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + + runner = CliRunner() + result = runner.invoke(hasher, ["hash", "-t", str(dir_a.parent)]) + assert result.exit_code == 0 + assert "test1" in result.output + + def test_hash_with_root_flag(self, tmp_path: Path) -> None: + """Hasher hash -r FOLDER should only print root hash.""" + dir_a = tmp_path / "dir_a" / "state_tests" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + + runner = CliRunner() + result = runner.invoke(hasher, ["hash", "-r", str(dir_a.parent)]) + assert result.exit_code == 0 + # Should only have one line with the hash + lines = [line for line in result.output.strip().split("\n") if line] + assert len(lines) == 1 + assert lines[0].startswith("0x") + + +class TestCompareDepthFlag: + """Test --depth flag for compare command.""" + + def test_depth_limits_output(self, tmp_path: Path) -> None: + """--depth should limit how deep the comparison goes.""" + dir_a = tmp_path / "dir_a" / "folder" / "subfolder" + dir_b = tmp_path / "dir_b" / "folder" / "subfolder" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + create_fixture(dir_b / "test.json", "test1", "0xdef456") + + runner = CliRunner() + + # depth=1 should show folder but not subfolder + result = runner.invoke( + hasher, + [ + "compare", + "--depth", + "1", + str(dir_a.parent.parent), + str(dir_b.parent.parent), + ], + ) + assert result.exit_code == 1 + assert "folder" in result.output + assert "subfolder" not in result.output + + def test_depth_2_shows_subfolders(self, tmp_path: Path) -> None: + """--depth 2 should show subfolders.""" + dir_a = tmp_path / "dir_a" / "folder" / "subfolder" + dir_b = tmp_path / "dir_b" / "folder" / "subfolder" + create_fixture(dir_a / "test.json", "test1", "0xabc123") + create_fixture(dir_b / "test.json", "test1", "0xdef456") + + runner = CliRunner() + + result = runner.invoke( + hasher, + [ + "compare", + "-d", + "2", + str(dir_a.parent.parent), + str(dir_b.parent.parent), + ], + ) + assert result.exit_code == 1 + assert "folder" in result.output + assert "subfolder" in result.output + + +class TestCompareHierarchy: + """Test that diff output preserves hierarchy.""" + + def test_full_paths_in_output(self, tmp_path: Path) -> None: + """Diff should show full paths to disambiguate items with same name.""" + # Create two folders each with a "shanghai" subfolder + dir_a = tmp_path / "dir_a" + dir_b = tmp_path / "dir_b" + create_fixture( + dir_a / "blockchain_tests" / "shanghai" / "test.json", + "test1", + "0xaaa111", + ) + create_fixture( + dir_a / "state_tests" / "shanghai" / "test.json", + "test1", + "0xbbb222", + ) + create_fixture( + dir_b / "blockchain_tests" / "shanghai" / "test.json", + "test1", + "0xccc333", + ) + create_fixture( + dir_b / "state_tests" / "shanghai" / "test.json", + "test1", + "0xddd444", + ) + + runner = CliRunner() + result = runner.invoke( + hasher, ["compare", "--depth", "2", str(dir_a), str(dir_b)] + ) + + assert result.exit_code == 1 + # Should show full paths, not just "shanghai" twice + assert "blockchain_tests/shanghai" in result.output + assert "state_tests/shanghai" in result.output + + +class TestHelpOptions: + """Test help options.""" + + def test_help_short(self) -> None: + """-h should show help.""" + runner = CliRunner() + result = runner.invoke(hasher, ["-h"]) + assert result.exit_code == 0 + assert "Hash folders of JSON fixtures" in result.output + + def test_help_long(self) -> None: + """--help should show help.""" + runner = CliRunner() + result = runner.invoke(hasher, ["--help"]) + assert result.exit_code == 0 + assert "Hash folders of JSON fixtures" in result.output + + def test_compare_help(self) -> None: + """Compare --help should show compare help.""" + runner = CliRunner() + result = runner.invoke(hasher, ["compare", "--help"]) + assert result.exit_code == 0 + assert "Compare two fixture directories" in result.output + + def test_hash_help(self) -> None: + """Hash --help should show hash help.""" + runner = CliRunner() + result = runner.invoke(hasher, ["hash", "--help"]) + assert result.exit_code == 0 + assert "Hash folders of JSON fixtures" in result.output