From 88085ecb344b73d15291ca2efdaac56abb78d74a Mon Sep 17 00:00:00 2001 From: Lawrence Chen Date: Wed, 27 May 2026 18:42:35 -0700 Subject: [PATCH] Gate visual screenshot harness failures --- .github/workflows/ci.yml | 5 ++- tests/test_visual_screenshots.py | 16 ++------- tests_v2/test_visual_screenshots.py | 55 ++++++++++++++++------------- 3 files changed, 37 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a61d61723ba8..f10fb368bd66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,9 +18,12 @@ jobs: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Validate WarpBuild runner guards + - name: Validate macOS runner guards run: ./tests/test_ci_self_hosted_guard.sh + - name: Validate Python test harness syntax + run: git ls-files 'tests/*.py' 'tests_v2/*.py' 'scripts/*.py' | xargs python3 -m py_compile + - name: Validate create-dmg version pinning run: ./tests/test_ci_create_dmg_pinned.sh diff --git a/tests/test_visual_screenshots.py b/tests/test_visual_screenshots.py index 62cd4cce26f2..eb0dfb96c14d 100644 --- a/tests/test_visual_screenshots.py +++ b/tests/test_visual_screenshots.py @@ -1368,13 +1368,6 @@ def generate_html_report(changes: list[StateChange]) -> None: # --------------------------------------------------------------------------- -def _is_known_non_blocking_failure(change: StateChange) -> bool: - """Return True for known flaky VM-only visual failures we still report but do not gate on.""" - if change.name == "Nested: Close Top of T-shape" and "VIEW_DETACHED" in (change.error or ""): - return True - return False - - def run_visual_tests(): changes: list[StateChange] = [] @@ -1486,25 +1479,20 @@ def run_visual_tests(): print("=" * 60) passed = sum(1 for c in changes if c.passed) failed_changes = [c for c in changes if not c.passed] - non_blocking_failed = [c for c in failed_changes if _is_known_non_blocking_failure(c)] - blocking_failed = [c for c in failed_changes if not _is_known_non_blocking_failure(c)] print(f" Passed: {passed}") print(f" Failed: {len(failed_changes)}") - if non_blocking_failed: - print(f" Non-blocking failed: {len(non_blocking_failed)}") print(f" Total: {len(changes)}") if failed_changes: print() print("Failed tests:") for c in failed_changes: - marker = " (non-blocking)" if _is_known_non_blocking_failure(c) else "" - print(f" - {c.name}{marker}: {c.error or 'unknown'}") + print(f" - {c.name}: {c.error or 'unknown'}") print() print(f"Report: {HTML_REPORT}") - return 0 if len(blocking_failed) == 0 else 1 + return 0 if len(failed_changes) == 0 else 1 if __name__ == "__main__": diff --git a/tests_v2/test_visual_screenshots.py b/tests_v2/test_visual_screenshots.py index a55a89f617d3..d8eeedb252e5 100644 --- a/tests_v2/test_visual_screenshots.py +++ b/tests_v2/test_visual_screenshots.py @@ -27,7 +27,7 @@ from pathlib import Path from datetime import datetime from dataclasses import dataclass -from typing import Optional, List +from typing import Optional, List, Union sys.path.insert(0, str(Path(__file__).parent)) from cmux import cmux @@ -383,7 +383,7 @@ def test_a3_split_down(client: cmux) -> StateChange: return change -def _close_and_verify(client: cmux, change: StateChange, close_idx: int, +def _close_and_verify(client: cmux, change: StateChange, close_idx: Union[int, str], expected: int, before_label: str, after_label: str) -> StateChange: """Shared logic: close a surface, verify count, verify responsiveness, capture after.""" change.before, change.before_state = capture(client, before_label) @@ -394,10 +394,22 @@ def _close_and_verify(client: cmux, change: StateChange, close_idx: int, change.error = f"Expected {expected} surface(s), got {surface_count(client)}" change.passed = False else: - # Functional blank-detection: verify every remaining terminal responds - blank_err = verify_all_responsive(client, after_label) - if blank_err: - change.error = f"BLANK: {blank_err}" + # Functional blank-detection can be transient while split-tree views settle. + last_blank_err = None + for verify_attempt in range(3): + blank_err = verify_all_responsive(client, after_label) + if not blank_err: + last_blank_err = None + break + last_blank_err = blank_err + if verify_attempt < 2: + try: + client.refresh_surfaces() + except Exception: + pass + time.sleep(0.8 + (0.4 * verify_attempt)) + if last_blank_err: + change.error = f"BLANK: {last_blank_err}" change.passed = False except Exception as e: change.error = str(e) @@ -532,19 +544,26 @@ def test_d11_nested_close_bottomright(client: cmux) -> StateChange: def test_d12_nested_close_top(client: cmux) -> StateChange: - """D12: Split down, split bottom right → close top pane.""" + """D12: Split down, split bottom right → close original top pane.""" change = StateChange( name="Nested: Close Top of T-shape", group="D", - description="Split down → split bottom right → close top (surface 0)", - command="split down; focus 1; split right; close 0", + description="Split down → split bottom right → close the original top surface by ID", + command="capture top-id; split down; focus 1; split right; close ", ) + surfaces = client.list_surfaces() + if not surfaces: + change.passed = False + change.error = "No initial surface" + return change + top_surface_id = surfaces[0][1] + client.new_split("down") time.sleep(SPLIT_WAIT) client.focus_surface(1) time.sleep(SHORT_WAIT) client.new_split("right") time.sleep(SPLIT_WAIT) - return _close_and_verify(client, change, 0, 2, "d12_before", "d12_after") + return _close_and_verify(client, change, top_surface_id, 2, "d12_before", "d12_after") def test_d13_4pane_close_second(client: cmux) -> StateChange: @@ -1310,13 +1329,6 @@ def generate_html_report(changes: list[StateChange]) -> None: # --------------------------------------------------------------------------- -def _is_known_non_blocking_failure(change: StateChange) -> bool: - """Return True for known flaky VM-only visual failures we still report but do not gate on.""" - if change.name == "Nested: Close Top of T-shape" and "VIEW_DETACHED" in (change.error or ""): - return True - return False - - def run_visual_tests(): changes: list[StateChange] = [] @@ -1428,25 +1440,20 @@ def run_visual_tests(): print("=" * 60) passed = sum(1 for c in changes if c.passed) failed_changes = [c for c in changes if not c.passed] - non_blocking_failed = [c for c in failed_changes if _is_known_non_blocking_failure(c)] - blocking_failed = [c for c in failed_changes if not _is_known_non_blocking_failure(c)] print(f" Passed: {passed}") print(f" Failed: {len(failed_changes)}") - if non_blocking_failed: - print(f" Non-blocking failed: {len(non_blocking_failed)}") print(f" Total: {len(changes)}") if failed_changes: print() print("Failed tests:") for c in failed_changes: - marker = " (non-blocking)" if _is_known_non_blocking_failure(c) else "" - print(f" - {c.name}{marker}: {c.error or 'unknown'}") + print(f" - {c.name}: {c.error or 'unknown'}") print() print(f"Report: {HTML_REPORT}") - return 0 if len(blocking_failed) == 0 else 1 + return 0 if len(failed_changes) == 0 else 1 if __name__ == "__main__":