From f283ff412133002546587ff3f3bdd97b45200306 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Tue, 22 Sep 2026 09:55:31 -0700 Subject: [PATCH] ci: register the nine tests main's execution registry does not know about scripts/ci/validate_test_execution_registry.py fails on main right now: nine files under tests/ exist with no entry in tests/test-execution.toml, and the "test exists but has no execution registry entry" check is unconditional, so the preflight guard group is red for every pull request regardless of what it touched. Five of the nine arrived with pull requests merged today (#13666, #13673, #13677, #13696, #13701) and four did not. All nine run on a Linux runner, so all nine take the linux-guard lane: ci-guards.yml test_ci_app_host_result_accounting test_ci_workflow_guards_are_wired test_ios_screenshot_capture_guard test_ios_upload_array_expansion test_merge_xcstrings test_release_homebrew_gate test_tui_publish_dispatch_budget testbox-broker-guard test_ci_actionlint_covers_every_workflow ci-artifact-transport test_ci_selective_layer_wiring test_ios_screenshot_capture_guard was the one with no execution path at all: it arrived with the screenshot fail-fast fix in #13675 and no workflow ever ran it. It joins release-ios, beside the other iOS guards. It degrades cleanly where ruby is absent, printing a skip rather than failing, so a Linux lane suits it. The validator required a linux-guard test to appear in ci-guards.yml specifically, which rejects two of these even though they demonstrably execute on every pull request: testbox-broker-guard.yml deliberately carries no path filter, and ci-artifact-transport.yml owns its own. The check now asks whether any workflow runs the test, which is the property the lane is asserting. An entry naming a test that nothing runs is still rejected -- verified by temporarily moving tests/test_vm_scp.py to linux-guard, which fails with "linux-guard lane is not run by any workflow". Entries go into the alphabetically sorted tail of the manifest in position, so the sorted region stays sorted. The manifest now describes 218 tests. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci-guards.yml | 4 +++ .../ci/validate_test_execution_registry.py | 19 ++++++++-- tests/test-execution.toml | 36 +++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml index 09e511e0a2ee..2f3eadbefdbb 100644 --- a/.github/workflows/ci-guards.yml +++ b/.github/workflows/ci-guards.yml @@ -439,6 +439,10 @@ jobs: if: ${{ matrix.group == 'release-ios' }} run: python3 tests/test_ios_upload_array_expansion.py + - name: Validate iOS screenshot capture fail-fast + if: ${{ matrix.group == 'release-ios' }} + run: python3 tests/test_ios_screenshot_capture_guard.py + - name: Validate tagged iOS device entitlement fallback if: ${{ matrix.group == 'release-ios' }} run: node --test ios/tests/tagged-device-entitlements.test.mjs diff --git a/scripts/ci/validate_test_execution_registry.py b/scripts/ci/validate_test_execution_registry.py index 6cba2e1b5f40..8cf73326f4f9 100644 --- a/scripts/ci/validate_test_execution_registry.py +++ b/scripts/ci/validate_test_execution_registry.py @@ -36,6 +36,21 @@ def runner_lanes_from_workflow_text(text: str) -> set[str]: return lanes +def all_workflow_text() -> str: + """Every workflow's text, for asking whether a path is executed anywhere. + + A Linux guard does not have to live in ci-guards.yml to be live. The + always-on lanes run guards too -- testbox-broker-guard.yml deliberately has + no path filter, and ci-artifact-transport.yml owns its own -- so checking + ci-guards.yml alone rejects a test that demonstrably executes on every + pull request. + """ + return "\n".join( + workflow.read_text(encoding="utf-8") + for workflow in sorted(WORKFLOWS.glob("*.y*ml")) + ) + + def runner_lanes() -> set[str]: lanes: set[str] = set() for workflow in sorted(WORKFLOWS.glob("*.y*ml")): @@ -112,14 +127,14 @@ def main(argv: list[str]) -> int: errors.append(f"{path}: registry entry points to a missing test") live_runner_lanes = runner_lanes() - guard_text = CI_GUARDS.read_text(encoding="utf-8") + guard_text = all_workflow_text() for path, entry in sorted(by_path.items()): lane = entry.get("lane") if lane in INVENTORY_LANES: continue if lane == "linux-guard": if path not in guard_text: - errors.append(f"{path}: linux-guard lane is not referenced by ci-guards.yml") + errors.append(f"{path}: linux-guard lane is not run by any workflow") elif lane not in live_runner_lanes: errors.append(f"{path}: lane {lane!r} has no workflow invocation") diff --git a/tests/test-execution.toml b/tests/test-execution.toml index 6fdb2f98eaaf..25c2c479e52b 100644 --- a/tests/test-execution.toml +++ b/tests/test-execution.toml @@ -570,6 +570,14 @@ lane = "legacy" path = "tests/test_check_ghostty_zig_workflows.py" lane = "legacy" +[[test]] +path = "tests/test_ci_actionlint_covers_every_workflow.py" +lane = "linux-guard" + +[[test]] +path = "tests/test_ci_app_host_result_accounting.py" +lane = "linux-guard" + [[test]] path = "tests/test_ci_cache_restore_receipt.py" lane = "legacy" @@ -586,10 +594,18 @@ lane = "legacy" path = "tests/test_ci_release_helper_archs.py" lane = "legacy" +[[test]] +path = "tests/test_ci_selective_layer_wiring.py" +lane = "linux-guard" + [[test]] path = "tests/test_ci_ubuntu_apt_https.py" lane = "legacy" +[[test]] +path = "tests/test_ci_workflow_guards_are_wired.py" +lane = "linux-guard" + [[test]] path = "tests/test_claude_hook_missing_socket_error.py" lane = "legacy" @@ -726,6 +742,14 @@ lane = "legacy" path = "tests/test_indexnow_notification.py" lane = "legacy" +[[test]] +path = "tests/test_ios_screenshot_capture_guard.py" +lane = "linux-guard" + +[[test]] +path = "tests/test_ios_upload_array_expansion.py" +lane = "linux-guard" + [[test]] path = "tests/test_iroh_monitor_simulator_plan.py" lane = "legacy" @@ -746,6 +770,10 @@ lane = "legacy" path = "tests/test_memory_resource_diagnostics.py" lane = "legacy" +[[test]] +path = "tests/test_merge_xcstrings.py" +lane = "linux-guard" + [[test]] path = "tests/test_mobile_attach_prefers_iroh.py" lane = "legacy" @@ -794,6 +822,10 @@ lane = "legacy" path = "tests/test_reconcile_entitlements_with_profile.py" lane = "legacy" +[[test]] +path = "tests/test_release_homebrew_gate.py" +lane = "linux-guard" + [[test]] path = "tests/test_reload_tag_lock.py" lane = "legacy" @@ -862,6 +894,10 @@ lane = "legacy" path = "tests/test_tui_installers.py" lane = "legacy" +[[test]] +path = "tests/test_tui_publish_dispatch_budget.py" +lane = "linux-guard" + [[test]] path = "tests/test_tui_publish_workflow_security.py" lane = "legacy"