diff --git a/scripts/ci/classify_failures.py b/scripts/ci/classify_failures.py index 8d3141ab89ca..93ff33ada1a4 100644 --- a/scripts/ci/classify_failures.py +++ b/scripts/ci/classify_failures.py @@ -97,6 +97,9 @@ def sig(name: str, verdict: str, pattern: str, why: str) -> Signature: "the compiled app-host products did not restore on this runner"), sig("app-host-preparation", MACHINE, r"Unexpected app-host preparation outcome", "the isolated app-host home was not prepared"), + sig("gui-token-unavailable", MACHINE, + r"^Could not take this Mac's gui token for the app-host tests \(take-gui exited ", + "the runner could not acquire the GUI token for app-host tests"), # The CLI and the package framework it links came from different builds: # the runner staged products from another job. Compiled together, they match. sig("mixed-products", MACHINE, r"dyld\[\d+\]: Symbol not found: ", diff --git a/tests/test_ci_classify_failures.py b/tests/test_ci_classify_failures.py index 611c9f2ae790..5621f8c5fa01 100644 --- a/tests/test_ci_classify_failures.py +++ b/tests/test_ci_classify_failures.py @@ -62,6 +62,12 @@ 2026-09-27T09:31:30.4083830Z ##[error]Process completed with exit code 65. """) +NO_CONSOLE_WARNING = ( + "No logged-in console user (or no passwordless sudo) on this runner; running in the current bootstrap. " + "XCTest will fail here if this runner has no GUI session." +) +GUI_TOKEN_UNAVAILABLE = "Could not take this Mac's gui token for the app-host tests (take-gui exited 1)." + PARAMETERIZED_TEST_FAILED = ( "2026-09-27T10:40:00.0000000Z ✘ Test mapsConnectionClosedStartupFailureToRetryableStatus(_:) recorded an issue " "with 1 argument diagnostic → \"Connection closed by 192.0.2.1 port 22\" at " @@ -112,6 +118,31 @@ def test_products_from_two_builds_outweigh_the_test_failures_they_cause(self) -> self.assertEqual((verdict, signature), (cf.MACHINE, "mixed-products")) self.assertIn("Symbol not found", cf.classify_text(MIXED_PRODUCTS)["evidence"]) + def test_a_gui_token_failure_is_the_machine(self) -> None: + log = f"##[error]{GUI_TOKEN_UNAVAILABLE}\n" + self.assertEqual(self.verdict(log), (cf.MACHINE, "gui-token-unavailable")) + self.assertIn(GUI_TOKEN_UNAVAILABLE, cf.classify_text(log)["evidence"]) + + def test_a_gui_token_failure_is_read_from_its_failure_annotation(self) -> None: + result = cf.classify_text("", [GUI_TOKEN_UNAVAILABLE]) + self.assertEqual((result["verdict"], result["signature"]), (cf.MACHINE, "gui-token-unavailable")) + + def test_an_ambiguous_console_warning_does_not_outweigh_test_failures(self) -> None: + log = f"##[warning]{NO_CONSOLE_WARNING}\n{TEST_FAILED}" + self.assertEqual(self.verdict(log), (cf.CODE, "swift-testing-issue")) + + def test_a_gui_token_diagnostic_in_assertion_output_is_the_code(self) -> None: + log = ("✘ Test reportsGUITokenFailure() recorded an issue at GUITokenTests.swift:12:5: " + f'Expectation failed: diagnostic → "{GUI_TOKEN_UNAVAILABLE}"\n' + "##[error]Process completed with exit code 1.\n") + self.assertEqual(self.verdict(log), (cf.CODE, "swift-testing-issue")) + + def test_an_echoed_gui_token_error_does_not_outweigh_test_failures(self) -> None: + log = ('##[group]Run "$helper" take-gui --wait 1800\n' + f'{ESC}[36;1mecho "::error::{GUI_TOKEN_UNAVAILABLE}"{ESC}[0m\n' + f"##[endgroup]\n{TEST_FAILED}") + self.assertEqual(self.verdict(log), (cf.CODE, "swift-testing-issue")) + def test_a_test_failure_on_a_healthy_runner_is_the_code(self) -> None: self.assertEqual(self.verdict(TEST_FAILED), (cf.CODE, "swift-testing-issue")) self.assertEqual(self.verdict(PARAMETERIZED_TEST_FAILED), (cf.CODE, "swift-testing-issue")) @@ -144,6 +175,8 @@ def test_a_machine_line_in_a_step_that_passed_does_not_count(self) -> None: 2026-09-27T10:00:03.0Z Warning: Failed to save: No space left on device """) self.assertEqual(self.verdict(log), (cf.CODE, "swift-testing-issue")) + gui_token_noise = log.replace("Warning: Failed to save: No space left on device", GUI_TOKEN_UNAVAILABLE) + self.assertEqual(self.verdict(gui_token_noise), (cf.CODE, "swift-testing-issue")) def test_a_group_a_step_titles_run_keeps_its_output(self) -> None: log = textwrap.dedent("""\ @@ -203,6 +236,14 @@ def test_every_failure_on_the_machine_reruns(self) -> None: self.assertTrue(rerun) self.assertIn("attempt 2", line) + def test_a_gui_token_failure_does_not_block_other_machine_retries(self) -> None: + report = self.report([]) + report["jobs"] = cf.classify_jobs( + [job(1, "macos / app-host unit tests (2/7)"), job(2, "macos / swift-package-tests")], + {1: (f"##[error]{GUI_TOKEN_UNAVAILABLE}\n", []), 2: (HOOK_REFUSED, [])}, + ) + self.assertTrue(cf.rerun_decision(report, self.LATEST)[0]) + def test_one_code_or_unknown_failure_keeps_the_run_red(self) -> None: for other in (cf.CODE, cf.UNKNOWN): rerun, line = cf.rerun_decision(self.report([cf.MACHINE, other]), self.LATEST)