Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions scripts/ci/classify_failures.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@ def sig(name: str, verdict: str, pattern: str, why: str) -> Signature:
"the compiled app-host products did not restore on this runner"),
sig("app-host-preparation", MACHINE, r"Unexpected app-host preparation outcome",
"the isolated app-host home was not prepared"),
sig("gui-token-unavailable", MACHINE,
r"^Could not take this Mac's gui token for the app-host tests \(take-gui exited ",
"the runner could not acquire the GUI token for app-host tests"),
# The CLI and the package framework it links came from different builds:
# the runner staged products from another job. Compiled together, they match.
sig("mixed-products", MACHINE, r"dyld\[\d+\]: Symbol not found: ",
Expand Down
41 changes: 41 additions & 0 deletions tests/test_ci_classify_failures.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,12 @@
2026-09-27T09:31:30.4083830Z ##[error]Process completed with exit code 65.
""")

NO_CONSOLE_WARNING = (
"No logged-in console user (or no passwordless sudo) on this runner; running in the current bootstrap. "
"XCTest will fail here if this runner has no GUI session."
)
GUI_TOKEN_UNAVAILABLE = "Could not take this Mac's gui token for the app-host tests (take-gui exited 1)."

PARAMETERIZED_TEST_FAILED = (
"2026-09-27T10:40:00.0000000Z ✘ Test mapsConnectionClosedStartupFailureToRetryableStatus(_:) recorded an issue "
"with 1 argument diagnostic → \"Connection closed by 192.0.2.1 port 22\" at "
Expand Down Expand Up @@ -112,6 +118,31 @@ def test_products_from_two_builds_outweigh_the_test_failures_they_cause(self) ->
self.assertEqual((verdict, signature), (cf.MACHINE, "mixed-products"))
self.assertIn("Symbol not found", cf.classify_text(MIXED_PRODUCTS)["evidence"])

def test_a_gui_token_failure_is_the_machine(self) -> None:
log = f"##[error]{GUI_TOKEN_UNAVAILABLE}\n"
self.assertEqual(self.verdict(log), (cf.MACHINE, "gui-token-unavailable"))
self.assertIn(GUI_TOKEN_UNAVAILABLE, cf.classify_text(log)["evidence"])

def test_a_gui_token_failure_is_read_from_its_failure_annotation(self) -> None:
result = cf.classify_text("", [GUI_TOKEN_UNAVAILABLE])
self.assertEqual((result["verdict"], result["signature"]), (cf.MACHINE, "gui-token-unavailable"))

def test_an_ambiguous_console_warning_does_not_outweigh_test_failures(self) -> None:
log = f"##[warning]{NO_CONSOLE_WARNING}\n{TEST_FAILED}"
self.assertEqual(self.verdict(log), (cf.CODE, "swift-testing-issue"))

def test_a_gui_token_diagnostic_in_assertion_output_is_the_code(self) -> None:
log = ("✘ Test reportsGUITokenFailure() recorded an issue at GUITokenTests.swift:12:5: "
f'Expectation failed: diagnostic → "{GUI_TOKEN_UNAVAILABLE}"\n'
"##[error]Process completed with exit code 1.\n")
self.assertEqual(self.verdict(log), (cf.CODE, "swift-testing-issue"))

def test_an_echoed_gui_token_error_does_not_outweigh_test_failures(self) -> None:
log = ('##[group]Run "$helper" take-gui --wait 1800\n'
f'{ESC}[36;1mecho "::error::{GUI_TOKEN_UNAVAILABLE}"{ESC}[0m\n'
f"##[endgroup]\n{TEST_FAILED}")
self.assertEqual(self.verdict(log), (cf.CODE, "swift-testing-issue"))

def test_a_test_failure_on_a_healthy_runner_is_the_code(self) -> None:
self.assertEqual(self.verdict(TEST_FAILED), (cf.CODE, "swift-testing-issue"))
self.assertEqual(self.verdict(PARAMETERIZED_TEST_FAILED), (cf.CODE, "swift-testing-issue"))
Expand Down Expand Up @@ -144,6 +175,8 @@ def test_a_machine_line_in_a_step_that_passed_does_not_count(self) -> None:
2026-09-27T10:00:03.0Z Warning: Failed to save: No space left on device
""")
self.assertEqual(self.verdict(log), (cf.CODE, "swift-testing-issue"))
gui_token_noise = log.replace("Warning: Failed to save: No space left on device", GUI_TOKEN_UNAVAILABLE)
self.assertEqual(self.verdict(gui_token_noise), (cf.CODE, "swift-testing-issue"))

def test_a_group_a_step_titles_run_keeps_its_output(self) -> None:
log = textwrap.dedent("""\
Expand Down Expand Up @@ -203,6 +236,14 @@ def test_every_failure_on_the_machine_reruns(self) -> None:
self.assertTrue(rerun)
self.assertIn("attempt 2", line)

def test_a_gui_token_failure_does_not_block_other_machine_retries(self) -> None:
report = self.report([])
report["jobs"] = cf.classify_jobs(
[job(1, "macos / app-host unit tests (2/7)"), job(2, "macos / swift-package-tests")],
{1: (f"##[error]{GUI_TOKEN_UNAVAILABLE}\n", []), 2: (HOOK_REFUSED, [])},
)
self.assertTrue(cf.rerun_decision(report, self.LATEST)[0])

def test_one_code_or_unknown_failure_keeps_the_run_red(self) -> None:
for other in (cf.CODE, cf.UNKNOWN):
rerun, line = cf.rerun_decision(self.report([cf.MACHINE, other]), self.LATEST)
Expand Down
Loading