diff --git a/.github/workflows/agent-notification-tests.yml b/.github/workflows/agent-notification-tests.yml index 82060df63ddf..0ab5503c849c 100644 --- a/.github/workflows/agent-notification-tests.yml +++ b/.github/workflows/agent-notification-tests.yml @@ -15,6 +15,7 @@ on: - cmuxTests/** - Packages/macOS/CMUXAgentLaunch/** - .github/workflows/test-depot.yml + - scripts/ci/classify-app-host-test-output.py - Sources/AgentJournalLifecycleCenter*.swift - .github/workflows/agent-notification-tests.yml permissions: diff --git a/scripts/ci/classify-app-host-test-output.py b/scripts/ci/classify-app-host-test-output.py index 77fcef8e1c2a..536774daf6c8 100644 --- a/scripts/ci/classify-app-host-test-output.py +++ b/scripts/ci/classify-app-host-test-output.py @@ -28,10 +28,20 @@ r"(?:test runner .*?(?:timed out|hung|failed)|" r"unexpected exit|communication with the test runner|" r"testmanagerd.*invalidated|Couldn't communicate with a helper|" - r"Fatal error:|Program crashed|Signal \d+|SIG[A-Z]+|" + r"Fatal error:|Program crashed|\*\*\*\s+Signal\s+\d+\b|" r"Idle timed out|Post-test timed out)", re.IGNORECASE, ) +_APP_HOST_SIGNAL_RE = re.compile( + r"(?:\*\*\*[^\n]*\bSignal\s+\d+\b|" + r"(?:received|terminated|killed|stopped|crashed|aborted|exited)[^\n]*" + r"\bsignal\s+\d+\b|^\s*signal\s+\d+\b|" + r"(?:\*\*\*[^\n]*|(?:received|terminated|killed|stopped|crashed)[^\n]*)" + r"\bSIG(?:ABRT|ALRM|BUS|CHLD|CONT|FPE|HUP|ILL|INT|IO|IOT|KILL|PIPE|POLL|" + r"PROF|QUIT|SEGV|STOP|SYS|TERM|TRAP|TSTP|TTIN|TTOU|URG|USR1|USR2|" + r"VTALRM|XCPU|XFSZ)\b)", + re.IGNORECASE, +) _ASSERTION_RE = re.compile( r"(?:✘ Test .* recorded an issue|Expectation failed|" r"XCTAssert.*failed|Test run with .* failed|" @@ -77,7 +87,9 @@ def diagnose(output: str, exit_code: Optional[int] = None) -> Dict[str, object]: and "program crashed" not in cleaned.lower() ): compile_line = cleaned - if app_host_line is None and _APP_HOST_FAILURE_RE.search(raw_line): + if app_host_line is None and ( + _APP_HOST_FAILURE_RE.search(raw_line) or _APP_HOST_SIGNAL_RE.search(raw_line) + ): app_host_line = _clean_line(raw_line) if assertion_line is None and _ASSERTION_RE.search(raw_line): assertion_line = _clean_line(raw_line) diff --git a/tests/test_ci_app_host_test_output.py b/tests/test_ci_app_host_test_output.py index af515a1e5596..ff25cddc6e67 100644 --- a/tests/test_ci_app_host_test_output.py +++ b/tests/test_ci_app_host_test_output.py @@ -79,6 +79,16 @@ def test_diagnoses_app_host_failure_before_tests(self) -> None: self.assertEqual(diagnosis["category"], "pre-test app-host failure") self.assertEqual(diagnosis["executed_tests"], 0) + def test_ignores_build_signature_when_finding_app_host_cause(self) -> None: + diagnosis = MODULE.diagnose( + "Build description Signal 5 is a digest label, not a crash marker\n" + "The test runner timed out while preparing to run tests.\n", + exit_code=65, + ) + + self.assertEqual(diagnosis["category"], "pre-test app-host failure") + self.assertIn("test runner timed out", diagnosis["first_causal_line"]) + def test_diagnoses_crash_before_tests_as_app_host_failure(self) -> None: diagnosis = MODULE.diagnose( "Fatal error: Initial workspace creation failed\n" @@ -89,6 +99,22 @@ def test_diagnoses_crash_before_tests_as_app_host_failure(self) -> None: self.assertEqual(diagnosis["category"], "pre-test app-host failure") self.assertIn("Fatal error", diagnosis["first_causal_line"]) + def test_diagnoses_contextual_signal_name_as_app_host_failure(self) -> None: + diagnosis = MODULE.diagnose( + "Received signal SIGABRT from the app host\n", + exit_code=65, + ) + + self.assertEqual(diagnosis["category"], "pre-test app-host failure") + + def test_diagnoses_numeric_signal_without_banner_in_crash_context(self) -> None: + diagnosis = MODULE.diagnose( + "terminated by signal 9\n", + exit_code=65, + ) + + self.assertEqual(diagnosis["category"], "pre-test app-host failure") + def test_diagnoses_assertion_failure_after_tests(self) -> None: diagnosis = MODULE.diagnose( "✘ Test notification() recorded an issue\n" @@ -203,6 +229,48 @@ def test_full_suite_keeps_nonzero_app_host_exit_red_with_clean_summary(self) -> self.assertNotEqual(completed.returncode, 0) + def test_selected_suite_requires_positive_summary_and_keeps_log_artifact(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = pathlib.Path(temporary_directory) + fake_ci = root / "scripts/ci" + fake_ci.mkdir(parents=True) + shutil.copy2(SCRIPT, fake_ci / SCRIPT.name) + fake_runner = fake_ci / "xcodebuild_noninteractive.py" + fake_runner.write_text( + "#!/usr/bin/env python3\n" + "import os\n" + "if os.environ.get('FAKE_TEST_MODE') == 'compile':\n" + " print('Sources/Foo.swift:1:1: error: cannot find Missing in scope')\n" + " raise SystemExit(65)\n" + "print('Test run with 2 tests in 1 suite passed after 0.01 seconds.')\n", + encoding="utf-8", + ) + fake_runner.chmod(0o755) + + for mode, expected_success in (("pass", True), ("compile", False)): + results = root / f"results-{mode}" + environment = { + **os.environ, + "UNIT_TEST_SUITES": "Foo", + "TEST_RESULTS_ROOT": str(results), + "FAKE_TEST_MODE": mode, + } + completed = subprocess.run( + ["bash", "-c", TEST_DEPOT_RUN_UNIT_TESTS], + cwd=root, + env=environment, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(completed.returncode == 0, expected_success) + self.assertTrue((results / "Foo.log").is_file()) + self.assertTrue((results / "all-suites.log").is_file()) + if expected_success: + self.assertIn("category=tests passed", completed.stdout) + else: + self.assertIn("category=pre-test build/setup failure", completed.stdout) + def test_singular_summary_is_supported(self) -> None: passed, _ = MODULE.classify("Executed 1 test, with 0 failures (0 unexpected)\n")