Follow-up: finish Agent notification failure diagnostics

.github/workflows/agent-notification-tests.yml

@@ -15,6 +15,7 @@ on:
       - cmuxTests/**
       - Packages/macOS/CMUXAgentLaunch/**
       - .github/workflows/test-depot.yml
+      - scripts/ci/classify-app-host-test-output.py
       - Sources/AgentJournalLifecycleCenter*.swift
       - .github/workflows/agent-notification-tests.yml
 permissions:

scripts/ci/classify-app-host-test-output.py

@@ -28,10 +28,20 @@
     r"(?:test runner .*?(?:timed out|hung|failed)|"
     r"unexpected exit|communication with the test runner|"
     r"testmanagerd.*invalidated|Couldn't communicate with a helper|"
-    r"Fatal error:|Program crashed|Signal \d+|SIG[A-Z]+|"
+    r"Fatal error:|Program crashed|\*\*\*\s+Signal\s+\d+\b|"
     r"Idle timed out|Post-test timed out)",
     re.IGNORECASE,
 )
+_APP_HOST_SIGNAL_RE = re.compile(
+    r"(?:\*\*\*[^\n]*\bSignal\s+\d+\b|"
+    r"(?:received|terminated|killed|stopped|crashed|aborted|exited)[^\n]*"
+    r"\bsignal\s+\d+\b|^\s*signal\s+\d+\b|"
+    r"(?:\*\*\*[^\n]*|(?:received|terminated|killed|stopped|crashed)[^\n]*)"
+    r"\bSIG(?:ABRT|ALRM|BUS|CHLD|CONT|FPE|HUP|ILL|INT|IO|IOT|KILL|PIPE|POLL|"
+    r"PROF|QUIT|SEGV|STOP|SYS|TERM|TRAP|TSTP|TTIN|TTOU|URG|USR1|USR2|"
+    r"VTALRM|XCPU|XFSZ)\b)",
+    re.IGNORECASE,
+)
 _ASSERTION_RE = re.compile(
     r"(?:✘ Test .* recorded an issue|Expectation failed|"
     r"XCTAssert.*failed|Test run with .* failed|"
@@ -77,7 +87,9 @@ def diagnose(output: str, exit_code: Optional[int] = None) -> Dict[str, object]:
                 and "program crashed" not in cleaned.lower()
             ):
                 compile_line = cleaned
-        if app_host_line is None and _APP_HOST_FAILURE_RE.search(raw_line):
+        if app_host_line is None and (
+            _APP_HOST_FAILURE_RE.search(raw_line) or _APP_HOST_SIGNAL_RE.search(raw_line)
+        ):
             app_host_line = _clean_line(raw_line)
         if assertion_line is None and _ASSERTION_RE.search(raw_line):
             assertion_line = _clean_line(raw_line)

tests/test_ci_app_host_test_output.py

@@ -79,6 +79,16 @@ def test_diagnoses_app_host_failure_before_tests(self) -> None:
         self.assertEqual(diagnosis["category"], "pre-test app-host failure")
         self.assertEqual(diagnosis["executed_tests"], 0)
 
+    def test_ignores_build_signature_when_finding_app_host_cause(self) -> None:
+        diagnosis = MODULE.diagnose(
+            "Build description Signal 5 is a digest label, not a crash marker\n"
+            "The test runner timed out while preparing to run tests.\n",
+            exit_code=65,
+        )
+
+        self.assertEqual(diagnosis["category"], "pre-test app-host failure")
+        self.assertIn("test runner timed out", diagnosis["first_causal_line"])
+
     def test_diagnoses_crash_before_tests_as_app_host_failure(self) -> None:
         diagnosis = MODULE.diagnose(
             "Fatal error: Initial workspace creation failed\n"
@@ -89,6 +99,22 @@ def test_diagnoses_crash_before_tests_as_app_host_failure(self) -> None:
         self.assertEqual(diagnosis["category"], "pre-test app-host failure")
         self.assertIn("Fatal error", diagnosis["first_causal_line"])
 
+    def test_diagnoses_contextual_signal_name_as_app_host_failure(self) -> None:
+        diagnosis = MODULE.diagnose(
+            "Received signal SIGABRT from the app host\n",
+            exit_code=65,
+        )
+
+        self.assertEqual(diagnosis["category"], "pre-test app-host failure")
+
+    def test_diagnoses_numeric_signal_without_banner_in_crash_context(self) -> None:
+        diagnosis = MODULE.diagnose(
+            "terminated by signal 9\n",
+            exit_code=65,
+        )
+
+        self.assertEqual(diagnosis["category"], "pre-test app-host failure")
+
     def test_diagnoses_assertion_failure_after_tests(self) -> None:
         diagnosis = MODULE.diagnose(
             "✘ Test notification() recorded an issue\n"
@@ -203,6 +229,48 @@ def test_full_suite_keeps_nonzero_app_host_exit_red_with_clean_summary(self) ->
 
         self.assertNotEqual(completed.returncode, 0)
 
+    def test_selected_suite_requires_positive_summary_and_keeps_log_artifact(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary_directory:
+            root = pathlib.Path(temporary_directory)
+            fake_ci = root / "scripts/ci"
+            fake_ci.mkdir(parents=True)
+            shutil.copy2(SCRIPT, fake_ci / SCRIPT.name)
+            fake_runner = fake_ci / "xcodebuild_noninteractive.py"
+            fake_runner.write_text(
+                "#!/usr/bin/env python3\n"
+                "import os\n"
+                "if os.environ.get('FAKE_TEST_MODE') == 'compile':\n"
+                "    print('Sources/Foo.swift:1:1: error: cannot find Missing in scope')\n"
+                "    raise SystemExit(65)\n"
+                "print('Test run with 2 tests in 1 suite passed after 0.01 seconds.')\n",
+                encoding="utf-8",
+            )
+            fake_runner.chmod(0o755)
+
+            for mode, expected_success in (("pass", True), ("compile", False)):
+                results = root / f"results-{mode}"
+                environment = {
+                    **os.environ,
+                    "UNIT_TEST_SUITES": "Foo",
+                    "TEST_RESULTS_ROOT": str(results),
+                    "FAKE_TEST_MODE": mode,
+                }
+                completed = subprocess.run(
+                    ["bash", "-c", TEST_DEPOT_RUN_UNIT_TESTS],
+                    cwd=root,
+                    env=environment,
+                    capture_output=True,
+                    text=True,
+                    check=False,
+                )
+                self.assertEqual(completed.returncode == 0, expected_success)
+                self.assertTrue((results / "Foo.log").is_file())
+                self.assertTrue((results / "all-suites.log").is_file())
+                if expected_success:
+                    self.assertIn("category=tests passed", completed.stdout)
+                else:
+                    self.assertIn("category=pre-test build/setup failure", completed.stdout)
+
     def test_singular_summary_is_supported(self) -> None:
         passed, _ = MODULE.classify("Executed 1 test, with 0 failures (0 unexpected)\n")