From f30f2e1d4619016c7714047e7021d5bec0efc1c6 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 15:05:25 -0700 Subject: [PATCH 1/4] test: agent notification workflow must trigger only for the suites it runs Co-Authored-By: Claude Fable 5.1 --- tests/test_ci_self_hosted_guard.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index 8a9b4bf247bf..6ad5e4057e69 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -1292,6 +1292,29 @@ check_signing_intermediate_imports check_signing_intermediate_helper_behavior check_sentry_cli_install_portability check_sentry_cli_helper_behavior +check_agent_notification_paths_cover_its_suites() { + # The workflow reruns suites that ci.yml's shards already run, so it should + # start only for changes that can affect them. Each suite it names needs its + # own path trigger; a blanket cmuxTests/** trigger starts it for every test. + local file="$ROOT_DIR/.github/workflows/agent-notification-tests.yml" suite failed=0 + if grep -qE "^[[:space:]]+- cmuxTests/\*\*[[:space:]]*$" "$file"; then + echo "FAIL: agent-notification-tests.yml must not trigger on all of cmuxTests/**" + failed=1 + fi + for suite in $(sed -n 's/^[[:space:]]*unit_test_suites:[[:space:]]*//p' "$file" | tr ',' ' '); do + # AgentNotificationRegressionTests -> cmuxTests/AgentNotification*.swift + if ! grep -E "^[[:space:]]+- cmuxTests/" "$file" | sed -E 's/^[[:space:]]+- cmuxTests\///; s/\*.*$//' | while read -r prefix; do + case "$suite" in "$prefix"*) exit 1 ;; esac + done; then + continue + fi + echo "FAIL: agent-notification-tests.yml runs $suite but no cmuxTests/ path trigger matches it" + failed=1 + done + [ "$failed" -eq 0 ] || exit 1 + echo "PASS: agent notification paths cover exactly the suites the workflow runs" +} + check_dmg_signing_uses_build_keychain check_create_dmg_uses_run_local_npm_prefix check_gui_smoke_unsupported_launch_handling @@ -1300,3 +1323,4 @@ check_no_ci_swift_package_skips check_web_db_behavior_tests check_web_test_runner_behavior check_tmux_terminal_nightly_isolation +check_agent_notification_paths_cover_its_suites From 73b9c5d82e7d932d9bc0c9c8899f70e5eb3dbe70 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 15:07:13 -0700 Subject: [PATCH 2/4] ci: start the agent notification lane only for the suites it runs The workflow triggered on all of cmuxTests/**, so almost any pull request with a test file took two more macOS runners to rerun suites that ci.yml's shards already run. Trigger on the files that define its five suites instead. The guard checks real coverage: every file that declares or extends one of those suites must match a path trigger. Co-Authored-By: Claude Fable 5.1 --- .../workflows/agent-notification-tests.yml | 10 +++- tests/test_ci_self_hosted_guard.sh | 46 +++++++++++-------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/.github/workflows/agent-notification-tests.yml b/.github/workflows/agent-notification-tests.yml index 0ab5503c849c..6df911765686 100644 --- a/.github/workflows/agent-notification-tests.yml +++ b/.github/workflows/agent-notification-tests.yml @@ -12,7 +12,15 @@ on: - Sources/TerminalNotification*.swift - Sources/AgentFeedSemanticInput.swift - Sources/Feed/** - - cmuxTests/** + # Every file that defines a suite in unit_test_suites below. ci.yml's shards + # run all of cmuxTests, so a blanket cmuxTests/** only adds a second run. + - cmuxTests/*Notification*.swift + - cmuxTests/AgentRelayTTYOwnership*.swift + - cmuxTests/AgentDeliveryTTYBinding*.swift + - cmuxTests/AgentJournalLifecycleCenter*.swift + - cmuxTests/FeedWaiterRegistry*.swift + - cmuxTests/ClaudeBackgroundWorkNotify*.swift + - cmuxTests/OpenCodeHookRegression*.swift - Packages/macOS/CMUXAgentLaunch/** - .github/workflows/test-depot.yml - scripts/ci/classify-app-host-test-output.py diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index 6ad5e4057e69..b6f59c1ee3b9 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -1294,25 +1294,33 @@ check_sentry_cli_install_portability check_sentry_cli_helper_behavior check_agent_notification_paths_cover_its_suites() { # The workflow reruns suites that ci.yml's shards already run, so it should - # start only for changes that can affect them. Each suite it names needs its - # own path trigger; a blanket cmuxTests/** trigger starts it for every test. - local file="$ROOT_DIR/.github/workflows/agent-notification-tests.yml" suite failed=0 - if grep -qE "^[[:space:]]+- cmuxTests/\*\*[[:space:]]*$" "$file"; then - echo "FAIL: agent-notification-tests.yml must not trigger on all of cmuxTests/**" - failed=1 - fi - for suite in $(sed -n 's/^[[:space:]]*unit_test_suites:[[:space:]]*//p' "$file" | tr ',' ' '); do - # AgentNotificationRegressionTests -> cmuxTests/AgentNotification*.swift - if ! grep -E "^[[:space:]]+- cmuxTests/" "$file" | sed -E 's/^[[:space:]]+- cmuxTests\///; s/\*.*$//' | while read -r prefix; do - case "$suite" in "$prefix"*) exit 1 ;; esac - done; then - continue - fi - echo "FAIL: agent-notification-tests.yml runs $suite but no cmuxTests/ path trigger matches it" - failed=1 - done - [ "$failed" -eq 0 ] || exit 1 - echo "PASS: agent notification paths cover exactly the suites the workflow runs" + # start only for changes that can affect them: every file that defines one of + # its suites must match a path trigger, and cmuxTests/** must not be one. + ROOT_DIR="$ROOT_DIR" python3 - <<'PY' +import fnmatch, os, re, sys +from pathlib import Path + +root = Path(os.environ["ROOT_DIR"]) +text = (root / ".github/workflows/agent-notification-tests.yml").read_text(encoding="utf-8") +paths = re.findall(r"^\s+- (cmuxTests/\S+)\s*$", text, flags=re.M) +suites = re.search(r"^\s*unit_test_suites:\s*(\S+)", text, flags=re.M).group(1).split(",") +errors = [] +if "cmuxTests/**" in paths: + errors.append("must not trigger on all of cmuxTests/**") +for suite in suites: + decl = re.compile(rf"^\s*(?:@\w+(?:\([^)]*\))?\s+)*(?:\w+\s+)*(?:class|struct|actor|extension)\s+{re.escape(suite)}\b", re.M) + files = [f for f in sorted((root / "cmuxTests").glob("*.swift")) if decl.search(f.read_text(encoding="utf-8", errors="ignore"))] + if not files: + errors.append(f"runs {suite}, which no file in cmuxTests defines") + for f in files: + rel = f"cmuxTests/{f.name}" + if not any(fnmatch.fnmatchcase(rel, p) for p in paths): + errors.append(f"runs {suite} but {rel} matches no path trigger") +for e in errors: + print(f"FAIL: agent-notification-tests.yml {e}") +sys.exit(1 if errors else 0) +PY + echo "PASS: agent notification paths cover every file of the suites the workflow runs" } check_dmg_signing_uses_build_keychain From 50299a84cbd514bd08d447553dfbf9900420c718 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 15:50:30 -0700 Subject: [PATCH 3/4] ci: trigger agent notification tests on the helpers its suites use The narrowed paths covered the suite files only. Five shared helper files they depend on now trigger the lane too, and the guard fails when a suite file names a top-level type from an uncovered cmuxTests file. Co-Authored-By: Claude Fable 5.1 --- .../workflows/agent-notification-tests.yml | 6 +++++ tests/test_ci_self_hosted_guard.sh | 22 ++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/agent-notification-tests.yml b/.github/workflows/agent-notification-tests.yml index 6df911765686..aa01c1ca6273 100644 --- a/.github/workflows/agent-notification-tests.yml +++ b/.github/workflows/agent-notification-tests.yml @@ -21,6 +21,12 @@ on: - cmuxTests/FeedWaiterRegistry*.swift - cmuxTests/ClaudeBackgroundWorkNotify*.swift - cmuxTests/OpenCodeHookRegression*.swift + # Shared helpers those suite files use. + - cmuxTests/AgentJournalTestSupport.swift + - cmuxTests/BundledCLILinkageTests.swift + - cmuxTests/ClaudeHookLiveDeliveryTargetTestSupport.swift + - cmuxTests/ClaudeHookSurfaceResolutionSwiftTests.swift + - cmuxTests/TerminalControllingTTYWaiter.swift - Packages/macOS/CMUXAgentLaunch/** - .github/workflows/test-depot.yml - scripts/ci/classify-app-host-test-output.py diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index b6f59c1ee3b9..6927a09140f6 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -1295,7 +1295,8 @@ check_sentry_cli_helper_behavior check_agent_notification_paths_cover_its_suites() { # The workflow reruns suites that ci.yml's shards already run, so it should # start only for changes that can affect them: every file that defines one of - # its suites must match a path trigger, and cmuxTests/** must not be one. + # its suites, and every helper file those name, must match a path trigger, + # and cmuxTests/** must not be one. ROOT_DIR="$ROOT_DIR" python3 - <<'PY' import fnmatch, os, re, sys from pathlib import Path @@ -1307,20 +1308,35 @@ suites = re.search(r"^\s*unit_test_suites:\s*(\S+)", text, flags=re.M).group(1). errors = [] if "cmuxTests/**" in paths: errors.append("must not trigger on all of cmuxTests/**") +sources = {f: f.read_text(encoding="utf-8", errors="ignore") for f in sorted((root / "cmuxTests").glob("*.swift"))} +suite_files = set() for suite in suites: decl = re.compile(rf"^\s*(?:@\w+(?:\([^)]*\))?\s+)*(?:\w+\s+)*(?:class|struct|actor|extension)\s+{re.escape(suite)}\b", re.M) - files = [f for f in sorted((root / "cmuxTests").glob("*.swift")) if decl.search(f.read_text(encoding="utf-8", errors="ignore"))] + files = [f for f, source in sources.items() if decl.search(source)] if not files: errors.append(f"runs {suite}, which no file in cmuxTests defines") + suite_files.update(files) for f in files: rel = f"cmuxTests/{f.name}" if not any(fnmatch.fnmatchcase(rel, p) for p in paths): errors.append(f"runs {suite} but {rel} matches no path trigger") +# A helper is a file whose top-level type the suite files name. Nested and +# private types are skipped: another file cannot reach them, and several test +# files declare a private type of the same name. +suite_text = "\n".join(sources[f] for f in suite_files) +top_level = re.compile(r"^(?:@\w+(?:\([^)]*\))?\s+)*(?:(?:final|internal|public|open)\s+)*(?:class|struct|enum|actor|protocol)\s+(\w+)", re.M) +for f, source in sources.items(): + if f in suite_files: + continue + used = sorted(n for n in set(top_level.findall(source)) if re.search(rf"\b{re.escape(n)}\b", suite_text)) + rel = f"cmuxTests/{f.name}" + if used and not any(fnmatch.fnmatchcase(rel, p) for p in paths): + errors.append(f"suites use {', '.join(used)} from {rel}, which matches no path trigger") for e in errors: print(f"FAIL: agent-notification-tests.yml {e}") sys.exit(1 if errors else 0) PY - echo "PASS: agent notification paths cover every file of the suites the workflow runs" + echo "PASS: agent notification paths cover every suite file and helper the workflow runs" } check_dmg_signing_uses_build_keychain From 5d05be45556be7ffa6fd1a8f33c928d87e557633 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 19 Sep 2026 16:02:04 -0700 Subject: [PATCH 4/4] ci: list the agent notification test files instead of globbing them The *Notification* pattern matched 45 test files the lane does not run. The triggers now name the 12 suite files and 7 helpers, and the guard fails when a cmuxTests trigger matches any other file. Co-Authored-By: Claude Fable 5.1 --- .../workflows/agent-notification-tests.yml | 25 ++++++++++++------- tests/test_ci_self_hosted_guard.sh | 11 +++++++- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.github/workflows/agent-notification-tests.yml b/.github/workflows/agent-notification-tests.yml index aa01c1ca6273..6a4fab19cbfb 100644 --- a/.github/workflows/agent-notification-tests.yml +++ b/.github/workflows/agent-notification-tests.yml @@ -12,20 +12,27 @@ on: - Sources/TerminalNotification*.swift - Sources/AgentFeedSemanticInput.swift - Sources/Feed/** - # Every file that defines a suite in unit_test_suites below. ci.yml's shards - # run all of cmuxTests, so a blanket cmuxTests/** only adds a second run. - - cmuxTests/*Notification*.swift - - cmuxTests/AgentRelayTTYOwnership*.swift - - cmuxTests/AgentDeliveryTTYBinding*.swift - - cmuxTests/AgentJournalLifecycleCenter*.swift - - cmuxTests/FeedWaiterRegistry*.swift - - cmuxTests/ClaudeBackgroundWorkNotify*.swift - - cmuxTests/OpenCodeHookRegression*.swift + # The files that define a suite in unit_test_suites below. ci.yml's shards + # run all of cmuxTests, so a broader pattern only adds a second run. + - cmuxTests/AgentDeliveryTTYBindingTests.swift + - cmuxTests/AgentJournalLifecycleCenterTests.swift + - cmuxTests/AgentNotificationLiveRetargetTests.swift + - cmuxTests/AgentNotificationMoveRaceTests.swift + - cmuxTests/AgentNotificationMutationBoundaryTests.swift + - cmuxTests/AgentRelayTTYOwnershipTests.swift + - cmuxTests/AgentSemanticNotificationDeliveryTests.swift + - cmuxTests/ClaudeBackgroundWorkNotifyTests.swift + - cmuxTests/FeedWaiterRegistryTests.swift + - cmuxTests/MemoryPressureNotificationTests.swift + - cmuxTests/NativeNotificationFallbackCommandTests.swift + - cmuxTests/OpenCodeHookRegressionTests.swift # Shared helpers those suite files use. + - cmuxTests/AgentHookTestNotificationPipeline.swift - cmuxTests/AgentJournalTestSupport.swift - cmuxTests/BundledCLILinkageTests.swift - cmuxTests/ClaudeHookLiveDeliveryTargetTestSupport.swift - cmuxTests/ClaudeHookSurfaceResolutionSwiftTests.swift + - cmuxTests/ClaudeNotificationStatusLifecycleTests.swift - cmuxTests/TerminalControllingTTYWaiter.swift - Packages/macOS/CMUXAgentLaunch/** - .github/workflows/test-depot.yml diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index 6927a09140f6..119a7098acfe 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -1296,7 +1296,7 @@ check_agent_notification_paths_cover_its_suites() { # The workflow reruns suites that ci.yml's shards already run, so it should # start only for changes that can affect them: every file that defines one of # its suites, and every helper file those name, must match a path trigger, - # and cmuxTests/** must not be one. + # and no cmuxTests trigger may match any other file. ROOT_DIR="$ROOT_DIR" python3 - <<'PY' import fnmatch, os, re, sys from pathlib import Path @@ -1324,14 +1324,23 @@ for suite in suites: # private types are skipped: another file cannot reach them, and several test # files declare a private type of the same name. suite_text = "\n".join(sources[f] for f in suite_files) +helper_files = set() top_level = re.compile(r"^(?:@\w+(?:\([^)]*\))?\s+)*(?:(?:final|internal|public|open)\s+)*(?:class|struct|enum|actor|protocol)\s+(\w+)", re.M) for f, source in sources.items(): if f in suite_files: continue used = sorted(n for n in set(top_level.findall(source)) if re.search(rf"\b{re.escape(n)}\b", suite_text)) rel = f"cmuxTests/{f.name}" + if used: + helper_files.add(f) if used and not any(fnmatch.fnmatchcase(rel, p) for p in paths): errors.append(f"suites use {', '.join(used)} from {rel}, which matches no path trigger") +# A trigger that also matches unrelated test files starts a second run of +# suites that ci.yml already ran. +for p in paths: + extra = sorted(f.name for f in sources if fnmatch.fnmatchcase(f"cmuxTests/{f.name}", p) and f not in suite_files | helper_files) + if extra: + errors.append(f"trigger {p} also matches unrelated files: {', '.join(extra[:5])}") for e in errors: print(f"FAIL: agent-notification-tests.yml {e}") sys.exit(1 if errors else 0)