Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 0 additions & 73 deletions .github/workflows/agent-notification-tests.yml

This file was deleted.

47 changes: 43 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,9 @@ jobs:
- name: Validate app-host xcodebuild attempt budget
run: ./tests/test_ci_app_host_xcodebuild_attempts.sh

- name: Validate notification semantics gates
run: python3 tests/test_ci_notification_semantics.py

- name: Validate app-host test failure classification
run: python3 tests/test_ci_app_host_test_output.py

Expand Down Expand Up @@ -1191,7 +1194,6 @@ jobs:
-xctestrun "$CMUX_APP_HOST_XCTESTRUN" \
-destination "platform=macOS" \
CMUX_SKIP_ZIG_BUILD=1 \
-only-testing:cmuxTests/AgentNotificationRegressionTests \
-only-testing:cmuxTests/DockNotificationAttentionTests \
-only-testing:cmuxTests/ClaudeHookLifecycleCleanupTests \
-only-testing:cmuxTests/ClaudeHookLiveDeliveryTargetTests \
Expand Down Expand Up @@ -1846,6 +1848,35 @@ jobs:
with:
bun-version: "1.3.6"

- name: Run agent notification semantics
if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }}
run: |
set -euo pipefail
# Reuse compile admission's product instead of rebuilding the app in
# agent-notification-tests.yml. Keep each suite in a fresh host and
# require positive execution, outside the tolerant broad batches.
command -v node
bun_path="$(command -v bun)"
bun_install="$(dirname "$(dirname "$bun_path")")"
for suite in \
AgentNotificationRegressionTests \
AgentJournalLifecycleCenterTests \
FeedWaiterRegistryTests \
ClaudeBackgroundWorkNotifyTests \
OpenCodeHookRegressionTests
do
test_log="$RUNNER_TEMP/cmux-notification-${suite}.txt"
scripts/ci/run-in-console-session.sh \
env "TEST_RUNNER_PATH=$PATH" "TEST_RUNNER_BUN_INSTALL=$bun_install" \
Comment thread
teamleaderleo marked this conversation as resolved.
scripts/ci/run-app-host-xcodebuild.sh \
-xctestrun "$CMUX_APP_HOST_XCTESTRUN" \
-destination "platform=macOS" \
CMUX_SKIP_ZIG_BUILD=1 \
-only-testing:"cmuxTests/$suite" \
test-without-building 2>&1 | tee "$test_log"
scripts/ci/require_selected_test_execution.sh "$test_log" "cmuxTests/$suite"
done

- name: Install pinned nushell (shell-integration tests)
if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }}
run: |
Expand Down Expand Up @@ -2357,6 +2388,14 @@ jobs:
exit 1
fi
echo "::group::swift test $pkgdir"
swift_test_args=(--package-path "$pkgdir")
# Preserve the notification workflow's warning gate without a
# second package build or changing the existing startup retry.
case "$pkg" in
CMUXAgentLaunch|CmuxAgentJournal)
swift_test_args+=(-Xswiftc -warnings-as-errors)
;;
esac
case "$pkg" in
# CmuxFoundation has several process-tree suites whose child
# fixtures share global process resources; run each suite in its
Expand All @@ -2366,7 +2405,7 @@ jobs:
;;
CmuxTerminal|CmuxTerminalCore)
test_status=0
output="$(swift test --package-path "$pkgdir" 2>&1)" || test_status=$?
output="$(swift test "${swift_test_args[@]}" 2>&1)" || test_status=$?
printf '%s\n' "$output"
if [ "$test_status" -ne 0 ]; then
if printf '%s\n' "$output" | grep -Eq 'Test run with [0-9]+ tests( in [0-9]+ suites)? passed' \
Expand All @@ -2385,15 +2424,15 @@ jobs:
# test failure: retry exactly once, and only when no test
# output was emitted.
test_status=0
output="$(swift test --package-path "$pkgdir" 2>&1)" || test_status=$?
output="$(swift test "${swift_test_args[@]}" 2>&1)" || test_status=$?
printf '%s\n' "$output"
if [ "$test_status" -ne 0 ] \
&& printf '%s\n' "$output" | grep -Fq 'Build complete!' \
&& printf '%s\n' "$output" | grep -Eq 'Exited with unexpected signal code [56]([^0-9]|$)' \
&& ! printf '%s\n' "$output" | grep -Eq '^(Test Suite|Test Case|◇ |↳ |✔ |✘ )'; then
echo "Test runner crashed at startup (runner flake); retrying $pkg once."
test_status=0
output="$(swift test --package-path "$pkgdir" 2>&1)" || test_status=$?
output="$(swift test "${swift_test_args[@]}" 2>&1)" || test_status=$?
printf '%s\n' "$output"
fi
if [ "$test_status" -ne 0 ]; then
Expand Down
5 changes: 5 additions & 0 deletions scripts/ci/cmux_unit_test_shard.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@
DEFAULT_TIMINGS_PATH = Path(__file__).resolve().parent / "cmux-unit-test-timings.json"
FALLBACK_TEST_MS = 200
FOCUSED_GATE_SELECTORS = {
"cmuxTests/AgentNotificationRegressionTests",
"cmuxTests/AgentJournalLifecycleCenterTests",
"cmuxTests/FeedWaiterRegistryTests",
"cmuxTests/ClaudeBackgroundWorkNotifyTests",
"cmuxTests/OpenCodeHookRegressionTests",
"cmuxTests/AgentRestoreLiveOwnerAdmissionTests",
"cmuxTests/BrowserSystemProxyMirrorTests",
"cmuxTests/CLISSHSessionAttachAnchorTests",
Expand Down
9 changes: 9 additions & 0 deletions scripts/ci/run-app-host-xcodebuild.sh
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,15 @@ fi
if [ -n "${GITHUB_ACTIONS:-}" ]; then
app_host_test_runner_environment+=("TEST_RUNNER_GITHUB_ACTIONS=$GITHUB_ACTIONS")
fi
# Focused app-host suites invoke Node/Bun-backed helpers from the test process.
# Xcode does not inherit these driver variables, so carry them through the
# TEST_RUNNER_ channel when the caller supplied them.
if [ -n "${TEST_RUNNER_PATH:-}" ]; then
app_host_test_runner_environment+=("TEST_RUNNER_PATH=$TEST_RUNNER_PATH")
fi
if [ -n "${TEST_RUNNER_BUN_INSTALL:-}" ]; then
app_host_test_runner_environment+=("TEST_RUNNER_BUN_INSTALL=$TEST_RUNNER_BUN_INSTALL")
fi
app_host_home=""
app_host_key=""
app_host_receipt_dir=""
Expand Down
13 changes: 13 additions & 0 deletions tests/test_ci_app_host_xcodebuild_retry.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ if [ "${CMUX_MOCK_XCODEBUILD_PROCESS:-0}" = "1" ]; then
"${TEST_RUNNER_CMUX_APP_HOST_KEY:-<unset>}" \
"${TEST_RUNNER_CMUX_APP_HOST_RECEIPT_DIR:-<unset>}" \
>> "$CMUX_CAPTURE_TEST_RUNNER_HOME_ENV"
if [ -n "${CMUX_CAPTURE_TEST_RUNNER_TOOL_ENV:-}" ]; then
printf '%s|%s\n' "${TEST_RUNNER_PATH-<unset>}" "${TEST_RUNNER_BUN_INSTALL-<unset>}" \
>> "$CMUX_CAPTURE_TEST_RUNNER_TOOL_ENV"
fi
config_home="${TEST_RUNNER_HOME:-${HOME:-/tmp}}"
config_category=default
config_message="reading configuration file"
Expand Down Expand Up @@ -250,12 +254,15 @@ CMUX_CAPTURE_XCODEBUILD_ARGS="$TMP_DIR/xcodebuild-args.log" \
CMUX_CAPTURE_TEST_RUNNER_ENV="$TMP_DIR/test-runner-env.log" \
CMUX_CAPTURE_XCODEBUILD_PARENT_ENV="$TMP_DIR/xcodebuild-parent-env.log" \
CMUX_CAPTURE_TEST_RUNNER_HOME_ENV="$TMP_DIR/test-runner-home-env.log" \
CMUX_CAPTURE_TEST_RUNNER_TOOL_ENV="$TMP_DIR/test-runner-tool-env.log" \
CMUX_MOCK_XCODEBUILD_PROCESS=1 \
CMUX_APP_HOST_XCODEBUILD_ATTEMPTS=2 \
CMUX_XCODEBUILD_NONINTERACTIVE_IDLE_TIMEOUT_SECONDS=0.1 \
CMUX_CI_APP_HOST_ISOLATION_REQUIRED=1 \
CMUX_APP_HOST_HOME="$APP_HOST_HOME" \
CMUX_APP_HOST_XDG_CONFIG_HOME="$APP_HOST_XDG_CONFIG_HOME" \
TEST_RUNNER_PATH="/ci/node/bin:/usr/bin" \
TEST_RUNNER_BUN_INSTALL="/ci/bun" \
CFFIXED_USER_HOME="$XCODE_PARENT_FIXED_HOME" \
XDG_CONFIG_HOME="$XCODE_PARENT_XDG_CONFIG_HOME" \
bash "$ROOT_DIR/scripts/ci/run-app-host-xcodebuild.sh" test >"$TMP_DIR/output.log" 2>&1
Expand Down Expand Up @@ -289,6 +296,12 @@ if [ "$runner_marker_count" -eq 0 ] || [ "$runner_marker_count" -ne "$invocation
exit 1
fi

if [ "$(grep -Fxc '/ci/node/bin:/usr/bin|/ci/bun' "$TMP_DIR/test-runner-tool-env.log" || true)" -ne "$invocation_count" ]; then
cat "$TMP_DIR/test-runner-tool-env.log"
echo "FAIL: focused test-runner tool paths must reach every app-host launch"
exit 1
fi

isolated_parent_count="$(awk -F '|' -v isolated="$APP_HOST_HOME" '
$1 == isolated { count += 1 }
END { print count + 0 }
Expand Down
Loading
Loading