From 994a62023a677fda5e584a24bc704f660ad9a5ce Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Mon, 3 Aug 2026 20:17:03 -0700 Subject: [PATCH 1/4] Test app-host CI failure accounting --- tests/test_ci_app_host_xcodebuild_retry.sh | 190 ++++++++++++++++++++- 1 file changed, 189 insertions(+), 1 deletion(-) diff --git a/tests/test_ci_app_host_xcodebuild_retry.sh b/tests/test_ci_app_host_xcodebuild_retry.sh index 088ecbaa2798..0594f5623d0a 100755 --- a/tests/test_ci_app_host_xcodebuild_retry.sh +++ b/tests/test_ci_app_host_xcodebuild_retry.sh @@ -51,4 +51,192 @@ if [ "$runner_marker_count" -eq 0 ] || [ "$runner_marker_count" -ne "$invocation exit 1 fi -echo "PASS: app-host xcodebuild wrapper retries idle timeouts" +cat > "$TMP_DIR/xcodebuild" <<'SH' +#!/usr/bin/env bash +printf 'invoked\n' >> "$CMUX_CAPTURE_XCODEBUILD_INVOCATIONS" +case "$CMUX_XCODEBUILD_NONINTERACTIVE_LOG_PATH" in + *-attempt-1.log) + printf '%s\n' \ + "SocketControlServer: Listening on /tmp/cmux-test.sock" \ + "Test Suite 'ExampleTests' failed" \ + "Executed 1 test, with 1 failure (0 unexpected)" \ + "Failed to establish communication with the test runner" + exit 65 + ;; + *) + printf '%s\n' \ + "SocketControlServer: Listening on /tmp/cmux-test.sock" \ + "Test Suite 'Selected tests' passed" \ + "Executed 1 test, with 0 failures (0 unexpected)" + ;; +esac +SH +chmod +x "$TMP_DIR/xcodebuild" + +regression_failures=0 +set +e +PATH="$TMP_DIR:$PATH" \ +RUNNER_TEMP="$TMP_DIR" \ +CMUX_CAPTURE_XCODEBUILD_INVOCATIONS="$TMP_DIR/assertion-retry-invocations.log" \ +CMUX_APP_HOST_XCODEBUILD_ATTEMPTS=2 \ + bash "$ROOT_DIR/scripts/ci/run-app-host-xcodebuild.sh" test >"$TMP_DIR/assertion-retry-output.log" 2>&1 +assertion_retry_status=$? +set -e + +if [ "$assertion_retry_status" -eq 0 ]; then + cat "$TMP_DIR/assertion-retry-output.log" + echo "FAIL: a clean retry masked an XCTest assertion failure from the first attempt" + regression_failures=$((regression_failures + 1)) +fi + +assertion_retry_invocations="$(wc -l < "$TMP_DIR/assertion-retry-invocations.log" | tr -d ' ')" +if [ "$assertion_retry_invocations" -ne 1 ]; then + cat "$TMP_DIR/assertion-retry-output.log" + echo "FAIL: an app-host assertion failure must stop retries; got $assertion_retry_invocations invocations" + regression_failures=$((regression_failures + 1)) +fi + +cat > "$TMP_DIR/xcodebuild" <<'SH' +#!/usr/bin/env bash +printf '%s\n' \ + "SocketControlServer: Listening on /tmp/cmux-test.sock" \ + "Test Suite 'ExampleTests' failed" \ + "Executed 1 test, with 1 failure (0 unexpected)" \ + "Restarting after unexpected exit, crash, or test timeout; summary will include totals from previous launches." \ + "Test Suite 'Selected tests' passed" \ + "Executed 1 test, with 0 failures (0 unexpected)" +SH +chmod +x "$TMP_DIR/xcodebuild" + +set +e +PATH="$TMP_DIR:$PATH" \ +RUNNER_TEMP="$TMP_DIR" \ +CMUX_APP_HOST_XCODEBUILD_ATTEMPTS=1 \ + bash "$ROOT_DIR/scripts/ci/run-app-host-xcodebuild.sh" test >"$TMP_DIR/xctest-summary-loss-output.log" 2>&1 +xctest_summary_loss_status=$? +set -e + +if [ "$xctest_summary_loss_status" -eq 0 ]; then + cat "$TMP_DIR/xctest-summary-loss-output.log" + echo "FAIL: a final clean XCTest summary masked an earlier assertion failure" + regression_failures=$((regression_failures + 1)) +fi + +cat > "$TMP_DIR/xcodebuild" <<'SH' +#!/usr/bin/env bash +printf '%s\n' \ + "SocketControlServer: Listening on /tmp/cmux-test.sock" \ + "Test Suite 'Selected tests' passed" \ + "Executed 1 test, with 0 failures (0 unexpected)" \ + "✘ Test example() recorded an issue at ExampleTests.swift:12:3: Expectation failed" \ + "✘ Test example() failed after 0.001 seconds with 1 issue." +SH +chmod +x "$TMP_DIR/xcodebuild" + +set +e +PATH="$TMP_DIR:$PATH" \ +RUNNER_TEMP="$TMP_DIR" \ +CMUX_APP_HOST_XCODEBUILD_ATTEMPTS=1 \ + bash "$ROOT_DIR/scripts/ci/run-app-host-xcodebuild.sh" test >"$TMP_DIR/swift-testing-summary-loss-output.log" 2>&1 +swift_testing_summary_loss_status=$? +set -e + +if [ "$swift_testing_summary_loss_status" -eq 0 ]; then + cat "$TMP_DIR/swift-testing-summary-loss-output.log" + echo "FAIL: a final clean XCTest summary masked a Swift Testing assertion failure" + regression_failures=$((regression_failures + 1)) +fi + +set +e +ROOT_DIR="$ROOT_DIR" python3 <<'PY' +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import sys +import tempfile + +import yaml + + +root = Path(os.environ["ROOT_DIR"]) +workflow = yaml.safe_load((root / ".github/workflows/ci.yml").read_text()) +steps = workflow["jobs"]["app-host-unit-tests"]["steps"] +run_unit_tests = next(step["run"] for step in steps if step.get("name") == "Run unit tests") +run_unit_tests = run_unit_tests.replace("${{ matrix.shard }}", "1") + +with tempfile.TemporaryDirectory() as raw_tmp: + fixture = Path(raw_tmp) + scripts = fixture / "scripts/ci" + scripts.mkdir(parents=True) + runner_temp = fixture / "runner-temp" + runner_temp.mkdir() + + (scripts / "cmux_unit_test_shard.py").write_text( + """\ +from pathlib import Path +import sys + +output = Path(sys.argv[sys.argv.index("--output") + 1]) +output.write_text("-only-testing:cmuxTests/ExampleTests\\n") +""" + ) + (scripts / "run-in-console-session.sh").write_text( + """\ +#!/usr/bin/env bash +set -euo pipefail +exec "$@" +""" + ) + (scripts / "run-app-host-xcodebuild.sh").write_text( + """\ +#!/usr/bin/env bash +printf '%s\\n' \\ + "Test Suite 'ExampleTests' failed" \\ + "Executed 2 tests, with 1 failure (0 unexpected)" \\ + "Test Suite 'Selected tests' passed" \\ + "Executed 2 tests, with 0 failures (0 unexpected)" +exit 65 +""" + ) + (scripts / "run-in-console-session.sh").chmod(0o755) + (scripts / "run-app-host-xcodebuild.sh").chmod(0o755) + + env = os.environ.copy() + env.update( + RUNNER_TEMP=str(runner_temp), + CMUX_DERIVED_DATA_PATH=str(fixture / "derived-data"), + CMUX_UNIT_TEST_TIMEOUT_SECONDS="30", + ) + result = subprocess.run( + ["bash", "-c", run_unit_tests], + cwd=fixture, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + +if result.returncode == 0: + print(result.stdout, end="") + print( + "FAIL: the app-host unit-test workflow treated an ordinary assertion " + "failure with '(0 unexpected)' as a pass" + ) + sys.exit(1) +PY +workflow_failure_status=$? +set -e + +if [ "$workflow_failure_status" -ne 0 ]; then + regression_failures=$((regression_failures + 1)) +fi + +if [ "$regression_failures" -ne 0 ]; then + echo "FAIL: detected $regression_failures app-host test failure-accounting regressions" + exit 1 +fi + +echo "PASS: app-host xcodebuild retries infrastructure failures without masking test failures" From 5b01f3c9cbf501a10f8ebd64c287df3ef7bb57ee Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Mon, 3 Aug 2026 20:46:27 -0700 Subject: [PATCH 2/4] Fail app-host CI on any test assertion --- .github/workflows/ci.yml | 84 ++++++++++----------------- scripts/ci/run-app-host-xcodebuild.sh | 16 +++++ 2 files changed, 48 insertions(+), 52 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8624d66f683b..5b9c5d1c0c4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -605,10 +605,9 @@ jobs: - name: Run agent chat transcript lifecycle regressions if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} run: | - # Swift Testing assertion failures are tolerated in the full sharded - # app-host suite. Keep hook-driven transcript resolution on a - # non-tolerant focused invocation so Feed ingress cannot regain an - # unbounded recursive filesystem scan. + # Keep hook-driven transcript resolution isolated from unrelated + # app-host crashes so Feed ingress cannot regain an unbounded + # recursive filesystem scan. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" for suite in \ @@ -630,10 +629,8 @@ jobs: - name: Run browser runtime viewport regression if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} run: | - # The tolerant full-suite step can accept ordinary assertion failures - # when XCTest reports "0 unexpected". Keep the real-WKWebView DOM - # viewport check non-tolerant so a layout-only screenshot workaround - # cannot make this regression appear green. + # Keep the real-WKWebView DOM viewport check isolated so an unrelated + # app-host abort cannot prevent it from executing. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" scripts/ci/run-in-console-session.sh \ @@ -651,8 +648,7 @@ jobs: if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} run: | # Use a dedicated app-host process so task_vm_info deltas only compare - # one versus five real Ghostty renderers in this workload. The focused - # invocation also makes footprint assertion failures non-tolerant. + # one versus five real Ghostty renderers in this workload. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" CMUX_RENDERER_MEMORY_REGRESSION=1 \ @@ -670,9 +666,8 @@ jobs: - name: Run notification routing regressions if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} run: | - # The tolerant full-suite step can accept assertion failures or stop - # after an app-host crash before this suite runs. Keep notification - # move/clear causality coverage on a non-tolerant focused invocation. + # Keep notification move/clear causality coverage isolated from + # unrelated app-host crashes in the full shard. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" scripts/ci/run-in-console-session.sh \ @@ -695,9 +690,8 @@ jobs: - name: Run Pi Feed ownership regressions if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} run: | - # Swift Testing assertion failures are tolerated in the full sharded - # app-host suite. Keep Feed ingestion and ownership on a non-tolerant - # focused invocation so valid Pi Feed targets cannot silently regress. + # Keep Feed ingestion and ownership isolated so an unrelated app-host + # abort cannot prevent these regressions from executing. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" # Each suite mutates process-global app/store state. Give every suite @@ -725,8 +719,8 @@ jobs: run: | # Closing a mirrored workspace must never kill the remote tmux session, # and --new-window must consolidate mirrors moved across source windows. - # Keep these destructive/topology regressions outside the tolerant full - # suite so an ordinary assertion failure cannot be accepted as expected. + # Keep these destructive/topology regressions in a dedicated app-host + # process so unrelated shard state cannot perturb them. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" scripts/ci/run-in-console-session.sh \ @@ -744,11 +738,9 @@ jobs: if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} run: | # Focused gate for https://github.com/manaflow-ai/cmux/issues/5888. - # The full "Run unit tests" step tolerates app-host crashes by - # parsing only the last test summary, so suites that run late can - # otherwise be skipped by an app-host abort that still reports - # "(0 unexpected)". Keep this targeted only-testing invocation as a - # non-tolerant guard for the system proxy mirror regression. + # Keep this targeted only-testing invocation so an unrelated + # app-host abort cannot prevent the system proxy mirror regression + # from executing. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" scripts/ci/run-in-console-session.sh \ @@ -782,13 +774,8 @@ jobs: if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} run: | # Focused gate for https://github.com/manaflow-ai/cmux/issues/7367. - # The full "Run unit tests" step tolerates any xcodebuild failure - # whose last test summary reports "(0 unexpected)", and test - # assertion failures never increment the unexpected count on these - # runners — so a failing regression assertion in the sharded run - # cannot turn the shard red. Keep this targeted only-testing - # invocation as a non-tolerant guard for the ssh-session-attach - # split-anchor regression. + # Keep the ssh-session-attach split-anchor regression isolated so an + # unrelated app-host abort cannot prevent it from executing. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" scripts/ci/run-in-console-session.sh \ @@ -806,9 +793,8 @@ jobs: if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} run: | # Focused gates for https://github.com/manaflow-ai/cmux/issues/7380 and #8054. - # The tolerant full-suite step can crash-skip late suites (see #5888 - # note above), so these regressions get a non-tolerant focused - # invocation. + # Keep these regressions isolated so an unrelated app-host abort + # cannot prevent them from executing. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" scripts/ci/run-in-console-session.sh \ @@ -827,9 +813,8 @@ jobs: if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} run: | # Focused gates for https://github.com/manaflow-ai/cmux/issues/7372. - # The tolerant full-suite step can crash-skip late suites (see #5888 - # note above), so the seed and immediate-update paths get a - # non-tolerant invocation. + # Keep the seed and immediate-update paths isolated so an unrelated + # app-host abort cannot prevent them from executing. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" scripts/ci/run-in-console-session.sh \ @@ -846,9 +831,8 @@ jobs: - name: Run remote tmux mirror layout identity regression if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} run: | - # The tolerant full-suite step may accept expected failures or stop - # after an app-host crash. Keep #7833's identity suite non-tolerant - # so a missing incremental-reconcile regression cannot pass the shard. + # Keep #7833's identity suite isolated so an unrelated app-host abort + # cannot prevent it from executing. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" scripts/ci/run-in-console-session.sh \ @@ -865,9 +849,8 @@ jobs: - name: Run socket ACL reload regression if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }} run: | - # The tolerant full-suite step can accept assertion failures or - # crash-skip late suites. Keep #7984's live reload and denial tests - # non-tolerant so a snapshotted socket ACL cannot pass the shard. + # Keep #7984's live reload and denial tests isolated so an unrelated + # app-host abort cannot prevent them from executing. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" scripts/ci/run-in-console-session.sh \ @@ -948,8 +931,8 @@ jobs: wait "$xcodebuild_pid" } - # Stream output via tee so CI logs are visible in real time, while still - # capturing for post-run analysis of expected vs unexpected failures. + # Stream output via tee so CI logs are visible in real time while still + # capturing package-resolution diagnostics for the bounded retry below. TEST_OUTPUT="$RUNNER_TEMP/cmux-unit-output-shard-${{ matrix.shard }}.txt" set +e run_unit_tests | tee "$TEST_OUTPUT" @@ -960,7 +943,9 @@ jobs: # SwiftPM binary artifact resolution can occasionally fail on ephemeral # runners with "Could not resolve package dependencies". Retry once after # clearing SwiftPM/DerivedData caches to recover from transient corruption. - if [ "$EXIT_CODE" -ne 0 ] && echo "$OUTPUT" | grep -q "Could not resolve package dependencies"; then + if [ "$EXIT_CODE" -ne 0 ] \ + && [[ "$OUTPUT" != *"FAIL: app-host test assertion failure detected"* ]] \ + && echo "$OUTPUT" | grep -q "Could not resolve package dependencies"; then echo "SwiftPM package resolution failed, clearing caches and retrying once" rm -rf ~/Library/Caches/org.swift.swiftpm mkdir -p ~/Library/Caches/org.swift.swiftpm @@ -974,13 +959,8 @@ jobs: fi if [ "$EXIT_CODE" -ne 0 ]; then - SUMMARY=$(echo "$OUTPUT" | grep "Executed.*tests.*with.*failures" | tail -1) - if echo "$SUMMARY" | grep -q "(0 unexpected)"; then - echo "All failures are expected, treating as pass" - else - echo "Unexpected test failures detected" - exit 1 - fi + echo "App-host unit tests failed with exit code $EXIT_CODE" + exit "$EXIT_CODE" fi - name: Run bundled Ghostty theme picker helper regression diff --git a/scripts/ci/run-app-host-xcodebuild.sh b/scripts/ci/run-app-host-xcodebuild.sh index 3b9a8a3749e5..a967b298412e 100755 --- a/scripts/ci/run-app-host-xcodebuild.sh +++ b/scripts/ci/run-app-host-xcodebuild.sh @@ -51,6 +51,11 @@ kill_stale_app_host() { pkill -f "${ci_app_host_root%/}/.*Build/Products/.*cmux DEV" 2>/dev/null || true } +test_assertion_failure_pattern='Executed [0-9]+ tests?, with [1-9][0-9]* failures?|Test Case .* failed|✘ Test .* (recorded an issue|failed after)|✘ Suite .* failed after|Test run with .* failed' +log_contains_test_assertion_failure() { + LC_ALL=C grep -Eq "$test_assertion_failure_pattern" "$1" +} + attempt=1 while [ "$attempt" -le "$max_attempts" ]; do log_path="${log_stem}-attempt-${attempt}.log" @@ -78,6 +83,17 @@ while [ "$attempt" -le "$max_attempts" ]; do exit 1 fi + # xcodebuild can restart an app host after a crash and finish with a clean + # summary for only the remaining tests. The noninteractive timeout helper can + # likewise return success after a passing XCTest summary even when a later + # Swift Testing phase reported issues. Inspect every attempt's complete log + # before considering either an infrastructure retry or a successful exit. + if log_contains_test_assertion_failure "$log_path"; then + echo "FAIL: app-host test assertion failure detected in attempt $attempt; refusing to continue" >&2 + LC_ALL=C grep -E "$test_assertion_failure_pattern" "$log_path" | tail -20 >&2 || true + exit 1 + fi + if [ "$status" -ne 0 ]; then retry_reason="" if [ "$status" -eq 124 ]; then From 69d3226d5e5c26bbf6ab5e7c188a5fe823142ab4 Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Mon, 3 Aug 2026 21:59:42 -0700 Subject: [PATCH 3/4] Sanitize app-host assertion diagnostics --- scripts/ci/run-app-host-xcodebuild.sh | 3 ++- tests/test_ci_app_host_xcodebuild_retry.sh | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/ci/run-app-host-xcodebuild.sh b/scripts/ci/run-app-host-xcodebuild.sh index a967b298412e..b092aa62e303 100755 --- a/scripts/ci/run-app-host-xcodebuild.sh +++ b/scripts/ci/run-app-host-xcodebuild.sh @@ -90,7 +90,8 @@ while [ "$attempt" -le "$max_attempts" ]; do # before considering either an infrastructure retry or a successful exit. if log_contains_test_assertion_failure "$log_path"; then echo "FAIL: app-host test assertion failure detected in attempt $attempt; refusing to continue" >&2 - LC_ALL=C grep -E "$test_assertion_failure_pattern" "$log_path" | tail -20 >&2 || true + marker_count="$(LC_ALL=C grep -Ec "$test_assertion_failure_pattern" "$log_path")" + echo "Detected ${marker_count} XCTest or Swift Testing failure markers." >&2 exit 1 fi diff --git a/tests/test_ci_app_host_xcodebuild_retry.sh b/tests/test_ci_app_host_xcodebuild_retry.sh index 0594f5623d0a..604d393ad261 100755 --- a/tests/test_ci_app_host_xcodebuild_retry.sh +++ b/tests/test_ci_app_host_xcodebuild_retry.sh @@ -89,6 +89,12 @@ if [ "$assertion_retry_status" -eq 0 ]; then regression_failures=$((regression_failures + 1)) fi +if ! grep -Fq "Detected 1 XCTest or Swift Testing failure markers." "$TMP_DIR/assertion-retry-output.log"; then + cat "$TMP_DIR/assertion-retry-output.log" + echo "FAIL: assertion diagnostics must report a sanitized marker count" + regression_failures=$((regression_failures + 1)) +fi + assertion_retry_invocations="$(wc -l < "$TMP_DIR/assertion-retry-invocations.log" | tr -d ' ')" if [ "$assertion_retry_invocations" -ne 1 ]; then cat "$TMP_DIR/assertion-retry-output.log" From 1f7639c9648679c09c98455940539194b66c687e Mon Sep 17 00:00:00 2001 From: austinpower1258 Date: Mon, 3 Aug 2026 22:54:11 -0700 Subject: [PATCH 4/4] Preserve app-host test failure status --- scripts/ci/run-app-host-xcodebuild.sh | 3 + tests/test_ci_app_host_xcodebuild_retry.sh | 88 ++++++++++++++++++---- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/scripts/ci/run-app-host-xcodebuild.sh b/scripts/ci/run-app-host-xcodebuild.sh index b092aa62e303..24bfc41d79e8 100755 --- a/scripts/ci/run-app-host-xcodebuild.sh +++ b/scripts/ci/run-app-host-xcodebuild.sh @@ -92,6 +92,9 @@ while [ "$attempt" -le "$max_attempts" ]; do echo "FAIL: app-host test assertion failure detected in attempt $attempt; refusing to continue" >&2 marker_count="$(LC_ALL=C grep -Ec "$test_assertion_failure_pattern" "$log_path")" echo "Detected ${marker_count} XCTest or Swift Testing failure markers." >&2 + if [ "$status" -ne 0 ]; then + exit "$status" + fi exit 1 fi diff --git a/tests/test_ci_app_host_xcodebuild_retry.sh b/tests/test_ci_app_host_xcodebuild_retry.sh index 604d393ad261..5c7f7ad416de 100755 --- a/tests/test_ci_app_host_xcodebuild_retry.sh +++ b/tests/test_ci_app_host_xcodebuild_retry.sh @@ -79,29 +79,89 @@ PATH="$TMP_DIR:$PATH" \ RUNNER_TEMP="$TMP_DIR" \ CMUX_CAPTURE_XCODEBUILD_INVOCATIONS="$TMP_DIR/assertion-retry-invocations.log" \ CMUX_APP_HOST_XCODEBUILD_ATTEMPTS=2 \ - bash "$ROOT_DIR/scripts/ci/run-app-host-xcodebuild.sh" test >"$TMP_DIR/assertion-retry-output.log" 2>&1 + bash "$ROOT_DIR/scripts/ci/run-app-host-xcodebuild.sh" test \ + >"$TMP_DIR/assertion-retry-stream.log" \ + 2>"$TMP_DIR/assertion-retry-diagnostics.log" assertion_retry_status=$? set -e -if [ "$assertion_retry_status" -eq 0 ]; then - cat "$TMP_DIR/assertion-retry-output.log" - echo "FAIL: a clean retry masked an XCTest assertion failure from the first attempt" +if [ "$assertion_retry_status" -ne 65 ]; then + cat "$TMP_DIR/assertion-retry-stream.log" + cat "$TMP_DIR/assertion-retry-diagnostics.log" + echo "FAIL: expected assertion failure status 65, got $assertion_retry_status" regression_failures=$((regression_failures + 1)) fi -if ! grep -Fq "Detected 1 XCTest or Swift Testing failure markers." "$TMP_DIR/assertion-retry-output.log"; then - cat "$TMP_DIR/assertion-retry-output.log" +if ! grep -Fq "Detected 1 XCTest or Swift Testing failure markers." "$TMP_DIR/assertion-retry-diagnostics.log"; then + cat "$TMP_DIR/assertion-retry-diagnostics.log" echo "FAIL: assertion diagnostics must report a sanitized marker count" regression_failures=$((regression_failures + 1)) fi +if grep -Fq "Test Suite 'ExampleTests' failed" "$TMP_DIR/assertion-retry-diagnostics.log" \ + || grep -Fq "Executed 1 test, with 1 failure (0 unexpected)" "$TMP_DIR/assertion-retry-diagnostics.log"; then + cat "$TMP_DIR/assertion-retry-diagnostics.log" + echo "FAIL: wrapper diagnostics must not repeat raw xcodebuild failure lines" + regression_failures=$((regression_failures + 1)) +fi + assertion_retry_invocations="$(wc -l < "$TMP_DIR/assertion-retry-invocations.log" | tr -d ' ')" if [ "$assertion_retry_invocations" -ne 1 ]; then - cat "$TMP_DIR/assertion-retry-output.log" + cat "$TMP_DIR/assertion-retry-diagnostics.log" echo "FAIL: an app-host assertion failure must stop retries; got $assertion_retry_invocations invocations" regression_failures=$((regression_failures + 1)) fi +cat > "$TMP_DIR/xcodebuild" <<'SH' +#!/usr/bin/env bash +printf 'invoked\n' >> "$CMUX_CAPTURE_XCODEBUILD_INVOCATIONS" +case "$CMUX_XCODEBUILD_NONINTERACTIVE_LOG_PATH" in + *-attempt-1.log) + printf '%s\n' \ + "SocketControlServer: Listening on /tmp/cmux-test.sock" \ + "Test Suite 'Selected tests' passed" \ + "Executed 1 test, with 0 failures (0 unexpected)" \ + "Failed to establish communication with the test runner" + exit 65 + ;; + *) + printf '%s\n' \ + "SocketControlServer: Listening on /tmp/cmux-test.sock" \ + "Test Suite 'Selected tests' passed" \ + "Executed 1 test, with 0 failures (0 unexpected)" + ;; +esac +SH +chmod +x "$TMP_DIR/xcodebuild" + +set +e +PATH="$TMP_DIR:$PATH" \ +RUNNER_TEMP="$TMP_DIR" \ +CMUX_CAPTURE_XCODEBUILD_INVOCATIONS="$TMP_DIR/communication-retry-invocations.log" \ +CMUX_APP_HOST_XCODEBUILD_ATTEMPTS=2 \ + bash "$ROOT_DIR/scripts/ci/run-app-host-xcodebuild.sh" test >"$TMP_DIR/communication-retry-output.log" 2>&1 +communication_retry_status=$? +set -e + +if [ "$communication_retry_status" -ne 0 ]; then + cat "$TMP_DIR/communication-retry-output.log" + echo "FAIL: communication-only failure did not recover on retry; got status $communication_retry_status" + regression_failures=$((regression_failures + 1)) +fi + +communication_retry_invocations="$(wc -l < "$TMP_DIR/communication-retry-invocations.log" | tr -d ' ')" +if [ "$communication_retry_invocations" -ne 2 ]; then + cat "$TMP_DIR/communication-retry-output.log" + echo "FAIL: communication-only failure must retry once; got $communication_retry_invocations invocations" + regression_failures=$((regression_failures + 1)) +fi + +if ! grep -Fq "Retrying app-host xcodebuild after test runner communication failure (attempt 1/2)" "$TMP_DIR/communication-retry-output.log"; then + cat "$TMP_DIR/communication-retry-output.log" + echo "FAIL: wrapper did not report the communication-only retry" + regression_failures=$((regression_failures + 1)) +fi + cat > "$TMP_DIR/xcodebuild" <<'SH' #!/usr/bin/env bash printf '%s\n' \ @@ -122,9 +182,9 @@ CMUX_APP_HOST_XCODEBUILD_ATTEMPTS=1 \ xctest_summary_loss_status=$? set -e -if [ "$xctest_summary_loss_status" -eq 0 ]; then +if [ "$xctest_summary_loss_status" -ne 1 ]; then cat "$TMP_DIR/xctest-summary-loss-output.log" - echo "FAIL: a final clean XCTest summary masked an earlier assertion failure" + echo "FAIL: expected synthesized XCTest assertion status 1, got $xctest_summary_loss_status" regression_failures=$((regression_failures + 1)) fi @@ -147,9 +207,9 @@ CMUX_APP_HOST_XCODEBUILD_ATTEMPTS=1 \ swift_testing_summary_loss_status=$? set -e -if [ "$swift_testing_summary_loss_status" -eq 0 ]; then +if [ "$swift_testing_summary_loss_status" -ne 1 ]; then cat "$TMP_DIR/swift-testing-summary-loss-output.log" - echo "FAIL: a final clean XCTest summary masked a Swift Testing assertion failure" + echo "FAIL: expected synthesized Swift Testing assertion status 1, got $swift_testing_summary_loss_status" regression_failures=$((regression_failures + 1)) fi @@ -225,11 +285,11 @@ exit 65 check=False, ) -if result.returncode == 0: +if result.returncode != 65: print(result.stdout, end="") print( - "FAIL: the app-host unit-test workflow treated an ordinary assertion " - "failure with '(0 unexpected)' as a pass" + "FAIL: expected app-host workflow status 65 for an ordinary assertion " + f"failure, got {result.returncode}" ) sys.exit(1) PY