Skip to content

ci: stop resolving SwiftPM in app-host test shards #13386

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
teamleaderleo wants to merge 1 commit into from
Closed

ci: stop resolving SwiftPM in app-host test shards #13386

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 1 addition & 107 deletions .github/workflows/ci-macos.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -861,17 +861,6 @@ jobs:
run: |
./scripts/install-rust-ci.sh

- name: Cache Swift packages
uses: ./.github/actions/cache-restore
with:
backend: ${{ inputs.cache_backend != 'default' && inputs.cache_backend || vars.CI_CACHE_BACKEND }}
path: .ci-source-packages
key: spm-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved') }}
restore-keys: spm-

- name: Sanitize Swift package cache
run: python3 scripts/ci/sanitize-xcode-source-packages-cache.py .ci-source-packages

- name: Prepare isolated DerivedData
run: |
set -euo pipefail
Expand Down Expand Up @@ -940,73 +929,6 @@ jobs:
id: prepare-app-host-home
run: scripts/ci/prepare-app-host-home.sh

- name: Resolve Swift packages
run: |
set -euo pipefail
SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages"
mkdir -p "$SOURCE_PACKAGES_DIR"
resolve_log="$RUNNER_TEMP/cmux-swiftpm-resolve-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-shard-${{ matrix.shard }}.log"

for attempt in 1 2 3; do
set +e
xcodebuild -project cmux.xcodeproj -scheme cmux-unit -configuration Debug \
-derivedDataPath "$CMUX_DERIVED_DATA_PATH" \
-clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \
-resolvePackageDependencies 2>&1 | tee "$resolve_log"
resolve_status="${PIPESTATUS[0]}"
set -e

if [ "$resolve_status" -eq 0 ]; then
# Guard against a stale/poisoned Swift-package cache: a restored
# .ci-source-packages can make resolve report success without the
# binary artifacts (Sparkle/Sentry) actually present, which then
# fails the build. Verify they materialized; if not, clear and retry.
if [ -d "$SOURCE_PACKAGES_DIR/artifacts/sparkle/Sparkle/Sparkle.xcframework" ] && [ -d "$SOURCE_PACKAGES_DIR/artifacts/sentry-cocoa/Sentry/Sentry.xcframework" ]; then
exit 0
fi
echo "Resolve succeeded but binary artifacts are missing (stale cache); clearing and retrying" >&2
rm -rf "$SOURCE_PACKAGES_DIR" # whole dir — resolve will not re-materialize artifacts into a partial tree
fi

# SwiftPM stores binary download destinations under the runner's
# global cache even when clonedSourcePackagesDirPath is isolated.
# A reused runner can leave the Iroh archive destination behind;
# SwiftPM then reports "already exists in file system" on every
# retry. Clear both resolver layers only for that cache collision,
# while preserving other failures for diagnosis.
if [ "$resolve_status" -ne 0 ] && grep -Fq "already exists in file system" "$resolve_log"; then
echo "SwiftPM artifact cache collision detected; clearing resolver caches before retry" >&2
rm -rf "$HOME/Library/Caches/org.swift.swiftpm"
mkdir -p "$HOME/Library/Caches/org.swift.swiftpm"
rm -rf "$SOURCE_PACKAGES_DIR"
mkdir -p "$SOURCE_PACKAGES_DIR"
fi

# Keep the existing three-attempt policy, but record resolver
# state when a package attempt fails so provider incidents are
# distinguishable from SwiftPM or cache defects.
scripts/ci/capture-network-diagnostics.sh || true
if [ "$attempt" -eq 3 ]; then
echo "Failed to resolve Swift packages after 3 attempts" >&2
exit 1
fi
echo "Package resolution failed on attempt $attempt, retrying..."
sleep $((attempt * 5))
done

- name: Restage package frameworks after resolution
run: |
set -euo pipefail
products="$CMUX_DERIVED_DATA_PATH/Build/Products/Debug"
if [ -L "$products/PackageFrameworks" ]; then
rm "$products/PackageFrameworks"
fi
mkdir -p "$products/PackageFrameworks"
framework_source="$(find "$products" -type d -name 'CmuxAgentJournal*_PackageProduct.framework' -print -quit 2>/dev/null || true)"
test -n "$framework_source"
rsync -aL "$(dirname "$framework_source")/" "$products/PackageFrameworks/"
test -f "$products/PackageFrameworks/CmuxAgentJournal_27B6EF8727F6C277_PackageProduct.framework/Versions/A/CmuxAgentJournal_27B6EF8727F6C277_PackageProduct"

- name: Run Ghostty split-theme appearance regression
if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_FOCUSED_REGRESSION_SHARD) }}
run: |
Expand Down Expand Up @@ -1857,33 +1779,6 @@ jobs:
set -e
collect_unit_test_output

# Retry package resolution only when the failed execution contains no
# evidence that tests began. This recovery can never replace a test
# verdict from the first execution.
if [ "$EXIT_CODE" -ne 0 ] \
&& grep -q "Could not resolve package dependencies" "$TEST_OUTPUT"; then
set +e
python3 scripts/ci/classify-app-host-test-output.py "$TEST_OUTPUT" --retry-safe
RETRY_SAFE=$?
set -e
if [ "$RETRY_SAFE" -eq 0 ]; then
echo "SwiftPM package resolution failed before tests; clearing caches and retrying once"
rm -rf ~/Library/Caches/org.swift.swiftpm
mkdir -p ~/Library/Caches/org.swift.swiftpm
xcodebuild -project cmux.xcodeproj -scheme cmux-unit -configuration Debug \
-derivedDataPath "$CMUX_DERIVED_DATA_PATH" \
-clonedSourcePackagesDirPath "$PWD/.ci-source-packages" \
-resolvePackageDependencies
set +e
run_unit_tests 2
EXIT_CODE=$?
set -e
collect_unit_test_output
else
echo "Preserving first execution failure; SwiftPM retry blocked after test execution evidence"
fi
fi

if [ "$EXIT_CODE" -ne 0 ]; then
echo "App-host unit-test batch failed with status ${EXIT_CODE}"
exit "$EXIT_CODE"
Expand All @@ -1893,8 +1788,7 @@ jobs:
if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_CLI_REGRESSION_SHARD) }}
run: |
set -euo pipefail
CMUX_SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" \
./tests/test_bundled_ghostty_theme_picker_helper.sh
./tests/test_bundled_ghostty_theme_picker_helper.sh

- name: Set up Bun for Pi extension dispatch regression
if: ${{ matrix.shard == fromJSON(env.CMUX_APP_HOST_CLI_REGRESSION_SHARD) }}
Expand Down
29 changes: 29 additions & 0 deletions tests/test_ci_product_publication.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,35 @@ def test_reuse_hit_keeps_consumer_validation_and_reports_metrics(self):
)


def test_app_host_shards_only_consume_the_admission_product(self):
job = self.workflow["jobs"]["app-host-unit-tests"]
steps = job["steps"]
names = [step["name"] for step in steps]

self.assertIn("needs.macos-compile-admission.outputs.artifact_id", str(job))
restore_index = names.index("Restore compiled app-host test product")
app_host_indices = [
index for index, step in enumerate(steps)
if "scripts/ci/run-app-host-xcodebuild.sh" in step.get("run", "")
]
self.assertTrue(app_host_indices)
self.assertLess(restore_index, min(app_host_indices))

run_text = "\n".join(step.get("run", "") for step in steps)
self.assertNotIn("-project cmux.xcodeproj", run_text)
self.assertNotIn("-resolvePackageDependencies", run_text)
self.assertNotIn(".ci-source-packages", str(job))
self.assertNotIn("Cache Swift packages", names)
self.assertNotIn("Resolve Swift packages", names)

for step in steps:
run = step.get("run", "")
if "scripts/ci/run-app-host-xcodebuild.sh" not in run:
continue
self.assertIn("-xctestrun", run, step["name"])
self.assertIn("test-without-building", run, step["name"])


def test_skipping_publication_keeps_admission_and_early_checks(self):
self.assertTrue(condition(self.job["if"], full_suite="false", publish="false"))
for name in ("Compile app-host test product", "Validate Swift warning budget",
Expand Down
127 changes: 33 additions & 94 deletions tests/test_ci_unit_test_spm_retry.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,107 +1,46 @@
#!/usr/bin/env bash
# Exercise the workflow resolver with controlled xcodebuild/cache failures.
# The legacy filename stays wired into workflow-guard-tests. Its contract now
# protects the build-once/test-many path: SwiftPM resolution belongs to compile
# admission, while app-host shards execute only the restored compiled product.
set -euo pipefail
cd "$(dirname "$0")/.."

python3 - <<'PY'
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
lines = Path('.github/workflows/ci-macos.yml').read_text().splitlines()
job_start = lines.index(' app-host-unit-tests:')
job_end = next(i for i in range(job_start + 1, len(lines))
if lines[i].startswith(' ') and not lines[i].startswith(' ') and lines[i].strip())
start = lines.index(' - name: Resolve Swift packages', job_start, job_end)
run_start = lines.index(' run: |', start, job_end) + 1
end = next(i for i in range(run_start, job_end)
if lines[i].startswith(' - name: '))
script = '\n'.join(line[10:] for line in lines[run_start:end])
script = script.replace('${{ matrix.shard }}', '1')
import re

class ResolverTests(unittest.TestCase):
def exercise(self, mode):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
bindir = root / 'bin'
bindir.mkdir()
home = root / 'home'
collision = home / 'Library/Caches/org.swift.swiftpm/artifacts/iroh'
collision.mkdir(parents=True)
marker = root / '.ci-source-packages/partial'
marker.parent.mkdir()
marker.write_text('stale')
diagnostics = root / 'scripts/ci/capture-network-diagnostics.sh'
diagnostics.parent.mkdir(parents=True)
diagnostics.write_text('#!/bin/bash\necho diagnostic >> "$FIXTURE_ROOT/diagnostics"\n')
diagnostics.chmod(0o755)
sleep = bindir / 'sleep'
sleep.write_text('#!/bin/bash\nexit 0\n')
sleep.chmod(0o755)
resolver = bindir / 'xcodebuild'
resolver.write_text('''#!/bin/bash
set -eu
count=0
[ ! -f "$FIXTURE_ROOT/count" ] || count=$(cat "$FIXTURE_ROOT/count")
count=$((count + 1))
echo "$count" > "$FIXTURE_ROOT/count"
if [ "$FIXTURE_MODE" = permanent ] || { [ "$FIXTURE_MODE" = collision ] && [ -d "$HOME/Library/Caches/org.swift.swiftpm/artifacts/iroh" ]; }; then
echo "failed downloading Iroh: $HOME/Library/Caches/org.swift.swiftpm/artifacts/iroh already exists in file system"
exit 74
fi
if [ "$FIXTURE_MODE" = network ]; then
echo 'Could not resolve package dependencies: host unavailable'
exit 74
fi
if [ "$FIXTURE_MODE" = missing ] && [ "$count" = 1 ]; then
exit 0
fi
mkdir -p .ci-source-packages/artifacts/sparkle/Sparkle/Sparkle.xcframework
mkdir -p .ci-source-packages/artifacts/sentry-cocoa/Sentry/Sentry.xcframework
''')
resolver.chmod(0o755)
env = dict(os.environ, HOME=str(home), RUNNER_TEMP=str(root),
GITHUB_RUN_ID='1', GITHUB_RUN_ATTEMPT='1',
CMUX_DERIVED_DATA_PATH=str(root / 'derived'),
FIXTURE_ROOT=str(root), FIXTURE_MODE=mode,
PATH=str(bindir) + os.pathsep + os.environ['PATH'])
result = subprocess.run(['bash', '-c', script], cwd=root, env=env,
text=True, capture_output=True, timeout=10)
return (result, int((root / 'count').read_text()),
collision.exists(), marker.exists())
workflow = Path(".github/workflows/ci-macos.yml").read_text(encoding="utf-8")

def test_collision_recovers_on_second_attempt(self):
result, attempts, collision, partial = self.exercise('collision')
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual(attempts, 2)
self.assertFalse(collision)
self.assertFalse(partial)
def job(name: str) -> str:
match = re.search(
rf"(?ms)^ {re.escape(name)}:\n(.*?)(?=^ [A-Za-z0-9_-]+:\n|\Z)",
workflow,
)
if match is None:
raise AssertionError(f"missing workflow job {name}")
return match.group(0)

def test_persistent_collision_stays_red_after_three_attempts(self):
result, attempts, _, _ = self.exercise('permanent')
self.assertNotEqual(result.returncode, 0)
self.assertEqual(attempts, 3)
admission = job("macos-compile-admission")
consumer = job("app-host-unit-tests")
restore = Path("scripts/ci/restore-app-host-test-product.sh").read_text(encoding="utf-8")

def test_network_failure_stays_red_without_clearing_cache(self):
result, attempts, collision, partial = self.exercise('network')
self.assertNotEqual(result.returncode, 0)
self.assertEqual(attempts, 3)
self.assertTrue(collision)
self.assertTrue(partial)
assert "scripts/ci/compile-app-host-test-product.sh resolve" in admission
assert "scripts/ci/compile-app-host-test-product.sh build" in admission
assert "Restore compiled app-host test product" in consumer
assert "test-without-building" in consumer

def test_success_keeps_cache(self):
result, attempts, collision, partial = self.exercise('success')
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual(attempts, 1)
self.assertTrue(collision)
self.assertTrue(partial)
for forbidden in (
"-resolvePackageDependencies",
".ci-source-packages",
"-project cmux.xcodeproj",
):
assert forbidden not in consumer, f"app-host consumer reintroduced {forbidden}"

def test_missing_binary_artifacts_retry(self):
result, attempts, collision, partial = self.exercise('missing')
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual(attempts, 2)
self.assertTrue(collision)
self.assertFalse(partial)
assert "PackageFrameworks" in restore
assert "app_host_test_products.py restore" in restore

unittest.main()
print(
"PASS: SwiftPM resolution stays in compile admission; "
"app-host shards consume restored compiled products"
)
PY
Loading