From 5fed793139987bb3ca2de6b1b5b2390931edefcb Mon Sep 17 00:00:00 2001 From: Abdulaziz Albahar <67667005+azooz2003-bit@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:02:59 -0700 Subject: [PATCH 01/12] Fix four automation-blocking regressions in the iOS connectivity gate Found by driving the relay-only release gate end to end on real staging: - MobileIrxSettingsController.irohSettingsUpdates() never yielded the initial snapshot, so the gate runner's path-policy check subscribed after the transport settled and hung to its deadline. - The irx settingsSnapshot() never populated selectedTransportPath (it stayed .unavailable while an admitted relay session was live); a new IrxConnection.selectedPath() accessor feeds it, classifying a relay as managed only when it matches a signed credential, fail-closed. - No script set CMUX_MOBILE_SOAK_OPEN_SELECTED_WORKSPACE, so gate-mode launches sat on the workspace list and readiness starved on selectedTerminalID; mobile-dev-launch now defaults it on in gate mode. - The What's New sheet covers the workspace UI on every fresh automated install; a DEBUG-only CMUX_UITEST_SUPPRESS_WHATS_NEW knob suppresses presentation without touching acknowledgement markers. Also logs each path-check snapshot so the next silent stall names itself. Co-Authored-By: Claude Fable 5 --- .../CmuxIrxTransport/IrxConnection.swift | 11 +++++++ .../MobileWhatsNewCenter.swift | 18 ++++++++++++ .../MobileIrohReleaseGateRunner.swift | 8 +++-- ...MobileIrxRuntimeComposition+Settings.swift | 29 ++++++++++++++++++- .../MobileIrxSettingsController.swift | 4 +++ scripts/mobile-dev-launch.sh | 2 ++ 6 files changed, 69 insertions(+), 3 deletions(-) diff --git a/Packages/Shared/CmuxIrxTransport/Sources/CmuxIrxTransport/IrxConnection.swift b/Packages/Shared/CmuxIrxTransport/Sources/CmuxIrxTransport/IrxConnection.swift index a7c07d90a7f2..b2404e645a59 100644 --- a/Packages/Shared/CmuxIrxTransport/Sources/CmuxIrxTransport/IrxConnection.swift +++ b/Packages/Shared/CmuxIrxTransport/Sources/CmuxIrxTransport/IrxConnection.swift @@ -386,6 +386,17 @@ public actor IrxConnection { return nil } + /// The selected QUIC path right now, structured for path attribution + /// (Settings and the release gate's path-policy check). `remoteAddress` + /// is the relay URL for relayed paths and the socket address otherwise. + public nonisolated func selectedPath() -> (isRelay: Bool, remoteAddress: String)? { + let paths = connection.paths() + guard let selected = paths.first(where: { $0.isSelected }) ?? paths.first else { + return nil + } + return (selected.isRelay, "\(selected.remoteAddr)") + } + /// The selected QUIC path right now, for relay attribution evidence. public nonisolated func selectedPathDescription() -> String { let paths = connection.paths() diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/MobileWhatsNewCenter.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/MobileWhatsNewCenter.swift index d95735c4d6a9..9adeebc92876 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/MobileWhatsNewCenter.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/MobileWhatsNewCenter.swift @@ -27,6 +27,16 @@ public final class MobileWhatsNewCenter { public typealias Loader = @Sendable (URL) async throws -> Data static let markerKey = "dev.cmux.mobile.whatsNew.newestAcknowledgedEntryId" + + #if DEBUG + /// `CMUX_UITEST_SUPPRESS_WHATS_NEW=1` (environment or launch argument) + /// keeps the launch sheet away during automated Debug runs. Debug-only, + /// mirroring the other `CMUX_UITEST_*` hooks in `UITestConfig`. + static var suppressedForAutomation: Bool { + ProcessInfo.processInfo.environment["CMUX_UITEST_SUPPRESS_WHATS_NEW"] == "1" + || ProcessInfo.processInfo.arguments.contains("CMUX_UITEST_SUPPRESS_WHATS_NEW=1") + } + #endif static let acknowledgedAnnouncementsKey = "dev.cmux.mobile.whatsNew.acknowledgedAnnouncementIds" static let cacheKey = "dev.cmux.mobile.whatsNew.remoteList.v1" static let requestPath = "/api/whats-new" @@ -220,6 +230,14 @@ public final class MobileWhatsNewCenter { /// advances past a page that was skipped this way unless a newer binary /// page was acknowledged above it). var unseenPages: [MobileWhatsNewPage] { + #if DEBUG + // Automated drivers (the Iroh release gate, the iOS e2e gate) run a + // fresh install every time, so the launch sheet would cover the + // workspace UI and block their readiness probes. The knob suppresses + // presentation only; markers are untouched, so a normal launch of the + // same container still shows the pages. + if Self.suppressedForAutomation { return [] } + #endif let acknowledged = acknowledgedAnnouncementIDs let unseenAnnouncements = announcementPages.filter { !acknowledged.contains($0.id) } let visible = visibleBinaryEntries diff --git a/ios/cmuxPackage/Sources/CmuxIrohReleaseGateSupport/MobileIrohReleaseGateRunner.swift b/ios/cmuxPackage/Sources/CmuxIrohReleaseGateSupport/MobileIrohReleaseGateRunner.swift index bb2a7e0e49cf..dc92c12b9d6d 100644 --- a/ios/cmuxPackage/Sources/CmuxIrohReleaseGateSupport/MobileIrohReleaseGateRunner.swift +++ b/ios/cmuxPackage/Sources/CmuxIrohReleaseGateSupport/MobileIrohReleaseGateRunner.swift @@ -479,10 +479,14 @@ final class MobileIrohReleaseGateRunner { failure: .timeout ) } - if let accepted = Self.acceptedPath( + let accepted = Self.acceptedPath( snapshot.selectedTransportPath, mode: configuration.mode - ) { + ) + mobileIrohReleaseGateLog.info( + "path-check path=\(String(describing: snapshot.selectedTransportPath), privacy: .public) accepted=\(accepted ?? "no", privacy: .public)" + ) + if let accepted { pathBeforeProbe = accepted break } diff --git a/ios/cmuxPackage/Sources/cmuxFeature/MobileIrxRuntimeComposition+Settings.swift b/ios/cmuxPackage/Sources/cmuxFeature/MobileIrxRuntimeComposition+Settings.swift index 2978472d6e62..2bccd95c70d4 100644 --- a/ios/cmuxPackage/Sources/cmuxFeature/MobileIrxRuntimeComposition+Settings.swift +++ b/ios/cmuxPackage/Sources/cmuxFeature/MobileIrxRuntimeComposition+Settings.swift @@ -15,8 +15,9 @@ extension MobileIrxRuntimeComposition { if activeScope == nil { status = .inactive } else if await endpointSupervisor?.boundEndpoint() != nil || directIsBound { status = .active } else { status = .starting } + let selectedPath = await liveSelectedTransportPath() guard (try? await assertScope(scope, epoch: currentEpoch)) != nil else { return .unavailable } - return CmxIrohSettingsSnapshot(runtimeStatus: status, + return CmxIrohSettingsSnapshot(runtimeStatus: status, selectedTransportPath: selectedPath, preference: .automatic, pathPreference: forceRelayOnly ? .relayOnly : .automatic, managedRelays: (cache?.relayCredentials ?? []).map { .init(id: $0.relayURL, provider: "cmux", region: "", url: $0.relayURL, isSelected: $0.relayURL == currentRelay) @@ -31,6 +32,32 @@ extension MobileIrxRuntimeComposition { failureDescription: lastFailure) } + /// The path application traffic uses right now, from the first admitted + /// peer session. A relay counts as managed only when it is one of this + /// account's signed relay credentials; any other relay stays + /// `.unavailable`, matching the fail-closed classifier the Iroh runtime + /// used before the v2 migration. Without this the snapshot always + /// reported `.unavailable`, so the release gate's relay-only path check + /// could never pass. + func liveSelectedTransportPath() async -> CmxIrohSelectedTransportPath { + for engine in enginesByPeer.values { + guard case .ready = await engine.currentState, + let session = await engine.currentSession(), + let path = session.connection.selectedPath() else { continue } + guard path.isRelay else { return .direct } + let normalized = Self.normalizedRelayURL(path.remoteAddress) + let managed = (cache?.relayCredentials ?? []).contains { + Self.normalizedRelayURL($0.relayURL) == normalized + } + return managed ? .managedRelay(provider: "cmux", region: "") : .unavailable + } + return .unavailable + } + + private static func normalizedRelayURL(_ url: String) -> String { + url.hasSuffix("/") ? String(url.dropLast()) : url + } + public func settingsUpdates() -> AsyncStream { changes() } public func refreshSettingsSnapshot() async { await invalidateDiscoverySnapshot() diff --git a/ios/cmuxPackage/Sources/cmuxFeature/MobileIrxSettingsController.swift b/ios/cmuxPackage/Sources/cmuxFeature/MobileIrxSettingsController.swift index fc0b4da25d52..99830063c941 100644 --- a/ios/cmuxPackage/Sources/cmuxFeature/MobileIrxSettingsController.swift +++ b/ios/cmuxPackage/Sources/cmuxFeature/MobileIrxSettingsController.swift @@ -25,7 +25,11 @@ public final class MobileIrxSettingsController: CmxIrohSettingsControlling { ) let irxTask = Task { @MainActor [weak self] in guard let self else { return } + // Yield the current snapshot first: a subscriber that arrives after + // the transport settled (the release-gate runner waiting for its + // path-policy check) must not hang until the next settings change. let changes = await irx.settingsUpdates() + continuation.yield(await irohSettingsSnapshot()) for await _ in changes { guard !Task.isCancelled else { return } continuation.yield(await irohSettingsSnapshot()) diff --git a/scripts/mobile-dev-launch.sh b/scripts/mobile-dev-launch.sh index e53bed28d3a1..a32da79af87a 100755 --- a/scripts/mobile-dev-launch.sh +++ b/scripts/mobile-dev-launch.sh @@ -401,6 +401,8 @@ if [[ "$TARGET" == "simulator" ]]; then SIMCTL_CHILD_CMUX_DOGFOOD_ATTACH_URL="$ATTACH_URL" \ SIMCTL_CHILD_CMUX_DOGFOOD_CLIENT_ID="$DOGFOOD_CLIENT_ID" \ SIMCTL_CHILD_CMUX_IROH_RELEASE_GATE_MODE="$IROH_RELEASE_GATE_MODE" \ + SIMCTL_CHILD_CMUX_UITEST_SUPPRESS_WHATS_NEW="$([[ -n "$IROH_RELEASE_GATE_MODE" ]] && printf 1 || printf '%s' "${CMUX_UITEST_SUPPRESS_WHATS_NEW:-0}")" \ + SIMCTL_CHILD_CMUX_MOBILE_SOAK_OPEN_SELECTED_WORKSPACE="$([[ -n "$IROH_RELEASE_GATE_MODE" ]] && printf '%s' "${CMUX_MOBILE_SOAK_OPEN_SELECTED_WORKSPACE:-1}" || printf '%s' "${CMUX_MOBILE_SOAK_OPEN_SELECTED_WORKSPACE:-0}")" \ SIMCTL_CHILD_CMUX_IROH_RELEASE_GATE_SCENARIO="${CMUX_IROH_RELEASE_GATE_SCENARIO:-standard}" \ SIMCTL_CHILD_CMUX_IROH_SOAK_PROFILE="${CMUX_IROH_SOAK_PROFILE:-}" \ SIMCTL_CHILD_CMUX_IROH_DISABLE_RELAY_CREDENTIAL_REFRESH="${CMUX_IROH_DISABLE_RELAY_CREDENTIAL_REFRESH:-0}" \ From 87eec29c73aedab6f8b7a47d3eec1d0d5e7a056c Mon Sep 17 00:00:00 2001 From: Abdulaziz Albahar <67667005+azooz2003-bit@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:02:59 -0700 Subject: [PATCH 02/12] Add the iOS e2e gate: two-machine workflow draft and terminal driver scripts/e2e/ios-e2e-run.sh drives sign-in-adjacent terminal use on a live paired sim/Mac and asserts BOTH sides of every step (Vision OCR of the rendered screen; the tagged Mac socket for what the real shell executed): echo round trip, output burst plus verified scrollback, alt-screen enter/exit, Ctrl-C, background/foreground replay, and input after reconnect. scripts/e2e/mac-host.sh holds a CI Mac runner on a done-file with a hard timeout (no GitHub API polling). ios-e2e.yml is the 4-job two-runner topology (Tailscale as control plane only); its pull_request trigger stays commented out until the check is approved for promotion. Verified twice back to back on tag e2eci against remote staging over the real relay, after the in-app gate probe passed with path=managed_relay. Co-Authored-By: Claude Fable 5 --- .github/workflows/ios-e2e.yml | 442 ++++++++++++++++++++++++++++++++++ docs/ci/ios-e2e.md | 127 ++++++++++ scripts/e2e/README.md | 82 +++++++ scripts/e2e/ios-e2e-run.sh | 304 +++++++++++++++++++++++ scripts/e2e/mac-host.sh | 70 ++++++ scripts/e2e/ocr.swift | 28 +++ 6 files changed, 1053 insertions(+) create mode 100644 .github/workflows/ios-e2e.yml create mode 100644 docs/ci/ios-e2e.md create mode 100644 scripts/e2e/README.md create mode 100755 scripts/e2e/ios-e2e-run.sh create mode 100755 scripts/e2e/mac-host.sh create mode 100644 scripts/e2e/ocr.swift diff --git a/.github/workflows/ios-e2e.yml b/.github/workflows/ios-e2e.yml new file mode 100644 index 000000000000..173fa4136c89 --- /dev/null +++ b/.github/workflows/ios-e2e.yml @@ -0,0 +1,442 @@ +name: iOS E2E + +# End-to-end Mac<->iPhone gate: a real Mac app on one runner, a real iOS +# simulator app on another, and the dev web backend on the durable tailnet VM. +# The iOS side signs in, pairs to the remote Mac through the backend, connects +# over Iroh, and drives a 6-step streamed-terminal script whose steps each +# cover a shipped regression (scripts/e2e/README.md). Topology, ACLs, secrets +# and the promotion plan live in docs/ci/ios-e2e.md. +# +# No workflow-level path filter on purpose: the `route` job decides skips so +# the `ios-e2e-status` aggregate always reports a deterministic conclusion. +# ci.yml takes the same approach — a paths: filter leaves the check MISSING +# on unrelated PRs, which branch protection cannot require; a routed skip +# still reports green. +on: + workflow_dispatch: + # The pull_request trigger stays off until the owner approves this check's + # promotion (see docs/ci/ios-e2e.md): tests enter a CI lane by proposal and + # approval, and the product-download stubs below would red every PR today. + # pull_request: + +permissions: + contents: read + +# A newer push to the same pull request replaces its run; dispatches never +# cancel each other (same shape as test-ios.yml). +concurrency: + group: ${{ github.event_name == 'pull_request' && format('ios-e2e-pr-{0}', github.event.pull_request.number) || format('ios-e2e-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + # The whole teardown handshake is one file touched over Tailscale SSH: the + # Mac host's wait loop watches this local path instead of polling the + # GitHub API for the iOS job's status. A ~25-minute per-PR status-poll loop + # would draw down the repo-wide API rate limit that every other workflow + # shares (secondary-rate-limit stalls have hit this repo's CI before), and + # a local file needs no token on the Mac at all. + CMUX_E2E_DONE_FILE: /tmp/e2e-done-${{ github.run_id }} + # Deterministic tailnet hostname for the Mac host. The iOS job must address + # the Mac while BOTH jobs are still running, and GitHub job outputs only + # publish when the producing job completes — so the name is derived from + # the run id up front rather than exchanged at runtime. + CMUX_E2E_MAC_TAILNET_HOSTNAME: cmux-e2e-mac-${{ github.run_id }} + CMUX_E2E_BACKEND_HOST: cmux-dev-backend-1.tail137216.ts.net + +jobs: + route: + # Decides whether the lane runs and which backend stack tag it uses. + # Cheap Linux layer so every PR gets a routed conclusion (see header). + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 5 + outputs: + run_e2e: ${{ steps.decide.outputs.run_e2e }} + backend_tag: ${{ steps.decide.outputs.backend_tag }} + web_changed: ${{ steps.decide.outputs.web_changed }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # Depth 2 reaches both parents of the PR merge commit, which is all + # the backend-tag path check below needs to diff. + fetch-depth: 2 + persist-credentials: false + + - name: Decide route and backend tag + id: decide + env: + EVENT_NAME: ${{ github.event_name }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + # TODO(ios-e2e): stub router. Integrate + # scripts/ci/detect_ci_change_areas.py (the classifier ci.yml's + # `changes` job uses) so docs-only and unrelated diffs skip both + # macOS runners; until then every routed run says "run" and the + # lane's cost is bounded by it not being required (shadow mode, + # docs/ci/ios-e2e.md#promotion-plan). + run_e2e=true + # Backend stack selection: a PR that changes web/ (API routes, + # services, drizzle schema — the code the stack actually runs) must + # not land its schema or API changes on the shared long-lived + # ci-main stack. It gets an isolated per-PR stack, tag ci, + # reused across pushes so the stack keeps its database (dev-backend + # semantics: re-ensuring a tag updates its source in place). + # TODO(ios-e2e): align this path set with the `web` area in + # scripts/ci/detect_ci_change_areas.py instead of one regex. + web_changed=false + if [ "$EVENT_NAME" = "pull_request" ]; then + if git rev-parse -q --verify HEAD^2 >/dev/null; then + # HEAD is the PR merge commit; parent 1 is the base branch, so + # this diff is exactly the PR's changed files, no API call. + if git diff --name-only HEAD^1 HEAD | grep -Eq '^web/'; then + web_changed=true + fi + else + # No merge commit to diff (detached/rebase edge): isolate + # rather than risk mutating the shared stack. + web_changed=true + fi + fi + if [ "$web_changed" = "true" ] && [ -n "${PR_NUMBER:-}" ]; then + backend_tag="ci${PR_NUMBER}" + else + backend_tag="ci-main" + fi + { + echo "run_e2e=$run_e2e" + echo "web_changed=$web_changed" + echo "backend_tag=$backend_tag" + } >> "$GITHUB_OUTPUT" + echo "route: run_e2e=$run_e2e web_changed=$web_changed backend_tag=$backend_tag" + + backend: + # Ensures a dev backend stack (web + Postgres) for this run's tag on the + # durable VM and proves the runner-side tailnet path to it before either + # macOS job spends its queue slot. + needs: route + # Every job that mounts secrets is fenced off from fork heads: a fork PR + # controls this workflow file's content, so it must never reach the + # tailnet OAuth client or the CI Stack account. success() must be spelled + # out — a custom `if:` replaces the implicit needs-succeeded condition. + # The aggregate reports forks as a neutral skip. + if: >- + success() + && needs.route.outputs.run_e2e == 'true' + && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 10 + outputs: + backend_url: ${{ steps.ensure.outputs.backend_url }} + steps: + - name: Join tailnet + # TODO(ios-e2e): pin to a commit SHA like the other third-party + # actions in this repo once the action version settles. + uses: tailscale/github-action@v4 + with: + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} + tags: tag:ci + + - name: Ping backend host + run: | + set -euo pipefail + # [infra-preflight] reds here are tailnet/ACL problems, never a + # product regression; the label keeps them out of flake triage + # (docs/ci/ios-e2e.md#infra-preflight-failure-labeling). + tailscale ping --timeout 5s -c 5 "$CMUX_E2E_BACKEND_HOST" || { + echo "::error::[infra-preflight] cannot reach $CMUX_E2E_BACKEND_HOST over the tailnet (ACL: tag:ci -> backend host)" + exit 1 + } + + - name: Ensure backend stack + id: ensure + env: + BACKEND_TAG: ${{ needs.route.outputs.backend_tag }} + run: | + set -euo pipefail + # TODO(ios-e2e): real ensure call. cmuxterm-hq's + # scripts/dev-backend.sh (shim for skills/infra/dev-backend/ + # dev-backend.sh) owns these stacks: `dev-backend.sh url --tag + # ` ensures the tagged web+Postgres Docker stack on the VM and + # prints its private Tailscale Serve URL; `status` shows the + # 12-running / 64-registered stack budget this job must respect + # before ensuring. It drives the VM's control API over SSH, so CI + # needs a dedicated deploy key — secret CMUX_DEV_BACKEND_SSH_KEY, + # NOT yet provisioned — loaded into an ssh-agent here and never + # written to disk or echoed. Until that lands, emit the serve + # origin shape so the drivers fail loudly against a missing stack + # instead of a missing variable. + backend_url="https://${CMUX_E2E_BACKEND_HOST}" + echo "backend_url=$backend_url" >> "$GITHUB_OUTPUT" + echo "ensure (stub): tag=$BACKEND_TAG url=$backend_url" >> "$GITHUB_STEP_SUMMARY" + + mac-host: + # Compiles nothing: reuses the prebuilt tagged Mac app, signs it into the + # CI Stack account, advertises it through the backend so the iOS client + # can discover it, then blocks on the done-file until the iOS job signals + # (or the bounded wait expires). Starts in PARALLEL with ios-e2e — the + # sim's sign-in/pair sequence retries until the Mac is advertised, so + # serializing the jobs would only add queue time. + needs: [route, backend] + if: >- + success() + && needs.route.outputs.run_e2e == 'true' + && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + runs-on: ${{ vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-26' }} + timeout-minutes: 45 + outputs: + # Debug/audit only. A job output is readable by dependents only after + # this job completes, and this job completes only after the signal — + # the iOS job addresses the Mac by CMUX_E2E_MAC_TAILNET_HOSTNAME (the + # deterministic name it joined with) instead. + tailnet_hostname: ${{ steps.tailnet-name.outputs.hostname }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # The app arrives prebuilt; the checkout is for the driver scripts. + submodules: false + + - name: Download tagged Mac app product + run: | + set -euo pipefail + # TODO(ios-e2e): reuse the compiled Mac app instead of building. + # ci-macos.yml's admission job is the pattern: `python3 + # scripts/ci/reuse_app_host_products.py key ` derives + # the reuse key from the source/Xcode fingerprint, then `... restore + # ` (GH_TOKEN in env) pulls a producer-`seal`ed + # Build/Products tree from a prior run's artifact or its R2 copy + # into an isolated DerivedData root, with hit/miss reasons on the + # step outputs. This job wants exactly that consumer path plus the + # tag stamping scripts/reload.sh applies (bundle + # com.cmuxterm.app.debug., socket /tmp/cmux-debug-.sock), + # and needs `permissions: actions: read` once wired. Until then, + # fail fast with the infra label rather than idle for 45 minutes. + echo "::error::[infra-preflight] Mac app product download not implemented (see reuse_app_host_products.py TODO)" + exit 1 + + - name: Join tailnet + # TODO(ios-e2e): pin to a commit SHA (see backend job); also confirm + # the action's macOS-runner support on the Blacksmith image. + uses: tailscale/github-action@v4 + with: + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} + tags: tag:ci + # Deterministic name (see workflow env): the parallel iOS job must + # be able to SSH here without a runtime hostname exchange. + hostname: ${{ env.CMUX_E2E_MAC_TAILNET_HOSTNAME }} + + - name: Publish tailnet hostname + id: tailnet-name + run: | + set -euo pipefail + # TODO(ios-e2e): the original design exchanged the hostname from + # `tailscale status --self --json` via a job output written before + # the wait. That cannot work for a PARALLEL consumer — job outputs + # publish only when the job completes, which here is after the + # signal — so the deterministic join hostname above is the real + # addressing mechanism. This output stays for logs, the aggregate, + # and any future sequential consumer. + hostname="$(tailscale status --self --json | python3 -c 'import json,sys; print(json.load(sys.stdin)["Self"]["DNSName"].rstrip("."))')" + echo "hostname=$hostname" >> "$GITHUB_OUTPUT" + echo "tailnet hostname: $hostname (expected prefix: $CMUX_E2E_MAC_TAILNET_HOSTNAME)" + + - name: Launch Mac host and wait for the iOS job + # Step ceiling above the driver's own ~25m bounded wait so a wedged + # driver can never consume the whole job timeout doing nothing. + timeout-minutes: 30 + env: + CMUX_E2E_TAG: ${{ needs.route.outputs.backend_tag }} + CMUX_DEV_BACKEND_URL: ${{ needs.backend.outputs.backend_url }} + # ~25m bounded done-file loop inside the driver; the done-file path + # itself comes from the workflow env (no GitHub API polling — see + # the CMUX_E2E_DONE_FILE comment at the top). + CMUX_E2E_WAIT_TIMEOUT_SECONDS: "1500" + # Dedicated CI Stack account, the same secret pair + # ios-streamed-validate.yml uses; the app's dev-secrets resolution + # reads CMUX_DOGFOOD_STACK_* from the environment first. Values + # travel only through the environment — never echoed, never argv, + # never written to disk. + CMUX_DOGFOOD_STACK_EMAIL: ${{ secrets.CMUX_DOGFOOD_STACK_EMAIL }} + CMUX_DOGFOOD_STACK_PASSWORD: ${{ secrets.CMUX_DOGFOOD_STACK_PASSWORD }} + run: ./scripts/e2e/mac-host.sh + + ios-e2e: + # The client half: fresh named simulator, prebuilt sim app, sign-in -> + # pair -> Iroh connect -> 6-step terminal script (scripts/e2e/README.md). + # Runs in parallel with mac-host (see that job's comment) and ALWAYS + # signals the Mac's done-file at the end, pass or fail, so the Mac never + # waits out its full timeout on a failed client. + needs: [route, backend] + if: >- + success() + && needs.route.outputs.run_e2e == 'true' + && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + # Simulator-capable pool: MACOS_RUNNER_IOS is the variable the iOS + # simulator lanes read (test-ios.yml, ios-streamed-validate.yml). + runs-on: ${{ vars.MACOS_RUNNER_IOS || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-26' }} + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + submodules: false + + - name: Select Xcode + # Canonical fleet selector (ranks by macOS SDK, aligns xcode-select); + # simctl needs a toolchain with an iOS 26 simulator runtime. + run: ./scripts/select-ci-xcode.sh + + - name: Ensure iOS simulator runtime + run: xcrun simctl list runtimes available | grep -Eq '\biOS\b' || xcodebuild -downloadPlatform iOS + + - name: Download iOS simulator app product + run: | + set -euo pipefail + # TODO(ios-e2e): reuse the iOS lane's compiled product instead of + # building. test-ios.yml's ios-simulator-build job stages and stamps + # the simulator Build/Products tree + # (scripts/ci/ios_simulator_test_product.py stamp) and uploads it + # as artifact `ios-test-product--` (tar.gz); + # its consumers download by artifact id, trying + # scripts/ci/parallel_artifact_download.py first. This lane needs a + # cross-workflow lookup of the newest compatible product for this + # head SHA (the way reuse_app_host_products.py locates Mac + # products), or a tagged ios/scripts/reload.sh-style build as the + # cold fallback. Caveat to resolve while wiring: that artifact is + # the TEST-host product — confirm its bundle id and entitlements + # suit the dogfood sign-in path (dev.cmux.ios.) or re-stamp. + # Needs `permissions: actions: read` once wired. Until then, fail + # fast with the infra label. + echo "::error::[infra-preflight] iOS sim app product download not implemented (see test-ios.yml product-upload TODO)" + exit 1 + + - name: Join tailnet + # TODO(ios-e2e): pin to a commit SHA (see backend job). + uses: tailscale/github-action@v4 + with: + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} + tags: tag:ci + hostname: cmux-e2e-ios-${{ github.run_id }} + + - name: Boot fresh named simulator + run: | + set -euo pipefail + # Per-run named sim so parallel runs on a shared mini never boot, + # install onto, or reset each other's device; the newest iPhone + # device type on the image keeps this from pinning a model name + # that ages out of the runner image. + DEVTYPE="$(xcrun simctl list devicetypes -j | python3 -c 'import json,sys; ts=[t["identifier"] for t in json.load(sys.stdin)["devicetypes"] if t.get("productFamily")=="iPhone"]; print(ts[-1])')" + UDID="$(xcrun simctl create "cmux-e2e-${GITHUB_RUN_ID}" "$DEVTYPE")" + xcrun simctl boot "$UDID" + xcrun simctl bootstatus "$UDID" -b + echo "CMUX_E2E_SIM_UDID=$UDID" >> "$GITHUB_ENV" + echo "sim: cmux-e2e-${GITHUB_RUN_ID} ($DEVTYPE) $UDID" + + - name: Run iOS E2E + env: + CMUX_E2E_TAG: ${{ needs.route.outputs.backend_tag }} + CMUX_DEV_BACKEND_URL: ${{ needs.backend.outputs.backend_url }} + CMUX_E2E_EVIDENCE_DIR: ${{ runner.temp }}/e2e-evidence + # Same CI Stack account as mac-host: pairing's same-account RPC + # gate requires both ends to resolve one account. Environment-only, + # never echoed. + CMUX_DOGFOOD_STACK_EMAIL: ${{ secrets.CMUX_DOGFOOD_STACK_EMAIL }} + CMUX_DOGFOOD_STACK_PASSWORD: ${{ secrets.CMUX_DOGFOOD_STACK_PASSWORD }} + run: ./scripts/e2e/ios-e2e-run.sh + + - name: Signal Mac host teardown + # Always, pass or fail: this touch is what releases the Mac host's + # done-file wait. It rides Tailscale SSH because the ACL grants + # tag:ci -> tag:ci on port 22 ONLY — deliberately nothing wider, so + # Iroh's path probing can never carry the terminal stream between the + # runners over the tailnet and silently bypass the transport this + # lane exists to gate (docs/ci/ios-e2e.md#tailscale-acl-requirements). + if: always() + run: | + set -euo pipefail + # TODO(ios-e2e): confirm the login user Tailscale SSH maps for the + # Blacksmith macOS runner account in the tailnet ACL ("runner" + # assumed here). Best-effort: a missed signal only costs the Mac + # its own bounded wait, so it must not repaint a green E2E red. + ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 \ + "runner@${CMUX_E2E_MAC_TAILNET_HOSTNAME}" \ + "touch '${CMUX_E2E_DONE_FILE}'" \ + || echo "::warning::[infra-preflight] could not signal ${CMUX_E2E_MAC_TAILNET_HOSTNAME}; its wait expires at its own ~25m bound" + + - name: Upload E2E evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ios-e2e-evidence-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/e2e-evidence + if-no-files-found: warn + retention-days: 7 + + - name: Delete simulator + if: always() + run: | + [ -n "${CMUX_E2E_SIM_UDID:-}" ] || exit 0 + xcrun simctl shutdown "$CMUX_E2E_SIM_UDID" 2>/dev/null || true + xcrun simctl delete "$CMUX_E2E_SIM_UDID" || true + + ios-e2e-status: + # The one conclusion branch protection will eventually require + # (docs/ci/ios-e2e.md#promotion-plan): green when the route skipped the + # lane or every needed job passed; red when the route said run and any + # needed job failed, was cancelled, or was skipped unexpectedly. Fork PRs + # are a neutral skip — the secret-fenced jobs cannot run there, and a red + # would block every outside contribution. + needs: [route, backend, mac-host, ios-e2e] + if: always() + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 5 + steps: + - name: Aggregate + env: + ROUTE_RESULT: ${{ needs.route.result }} + RUN_E2E: ${{ needs.route.outputs.run_e2e }} + BACKEND_TAG: ${{ needs.route.outputs.backend_tag }} + BACKEND_RESULT: ${{ needs.backend.result }} + MAC_RESULT: ${{ needs.mac-host.result }} + IOS_RESULT: ${{ needs.ios-e2e.result }} + IS_FORK: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + run: | + set -euo pipefail + { + echo "### iOS E2E" + echo "- route: $ROUTE_RESULT (run_e2e=${RUN_E2E:-}, backend tag=${BACKEND_TAG:-})" + echo "- backend: $BACKEND_RESULT · mac-host: $MAC_RESULT · ios-e2e: $IOS_RESULT" + } >> "$GITHUB_STEP_SUMMARY" + if [ "$IS_FORK" = "true" ]; then + # Neutral skip: report it and pass without judging jobs the fork + # fence intentionally skipped. + echo "- conclusion: neutral skip (fork PR; secret-fenced jobs cannot run)" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + if [ "$ROUTE_RESULT" != "success" ]; then + echo "::error::route job did not succeed ($ROUTE_RESULT)" + exit 1 + fi + if [ "$RUN_E2E" != "true" ]; then + echo "- conclusion: pass (route skipped the lane)" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + fail=0 + for pair in "backend:$BACKEND_RESULT" "mac-host:$MAC_RESULT" "ios-e2e:$IOS_RESULT"; do + job="${pair%%:*}"; result="${pair#*:}" + if [ "$result" != "success" ]; then + echo "::error::$job: $result (needed because route said run)" + fail=1 + fi + done + if [ "$fail" -ne 0 ]; then + echo "- conclusion: FAIL" >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + echo "- conclusion: pass" >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/ci/ios-e2e.md b/docs/ci/ios-e2e.md new file mode 100644 index 000000000000..71fee40eb313 --- /dev/null +++ b/docs/ci/ios-e2e.md @@ -0,0 +1,127 @@ +# iOS E2E gate + +[.github/workflows/ios-e2e.yml](../../.github/workflows/ios-e2e.yml) proves +the whole Mac-to-iPhone product path on a pull request: a real Mac app on one +runner, a real iOS simulator app on another, the dev web backend on the +durable tailnet VM, then sign-in → pairing → an Iroh connection → a scripted +streamed terminal session. The per-step driver contract and the regression +each step covers live in [scripts/e2e/README.md](../../scripts/e2e/README.md). + +## Topology + +``` + GitHub Actions run + ┌─────────────────────────────────────────────────────────────┐ + │ route (Linux)ci──▶ backend (Linux)ci──────────────┐ │ + │ │ ensure stack ci/ci-main │ + │ ▼ ▼ │ + │ ┌── mac-host (macOS) ──┐ ┌── ios-e2e (macOS) ─┐ + │ │ tagged cmux DEV app │ │ fresh named sim │ + │ │ signed-in, advertised│ │ sign-in, pair │ + │ │ waits on done-file │ │ 6-step terminal │ + │ └───────▲──────────────┘ └──────┬─────────────┘ + └────────────────────│───────────────────────── │─────────────┘ + │ (2) touch done-file │ + │ Tailscale SSH, port 22 │ + tailnet │ tag:ci -> tag:ci │ + ─────────────────────┴──────────────┬───────────┴────────────── + │ (1) HTTPS web API: sign-in, + ▼ pairing ticket, advertise + cmux-dev-backend-1.tail137216.ts.net + (per-tag web + Postgres Docker stacks) + + iOS ⇄ Mac terminal data itself flows over IROH + (relay or direct), never over the tailnet — the + ACL below makes the shortcut impossible. +``` + +## Job graph + +| Job | Runner | Timeout | Does | +| --- | --- | --- | --- | +| `route` | Linux (`blacksmith-4vcpu-ubuntu-2404`) | 5m | Decides `run_e2e` (stub: always true, `detect_ci_change_areas.py` integration pending) and the backend tag: `ci` when `web/` changed, shared `ci-main` otherwise. | +| `backend` | Linux | 10m | Joins the tailnet (`tailscale/github-action@v4`, tag:ci), pings the backend host, ensures the tagged stack (stub; real call is cmuxterm-hq `scripts/dev-backend.sh url --tag ` over SSH). | +| `mac-host` | macOS (`MACOS_RUNNER_PR` or `blacksmith-6vcpu-macos-26`) | 45m | Downloads the prebuilt Mac app (reuse pending), joins the tailnet under the deterministic name `cmux-e2e-mac-`, launches signed into the CI Stack account, advertises through the backend, waits on `/tmp/e2e-done-` (bounded ~25m). | +| `ios-e2e` | macOS (`MACOS_RUNNER_IOS` fallback chain) | 45m | Downloads the sim app product (pending), boots a fresh per-run simulator, runs `scripts/e2e/ios-e2e-run.sh`, then ALWAYS signals the Mac's done-file over Tailscale SSH, uploads evidence, deletes the sim. | +| `ios-e2e-status` | Linux | 5m | `if: always()` aggregate; the only check to require. | + +`mac-host` and `ios-e2e` both need only `backend` and run in parallel: the +sim's sign-in/pair sequence retries until the Mac is advertised, so +serializing them would just add queue time. + +Teardown is a local done-file touched over Tailscale SSH, never GitHub API +polling from the Mac's wait loop: a ~25-minute per-PR status poll would draw +down the repo-wide API rate limit every workflow shares, and the file needs +no token on the Mac. + +`ios-e2e-status` semantics: green when the route skipped the lane or every +needed job passed; red when the route said run and any needed job failed, was +cancelled, or was skipped unexpectedly; neutral-skip green on fork PRs (the +secret-fenced jobs cannot run there). It writes the route decision and each +job's result to the step summary. + +## Secrets + +| Secret | Jobs | Purpose | +| --- | --- | --- | +| `TS_OAUTH_CLIENT_ID` / `TS_OAUTH_SECRET` | backend, mac-host, ios-e2e | Tailnet OAuth join, tag:ci. | +| `CMUX_DOGFOOD_STACK_EMAIL` / `CMUX_DOGFOOD_STACK_PASSWORD` | mac-host, ios-e2e | Dedicated CI Stack account, same pair as ios-streamed-validate.yml; both ends must resolve one account for pairing's same-account RPC gate. | +| `CMUX_DEV_BACKEND_SSH_KEY` | backend | **Not yet provisioned.** Deploy key for the VM's dev-backend control API, needed by the real ensure call. | + +Secrets travel only through step environments, never argv, never echoed. +Every secret-mounting job is fenced with +`github.event.pull_request.head.repo.full_name == github.repository`, because +a fork PR controls the workflow file's own content; the aggregate reports +forks as a neutral skip instead of a red. + +## Tailscale ACL requirements + +- `tag:ci` → `cmux-dev-backend-1.tail137216.ts.net` on 443 (Tailscale Serve + web API for sign-in/pairing/advertise) and 22 (dev-backend control SSH, + once the ensure call lands). +- `tag:ci` → `tag:ci` on port 22 ONLY (Tailscale SSH, for the done-file + signal), with an SSH rule mapping to the runner login user. + +The narrowness of the second rule is load-bearing: if runners could reach +each other on arbitrary ports, Iroh's path probing could discover the +runners' Tailscale IPs and carry the terminal stream host-to-host over the +tailnet. The run would go green while testing a transport path no customer +has, which is exactly the false confidence this lane exists to eliminate. +Port 22 alone is useless to Iroh and sufficient for one `touch`. + +## Infra-preflight failure labeling + +Steps that can only fail for infrastructure reasons — tailnet join, backend +ping/ensure, product downloads, simulator boot, the teardown signal — emit +errors prefixed `[infra-preflight]`. Triage rule: an `[infra-preflight]` red +is a fleet/ACL/cache problem for CI infra, never a product regression, and it +does not count against the lane's flake budget during shadow. A red with no +`[infra-preflight]` marker is the E2E itself and gets a +`E2E FAIL step=` line naming the failed step +(see [scripts/e2e/README.md](../../scripts/e2e/README.md)). + +## Promotion plan + +1. **Shadow.** The workflow runs on every PR (route-gated, not required) and + on dispatch. Expected red until the TODOs land, in this order: real + router via `scripts/ci/detect_ci_change_areas.py`; backend ensure with + `CMUX_DEV_BACKEND_SSH_KEY`; Mac app product reuse + (`scripts/ci/reuse_app_host_products.py` consumer path); iOS sim product + reuse (test-ios.yml's `ios-test-product-*` artifact); the two driver + scripts. During shadow, track pass rate and `[infra-preflight]` rate + separately. +2. **Required inside the ios aggregate.** Once the lane holds a stable pass + rate with infra-preflight reds at fleet-noise level, `ios-e2e-status` + joins the required iOS aggregate check rather than becoming its own + branch-protection entry, keeping one required conclusion per area. The + neutral-skip semantics (route skip, fork PRs) already match what a + required check needs. + +Dictionary: **aggregate** — the single always-run job whose conclusion +branch protection requires on behalf of a lane's many conditional jobs; +**shadow** — running a check on every PR without requiring it, to measure +reliability before it can block merges; **done-file** — the local file whose +appearance releases the Mac host's bounded wait, our GitHub-API-free +teardown handshake; **infra-preflight** — a labeled failure in environment +setup (tailnet, cache, backend, simulator) as opposed to the product path +under test. diff --git a/scripts/e2e/README.md b/scripts/e2e/README.md new file mode 100644 index 000000000000..cf360244d2f4 --- /dev/null +++ b/scripts/e2e/README.md @@ -0,0 +1,82 @@ +# scripts/e2e — iOS E2E drivers + +Driver contract for [.github/workflows/ios-e2e.yml](../../.github/workflows/ios-e2e.yml). +The workflow owns runner selection, tailnet join, product download, the +backend stack, evidence upload, and the teardown signal; these scripts own +everything on the runner between "app product on disk" and "verdict". The +interface is environment variables only, no flags — keep it stable, the +workflow and the scripts land from different PRs. + +## mac-host.sh + +Launches the tagged Mac app, signs it into the CI Stack account, advertises it +through the dev backend so the iOS client can discover and pair with it, then +blocks until the iOS job signals completion. + +| Env | Meaning | +| --- | --- | +| `CMUX_E2E_TAG` | Shared dev tag for this run (`ci` or `ci-main`). Names the app bundle (`com.cmuxterm.app.debug.`), the debug socket (`/tmp/cmux-debug-.sock`), and the backend stack. | +| `CMUX_DEV_BACKEND_URL` | Web API origin of the ensured backend stack (private Tailscale Serve URL on the durable VM). | +| `CMUX_E2E_DONE_FILE` | Absolute path of the teardown file. Poll for it locally (sleep loop); the iOS job touches it over Tailscale SSH. Never substitute GitHub API status polling — a ~25-minute per-PR poll loop draws down the repo-wide API rate limit, and the file needs no token. | +| `CMUX_E2E_WAIT_TIMEOUT_SECONDS` | Optional bound on the done-file wait; default 1500 (~25m). Expiry exits nonzero. | +| `CMUX_DOGFOOD_STACK_EMAIL` / `CMUX_DOGFOOD_STACK_PASSWORD` | Dedicated CI Stack account (the pair ios-streamed-validate.yml uses; the app's dev-secrets resolution reads `CMUX_DOGFOOD_STACK_*` from the environment first). Never echo, never pass on argv, never write to disk. | + +Exit 0 means the app launched, signed in, advertised, and the done-file +appeared in time. On failure exit nonzero and name the phase on the last +stderr line: `launch`, `sign-in`, `advertise`, or `wait-timeout`. + +## ios-e2e-run.sh + +Signs the simulator app in, pairs it to the remote Mac through the backend, +connects over Iroh, and drives the 6-step terminal script against a real +streamed terminal. + +| Env | Meaning | +| --- | --- | +| `CMUX_E2E_TAG` | Same shared tag as the Mac host (bundle `dev.cmux.ios.`); pairing is tag-scoped, so a tag mismatch can never pair. | +| `CMUX_DEV_BACKEND_URL` | Web API origin used for sign-in and pairing. | +| `CMUX_E2E_SIM_UDID` | The freshly created, booted simulator this run owns. Pass it to every simctl/idb call; never resolve by name. | +| `CMUX_E2E_EVIDENCE_DIR` | Directory for screenshots, streamed-grid text dumps, and device logs; the workflow uploads it verbatim (`if: always()`). Write a capture at every step boundary, pass or fail. | +| `CMUX_DOGFOOD_STACK_EMAIL` / `CMUX_DOGFOOD_STACK_PASSWORD` | Same account as the Mac host — pairing's same-account RPC gate requires both ends to resolve one account. Same secrecy rules. | + +On failure exit nonzero and print `E2E FAIL step=` as the last stderr +line, where `` is a step id below or `sign-in`, `pair`, `connect` for the +setup phases. + +### The 6-step terminal script + +Each step covers a shipped regression; do not weaken a step without replacing +its coverage. + +1. `marker-1` — type `echo E2E--A` into the streamed terminal and assert + the echoed marker renders in the grid within a bounded wait. Proves the + full live keystroke path: iOS key → Iroh → Mac PTY → stream → grid. + Regression: input echo stall, caught only by marker-echo liveness + ([#12927](https://github.com/manaflow-ai/cmux/pull/12927)). +2. `burst-scrollback` — run `seq 1 5000`, wait for the tail, scroll back and + assert an early line and the final line are both intact. Proves ordered + byte-tee append and scrollback integrity under burst output. + Regression: O(chunk²) byte-tee append and viewport livelock + ([#13432](https://github.com/manaflow-ai/cmux/pull/13432)). +3. `alt-screen` — open `less` on a real file, assert the alt-screen UI + rendered, quit with `q`, assert the primary screen (step 2's tail) is + restored. Proves the atomic alt-screen swap both directions. + Regression: alt-screen transition freeze + ([#12844](https://github.com/manaflow-ai/cmux/pull/12844)). +4. `interrupt` — start `sleep 300`, send Ctrl-C, assert the prompt returns. + Proves control-byte delivery works independently of the output path; an + interrupt that only lands on an idle stream is broken. +5. `replay` — background the iOS app (or drop the connection), generate + output on the Mac side, foreground, and assert the reconnected grid + replays the missed content rather than staying blank. + Regression: black-holed QUIC path kept installed, terminal blank on replay + ([#14030](https://github.com/manaflow-ai/cmux/pull/14030)). +6. `marker-2` — type `echo E2E--B` and assert it echoes. Proves the + session is still live for INPUT after the churn of steps 2–5: reconnect + and recovery must not have wedged the transport behind a cooldown. + Regression: pre-bootstrap recovery armed a cooldown that filtered Iroh + and stalled the fresh session + ([#14124](https://github.com/manaflow-ai/cmux/pull/14124)). + +The workflow — not this script — signals the Mac host's done-file over +Tailscale SSH after this script exits, pass or fail. diff --git a/scripts/e2e/ios-e2e-run.sh b/scripts/e2e/ios-e2e-run.sh new file mode 100755 index 000000000000..f515a7b6a696 --- /dev/null +++ b/scripts/e2e/ios-e2e-run.sh @@ -0,0 +1,304 @@ +#!/usr/bin/env bash +# iOS e2e terminal driver: the six-step terminal script of the PR e2e gate. +# +# Runs against an ALREADY signed-in, paired, connected tagged pair — locally +# the pair `scripts/run-iroh-release-gate.sh --keep-simulator` leaves behind, +# on CI the pair the ios-e2e workflow launches. This script only drives and +# asserts; it never builds, signs in, or pairs. +# +# Every step asserts BOTH sides of the transport: +# - phone side: simulator screenshot + Vision OCR (what actually rendered) +# - Mac side: tagged debug socket (what the real shell actually received) +# One side alone can lie (an echo can render locally without reaching the +# Mac; the Mac can accept input the phone never repaints after). +# +# Steps and the shipped regression class each one guards: +# 1 echo marker round trip input stall (cmux #12927) +# 2 burst output + scrollback byte-tee append (cmux #13432) +# 3 alt-screen enter/exit alt-screen freeze (cmux #12844) +# 4 Ctrl-C a running command control keys cross the transport +# 5 background/foreground blank replay (cmux #14030) +# 6 marker after reconnect recovery cooldown (cmux #14124) +# +# Waits are bounded polls on observable state (OCR text or Mac screen text), +# never fixed sleeps standing in for synchronization. Failures name the step. +set -euo pipefail + +TAG="" +SIM_UDID="" +EVIDENCE_DIR="" +BUNDLE_ID="" +STEP_TIMEOUT=45 + +usage() { + cat <<'EOF' +Usage: scripts/e2e/ios-e2e-run.sh --tag --sim-udid --evidence-dir + [--bundle-id ] [--step-timeout ] +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --tag) TAG="${2:-}"; shift 2 ;; + --sim-udid) SIM_UDID="${2:-}"; shift 2 ;; + --evidence-dir) EVIDENCE_DIR="${2:-}"; shift 2 ;; + --bundle-id) BUNDLE_ID="${2:-}"; shift 2 ;; + --step-timeout) STEP_TIMEOUT="${2:-}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "error: unknown argument '$1'" >&2; usage >&2; exit 2 ;; + esac +done +[[ -n "$TAG" && -n "$SIM_UDID" && -n "$EVIDENCE_DIR" ]] || { usage >&2; exit 2; } + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +SOCKET="/tmp/cmux-debug-${TAG}.sock" +AXE="${CMUX_E2E_AXE:-axe}" +mkdir -p "$EVIDENCE_DIR" + +# --- evidence + assertion helpers ------------------------------------------- + +STEP_NAME="preflight" +STEP_INDEX=0 +TIMINGS_FILE="$EVIDENCE_DIR/steps.jsonl" +: > "$TIMINGS_FILE" + +fail() { + echo "E2E FAIL [$STEP_NAME]: $*" >&2 + shot "failure" + exit 1 +} + +step() { + STEP_INDEX=$((STEP_INDEX + 1)) + STEP_NAME="$1" + STEP_STARTED="$(date +%s)" + echo "== step $STEP_INDEX: $STEP_NAME" +} + +step_done() { + local now + now="$(date +%s)" + printf '{"step":%d,"name":"%s","seconds":%d}\n' \ + "$STEP_INDEX" "$STEP_NAME" "$((now - STEP_STARTED))" >> "$TIMINGS_FILE" + shot "done" +} + +shot() { + xcrun simctl io "$SIM_UDID" screenshot \ + "$EVIDENCE_DIR/$(printf '%02d' "$STEP_INDEX")-$STEP_NAME-$1.png" 2>/dev/null || true +} + +# Compile the Vision OCR helper once per run (plain swiftc, no xcodebuild). +OCR_BIN="$EVIDENCE_DIR/.ocr" +ocr_build() { + [[ -x "$OCR_BIN" ]] && return 0 + swiftc -O "$SCRIPT_DIR/ocr.swift" -o "$OCR_BIN" +} + +phone_text() { + local png="$EVIDENCE_DIR/.probe.png" + xcrun simctl io "$SIM_UDID" screenshot "$png" >/dev/null 2>&1 || return 1 + "$OCR_BIN" "$png" 2>/dev/null || true +} + +mac_text() { + CMUX_TAG="$TAG" "$REPO_ROOT/scripts/cmux-debug-cli.sh" read-screen 2>/dev/null || true +} + +# wait_for