diff --git a/.github/test-determinism-allowlist.txt b/.github/test-determinism-allowlist.txt index 923163d72e9b..52e7d35d489b 100644 --- a/.github/test-determinism-allowlist.txt +++ b/.github/test-determinism-allowlist.txt @@ -7,5 +7,5 @@ Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/OnboardingMacDiscove cmuxTests/WorkspaceForkConversationContextMenuTests.swift assert-on-duration grandfathered cmuxTests/WorkspaceForkConversationContextMenuTests.swift sleep-then-assert grandfathered tests/test_ci_sparkle_build_monotonic.sh live-network-host release pretag guard intentionally probes the published appcast and soft-passes offline -Packages/Shared/CMUXMobileCore/Tests/CMUXMobileCoreTests/CmxRetryAfterPolicyTests.swift sleep-then-assert upstream retry policy timing assertions +Packages/Shared/CMUXMobileCore/Tests/CMUXMobileCoreTests/CmxRetryAfterPolicyTests.swift sleep-then-assert virtual-clock injection makes these large delays non-blocking and deterministic Packages/iOS/CmuxMobileShell/Tests/CmuxMobileShellTests/MobileTerminalLaneCoordinatorTests.swift sleep-then-assert upstream coordinator readiness timing assertion diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea0915087c79..097224fa05fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -332,6 +332,21 @@ jobs: - name: Validate release-build timeout guard run: ./tests/test_ci_release_build_timeout.sh + - name: Validate reusable workflow permission grants + run: python3 tests/test_ci_reusable_workflow_permissions.py + + - name: Validate release does not gate on iOS screenshot capture + run: ./tests/test_ci_release_ios_screenshots_decoupled.sh + + - name: Validate Sparkle monotonic guard modes + run: ./tests/test_sparkle_build_monotonic_modes.sh + + - name: Validate release tunnel extension identifiers + run: ./tests/test_ci_release_tunnel_identifiers.sh + + - name: Validate Sparkle appcast generation without previous archives + run: ./tests/test_sparkle_generate_appcast_no_deltas.sh + - name: Validate markdown viewer asset compression run: ./tests/test_compress_markdown_viewer_assets.sh @@ -916,22 +931,45 @@ jobs: # Ghostty-backed panels. Run the close and placement suites in separate # app-host processes so renderer teardown from one topology workload # cannot crash the next before its assertions execute. + # + # The app host still crashes intermittently mid-suite here + # (https://github.com/manaflow-ai/cmux/issues/9348): the crash point + # moves between tests and the relaunched host passes every remaining + # test. Rerun a suite once, and only when xcodebuild reports that the + # host exited, crashed, or timed out. An assertion failure never earns + # a rerun, and a second crash still fails the shard. set -euo pipefail SOURCE_PACKAGES_DIR="$PWD/.ci-source-packages" for suite in \ RemoteTmuxMirrorCloseDetachTests \ RemoteTmuxMirrorDedicatedPlacementTests do - scripts/ci/run-in-console-session.sh \ - scripts/ci/run-app-host-xcodebuild.sh \ - -project cmux.xcodeproj -scheme cmux-unit -configuration Debug \ - -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ - -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ - -disableAutomaticPackageResolution \ - -destination "platform=macOS" \ - CMUX_SKIP_ZIG_BUILD=1 \ - -only-testing:"cmuxTests/$suite" \ - test + for attempt in 1 2; do + suite_output="$RUNNER_TEMP/cmux-remote-tmux-mirror-${suite}-attempt-${attempt}.txt" + set +e + scripts/ci/run-in-console-session.sh \ + scripts/ci/run-app-host-xcodebuild.sh \ + -project cmux.xcodeproj -scheme cmux-unit -configuration Debug \ + -derivedDataPath "$CMUX_DERIVED_DATA_PATH" \ + -clonedSourcePackagesDirPath "$SOURCE_PACKAGES_DIR" \ + -disableAutomaticPackageResolution \ + -destination "platform=macOS" \ + CMUX_SKIP_ZIG_BUILD=1 \ + -only-testing:"cmuxTests/$suite" \ + test 2>&1 | tee "$suite_output" + suite_status="${PIPESTATUS[0]}" + set -e + if [ "$suite_status" -eq 0 ]; then + break + fi + if [ "$attempt" -eq 1 ] \ + && grep -Fq 'Restarting after unexpected exit, crash, or test timeout' "$suite_output"; then + echo "::warning::app host crashed while running cmuxTests/${suite} (https://github.com/manaflow-ai/cmux/issues/9348); rerunning the suite once" + continue + fi + echo "cmuxTests/${suite} failed with status ${suite_status} on attempt ${attempt}" + exit "$suite_status" + done done - name: Run browser system proxy mirror regression diff --git a/.github/workflows/ios-screenshots.yml b/.github/workflows/ios-screenshots.yml index 4512a96b8b0d..398ef5d7f03c 100644 --- a/.github/workflows/ios-screenshots.yml +++ b/.github/workflows/ios-screenshots.yml @@ -53,9 +53,18 @@ on: required: false default: false type: boolean +# Capture-only needs nothing beyond reading the repository: checkout runs with +# persist-credentials disabled, and artifact uploads use the runner's artifact +# token rather than GITHUB_TOKEN. Nothing here dispatches, cancels or deletes +# runs, so `actions: write` was never exercised, and declaring it broke every +# caller that grants less: GitHub validates a reusable workflow's permissions +# against the calling job at parse time, and release.yml failed at startup +# with "is requesting 'actions: write', but is only allowed 'actions: none'" +# (https://github.com/manaflow-ai/cmux/issues/12149). Keep this block at the +# minimum the steps use; tests/test_ci_reusable_workflow_permissions.py checks +# every local caller against it. permissions: contents: read - actions: write jobs: screenshots: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6e6a6d126ded..fadd70d0e8f1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,12 +17,26 @@ env: jobs: generate-ios-screenshots: name: Generate iOS App Store screenshots + # Runs alongside the macOS build so every stable tag still ships screenshot + # artifacts captured at the exact release ref. It is deliberately not a + # dependency of build-sign-notarize: the DMG consumes nothing from it, and a + # slow or flaky simulator capture (up to 300 minutes) must neither delay nor + # fail a macOS release (https://github.com/manaflow-ai/cmux/issues/12149). + # A failed capture still turns the run red, so it stays visible. + # + # Least privilege: capture-only reads the repository and nothing else, so + # this job hands the callee neither the release token scopes nor the + # repository secrets. The App Store Connect upload path inside the called + # workflow is gated to workflow_dispatch from main and is unreachable from + # a release run, so `secrets: inherit` would only have exposed the signing, + # notarization, Sparkle and R2 secrets to a UI-test job. + permissions: + contents: read uses: ./.github/workflows/ios-screenshots.yml with: ref: ${{ github.ref }} languages: "en-US,de-DE,fr-FR,ar-SA,es-ES,zh-Hant,zh-Hans,ko,ja" upload: false - secrets: inherit build-ghostty-cli-helper: runs-on: ${{ vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }} @@ -72,9 +86,7 @@ jobs: retention-days: 3 build-sign-notarize: - needs: - - build-ghostty-cli-helper - - generate-ios-screenshots + needs: build-ghostty-cli-helper # Build the app on macOS 26 so SDK-gated SwiftUI Liquid Glass code compiles # into stable releases. The real universal Ghostty CLI helper is built on # macOS 15 above because Zig 0.15.2 cannot link it on macOS 26. @@ -84,7 +96,11 @@ jobs: # Notarization wait times vary on Apple's side. The standalone Computer Use # helper starts as soon as the unsigned app exists and overlaps the remaining # release preparation; the outer app and final DMG still serialize afterward. - timeout-minutes: 60 + # v0.64.22 took 40 minutes before the Cloud tunnel extension, its Go engine + # build and the universal diff sidecar joined this job, and the helper's + # Gatekeeper propagation wait alone can now take twenty minutes, so 60 left + # no headroom. Keep at least 25 percent above the measured duration. + timeout-minutes: 90 steps: - name: Clear stale git locks (self-hosted reused workspace) shell: bash @@ -104,6 +120,12 @@ jobs: submodules: recursive - name: Validate Sparkle build number is monotonic + # A tag push is about to publish, so a stale build number must fail. + # A non-tag workflow_dispatch is the built-in dry run: it publishes + # nothing and normally runs from a branch that has not been bumped yet, + # so the same condition only warns there. + env: + CMUX_SPARKLE_MONOTONIC_MODE: ${{ startsWith(github.ref, 'refs/tags/') && 'enforce' || 'warn' }} run: ./tests/test_ci_sparkle_build_monotonic.sh - name: Guard immutable release assets @@ -339,7 +361,11 @@ jobs: run: | set -euo pipefail APP="build-universal/Build/Products/Release/cmux.app" - ./scripts/normalize-system-extension-bundle.sh "$APP" "7WLXT3NR37.com.cmuxterm.app.tunnel" + # The Release build's tunnel extension is com.cmuxterm.app.tunnel: the + # bundle identifier carries no team prefix (only NEMachServiceName and + # the profile's App ID do), exactly as nightly ships .tunnel. + # tests/test_ci_release_tunnel_identifiers.sh pins this to the project. + ./scripts/normalize-system-extension-bundle.sh "$APP" "com.cmuxterm.app.tunnel" APP_BINARY="build-universal/Build/Products/Release/cmux.app/Contents/MacOS/cmux" CLI_BINARY="build-universal/Build/Products/Release/cmux.app/Contents/Resources/bin/cmux" HELPER_BINARY="build-universal/Build/Products/Release/cmux.app/Contents/Resources/bin/ghostty" @@ -355,7 +381,7 @@ jobs: [[ "$APP_ARCHS" == *arm64* && "$APP_ARCHS" == *x86_64* ]] [[ "$CLI_ARCHS" == *arm64* && "$CLI_ARCHS" == *x86_64* ]] [[ "$HELPER_ARCHS" == *arm64* && "$HELPER_ARCHS" == *x86_64* ]] - TUNNEL_BINARY="$APP/Contents/Library/SystemExtensions/7WLXT3NR37.com.cmuxterm.app.tunnel.systemextension/Contents/MacOS/cmuxTunnel" + TUNNEL_BINARY="$APP/Contents/Library/SystemExtensions/com.cmuxterm.app.tunnel.systemextension/Contents/MacOS/cmuxTunnel" [ -x "$TUNNEL_BINARY" ] || { echo "Cloud tunnel extension binary not found at $TUNNEL_BINARY" >&2; exit 1; } TUNNEL_ARCHS="$(lipo -archs "$TUNNEL_BINARY")" echo "Tunnel extension architectures: $TUNNEL_ARCHS" @@ -563,6 +589,13 @@ jobs: exit 1 fi ./scripts/sparkle_generate_appcast.sh cmux-macos.dmg "$GITHUB_REF_NAME" appcast.xml + # macOS /bin/bash 3.2 can report "unbound variable" from a script with + # an EXIT trap and still exit 0; dry run 34227505375 (2026-09-08) + # uploaded a DMG with no appcast that way. The feed must exist, carry + # an EdDSA signature and point at this DMG before anything uploads it. + test -s appcast.xml + grep -q 'sparkle:edSignature' appcast.xml + grep -q 'cmux-macos.dmg' appcast.xml - name: Upload build artifacts (dry-run) if: steps.guard_release_assets.outputs.skip_upload != 'true' && github.event_name == 'workflow_dispatch' diff --git a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohEndpointServerTests.swift b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohEndpointServerTests.swift index e1ef5a1a260f..b6bd291dc1ec 100644 --- a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohEndpointServerTests.swift +++ b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohEndpointServerTests.swift @@ -340,28 +340,35 @@ struct CmxIrohEndpointServerTests { _ = try await supervisor.activate() let blocker = EndpointServerHandlerBlocker() let recorder = EndpointServerRecorder() + let first = TestIrohConnection( + remoteIdentity: remoteIdentity, + bidirectionalStreams: [] + ) + let replacement = TestIrohConnection( + remoteIdentity: remoteIdentity, + bidirectionalStreams: [] + ) let server = CmxIrohEndpointServer(supervisor: supervisor) { connection, generation, admission in + // Admit before recording so the test's `recorder.next()` below + // proves `first` is already active when `replacement` arrives, + // and promote only the replacement. Keying the promotion off the + // recorded count let the first handler observe the second record + // before its own check and promote `first` instead, which + // superseded the replacement and failed its markUsable (CI run + // 34414741413, swift-package-tests). + #expect(await admission()) await recorder.record( identity: await connection.remoteIdentity(), generation: generation ) - #expect(await admission()) - if await recorder.recordedCount() == 2 { + if (connection as? TestIrohConnection) === replacement { #expect(await admission.markUsable()) } await blocker.wait() } - let first = TestIrohConnection( - remoteIdentity: remoteIdentity, - bidirectionalStreams: [] - ) - let replacement = TestIrohConnection( - remoteIdentity: remoteIdentity, - bidirectionalStreams: [] - ) var firstCloses = await first.closeEvents().makeAsyncIterator() await server.start() diff --git a/cmuxTests/CLICoderouterCommandTests.swift b/cmuxTests/CLICoderouterCommandTests.swift index e3771226e589..41c4ba87998e 100644 --- a/cmuxTests/CLICoderouterCommandTests.swift +++ b/cmuxTests/CLICoderouterCommandTests.swift @@ -65,7 +65,7 @@ extension CLINotifyProcessIntegrationRegressionTests { unlink(socketPath) } - let serverHandled = startMockServer(listenerFD: listenerFD, state: state) { line in + let respond: @Sendable (String) -> String = { line in guard let payload = self.jsonObject(line), let id = payload["id"] as? String, let method = payload["method"] as? String else { @@ -81,6 +81,18 @@ extension CLINotifyProcessIntegrationRegressionTests { error: ["code": "unexpected", "message": "Unexpected method \(method)"] ) } + // A test that expects the CLI to stay off the socket must not hold a + // case-bound expectation it never waits on: the shared accept loop + // fulfills it when the listener closes below, and XCTest reports a + // fulfilled-but-unwaited expectation as an unexpected failure, which + // the app-host batch classifier turns into a red shard. + let serverHandled: XCTestExpectation? + if waitForSocket { + serverHandled = startMockServer(listenerFD: listenerFD, state: state, handler: respond) + } else { + serverHandled = nil + startDetachedMockServer(listenerFD: listenerFD, state: state, handler: respond) + } var environment = ProcessInfo.processInfo.environment environment["CMUX_SOCKET_PATH"] = socketPath @@ -98,7 +110,7 @@ extension CLINotifyProcessIntegrationRegressionTests { standardInput: standardInput, timeout: 5 ) - if waitForSocket { + if let serverHandled { wait(for: [serverHandled], timeout: 5) } return (result, state) diff --git a/cmuxTests/CodexAppServerSessionTests.swift b/cmuxTests/CodexAppServerSessionTests.swift index 630b4a471f29..f4a748a45d79 100644 --- a/cmuxTests/CodexAppServerSessionTests.swift +++ b/cmuxTests/CodexAppServerSessionTests.swift @@ -722,6 +722,11 @@ struct CodexAppServerSessionTests { func testClaudeStreamJSONAccumulatorTracksDeltaTextPerAssistantMessage() { var accumulator = ClaudeStreamJSONAccumulator() + expectEqual( + accumulator.consumeLine( + #"{"type":"message_start","message":{"id":"msg_1","role":"assistant"}}"#), + [] + ) expectEqual( accumulator.consumeLine( #"{"type":"content_block_delta","delta":{"type":"text_delta","text":"first"}}"#), @@ -733,6 +738,11 @@ struct CodexAppServerSessionTests { ), [" done"] ) + expectEqual( + accumulator.consumeLine( + #"{"type":"message_start","message":{"id":"msg_2","role":"assistant"}}"#), + [] + ) expectEqual( accumulator.consumeLine( #"{"type":"content_block_delta","delta":{"type":"text_delta","text":"second"}}"#), diff --git a/cmuxTests/SSHPTYAttachReconnectInputFilterTests.swift b/cmuxTests/SSHPTYAttachReconnectInputFilterTests.swift index a08a0e88b06b..4a0a73fcc02a 100644 --- a/cmuxTests/SSHPTYAttachReconnectInputFilterTests.swift +++ b/cmuxTests/SSHPTYAttachReconnectInputFilterTests.swift @@ -2,7 +2,13 @@ import Darwin import Foundation import Testing -@Suite struct SSHPTYAttachReconnectInputFilterTests { +// Serialized: each stdin-pump test parks a Swift Testing cooperative thread in +// a blocking wait while the pump under test is a detached task that needs one +// of those same threads. Running the five pump tests concurrently can leave +// no thread for any pump, and the suite then hangs until CI's idle timeout +// (run 34414741413, shard 3, three attempts). Every wait below is also +// bounded so a starved pump fails the test instead of the batch. +@Suite(.serialized) struct SSHPTYAttachReconnectInputFilterTests { @Test func deadlineFlushesPendingAndStopsStripping() { var expired = false let filter = SSHPTYAttachReconnectInputFilter( @@ -69,7 +75,7 @@ import Testing let lateProbeReply = Data("\u{1B}[1;1R".utf8) let forwardedInput = Data("printf keep\n".utf8) try writeAll(fd: inputPipe[1], data: lateProbeReply + forwardedInput) - control?.stopFiltering() + #expect(control?.stopFiltering(timeoutMilliseconds: Self.waitTimeoutMilliseconds) == true) Darwin.close(inputPipe[1]) inputPipe[1] = -1 @@ -95,7 +101,7 @@ import Testing ) #expect(control != nil) - control?.stopFiltering() + #expect(control?.stopFiltering(timeoutMilliseconds: Self.waitTimeoutMilliseconds) == true) let liveProbeReply = Data("\u{1B}[2;2R".utf8) try writeAll(fd: inputPipe[1], data: liveProbeReply) Darwin.close(inputPipe[1]) @@ -127,7 +133,7 @@ import Testing try writeAll(fd: inputPipe[1], data: normalInput) #expect(try readExactly(fd: bridgePair[1], count: normalInput.count) == normalInput) - control?.stopFiltering() + #expect(control?.stopFiltering(timeoutMilliseconds: Self.waitTimeoutMilliseconds) == true) let liveProbeReply = Data("\u{1B}[3;3R".utf8) try writeAll(fd: inputPipe[1], data: liveProbeReply) Darwin.close(inputPipe[1]) @@ -197,10 +203,35 @@ import Testing } } + /// Longer than the pump's reconnect probe deadline, which the EOF reads + /// below legitimately wait out, and short enough that a pump that never + /// got scheduled fails the test well inside CI's 300s idle timeout. + private static let waitTimeoutMilliseconds: Int32 = 30_000 + + private struct WaitTimedOut: Error {} + + private func waitReadable(fd: Int32) throws { + let events = Int16(POLLIN | POLLHUP | POLLERR | POLLNVAL) + var pollFD = pollfd(fd: fd, events: events, revents: 0) + while true { + let result = Darwin.poll(&pollFD, 1, Self.waitTimeoutMilliseconds) + if result > 0 { + return + } + if result == 0 { + throw WaitTimedOut() + } + if errno != EINTR { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + } + } + private func readUntilEOF(fd: Int32) throws -> Data { var output = Data() var buffer = [UInt8](repeating: 0, count: 1024) while true { + try waitReadable(fd: fd) let count = Darwin.read(fd, &buffer, buffer.count) if count > 0 { output.append(contentsOf: buffer.prefix(count)) @@ -217,6 +248,7 @@ import Testing var buffer = [UInt8](repeating: 0, count: 1024) while output.count < expectedCount { let remaining = expectedCount - output.count + try waitReadable(fd: fd) let count = Darwin.read(fd, &buffer, min(buffer.count, remaining)) if count > 0 { output.append(contentsOf: buffer.prefix(count)) diff --git a/scripts/ci/check_reusable_workflow_permissions.py b/scripts/ci/check_reusable_workflow_permissions.py new file mode 100755 index 000000000000..9955139d53d8 --- /dev/null +++ b/scripts/ci/check_reusable_workflow_permissions.py @@ -0,0 +1,590 @@ +#!/usr/bin/env python3 +"""Reject local reusable-workflow calls whose callee asks for more than the caller grants. + +GitHub resolves ``jobs..uses: ./.github/workflows/`` while it parses the *caller* +workflow. The called workflow's ``permissions`` (its workflow-level block and every job-level +block, whether or not the job is gated by ``if:``) may only keep or reduce what the calling job +grants. Any scope that asks for more fails the caller before a single job starts:: + + Invalid workflow file: ... is requesting 'actions: write', but is only allowed 'actions: none'. + +That is a ``startup_failure``: no job runs, and a tag push fails exactly like a manual dispatch. +https://github.com/manaflow-ai/cmux/issues/12149 blocked the stable macOS release this way after +``release.yml`` started calling ``ios-screenshots.yml``. + +Rules mirrored here (https://docs.github.com/en/actions/how-tos/sharing-automations/reuse-workflows: +"permissions can only be maintained or reduced, not elevated, throughout the chain"): + +* The calling job's grant is its own ``permissions`` block, else the caller workflow's block, else + the repository default (``--default-workflow-permissions``). A declared block sets every unlisted + scope to ``none``; ``metadata: read`` is always granted; the default never grants ``id-token``. +* The callee's request is the per-scope maximum over its workflow-level block and every job-level + block. A callee that declares no ``permissions`` anywhere inherits the caller's grant. +* Nested calls are checked with the intermediate job's grant. +* Only ``./.github/workflows/...`` callees are checked; cross-repository callees cannot be read here. + +No third-party modules: workflow files are read with a small YAML-subset reader that understands +the block shapes GitHub Actions accepts (block scalars, indentless sequences, flow mappings, quoted +keys, comments). Sequences stay opaque because nothing under ``steps`` matters to this check. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Optional, Union + +LEVELS = {"none": 0, "read": 1, "write": 2} +LEVEL_NAMES = {value: name for name, value in LEVELS.items()} + +# https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#permissions +SCOPES = ( + "actions", + "attestations", + "checks", + "contents", + "deployments", + "discussions", + "id-token", + "issues", + "metadata", + "models", + "packages", + "pages", + "pull-requests", + "repository-projects", + "security-events", + "statuses", +) + +LOCAL_USES_PREFIX = "./.github/workflows/" + +Permissions = dict[str, int] + + +class WorkflowSyntaxError(ValueError): + """A workflow file uses a shape this reader (or GitHub) does not accept.""" + + +# -------------------------------------------------------------------------------------- +# YAML subset reader +# -------------------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class BlockScalar: + """Opaque ``|`` / ``>`` scalar; its text never matters to the permission check.""" + + indicator: str + + +@dataclass(frozen=True) +class _Line: + number: int + indent: int + text: str + + +def _split_lines(text: str) -> list[_Line]: + lines: list[_Line] = [] + for number, raw in enumerate(text.splitlines(), start=1): + raw = raw.rstrip("\r") + stripped = raw.lstrip(" ") + lines.append(_Line(number, len(raw) - len(stripped), stripped.rstrip())) + return lines + + +def _is_insignificant(line: _Line) -> bool: + text = line.text + if not text or text.startswith("#"): + return True + if line.indent == 0 and (text == "---" or text.startswith("--- ") or text == "..." or text.startswith("%")): + return True + return False + + +def _is_sequence_item(text: str) -> bool: + return text == "-" or text.startswith("- ") + + +def _strip_inline_comment(value: str) -> str: + quote: Optional[str] = None + for index, char in enumerate(value): + if quote: + if char == quote: + quote = None + elif char in ("'", '"'): + quote = char + elif char == "#" and (index == 0 or value[index - 1] in " \t"): + return value[:index].rstrip() + return value.rstrip() + + +def _unquote(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + inner = value[1:-1] + return inner.replace("''", "'") if value[0] == "'" else inner.replace('\\"', '"') + return value + + +def _split_key(text: str) -> Optional[tuple[str, str]]: + """Split ``key: rest`` at the first unquoted ``:`` followed by whitespace or end of line.""" + quote: Optional[str] = None + for index, char in enumerate(text): + if quote: + if char == quote: + quote = None + continue + if index == 0 and char in ("'", '"'): + quote = char + continue + if char == ":" and (index + 1 == len(text) or text[index + 1] in " \t"): + return _unquote(text[:index]), text[index + 1 :].strip() + return None + + +def _split_flow_items(body: str) -> list[str]: + items: list[str] = [] + depth = 0 + quote: Optional[str] = None + current: list[str] = [] + for char in body: + if quote: + current.append(char) + if char == quote: + quote = None + continue + if char in ("'", '"'): + quote = char + elif char in "{[": + depth += 1 + elif char in "}]": + depth -= 1 + elif char == "," and depth == 0: + items.append("".join(current).strip()) + current = [] + continue + current.append(char) + tail = "".join(current).strip() + if tail: + items.append(tail) + return [item for item in items if item] + + +class _Reader: + def __init__(self, text: str, name: str) -> None: + self.lines = _split_lines(text) + self.index = 0 + self.name = name + + # -- cursor helpers ------------------------------------------------------------- + + def _skip_insignificant(self) -> Optional[_Line]: + while self.index < len(self.lines) and _is_insignificant(self.lines[self.index]): + self.index += 1 + return self.lines[self.index] if self.index < len(self.lines) else None + + def _consume_deeper(self, indent: int) -> None: + """Swallow continuation lines (block scalar text, wrapped plain scalars, nested comments).""" + while self.index < len(self.lines): + line = self.lines[self.index] + if line.text == "" or line.indent > indent: + self.index += 1 + continue + return + + # -- structures ----------------------------------------------------------------- + + def parse_document(self) -> dict: + first = self._skip_insignificant() + if first is None: + return {} + if _is_sequence_item(first.text): + raise WorkflowSyntaxError(f"{self.name}: workflow root must be a mapping") + return self.parse_mapping(first.indent) + + def parse_mapping(self, indent: int) -> dict: + mapping: dict = {} + while True: + line = self._skip_insignificant() + if line is None or line.indent < indent: + return mapping + if line.indent > indent or _is_sequence_item(line.text): + if line.indent == indent: + # An indentless sequence belongs to the parent key, not to this mapping. + return mapping + # Stray deeper content (a wrapped scalar we could not attribute); skip it. + self.index += 1 + continue + split = _split_key(line.text) + self.index += 1 + if split is None: + continue + key, rest = split + mapping[key] = self._parse_value(rest, indent) + + def parse_sequence(self, indent: int) -> list[str]: + """Opaque sequence: returns the first line of every item, nothing nested.""" + items: list[str] = [] + while self.index < len(self.lines): + line = self.lines[self.index] + if line.text == "" or line.indent > indent: + self.index += 1 + continue + if line.indent == indent and _is_sequence_item(line.text): + items.append(_unquote(_strip_inline_comment(line.text[1:].strip()))) + self.index += 1 + continue + if line.indent == indent and line.text.startswith("#"): + self.index += 1 + continue + return items + return items + + def _parse_value(self, rest: str, indent: int) -> Union[None, str, dict, list, BlockScalar]: + value = _strip_inline_comment(rest) + if value == "": + nxt = self._skip_insignificant() + if nxt is not None and nxt.indent > indent: + if _is_sequence_item(nxt.text): + return self.parse_sequence(nxt.indent) + return self.parse_mapping(nxt.indent) + if nxt is not None and nxt.indent == indent and _is_sequence_item(nxt.text): + return self.parse_sequence(indent) + return None + if value[0] in "|>": + self._consume_deeper(indent) + return BlockScalar(value) + if value[0] == "{": + return self._parse_flow_mapping(self._collect_flow(value, "{", "}")) + if value[0] == "[": + body = self._collect_flow(value, "[", "]").strip()[1:-1] + return [_unquote(item) for item in _split_flow_items(body)] + self._consume_deeper(indent) + return _unquote(value) + + def _collect_flow(self, first: str, open_char: str, close_char: str) -> str: + text = first + while text.count(open_char) > text.count(close_char) and self.index < len(self.lines): + text += " " + _strip_inline_comment(self.lines[self.index].text) + self.index += 1 + if text.count(open_char) != text.count(close_char): + raise WorkflowSyntaxError(f"{self.name}: unterminated flow collection: {first}") + return text + + def _parse_flow_mapping(self, text: str) -> dict: + body = text.strip() + if not (body.startswith("{") and body.endswith("}")): + raise WorkflowSyntaxError(f"{self.name}: malformed flow mapping: {text}") + mapping: dict = {} + for item in _split_flow_items(body[1:-1]): + split = _split_key(item) + if split is None: + raise WorkflowSyntaxError(f"{self.name}: malformed flow mapping entry: {item}") + key, rest = split + mapping[key] = _unquote(rest) + return mapping + + +def parse_workflow(path: Path) -> dict: + return _Reader(path.read_text(encoding="utf-8"), path.name).parse_document() + + +# -------------------------------------------------------------------------------------- +# Permission semantics +# -------------------------------------------------------------------------------------- + + +def normalize_permissions(value: object, where: str) -> Optional[Permissions]: + """Turn a ``permissions`` value into ``{scope: level}``; ``None`` when not declared.""" + if value is None: + return None + if isinstance(value, str): + if value == "read-all": + return {scope: LEVELS["read"] for scope in SCOPES} + if value == "write-all": + return {scope: LEVELS["write"] for scope in SCOPES} + raise WorkflowSyntaxError(f"{where}: unsupported permissions value {value!r}") + if isinstance(value, dict): + result: Permissions = {} + for scope, level in value.items(): + if not isinstance(level, str) or level not in LEVELS: + raise WorkflowSyntaxError(f"{where}: unsupported permission level {scope}: {level!r}") + result[scope] = LEVELS[level] + return result + raise WorkflowSyntaxError(f"{where}: permissions must be a mapping, read-all or write-all") + + +def default_grant(default_workflow_permissions: str) -> Permissions: + """Token permissions when a workflow declares none (the repository Actions setting).""" + if default_workflow_permissions == "write": + grant = {scope: LEVELS["write"] for scope in SCOPES} + grant["id-token"] = LEVELS["none"] + grant["metadata"] = LEVELS["read"] + return grant + if default_workflow_permissions == "read": + return {"contents": LEVELS["read"], "packages": LEVELS["read"], "metadata": LEVELS["read"]} + raise ValueError(f"unknown default workflow permissions: {default_workflow_permissions!r}") + + +def _with_metadata_floor(grant: Permissions) -> Permissions: + result = dict(grant) + result["metadata"] = max(result.get("metadata", 0), LEVELS["read"]) + return result + + +def format_permissions(permissions: Permissions) -> str: + if not permissions: + return "(none)" + return ", ".join(f"{scope}: {LEVEL_NAMES[level]}" for scope, level in sorted(permissions.items())) + + +@dataclass(frozen=True) +class Request: + scope: str + level: int + origin: str + + +@dataclass(frozen=True) +class Grant: + permissions: Permissions + origin: str + + +@dataclass(frozen=True) +class Edge: + caller: Path + job: str + callee: Path + grant: Grant + requests: tuple[Request, ...] + # Workflow files from the top-level run down to ``caller``; longer than one entry for nested calls. + chain: tuple[str, ...] + + @property + def site(self) -> tuple[Path, str]: + return (self.caller, self.job) + + +def _jobs(document: dict, where: str) -> dict: + jobs = document.get("jobs") + if jobs is None: + return {} + if not isinstance(jobs, dict): + raise WorkflowSyntaxError(f"{where}: jobs must be a mapping") + return {name: job for name, job in jobs.items() if isinstance(job, dict)} + + +def _job_uses(job: dict) -> Optional[str]: + uses = job.get("uses") + return uses if isinstance(uses, str) else None + + +def is_local_reusable(uses: Optional[str]) -> bool: + return bool(uses) and uses.startswith(LOCAL_USES_PREFIX) + + +def is_directly_runnable(document: dict) -> bool: + """True when some trigger other than ``workflow_call`` can start this workflow on its own.""" + triggers = document.get("on") + if isinstance(triggers, str): + names = {triggers} + elif isinstance(triggers, (list, dict)): + names = set(triggers) + else: + return False + return bool(names - {"workflow_call"}) + + +def resolve_callee(uses: str, workflows_dir: Path) -> Path: + return workflows_dir / uses[len(LOCAL_USES_PREFIX) :] + + +def callee_requests(document: dict, label: str) -> list[Request]: + """Every scope the callee declares anywhere, at the highest level it asks for.""" + highest: dict[str, Request] = {} + + def record(block: Optional[Permissions], origin: str) -> None: + if block is None: + return + for scope, level in block.items(): + current = highest.get(scope) + if current is None or level > current.level: + highest[scope] = Request(scope, level, origin) + + record( + normalize_permissions(document.get("permissions"), f"{label} workflow-level permissions"), + f"{label} workflow-level permissions", + ) + for job_name, job in _jobs(document, label).items(): + record( + normalize_permissions(job.get("permissions"), f"{label} job '{job_name}' permissions"), + f"{label} job '{job_name}' permissions", + ) + return sorted(highest.values(), key=lambda request: request.scope) + + +def job_grant( + document: dict, + job: dict, + label: str, + job_name: str, + inherited: Optional[Grant], + default_workflow_permissions: str, +) -> Grant: + """What the token handed to ``job`` may do: job block, else workflow block, else inherited/default.""" + job_level = normalize_permissions(job.get("permissions"), f"{label} job '{job_name}' permissions") + if job_level is not None: + return Grant(_with_metadata_floor(job_level), f"{label} job '{job_name}' permissions") + workflow_level = normalize_permissions(document.get("permissions"), f"{label} workflow-level permissions") + if workflow_level is not None: + return Grant(_with_metadata_floor(workflow_level), f"{label} workflow-level permissions") + if inherited is not None: + return Grant(inherited.permissions, f"{inherited.origin} (inherited)") + return Grant( + default_grant(default_workflow_permissions), + f"repository default workflow permissions ({default_workflow_permissions})", + ) + + +def check_workflows_dir(workflows_dir: Path, default_workflow_permissions: str) -> tuple[list[Edge], list[str]]: + """Return every checked local call and the failures GitHub would raise at startup. + + Directly runnable workflows are walked as top-level runs; a ``workflow_call``-only file is + checked in the context of each caller (its grant may be inherited), and one nobody calls is + checked on its own so no ``uses:`` site is silently skipped. + """ + edges: list[Edge] = [] + failures: list[str] = [] + seen: set[tuple[tuple[str, ...], str, str]] = set() + workflows = sorted(path for path in workflows_dir.iterdir() if path.suffix in {".yml", ".yaml"} and path.is_file()) + parsed: dict[Path, dict] = {} + called: set[Path] = set() + + def load(path: Path) -> dict: + if path not in parsed: + parsed[path] = parse_workflow(path) + return parsed[path] + + def check_call(caller: Path, job_name: str, job: dict, grant: Grant, chain: tuple[Path, ...]) -> None: + uses = _job_uses(job) + if not is_local_reusable(uses): + return + assert uses is not None + callee = resolve_callee(uses, workflows_dir) + chain_names = tuple(path.name for path in chain) + key = (chain_names, job_name, uses) + if key in seen: + return + seen.add(key) + via = "" if len(chain) == 1 else f" (reached via {' -> '.join(chain_names[:-1])})" + prefix = f"{caller.name}{via}: job '{job_name}' uses {uses}" + if not callee.is_file(): + failures.append(f"{prefix}, but that workflow file does not exist") + return + if callee in chain: + loop = " -> ".join(path.name for path in (*chain, callee)) + failures.append(f"{prefix}, which closes a reusable-workflow cycle ({loop})") + return + called.add(callee) + callee_document = load(callee) + requests = tuple(callee_requests(callee_document, callee.name)) + edges.append(Edge(caller, job_name, callee, grant, requests, chain_names)) + for request in requests: + allowed = grant.permissions.get(request.scope, LEVELS["none"]) + if request.level > allowed: + failures.append( + f"{prefix}, which requests '{request.scope}: {LEVEL_NAMES[request.level]}' " + f"({request.origin}) but the calling job only allows " + f"'{request.scope}: {LEVEL_NAMES[allowed]}' ({grant.origin}). " + "GitHub rejects the caller at startup; reduce the callee or widen the calling job." + ) + for nested_name, nested_job in _jobs(callee_document, callee.name).items(): + if not is_local_reusable(_job_uses(nested_job)): + continue + nested_grant = job_grant( + callee_document, nested_job, callee.name, nested_name, grant, default_workflow_permissions + ) + check_call(callee, nested_name, nested_job, nested_grant, (*chain, callee)) + + def walk_top_level(caller: Path) -> None: + document = load(caller) + for job_name, job in _jobs(document, caller.name).items(): + if not is_local_reusable(_job_uses(job)): + continue + grant = job_grant(document, job, caller.name, job_name, None, default_workflow_permissions) + check_call(caller, job_name, job, grant, (caller,)) + + runnable = [path for path in workflows if is_directly_runnable(load(path))] + for caller in runnable: + walk_top_level(caller) + for orphan in workflows: + if orphan in runnable or orphan in called: + continue + walk_top_level(orphan) + return edges, failures + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--workflows-dir", + type=Path, + default=Path(__file__).resolve().parents[2] / ".github" / "workflows", + help="directory holding the workflow files (default: this repository's .github/workflows)", + ) + parser.add_argument( + "--default-workflow-permissions", + choices=("read", "write"), + default="write", + help=( + "GITHUB_TOKEN default for callers that declare no permissions; mirrors the repository " + "Actions setting (manaflow-ai/cmux: write). Use read to model a restricted repository." + ), + ) + parser.add_argument("--verbose", action="store_true", help="print every checked call") + args = parser.parse_args(argv) + + if not args.workflows_dir.is_dir(): + print(f"FAIL: workflows directory not found: {args.workflows_dir}", file=sys.stderr) + return 2 + try: + edges, failures = check_workflows_dir(args.workflows_dir, args.default_workflow_permissions) + except WorkflowSyntaxError as error: + print(f"FAIL: {error}", file=sys.stderr) + return 2 + + if args.verbose: + for edge in edges: + requested = ", ".join(f"{r.scope}: {LEVEL_NAMES[r.level]}" for r in edge.requests) or "(inherits caller)" + via = "" if len(edge.chain) == 1 else f" (reached via {' -> '.join(edge.chain[:-1])})" + print( + f"{edge.caller.name}{via}: job '{edge.job}' -> {edge.callee.name}\n" + f" grant [{edge.grant.origin}]: {format_permissions(edge.grant.permissions)}\n" + f" request: {requested}" + ) + for failure in failures: + print(f"FAIL: {failure}", file=sys.stderr) + sites = len({edge.site for edge in edges}) + callers = len({edge.caller for edge in edges}) + if failures: + print( + f"FAIL: {len(failures)} reusable-workflow permission mismatch(es) across " + f"{sites} local call site(s) in {args.workflows_dir}", + file=sys.stderr, + ) + return 1 + print( + f"PASS: every local reusable-workflow call stays within its caller's permissions " + f"({sites} call site(s) in {callers} caller workflow(s))" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/notarize-computer-use-helper.sh b/scripts/ci/notarize-computer-use-helper.sh index 6a977d016391..201e256a0cb6 100755 --- a/scripts/ci/notarize-computer-use-helper.sh +++ b/scripts/ci/notarize-computer-use-helper.sh @@ -47,13 +47,17 @@ DITTO_TOOL="${CMUX_DITTO_TOOL:-/usr/bin/ditto}" XCRUN_TOOL="${CMUX_XCRUN_TOOL:-xcrun}" CODESIGN_TOOL="${CMUX_CODESIGN_TOOL:-/usr/bin/codesign}" SPCTL_TOOL="${CMUX_SPCTL_TOOL:-spctl}" -# Gatekeeper keeps negative assessments in a cache keyed by the helper's code -# directory hash. The helper has the same CDHash before and after stapling, so -# an assessment made before the ticket is attached can otherwise be replayed -# after stapling and reported as "Unnotarized Developer ID". Force every poll -# to assess the copied helper without consulting or populating that cache; -# retries still cover propagation of the fresh ticket to Apple's service. -GATEKEEPER_ASSESS_ATTEMPTS="${CMUX_GATEKEEPER_ASSESS_ATTEMPTS:-20}" +# Gatekeeper learns about a fresh notarization ticket from Apple's CDN, which +# lags the notarytool "Accepted" status: usually by a minute or two, but +# nightly run 34208928547 (2026-09-08) was still rejected 4m50s after +# "Accepted" and failed on the previous five-minute budget. A stapled, valid +# helper can therefore assess as "Unnotarized Developer ID" for a while. Poll +# until it is accepted or the budget runs out. The default budget is twenty +# minutes (80 x 15s): a good ticket leaves the loop on its first acceptance, +# so a larger budget only lengthens how long a genuinely rejected helper takes +# to fail, whereas a short budget fails good releases whenever the CDN lags. +# Both knobs stay env-configurable; the calling job's timeout must cover them. +GATEKEEPER_ASSESS_ATTEMPTS="${CMUX_GATEKEEPER_ASSESS_ATTEMPTS:-80}" GATEKEEPER_ASSESS_DELAY_SECONDS="${CMUX_GATEKEEPER_ASSESS_DELAY_SECONDS:-15}" assess_with_gatekeeper() { @@ -62,6 +66,9 @@ assess_with_gatekeeper() { if "$SPCTL_TOOL" -a -vv --ignore-cache --no-cache --type execute "$target"; then return 0 fi + if [ "$attempt" -eq 1 ]; then + echo "Gatekeeper propagation budget: $GATEKEEPER_ASSESS_ATTEMPTS attempts x ${GATEKEEPER_ASSESS_DELAY_SECONDS}s (about $((GATEKEEPER_ASSESS_ATTEMPTS * GATEKEEPER_ASSESS_DELAY_SECONDS / 60)) minutes)" + fi if [ "$attempt" -ge "$GATEKEEPER_ASSESS_ATTEMPTS" ]; then echo "Gatekeeper still rejects $target after $attempt attempts" >&2 return 3 diff --git a/scripts/sparkle_generate_appcast.sh b/scripts/sparkle_generate_appcast.sh index 99aae6e6ae6b..6b697e3088a6 100755 --- a/scripts/sparkle_generate_appcast.sh +++ b/scripts/sparkle_generate_appcast.sh @@ -89,11 +89,17 @@ printf "%s" "$padded_key" > "$key_file" generated_appcast_path="$archives_dir/$(basename "$OUT_PATH")" +# ${arr[@]+"${arr[@]}"} expands to nothing when the array is empty. A bare +# "${delta_args[@]}" is an "unbound variable" error under `set -u` in bash 3.2 +# (macOS /bin/bash), and with the EXIT trap above bash 3.2 then exits 0, so the +# stable release lane (no previous archives) silently produced no appcast +# (release dry run 34227505375, 2026-09-08). Nightly never hit it because it +# always has previous archives. "$generate_appcast" \ --ed-key-file "$key_file" \ --download-url-prefix "$DOWNLOAD_URL_PREFIX" \ --full-release-notes-url "$RELEASE_NOTES_URL" \ - "${delta_args[@]}" \ + ${delta_args[@]+"${delta_args[@]}"} \ "$archives_dir" if [[ ! -f "$generated_appcast_path" ]]; then diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 6fe1f5d8811c..3f58c9381bda 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -848,6 +848,112 @@ def test_app_host_multi_batch_failure_cannot_reuse_prior_expected_summary() -> N assert "simulated app-host crash before test summary" in result.stdout +def run_remote_tmux_mirror_step( + outcomes: list[str], +) -> tuple[subprocess.CompletedProcess[str], int]: + """Run the remote tmux mirror focused gate against a fake console runner. + + ``outcomes`` lists what each xcodebuild invocation reports, in order: + ``pass``; ``crash`` (xcodebuild restarted the app host, exit 65); or + ``fail`` (an assertion failure with the host alive, exit 65). Returns the + step result and how many times the runner was invoked. + """ + script = workflow_job_step_script( + "app-host-unit-tests", "Run remote tmux mirror detach and placement regressions" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + runner_temp = root / "runner" + ci_scripts = root / "scripts" / "ci" + runner_temp.mkdir() + ci_scripts.mkdir(parents=True) + outcomes_file = root / "outcomes" + outcomes_file.write_text("\n".join(outcomes) + "\n", encoding="utf-8") + counter = root / "invocations" + + console_runner = ci_scripts / "run-in-console-session.sh" + console_runner.write_text( + """ +#!/bin/bash +set -euo pipefail +counter="${CMUX_TEST_INVOCATION_COUNTER:?}" +iteration=0 +if [ -f "$counter" ]; then + iteration="$(cat "$counter")" +fi +iteration=$((iteration + 1)) +printf '%s\\n' "$iteration" > "$counter" +outcome="$(sed -n "${iteration}p" "${CMUX_TEST_OUTCOMES:?}")" +printf 'invocation %s: %s\\n' "$iteration" "$*" +case "$outcome" in + pass) + echo "Executed 7 tests, with 0 failures (0 unexpected)" + exit 0 + ;; + crash) + echo "Restarting after unexpected exit, crash, or test timeout; summary will include totals from previous launches." + echo "Executed 7 tests, with 1 failure (1 unexpected)" + exit 65 + ;; + fail) + echo "Executed 7 tests, with 1 failure (0 unexpected)" + exit 65 + ;; + *) + echo "unexpected extra invocation ${iteration}" >&2 + exit 97 + ;; +esac +""".lstrip(), + encoding="utf-8", + ) + console_runner.chmod(0o755) + + result = subprocess.run( + ["bash", "-c", script], + cwd=root, + env={ + **os.environ, + "RUNNER_TEMP": str(runner_temp), + "CMUX_DERIVED_DATA_PATH": str(root / "derived-data"), + "CMUX_TEST_INVOCATION_COUNTER": str(counter), + "CMUX_TEST_OUTCOMES": str(outcomes_file), + }, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + invocations = int(counter.read_text(encoding="utf-8").strip()) if counter.exists() else 0 + return result, invocations + + +def test_remote_tmux_mirror_gate_reruns_a_suite_once_after_an_app_host_crash() -> None: + # The close suite crashes once and passes on its rerun; the placement + # suite then runs and passes, so the step is green with three invocations. + result, invocations = run_remote_tmux_mirror_step(["crash", "pass", "pass"]) + + assert result.returncode == 0, result.stdout + result.stderr + assert invocations == 3, result.stdout + assert "rerunning the suite once" in result.stdout + assert "cmuxTests/RemoteTmuxMirrorDedicatedPlacementTests" in result.stdout + + +def test_remote_tmux_mirror_gate_never_reruns_an_assertion_failure() -> None: + result, invocations = run_remote_tmux_mirror_step(["fail", "pass", "pass"]) + + assert result.returncode == 65, result.stdout + result.stderr + assert invocations == 1, result.stdout + assert "rerunning the suite once" not in result.stdout + + +def test_remote_tmux_mirror_gate_fails_after_a_second_crash() -> None: + result, invocations = run_remote_tmux_mirror_step(["crash", "crash", "pass"]) + + assert result.returncode == 65, result.stdout + result.stderr + assert invocations == 2, result.stdout + + def test_app_host_rejects_failed_or_empty_shard_generation() -> None: for shard_mode in ("fail", "empty"): result, runner_invoked = run_app_host_unit_test_step(shard_mode) diff --git a/tests/test_ci_release_ios_screenshots_decoupled.sh b/tests/test_ci_release_ios_screenshots_decoupled.sh new file mode 100755 index 000000000000..f3df4403272d --- /dev/null +++ b/tests/test_ci_release_ios_screenshots_decoupled.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Regression guard for https://github.com/manaflow-ai/cmux/issues/12149. +# The stable macOS release must not wait on, or fail because of, the iOS App +# Store screenshot capture: build-sign-notarize consumes nothing from it, and +# the capture is a long simulator run on shared macOS runners. Keep the capture +# as a sibling job in release.yml (every tag still gets screenshots at the +# release ref) but never as a dependency of the DMG pipeline. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +WORKFLOW="${CMUX_RELEASE_WORKFLOW_FILE:-$ROOT_DIR/.github/workflows/release.yml}" + +job_block() { + awk -v job="$1" ' + $0 == " " job ":" { in_job = 1; next } + in_job && /^ [A-Za-z0-9_-]+:/ { exit } + in_job { print } + ' "$WORKFLOW" +} + +screenshots="$(job_block generate-ios-screenshots)" +if [ -z "$screenshots" ] || ! grep -Eq '^ uses: \./\.github/workflows/ios-screenshots\.yml$' <<<"$screenshots"; then + echo "FAIL: release.yml must keep the generate-ios-screenshots job calling ./.github/workflows/ios-screenshots.yml" >&2 + exit 1 +fi + +notarize="$(job_block build-sign-notarize)" +if [ -z "$notarize" ]; then + echo "FAIL: build-sign-notarize job not found in $WORKFLOW" >&2 + exit 1 +fi + +# needs: may be a scalar (`needs: job`) or a block sequence (`needs:` + `- job` lines). +needs="$( + awk ' + /^ needs:/ { in_needs = 1; sub(/^ needs:[[:space:]]*/, ""); if ($0 != "") print; next } + in_needs && /^ - / { sub(/^ - /, ""); print; next } + in_needs { in_needs = 0 } + ' <<<"$notarize" | sed -E 's/[[:space:]]*#.*$//; s/^\[//; s/\]$//; s/,/\n/g' | sed -E 's/^[[:space:]]+|[[:space:]]+$//g' | sed '/^$/d' +)" + +if ! grep -qx 'build-ghostty-cli-helper' <<<"$needs"; then + echo "FAIL: build-sign-notarize must keep needs: build-ghostty-cli-helper (got: $(tr '\n' ' ' <<<"$needs"))" >&2 + exit 1 +fi + +if grep -qx 'generate-ios-screenshots' <<<"$needs"; then + cat >&2 <<'MSG' +FAIL: build-sign-notarize must not depend on generate-ios-screenshots. + + The macOS DMG never consumes the screenshot artifacts, and the capture is + a long simulator run that must not delay or fail a stable release. If the + release really needs the screenshots, download them in a step instead of + gating the whole job on the capture, and update this guard with the reason. +MSG + exit 1 +fi + +echo "PASS: build-sign-notarize does not wait on iOS screenshot capture (needs: $(tr '\n' ' ' <<<"$needs"| sed 's/ $//'))" diff --git a/tests/test_ci_release_tunnel_identifiers.sh b/tests/test_ci_release_tunnel_identifiers.sh new file mode 100755 index 000000000000..bf8c3b53aed0 --- /dev/null +++ b/tests/test_ci_release_tunnel_identifiers.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Regression guard for the stable release's Cloud tunnel extension naming. +# +# release.yml hardcodes three identifiers for the bundled system extension: +# the bundle identifier passed to scripts/normalize-system-extension-bundle.sh, +# the .systemextension directory it then verifies, and the App ID +# (team prefix + bundle id) that the provisioning profile must name. #11789 +# passed the team-prefixed App ID as the bundle identifier, so the first +# release dry run after #12149 failed in "Verify binary architectures": +# system extension identifier is 'com.cmuxterm.app.tunnel', expected +# '7WLXT3NR37.com.cmuxterm.app.tunnel' +# Derive all three from the Xcode project so the workflow cannot drift again. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +WORKFLOW="${CMUX_RELEASE_WORKFLOW_FILE:-$ROOT_DIR/.github/workflows/release.yml}" +PROJECT="${CMUX_PROJECT_FILE:-$ROOT_DIR/cmux.xcodeproj/project.pbxproj}" + +# The tunnel target's Release identifier is the one without a .debug segment. +bundle_id="$(grep -E 'PRODUCT_BUNDLE_IDENTIFIER = [A-Za-z0-9.-]+\.tunnel;' "$PROJECT" | sed -E 's/.*= ([A-Za-z0-9.-]+);.*/\1/' | grep -v '\.debug\.' | sort -u)" +if [ "$(wc -l <<<"$bundle_id" | tr -d ' ')" != "1" ] || [ -z "$bundle_id" ]; then + echo "FAIL: expected exactly one non-debug tunnel PRODUCT_BUNDLE_IDENTIFIER in $PROJECT, got: $(tr '\n' ' ' <<<"$bundle_id")" >&2 + exit 1 +fi +team_prefix="$(grep -E 'CMUX_TEAM_ID_PREFIX = "[A-Z0-9]+\.";' "$PROJECT" | sed -E 's/.*= "([A-Z0-9]+\.)";.*/\1/' | sort -u)" +if [ "$(wc -l <<<"$team_prefix" | tr -d ' ')" != "1" ] || [ -z "$team_prefix" ]; then + echo "FAIL: expected exactly one CMUX_TEAM_ID_PREFIX in $PROJECT, got: $(tr '\n' ' ' <<<"$team_prefix")" >&2 + exit 1 +fi +app_id="${team_prefix}${bundle_id}" + +verify_job="$( + awk ' + /^ build-sign-notarize:/ { in_job = 1; next } + in_job && /^ [A-Za-z0-9_-]+:/ { exit } + in_job { print } + ' "$WORKFLOW" +)" +[ -n "$verify_job" ] || { echo "FAIL: build-sign-notarize job not found in $WORKFLOW" >&2; exit 1; } + +normalize_calls="$(grep -E 'normalize-system-extension-bundle\.sh' <<<"$verify_job" || true)" +if [ -z "$normalize_calls" ]; then + echo "FAIL: build-sign-notarize must normalize the system extension bundle name" >&2 + exit 1 +fi +if ! grep -Eq "normalize-system-extension-bundle\.sh \"\\\$APP\" \"$bundle_id\"$" <<<"$normalize_calls"; then + echo "FAIL: normalize-system-extension-bundle.sh must receive the Release bundle identifier '$bundle_id' (not the App ID), got:" >&2 + echo "$normalize_calls" >&2 + exit 1 +fi + +if ! grep -Eq "SystemExtensions/$bundle_id\.systemextension/Contents/MacOS/" <<<"$verify_job"; then + echo "FAIL: build-sign-notarize must verify the tunnel binary under SystemExtensions/$bundle_id.systemextension" >&2 + exit 1 +fi +if grep -Eq "SystemExtensions/$app_id\.systemextension" <<<"$verify_job"; then + echo "FAIL: the system extension directory is named after the bundle identifier, never the team-prefixed App ID" >&2 + exit 1 +fi + +embed_args="$(grep -A3 'embed-tunnel-extension-profile.sh' <<<"$verify_job" | grep -E '^[[:space:]]+"[A-Za-z0-9.]+\.tunnel" \\$' | sed -E 's/^[[:space:]]+"([^"]+)".*/\1/' || true)" +if [ "$embed_args" != "$app_id" ]; then + echo "FAIL: embed-tunnel-extension-profile.sh must check the profile against the App ID '$app_id', got '${embed_args:-}'" >&2 + exit 1 +fi + +echo "PASS: release.yml names the tunnel extension $bundle_id (bundle) and $app_id (profile App ID)" diff --git a/tests/test_ci_reusable_workflow_permissions.py b/tests/test_ci_reusable_workflow_permissions.py new file mode 100644 index 000000000000..f1a848fdaeeb --- /dev/null +++ b/tests/test_ci_reusable_workflow_permissions.py @@ -0,0 +1,494 @@ +#!/usr/bin/env python3 +"""Behavioral tests for the reusable-workflow permission guard. + +Regression test for https://github.com/manaflow-ai/cmux/issues/12149: release.yml called +ios-screenshots.yml, which declared `actions: write` while the caller granted no `actions` +scope at all. GitHub refused the whole release workflow at parse time (startup_failure), so a +v* tag push could not build. The guard under test reproduces GitHub's rule for every local +`uses: ./.github/workflows/*.yml` call so that shape can never land again. +""" + +from __future__ import annotations + +import importlib.util +import re +import subprocess +import sys +import tempfile +import textwrap +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CHECKER = ROOT / "scripts" / "ci" / "check_reusable_workflow_permissions.py" +WORKFLOWS_DIR = ROOT / ".github" / "workflows" +CI_WORKFLOW = WORKFLOWS_DIR / "ci.yml" + +spec = importlib.util.spec_from_file_location("check_reusable_workflow_permissions", CHECKER) +assert spec and spec.loader +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +# The manaflow-ai/cmux Actions setting "Workflow permissions" is "Read and write" (verified via +# `gh api repos/manaflow-ai/cmux/actions/permissions/workflow` on 2026-09-08). It only matters for +# callers that declare no `permissions` block anywhere; every current caller declares one. +REPOSITORY_DEFAULT_WORKFLOW_PERMISSIONS = "write" + +_keepalive: list[tempfile.TemporaryDirectory[str]] = [] + + +def workflows_tree(files: dict[str, str]) -> Path: + tmp = tempfile.TemporaryDirectory(prefix="cmux-reusable-perms-") + _keepalive.append(tmp) + workflows = Path(tmp.name) / ".github" / "workflows" + workflows.mkdir(parents=True) + for name, text in files.items(): + (workflows / name).write_text(textwrap.dedent(text), encoding="utf-8") + return workflows + + +def run_cli(workflows: Path, *, default: str = "write", verbose: bool = False) -> subprocess.CompletedProcess[str]: + command = [ + sys.executable, + str(CHECKER), + "--workflows-dir", + str(workflows), + "--default-workflow-permissions", + default, + ] + if verbose: + command.append("--verbose") + return subprocess.run(command, capture_output=True, text=True, check=False) + + +def check(files: dict[str, str], *, default: str = "write") -> tuple[list, list[str]]: + return module.check_workflows_dir(workflows_tree(files), default) + + +def failures_for(files: dict[str, str], *, default: str = "write") -> list[str]: + return check(files, default=default)[1] + + +RELEASE_SHAPE_BEFORE_FIX = { + # release.yml as of main on 2026-09-08 (permissions and the reusable call, verbatim shape). + "release.yml": """\ + name: Release macOS app + + on: + push: + tags: + - "v*" + workflow_dispatch: + + permissions: + contents: write + attestations: write + id-token: write + + jobs: + generate-ios-screenshots: + name: Generate iOS App Store screenshots + uses: ./.github/workflows/ios-screenshots.yml + with: + ref: ${{ github.ref }} + languages: "en-US,de-DE,fr-FR,ar-SA,es-ES,zh-Hant,zh-Hans,ko,ja" + upload: false + secrets: inherit + + build-sign-notarize: + needs: + - generate-ios-screenshots + runs-on: macos-26 + steps: + - run: echo build + """, + # ios-screenshots.yml as of main on 2026-09-08: the callee asked for a scope the caller lacks. + "ios-screenshots.yml": """\ + name: iOS App Store screenshots + + on: + workflow_call: + inputs: + ref: + type: string + default: "" + workflow_dispatch: + permissions: + contents: read + actions: write + + jobs: + screenshots: + runs-on: macos-26 + steps: + - uses: actions/checkout@v6 + - run: fastlane screenshots + """, +} + + +RELEASE_SHAPE_BEFORE_FIX = {name: textwrap.dedent(text) for name, text in RELEASE_SHAPE_BEFORE_FIX.items()} + + +def test_issue_12149_release_shape_is_rejected_before_any_job_runs() -> None: + result = run_cli(workflows_tree(RELEASE_SHAPE_BEFORE_FIX)) + + assert result.returncode == 1, result + assert "PASS" not in result.stdout, result.stdout + failure = result.stderr + assert "release.yml: job 'generate-ios-screenshots' uses ./.github/workflows/ios-screenshots.yml" in failure, failure + assert "requests 'actions: write' (ios-screenshots.yml workflow-level permissions)" in failure, failure + assert "only allows 'actions: none' (release.yml workflow-level permissions)" in failure, failure + assert "1 reusable-workflow permission mismatch(es) across 1 local call site(s)" in failure, failure + + +def test_callee_that_keeps_or_reduces_the_grant_passes() -> None: + fixed = dict(RELEASE_SHAPE_BEFORE_FIX) + fixed["ios-screenshots.yml"] = fixed["ios-screenshots.yml"].replace(" actions: write\n", "") + + result = run_cli(workflows_tree(fixed), verbose=True) + + assert result.returncode == 0, result + assert result.stderr == "", result.stderr + assert "PASS: every local reusable-workflow call stays within its caller's permissions (1 call site(s) in 1 caller workflow(s))" in result.stdout, result.stdout + assert "request: contents: read" in result.stdout, result.stdout + + +def test_callee_job_level_permissions_count_even_when_the_job_is_gated() -> None: + # Mirrors cmux-tui-build-package.yml: its attest job is optional at runtime, but GitHub still + # validates the declared job-level ceiling against the caller (see 4b9720dc750). + callee = """\ + on: + workflow_call: + permissions: {} + jobs: + build: + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - run: echo build + attest: + if: ${{ false }} + permissions: + contents: read + id-token: write + attestations: write + runs-on: ubuntu-latest + steps: + - run: echo attest + """ + caller_without_ceiling = textwrap.dedent("""\ + on: push + permissions: {} + jobs: + package: + permissions: + contents: read + uses: ./.github/workflows/package.yml + """) + failures = failures_for({"caller.yml": caller_without_ceiling, "package.yml": callee}) + assert len(failures) == 2, failures + assert any("requests 'id-token: write' (package.yml job 'attest' permissions)" in f for f in failures), failures + assert any("requests 'attestations: write' (package.yml job 'attest' permissions)" in f for f in failures), failures + assert all("only allows" in f and "(caller.yml job 'package' permissions)" in f for f in failures), failures + + caller_with_ceiling = caller_without_ceiling.replace( + " contents: read\n", " contents: read\n id-token: write\n attestations: write\n" + ) + assert failures_for({"caller.yml": caller_with_ceiling, "package.yml": callee}) == [] + + +def test_caller_job_block_replaces_the_workflow_block_entirely() -> None: + caller = """\ + on: push + permissions: + contents: write + packages: write + jobs: + call: + permissions: + contents: read + uses: ./.github/workflows/callee.yml + """ + callee = """\ + on: + workflow_call: + permissions: + contents: read + packages: read + jobs: + run: + runs-on: ubuntu-latest + steps: + - run: echo hi + """ + failures = failures_for({"caller.yml": caller, "callee.yml": callee}) + + assert len(failures) == 1, failures + assert "requests 'packages: read' (callee.yml workflow-level permissions)" in failures[0], failures + assert "only allows 'packages: none' (caller.yml job 'call' permissions)" in failures[0], failures + + +def test_callee_without_any_permissions_block_inherits_the_caller() -> None: + caller = """\ + on: push + permissions: + contents: read + jobs: + call: + uses: ./.github/workflows/callee.yml + """ + callee = """\ + on: + workflow_call: + jobs: + one: + runs-on: ubuntu-latest + steps: + - run: echo one + two: + runs-on: ubuntu-latest + steps: + - run: echo two + """ + edges, failures = check({"caller.yml": caller, "callee.yml": callee}) + + assert failures == [] + assert len(edges) == 1 and edges[0].requests == (), edges + + +def test_shorthand_permission_blocks() -> None: + def caller(block: str) -> str: + return f"""\ + on: push + permissions: {block} + jobs: + call: + uses: ./.github/workflows/callee.yml + """ + + def callee(block: str) -> str: + return f"""\ + on: + workflow_call: + permissions: {block} + jobs: + run: + runs-on: ubuntu-latest + steps: + - run: echo hi + """ + + read_all_vs_contents_read = failures_for({"caller.yml": caller("{ contents: read }"), "callee.yml": callee("read-all")}) + assert any("requests 'actions: read'" in f and "only allows 'actions: none'" in f for f in read_all_vs_contents_read), read_all_vs_contents_read + assert not any("'contents: read'" in f and "only allows" in f for f in read_all_vs_contents_read), read_all_vs_contents_read + + assert failures_for({"caller.yml": caller("write-all"), "callee.yml": callee("read-all")}) == [] + assert failures_for({"caller.yml": caller("write-all"), "callee.yml": callee("write-all")}) == [] + assert failures_for({"caller.yml": caller("{}"), "callee.yml": callee("{}")}) == [] + + nothing_vs_contents_read = failures_for({"caller.yml": caller("{}"), "callee.yml": callee("{ contents: read }")}) + assert len(nothing_vs_contents_read) == 1, nothing_vs_contents_read + assert "requests 'contents: read'" in nothing_vs_contents_read[0] and "only allows 'contents: none'" in nothing_vs_contents_read[0], nothing_vs_contents_read + + +def test_caller_without_permissions_uses_the_repository_default() -> None: + caller = """\ + on: push + jobs: + call: + uses: ./.github/workflows/callee.yml + """ + + def callee(scope: str, level: str) -> str: + return f"""\ + on: + workflow_call: + permissions: + {scope}: {level} + jobs: + run: + runs-on: ubuntu-latest + steps: + - run: echo hi + """ + + contents_write = {"caller.yml": caller, "callee.yml": callee("contents", "write")} + assert failures_for(contents_write, default="write") == [] + restricted = failures_for(contents_write, default="read") + assert len(restricted) == 1 and "repository default workflow permissions (read)" in restricted[0], restricted + + # The default token never carries id-token, whichever repository setting applies. + for default in ("write", "read"): + id_token = failures_for({"caller.yml": caller, "callee.yml": callee("id-token", "write")}, default=default) + assert len(id_token) == 1 and "only allows 'id-token: none'" in id_token[0], (default, id_token) + # metadata: read is always available. + assert failures_for({"caller.yml": caller, "callee.yml": callee("metadata", "read")}, default=default) == [] + + +def test_nested_calls_are_checked_with_the_intermediate_job_grant() -> None: + top = textwrap.dedent("""\ + on: push + permissions: + contents: read + id-token: write + jobs: + call: + uses: ./.github/workflows/middle.yml + """) + middle_without_ceiling = textwrap.dedent("""\ + on: + workflow_call: + permissions: + contents: read + jobs: + inner: + uses: ./.github/workflows/leaf.yml + """) + leaf = """\ + on: + workflow_call: + permissions: + contents: read + id-token: write + jobs: + run: + runs-on: ubuntu-latest + steps: + - run: echo leaf + """ + edges, failures = check({"top.yml": top, "middle.yml": middle_without_ceiling, "leaf.yml": leaf}) + + assert [(edge.caller.name, edge.job, edge.callee.name) for edge in edges] == [ + ("top.yml", "call", "middle.yml"), + ("middle.yml", "inner", "leaf.yml"), + ], edges + assert len(failures) == 1, failures + assert "middle.yml (reached via top.yml): job 'inner' uses ./.github/workflows/leaf.yml" in failures[0], failures + assert "requests 'id-token: write' (leaf.yml workflow-level permissions)" in failures[0], failures + assert "only allows 'id-token: none' (middle.yml workflow-level permissions)" in failures[0], failures + + middle_with_ceiling = middle_without_ceiling.replace( + " inner:\n", " inner:\n permissions:\n contents: read\n id-token: write\n" + ) + assert failures_for({"top.yml": top, "middle.yml": middle_with_ceiling, "leaf.yml": leaf}) == [] + + # The intermediate job's own ceiling is itself a request against the top-level caller. + top_without_id_token = top.replace(" id-token: write\n", "") + outer = failures_for({"top.yml": top_without_id_token, "middle.yml": middle_with_ceiling, "leaf.yml": leaf}) + assert any("top.yml: job 'call' uses ./.github/workflows/middle.yml" in f and "(middle.yml job 'inner' permissions)" in f for f in outer), outer + + +def test_external_reusable_workflows_are_skipped() -> None: + caller = """\ + on: push + permissions: {} + jobs: + external: + uses: octo-org/octo-repo/.github/workflows/build.yml@v1 + action_step: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + """ + edges, failures = check({"caller.yml": caller}) + + assert edges == [] and failures == [] + + +def test_missing_local_callee_fails() -> None: + caller = """\ + on: push + permissions: + contents: read + jobs: + call: + uses: ./.github/workflows/does-not-exist.yml + """ + failures = failures_for({"caller.yml": caller}) + + assert len(failures) == 1 and "does-not-exist.yml, but that workflow file does not exist" in failures[0], failures + + +def test_reader_ignores_lookalike_text_and_accepts_github_yaml_shapes() -> None: + callee = ( + "---\r\n" + "name: shapes\r\n" + "on:\r\n" + " push:\r\n" + " branches:\r\n" + " - main\r\n" + " workflow_call:\r\n" + '"permissions": { contents: read } # flow mapping, quoted key, inline comment\r\n' + "jobs:\r\n" + " run:\r\n" + " runs-on: ubuntu-latest\r\n" + " steps:\r\n" + " - name: Print a permissions block that must not be read as one\r\n" + " run: |\r\n" + " cat <<'EOF'\r\n" + " permissions:\r\n" + " actions: write\r\n" + " EOF\r\n" + " # a comment between steps\r\n" + " - run: echo 'permissions: write-all'\r\n" + " other:\r\n" + " permissions:\r\n" + " contents: read # trailing comment\r\n" + " packages: 'read'\r\n" + " runs-on: ubuntu-latest\r\n" + " steps:\r\n" + " - run: echo other\r\n" + ) + caller = """\ + on: push + permissions: + contents: write + packages: read + jobs: + call: + uses: './.github/workflows/callee.yml' + with: + example: "value: with colon" + """ + workflows = workflows_tree({"caller.yml": caller, "callee.yml": callee}) + parsed = module.parse_workflow(workflows / "callee.yml") + + assert parsed["permissions"] == {"contents": "read"}, parsed + assert set(parsed["jobs"]) == {"run", "other"}, parsed + assert parsed["jobs"]["other"]["permissions"] == {"contents": "read", "packages": "read"}, parsed + assert "permissions" not in parsed["jobs"]["run"], parsed + requests = module.callee_requests(parsed, "callee.yml") + assert {(r.scope, r.level) for r in requests} == {("contents", 1), ("packages", 1)}, requests + + edges, failures = module.check_workflows_dir(workflows, "write") + assert failures == [] and len(edges) == 1, (edges, failures) + + +def test_repository_workflows_stay_within_their_callers_grants() -> None: + result = run_cli(WORKFLOWS_DIR, default=REPOSITORY_DEFAULT_WORKFLOW_PERMISSIONS) + + assert result.returncode == 0, f"{result.stdout}\n{result.stderr}" + assert result.stdout.startswith("PASS:"), result.stdout + + # Behavioral completeness: every local `uses:` job in the tree was actually checked. + pattern = re.compile(r"^\s+uses:\s*['\"]?\./\.github/workflows/", re.MULTILINE) + expected_calls = sum(len(pattern.findall(path.read_text(encoding="utf-8"))) for path in WORKFLOWS_DIR.glob("*.yml")) + assert expected_calls > 0 + assert f"({expected_calls} call site(s) in" in result.stdout, (expected_calls, result.stdout) + + +def test_ci_runs_this_guard_in_workflow_guard_tests() -> None: + text = CI_WORKFLOW.read_text(encoding="utf-8") + match = re.search(r"(?ms)^ workflow-guard-tests:\n(.*?)(?=^ [A-Za-z0-9_-]+:\n|\Z)", text) + assert match is not None, "workflow-guard-tests job missing from ci.yml" + + assert "run: python3 tests/test_ci_reusable_workflow_permissions.py" in match.group(1), match.group(1) + + +if __name__ == "__main__": + for name, value in sorted(globals().items()): + if name.startswith("test_") and callable(value): + value() + print("PASS: reusable workflow permission guard") diff --git a/tests/test_ci_sparkle_build_monotonic.sh b/tests/test_ci_sparkle_build_monotonic.sh index e4063619a58f..6f689636a26f 100755 --- a/tests/test_ci_sparkle_build_monotonic.sh +++ b/tests/test_ci_sparkle_build_monotonic.sh @@ -8,12 +8,30 @@ # compares CFBundleVersion (CURRENT_PROJECT_VERSION) against # — the marketing string is informational only. # -# If the published appcast cannot be fetched (e.g. offline CI runner), the -# test soft-passes with a warning so it never blocks unrelated work. +# Modes (CMUX_SPARKLE_MONOTONIC_MODE): +# enforce (default) - a stale build number fails, and so does an appcast that +# cannot be fetched: a tag push is about to publish, and a +# missing signal must fail closed rather than let a stale +# build number reach users. Tag pushes and the local +# pre-tag guard use this. +# warn - a stale or unknown published build is reported but does +# not fail. release.yml selects this for a non-tag +# workflow_dispatch dry run, which publishes nothing and is +# expected to run from a branch that has not been bumped. set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" -PROJECT_FILE="$ROOT_DIR/cmux.xcodeproj/project.pbxproj" +PROJECT_FILE="${CMUX_SPARKLE_PROJECT_FILE:-$ROOT_DIR/cmux.xcodeproj/project.pbxproj}" +APPCAST_URL="${CMUX_SPARKLE_APPCAST_URL:-https://github.com/manaflow-ai/cmux/releases/latest/download/appcast.xml}" +MODE="${CMUX_SPARKLE_MONOTONIC_MODE:-enforce}" + +case "$MODE" in + enforce|warn) ;; + *) + echo "FAIL: CMUX_SPARKLE_MONOTONIC_MODE must be 'enforce' or 'warn' (got '$MODE')" >&2 + exit 1 + ;; +esac if [[ ! -f "$PROJECT_FILE" ]]; then echo "FAIL: $PROJECT_FILE not found" >&2 @@ -35,18 +53,42 @@ if [[ "$MISMATCHED" != "1" ]]; then exit 1 fi +# Retry transient fetch failures so enforce mode does not fail a real release +# on a blip; the retry knobs exist so tests can exercise the unreachable path fast. PUBLISHED_BUILD=$(curl -fsSL --max-time 15 \ - https://github.com/manaflow-ai/cmux/releases/latest/download/appcast.xml 2>/dev/null \ + --retry "${CMUX_SPARKLE_APPCAST_RETRIES:-3}" --retry-delay "${CMUX_SPARKLE_APPCAST_RETRY_DELAY:-2}" --retry-all-errors \ + "$APPCAST_URL" 2>/dev/null \ | sed -n 's#.*\([0-9][0-9]*\).*#\1#p' \ | head -n1 || true) if ! [[ "$PUBLISHED_BUILD" =~ ^[0-9]+$ ]]; then - echo "WARN: could not fetch latest published Sparkle build; skipping monotonic check" - echo "PASS (soft): local CURRENT_PROJECT_VERSION=$LOCAL_BUILD" - exit 0 + if [[ "$MODE" == "warn" ]]; then + echo "WARN: could not fetch latest published Sparkle build; skipping monotonic check" + echo "PASS (soft): local CURRENT_PROJECT_VERSION=$LOCAL_BUILD" + exit 0 + fi + cat >&2 <&2 < "$LOG" rm -f "$TMP_DIR/spctl-count" -if ! CMUX_TEST_SPCTL_COUNT_FILE="$TMP_DIR/spctl-count" CMUX_TEST_SPCTL_REJECTS=2 run_helper >/dev/null 2>&1; then +if ! CMUX_TEST_SPCTL_COUNT_FILE="$TMP_DIR/spctl-count" CMUX_TEST_SPCTL_REJECTS=2 run_helper >"$TMP_DIR/poll.out" 2>&1; then echo "FAIL: helper notarization gave up while the Gatekeeper ticket was still propagating" >&2 exit 1 fi @@ -222,6 +222,33 @@ if [ "$(grep -c '^spctl -a -vv --ignore-cache --no-cache --type execute .*/stand exit 1 fi +# The first rejection announces the whole budget so a log reader can tell a +# propagation wait from a hang. +if ! grep -Eq '^Gatekeeper propagation budget: [0-9]+ attempts x [0-9]+s \(about [0-9]+ minutes\)$' "$TMP_DIR/poll.out"; then + echo "FAIL: Gatekeeper polling must announce its attempt budget on the first rejection" >&2 + exit 1 +fi + +# The default budget must cover Apple's CDN propagation tail for a stable +# release: nightly run 34208928547 (2026-09-08) was still rejected 4m50s +# after notarytool reported Accepted and failed on a 20 x 15s budget. Keep the +# default at twenty minutes or more, polled often enough that a landed ticket +# is noticed within half a minute, and keep both knobs env-configurable. +default_attempts="$(sed -n 's/^GATEKEEPER_ASSESS_ATTEMPTS="\${CMUX_GATEKEEPER_ASSESS_ATTEMPTS:-\([0-9][0-9]*\)}"$/\1/p' "$SCRIPT")" +default_delay="$(sed -n 's/^GATEKEEPER_ASSESS_DELAY_SECONDS="\${CMUX_GATEKEEPER_ASSESS_DELAY_SECONDS:-\([0-9][0-9]*\)}"$/\1/p' "$SCRIPT")" +if ! [[ "$default_attempts" =~ ^[0-9]+$ && "$default_delay" =~ ^[0-9]+$ ]]; then + echo "FAIL: Gatekeeper attempt and delay defaults must be env-configurable numeric literals (got '$default_attempts' x '$default_delay')" >&2 + exit 1 +fi +if (( default_attempts * default_delay < 1200 )); then + echo "FAIL: default Gatekeeper propagation budget is $((default_attempts * default_delay))s; a stable release needs at least 1200s" >&2 + exit 1 +fi +if (( default_delay > 30 )); then + echo "FAIL: Gatekeeper poll interval ${default_delay}s is too coarse; poll at least every 30s" >&2 + exit 1 +fi + # A ticket that never propagates within the budget still fails the release. : > "$LOG" rm -f "$TMP_DIR/spctl-count" diff --git a/tests/test_sparkle_build_monotonic_modes.sh b/tests/test_sparkle_build_monotonic_modes.sh new file mode 100755 index 000000000000..995728fcf782 --- /dev/null +++ b/tests/test_sparkle_build_monotonic_modes.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Behavioral test for tests/test_ci_sparkle_build_monotonic.sh. +# +# release.yml runs that guard at the top of build-sign-notarize. A tag push must +# fail on a stale build number (Sparkle would never offer the update), but a +# non-tag workflow_dispatch is the pipeline's dry run: it publishes nothing and +# runs from a branch that has not been bumped, so the same condition must only +# warn there (https://github.com/manaflow-ai/cmux/issues/12149). Exercise both +# modes against fixture project files and a local appcast. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +GUARD="$ROOT_DIR/tests/test_ci_sparkle_build_monotonic.sh" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +project_with_build() { + local build="$1" + local file="$TMP_DIR/project-$build.pbxproj" + cat > "$file" < "$APPCAST" <<'XML' + + + + + 0.64.22 + 250 + 0.64.22 + + + +XML + +run_guard() { + local mode="$1" project="$2" appcast="$3" + CMUX_SPARKLE_MONOTONIC_MODE="$mode" \ + CMUX_SPARKLE_PROJECT_FILE="$project" \ + CMUX_SPARKLE_APPCAST_URL="$appcast" \ + CMUX_SPARKLE_APPCAST_RETRIES=0 \ + CMUX_SPARKLE_APPCAST_RETRY_DELAY=0 \ + "$GUARD" +} + +STALE="$(project_with_build 250)" +FRESH="$(project_with_build 251)" + +# Tag push semantics: a stale build number fails. +if output="$(run_guard enforce "$STALE" "file://$APPCAST" 2>&1)"; then + echo "FAIL: enforce mode accepted CURRENT_PROJECT_VERSION equal to the published build" >&2 + exit 1 +fi +if ! grep -q "must be strictly greater than" <<<"$output"; then + echo "FAIL: enforce mode did not explain the stale build number: $output" >&2 + exit 1 +fi + +# The default is enforce, so callers that pass no mode (release-pretag-guard.sh) still fail. +if CMUX_SPARKLE_PROJECT_FILE="$STALE" CMUX_SPARKLE_APPCAST_URL="file://$APPCAST" "$GUARD" >/dev/null 2>&1; then + echo "FAIL: the guard must enforce by default" >&2 + exit 1 +fi + +# Dry-run semantics: the same stale build number only warns and exits 0. +if ! output="$(run_guard warn "$STALE" "file://$APPCAST" 2>&1)"; then + echo "FAIL: warn mode must not fail on a stale build number: $output" >&2 + exit 1 +fi +if ! grep -q "^WARN: CURRENT_PROJECT_VERSION (250) is not greater than" <<<"$output"; then + echo "FAIL: warn mode did not report the stale build number: $output" >&2 + exit 1 +fi +if ! grep -q "PASS (warn mode)" <<<"$output"; then + echo "FAIL: warn mode did not report its tolerated pass: $output" >&2 + exit 1 +fi + +# A bumped build number passes in both modes. +for mode in enforce warn; do + if ! output="$(run_guard "$mode" "$FRESH" "file://$APPCAST" 2>&1)"; then + echo "FAIL: $mode mode rejected a bumped build number: $output" >&2 + exit 1 + fi + if ! grep -q "^PASS: local CURRENT_PROJECT_VERSION=251 > published Sparkle build=250" <<<"$output"; then + echo "FAIL: $mode mode did not report the monotonic pass: $output" >&2 + exit 1 + fi +done + +# An unreachable appcast is a missing signal. A tag push (enforce, also the +# default) must fail closed rather than publish a build number it cannot compare; +# only an explicit warn-mode dry run tolerates it. +if output="$(run_guard enforce "$FRESH" "file://$TMP_DIR/missing-appcast.xml" 2>&1)"; then + echo "FAIL: enforce mode must fail when the published appcast cannot be fetched: $output" >&2 + exit 1 +fi +if ! grep -q "^FAIL: could not fetch the latest published Sparkle build" <<<"$output"; then + echo "FAIL: enforce mode did not explain the unreachable appcast: $output" >&2 + exit 1 +fi +if CMUX_SPARKLE_PROJECT_FILE="$FRESH" CMUX_SPARKLE_APPCAST_URL="file://$TMP_DIR/missing-appcast.xml" \ + CMUX_SPARKLE_APPCAST_RETRIES=0 CMUX_SPARKLE_APPCAST_RETRY_DELAY=0 "$GUARD" >/dev/null 2>&1; then + echo "FAIL: the default mode must fail closed on an unreachable appcast" >&2 + exit 1 +fi +if ! output="$(run_guard warn "$STALE" "file://$TMP_DIR/missing-appcast.xml" 2>&1)"; then + echo "FAIL: warn mode must soft-pass an unreachable appcast: $output" >&2 + exit 1 +fi +if ! grep -q "PASS (soft)" <<<"$output"; then + echo "FAIL: warn mode did not soft-pass the unreachable appcast: $output" >&2 + exit 1 +fi + +# An unknown mode is a configuration error, never a silent pass. +if run_guard sometimes "$FRESH" "file://$APPCAST" >/dev/null 2>&1; then + echo "FAIL: an unknown CMUX_SPARKLE_MONOTONIC_MODE must fail" >&2 + exit 1 +fi + +# release.yml must select the mode from the ref: enforce on tags, warn otherwise. +RELEASE_WORKFLOW="$ROOT_DIR/.github/workflows/release.yml" +if ! grep -Fq "CMUX_SPARKLE_MONOTONIC_MODE: \${{ startsWith(github.ref, 'refs/tags/') && 'enforce' || 'warn' }}" "$RELEASE_WORKFLOW"; then + echo "FAIL: release.yml must run the monotonic guard in enforce mode for tags and warn mode for dry runs" >&2 + exit 1 +fi + +echo "PASS: Sparkle monotonic guard enforces (and fails closed) for tag pushes and warns for dry runs" diff --git a/tests/test_sparkle_generate_appcast_no_deltas.sh b/tests/test_sparkle_generate_appcast_no_deltas.sh new file mode 100755 index 000000000000..c77f6144d026 --- /dev/null +++ b/tests/test_sparkle_generate_appcast_no_deltas.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Behavioral test for scripts/sparkle_generate_appcast.sh on the stable release +# path, where no previous archives exist and therefore no delta arguments. +# +# Release dry run 34227505375 (2026-09-08) uploaded a DMG with no appcast: the +# script expanded an empty array as "${delta_args[@]}", which is an "unbound +# variable" error under `set -u` in bash 3.2 (macOS /bin/bash), and the EXIT +# trap made bash 3.2 exit 0 anyway, so the workflow step passed. Nightly never +# hit it because it always has previous archives. Drive the script with fake +# git/xcodebuild/generate_appcast tools under every bash on this machine +# (macOS /bin/bash 3.2 reproduces the bug; bash 4.4+ never did) and require a +# signed appcast to land at the requested output path. +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +SCRIPT="$ROOT_DIR/scripts/sparkle_generate_appcast.sh" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/cmux-appcast-no-deltas.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT +FAKE_BIN="$TMP_DIR/bin" +mkdir -p "$FAKE_BIN" +fail() { echo "FAIL: $*" >&2; exit 1; } + +# `git clone ... `: pretend the Sparkle checkout exists. +cat > "$FAKE_BIN/git" <<'GIT' +#!/usr/bin/env bash +set -euo pipefail +[ "${1:-}" = "clone" ] || { echo "fake git: unexpected $*" >&2; exit 1; } +mkdir -p "${@: -1}" +GIT + +# `xcodebuild ... -scheme ... -derivedDataPath ... build`: drop a +# fake tool binary where the script expects the Release product. +cat > "$FAKE_BIN/xcodebuild" <<'XC' +#!/usr/bin/env bash +set -euo pipefail +scheme=""; derived="" +while [ $# -gt 0 ]; do + case "$1" in + -scheme) scheme="$2"; shift ;; + -derivedDataPath) derived="$2"; shift ;; + esac + shift +done +[ -n "$scheme" ] && [ -n "$derived" ] || { echo "fake xcodebuild: missing -scheme/-derivedDataPath" >&2; exit 1; } +mkdir -p "$derived/Build/Products/Release" +cp "$CMUX_TEST_FAKE_TOOLS/$scheme" "$derived/Build/Products/Release/$scheme" +chmod +x "$derived/Build/Products/Release/$scheme" +XC + +FAKE_TOOLS="$TMP_DIR/tools" +mkdir -p "$FAKE_TOOLS" +# generate_appcast: record argv (one per line, so an empty argument is visible), +# then write a signed feed for the DMG found in the archives dir (last argument). +cat > "$FAKE_TOOLS/generate_appcast" <<'GA' +#!/usr/bin/env bash +set -euo pipefail +: > "$CMUX_TEST_ARGV_LOG" +for arg in "$@"; do printf '%s\n' "$arg" >> "$CMUX_TEST_ARGV_LOG"; done +archives="${@: -1}" +dmg="$(find "$archives" -maxdepth 1 -name '*.dmg' | sort | tail -n 1)" +[ -n "$dmg" ] || { echo "fake generate_appcast: no dmg in $archives" >&2; exit 1; } +name="$(basename "$dmg")" +cat > "$archives/appcast.xml" < + + + + 102 + + + + +XML +GA +cat > "$FAKE_TOOLS/sign_update" <<'SU' +#!/usr/bin/env bash +echo "fixture-signature" +SU +chmod +x "$FAKE_BIN"/* "$FAKE_TOOLS"/* + +run_script() { + local bash_bin="$1" out="$2" + shift 2 + PATH="$FAKE_BIN:$PATH" \ + CMUX_TEST_FAKE_TOOLS="$FAKE_TOOLS" \ + CMUX_TEST_ARGV_LOG="$TMP_DIR/argv.log" \ + SPARKLE_PRIVATE_KEY="Zml4dHVyZS1rZXk" \ + "$@" \ + "$bash_bin" "$SCRIPT" "$TMP_DIR/cmux-macos.dmg" "v0.0.0-test" "$out" +} + +printf 'dmg' > "$TMP_DIR/cmux-macos.dmg" + +# Every bash on this machine: /bin/bash is 3.2 on macOS runners (the bug), and +# whichever bash `env` resolves is what the shebang would pick. +candidates=() +[ -x /bin/bash ] && candidates+=(/bin/bash) +resolved="$(command -v bash)" +if [ -n "$resolved" ] && [ "$resolved" != "/bin/bash" ]; then candidates+=("$resolved"); fi +[ "${#candidates[@]}" -gt 0 ] || fail "no bash found" + +for bash_bin in "${candidates[@]}"; do + version="$("$bash_bin" -c 'echo "${BASH_VERSION%%(*}"')" + out_dir="$TMP_DIR/out-$(echo "$bash_bin" | tr '/' '_')" + mkdir -p "$out_dir" + + # Stable release path: no previous archives, so no delta arguments. + if ! run_script "$bash_bin" "$out_dir/appcast.xml" env -u SPARKLE_PREVIOUS_ARCHIVES_DIR >"$out_dir/run.log" 2>&1; then + fail "bash $version: script failed on the no-previous-archives path: $(tail -n 5 "$out_dir/run.log")" + fi + [ -s "$out_dir/appcast.xml" ] || fail "bash $version: no appcast written to the requested output path" + grep -q 'sparkle:edSignature' "$out_dir/appcast.xml" || fail "bash $version: appcast lacks sparkle:edSignature" + grep -q 'cmux-macos.dmg' "$out_dir/appcast.xml" || fail "bash $version: appcast does not reference the DMG" + grep -q "unbound variable" "$out_dir/run.log" && fail "bash $version: script still reports an unbound variable" + grep -qx -- "--maximum-deltas" "$TMP_DIR/argv.log" && fail "bash $version: delta arguments passed although there were no previous archives" + grep -qx "" "$TMP_DIR/argv.log" && fail "bash $version: generate_appcast received an empty argument" + + # Nightly path: previous archives present, delta arguments still flow through. + mkdir -p "$TMP_DIR/previous" + printf 'old' > "$TMP_DIR/previous/cmux-macos-101.dmg" + if ! run_script "$bash_bin" "$out_dir/appcast-deltas.xml" env SPARKLE_PREVIOUS_ARCHIVES_DIR="$TMP_DIR/previous" SPARKLE_MAXIMUM_DELTAS=1 >"$out_dir/run-deltas.log" 2>&1; then + fail "bash $version: script failed with previous archives: $(tail -n 5 "$out_dir/run-deltas.log")" + fi + [ -s "$out_dir/appcast-deltas.xml" ] || fail "bash $version: no appcast written on the delta path" + paste -sd' ' "$TMP_DIR/argv.log" | grep -q -- "--maximum-deltas 1 " || fail "bash $version: --maximum-deltas 1 not passed with previous archives: $(paste -sd' ' "$TMP_DIR/argv.log")" + echo "ok: bash $version generates a signed appcast with and without previous archives" +done + +# release.yml must not trust the generator's exit status alone (bash 3.2 masks it). +RELEASE_WORKFLOW="$ROOT_DIR/.github/workflows/release.yml" +step="$(awk '/sparkle_generate_appcast.sh cmux-macos.dmg/{p=1} p{print} p&&/^ - name:/{exit}' "$RELEASE_WORKFLOW")" +grep -q 'test -s appcast.xml' <<<"$step" || fail "release.yml must verify appcast.xml exists after generation" +grep -q "grep -q 'sparkle:edSignature' appcast.xml" <<<"$step" || fail "release.yml must verify the appcast is signed after generation" + +echo "PASS: sparkle_generate_appcast.sh produces a signed appcast on the no-delta release path under every local bash"