Retry transient Sparkle download failures

.github/swift-file-length-budget.tsv

@@ -2,7 +2,7 @@
 # Format: max_lines<TAB>relative path
 # Reduce counts as files shrink. CI fails if tracked files exceed this budget.
 34427	CLI/cmux.swift
-17908	Sources/AppDelegate.swift
+17906	Sources/AppDelegate.swift
 16491	Sources/ContentView.swift
 14170	Sources/TerminalController.swift
 12917	Sources/Workspace.swift

Packages/macOS/CmuxUpdater/Sources/CmuxUpdater/UpdateActionDelegate.swift

@@ -8,7 +8,11 @@
 public protocol UpdateActionDelegate: AnyObject {
     /// The user asked to retry after an update error. The host should re-initiate a check
     /// through its normal entry point.
-    func updaterRequestsRetryCheckForUpdates()
+    ///
+    /// - Parameter preservingInstallIntent: Whether the interrupted Sparkle session had already
+    ///   moved past update selection into an install/download phase and the retry should
+    ///   auto-confirm the same update after Sparkle finds it again.
+    func updaterRequestsRetryCheckForUpdates(preservingInstallIntent: Bool)
 
     /// Sparkle is about to relaunch the app to finish installing. The host should persist
     /// session state, stop its terminal/runtime, and invalidate restorable state so the

Packages/macOS/CmuxUpdater/Sources/CmuxUpdater/UpdateController.swift

@@ -202,12 +202,33 @@ public final class UpdateController {
         checkForUpdatesWhenReady()
     }
 
-    private func performCheckForUpdates() {
+    /// Retry after a transient Sparkle download failure. Unlike a user-started fresh check,
+    /// this preserves any in-progress install/attempt intent so a retried archive download can
+    /// continue silently after Sparkle finds the same update again.
+    ///
+    /// The driver passes `preservingInstallIntent: true` when the failure occurred during a
+    /// download/extract/install phase. We additionally OR in ``AttemptUpdateCoordinator/isMonitoring``
+    /// so intent is preserved when the failure lands during the coordinator's own re-resolve
+    /// check (issue #6366), before the install state machine reaches a downloading phase.
+    ///
+    /// - Parameter preservingInstallIntent: Whether the retry request came from a Sparkle
+    ///   install/download phase whose update choice should be auto-confirmed again.
+    public func retryAfterTransientFailure(preservingInstallIntent: Bool = false) {
+        let shouldPreserveInstallIntent = preservingInstallIntent || attemptCoordinator.isMonitoring
+        log.append("retrying update after transient download failure (preserveInstallIntent=\(shouldPreserveInstallIntent))")
+        checkForUpdatesWhenReady(preservingInstallIntent: shouldPreserveInstallIntent)
+    }
+
+    private func performCheckForUpdates(preservingInstallIntent: Bool = false) {
         startUpdaterIfNeeded()
         ensureSparkleInstallationCache()
         // Cancel any pending deferred re-check on every path so a stale one can't fire a
         // duplicate checkForUpdates() after this new check starts.
         recheckTask?.cancel()
+        if preservingInstallIntent {
+            updater.checkForUpdates()
+            return
+        }
         if model.state == .idle {
             updater.checkForUpdates()
             return
@@ -228,7 +249,7 @@ public final class UpdateController {
     }
 
     /// Check for updates once the updater reports it can.
-    private func checkForUpdatesWhenReady() {
+    private func checkForUpdatesWhenReady(preservingInstallIntent: Bool = false) {
         if isDevLikeBundle {
             // DEV/staging builds are not on the public release train. A manual check (menu,
             // custom UI, or attempt-and-install) must not query the public appcast or offer the
@@ -246,22 +267,22 @@ public final class UpdateController {
         let canCheck = updater.canCheckForUpdates
         log.append("checkForUpdatesWhenReady invoked (canCheck=\(canCheck))")
         if canCheck {
-            performCheckForUpdates()
+            performCheckForUpdates(preservingInstallIntent: preservingInstallIntent)
             return
         }
         if model.state.isIdle {
             model.setState(.checking(.init(cancel: {})))
         }
-        waitForReadinessThenCheck()
+        waitForReadinessThenCheck(preservingInstallIntent: preservingInstallIntent)
     }
 
-    private func waitForReadinessThenCheck() {
+    private func waitForReadinessThenCheck(preservingInstallIntent: Bool = false) {
         readyCheckTask = Task { @MainActor [weak self] in
             guard let self else { return }
             var remaining = self.readyRetryCount
             while remaining > 0 {
                 if self.updater.canCheckForUpdates {
-                    self.performCheckForUpdates()
+                    self.performCheckForUpdates(preservingInstallIntent: preservingInstallIntent)
                     return
                 }
                 remaining -= 1

Packages/macOS/CmuxUpdater/Sources/CmuxUpdater/UpdateDriver.swift

@@ -14,6 +14,7 @@ final class UpdateDriver: NSObject, @preconcurrency SPUUserDriver {
     let model: UpdateStateModel
     let log: any UpdateLogging
     private let clock: any UpdateClock
+    private let retryPolicy: UpdateRetryPolicy
     /// Whether the running build is a cmux DEV/staging build that is not on the public release
     /// train. When `true`, the driver must never surface the public appcast's update pill (see
     /// ``UpdateController/isDevLikeBundleIdentifier(_:)``).
@@ -26,19 +27,28 @@ final class UpdateDriver: NSObject, @preconcurrency SPUUserDriver {
     private var lastCheckStart: Date?
     private var pendingCheckTransitionTask: Task<Void, Never>?
     private var checkTimeoutTask: Task<Void, Never>?
+    private var transientErrorRetryTask: Task<Void, Never>?
+    private var transientErrorRetryFailureCount = 0
+    private var shouldPreserveRetryStateForNextCheck = false
     private(set) var lastFeedURLString: String?
 
-    init(model: UpdateStateModel, log: any UpdateLogging, clock: any UpdateClock, isDevLikeBundle: Bool = false) {
+    init(model: UpdateStateModel,
+         log: any UpdateLogging,
+         clock: any UpdateClock,
+         retryPolicy: UpdateRetryPolicy = UpdateRetryPolicy(),
+         isDevLikeBundle: Bool = false) {
         self.model = model
         self.log = log
         self.clock = clock
+        self.retryPolicy = retryPolicy
         self.isDevLikeBundle = isDevLikeBundle
         super.init()
     }
 
     deinit {
         pendingCheckTransitionTask?.cancel()
         checkTimeoutTask?.cancel()
+        transientErrorRetryTask?.cancel()
     }
 
     func show(_ request: SPUUpdatePermissionRequest,
@@ -79,6 +89,7 @@ final class UpdateDriver: NSObject, @preconcurrency SPUUserDriver {
 
     func showUpdateNotFoundWithError(_ error: any Error,
                                      acknowledgement: @escaping () -> Void) {
+        resetTransientErrorRetryState()
         log.append("show update not found: \(formatErrorForLog(error))")
         setStateAfterMinimumCheckDelay(.notFound(.init(acknowledgement: acknowledgement)))
     }
@@ -87,11 +98,15 @@ final class UpdateDriver: NSObject, @preconcurrency SPUUserDriver {
                           acknowledgement: @escaping () -> Void) {
         let details = formatErrorForLog(error)
         log.append("show updater error: \(details)")
+        if scheduleTransientErrorRetryIfNeeded(for: error) {
+            acknowledgement()
+            return
+        }
         setState(.error(.init(
             error: error,
             retry: { [weak self] in
                 self?.model.setState(.idle)
-                self?.actionDelegate?.updaterRequestsRetryCheckForUpdates()
+                self?.actionDelegate?.updaterRequestsRetryCheckForUpdates(preservingInstallIntent: false)
             },
             dismiss: { [weak self] in
                 self?.model.setState(.idle)
@@ -133,6 +148,7 @@ final class UpdateDriver: NSObject, @preconcurrency SPUUserDriver {
     }
 
     func showDownloadDidStartExtractingUpdate() {
+        resetTransientErrorRetryState()
         log.append("show extraction started")
         setState(.extracting(.init(progress: 0)))
     }
@@ -158,6 +174,7 @@ final class UpdateDriver: NSObject, @preconcurrency SPUUserDriver {
     }
 
     func showUpdateInstalledAndRelaunched(_ relaunched: Bool, acknowledgement: @escaping () -> Void) {
+        resetTransientErrorRetryState()
         log.append("show update installed (relaunched=\(relaunched))")
         setState(.idle)
         acknowledgement()
@@ -187,6 +204,13 @@ final class UpdateDriver: NSObject, @preconcurrency SPUUserDriver {
     // MARK: - State transition helpers
 
     private func beginChecking(cancel: @escaping () -> Void) {
+        transientErrorRetryTask?.cancel()
+        transientErrorRetryTask = nil
+        if shouldPreserveRetryStateForNextCheck {
+            shouldPreserveRetryStateForNextCheck = false
+        } else {
+            transientErrorRetryFailureCount = 0
+        }
         model.setOverrideState(nil)
         pendingCheckTransitionTask?.cancel()
         pendingCheckTransitionTask = nil
@@ -197,6 +221,62 @@ final class UpdateDriver: NSObject, @preconcurrency SPUUserDriver {
         scheduleCheckTimeout()
     }
 
+    private func scheduleTransientErrorRetryIfNeeded(for error: any Error) -> Bool {
+        guard actionDelegate != nil else { return false }
+        let nextFailureCount = transientErrorRetryFailureCount + 1
+        guard let retryDelay = retryPolicy.delay(afterFailureNumber: nextFailureCount, for: error) else {
+            if retryPolicy.isTransientDownloadError(error) {
+                log.append("transient update download error retries exhausted (\(transientErrorRetryFailureCount)/\(retryPolicy.maximumRetryCount))")
+            }
+            return false
+        }
+
+        let preservingInstallIntent = shouldPreserveInstallIntentForTransientRetry()
+        transientErrorRetryFailureCount = nextFailureCount
+        transientErrorRetryTask?.cancel()
+        log.append("transient update download error; retry \(nextFailureCount)/\(retryPolicy.maximumRetryCount) in \(String(format: "%.0f", retryDelay))s")
+        setState(.checking(.init(cancel: { [weak self] in
+            self?.cancelPendingTransientErrorRetry()
+        })))
+
+        transientErrorRetryTask = Task { @MainActor [weak self] in
+            guard let self else { return }
+            // Bounded, cancellable retry backoff via the injected clock.
+            try? await self.clock.sleep(for: .seconds(retryDelay))
+            guard !Task.isCancelled else { return }
+            self.transientErrorRetryTask = nil
+            self.shouldPreserveRetryStateForNextCheck = true
+            self.actionDelegate?.updaterRequestsRetryCheckForUpdates(preservingInstallIntent: preservingInstallIntent)
+        }
+        return true
+    }
+
+    private func shouldPreserveInstallIntentForTransientRetry() -> Bool {
+        switch model.state {
+        case .downloading, .extracting, .installing:
+            return true
+        default:
+            return false
+        }
+    }
+
+    private func cancelPendingTransientErrorRetry() {
+        transientErrorRetryTask?.cancel()
+        transientErrorRetryTask = nil
+        guard !shouldPreserveRetryStateForNextCheck else { return }
+        transientErrorRetryFailureCount = 0
+        if case .checking = model.state {
+            model.setState(.idle)
+        }
+    }
+
+    private func resetTransientErrorRetryState() {
+        transientErrorRetryTask?.cancel()
+        transientErrorRetryTask = nil
+        transientErrorRetryFailureCount = 0
+        shouldPreserveRetryStateForNextCheck = false
+    }
+
     private func setStateAfterMinimumCheckDelay(_ newState: UpdateState) {
         pendingCheckTransitionTask?.cancel()
         pendingCheckTransitionTask = nil

Packages/macOS/CmuxUpdater/Sources/CmuxUpdater/UpdateRetryPolicy.swift

@@ -0,0 +1,106 @@
+import Foundation
+@preconcurrency import Sparkle
+
+/// Classifies transient update download failures and maps them to bounded retry delays.
+struct UpdateRetryPolicy {
+    /// The retry delays for consecutive transient failures.
+    let retryDelays: [TimeInterval]
+
+    /// Creates a retry policy.
+    init(retryDelays: [TimeInterval] = [1, 3, 8]) {
+        self.retryDelays = retryDelays
+    }
+
+    /// The number of automatic retries allowed for one update operation.
+    var maximumRetryCount: Int {
+        retryDelays.count
+    }
+
+    /// Returns the retry delay for a 1-based failure number, or `nil` if this error should
+    /// surface immediately.
+    func delay(afterFailureNumber failureNumber: Int, for error: any Swift.Error) -> TimeInterval? {
+        guard isTransientDownloadError(error) else { return nil }
+        guard failureNumber > 0, failureNumber <= retryDelays.count else { return nil }
+        return retryDelays[failureNumber - 1]
+    }
+
+    /// Whether the error is a Sparkle download failure caused by a transient network/server
+    /// condition such as GitHub's release CDN returning HTTP 504.
+    func isTransientDownloadError(_ error: any Swift.Error) -> Bool {
+        let errors = relatedErrors(startingAt: error as NSError)
+        guard errors.contains(where: isSparkleDownloadError) else { return false }
+        return errors.contains(where: hasTransientNetworkSignal)
+    }
+
+    private func isSparkleDownloadError(_ error: NSError) -> Bool {
+        error.domain == SUSparkleErrorDomain && error.code == 2001
+    }
+
+    private func hasTransientNetworkSignal(_ error: NSError) -> Bool {
+        if isTransientURLLoadingError(error) {
+            return true
+        }
+        return transientHTTPStatusCode(in: error).map(isTransientHTTPStatusCode) ?? false
+    }
+
+    private func isTransientURLLoadingError(_ error: NSError) -> Bool {
+        guard error.domain == NSURLErrorDomain else { return false }
+        switch error.code {
+        case NSURLErrorTimedOut,
+             NSURLErrorCannotConnectToHost,
+             NSURLErrorDNSLookupFailed,
+             NSURLErrorNetworkConnectionLost,
+             NSURLErrorCannotLoadFromNetwork:
+            return true
+        default:
+            return false
+        }
+    }
+
+    private func isTransientHTTPStatusCode(_ statusCode: Int) -> Bool {
+        statusCode == 408 || statusCode == 429 || (500...599).contains(statusCode)
+    }
+
+    private func transientHTTPStatusCode(in error: NSError) -> Int? {
+        let text = [
+            error.localizedDescription,
+            (error.userInfo[NSLocalizedFailureReasonErrorKey] as? String) ?? "",
+            (error.userInfo[NSLocalizedRecoverySuggestionErrorKey] as? String) ?? "",
+        ].joined(separator: "\n")
+
+        return firstTransientHTTPStatusCode(in: text)
+    }
+
+    private func firstTransientHTTPStatusCode(in text: String) -> Int? {
+        let patterns = [
+            #"\((\d{3})\)"#,
+            #"(?i)\bHTTP\s+(\d{3})\b"#,
+        ]
+        let nsText = text as NSString
+        let fullRange = NSRange(location: 0, length: nsText.length)
+        for pattern in patterns {
+            guard let regex = try? NSRegularExpression(pattern: pattern) else { continue }
+            let matches = regex.matches(in: text, range: fullRange)
+            for match in matches where match.numberOfRanges > 1 {
+                let status = nsText.substring(with: match.range(at: 1))
+                guard let statusCode = Int(status) else { continue }
+                guard isTransientHTTPStatusCode(statusCode) else { continue }
+                return statusCode
+            }
+        }
+        return nil
+    }
+
+    private func relatedErrors(startingAt error: NSError) -> [NSError] {
+        var errors = [error]
+        var current = error
+        var remainingDepth = 8
+        while remainingDepth > 0,
+              let underlying = current.userInfo[NSUnderlyingErrorKey] as? NSError {
+            errors.append(underlying)
+            current = underlying
+            remainingDepth -= 1
+        }
+        return errors
+    }
+}

Packages/macOS/CmuxUpdater/Tests/CmuxUpdaterTests/ImmediateUpdateClock.swift

@@ -0,0 +1,6 @@
+import Foundation
+@testable import CmuxUpdater
+
+struct ImmediateUpdateClock: UpdateClock {
+    func sleep(for duration: Duration) async throws {}
+}

Packages/macOS/CmuxUpdater/Tests/CmuxUpdaterTests/NullUpdateLog.swift

@@ -0,0 +1,6 @@
+@testable import CmuxUpdater
+
+struct NullUpdateLog: UpdateLogging {
+    func append(_ message: String) {}
+    func logPath() -> String { "/tmp/cmux-update-test.log" }
+}