diff --git a/Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/CmxIrohSettingsControlling.swift b/Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/CmxIrohSettingsControlling.swift index 87b08240c68b..b7ad2664849f 100644 --- a/Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/CmxIrohSettingsControlling.swift +++ b/Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/CmxIrohSettingsControlling.swift @@ -41,6 +41,10 @@ public protocol CmxIrohSettingsControlling: AnyObject { /// Erases the in-memory connection timeline and rotates its report session. func clearIrohDiagnosticReport() async + + /// The archived report from the previous process launch, if one exists. + /// Exports include it so a drop that preceded a relaunch stays diagnosable. + func irohPreviousLaunchDiagnosticReport() async -> DiagnosticReport? } public extension CmxIrohSettingsControlling { @@ -56,6 +60,10 @@ public extension CmxIrohSettingsControlling { .empty } + func irohPreviousLaunchDiagnosticReport() async -> DiagnosticReport? { + nil + } + func exportIrohDiagnosticReport() async -> Data { Data() } diff --git a/Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/DiagnosticEventCode.swift b/Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/DiagnosticEventCode.swift index 7bf269c48714..e68980f51888 100644 --- a/Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/DiagnosticEventCode.swift +++ b/Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/DiagnosticEventCode.swift @@ -173,4 +173,18 @@ public enum DiagnosticEventCode: UInt16, Sendable, Codable, CaseIterable { /// process-local session correlation ID. The event contains no peer or route /// identity. case transportSessionLifecycle = 51 + /// The app's scene phase changed. `a` is ``DiagnosticAppLifecyclePhase``. + /// Session drops that follow a backgrounding within seconds are suspension + /// casualties, not network failures; this event makes that attributable. + case appLifecycleChanged = 52 + /// Device reachability changed. `a` is 1 when a usable network path + /// exists, else 0. Correlates drops with WiFi/cellular transitions. + case reachabilityChanged = 53 +} + +/// Scene phase carried by ``DiagnosticEventCode/appLifecycleChanged``. +public enum DiagnosticAppLifecyclePhase: Int, Sendable, Codable, CaseIterable { + case background = 0 + case active = 1 + case inactive = 2 } diff --git a/Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/DiagnosticReportArchive.swift b/Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/DiagnosticReportArchive.swift new file mode 100644 index 000000000000..683d8d35fff9 --- /dev/null +++ b/Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/DiagnosticReportArchive.swift @@ -0,0 +1,61 @@ +public import Foundation + +/// Persists the most recent diagnostic snapshot across process launches. +/// +/// The in-memory diagnostic ring dies with the process, so the events around +/// a connection drop were unrecoverable once the user relaunched before +/// exporting. The archive keeps exactly one previous-launch report on disk +/// (bounded, privacy-safe integers only, same vocabulary as the live ring) so +/// an export can include what happened before the current launch. +public struct DiagnosticReportArchive: Sendable { + /// Upper bound for a stored report file; a report of maximum event count + /// encodes far below this, so hitting it means corruption, not data. + public static let maximumFileByteCount = 1_024 * 1_024 + + public let fileURL: URL + + public init(fileURL: URL) { + self.fileURL = fileURL + } + + /// Default location inside Application Support. + public static func defaultArchive( + fileManager: FileManager = .default + ) -> DiagnosticReportArchive? { + guard let base = fileManager.urls( + for: .applicationSupportDirectory, + in: .userDomainMask + ).first else { return nil } + do { + try fileManager.createDirectory(at: base, withIntermediateDirectories: true) + } catch { + return nil + } + return DiagnosticReportArchive( + fileURL: base.appendingPathComponent("cmux-diagnostic-report.json") + ) + } + + /// Atomically replaces the stored report. Empty reports are not worth a + /// write and would only erase a more useful previous snapshot. + public func save(_ report: DiagnosticReport) { + guard !report.events.isEmpty else { return } + guard let data = try? JSONEncoder().encode(report), + data.count <= Self.maximumFileByteCount else { return } + try? data.write(to: fileURL, options: .atomic) + } + + /// Loads the previous process's report, or `nil` when absent or invalid. + public func load() -> DiagnosticReport? { + guard let data = try? Data(contentsOf: fileURL), + data.count <= Self.maximumFileByteCount, + let report = try? JSONDecoder().decode(DiagnosticReport.self, from: data), + !report.events.isEmpty else { return nil } + return report + } + + /// Removes the stored report (sign-out / account erase). + public func clear(fileManager: FileManager = .default) { + try? fileManager.removeItem(at: fileURL) + } +} diff --git a/Packages/Shared/CMUXMobileCore/Tests/CMUXMobileCoreTests/DiagnosticReportArchiveTests.swift b/Packages/Shared/CMUXMobileCore/Tests/CMUXMobileCoreTests/DiagnosticReportArchiveTests.swift new file mode 100644 index 000000000000..487be6af4001 --- /dev/null +++ b/Packages/Shared/CMUXMobileCore/Tests/CMUXMobileCoreTests/DiagnosticReportArchiveTests.swift @@ -0,0 +1,62 @@ +import Foundation +import Testing +@testable import CMUXMobileCore + +@Suite +struct DiagnosticReportArchiveTests { + private func temporaryArchive() -> DiagnosticReportArchive { + DiagnosticReportArchive( + fileURL: FileManager.default.temporaryDirectory + .appendingPathComponent("cmuxdiag-archive-\(UUID().uuidString).json") + ) + } + + private func report(eventCount: Int) -> DiagnosticReport { + DiagnosticReport( + role: .mobileClient, + anchorWallNanos: 1, + anchorMonotonicNanos: 1, + buildStamp: "test", + events: (0 ..< eventCount).map { + DiagnosticEvent(code: .connect, tNanos: UInt64($0 + 1)) + } + ) + } + + @Test + func savesAndReloadsAcrossInstances() { + let archive = temporaryArchive() + defer { archive.clear() } + archive.save(report(eventCount: 3)) + + let reloaded = DiagnosticReportArchive(fileURL: archive.fileURL).load() + #expect(reloaded?.events.count == 3) + #expect(reloaded?.role == .mobileClient) + } + + @Test + func emptyReportDoesNotReplaceStoredSnapshot() { + let archive = temporaryArchive() + defer { archive.clear() } + archive.save(report(eventCount: 2)) + archive.save(report(eventCount: 0)) + + #expect(archive.load()?.events.count == 2) + } + + @Test + func clearRemovesTheSnapshot() { + let archive = temporaryArchive() + archive.save(report(eventCount: 1)) + archive.clear() + #expect(archive.load() == nil) + } + + @Test + func corruptFileLoadsAsNil() throws { + let archive = temporaryArchive() + defer { archive.clear() } + try Data("not json".utf8).write(to: archive.fileURL) + #expect(archive.load() == nil) + } +} diff --git a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohBrokerCooldown.swift b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohBrokerCooldown.swift new file mode 100644 index 000000000000..92a75a6aa8f0 --- /dev/null +++ b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohBrokerCooldown.swift @@ -0,0 +1,92 @@ +public import CMUXMobileCore +public import Foundation + +/// Account-scoped cooldown honoring a broker's Retry-After directive across +/// endpoint activation attempts. +/// +/// The credential coordinator already honors Retry-After inside one activation, +/// but a torn-down runtime discards that state, so every reconnect attempt +/// re-ran the full broker call set and kept the server's rate-limit window +/// exhausted. This value survives between activations: while active, callers +/// skip broker work entirely and surface the remaining delay to the reconnect +/// scheduler. +public struct CmxIrohBrokerCooldown: Equatable, Sendable { + public private(set) var accountID: String? + public private(set) var retryAt: Date? + + public init() {} + + /// Records a server directive, keeping the later of two overlapping floors. + /// A directive for a different account replaces the previous floor. + @discardableResult + public mutating func record( + accountID: String, + retryAfterSeconds: Int, + now: Date + ) -> Date { + let proposed = now.addingTimeInterval(TimeInterval(max(1, retryAfterSeconds))) + if self.accountID != accountID { + self.accountID = accountID + retryAt = proposed + return proposed + } + if let retryAt, retryAt >= proposed { return retryAt } + retryAt = proposed + return proposed + } + + /// Whole seconds until the floor expires, or `nil` when no floor applies. + /// Expired floors are cleared on read so state cannot go stale. + public mutating func remainingSeconds(accountID: String, now: Date) -> Int? { + guard self.accountID == accountID, let retryAt else { return nil } + let remaining = retryAt.timeIntervalSince(now) + guard remaining > 0 else { + clear() + return nil + } + return Int(remaining.rounded(.up)) + } + + /// Clears the floor, optionally only when it belongs to one account. + public mutating func clear(accountID: String? = nil) { + guard accountID == nil || self.accountID == accountID else { return } + self.accountID = nil + retryAt = nil + } +} + +public extension CmxIrohBrokerCooldown { + /// Floor applied to a 429 whose response carried no Retry-After header. + static let defaultRateLimitedSeconds = 60 + + /// Seconds of cooldown one broker failure demands, or `nil` when the + /// error is not a rate-limit signal. Prefers the server's own Retry-After + /// directive; a bare 429 still arms a short default floor so a missing + /// header can never reopen the retry storm. + static func directiveSeconds(for error: any Error) -> Int? { + if let retryAfterSeconds = (error as? any CmxRetryAfterProviding)? + .retryAfterSeconds { + return retryAfterSeconds + } + if case let .rejected(statusCode, _)? = error as? CmxIrohTrustBrokerClientError, + statusCode == 429 { + return defaultRateLimitedSeconds + } + return nil + } +} + +/// Thrown instead of a bare inactive-runtime error while a broker cooldown is +/// active, so the reconnect scheduler can adopt the server's floor instead of +/// its short transient backoff. +public struct CmxIrohBrokerCooldownError: CmxRetryAfterProviding, Equatable { + public let retryAfterSeconds: Int? + + public init(retryAfterSeconds: Int) { + self.retryAfterSeconds = max(1, retryAfterSeconds) + } +} + +extension CmxIrohBrokerCooldownError: DiagnosticFailureProviding { + public var diagnosticFailureKind: DiagnosticFailureKind { .policyUnavailable } +} diff --git a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime+Lifecycle.swift b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime+Lifecycle.swift index 94013f53e751..88a685cda4aa 100644 --- a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime+Lifecycle.swift +++ b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime+Lifecycle.swift @@ -77,6 +77,10 @@ extension CmxIrohClientRuntime { } func validateRelayFleet(_ fleet: [String]) throws { + // Without a verified managed fleet (relay policy unavailable) there is + // nothing to cross-check and no relay will be configured; activation + // continues on direct paths instead of failing closed here. + guard !managedRelayURLs.isEmpty else { return } guard fleet.count == managedRelayURLs.count, Set(fleet) == managedRelayURLs else { throw CmxIrohClientRuntimeError.relayFleetMismatch @@ -103,6 +107,6 @@ extension CmxIrohClientRuntime { } static func isConnectivity(_ error: any Error) -> Bool { - (error as? CmxIrohTrustBrokerClientError) == .connectivity + CmxIrohTrustBrokerClientError.preservesVerifiedPolicyDuringRefresh(error) } } diff --git a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime+Policy.swift b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime+Policy.swift index f09915286c85..31767fa65974 100644 --- a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime+Policy.swift +++ b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime+Policy.swift @@ -4,7 +4,8 @@ public import Foundation extension CmxIrohClientRuntime { func resolvePolicy( expectedEndpointID: CmxIrohPeerIdentity, - revision: UInt64 + revision: UInt64, + allowRegistrationDiscoveryFallback: Bool = false ) async throws -> ResolvedPolicy { try await pendingRevocations.revokePending( accountID: configuration.accountID, @@ -47,13 +48,18 @@ extension CmxIrohClientRuntime { pairingEnabled: false, capabilities: configuration.capabilities ) - let offlineExpectation = try offlinePolicyCache.map { _ in - try CmxIrohClientOfflinePolicyExpectation( - accountID: configuration.accountID, - localBindingExpectation: expectation, - managedRelayURLs: managedRelayURLs - ) - } + // Without a managed relay fleet (policy unavailable or direct-only) + // there is no relay bootstrap to cache offline; activation proceeds + // with direct paths instead of failing the expectation's fleet check. + let offlineExpectation: CmxIrohClientOfflinePolicyExpectation? = + try offlinePolicyCache.flatMap { _ in + guard !managedRelayURLs.isEmpty else { return nil } + return try CmxIrohClientOfflinePolicyExpectation( + accountID: configuration.accountID, + localBindingExpectation: expectation, + managedRelayURLs: managedRelayURLs + ) + } let signer = try CmxIrohRegistrationSigner( identity: configuration.identity, endpointID: expectedEndpointID.endpointID @@ -63,6 +69,32 @@ extension CmxIrohClientRuntime { do { registration = try await broker.register(prepared: prepared, signer: signer) } catch { + if allowRegistrationDiscoveryFallback, + Self.canResolveRegistrationFromDiscovery(error) { + let discovery = try await broker.discover() + try requireCurrent(revision) + guard discovery.routeContractVersion == payload.routeContractVersion else { + throw CmxIrohClientRuntimeError.routeContractMismatch + } + try validateRelayFleet(discovery.relayFleet) + let localMatches = discovery.bindings.filter(expectation.matches) + guard localMatches.count == 1, + let discovered = localMatches.first else { + throw CmxIrohClientRuntimeError.localBindingMissingFromDiscovery + } + return ResolvedPolicy( + registration: CmxIrohRegistrationResponse( + binding: discovered, + relay: .unavailable + ), + discovery: discovery, + binding: discovered, + expectation: expectation, + offlineExpectation: offlineExpectation, + cachedTargetBindings: [], + cachedLANRendezvous: nil + ) + } guard Self.isConnectivity(error), let cached = try await offlineBootstrap( expectation: offlineExpectation, @@ -123,6 +155,25 @@ extension CmxIrohClientRuntime { ) } + private static func canResolveRegistrationFromDiscovery(_ error: any Error) -> Bool { + guard let brokerError = error as? CmxIrohTrustBrokerClientError else { + return false + } + switch brokerError { + case .rateLimited: + return true + case let .rejected(statusCode, _): + return statusCode == 429 + case .connectivity, + .invalidBaseURL, + .missingAuthentication, + .invalidAuthentication, + .nonHTTPResponse, + .invalidResponse: + return false + } + } + func offlineBootstrap( expectation: CmxIrohClientOfflinePolicyExpectation?, confirmedLocalBinding: CmxIrohBrokerBinding? @@ -199,7 +250,8 @@ extension CmxIrohClientRuntime { automaticRefreshEnabled: automaticRelayCredentialRefreshEnabled, credentialDidInstall: { [handleRelayCredential] response in await handleRelayCredential(response, policy.binding) - } + }, + rateLimitedDirective: handleRelayRateLimit ) relayCoordinator = coordinator } @@ -213,7 +265,8 @@ extension CmxIrohClientRuntime { bindingID: policy.binding.bindingID, endpointIdentity: policy.binding.endpointID, bootstrap: bootstrap, - waitForInitialCredential: requiresRelayReadiness + waitForInitialCredential: requiresRelayReadiness, + mintNotBefore: configuration.relayCredentialMintNotBefore ) } catch { if requiresRelayReadiness { throw error } diff --git a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime+RelayPolicy.swift b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime+RelayPolicy.swift index 6cfec26057a8..4a49983fa22a 100644 --- a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime+RelayPolicy.swift +++ b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime+RelayPolicy.swift @@ -109,7 +109,8 @@ extension CmxIrohClientRuntime { automaticRefreshEnabled: automaticRelayCredentialRefreshEnabled, credentialDidInstall: { [handleRelayCredential] response in await handleRelayCredential(response, binding) - } + }, + rateLimitedDirective: handleRelayRateLimit ) relayCoordinator = coordinator do { diff --git a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime.swift b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime.swift index ca23ef6a3410..47909dc349bd 100644 --- a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime.swift +++ b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntime.swift @@ -81,6 +81,9 @@ public actor CmxIrohClientRuntime { let handleBinding: BindingHandler let handleCachedBindings: CachedBindingsHandler let handleRelayCredential: RelayCredentialHandler + /// Forwards a broker Retry-After observed by the credential coordinator so + /// the composition-owned cooldown survives runtime teardown. + let handleRelayRateLimit: (@Sendable (Int) async -> Void)? let handleLocalDeactivation: LocalDeactivationHandler let handlePolicyInvalidation: PolicyInvalidationHandler @@ -141,6 +144,7 @@ public actor CmxIrohClientRuntime { handleBinding: @escaping BindingHandler = { _, _ in true }, handleCachedBindings: @escaping CachedBindingsHandler = { _, _ in }, handleRelayCredential: @escaping RelayCredentialHandler = { _, _ in }, + handleRelayRateLimit: (@Sendable (Int) async -> Void)? = nil, handleLocalDeactivation: @escaping LocalDeactivationHandler = {}, handlePolicyInvalidation: @escaping PolicyInvalidationHandler = {} ) throws { @@ -181,6 +185,7 @@ public actor CmxIrohClientRuntime { self.handleBinding = handleBinding self.handleCachedBindings = handleCachedBindings self.handleRelayCredential = handleRelayCredential + self.handleRelayRateLimit = handleRelayRateLimit self.handleLocalDeactivation = handleLocalDeactivation self.handlePolicyInvalidation = handlePolicyInvalidation transportFactory = CmxIrohByteTransportFactory(sessionPool: sessionPool) @@ -325,7 +330,8 @@ public actor CmxIrohClientRuntime { } let policy = try await resolvePolicy( expectedEndpointID: endpointID, - revision: revision + revision: revision, + allowRegistrationDiscoveryFallback: true ) try requireCurrent(revision) await sessionPool.activate( diff --git a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntimeConfiguration.swift b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntimeConfiguration.swift index a9d192bcb2ab..fe2b54c87372 100644 --- a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntimeConfiguration.swift +++ b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohClientRuntimeConfiguration.swift @@ -1,4 +1,5 @@ internal import CMUXMobileCore +public import Foundation /// Stable, account-and-build-scoped inputs for one iOS Iroh lifecycle. public struct CmxIrohClientRuntimeConfiguration: Equatable, Sendable { @@ -33,6 +34,9 @@ public struct CmxIrohClientRuntimeConfiguration: Equatable, Sendable { /// A previously validated endpoint-scoped relay credential, when available. public let cachedRelayCredential: CmxIrohRelayTokenResponse? + /// Earliest time the credential coordinator may mint against the broker, + /// carried from a rate-limited bootstrap on the same activation. + public let relayCredentialMintNotBefore: Date? /// Creates an immutable iOS client lifecycle configuration. /// @@ -59,7 +63,8 @@ public struct CmxIrohClientRuntimeConfiguration: Equatable, Sendable { capabilities: [String], managedRelayURLs: Set, endpointRelayProfile: CmxIrohEndpointRelayProfile? = nil, - cachedRelayCredential: CmxIrohRelayTokenResponse? = nil + cachedRelayCredential: CmxIrohRelayTokenResponse? = nil, + relayCredentialMintNotBefore: Date? = nil ) { self.accountID = accountID self.deviceID = cmxCanonicalDeviceID(deviceID) @@ -71,5 +76,6 @@ public struct CmxIrohClientRuntimeConfiguration: Equatable, Sendable { self.managedRelayURLs = managedRelayURLs self.endpointRelayProfile = endpointRelayProfile self.cachedRelayCredential = cachedRelayCredential + self.relayCredentialMintNotBefore = relayCredentialMintNotBefore } } diff --git a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohDiagnosticFailure.swift b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohDiagnosticFailure.swift index 29bcc394bcfe..ab0e8452ce57 100644 --- a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohDiagnosticFailure.swift +++ b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohDiagnosticFailure.swift @@ -203,3 +203,29 @@ extension CmxIrohKeychainCredentialStoreError: DiagnosticFailureProviding { extension CmxIrohKeychainIdentityStoreError: DiagnosticFailureProviding { public var diagnosticFailureKind: DiagnosticFailureKind { .credentialUnavailable } } + +extension CmxIrohClientOfflinePolicyCacheError: DiagnosticFailureProviding { + public var diagnosticFailureKind: DiagnosticFailureKind { + switch self { + case .invalidExpectation, .invalidPolicy, .policyMismatch: + .policyUnavailable + case .invalidGrantEnvelope: + .protocolViolation + } + } +} + +extension CmxIrohHostPolicyCacheError: DiagnosticFailureProviding { + public var diagnosticFailureKind: DiagnosticFailureKind { + switch self { + case .invalidExpectation, .invalidPolicy, .policyMismatch: + .policyUnavailable + case .invalidAttestationEnvelope: + .protocolViolation + } + } +} + +extension CmxIrohLocalBindingExpectationError: DiagnosticFailureProviding { + public var diagnosticFailureKind: DiagnosticFailureKind { .protocolViolation } +} diff --git a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohRegistryContextProvider.swift b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohRegistryContextProvider.swift index d2373f6ac1d6..66076919ba18 100644 --- a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohRegistryContextProvider.swift +++ b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohRegistryContextProvider.swift @@ -142,7 +142,11 @@ public actor CmxIrohRegistryContextProvider: CmxIrohClientContextProvider { guard discovery.routeContractVersion == 1 else { throw CmxIrohRegistryContextError.incompatibleContract } - guard Set(discovery.relayFleet) == managedRelayURLs else { + // Without a verified managed fleet there is nothing to cross-check and + // allowedRouteRelayURLs is empty, so no relay hint survives filtering; + // direct dial plans stay valid while relays remain unusable. + guard managedRelayURLs.isEmpty + || Set(discovery.relayFleet) == managedRelayURLs else { throw CmxIrohRegistryContextError.relayFleetMismatch } lanAuthorities.removeAll(keepingCapacity: false) @@ -668,6 +672,6 @@ public actor CmxIrohRegistryContextProvider: CmxIrohClientContextProvider { } private static func isConnectivity(_ error: any Error) -> Bool { - (error as? CmxIrohTrustBrokerClientError) == .connectivity + CmxIrohTrustBrokerClientError.preservesVerifiedPolicyDuringRefresh(error) } } diff --git a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohRelayCredentialCoordinator.swift b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohRelayCredentialCoordinator.swift index 56225402a0ae..edc1e88d34b6 100644 --- a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohRelayCredentialCoordinator.swift +++ b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohRelayCredentialCoordinator.swift @@ -37,6 +37,11 @@ public actor CmxIrohRelayCredentialCoordinator { private let retryJitter: @Sendable () -> Double private let automaticRefreshEnabled: Bool private let credentialDidInstall: @Sendable (CmxIrohRelayTokenResponse) async -> Void + private let rateLimitedDirective: (@Sendable (Int) async -> Void)? + /// Coordinator-owned earliest next mint. Every schedule and on-demand + /// refresh honors it, and every observed Retry-After extends it, so no + /// path can spend a guaranteed rejection inside the server's floor. + private var mintFloor: Date? private var binding: Binding? private var installedCredential: InstalledCredential? private var lifecycleRevision: UInt64 = 0 @@ -65,7 +70,8 @@ public actor CmxIrohRelayCredentialCoordinator { automaticRefreshEnabled: Bool = true, credentialDidInstall: @escaping @Sendable ( CmxIrohRelayTokenResponse - ) async -> Void = { _ in } + ) async -> Void = { _ in }, + rateLimitedDirective: (@Sendable (Int) async -> Void)? = nil ) { self.supervisor = supervisor self.broker = broker @@ -77,6 +83,28 @@ public actor CmxIrohRelayCredentialCoordinator { self.retryJitter = retryJitter self.automaticRefreshEnabled = automaticRefreshEnabled self.credentialDidInstall = credentialDidInstall + self.rateLimitedDirective = rateLimitedDirective + } + + /// Latest of the supplied deadline and the coordinator-owned mint floor. + private func flooredDeadline(_ deadline: Date?) -> Date? { + guard let mintFloor, mintFloor > clock.now() else { return deadline } + guard let deadline else { return mintFloor } + return max(deadline, mintFloor) + } + + /// Extends the mint floor from an observed Retry-After and forwards the + /// directive to the owner so it survives runtime teardown. + private func noteRateLimited(_ error: any Error) { + guard let seconds = (error as? CmxIrohTrustBrokerClientError)? + .retryAfterSeconds else { return } + let proposed = clock.now().addingTimeInterval(TimeInterval(seconds)) + if mintFloor.map({ proposed > $0 }) ?? true { + mintFloor = proposed + } + if let rateLimitedDirective { + Task { await rateLimitedDirective(seconds) } + } } /// Starts refresh scheduling for one exact registered endpoint binding. @@ -90,7 +118,8 @@ public actor CmxIrohRelayCredentialCoordinator { bindingID: String, endpointIdentity: CmxIrohPeerIdentity, bootstrap: CmxIrohRelayTokenResponse? = nil, - waitForInitialCredential: Bool = false + waitForInitialCredential: Bool = false, + mintNotBefore: Date? = nil ) async throws { lifecycleRevision &+= 1 let revision = lifecycleRevision @@ -100,6 +129,9 @@ public actor CmxIrohRelayCredentialCoordinator { let expectedBinding = Binding(id: bindingID, endpointIdentity: endpointIdentity) binding = expectedBinding installedCredential = nil + if let mintNotBefore { + mintFloor = [mintFloor, mintNotBefore].compactMap { $0 }.max() + } if let bootstrap { do { @@ -128,6 +160,22 @@ public actor CmxIrohRelayCredentialCoordinator { return } } + // A Retry-After floor from the caller's own bootstrap attempt covers + // this same endpoint; the first mint waits it out instead of spending + // a guaranteed rejection. + if let mintFloor, mintFloor > clock.now() { + if waitForInitialCredential { + try await installInitialCredentialAfterRetry( + binding: expectedBinding, + revision: revision, + firstRetry: mintFloor, + initialFailureCount: 0 + ) + } else { + startLoopIfEnabled(revision: revision, firstRefresh: mintFloor) + } + return + } do { let response = try await broker.issueRelayToken( bindingID: bindingID, @@ -143,6 +191,7 @@ public actor CmxIrohRelayCredentialCoordinator { guard isCurrent(revision), !Task.isCancelled else { throw CancellationError() } + noteRateLimited(error) let delay = retryDelay(failureCount: 0, error: error) let firstRetry = retryDeadline( now: clock.now(), @@ -173,7 +222,7 @@ public actor CmxIrohRelayCredentialCoordinator { firstRetry: Date?, initialFailureCount: Int ) async throws { - var deadline = firstRetry + var deadline = flooredDeadline(firstRetry) var failureCount = initialFailureCount while isCurrent(revision), !Task.isCancelled { if let deadline { @@ -198,6 +247,7 @@ public actor CmxIrohRelayCredentialCoordinator { guard isCurrent(revision), !Task.isCancelled else { throw CancellationError() } + noteRateLimited(error) let delay = retryDelay(failureCount: failureCount, error: error) deadline = retryDeadline( now: clock.now(), @@ -243,6 +293,13 @@ public actor CmxIrohRelayCredentialCoordinator { } guard automaticRefreshEnabled else { return } let now = clock.now() + if let mintFloor, now < mintFloor { + // Inside the server's floor an on-demand mint is a guaranteed + // rejection; keep the scheduled loop aimed at the floor instead. + refreshTask?.cancel() + startLoopIfEnabled(revision: lifecycleRevision, firstRefresh: mintFloor) + return + } if let installedCredential, now < installedCredential.refreshAfter, installedCredential.expiresAt.timeIntervalSince(now) @@ -262,6 +319,7 @@ public actor CmxIrohRelayCredentialCoordinator { throw CancellationError() } refreshTask?.cancel() + noteRateLimited(error) let delay = retryDelay(failureCount: 0, error: error) startLoopIfEnabled( revision: revision, @@ -283,6 +341,7 @@ public actor CmxIrohRelayCredentialCoordinator { initialFailureCount: Int = 0 ) { guard automaticRefreshEnabled else { return } + let firstRefresh = flooredDeadline(firstRefresh) refreshTask = Task { [weak self] in await self?.run( revision: revision, @@ -320,6 +379,7 @@ public actor CmxIrohRelayCredentialCoordinator { } catch { guard isCurrent(revision), !Task.isCancelled else { return } let now = clock.now() + noteRateLimited(error) let delay = retryDelay(failureCount: failureCount, error: error) deadline = retryDeadline( now: now, diff --git a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohRelayPolicyService.swift b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohRelayPolicyService.swift index c1335bf730a5..e27527cfdcea 100644 --- a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohRelayPolicyService.swift +++ b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohRelayPolicyService.swift @@ -36,15 +36,45 @@ public actor CmxIrohRelayPolicyService { trustRoot: CmxIrohRelayPolicyTrustRoot, now: Date = Date() ) async throws -> CmxIrohEffectiveRelayPolicy { + try await refreshWithCredential( + endpointID: endpointID, + accountID: accountID, + trustRoot: trustRoot, + now: now + ).effective + } + + /// One resolved bootstrap: the effective policy plus the broker-minted + /// relay credential from the same response, so activation can install the + /// credential without a second mint request. + public struct RefreshOutcome: Sendable { + public let effective: CmxIrohEffectiveRelayPolicy + public let relayCredential: CmxIrohRelayTokenResponse? + } + + /// Fetches and installs the broker's current relay bootstrap response, + /// returning the minted credential alongside the effective policy. + public func refreshWithCredential( + endpointID: CmxIrohPeerIdentity, + accountID: String, + trustRoot: CmxIrohRelayPolicyTrustRoot, + now: Date = Date() + ) async throws -> RefreshOutcome { guard let broker else { throw CmxIrohRelayPolicyServiceError.brokerUnavailable } let bootstrap = try await broker.issueRelayBootstrap(endpointID: endpointID) - return try await install( + let effective = try await install( response: bootstrap.relayPolicy, accountID: accountID, trustRoot: trustRoot, relayCredential: bootstrap.relayToken, now: now ) + return RefreshOutcome( + effective: effective, + // The resolution-validated credential, not the raw response: a + // rejected bootstrap must never displace a valid cached one. + relayCredential: effective.relayBootstrap + ) } /// Verifies and resolves one broker response without replacing last-known-good diff --git a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohTrustBrokerClient.swift b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohTrustBrokerClient.swift index ca757a0500fe..b48e06f60da1 100644 --- a/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohTrustBrokerClient.swift +++ b/Packages/Shared/CmuxIrohTransport/Sources/CmuxIrohTransport/CmxIrohTrustBrokerClient.swift @@ -86,11 +86,17 @@ public actor CmxIrohTrustBrokerClient: CmxIrohRelayPolicyServing { } } private struct BrokerError: Decodable { let error: String } + private struct RouteCooldown: Sendable { + let code: String? + let retryUntil: Date + } private let baseURL: URL private let tokenSource: CmxIrohBrokerTokenSource private let transport: any CmxIrohHTTPTransport private let requestTimeout: TimeInterval + private let now: @Sendable () -> Date + private var routeCooldowns: [String: RouteCooldown] = [:] /// Creates a client that rejects cleartext non-loopback API origins. public init( @@ -102,7 +108,8 @@ public actor CmxIrohTrustBrokerClient: CmxIrohRelayPolicyServing { baseURL: baseURL, tokenSource: tokenSource, transport: CmxIrohURLSessionTransport(), - requestTimeout: requestTimeout + requestTimeout: requestTimeout, + now: Date.init ) } @@ -111,7 +118,8 @@ public actor CmxIrohTrustBrokerClient: CmxIrohRelayPolicyServing { baseURL: URL, tokenSource: CmxIrohBrokerTokenSource, transport: any CmxIrohHTTPTransport, - requestTimeout: TimeInterval = 10 + requestTimeout: TimeInterval = 10, + now: @escaping @Sendable () -> Date = Date.init ) throws { guard Self.isAllowedBaseURL(baseURL), requestTimeout > 0 else { throw CmxIrohTrustBrokerClientError.invalidBaseURL @@ -120,6 +128,7 @@ public actor CmxIrohTrustBrokerClient: CmxIrohRelayPolicyServing { self.tokenSource = tokenSource self.transport = transport self.requestTimeout = requestTimeout + self.now = now } public func issueChallenge( @@ -264,6 +273,7 @@ public actor CmxIrohTrustBrokerClient: CmxIrohRelayPolicyServing { method: String, body: Data? ) async throws -> Response { + try applyRouteCooldown(path: path, method: method) let accessToken = await tokenSource.accessToken() let refreshToken = await tokenSource.refreshToken() guard let accessToken, let refreshToken else { @@ -303,6 +313,12 @@ public actor CmxIrohTrustBrokerClient: CmxIrohRelayPolicyServing { let retryAfterSeconds = Self.retryAfterSeconds( http.value(forHTTPHeaderField: "Retry-After") ) { + recordRouteCooldown( + path: path, + method: method, + code: code, + retryAfterSeconds: retryAfterSeconds + ) throw CmxIrohTrustBrokerClientError.rateLimited( code: code, retryAfterSeconds: retryAfterSeconds @@ -322,6 +338,36 @@ public actor CmxIrohTrustBrokerClient: CmxIrohRelayPolicyServing { } } + private func applyRouteCooldown(path: String, method: String) throws { + let key = routeCooldownKey(path: path, method: method) + guard let cooldown = routeCooldowns[key] else { return } + let remaining = cooldown.retryUntil.timeIntervalSince(now()) + guard remaining > 0 else { + routeCooldowns[key] = nil + return + } + throw CmxIrohTrustBrokerClientError.rateLimited( + code: cooldown.code, + retryAfterSeconds: max(1, Int(ceil(remaining))) + ) + } + + private func recordRouteCooldown( + path: String, + method: String, + code: String?, + retryAfterSeconds: Int + ) { + routeCooldowns[routeCooldownKey(path: path, method: method)] = RouteCooldown( + code: code, + retryUntil: now().addingTimeInterval(TimeInterval(retryAfterSeconds)) + ) + } + + private func routeCooldownKey(path: String, method: String) -> String { + "\(method.uppercased()) \(path)" + } + private static func isAllowedBaseURL(_ url: URL) -> Bool { guard let components = URLComponents(url: url, resolvingAgainstBaseURL: false), let scheme = components.scheme?.lowercased(), @@ -346,7 +392,7 @@ public actor CmxIrohTrustBrokerClient: CmxIrohRelayPolicyServing { !value.isEmpty, value.utf8.allSatisfy({ (48 ... 57).contains($0) }), let seconds = Int(value), - (1 ... 3_600).contains(seconds), + (1 ... 86_400).contains(seconds), String(seconds) == value else { return nil } diff --git a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohBrokerCooldownTests.swift b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohBrokerCooldownTests.swift new file mode 100644 index 000000000000..6ebaa7eaddca --- /dev/null +++ b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohBrokerCooldownTests.swift @@ -0,0 +1,115 @@ +import CMUXMobileCore +import Foundation +import Testing +@testable import CmuxIrohTransport + +@Suite +struct CmxIrohBrokerCooldownTests { + private let start = Date(timeIntervalSince1970: 1_782_000_000) + + @Test + func recordsAndReportsRemainingWholeSeconds() { + var cooldown = CmxIrohBrokerCooldown() + cooldown.record(accountID: "a", retryAfterSeconds: 600, now: start) + + let remaining = cooldown.remainingSeconds( + accountID: "a", + now: start.addingTimeInterval(0.5) + ) + #expect(remaining == 600) + #expect(cooldown.remainingSeconds( + accountID: "a", + now: start.addingTimeInterval(599.2) + ) == 1) + } + + @Test + func expiresOnReadAndStaysClear() { + var cooldown = CmxIrohBrokerCooldown() + cooldown.record(accountID: "a", retryAfterSeconds: 10, now: start) + + #expect(cooldown.remainingSeconds( + accountID: "a", + now: start.addingTimeInterval(10) + ) == nil) + #expect(cooldown.retryAt == nil) + #expect(cooldown.remainingSeconds(accountID: "a", now: start) == nil) + } + + @Test + func overlappingDirectivesKeepTheLaterFloor() { + var cooldown = CmxIrohBrokerCooldown() + cooldown.record(accountID: "a", retryAfterSeconds: 600, now: start) + cooldown.record(accountID: "a", retryAfterSeconds: 10, now: start.addingTimeInterval(5)) + + #expect(cooldown.remainingSeconds( + accountID: "a", + now: start.addingTimeInterval(5) + ) == 595) + } + + @Test + func laterDirectiveExtendsTheFloor() { + var cooldown = CmxIrohBrokerCooldown() + cooldown.record(accountID: "a", retryAfterSeconds: 10, now: start) + cooldown.record(accountID: "a", retryAfterSeconds: 600, now: start.addingTimeInterval(5)) + + #expect(cooldown.remainingSeconds( + accountID: "a", + now: start.addingTimeInterval(6) + ) == 599) + } + + @Test + func accountSwitchReplacesTheFloor() { + var cooldown = CmxIrohBrokerCooldown() + cooldown.record(accountID: "a", retryAfterSeconds: 600, now: start) + cooldown.record(accountID: "b", retryAfterSeconds: 30, now: start) + + #expect(cooldown.remainingSeconds(accountID: "a", now: start) == nil) + #expect(cooldown.remainingSeconds(accountID: "b", now: start) == 30) + } + + @Test + func scopedClearOnlyDropsMatchingAccount() { + var cooldown = CmxIrohBrokerCooldown() + cooldown.record(accountID: "a", retryAfterSeconds: 600, now: start) + + cooldown.clear(accountID: "b") + #expect(cooldown.remainingSeconds(accountID: "a", now: start) == 600) + + cooldown.clear(accountID: "a") + #expect(cooldown.remainingSeconds(accountID: "a", now: start) == nil) + } + + @Test + func nonPositiveDirectiveStillArmsOneSecond() { + var cooldown = CmxIrohBrokerCooldown() + cooldown.record(accountID: "a", retryAfterSeconds: 0, now: start) + #expect(cooldown.remainingSeconds(accountID: "a", now: start) == 1) + } + + @Test + func directiveSecondsPrefersRetryAfterAndDefaultsBare429() { + #expect(CmxIrohBrokerCooldown.directiveSeconds( + for: CmxIrohTrustBrokerClientError.rateLimited(code: nil, retryAfterSeconds: 600) + ) == 600) + #expect(CmxIrohBrokerCooldown.directiveSeconds( + for: CmxIrohTrustBrokerClientError.rejected(statusCode: 429, code: nil) + ) == CmxIrohBrokerCooldown.defaultRateLimitedSeconds) + #expect(CmxIrohBrokerCooldown.directiveSeconds( + for: CmxIrohTrustBrokerClientError.rejected(statusCode: 503, code: nil) + ) == nil) + #expect(CmxIrohBrokerCooldown.directiveSeconds( + for: CmxIrohTrustBrokerClientError.connectivity + ) == nil) + } + + @Test + func cooldownErrorProvidesRetryAfterAndFailureKind() { + let error = CmxIrohBrokerCooldownError(retryAfterSeconds: 42) + #expect(error.retryAfterSeconds == 42) + #expect(error.diagnosticFailureKind == .policyUnavailable) + #expect(CmxIrohBrokerCooldownError(retryAfterSeconds: 0).retryAfterSeconds == 1) + } +} diff --git a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohClientRuntimeEmptyFleetTests.swift b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohClientRuntimeEmptyFleetTests.swift new file mode 100644 index 000000000000..2283ce84da8a --- /dev/null +++ b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohClientRuntimeEmptyFleetTests.swift @@ -0,0 +1,54 @@ +import CMUXMobileCore +import Testing +@testable import CmuxIrohTransport + +@Suite +struct CmxIrohClientRuntimeEmptyFleetTests { + /// A device whose relay policy is unavailable (empty managed fleet) must + /// still activate for registration and direct paths instead of failing + /// the offline-policy expectation before any broker call. Field reports + /// showed this as an unclassified `endpointFailed` immediately after a + /// rate-limited policy refresh, which made reconnection impossible even + /// on the local network. + @Test + func startWithEmptyManagedFleetAndOfflineCacheActivatesForDirectPaths() async throws { + let fixture = try ClientRuntimeTestFixture() + let endpoint = TestIrohEndpoint(identity: fixture.endpointID) + let broker = TestIrohClientBroker( + binding: fixture.binding, + discovery: fixture.discovery, + relay: fixture.relayResponse() + ) + let recorder = ClientRuntimeTestRecorder() + let configuration = CmxIrohClientRuntimeConfiguration( + accountID: fixture.configuration.accountID, + deviceID: fixture.configuration.deviceID, + appInstanceID: fixture.configuration.appInstanceID, + tag: fixture.configuration.tag, + displayName: fixture.configuration.displayName, + identity: fixture.identity, + capabilities: fixture.configuration.capabilities, + managedRelayURLs: [] + ) + let runtime = try CmxIrohClientRuntime( + factory: TestIrohEndpointFactory(endpoints: [endpoint]), + broker: broker, + configuration: configuration, + pendingRevocations: fixture.pendingRevocations(), + offlinePolicyCache: CmxIrohClientOfflinePolicyCache( + secureStore: TestSecureCredentialStore() + ), + now: { fixture.now }, + handleBinding: { _, _ in + await recorder.recordBinding() + return true + }, + handleRelayCredential: { _, _ in await recorder.recordRelay() } + ) + + try await runtime.start() + + #expect(await runtime.snapshot().state == .active) + #expect(await broker.observedRegistrations().isEmpty == false) + } +} diff --git a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohClientRuntimeTests.swift b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohClientRuntimeTests.swift index 6ab15017bc7a..0847000ba821 100644 --- a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohClientRuntimeTests.swift +++ b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohClientRuntimeTests.swift @@ -148,6 +148,39 @@ struct CmxIrohClientRuntimeTests { await runtime.stop() } + @Test + func rateLimitedRegistrationStartsFromFreshAuthenticatedDiscovery() async throws { + let fixture = try ClientRuntimeTestFixture() + let endpoint = TestIrohEndpoint(identity: fixture.endpointID) + let broker = TestIrohClientBroker( + binding: fixture.binding, + discovery: fixture.discovery, + relay: fixture.relayResponse(), + registrationError: CmxIrohTrustBrokerClientError.rateLimited( + code: "device_registration_hour_quota", + retryAfterSeconds: 600 + ) + ) + let runtime = try CmxIrohClientRuntime( + factory: TestIrohEndpointFactory(endpoints: [endpoint]), + broker: broker, + configuration: fixture.configuration, + pendingRevocations: fixture.pendingRevocations(), + now: { fixture.now } + ) + + try await runtime.start() + + let snapshot = await runtime.snapshot() + #expect(snapshot.state == .active) + #expect(snapshot.endpointID == fixture.endpointID) + #expect(snapshot.bindingID == fixture.binding.bindingID) + #expect(await broker.observedRegistrations().count == 1) + #expect(await broker.observedDiscoveryCount() == 1) + #expect(await endpoint.observedCloseCallCount() == 0) + await runtime.stop() + } + @Test func rejectedCatalogPublicationCannotAdvanceLiveDiscoveryGeneration() async throws { let fixture = try ClientRuntimeTestFixture() diff --git a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRegistryContextProviderPolicyTests.swift b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRegistryContextProviderPolicyTests.swift index 8eccc8fcceb4..1ecade34990d 100644 --- a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRegistryContextProviderPolicyTests.swift +++ b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRegistryContextProviderPolicyTests.swift @@ -6,6 +6,35 @@ import Testing @testable import CmuxIrohTransport extension CmxIrohRegistryContextProviderTests { + /// With no verified managed fleet (relay policy unavailable) the discovery + /// fleet cannot be cross-checked; direct dial plans must still resolve so + /// LAN reconnects work while relays stay unusable. + @Test + func emptyManagedFleetStillResolvesDirectDialContext() async throws { + let fixture = try RegistryFixture() + let broker = TestIrohRegistryBroker( + discovery: try fixture.discovery( + targetHints: [], + relayFleet: [fixture.relayURL] + ), + pairGrantResponses: [try fixture.pairGrantResponse( + issuedAt: fixture.nowSeconds, + expiresAt: fixture.nowSeconds + 7 * 24 * 60 * 60 + )] + ) + let provider = CmxIrohRegistryContextProvider( + supervisor: try await fixture.activeSupervisor(), + broker: broker, + localBindingExpectation: try fixture.localExpectation(), + managedRelayURLs: [], + activeNetworkProfiles: { [] }, + now: { fixture.now } + ) + + let context = try await provider.context(for: fixture.request(hints: [])) + #expect(context.dialPlan.privateFallbackPaths.isEmpty) + } + @Test func discoveryMustPublishTheExactConfiguredRelayFleet() async throws { let fixture = try RegistryFixture() @@ -203,6 +232,55 @@ extension CmxIrohRegistryContextProviderTests { #expect(await broker.pairGrantRequestCount() == 1) } + @Test + func pairGrantRateLimitReusesCachedGrantAfterFreshDiscovery() async throws { + let fixture = try RegistryFixture() + let discovery = try fixture.discovery(targetHints: []) + let grant = try fixture.pairGrantResponse( + issuedAt: fixture.nowSeconds, + expiresAt: fixture.nowSeconds + 7 * 24 * 60 * 60 + ) + let store = TestSecureCredentialStore() + let cache = CmxIrohClientOfflinePolicyCache(secureStore: store) + let expectation = try fixture.offlineExpectation() + try await cache.save( + localBinding: discovery.bindings[0], + targetBinding: discovery.bindings[1], + discovery: discovery, + pairGrant: grant, + for: expectation, + now: fixture.now + ) + let broker = TestIrohRegistryBroker( + discovery: discovery, + pairGrantResponses: [], + pairGrantError: CmxIrohTrustBrokerClientError.rateLimited( + code: "pair_grant_hour_quota", + retryAfterSeconds: 600 + ) + ) + let provider = CmxIrohRegistryContextProvider( + supervisor: try await fixture.activeSupervisor(), + broker: broker, + localBindingExpectation: try fixture.localExpectation(), + managedRelayURLs: [fixture.relayURL], + activeNetworkProfiles: { [] }, + offlinePolicy: try CmxIrohClientOfflinePolicyContext( + cache: cache, + expectation: expectation, + localBinding: discovery.bindings[0] + ), + now: { fixture.now } + ) + + let context = try await provider.context(for: fixture.request(hints: [])) + + #expect(context.credential.pairGrantToken == grant.grant) + #expect(await broker.discoveryRequestCount() == 1) + #expect(await broker.pairGrantRequestCount() == 1) + #expect(await store.readCount() > 0) + } + @Test func authenticatedBrokerFailuresNeverConsultOfflinePolicy() async throws { let fixture = try RegistryFixture() diff --git a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRegistryContextProviderTests.swift b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRegistryContextProviderTests.swift index b2c9bb69d422..e93f458f4bf0 100644 --- a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRegistryContextProviderTests.swift +++ b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRegistryContextProviderTests.swift @@ -233,6 +233,7 @@ actor TestIrohRegistryBroker: CmxIrohRegistryServing { private var discoveryResponse: CmxIrohDiscoveryResponse private var responses: [CmxIrohPairGrantResponse] + private var discoveryRequests = 0 private var pairGrantRequests: [PairGrantRequest] = [] private let discoveryError: (any Error)? private let pairGrantError: (any Error)? @@ -250,6 +251,7 @@ actor TestIrohRegistryBroker: CmxIrohRegistryServing { } func discover() throws -> CmxIrohDiscoveryResponse { + discoveryRequests += 1 if let discoveryError { throw discoveryError } return discoveryResponse } @@ -275,6 +277,10 @@ actor TestIrohRegistryBroker: CmxIrohRegistryServing { pairGrantRequests } + func discoveryRequestCount() -> Int { + discoveryRequests + } + func pairGrantRequestCount() -> Int { pairGrantRequests.count } diff --git a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRelayCredentialCoordinatorTests.swift b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRelayCredentialCoordinatorTests.swift index 5d61ca30ac44..150ef7fda9d0 100644 --- a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRelayCredentialCoordinatorTests.swift +++ b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRelayCredentialCoordinatorTests.swift @@ -5,6 +5,84 @@ import Testing @Suite struct CmxIrohRelayCredentialCoordinatorTests { + /// A Retry-After floor from the caller's own bootstrap attempt covers the + /// same relay-token endpoint, so activation without a bootstrap must wait + /// out the floor instead of minting into a guaranteed 429. + @Test + func mintNotBeforeDefersTheFirstMintPastTheFloor() async throws { + let fixture = try RelayCoordinatorFixture() + let endpoint = TestIrohEndpoint(identity: fixture.identity) + let supervisor = try await fixture.activeSupervisor(endpoint: endpoint) + let broker = TestRelayTokenBroker(steps: []) + let clock = TestRelayClock(now: fixture.now) + var clockEvents = clock.events().makeAsyncIterator() + let coordinator = CmxIrohRelayCredentialCoordinator( + supervisor: supervisor, + broker: broker, + managedRelayURLs: Set(fixture.relayURLs), + clock: clock, + jitter: { _, refreshAfter in refreshAfter }, + retryJitter: { 0 }, + credentialDidInstall: { _ in } + ) + let floor = fixture.now.addingTimeInterval(600) + + try await coordinator.activate( + bindingID: fixture.bindingID, + endpointIdentity: fixture.identity, + mintNotBefore: floor + ) + + guard case let .sleep(deadline) = await clockEvents.next() else { + Issue.record("Expected the deferred mint sleep") + return + } + #expect(deadline == floor) + #expect(await broker.observedEndpointIDs().isEmpty) + await coordinator.deactivate() + #expect(await clockEvents.next() == .cancelled) + } + + /// Even a successfully installed bootstrap must not schedule its refresh + /// before the mint floor: the credential's refreshAfter can precede a + /// Retry-After the caller just received from the same endpoint. + @Test + func bootstrapRefreshScheduleNeverPrecedesTheMintFloor() async throws { + let fixture = try RelayCoordinatorFixture() + let endpoint = TestIrohEndpoint(identity: fixture.identity) + let supervisor = try await fixture.activeSupervisor(endpoint: endpoint) + let broker = TestRelayTokenBroker(steps: []) + let clock = TestRelayClock(now: fixture.now) + var clockEvents = clock.events().makeAsyncIterator() + let response = try fixture.response() + let coordinator = CmxIrohRelayCredentialCoordinator( + supervisor: supervisor, + broker: broker, + managedRelayURLs: Set(fixture.relayURLs), + clock: clock, + jitter: { _, refreshAfter in refreshAfter }, + retryJitter: { 0 }, + credentialDidInstall: { _ in } + ) + let floor = fixture.refreshAfter.addingTimeInterval(120) + + try await coordinator.activate( + bindingID: fixture.bindingID, + endpointIdentity: fixture.identity, + bootstrap: response, + mintNotBefore: floor + ) + + guard case let .sleep(deadline) = await clockEvents.next() else { + Issue.record("Expected the floored refresh sleep") + return + } + #expect(deadline == floor) + #expect(await broker.observedEndpointIDs().isEmpty) + await coordinator.deactivate() + #expect(await clockEvents.next() == .cancelled) + } + @Test func bootstrapInstallsCompleteFleetBeforeSleepingUntilRefresh() async throws { let fixture = try RelayCoordinatorFixture() diff --git a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRelayPolicyServiceRefreshTests.swift b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRelayPolicyServiceRefreshTests.swift new file mode 100644 index 000000000000..a7c047ebbf9b --- /dev/null +++ b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohRelayPolicyServiceRefreshTests.swift @@ -0,0 +1,106 @@ +import CMUXMobileCore +import Foundation +import Testing +@testable import CmuxIrohTransport + +private actor RefreshBootstrapBroker: CmxIrohRelayPolicyServing { + private let bootstrap: CmxIrohRelayBootstrapResponse + private(set) var bootstrapRequestCount = 0 + + init(bootstrap: CmxIrohRelayBootstrapResponse) { + self.bootstrap = bootstrap + } + + func issueRelayBootstrap( + endpointID _: CmxIrohPeerIdentity + ) async throws -> CmxIrohRelayBootstrapResponse { + bootstrapRequestCount += 1 + return bootstrap + } + + func relayPreference() async throws -> CmxIrohRelayPreferenceResponse { + throw CmxIrohRelayPolicyServiceError.brokerUnavailable + } + + func updateRelayPreference( + _: CmxIrohRelayPreferenceUpdateRequest + ) async throws -> CmxIrohRelayPreferenceResponse { + throw CmxIrohRelayPolicyServiceError.brokerUnavailable + } +} + +@Suite +struct CmxIrohRelayPolicyServiceRefreshTests { + @Test + func refreshWithCredentialReturnsMintedCredentialWithEffectivePolicy() async throws { + let fixture = RelayPolicyServiceTestFixture() + let credential = fixture.relayCredential() + let broker = RefreshBootstrapBroker( + bootstrap: CmxIrohRelayBootstrapResponse( + relayToken: credential, + relayPolicy: try CmxIrohRelayPolicyResponse( + policy: fixture.token(sequence: 1), + preference: .automatic, + preferenceRevision: 1 + ) + ) + ) + let service = CmxIrohRelayPolicyService( + policyCache: CmxIrohRelayPolicyCache(secureStore: TestSecureCredentialStore()), + preferenceStore: CmxIrohRelayPreferenceStore(secureStore: TestSecureCredentialStore()), + credentialStore: CmxIrohCustomRelayCredentialStore( + secureStore: TestSecureCredentialStore() + ), + broker: broker + ) + + let outcome = try await service.refreshWithCredential( + endpointID: try CmxIrohPeerIdentity( + endpointID: String(repeating: "a", count: 64) + ), + accountID: "account-a", + trustRoot: fixture.firstTrustRoot, + now: fixture.now + ) + + #expect(outcome.relayCredential == credential) + #expect(outcome.effective.endpointRelayProfile.allowedRelayURLs + == Set(fixture.relayURLs)) + #expect(await broker.bootstrapRequestCount == 1) + } + + @Test + func refreshDelegatesToRefreshWithCredential() async throws { + let fixture = RelayPolicyServiceTestFixture() + let broker = RefreshBootstrapBroker( + bootstrap: CmxIrohRelayBootstrapResponse( + relayToken: nil, + relayPolicy: try CmxIrohRelayPolicyResponse( + policy: fixture.token(sequence: 1), + preference: .automatic, + preferenceRevision: 1 + ) + ) + ) + let service = CmxIrohRelayPolicyService( + policyCache: CmxIrohRelayPolicyCache(secureStore: TestSecureCredentialStore()), + preferenceStore: CmxIrohRelayPreferenceStore(secureStore: TestSecureCredentialStore()), + credentialStore: CmxIrohCustomRelayCredentialStore( + secureStore: TestSecureCredentialStore() + ), + broker: broker + ) + + let effective = try await service.refresh( + endpointID: try CmxIrohPeerIdentity( + endpointID: String(repeating: "b", count: 64) + ), + accountID: "account-a", + trustRoot: fixture.firstTrustRoot, + now: fixture.now + ) + + #expect(effective.endpointRelayProfile.allowedRelayURLs == Set(fixture.relayURLs)) + #expect(await broker.bootstrapRequestCount == 1) + } +} diff --git a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohTrustBrokerClientNetworkTests.swift b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohTrustBrokerClientNetworkTests.swift index 07e5b5a9c178..dc6362d29dc2 100644 --- a/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohTrustBrokerClientNetworkTests.swift +++ b/Packages/Shared/CmuxIrohTransport/Tests/CmuxIrohTransportTests/CmxIrohTrustBrokerClientNetworkTests.swift @@ -10,11 +10,15 @@ extension CmxIrohTrustBrokerClientTests { code: "rate_limited", retryAfterSeconds: 600 )), + ("86400", CmxIrohTrustBrokerClientError.rateLimited( + code: "rate_limited", + retryAfterSeconds: 86_400 + )), ("0", CmxIrohTrustBrokerClientError.rejected( statusCode: 429, code: "rate_limited" )), - ("3601", CmxIrohTrustBrokerClientError.rejected( + ("86401", CmxIrohTrustBrokerClientError.rejected( statusCode: 429, code: "rate_limited" )), @@ -38,6 +42,51 @@ extension CmxIrohTrustBrokerClientTests { } } + @Test + func rateLimitSuppressesConcurrentSameRouteRequestsWithoutBlockingOtherRoutes() async throws { + let transport = RouteRecordingBrokerTransport(responsesByPath: [ + "/api/devices/iroh": [ + .json( + status: 429, + body: #"{"error":"rate_limited"}"#, + headers: ["Retry-After": "600"] + ), + ], + "/api/relay/preferences": [ + .json( + status: 200, + body: #"{"preference":{"mode":"automatic"},"preferenceRevision":0}"# + ), + ], + ]) + let client = try makeNetworkClient(transport: transport) + + await #expect(throws: CmxIrohTrustBrokerClientError.rateLimited( + code: "rate_limited", + retryAfterSeconds: 600 + )) { + _ = try await client.discover() + } + + await withTaskGroup(of: Void.self) { group in + for _ in 0 ..< 2 { + group.addTask { + do { + _ = try await client.discover() + Issue.record("Expected the active route cooldown to reject discovery") + } catch {} + } + } + } + + let preference = try await client.relayPreference() + #expect(preference.preference == .automatic) + #expect(await transport.requests().map { $0.url?.path } == [ + "/api/devices/iroh", + "/api/relay/preferences", + ]) + } + @Test func missingAuthFailsBeforeAnyNetworkRequest() async throws { let transport = RecordingBrokerTransport(responses: []) @@ -118,7 +167,7 @@ extension CmxIrohTrustBrokerClientTests { } private func makeNetworkClient( - transport: RecordingBrokerTransport + transport: any CmxIrohHTTPTransport ) throws -> CmxIrohTrustBrokerClient { try CmxIrohTrustBrokerClient( baseURL: #require(URL(string: "https://cmux.example")), @@ -132,3 +181,39 @@ extension CmxIrohTrustBrokerClientTests { refreshToken: { "refresh" } ) } + +private actor RouteRecordingBrokerTransport: CmxIrohHTTPTransport { + enum TestError: Error { + case invalidRequest + case unexpectedRequest(String) + } + + private var responsesByPath: [String: [RecordingBrokerTransport.Response]] + private var captured: [URLRequest] = [] + + init(responsesByPath: [String: [RecordingBrokerTransport.Response]]) { + self.responsesByPath = responsesByPath + } + + func data(for request: URLRequest) async throws -> (Data, URLResponse) { + guard let url = request.url else { throw TestError.invalidRequest } + captured.append(request) + guard var pending = responsesByPath[url.path], !pending.isEmpty else { + throw TestError.unexpectedRequest(url.path) + } + let response = pending.removeFirst() + responsesByPath[url.path] = pending + guard let http = HTTPURLResponse( + url: url, + statusCode: response.status, + httpVersion: nil, + headerFields: ["Content-Type": "application/json"] + .merging(response.headers) { _, new in new } + ) else { + throw TestError.invalidRequest + } + return (response.body, http) + } + + func requests() -> [URLRequest] { captured } +} diff --git a/Packages/iOS/CmuxMobileDiagnostics/Sources/CmuxMobileDiagnostics/MobileDebugLog.swift b/Packages/iOS/CmuxMobileDiagnostics/Sources/CmuxMobileDiagnostics/MobileDebugLog.swift index 5132346b389d..125d0aaff0d8 100644 --- a/Packages/iOS/CmuxMobileDiagnostics/Sources/CmuxMobileDiagnostics/MobileDebugLog.swift +++ b/Packages/iOS/CmuxMobileDiagnostics/Sources/CmuxMobileDiagnostics/MobileDebugLog.swift @@ -26,18 +26,43 @@ public struct MobileDebugLog: Sendable { sink: MobileDebugLogSink( fileURL: logFileURL, fileHeader: fileHeader, - installCrashCapture: true + installCrashCapture: true, + startsFileLoggingEnabled: Self.startsFileLoggingEnabled ) ) }() - /// Default DEBUG-build file location for the durable iOS debug log. + /// UserDefaults key for the user-visible verbose-log opt-in. Release + /// builds write to disk only while this is set; DEBUG builds always log. + public static let verboseLogDefaultsKey = "cmux.diagnostics.verbose-log-enabled" + + private static var startsFileLoggingEnabled: Bool { + #if DEBUG + true + #else + UserDefaults.standard.bool(forKey: verboseLogDefaultsKey) + #endif + } + + /// Turns durable file logging on or off for the shared sink and persists + /// the choice for the next launch. Returns whether the sink accepted it; + /// a failed enable is not persisted, so the toggle cannot claim to be + /// recording when no file could be opened. + @discardableResult + public func setFileLogging(enabled: Bool) async -> Bool { + let accepted = await sink.setFileLogging(enabled: enabled) + if accepted { + UserDefaults.standard.set(enabled, forKey: Self.verboseLogDefaultsKey) + } + return accepted + } + + /// File location for the durable iOS debug log. /// - /// DEBUG builds write to `Application Support/cmux-debug.log` inside the app - /// container. Release builds always return `nil`, so callers cannot enable - /// durable debug logging by accident. + /// All builds resolve `Application Support/cmux-debug.log` inside the app + /// container; whether anything is WRITTEN there is controlled separately + /// (always in DEBUG, only behind the user's verbose-log opt-in in Release). public static let logFileURL: URL? = { - #if DEBUG let fileManager = FileManager.default guard let applicationSupportURL = fileManager.urls( for: .applicationSupportDirectory, @@ -51,14 +76,13 @@ public struct MobileDebugLog: Sendable { withIntermediateDirectories: true ) let fileURL = applicationSupportURL.appendingPathComponent("cmux-debug.log") + #if DEBUG NSLog("cmux debug log file: %@", fileURL.path) + #endif return fileURL } catch { return nil } - #else - return nil - #endif }() /// The actor that owns the ring buffer and broadcast stream. diff --git a/Packages/iOS/CmuxMobileDiagnostics/Sources/CmuxMobileDiagnostics/MobileDebugLogSink.swift b/Packages/iOS/CmuxMobileDiagnostics/Sources/CmuxMobileDiagnostics/MobileDebugLogSink.swift index 81e945b8b6bb..defcf97b2dd9 100644 --- a/Packages/iOS/CmuxMobileDiagnostics/Sources/CmuxMobileDiagnostics/MobileDebugLogSink.swift +++ b/Packages/iOS/CmuxMobileDiagnostics/Sources/CmuxMobileDiagnostics/MobileDebugLogSink.swift @@ -47,7 +47,8 @@ public actor MobileDebugLogSink { fileURL: URL? = nil, fileHeader: String? = nil, maxFileBytes: Int = 5_000_000, - installCrashCapture: Bool = false + installCrashCapture: Bool = false, + startsFileLoggingEnabled: Bool = true ) { self.capacity = capacity self.now = now @@ -57,7 +58,9 @@ public actor MobileDebugLogSink { self.maxFileBytes = maxFileBytes self.fileBytesWritten = 0 self.crashCaptureInstalled = false - if let fileURL, let openedLogFile = Self.openLogFile(at: fileURL, header: fileHeader) { + if startsFileLoggingEnabled, + let fileURL, + let openedLogFile = Self.openLogFile(at: fileURL, header: fileHeader) { self.fileHandle = openedLogFile.fileHandle self.fileLoggingEnabled = true self.fileBytesWritten = openedLogFile.byteCount @@ -217,6 +220,22 @@ public actor MobileDebugLogSink { return true } + /// Turns durable file logging on or off at runtime. + /// + /// Enabling rotates any existing generation and opens a fresh file at the + /// sink's configured location; it reports `false` when the sink has no + /// file location or the file cannot be opened. Disabling closes the handle + /// and stops writes while the on-disk generations remain exportable. + @discardableResult + public func setFileLogging(enabled: Bool) -> Bool { + if enabled { + if fileLoggingEnabled { return true } + return rotateLogFile() + } + disableFileLogging() + return true + } + private func disableFileLogging() { fileLoggingEnabled = false closeFileHandle() diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift index 04f07dc90b1f..a5108b90df3c 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite+ConnectionRecovery.swift @@ -35,6 +35,11 @@ extension MobileShellComposite { // connection so a moving network repaints instead of going stale. for await _ in reachability.pathChanges() { guard let self, !Task.isCancelled else { return } + let isOnline = await reachability.isOnline + self.diagnosticLog?.record(DiagnosticEvent( + .reachabilityChanged, + a: isOnline ? 1 : 0 + )) self.recoverMobileConnection(trigger: .networkChange) } } @@ -171,7 +176,8 @@ extension MobileShellComposite { diagnosticLog?.record(DiagnosticEvent( .recoveryStarted, a: activeRoute.map { DiagnosticTransportKind($0.kind).rawValue } - ?? DiagnosticTransportKind.unknown.rawValue + ?? DiagnosticTransportKind.unknown.rawValue, + b: trigger.diagnosticCode )) applyConnectionRecoveryOwnerState() let stackUserID = lastReconnectStackUserID ?? identityProvider?.currentUserID diff --git a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift index cffd1fc76a33..a6fe6447b5d1 100644 --- a/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift +++ b/Packages/iOS/CmuxMobileShell/Sources/CmuxMobileShell/MobileShellComposite.swift @@ -1587,6 +1587,23 @@ public final class MobileShellComposite: MobileTerminalOutputSinking { var reschedulesSecondaryAggregation: Bool { self != .presencePush } + /// Stable integer carried in ``DiagnosticEventCode/recoveryStarted``'s + /// `b` slot so an export names WHY each recovery cycle began. Values + /// are append-only; never renumber. + var diagnosticCode: Int { + switch self { + case .networkChange: 1 + case .manual: 2 + case .presencePush: 3 + case .foreground: 4 + case .liveness: 5 + case .eventStreamEnded: 6 + case .subscriptionStartFailed: 7 + case .transportWriteTimedOut: 8 + case .automaticBackoffExpired: 9 + } + } + var description: String { switch self { case .networkChange: return "networkChange" diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/MobileIrohSettingsModel.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/MobileIrohSettingsModel.swift index d94c08f0667c..c63c231fab37 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/MobileIrohSettingsModel.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/MobileIrohSettingsModel.swift @@ -1,5 +1,7 @@ #if os(iOS) import CMUXMobileCore +import CmuxMobileDiagnostics +import Foundation import Observation @MainActor @@ -13,8 +15,26 @@ final class MobileIrohSettingsModel { private(set) var testResults: [String: CmxIrohRelayTestResult] = [:] private(set) var diagnosticReport = DiagnosticReport.empty private(set) var diagnosticExportText = "" + private(set) var verboseLogEnabled = UserDefaults.standard.bool( + forKey: MobileDebugLog.verboseLogDefaultsKey + ) private var diagnosticReloadGeneration: UInt64 = 0 + /// The durable verbose log file, offered for sharing once it exists. + var verboseLogShareURL: URL? { + guard let url = MobileDebugLog.logFileURL, + FileManager.default.fileExists(atPath: url.path) else { return nil } + return url + } + + func setVerboseLog(_ enabled: Bool) async { + verboseLogEnabled = enabled + let accepted = await MobileDebugLog.shared.setFileLogging(enabled: enabled) + if !accepted { + verboseLogEnabled = false + } + } + init(controller: any CmxIrohSettingsControlling) { self.controller = controller } @@ -119,11 +139,16 @@ final class MobileIrohSettingsModel { diagnosticReloadGeneration &+= 1 let generation = diagnosticReloadGeneration let report = await controller.irohDiagnosticReport() + let previous = await controller.irohPreviousLaunchDiagnosticReport() guard generation == diagnosticReloadGeneration else { return } diagnosticReport = report - diagnosticExportText = report.events.isEmpty - ? "" - : String(decoding: report.compactExport(), as: UTF8.self) + // The export carries the previous launch's archived block first so a + // drop that happened before a relaunch stays in the shared timeline. + let blocks = [previous, report].compactMap { block -> String? in + guard let block, !block.events.isEmpty else { return nil } + return String(decoding: block.compactExport(), as: UTF8.self) + } + diagnosticExportText = blocks.joined(separator: "\n") } } #endif diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/MobileIrohSettingsView.swift b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/MobileIrohSettingsView.swift index e78b60fd9b8e..4e4e8d87cca0 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/MobileIrohSettingsView.swift +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/MobileIrohSettingsView.swift @@ -137,6 +137,11 @@ struct MobileIrohSettingsView: View { eventCount: model.diagnosticReport.events.count, exportText: model.diagnosticExportText, needsAttention: !model.snapshot.staleRelayIDs.isEmpty || model.snapshot.failureDescription != nil, + verboseLogEnabled: model.verboseLogEnabled, + verboseLogShareURL: model.verboseLogShareURL, + setVerboseLog: { enabled in + Task { await model.setVerboseLog(enabled) } + }, refresh: model.refresh, clear: { Task { await model.clearDiagnosticReport() } @@ -447,6 +452,9 @@ private struct MobileIrohDiagnosticsSection: View { let eventCount: Int let exportText: String let needsAttention: Bool + let verboseLogEnabled: Bool + let verboseLogShareURL: URL? + let setVerboseLog: (Bool) -> Void let refresh: () -> Void let clear: () -> Void @@ -513,6 +521,37 @@ private struct MobileIrohDiagnosticsSection: View { .disabled(exportText.isEmpty) .accessibilityIdentifier("MobileIrohShareDiagnosticReport") + Toggle(isOn: Binding( + get: { verboseLogEnabled }, + set: setVerboseLog + )) { + Text(L10n.string( + "mobile.iroh.diagnostics.verboseLog", + defaultValue: "Verbose Connection Log" + )) + } + .accessibilityIdentifier("MobileIrohVerboseLogToggle") + if verboseLogEnabled { + Text(L10n.string( + "mobile.iroh.diagnostics.verboseLog.footer", + defaultValue: "Records detailed connection activity to a file on this device for troubleshooting. Terminal contents and credentials are never written." + )) + .font(.footnote) + .foregroundStyle(.secondary) + } + if let verboseLogShareURL { + ShareLink(item: verboseLogShareURL) { + Label( + L10n.string( + "mobile.iroh.diagnostics.shareVerboseLog", + defaultValue: "Share Verbose Log" + ), + systemImage: "doc.text" + ) + } + .accessibilityIdentifier("MobileIrohShareVerboseLog") + } + Button(role: .destructive) { showsClearConfirmation = true } label: { diff --git a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/Resources/Localizable.xcstrings b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/Resources/Localizable.xcstrings index f844cca556ed..6b783514c618 100644 --- a/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/Resources/Localizable.xcstrings +++ b/Packages/iOS/CmuxMobileShellUI/Sources/CmuxMobileShellUI/Resources/Localizable.xcstrings @@ -830,6 +830,27 @@ "ja" : { "stringUnit" : { "state" : "translated", "value" : "安全なレポートを共有" } } } }, + "mobile.iroh.diagnostics.shareVerboseLog" : { + "extractionState" : "manual", + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Share Verbose Log" } }, + "ja" : { "stringUnit" : { "state" : "translated", "value" : "詳細ログを共有" } } + } + }, + "mobile.iroh.diagnostics.verboseLog" : { + "extractionState" : "manual", + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Verbose Connection Log" } }, + "ja" : { "stringUnit" : { "state" : "translated", "value" : "詳細な接続ログ" } } + } + }, + "mobile.iroh.diagnostics.verboseLog.footer" : { + "extractionState" : "manual", + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Records detailed connection activity to a file on this device for troubleshooting. Terminal contents and credentials are never written." } }, + "ja" : { "stringUnit" : { "state" : "translated", "value" : "トラブルシューティングのため、接続の詳細な動作をこのデバイス上のファイルに記録します。ターミナルの内容や認証情報は書き込まれません。" } } + } + }, "mobile.iroh.diagnostics.failure.none" : { "extractionState" : "manual", "localizations" : { diff --git a/Sources/Mobile/MobileHostIrohRuntime+Activation.swift b/Sources/Mobile/MobileHostIrohRuntime+Activation.swift index 918ff39c79d2..672523db21da 100644 --- a/Sources/Mobile/MobileHostIrohRuntime+Activation.swift +++ b/Sources/Mobile/MobileHostIrohRuntime+Activation.swift @@ -7,6 +7,16 @@ import Foundation @MainActor extension MobileHostIrohRuntime { func activate(accountID: String, revision: UInt64) async throws { + if let remaining = brokerCooldown.remainingSeconds( + accountID: accountID, + now: Date() + ) { + diagnosticLog.record(DiagnosticEvent( + .retryScheduled, + ms: UInt32(clamping: remaining * 1_000) + )) + throw CmxIrohBrokerCooldownError(retryAfterSeconds: remaining) + } guard let auth else { throw CmxIrohHostRuntimeError.inactive } let tag = Self.currentTag() let appInstanceID = try await appInstances.appInstanceID( @@ -107,6 +117,7 @@ extension MobileHostIrohRuntime { let managedRelayURLs: Set let resolvedPolicyService: CmxIrohRelayPolicyService? let resolvedEffectivePolicy: CmxIrohEffectiveRelayPolicy? + var freshRelayCredential: CmxIrohRelayTokenResponse? if let relayPolicyTrustRoot { let service = CmxIrohRelayPolicyService( policyCache: relayPolicyCache, @@ -117,14 +128,17 @@ extension MobileHostIrohRuntime { let effective: CmxIrohEffectiveRelayPolicy diagnosticLog.record(DiagnosticEvent(.relayPolicyRefreshStarted)) do { - effective = try await service.refresh( + let outcome = try await service.refreshWithCredential( endpointID: derivedEndpointID, accountID: accountID, trustRoot: relayPolicyTrustRoot, now: Date() ) + effective = outcome.effective + freshRelayCredential = outcome.relayCredential diagnosticLog.record(DiagnosticEvent(.relayPolicyRefreshSucceeded)) } catch { + recordBrokerCooldown(for: error, accountID: accountID) diagnosticLog.record(DiagnosticEvent( .relayPolicyRefreshFailed, b: Self.diagnosticFailureKind(for: error).rawValue @@ -162,6 +176,9 @@ extension MobileHostIrohRuntime { let compatibleCachedRelay = cachedRelay.flatMap { relay in Set(relay.relayFleet) == managedRelayURLs ? relay : nil } + let freshCompatibleRelay = freshRelayCredential.flatMap { relay in + Set(relay.relayFleet) == managedRelayURLs ? relay : nil + } let configuration = CmxIrohHostRuntimeConfiguration( accountID: accountID, deviceID: deviceID, @@ -179,7 +196,7 @@ extension MobileHostIrohRuntime { ), managedRelayURLs: managedRelayURLs, endpointRelayProfile: endpointRelayProfile, - cachedRelayCredential: compatibleCachedRelay, + cachedRelayCredential: freshCompatibleRelay ?? compatibleCachedRelay, cachedHostPolicy: cachedHostPolicy ) let credentialRepository = brokerCredentials @@ -367,6 +384,7 @@ extension MobileHostIrohRuntime { runtime = hostRuntime activeAccountID = accountID activeAppInstanceID = appInstanceID + brokerCooldown.clear(accountID: accountID) diagnosticLog.record(DiagnosticEvent( .endpointActive, a: DiagnosticTransportKind.iroh.rawValue diff --git a/Sources/Mobile/MobileHostIrohRuntime.swift b/Sources/Mobile/MobileHostIrohRuntime.swift index 92540465b668..a9726879e139 100644 --- a/Sources/Mobile/MobileHostIrohRuntime.swift +++ b/Sources/Mobile/MobileHostIrohRuntime.swift @@ -115,6 +115,7 @@ final class MobileHostIrohRuntime { var signOutPreparationTask: Task? var signOutPreparationRevision: UInt64 = 0 var lifecycleRevision: UInt64 = 0 + var brokerCooldown = CmxIrohBrokerCooldown() private init() { diagnosticLog = DiagnosticLog( @@ -268,6 +269,7 @@ final class MobileHostIrohRuntime { } catch is CancellationError { return } catch { + recordBrokerCooldown(for: error, accountID: targetAccountID) diagnosticLog.record(DiagnosticEvent( .endpointFailed, a: DiagnosticTransportKind.iroh.rawValue, @@ -279,6 +281,17 @@ final class MobileHostIrohRuntime { } } + func recordBrokerCooldown(for error: any Error, accountID: String) { + guard let retryAfterSeconds = CmxIrohBrokerCooldown.directiveSeconds( + for: error + ) else { return } + brokerCooldown.record( + accountID: accountID, + retryAfterSeconds: retryAfterSeconds, + now: Date() + ) + } + nonisolated static func diagnosticFailureKind( for error: any Error ) -> DiagnosticFailureKind { diff --git a/ios/cmux/AppCompositionRoot.swift b/ios/cmux/AppCompositionRoot.swift index a2162c4c8e28..53d8f0ddfc90 100644 --- a/ios/cmux/AppCompositionRoot.swift +++ b/ios/cmux/AppCompositionRoot.swift @@ -183,6 +183,10 @@ final class AppCompositionRoot { } emitter.capture("ios_app_foregrounded", foregroundProps) hasForegrounded = true + case .inactive: + // The switcher opened; a swipe-kill from here may skip the + // background transition entirely, so snapshot diagnostics now. + iroh.archiveDiagnostics() case .background: iroh.didEnterBackground() let now = Date() @@ -200,8 +204,6 @@ final class AppCompositionRoot { } // Force a flush before the OS may suspend us, so queued events survive. Task { await emitter.flush() } - case .inactive: - break @unknown default: break } diff --git a/ios/cmuxPackage/Sources/cmuxFeature/MobileIrohRuntimeComposition.swift b/ios/cmuxPackage/Sources/cmuxFeature/MobileIrohRuntimeComposition.swift index d56892e5e6cb..a0bc7d653558 100644 --- a/ios/cmuxPackage/Sources/cmuxFeature/MobileIrohRuntimeComposition.swift +++ b/ios/cmuxPackage/Sources/cmuxFeature/MobileIrohRuntimeComposition.swift @@ -152,6 +152,9 @@ public final class MobileIrohRuntimeComposition: private var observedAuthState: MobileIrohAuthState? private var observedAccountID: String? { observedAuthState?.accountID } private var activeAccountID: String? + private var brokerCooldown = CmxIrohBrokerCooldown() + private let diagnosticArchive = DiagnosticReportArchive.defaultArchive() + private var previousLaunchDiagnosticReport: DiagnosticReport?? private var lastKnownBindingAccountID: String? private var lastKnownBindingTag: String? private var lastKnownBindingID: String? @@ -487,7 +490,7 @@ public final class MobileIrohRuntimeComposition: for request: CmxByteTransportRequest ) async throws -> any CmxByteTransport { await prepareForConnection() - guard let runtime else { throw CmxIrohClientRuntimeError.inactive } + let runtime = try runtimeForDial() return try runtime.transportFactory.makeTransport(for: request) } @@ -504,7 +507,7 @@ public final class MobileIrohRuntimeComposition: priority: Int32 ) async throws -> CmxIrohBidirectionalStream { await prepareForConnection() - guard let runtime else { throw CmxIrohClientRuntimeError.inactive } + let runtime = try runtimeForDial() return try await runtime.openBidirectionalLane( for: request, lane: lane, @@ -560,14 +563,80 @@ public final class MobileIrohRuntimeComposition: for request: CmxByteTransportRequest ) async throws -> CmxIndependentEventByteStream { await prepareForConnection() - guard let runtime else { throw CmxIrohClientRuntimeError.inactive } + let runtime = try runtimeForDial() return try await runtime.serverEventByteStream(for: request) } + private func runtimeForDial() throws -> CmxIrohClientRuntime { + if let runtime { return runtime } + if let accountID = observedAccountID ?? activeAccountID, + let remaining = brokerCooldown.remainingSeconds( + accountID: accountID, + now: now() + ) { + throw CmxIrohBrokerCooldownError(retryAfterSeconds: remaining) + } + throw CmxIrohClientRuntimeError.inactive + } + + private func recordBrokerCooldown(for error: any Error, accountID: String) { + // The gate's own synthesized error echoes the remaining floor; treating + // it as a fresh directive would slide the deadline forward on every + // sub-second attempt. + guard !(error is CmxIrohBrokerCooldownError), + let retryAfterSeconds = CmxIrohBrokerCooldown.directiveSeconds( + for: error + ) else { return } + brokerCooldown.record( + accountID: accountID, + retryAfterSeconds: retryAfterSeconds, + now: now() + ) + } + /// Preserves the endpoint when iOS backgrounds the scene. + /// Archives the diagnostic ring without touching the runtime. Called on + /// scene inactivation (the app switcher opening) so a force-quit that + /// never delivers a background transition still leaves the previous + /// launch's events exportable. + public func archiveDiagnostics() { + diagnosticLog?.record(DiagnosticEvent( + .appLifecycleChanged, + a: DiagnosticAppLifecyclePhase.inactive.rawValue + )) + persistDiagnosticsSnapshot() + } + + /// Reads the previous launch's archive (once) and replaces it with the + /// current ring, off the main actor: backgrounding must not spend the + /// suspension window on filesystem work. + private func persistDiagnosticsSnapshot() { + guard let diagnosticLog, let diagnosticArchive else { return } + let needsPreviousLoad = previousLaunchDiagnosticReport == nil + Task.detached(priority: .utility) { [weak self] in + let previous = needsPreviousLoad ? diagnosticArchive.load() : nil + if needsPreviousLoad { + await self?.cachePreviousLaunchReport(previous) + } + diagnosticArchive.save(await diagnosticLog.snapshot()) + } + } + + private func cachePreviousLaunchReport(_ report: DiagnosticReport?) { + guard previousLaunchDiagnosticReport == nil else { return } + previousLaunchDiagnosticReport = .some(report) + } + public func didEnterBackground() { + diagnosticLog?.record(DiagnosticEvent( + .appLifecycleChanged, + a: DiagnosticAppLifecyclePhase.background.rawValue + )) guard signOutPhase.allowsLifecycle else { return } sceneTransitionTask?.cancel() + // Archive the diagnostic ring so a later relaunch keeps the events + // around a drop exportable. + persistDiagnosticsSnapshot() let runtime = runtime sceneTransitionTask = Task { await runtime?.didEnterBackground() @@ -576,6 +645,10 @@ public final class MobileIrohRuntimeComposition: /// Health-checks and refreshes the preserved endpoint on foreground return. public func didBecomeActive() { + diagnosticLog?.record(DiagnosticEvent( + .appLifecycleChanged, + a: DiagnosticAppLifecyclePhase.active.rawValue + )) guard signOutPhase.allowsLifecycle else { return } sceneTransitionTask?.cancel() let auth = auth @@ -678,6 +751,9 @@ public final class MobileIrohRuntimeComposition: selectedPathObservationTask?.cancel() selectedPathObservationTask = nil activeAccountID = nil + brokerCooldown.clear() + diagnosticArchive?.clear() + previousLaunchDiagnosticReport = .some(nil) let fallbackBindingID = lastKnownBindingID let preparation: CmxIrohClientSignOutPreparation if let previousRuntime { @@ -1109,6 +1185,11 @@ public final class MobileIrohRuntimeComposition: selectedPathObservationTask?.cancel() selectedPathObservationTask = nil activeAccountID = nil + if shouldErase { + brokerCooldown.clear() + diagnosticArchive?.clear() + previousLaunchDiagnosticReport = .some(nil) + } await lanPeerDiscovery?.stop() if let previousRuntime { if shouldErase { @@ -1151,6 +1232,7 @@ public final class MobileIrohRuntimeComposition: } catch is CancellationError { return } catch { + recordBrokerCooldown(for: error, accountID: targetAccountID) diagnosticLog?.record(DiagnosticEvent( .endpointFailed, a: DiagnosticTransportKind.iroh.rawValue, @@ -1163,6 +1245,16 @@ public final class MobileIrohRuntimeComposition: } private func activate(accountID: String, revision: UInt64) async throws { + if let remaining = brokerCooldown.remainingSeconds( + accountID: accountID, + now: now() + ) { + diagnosticLog?.record(DiagnosticEvent( + .retryScheduled, + ms: UInt32(clamping: remaining * 1_000) + )) + throw CmxIrohBrokerCooldownError(retryAfterSeconds: remaining) + } guard let auth else { throw CmxIrohClientRuntimeError.inactive } let appInstanceID = try await appInstances.appInstanceID( accountID: accountID, @@ -1235,6 +1327,7 @@ public final class MobileIrohRuntimeComposition: let managedRelayURLs: Set let resolvedPolicyService: CmxIrohRelayPolicyService? let resolvedEffectivePolicy: CmxIrohEffectiveRelayPolicy? + var freshRelayCredential: CmxIrohRelayTokenResponse? if let relayPolicyTrustRoot { let service = CmxIrohRelayPolicyService( policyCache: relayPolicyCache, @@ -1245,14 +1338,17 @@ public final class MobileIrohRuntimeComposition: let effective: CmxIrohEffectiveRelayPolicy diagnosticLog?.record(DiagnosticEvent(.relayPolicyRefreshStarted)) do { - effective = try await service.refresh( + let outcome = try await service.refreshWithCredential( endpointID: endpointID, accountID: accountID, trustRoot: relayPolicyTrustRoot, now: now() ) + effective = outcome.effective + freshRelayCredential = outcome.relayCredential diagnosticLog?.record(DiagnosticEvent(.relayPolicyRefreshSucceeded)) } catch { + recordBrokerCooldown(for: error, accountID: accountID) diagnosticLog?.record(DiagnosticEvent( .relayPolicyRefreshFailed, b: Self.diagnosticFailureKind(for: error).rawValue @@ -1290,6 +1386,13 @@ public final class MobileIrohRuntimeComposition: let compatibleCachedRelay = cachedRelay.flatMap { relay in Set(relay.relayFleet) == managedRelayURLs ? relay : nil } + let freshCompatibleRelay = freshRelayCredential.flatMap { relay in + Set(relay.relayFleet) == managedRelayURLs ? relay : nil + } + let relayCredentialMintNotBefore = brokerCooldown.remainingSeconds( + accountID: accountID, + now: now() + ).map { now().addingTimeInterval(TimeInterval($0)) } let configuration = CmxIrohClientRuntimeConfiguration( accountID: accountID, deviceID: deviceID, @@ -1300,7 +1403,8 @@ public final class MobileIrohRuntimeComposition: capabilities: Self.capabilities, managedRelayURLs: managedRelayURLs, endpointRelayProfile: endpointRelayProfile, - cachedRelayCredential: compatibleCachedRelay + cachedRelayCredential: freshCompatibleRelay ?? compatibleCachedRelay, + relayCredentialMintNotBefore: relayCredentialMintNotBefore ) let credentialRepository = brokerCredentials let routeCatalog = routeCatalog @@ -1315,6 +1419,13 @@ public final class MobileIrohRuntimeComposition: factory: endpointFactoryProvider(transportVerificationMode), broker: broker, configuration: configuration, + handleRelayRateLimit: { @MainActor [weak self] seconds in + self?.brokerCooldown.record( + accountID: accountID, + retryAfterSeconds: seconds, + now: self?.now() ?? Date() + ) + }, pendingRevocations: pendingRevocations, protocolConfiguration: Self.protocolConfiguration( for: transportVerificationMode @@ -1455,6 +1566,14 @@ public final class MobileIrohRuntimeComposition: } self.runtime = runtime activeAccountID = accountID + // The floor survives a successful direct-only activation: live dials + // bypass the gate through the active runtime, while a torn-down + // runtime must not reach the rate-limited broker before Retry-After + // expires. It also seeds the policy refresh scheduler below. + let pendingRelayRetryAfterSeconds = brokerCooldown.remainingSeconds( + accountID: accountID, + now: now() + ) diagnosticLog?.record(DiagnosticEvent(.endpointActive, a: DiagnosticTransportKind.iroh.rawValue)) relayPolicyService = resolvedPolicyService relayPolicyEffective = resolvedEffectivePolicy @@ -1475,7 +1594,8 @@ public final class MobileIrohRuntimeComposition: accountID: accountID, endpointID: endpointID, trustRoot: relayPolicyTrustRoot, - revision: revision + revision: revision, + initialRetryAfterSeconds: pendingRelayRetryAfterSeconds ) publishIrohSettingsUpdate() } @@ -2017,9 +2137,18 @@ extension MobileIrohRuntimeComposition: CmxIrohSettingsControlling { public func clearIrohDiagnosticReport() async { await diagnosticLog?.clear() + diagnosticArchive?.clear() + previousLaunchDiagnosticReport = .some(nil) publishIrohSettingsUpdate() } + public func irohPreviousLaunchDiagnosticReport() async -> DiagnosticReport? { + if let cached = previousLaunchDiagnosticReport { return cached } + let loaded = diagnosticArchive?.load() + previousLaunchDiagnosticReport = .some(loaded) + return loaded + } + private func observeRelayPolicyDiagnostics( service: CmxIrohRelayPolicyService?, accountID: String, @@ -2075,15 +2204,24 @@ extension MobileIrohRuntimeComposition: CmxIrohSettingsControlling { accountID: String, endpointID: CmxIrohPeerIdentity, trustRoot: CmxIrohRelayPolicyTrustRoot?, - revision: UInt64 + revision: UInt64, + initialRetryAfterSeconds: Int? = nil ) { relayPolicyRefreshTask?.cancel() guard let service, let trustRoot else { relayPolicyRefreshTask = nil return } + // A Retry-After from the activation-time bootstrap failure seeds the + // first attempt so the scheduler's default cadence cannot retry the + // rate-limited endpoint early. The credential coordinator owns the + // floor-expiry mint; this refresh follows staggered so the two + // consumers of the same endpoint never wake together. + let initialRetryAt = initialRetryAfterSeconds.map { + now().addingTimeInterval(TimeInterval($0) + 30) + } relayPolicyRefreshTask = Task { @MainActor [weak self] in - var retryAt: Date? + var retryAt: Date? = initialRetryAt var failureCount = 0 var relayAuthorityExpired = false while !Task.isCancelled { diff --git a/ios/cmuxPackage/Tests/cmuxFeatureTests/MobileIrohRuntimeCompositionCooldownTests.swift b/ios/cmuxPackage/Tests/cmuxFeatureTests/MobileIrohRuntimeCompositionCooldownTests.swift new file mode 100644 index 000000000000..fb80d2b99ce0 --- /dev/null +++ b/ios/cmuxPackage/Tests/cmuxFeatureTests/MobileIrohRuntimeCompositionCooldownTests.swift @@ -0,0 +1,902 @@ +import AuthenticationServices +import CMUXAuthCore +import CMUXMobileCore +import CmuxAuthRuntime +import CmuxIrohTransport +import CryptoKit +import Foundation +import Synchronization +import Testing + +@testable import cmuxFeature + +@MainActor +@Suite("Mobile Iroh runtime composition broker cooldown", .serialized) +struct MobileIrohRuntimeCompositionCooldownTests { + /// Re-drives the lifecycle until the broker fake has seen activity (or the + /// runtime activated). Auth observation and reconcile coalesce across + /// main-actor tasks, so a single prepareForConnection can settle before + /// the first activation lands; short bounded sleeps (max ~5s) let every + /// executor drain between attempts. + private func settleActivation( + _ fixture: MobileIrohCooldownFixture, + until condition: @escaping () async -> Bool + ) async { + for _ in 0 ..< 500 { + if await condition() { return } + try? await Task.sleep(nanoseconds: 10_000_000) + await fixture.composition.prepareForConnection() + } + } + + @Test + func rateLimitFloorsActivationAndSurfacesRetryAfter() async throws { + let fixture = try await MobileIrohCooldownFixture.make( + registrationError: CmxIrohTrustBrokerClientError.rateLimited( + code: nil, + retryAfterSeconds: 600 + ) + ) + + await settleActivation(fixture) { + await fixture.broker.totalRequestCount() >= 1 + } + let flooredRequestCount = await fixture.broker.totalRequestCount() + #expect(flooredRequestCount >= 1) + #expect((await fixture.diagnosticLog.snapshot()).events.contains { + $0.code == .endpointFailed + }) + + // Within the Retry-After floor every further attempt must be free. + await fixture.composition.prepareForConnection() + await fixture.composition.prepareForConnection() + #expect(await fixture.broker.totalRequestCount() == flooredRequestCount) + + let dialError: any Error + do { + _ = try await fixture.composition.transport(for: fixture.request) + Issue.record("Expected transport creation to fail while activation is cooling down") + return + } catch { + dialError = error + } + #expect(await fixture.broker.totalRequestCount() == flooredRequestCount) + #expect((dialError as? any CmxRetryAfterProviding)?.retryAfterSeconds ?? 0 > 0) + + fixture.clock.advance(by: 601) + await settleActivation(fixture) { + await fixture.broker.totalRequestCount() > flooredRequestCount + } + #expect(await fixture.broker.totalRequestCount() == flooredRequestCount + 1) + } + + @Test + func cooldownGateDoesNotExtendItsOwnDeadline() async throws { + let fixture = try await MobileIrohCooldownFixture.make( + registrationError: CmxIrohTrustBrokerClientError.rateLimited( + code: nil, + retryAfterSeconds: 600 + ) + ) + await settleActivation(fixture) { + await fixture.broker.totalRequestCount() >= 1 + } + let flooredRequestCount = await fixture.broker.totalRequestCount() + + // Hammering the gate late in the window throws its synthesized + // cooldown error each time; those echoes must not push the deadline + // forward or the floor would never expire under retry pressure. + fixture.clock.advance(by: 590) + for _ in 0 ..< 5 { + await fixture.composition.prepareForConnection() + } + #expect(await fixture.broker.totalRequestCount() == flooredRequestCount) + + fixture.clock.advance(by: 11) + await settleActivation(fixture) { + await fixture.broker.totalRequestCount() > flooredRequestCount + } + #expect(await fixture.broker.totalRequestCount() == flooredRequestCount + 1) + } + + @Test + func nonRateLimitedFailureKeepsInactiveDialBehavior() async throws { + let fixture = try await MobileIrohCooldownFixture.make( + registrationError: MobileIrohCooldownTestError.unavailable + ) + + await settleActivation(fixture) { + await fixture.broker.totalRequestCount() >= 1 + } + let settledRequestCount = await fixture.broker.totalRequestCount() + + let transportError: any Error + do { + _ = try await fixture.composition.transport(for: fixture.request) + Issue.record("Expected inactive transport creation to fail") + return + } catch { + transportError = error + } + #expect(transportError as? CmxIrohClientRuntimeError == .inactive) + #expect((transportError as? any CmxRetryAfterProviding)?.retryAfterSeconds == nil) + + let laneError: any Error + do { + _ = try await fixture.composition.openBidirectionalLane( + for: fixture.request, + lane: .terminal( + resourceID: try CmxIrohResourceID( + "terminal:123e4567-e89b-42d3-a456-426614174099" + ), + cursor: nil + ), + priority: 0 + ) + Issue.record("Expected inactive lane creation to fail") + return + } catch { + laneError = error + } + #expect(laneError as? CmxIrohClientRuntimeError == .inactive) + #expect((laneError as? any CmxRetryAfterProviding)?.retryAfterSeconds == nil) + + let eventStreamError: any Error + do { + _ = try await fixture.composition.serverEventByteStream(for: fixture.request) + Issue.record("Expected inactive event stream creation to fail") + return + } catch { + eventStreamError = error + } + #expect(eventStreamError as? CmxIrohClientRuntimeError == .inactive) + #expect((eventStreamError as? any CmxRetryAfterProviding)?.retryAfterSeconds == nil) + // No cooldown for non-rate-limited failures: each dial retried a + // fresh activation and reached the broker again. + #expect(await fixture.broker.totalRequestCount() > settledRequestCount) + } + + @Test + func freshRelayBootstrapCredentialAvoidsSecondMint() async throws { + let fixture = try await MobileIrohCooldownFixture.makeSuccessfulBootstrap() + + await settleActivation(fixture) { + fixture.composition.runtime != nil + } + + #expect(fixture.composition.runtime != nil) + #expect(await fixture.broker.bootstrapRequestCount() >= 1) + #expect(await fixture.broker.relayTokenRequestCount() == 0) + } + + @Test + func activeRuntimeRateLimitFloorsPolicyRefreshAndDiscovery() async throws { + let fixture = try await MobileIrohCooldownFixture.makeSuccessfulBootstrap() + await settleActivation(fixture) { + fixture.composition.runtime != nil + } + #expect(fixture.composition.runtime != nil) + + let requestCountBeforeRateLimit = await fixture.broker.totalRequestCount() + let bootstrapCountBeforeRateLimit = await fixture.broker.bootstrapRequestCount() + await fixture.broker.setRelayBootstrapRateLimit(retryAfterSeconds: 600) + + await fixture.composition.refreshIrohSettings() + let requestCountAtFloor = await fixture.broker.totalRequestCount() + let bootstrapCountAtFloor = await fixture.broker.bootstrapRequestCount() + #expect(requestCountAtFloor == requestCountBeforeRateLimit + 1) + #expect(bootstrapCountAtFloor == bootstrapCountBeforeRateLimit + 1) + + // This is the field loop: settings refresh and discovery can both + // re-enter the broker while the already-active runtime reconnects. + // The first 429 owns the whole account-scoped request floor. + await fixture.composition.refreshIrohSettings() + await fixture.composition.prepareForConnection() + _ = await fixture.composition.discoverLiveMacs() + _ = await fixture.composition.discoverLiveMacs() + await fixture.composition.refreshIrohSettings() + + #expect(await fixture.broker.totalRequestCount() == requestCountAtFloor) + #expect(await fixture.broker.bootstrapRequestCount() == bootstrapCountAtFloor) + } + + @Test + func activeRuntimeRateLimitSurvivesCompositionRecreation() async throws { + let fixture = try await MobileIrohCooldownFixture.makeSuccessfulBootstrap() + await settleActivation(fixture) { + fixture.composition.runtime != nil + } + #expect(fixture.composition.runtime != nil) + + await fixture.broker.setRelayBootstrapRateLimit(retryAfterSeconds: 600) + await fixture.composition.refreshIrohSettings() + let requestCountAtFloor = await fixture.broker.totalRequestCount() + let diagnosticCountAtFloor = (await fixture.diagnosticLog.snapshot()).events.count + + // Rebuild the process-owned composition over the same injected + // UserDefaults domain and repositories. A persisted floor must be + // restored before the replacement can perform any broker work. + let recreated = fixture.recreatingComposition() + await settleActivation(recreated) { + if recreated.composition.runtime != nil { return true } + if await recreated.broker.totalRequestCount() > requestCountAtFloor { return true } + let events = (await recreated.diagnosticLog.snapshot()).events + return events.dropFirst(diagnosticCountAtFloor).contains { + $0.code == .endpointFailed + } + } + + #expect(await recreated.broker.totalRequestCount() == requestCountAtFloor) + do { + _ = try await recreated.composition.transport(for: recreated.request) + Issue.record("Expected the recreated composition to restore the broker cooldown") + } catch { + #expect((error as? any CmxRetryAfterProviding)?.retryAfterSeconds ?? 0 > 0) + } + #expect(await recreated.broker.totalRequestCount() == requestCountAtFloor) + } +} + +private enum MobileIrohCooldownTestError: Error { + case unavailable +} + +private final class MobileIrohCooldownTestClock: Sendable { + private let seconds: Atomic + + init(_ date: Date) { + seconds = Atomic(Int64(date.timeIntervalSince1970)) + } + + func now() -> Date { + Date(timeIntervalSince1970: TimeInterval(seconds.load(ordering: .relaxed))) + } + + func advance(by interval: Int64) { + seconds.store( + seconds.load(ordering: .relaxed) + interval, + ordering: .relaxed + ) + } +} + +@MainActor +private struct MobileIrohCooldownFixture { + static let accountID = "account-a" + static let deviceID = "123e4567-e89b-42d3-a456-426614174071" + static let appInstanceUUID = UUID( + uuidString: "123e4567-e89b-42d3-a456-426614174072" + )! + static let tag = "test" + // Anchored to the wall clock: the runtime constructs its relay credential + // coordinator with a real-time clock, so fixture credentials must be valid + // against real time. Cooldown timing still flows through the injected clock. + static let now = Date() + + let composition: MobileIrohRuntimeComposition + let broker: MobileIrohCooldownBroker + let clock: MobileIrohCooldownTestClock + let request: CmxByteTransportRequest + let diagnosticLog: DiagnosticLog + /// The composition observes auth weakly (the app shell owns the + /// coordinator); the fixture must retain it or every reconcile silently + /// no-ops against a deallocated coordinator. + let auth: AuthCoordinator + private let compositionFactory: @MainActor () -> MobileIrohRuntimeComposition + + static func make( + registrationError: any Error + ) async throws -> Self { + try await make(registrationError: registrationError, relayPolicy: nil) + } + + static func makeSuccessfulBootstrap() async throws -> Self { + let policy = MobileIrohCooldownRelayPolicyFixture(now: now) + return try await make( + registrationError: nil, + relayPolicy: policy + ) + } + + private static func make( + registrationError: (any Error)?, + relayPolicy: MobileIrohCooldownRelayPolicyFixture? + ) async throws -> Self { + let suiteName = "MobileIrohRuntimeCompositionCooldownTests.\(UUID().uuidString)" + let defaults = try #require(UserDefaults(suiteName: suiteName)) + defaults.removePersistentDomain(forName: suiteName) + let installState = CmxIrohUserDefaultsInstallStateStore(defaults: defaults) + let stableAppInstanceUUID = appInstanceUUID + let appInstances = CmxIrohAppInstanceRepository( + store: installState, + makeUUID: { stableAppInstanceUUID } + ) + let identities = CmxIrohIdentityRepository( + secureStore: MobileIrohCooldownIdentityStore(), + installState: installState, + randomBytes: { Data(repeating: 7, count: 32) }, + marker: { "cooldown-test-install" } + ) + let appInstanceID = try await appInstances.appInstanceID( + accountID: accountID, + tag: tag + ) + let identity = try await identities.identity( + accountID: accountID, + appInstanceID: appInstanceID + ) + let privateKey = try Curve25519.Signing.PrivateKey( + rawRepresentation: identity.secretKey.bytes + ) + let endpointID = try CmxIrohPeerIdentity( + endpointID: privateKey.publicKey.rawRepresentation + .map { String(format: "%02x", $0) } + .joined() + ) + let registration = try registrationResponse( + appInstanceID: appInstanceID, + endpointID: endpointID + ) + let discovery = try discoveryResponse( + binding: registration.binding, + relayFleet: relayPolicy?.relayURLs ?? ["https://usc1.relay.cmux.dev/"] + ) + let broker = MobileIrohCooldownBroker( + registrationError: registrationError, + registration: registration, + discovery: discovery, + bootstrap: try relayPolicy?.bootstrap() + ) + let credentialStore = MobileIrohCooldownCredentialStore() + let clock = MobileIrohCooldownTestClock(now) + let diagnosticLog = DiagnosticLog(capacity: 64, role: .mobileClient) + let stableDeviceID = deviceID + let brokerCredentials = CmxIrohBrokerCredentialRepository( + secureStore: credentialStore, + installState: installState + ) + let pendingRevocations = CmxIrohPendingRevocationOutbox( + secureStore: MobileIrohCooldownCredentialStore() + ) + let offlinePolicies = CmxIrohClientOfflinePolicyCache( + secureStore: MobileIrohCooldownCredentialStore() + ) + let relayPolicyCache = CmxIrohRelayPolicyCache( + secureStore: MobileIrohCooldownCredentialStore() + ) + let relayPreferenceStore = CmxIrohRelayPreferenceStore( + secureStore: MobileIrohCooldownCredentialStore() + ) + let customRelayCredentials = CmxIrohCustomRelayCredentialStore( + secureStore: MobileIrohCooldownCredentialStore() + ) + let relayPolicyTrustRoot = try relayPolicy?.trustRoot() + let endpointFactory = MobileIrohCooldownEndpointFactory(identity: endpointID) + let compositionFactory: @MainActor () -> MobileIrohRuntimeComposition = { + MobileIrohRuntimeComposition( + appInstances: appInstances, + identities: identities, + brokerCredentials: brokerCredentials, + pendingRevocations: pendingRevocations, + offlinePolicies: offlinePolicies, + relayPolicyCache: relayPolicyCache, + relayPreferenceStore: relayPreferenceStore, + customRelayCredentials: customRelayCredentials, + relayPolicyTrustRoot: relayPolicyTrustRoot, + endpointFactory: endpointFactory, + brokerFactory: { _ in broker }, + deviceID: { stableDeviceID }, + tag: tag, + now: { clock.now() }, + diagnosticLog: diagnosticLog, + debugDefaults: defaults + ) + } + let composition = compositionFactory() + let authClient = MobileIrohCooldownAuthClient( + user: CMUXAuthUser( + id: accountID, + primaryEmail: "a@example.com", + displayName: "A" + ) + ) + let authStore = MobileIrohCooldownAuthKeyValueStore() + let auth = AuthCoordinator( + client: authClient, + sessionCache: CMUXAuthSessionCache( + keyValueStore: authStore, + key: "has-tokens" + ), + userCache: CMUXAuthIdentityStore( + keyValueStore: authStore, + key: "cached-user" + ), + teamSelection: CMUXAuthTeamSelectionStore( + keyValueStore: authStore, + key: "selected-team" + ), + anchor: MobileIrohCooldownAuthAnchor(), + config: AuthConfig( + stack: CMUXAuthConfig( + projectId: "test", + publishableClientKey: "test" + ), + magicLinkCallbackURL: "http://localhost/auth/callback", + apiBaseURL: "http://localhost" + ), + launch: AuthLaunchOptions( + clearAuthRequested: false, + mockDataEnabled: false, + environment: [:], + includesDevAuth: false + ) + ) + try await auth.signInWithPassword(email: "a@example.com", password: "pw") + composition.configure(auth: auth) + + return Self( + composition: composition, + broker: broker, + clock: clock, + request: try request(), + diagnosticLog: diagnosticLog, + auth: auth, + compositionFactory: compositionFactory + ) + } + + func recreatingComposition() -> Self { + let recreated = compositionFactory() + recreated.configure(auth: auth) + return Self( + composition: recreated, + broker: broker, + clock: clock, + request: request, + diagnosticLog: diagnosticLog, + auth: auth, + compositionFactory: compositionFactory + ) + } + + private static func request() throws -> CmxByteTransportRequest { + CmxByteTransportRequest( + route: try CmxAttachRoute( + id: "iroh", + kind: .iroh, + endpoint: .peer( + identity: CmxIrohPeerIdentity( + endpointID: String(repeating: "a", count: 64) + ), + pathHints: [] + ), + priority: 0 + ), + expectedPeerDeviceID: "123e4567-e89b-42d3-a456-426614174074", + authorizationMode: .transportAdmission + ) + } + + private static func registrationResponse( + appInstanceID: String, + endpointID: CmxIrohPeerIdentity + ) throws -> CmxIrohRegistrationResponse { + let object: [String: Any] = [ + "binding": bindingObject( + appInstanceID: appInstanceID, + endpointID: endpointID + ), + "relay": ["status": "not_requested"], + ] + return try JSONDecoder().decode( + CmxIrohRegistrationResponse.self, + from: JSONSerialization.data(withJSONObject: object) + ) + } + + private static func discoveryResponse( + binding: CmxIrohBrokerBinding, + relayFleet: [String] + ) throws -> CmxIrohDiscoveryResponse { + let rendezvousKey = Data(repeating: 0, count: 32) + .base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: "") + let bindingData = try JSONEncoder().encode(binding) + let bindingJSON = try #require( + JSONSerialization.jsonObject(with: bindingData) as? [String: Any] + ) + let object: [String: Any] = [ + "route_contract_version": 1, + "bindings": [bindingJSON], + "relay_fleet": relayFleet, + "lan_rendezvous": ["generation": 1, "key": rendezvousKey], + "grant_verification_keys": [ + "version": 1, + "current_kid": "test-key", + "keys": [[ + "kid": "test-key", + "alg": "EdDSA", + "spki_der_base64": "AA==", + ]], + ], + ] + return try JSONDecoder().decode( + CmxIrohDiscoveryResponse.self, + from: JSONSerialization.data(withJSONObject: object) + ) + } + + private static func bindingObject( + appInstanceID: String, + endpointID: CmxIrohPeerIdentity + ) -> [String: Any] { + [ + "binding_id": "123e4567-e89b-42d3-a456-426614174073", + "device_id": deviceID, + "app_instance_id": appInstanceID, + "tag": tag, + "platform": "ios", + "endpoint_id": endpointID.endpointID, + "identity_generation": 1, + "pairing_enabled": false, + "capabilities": ["mobile-rpc-v1", "multistream-v1"], + "path_hints": [], + "last_seen_at": ISO8601DateFormatter().string(from: now), + ] + } +} + +private struct MobileIrohCooldownRelayPolicyFixture { + let privateKey = Curve25519.Signing.PrivateKey() + let now: Date + let relayURLs = ["https://usc1.relay.cmux.dev/"] + + func trustRoot() throws -> CmxIrohRelayPolicyTrustRoot { + try CmxIrohRelayPolicyTrustRoot(keys: [ + CmxIrohRelayPolicyVerificationKey( + keyID: "policy-first", + rawPublicKeyBase64: privateKey.publicKey.rawRepresentation + .base64EncodedString() + ), + ]) + } + + func bootstrap() throws -> CmxIrohRelayBootstrapResponse { + CmxIrohRelayBootstrapResponse( + relayToken: relayCredential(), + relayPolicy: try CmxIrohRelayPolicyResponse( + policy: signedPolicy(), + preference: .automatic, + preferenceRevision: 1 + ) + ) + } + + func relayCredential() -> CmxIrohRelayTokenResponse { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return CmxIrohRelayTokenResponse( + token: "aaaa", + expiresAt: formatter.string(from: now.addingTimeInterval(3_600)), + refreshAfter: formatter.string(from: now.addingTimeInterval(1_800)), + relayFleet: relayURLs + ) + } + + private func signedPolicy() throws -> String { + let header = try JSONSerialization.data( + withJSONObject: [ + "alg": "EdDSA", + "typ": "cmux-relay-policy-v1+jwt", + "kid": "policy-first", + ], + options: [.sortedKeys] + ) + let descriptors = relayURLs.map { url in + [ + "id": "cmux-us", + "provider": "cmux", + "region": "us-central1", + "url": url, + ] + } + let nowSeconds = Int64(now.timeIntervalSince1970) + let payload = try JSONSerialization.data( + withJSONObject: [ + "version": 1, + "jti": "123e4567-e89b-42d3-a456-426614174000", + "sequence": 1, + "iat": nowSeconds, + "nbf": nowSeconds, + "exp": nowSeconds + 3_600, + "aud": "cmux-iroh-relay-policy", + "relay_protocol": "iroh-relay-v1", + "relays": descriptors, + ], + options: [.sortedKeys] + ) + let input = "\(Self.base64URL(header)).\(Self.base64URL(payload))" + let signature = try privateKey.signature(for: Data(input.utf8)) + return "\(input).\(Self.base64URL(signature))" + } + + private static func base64URL(_ data: Data) -> String { + data.base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: "") + } +} + +private actor MobileIrohCooldownBroker: + CmxIrohClientBrokerServing, + CmxIrohRelayPolicyServing +{ + private let registrationError: (any Error)? + private let registration: CmxIrohRegistrationResponse + private let discoveryResponse: CmxIrohDiscoveryResponse + private let bootstrap: CmxIrohRelayBootstrapResponse? + private var relayBootstrapRetryAfterSeconds: Int? + private var totalRequests = 0 + private var bootstrapRequests = 0 + private var relayTokenRequests = 0 + + init( + registrationError: (any Error)?, + registration: CmxIrohRegistrationResponse, + discovery: CmxIrohDiscoveryResponse, + bootstrap: CmxIrohRelayBootstrapResponse? + ) { + self.registrationError = registrationError + self.registration = registration + discoveryResponse = discovery + self.bootstrap = bootstrap + } + + func setRelayBootstrapRateLimit(retryAfterSeconds: Int) { + relayBootstrapRetryAfterSeconds = retryAfterSeconds + } + + func register( + prepared _: CmxIrohPreparedRegistration, + signer _: CmxIrohRegistrationSigner + ) throws -> CmxIrohRegistrationResponse { + totalRequests += 1 + if let registrationError { throw registrationError } + return registration + } + + func discover() -> CmxIrohDiscoveryResponse { + totalRequests += 1 + return discoveryResponse + } + + func issuePairGrant( + initiatorBindingID _: String, + acceptorBindingID _: String + ) throws -> CmxIrohPairGrantResponse { + totalRequests += 1 + throw MobileIrohCooldownTestError.unavailable + } + + func issueRelayToken( + bindingID _: String, + endpointID _: CmxIrohPeerIdentity + ) throws -> CmxIrohRelayTokenResponse { + totalRequests += 1 + relayTokenRequests += 1 + guard let credential = bootstrap?.relayToken else { + throw MobileIrohCooldownTestError.unavailable + } + return credential + } + + func revoke(bindingID _: String) { + totalRequests += 1 + } + + func issueRelayBootstrap( + endpointID _: CmxIrohPeerIdentity + ) throws -> CmxIrohRelayBootstrapResponse { + totalRequests += 1 + bootstrapRequests += 1 + if let relayBootstrapRetryAfterSeconds { + throw CmxIrohTrustBrokerClientError.rateLimited( + code: nil, + retryAfterSeconds: relayBootstrapRetryAfterSeconds + ) + } + guard let bootstrap else { throw MobileIrohCooldownTestError.unavailable } + return bootstrap + } + + func relayPreference() throws -> CmxIrohRelayPreferenceResponse { + totalRequests += 1 + throw MobileIrohCooldownTestError.unavailable + } + + func updateRelayPreference( + _: CmxIrohRelayPreferenceUpdateRequest + ) throws -> CmxIrohRelayPreferenceResponse { + totalRequests += 1 + throw MobileIrohCooldownTestError.unavailable + } + + func totalRequestCount() -> Int { totalRequests } + func bootstrapRequestCount() -> Int { bootstrapRequests } + func relayTokenRequestCount() -> Int { relayTokenRequests } +} + +private actor MobileIrohCooldownEndpointFactory: CmxIrohEndpointFactory { + private let identity: CmxIrohPeerIdentity + + init(identity: CmxIrohPeerIdentity) { + self.identity = identity + } + + func bind( + configuration _: CmxIrohEndpointConfiguration + ) -> any CmxIrohEndpoint { + MobileIrohCooldownEndpoint(identity: identity) + } +} + +private actor MobileIrohCooldownEndpoint: CmxIrohEndpoint { + private let peerIdentity: CmxIrohPeerIdentity + + init(identity: CmxIrohPeerIdentity) { + peerIdentity = identity + } + + func identity() -> CmxIrohPeerIdentity { peerIdentity } + + func address() -> CmxIrohEndpointAddress { + CmxIrohEndpointAddress(identity: peerIdentity, pathHints: []) + } + + func connect( + to _: CmxIrohEndpointAddress, + alpn _: Data + ) throws -> any CmxIrohConnection { + throw MobileIrohCooldownTestError.unavailable + } + + func accept() -> (any CmxIrohConnection)? { nil } + func replaceRelays(_: [CmxIrohRelayConfiguration]) {} + + func healthEvents() -> AsyncStream { + AsyncStream { $0.finish() } + } + + func isHealthy() -> Bool { true } + func close() {} +} + +private actor MobileIrohCooldownCredentialStore: CmxIrohSecureCredentialStoring { + private var storage: [String: Data] = [:] + + func read(account: String) -> Data? { storage[account] } + + func write( + _ data: Data, + account: String, + accessibility _: CmxIrohSecureCredentialAccessibility + ) { + storage[account] = data + } + + func delete(account: String) { storage[account] = nil } + func deleteAll() { storage.removeAll() } +} + +// The synchronous storage protocol is used only by one identity repository actor. +private final class MobileIrohCooldownIdentityStore: CmxIrohSecureIdentityStoring, + @unchecked Sendable +{ + nonisolated(unsafe) private var storage: [String: Data] = [:] + + func read(account: String) -> Data? { storage[account] } + func write(_ data: Data, account: String) { storage[account] = data } + func delete(account: String) { storage[account] = nil } + func deleteAll() { storage.removeAll() } +} + +private final class MobileIrohCooldownAuthKeyValueStore: CMUXAuthKeyValueStore { + private var storage: [String: Any] = [:] + + func bool(forKey defaultName: String) -> Bool { + storage[defaultName] as? Bool ?? false + } + + func data(forKey defaultName: String) -> Data? { + storage[defaultName] as? Data + } + + func string(forKey defaultName: String) -> String? { + storage[defaultName] as? String + } + + func set(_ value: Any?, forKey defaultName: String) { + storage[defaultName] = value + } + + func removeObject(forKey defaultName: String) { + storage[defaultName] = nil + } +} + +private final class MobileIrohCooldownAuthAnchor: NSObject, + AuthPresentationAnchoring, + @unchecked Sendable +{ + func presentationAnchor( + for session: ASWebAuthenticationSession + ) -> ASPresentationAnchor { + ASPresentationAnchor() + } + + func presentationAnchor( + for controller: ASAuthorizationController + ) -> ASPresentationAnchor { + ASPresentationAnchor() + } +} + +private actor MobileIrohCooldownAuthClient: AuthClient { + private var access: String? = "access" + private var refresh: String? = "refresh" + private let user: CMUXAuthUser + + init(user: CMUXAuthUser) { + self.user = user + } + + func accessToken() -> String? { access } + func refreshToken() -> String? { refresh } + func forceRefreshAccessToken() -> String? { access } + func currentUser(throwOnMissing _: Bool) -> CMUXAuthUser? { user } + func listTeams() -> [CMUXAuthTeam] { [] } + func sendMagicLinkEmail(email _: String, callbackURL _: String) -> String { "nonce" } + + func signInWithMagicLink(code _: String) { + access = "access" + refresh = "refresh" + } + + func signInWithCredential(email _: String, password _: String) { + access = "access" + refresh = "refresh" + } + + func signInWithOAuth( + provider _: String, + anchor _: any AuthPresentationAnchoring + ) { + access = "access" + refresh = "refresh" + } + + func storedAccessToken() -> String? { access } + + func clearLocalSession() { + access = nil + refresh = nil + } + + func clearLocalSession(ifRefreshTokenMatches refreshToken: String) { + guard refresh == refreshToken else { return } + access = nil + refresh = nil + } + + func revokeSession(accessToken _: String?, refreshToken _: String?) {} + + func freshAccessToken( + accessToken: String?, + refreshToken _: String + ) -> String? { + accessToken + } +} diff --git a/web/app/api/relay/token/route.ts b/web/app/api/relay/token/route.ts index 17bde188219d..2129181309e2 100644 --- a/web/app/api/relay/token/route.ts +++ b/web/app/api/relay/token/route.ts @@ -118,15 +118,6 @@ export async function handleRelayTokenRequest( if (!user) return unauthorized(); try { - await runRelayEffect(enforceRelayRateLimit({ - request, - accountId: user.id, - ruleId: deps.rateLimitRuleId(), - check: deps.checkRateLimit, - isVercel: deps.isVercel(), - retryAfterSeconds: RELAY_TOKEN_RATE_LIMIT_RETRY_AFTER_SECONDS, - })); - const key = deps.signingKey(); if (!key) return jsonResponse({ error: "relay_token_not_configured" }, 503); @@ -142,6 +133,19 @@ export async function handleRelayTokenRequest( return jsonResponse({ error: "invalid_endpoint_id" }, 400); } + // Rate limited per account+endpoint so one storming device only starves + // itself; runs after validation so malformed requests never consume the + // per-device budget. + await runRelayEffect(enforceRelayRateLimit({ + request, + accountId: user.id, + devicePartition: rawEndpointId.toLowerCase(), + ruleId: deps.rateLimitRuleId(), + check: deps.checkRateLimit, + isVercel: deps.isVercel(), + retryAfterSeconds: RELAY_TOKEN_RATE_LIMIT_RETRY_AFTER_SECONDS, + })); + const nowSeconds = deps.nowSeconds(); const policy = await deps.signedPolicy(user.id, nowSeconds); const relayUrls = policy.payload.relays.map((relay) => relay.url); diff --git a/web/services/relay/http.ts b/web/services/relay/http.ts index 55c35a53d5eb..4839a9eda83a 100644 --- a/web/services/relay/http.ts +++ b/web/services/relay/http.ts @@ -22,6 +22,12 @@ export async function runRelayEffect( export function enforceRelayRateLimit(input: { readonly request: Request; readonly accountId: string; + /** + * Optional per-device partition (endpoint id). When present the budget is + * per account+device, so one storming device cannot starve the account's + * other phones, simulators, and tagged builds. + */ + readonly devicePartition?: string; readonly ruleId: string | undefined; readonly check: RelayRateLimitCheck; readonly isVercel?: boolean; @@ -39,7 +45,9 @@ export function enforceRelayRateLimit(input: { return Effect.tryPromise({ try: () => input.check(ruleId, { request: input.request, - rateLimitKey: input.accountId, + rateLimitKey: input.devicePartition + ? `${input.accountId}:${input.devicePartition}` + : input.accountId, }), catch: () => new RelayRateLimitError({ code: "rate_limit_unavailable" }), }).pipe( diff --git a/web/tests/relay-token-route.test.ts b/web/tests/relay-token-route.test.ts index ae1261e4edf1..baa75064a92e 100644 --- a/web/tests/relay-token-route.test.ts +++ b/web/tests/relay-token-route.test.ts @@ -307,23 +307,43 @@ describe("POST /api/relay/token", () => { expect(invalid.status).toBe(400); }); - test("rate limits by authenticated account and fails closed", async () => { + test("rate limits per account and endpoint and fails closed", async () => { let key: string | undefined; + let checks = 0; const limited = await handleRelayTokenRequest( request({ endpointId: ENDPOINT_ID }), deps({ isVercel: () => true, rateLimitRuleId: () => "relay-token", checkRateLimit: async (_id, options) => { + checks += 1; key = options.rateLimitKey; return { rateLimited: true }; }, }), ); expect(limited.status).toBe(429); - expect(key).toBe("account-a"); + // Partitioned per device: a storming endpoint starves only itself, never + // the account's other phones, simulators, or tagged builds. + expect(key).toBe(`account-a:${ENDPOINT_ID.toLowerCase()}`); expect(limited.headers.get("retry-after")).toBe("600"); + // Malformed requests are rejected before the limiter and never consume + // the per-device budget. + const invalid = await handleRelayTokenRequest( + request({ endpointId: "not-an-endpoint" }), + deps({ + isVercel: () => true, + rateLimitRuleId: () => "relay-token", + checkRateLimit: async () => { + checks += 1; + return { rateLimited: true }; + }, + }), + ); + expect(invalid.status).toBe(400); + expect(checks).toBe(1); + const unavailable = await handleRelayTokenRequest( request({ endpointId: ENDPOINT_ID }), deps({ isVercel: () => true, rateLimitRuleId: () => undefined }),