telemetry(ios): make a stalled terminal replay visible in Axiom

Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/DiagnosticTerminalTrace.swift

@@ -18,6 +18,13 @@ public enum DiagnosticTerminalTracePhase: Int, Sendable, Codable, CaseIterable {
     case applied = 7
     case failed = 8
     case discarded = 9
+    /// The operation is still in flight past a stall threshold.
+    ///
+    /// Unlike every other phase this is not terminal: it is emitted by a
+    /// probe while the request is outstanding, so an operation that never
+    /// settles still produces evidence. A settled operation emits its real
+    /// terminal phase afterwards.
+    case stalled = 10
 }
 
 /// A short opaque ID that can safely cross the mobile RPC boundary.

Packages/Shared/CMUXMobileCore/Sources/CMUXMobileCore/MobileTerminalReplayTrace.swift

@@ -0,0 +1,102 @@
+import Foundation
+
+/// Why a terminal replay was requested.
+///
+/// A replay is the only path that repaints a surface the phone has just
+/// rebuilt blank, so the trigger is the difference between "the user is
+/// looking at a blank terminal" and "the user is looking at slightly stale
+/// text". Values are stable on the wire: they are packed into the terminal
+/// trace's payload slot and read back in the analytics pipeline.
+/// Raw values are never reused: a removed case leaves its number retired so a
+/// newer producer cannot be misread as an older codepath.
+public enum MobileTerminalReplayTrigger: Int, Sendable, Codable, CaseIterable {
+    /// No reason was recorded. Nothing produces this today; it is the
+    /// well-defined zero so an empty payload slot decodes without inventing a
+    /// codepath.
+    case unknown = 0
+    /// The output stream reset; the surface was rebuilt blank.
+    case outputReset = 1
+    /// The render pipeline reset; the surface was rebuilt blank.
+    case renderPipelineReset = 2
+    /// A viewport transition armed a barrier and re-requested state.
+    case viewportTransition = 3
+    /// A render-grid delta did not chain onto the delivered revision.
+    case revisionChainBreak = 4
+    /// A render-grid delta did not chain onto the delivered history rows.
+    case historyChainBreak = 5
+    /// First attach to a surface with no delivered baseline.
+    case coldAttach = 6
+    /// A previous replay attempt failed or came back unusable.
+    case failureRetry = 7
+    /// The phone dropped a delivered frame before it reached the grid.
+    case droppedFrame = 8
+    /// The grid apply contract rejected a frame at paint time.
+    case applyFenceFailure = 9
+    /// Pending input never echoed, so the mirror is presumed diverged.
+    case pendingInputDrop = 10
+    /// The event subscription was re-established.
+    case resubscribe = 11
+    /// The Mac left the alternate screen, so the primary baseline is unknown.
+    case screenTransition = 14
+    /// A render-grid delta arrived with no delivered baseline to patch.
+    case missingBaseline = 15
+    /// A gap in the byte stream needs an authoritative screen to verify it.
+    case byteGap = 16
+}
+
+/// Categorical context recorded alongside one replay trace.
+///
+/// The terminal trace event carries a single integer payload slot, so this
+/// packs the fields that decide whether a slow replay is user-visible. The
+/// encoding is stable on the wire and round-trips through ``encoded``.
+public struct MobileTerminalReplayTraceContext: Equatable, Sendable {
+    /// Highest retry attempt the encoding can represent.
+    public static let maxAttempt = 15
+
+    /// Why this replay was requested.
+    public let trigger: MobileTerminalReplayTrigger
+    /// Whether the surface had been rebuilt blank when the replay was
+    /// requested. A slow replay on a blank surface is the blank-screen stall;
+    /// a slow replay on a painted surface only holds back fresh output.
+    public let surfaceIsBlank: Bool
+    /// Whether a replay barrier is suppressing live output for this surface.
+    public let barrierActive: Bool
+    /// Zero-based retry index within the current replay episode.
+    public let attempt: Int
+
+    public init(
+        trigger: MobileTerminalReplayTrigger,
+        surfaceIsBlank: Bool,
+        barrierActive: Bool,
+        attempt: Int
+    ) {
+        self.trigger = trigger
+        self.surfaceIsBlank = surfaceIsBlank
+        self.barrierActive = barrierActive
+        self.attempt = min(max(0, attempt), Self.maxAttempt)
+    }
+
+    /// Packs the context into one non-negative integer payload slot.
+    public var encoded: Int {
+        var value = trigger.rawValue & 0xFF
+        if surfaceIsBlank { value |= 1 << 8 }
+        if barrierActive { value |= 1 << 9 }
+        value |= (attempt & 0xF) << 10
+        return value
+    }
+
+    /// Unpacks a context previously produced by ``encoded``.
+    ///
+    /// Returns `nil` for a negative value or an unknown trigger so a future
+    /// producer cannot be silently misread as `unknown` by an older consumer.
+    public init?(encoded: Int) {
+        guard encoded >= 0,
+              let trigger = MobileTerminalReplayTrigger(rawValue: encoded & 0xFF) else {
+            return nil
+        }
+        self.trigger = trigger
+        self.surfaceIsBlank = (encoded & (1 << 8)) != 0
+        self.barrierActive = (encoded & (1 << 9)) != 0
+        self.attempt = (encoded >> 10) & 0xF
+    }
+}

Packages/Shared/CMUXMobileCore/Tests/CMUXMobileCoreTests/MobileTerminalReplayTraceContextTests.swift

@@ -0,0 +1,65 @@
+import Testing
+
+@testable import CMUXMobileCore
+
+@Suite("Replay trace context encoding")
+struct MobileTerminalReplayTraceContextTests {
+    @Test func everyTriggerRoundTripsWithBothFlags() {
+        for trigger in MobileTerminalReplayTrigger.allCases {
+            for blank in [true, false] {
+                for barrier in [true, false] {
+                    let context = MobileTerminalReplayTraceContext(
+                        trigger: trigger,
+                        surfaceIsBlank: blank,
+                        barrierActive: barrier,
+                        attempt: 3
+                    )
+                    #expect(MobileTerminalReplayTraceContext(encoded: context.encoded) == context)
+                }
+            }
+        }
+    }
+
+    @Test func attemptClampsToTheEncodableRange() {
+        let context = MobileTerminalReplayTraceContext(
+            trigger: .failureRetry,
+            surfaceIsBlank: true,
+            barrierActive: true,
+            attempt: 99
+        )
+        #expect(context.attempt == MobileTerminalReplayTraceContext.maxAttempt)
+        #expect(MobileTerminalReplayTraceContext(encoded: context.encoded) == context)
+    }
+
+    @Test func negativeAttemptClampsToZero() {
+        let context = MobileTerminalReplayTraceContext(
+            trigger: .coldAttach,
+            surfaceIsBlank: false,
+            barrierActive: false,
+            attempt: -4
+        )
+        #expect(context.attempt == 0)
+        #expect(context.encoded == MobileTerminalReplayTrigger.coldAttach.rawValue)
+    }
+
+    /// An older consumer must not read a future trigger as `unknown`: that
+    /// would silently attribute a new codepath's stalls to the wrong bucket.
+    @Test func unknownTriggerDecodesToNilRatherThanUnknown() {
+        let futureTrigger = 0xFE
+        #expect(MobileTerminalReplayTraceContext(encoded: futureTrigger) == nil)
+        #expect(MobileTerminalReplayTraceContext(encoded: -1) == nil)
+    }
+
+    @Test func flagsAreIndependentOfTheTriggerBits() {
+        let blankOnly = MobileTerminalReplayTraceContext(
+            trigger: .outputReset, surfaceIsBlank: true, barrierActive: false, attempt: 0
+        )
+        let barrierOnly = MobileTerminalReplayTraceContext(
+            trigger: .outputReset, surfaceIsBlank: false, barrierActive: true, attempt: 0
+        )
+        #expect(blankOnly.encoded != barrierOnly.encoded)
+        #expect(MobileTerminalReplayTraceContext(encoded: blankOnly.encoded)?.surfaceIsBlank == true)
+        #expect(MobileTerminalReplayTraceContext(encoded: barrierOnly.encoded)?.surfaceIsBlank == false)
+        #expect(MobileTerminalReplayTraceContext(encoded: barrierOnly.encoded)?.barrierActive == true)
+    }
+}

Packages/iOS/CmuxMobileAnalytics/Sources/CmuxMobileAnalytics/MobileTerminalTraceReporter.swift

@@ -17,12 +17,31 @@ public final class MobileTerminalTraceReporter: Sendable {
     private struct Start: Sendable {
         let operation: DiagnosticTerminalTraceOperation
         let tNanos: UInt64
+        let replayContext: MobileTerminalReplayTraceContext?
+        /// Background-transition count when this operation began.
+        let backgroundEpoch: UInt64
     }
 
     private struct State: Sendable {
         var starts: [UInt64: Start] = [:]
         var windowStart: UInt64 = 0
         var emittedInWindow = 0
+        /// Counts transitions out of the foreground.
+        ///
+        /// Comparing this against the value recorded at `started` answers the
+        /// question the flag below cannot: an operation that began on screen,
+        /// spent an hour suspended and settled after reactivation is in the
+        /// foreground when it reports, but its elapsed time is not screen
+        /// time. Only an unchanged count means the whole trace was on screen.
+        var backgroundEpoch: UInt64 = 0
+        /// Whether the app was in the foreground when the phase was recorded.
+        ///
+        /// A suspended app runs no code, so an operation that spans
+        /// suspension accrues wall-clock time it never spent waiting on
+        /// screen. Without this flag a stall of a few foreground seconds and
+        /// one that sat in a pocket for an hour are indistinguishable in
+        /// Axiom, and any percentile over the mix is meaningless.
+        var isForeground = true
     }
 
     private final class StateStore: @unchecked Sendable {
@@ -46,6 +65,13 @@ public final class MobileTerminalTraceReporter: Sendable {
             }
         }
 
+        func setForeground(_ active: Bool) {
+            queue.async { [self] in
+                if !active, state.isForeground { state.backgroundEpoch &+= 1 }
+                state.isForeground = active
+            }
+        }
+
         func drain() async {
             await withCheckedContinuation { continuation in
                 queue.async { continuation.resume() }
@@ -59,6 +85,8 @@ public final class MobileTerminalTraceReporter: Sendable {
         let terminalPhase: DiagnosticTerminalTracePhase
         let durationMilliseconds: UInt32
         let outcome: String
+        let replayContext: MobileTerminalReplayTraceContext?
+        let isForeground: Bool
     }
 
     private let emitter: any AnalyticsEmitting
@@ -79,6 +107,12 @@ public final class MobileTerminalTraceReporter: Sendable {
         }
     }
 
+    /// Records the app lifecycle edge so each emitted row says whether its
+    /// elapsed time was spent on screen.
+    public func setForeground(_ active: Bool) {
+        state.setForeground(active)
+    }
+
     public func flush() async {
         await state.drain()
         await emitter.flush()
@@ -103,9 +137,33 @@ public final class MobileTerminalTraceReporter: Sendable {
                let oldest = state.starts.min(by: { $0.value.tNanos < $1.value.tNanos })?.key {
                 state.starts.removeValue(forKey: oldest)
             }
-            state.starts[traceID.rawValue] = Start(operation: operation, tNanos: event.tNanos)
+            state.starts[traceID.rawValue] = Start(
+                operation: operation,
+                tNanos: event.tNanos,
+                replayContext: event.c.flatMap(MobileTerminalReplayTraceContext.init(encoded:)),
+                backgroundEpoch: state.backgroundEpoch
+            )
             return nil
         }
+        // A stall report is the only non-terminal emission: the operation is
+        // still outstanding, so the pending start must survive for the phase
+        // that eventually settles it. Without this an operation that never
+        // settles produced no row at all.
+        if phase == .stalled {
+            guard let duration = event.ms else { return nil }
+            guard admitEmission(at: event.tNanos, state: &state) else { return nil }
+            let start = state.starts[traceID.rawValue]
+            return Observation(
+                traceID: traceID,
+                operation: start?.operation ?? operation,
+                terminalPhase: phase,
+                durationMilliseconds: duration,
+                outcome: "stalled",
+                replayContext: event.c.flatMap(MobileTerminalReplayTraceContext.init(encoded:))
+                    ?? start?.replayContext,
+                isForeground: stayedForeground(start, state: state)
+            )
+        }
         guard phase == .applied || phase == .failed || phase == .discarded else { return nil }
         let start = state.starts.removeValue(forKey: traceID.rawValue)
         // Prefer the monotonic diagnostic timestamps whenever the start phase
@@ -124,10 +182,22 @@ public final class MobileTerminalTraceReporter: Sendable {
             operation: start?.operation ?? operation,
             terminalPhase: phase,
             durationMilliseconds: duration,
-            outcome: outcome
+            outcome: outcome,
+            replayContext: start?.replayContext,
+            isForeground: stayedForeground(start, state: state)
         )
     }
 
+    /// Whether the whole operation stayed on screen.
+    ///
+    /// Conservative when the start was dropped under admission pressure: an
+    /// operation whose beginning is unknown cannot claim its elapsed time was
+    /// screen time.
+    private static func stayedForeground(_ start: Start?, state: State) -> Bool {
+        guard let start else { return false }
+        return state.isForeground && start.backgroundEpoch == state.backgroundEpoch
+    }
+
     private static func admitEmission(at now: UInt64, state: inout State) -> Bool {
         if state.windowStart == 0 || now < state.windowStart
             || now - state.windowStart >= 60 * 1_000_000_000 {
@@ -140,14 +210,23 @@ public final class MobileTerminalTraceReporter: Sendable {
     }
 
     private static func properties(for observation: Observation) -> [String: AnalyticsValue] {
-        [
+        var properties: [String: AnalyticsValue] = [
             "phase": .string(tracePhase),
             "outcome": .string(observation.outcome),
             "duration_ms": .int(Int(observation.durationMilliseconds)),
             "user_usable": .bool(false),
             "trace_id": .string(observation.traceID.stringValue),
             "operation": .string(String(describing: observation.operation)),
             "terminal_phase": .string(String(describing: observation.terminalPhase)),
+            "app_foreground": .bool(observation.isForeground),
         ]
+        if let context = observation.replayContext {
+            properties["replay_trigger"] = .string(String(describing: context.trigger))
+            // The field that separates a blank terminal from a stale one.
+            properties["surface_blank"] = .bool(context.surfaceIsBlank)
+            properties["barrier_active"] = .bool(context.barrierActive)
+            properties["replay_attempt"] = .int(context.attempt)
+        }
+        return properties
     }
 }