diff --git a/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Wire/ControlCommandExecutionPolicy.swift b/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Wire/ControlCommandExecutionPolicy.swift index c6aeaae69bdb..23553c255a73 100644 --- a/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Wire/ControlCommandExecutionPolicy.swift +++ b/Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Wire/ControlCommandExecutionPolicy.swift @@ -159,6 +159,12 @@ public enum ControlCommandExecutionPolicy: Sendable, Equatable { // connection-owned shutdown path, which awaits asynchronous writers. // Keep that wait off the main actor. "debug.mobile.transport.disconnect", + // debug.surface.screenshot blocks its worker on the renderer's + // presented-frame acknowledgment, which is delivered on the main + // thread; running it on the main actor would deadlock for the full + // capture timeout. UI/model access inside the handler stays on main + // via v2MainSync and the main-thread presented callback. + "debug.surface.screenshot", // Browser automation methods that wait on page JavaScript, WebKit // cookies, or capture callbacks run on the socket worker: on the main // actor they block SwiftUI updates for their full duration, and on a diff --git a/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandExecutionPolicyTests.swift b/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandExecutionPolicyTests.swift index 4b094905fe65..50a711b888d5 100644 --- a/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandExecutionPolicyTests.swift +++ b/Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandExecutionPolicyTests.swift @@ -33,7 +33,8 @@ struct ControlCommandExecutionPolicyTests { "workspace.remote.pty_bridge", "workspace.env", "sidebar.custom.reload", "sidebar.custom.open", "debug.sidebar.simulate_drag", "debug.mobile.transport.disconnect", - "debug.window.screenshot", "mobile.attach_ticket.create", + "debug.window.screenshot", "debug.surface.screenshot", + "mobile.attach_ticket.create", "mobile.terminal.set_font", "mobile.task.models.list", // JavaScript-evaluating browser methods block on page JS and must // not hold the main actor (see socketWorkerMethods rationale). diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift index 98ecc8a1e5e6..7f20e96a853d 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift @@ -240,6 +240,10 @@ extension TerminalSurface { on: runtimeSurface, callbackContext: callbackContext ) + installRenderPresentedObservation( + on: runtimeSurface, + callbackContext: callbackContext + ) } #endif } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+PresentedFrameCapture.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+PresentedFrameCapture.swift new file mode 100644 index 000000000000..9dbb5166e2d8 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+PresentedFrameCapture.swift @@ -0,0 +1,130 @@ +public import Foundation +internal import GhosttyKit +internal import CmuxTerminalCore +internal import CmuxFoundation + +/// One registered waiter for a tokened render-presented acknowledgment. +/// +/// `expiresAt` bounds bookkeeping only: a waiter whose forced draw was skipped +/// by the renderer (unrealized renderer, zero-sized surface, size-discarded +/// layer assignment) never receives its callback, so stale entries are pruned +/// opportunistically on later registrations instead of leaking until surface +/// teardown. The caller's own timeout (the socket-worker callback awaiter) +/// remains the authoritative failure signal. +struct TerminalSurfacePresentedFrameWaiter { + let expiresAt: Date + let onPresented: @MainActor () -> Void +} + +/// The libghostty render-presented callback (cmux fork C API). Ghostty invokes +/// it on the main thread, in the same dispatched block that assigned the +/// frame's IOSurface to the layer, so the layer's `contents` observed from the +/// waiter is at least as new as the acknowledged frame. +private let terminalGhosttyRenderPresentedCallback: + ghostty_render_presented_cb = { userdata, token in + guard let userdata else { return } + let userdataAddress = UInt(bitPattern: userdata) + MainActor.assumeIsolated { + guard let mainActorUserdata = + UnsafeMutableRawPointer(bitPattern: userdataAddress) else { + return + } + let context = Unmanaged + .fromOpaque(mainActorUserdata) + .takeUnretainedValue() + guard let surface = + context.surfaceController as? TerminalSurface else { + return + } + surface.deliverRenderPresentedToken(token) + } + } + +extension TerminalSurface { + /// The lifetime bound for stale-waiter pruning. Comfortably above every + /// socket-side capture timeout so a pruned entry can never race a waiter + /// whose caller is still blocked. + private static let presentedFrameWaiterLifetime: TimeInterval = 60 + + /// Monotonic token source for tokened renders. An atomic (not main-actor + /// state) so nonisolated callers can mint tokens before hopping to main. + private static let presentedFrameTokenSource = AtomicUInt64Value(1) + + /// Mints a process-unique nonzero token for one tokened render. + public static func makePresentedFrameToken() -> UInt64 { + presentedFrameTokenSource.wrappingIncrementRelaxed() + } + + /// Installs the per-runtime-surface render-presented callback. Called once + /// directly after `ghostty_surface_new`, mirroring + /// `installFontSizeActionObservation`. + @MainActor + func installRenderPresentedObservation( + on runtimeSurface: ghostty_surface_t, + callbackContext: Unmanaged + ) { + precondition( + ghostty_surface_set_render_presented_callback( + runtimeSurface, + terminalGhosttyRenderPresentedCallback, + callbackContext.toOpaque() + ), + "Each Ghostty surface installs one render-presented callback" + ) + } + + /// Requests one forced renderer-thread frame whose presentation is + /// acknowledged with `token`, invoking `onPresented` on the main actor + /// after the exact frame's IOSurface has been assigned to the layer. + /// + /// Returns false without retaining `onPresented` when the surface has no + /// live runtime pointer or the runtime refused the request (no callback + /// installed, or another tokened draw is still pending). The caller owns + /// timeout handling; `cancelPresentedFrameWaiter` withdraws a waiter whose + /// caller gave up. + @MainActor + public func requestPresentedFrame( + token: UInt64, + onPresented: @escaping @MainActor () -> Void + ) -> Bool { + guard let surface = liveSurfaceForGhosttyAccess( + reason: "renderer.requestPresentedFrame" + ) else { return false } + prunePresentedFrameWaiters(now: Date()) + pendingRenderPresentedWaiters[token] = TerminalSurfacePresentedFrameWaiter( + expiresAt: Date().addingTimeInterval(Self.presentedFrameWaiterLifetime), + onPresented: onPresented + ) + guard ghostty_surface_request_render_with_token(surface, token) else { + pendingRenderPresentedWaiters.removeValue(forKey: token) + return false + } + return true + } + + /// Withdraws a waiter whose caller timed out or was cancelled. + @MainActor + public func cancelPresentedFrameWaiter(token: UInt64) { + pendingRenderPresentedWaiters.removeValue(forKey: token) + } + + /// Completes the waiter registered for `token`, if any. Tokens without a + /// waiter (already cancelled, or minted by another consumer of the tokened + /// render API) are ignored. + @MainActor + func deliverRenderPresentedToken(_ token: UInt64) { + guard let waiter = + pendingRenderPresentedWaiters.removeValue(forKey: token) else { + return + } + waiter.onPresented() + } + + @MainActor + private func prunePresentedFrameWaiters(now: Date) { + guard !pendingRenderPresentedWaiters.isEmpty else { return } + pendingRenderPresentedWaiters = pendingRenderPresentedWaiters.filter { + $0.value.expiresAt > now + } + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index c3a1f3b6766c..8f4715531edc 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -693,6 +693,10 @@ extension TerminalSurface { on: createdSurface, callbackContext: surfaceCallbackContext ) + installRenderPresentedObservation( + on: createdSurface, + callbackContext: surfaceCallbackContext + ) if source == .scheduledRestore || source == .inputDemand { requiresRestoreSpawnPacing = false } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift index 5456f35f8673..c1d49304302a 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift @@ -314,6 +314,14 @@ public final class TerminalSurface: Identifiable, ObservableObject { /// state unless the workspace focus path requests it. var desiredFocusState: Bool = false + /// Pending waiters for tokened render-presented acknowledgments, keyed by + /// the token passed to `ghostty_surface_request_render_with_token`. Filled + /// and drained on the main actor (the libghostty presented callback runs + /// on main, in the same block as the layer's IOSurface assignment). + /// See `TerminalSurface+PresentedFrameCapture.swift`. + var pendingRenderPresentedWaiters: + [UInt64: TerminalSurfacePresentedFrameWaiter] = [:] + /// Bumped after every completed runtime clipboard read. public internal(set) var clipboardReadGeneration = 0 #if DEBUG diff --git a/Sources/GhosttyTerminalView.swift b/Sources/GhosttyTerminalView.swift index 4febc6877eb8..0cabc8bbd363 100644 --- a/Sources/GhosttyTerminalView.swift +++ b/Sources/GhosttyTerminalView.swift @@ -11308,6 +11308,71 @@ final class GhosttySurfaceScrollView: NSView { ) } + /// One deep-copied presented terminal frame for the offscreen screenshot RPC. + struct DebugPresentedFrameImage { + /// Deep copy of the presented IOSurface pixels, tagged Display P3 (the + /// Ghostty Metal renderer draws every frame into a Display P3 BGRA8 + /// target; see `ghostty/src/renderer/metal/Target.zig`). + let image: CGImage + /// The layer's contents scale (the native backing scale, 2 on Retina). + let backingScale: CGFloat + } + + /// Deep-copies the renderer's presented IOSurface — the exact bytes the + /// compositor would show for the terminal grid area — into a CGImage + /// tagged with the renderer's true color space (Display P3), at native + /// backing resolution. Unlike `CGWindowListCreateImage`, this needs no + /// Screen Recording permission, no window-server compositing, and works + /// while the window is occluded, on another Space, or never ordered front. + func debugCopyPresentedFrameImage() -> DebugPresentedFrameImage? { + guard let modelLayer = surfaceView.layer else { return nil } + let layer = modelLayer.presentation() ?? modelLayer + guard let contents = layer.contents else { return nil } + + let cf = contents as CFTypeRef + guard CFGetTypeID(cf) == IOSurfaceGetTypeID() else { return nil } + let surfaceRef = (contents as! IOSurfaceRef) + + let width = Int(IOSurfaceGetWidth(surfaceRef)) + let height = Int(IOSurfaceGetHeight(surfaceRef)) + let bytesPerRow = Int(IOSurfaceGetBytesPerRow(surfaceRef)) + guard width > 0, height > 0, bytesPerRow > 0 else { return nil } + + IOSurfaceLock(surfaceRef, [.readOnly], nil) + defer { IOSurfaceUnlock(surfaceRef, [.readOnly], nil) } + + let base = IOSurfaceGetBaseAddress(surfaceRef) + let size = bytesPerRow * height + let data = Data(bytes: base, count: size) + + guard let provider = CGDataProvider(data: data as CFData), + let colorSpace = CGColorSpace(name: CGColorSpace.displayP3) else { + return nil + } + let bitmapInfo = CGBitmapInfo.byteOrder32Little.union( + CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedFirst.rawValue) + ) + + guard let image = CGImage( + width: width, + height: height, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerRow: bytesPerRow, + space: colorSpace, + bitmapInfo: bitmapInfo, + provider: provider, + decode: nil, + shouldInterpolate: false, + intent: .defaultIntent + ) else { return nil } + + return DebugPresentedFrameImage( + image: image, + backingScale: max(1.0, layer.contentsScale) + ) + } + /// Sample the IOSurface backing the terminal layer (if any) to detect a transient blank frame /// without using screenshots/screen recording permissions. func debugSampleIOSurface(normalizedCrop: CGRect) -> DebugFrameSample? { diff --git a/Sources/TerminalController+DebugMethodNames.swift b/Sources/TerminalController+DebugMethodNames.swift index 8571346e2fc7..6efa97afaa0f 100644 --- a/Sources/TerminalController+DebugMethodNames.swift +++ b/Sources/TerminalController+DebugMethodNames.swift @@ -44,6 +44,7 @@ extension TerminalController { "debug.session_snapshot_benchmark", "debug.session_snapshot_seed_scrollback", "debug.window.screenshot", + "debug.surface.screenshot", "debug.terminal.simulate_file_drop", "debug.sidebar.simulate_drag", "debug.mobile.transport.disconnect", diff --git a/Sources/TerminalController+SurfaceScreenshot.swift b/Sources/TerminalController+SurfaceScreenshot.swift new file mode 100644 index 000000000000..8ba84ca3abda --- /dev/null +++ b/Sources/TerminalController+SurfaceScreenshot.swift @@ -0,0 +1,226 @@ +import AppKit +import CmuxTerminal +import CoreGraphics +import Foundation +import ImageIO +import UniformTypeIdentifiers + +#if DEBUG +/// `debug.surface.screenshot` — ground-truth capture of one terminal surface +/// without the window server. +/// +/// Mechanism: the socket worker asks the surface for one forced tokened render +/// (`ghostty_surface_request_render_with_token`, cmux fork API). The renderer +/// thread rebuilds frame data from the current terminal state and draws it, +/// deliberately ignoring the occlusion gate; libghostty acknowledges the token +/// on the main thread in the same block that assigns the frame's IOSurface to +/// the layer. The waiter then deep-copies that IOSurface — the exact bytes the +/// compositor would composite for the terminal grid area — at native backing +/// resolution (2x on Retina), tagged Display P3 (the renderer's true target +/// color space; see `ghostty/src/renderer/metal/Target.zig`). +/// +/// The path never activates the app, never changes key window, never reorders +/// windows, needs no Screen Recording permission, and works while the window +/// is fully occluded, on another Space, or never ordered front. +extension TerminalController { + /// One captured presented frame plus the visibility evidence observed on + /// the main thread at capture time (so callers can prove the capture + /// happened while occluded and non-frontmost). + private struct V2SurfaceScreenshotFrame: @unchecked Sendable { + let surfaceId: UUID + let image: CGImage + let backingScale: CGFloat + let windowOcclusionVisible: Bool + let appActive: Bool + let windowFrame: CGRect? + } + + private enum V2SurfaceScreenshotOutcome: @unchecked Sendable { + case frame(V2SurfaceScreenshotFrame) + case failure(code: String, message: String) + } + + nonisolated func v2DebugSurfaceScreenshot(params: [String: Any]) -> V2CallResult { + let surfaceArg = ((params["surface_id"] as? String) ?? "") + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !surfaceArg.isEmpty else { + return .err(code: "invalid_params", message: "Missing surface_id", data: nil) + } + let requestedScale: Double? + if let raw = params["scale"] { + guard let value = (raw as? Double) ?? (raw as? Int).map(Double.init), + value > 0, value <= 8 else { + return .err(code: "invalid_params", message: "scale must be a number in (0, 8]", data: nil) + } + requestedScale = value + } else { + requestedScale = nil + } + let label = ((params["label"] as? String) ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let explicitPath = ((params["path"] as? String) ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + + let outcome: V2SurfaceScreenshotOutcome? = socketAwaitCallback(timeout: 10.0) { finish in + v2MainSync { + self.v2StartDebugSurfaceScreenshot(surfaceArg: surfaceArg, finish: finish) + } + } + + guard let outcome else { + return .err( + code: "timeout", + message: "Timed out waiting for the renderer to present the requested frame", + data: nil + ) + } + let frame: V2SurfaceScreenshotFrame + switch outcome { + case .failure(let code, let message): + return .err(code: code, message: message, data: nil) + case .frame(let value): + frame = value + } + + // Native capture by default; an explicit scale resamples in Display P3 + // (output pixels = logical points * scale, so scale==backingScale is + // the native passthrough). + let outputImage: CGImage + let outputScale: Double + if let requestedScale, abs(requestedScale - Double(frame.backingScale)) > 0.001 { + let factor = requestedScale / Double(frame.backingScale) + guard let scaled = Self.v2ResampleDisplayP3(frame.image, factor: factor) else { + return .err(code: "internal_error", message: "Failed to resample capture", data: nil) + } + outputImage = scaled + outputScale = requestedScale + } else { + outputImage = frame.image + outputScale = Double(frame.backingScale) + } + + let outputURL: URL + if !explicitPath.isEmpty { + outputURL = URL(fileURLWithPath: (explicitPath as NSString).expandingTildeInPath) + } else { + let outputDir = FileManager.default.temporaryDirectory + .appendingPathComponent("cmux-screenshots") + try? FileManager.default.createDirectory(at: outputDir, withIntermediateDirectories: true) + let timestampMs = Int(Date().timeIntervalSince1970 * 1000) + let shortId = String(UUID().uuidString.prefix(8)) + let base = label.isEmpty ? "surface" : label + outputURL = outputDir.appendingPathComponent("\(base)-\(timestampMs)-\(shortId).png") + } + + guard Self.v2WritePNG(outputImage, to: outputURL) else { + return .err(code: "internal_error", message: "Failed to encode or write PNG", data: nil) + } + + var result: [String: Any] = [ + "surface_id": frame.surfaceId.uuidString, + "path": outputURL.path, + "width_px": outputImage.width, + "height_px": outputImage.height, + "points_width": Double(frame.image.width) / Double(frame.backingScale), + "points_height": Double(frame.image.height) / Double(frame.backingScale), + "scale": outputScale, + "native_scale": Double(frame.backingScale), + "color_space": "display-p3", + "window_occlusion_visible": frame.windowOcclusionVisible, + "app_active": frame.appActive, + ] + if let windowFrame = frame.windowFrame { + result["window_frame"] = [ + "x": Double(windowFrame.origin.x), + "y": Double(windowFrame.origin.y), + "width": Double(windowFrame.width), + "height": Double(windowFrame.height), + ] + } + return .ok(result) + } + + @MainActor + private func v2StartDebugSurfaceScreenshot( + surfaceArg: String, + finish: @escaping (V2SurfaceScreenshotOutcome) -> Void + ) { + guard let tabManager else { + finish(.failure(code: "unavailable", message: "TabManager not available")) + return + } + guard let tabId = tabManager.selectedTabId, + let tab = tabManager.tabs.first(where: { $0.id == tabId }) else { + finish(.failure(code: "not_found", message: "No tab selected")) + return + } + guard let panelId = resolveSurfaceId(from: surfaceArg, tab: tab), + let terminalPanel = tab.terminalInputTarget(forPanelID: panelId)?.panel else { + finish(.failure(code: "not_found", message: "Terminal surface not found")) + return + } + + let view = terminalPanel.hostedView + let surface = terminalPanel.surface + let capturedSurfaceId = panelId + let token = TerminalSurface.makePresentedFrameToken() + let accepted = surface.requestPresentedFrame(token: token) { [weak view] in + guard let view, + let presented = view.debugCopyPresentedFrameImage() else { + finish(.failure( + code: "internal_error", + message: "Failed to read the presented IOSurface" + )) + return + } + let window = view.window + finish(.frame(V2SurfaceScreenshotFrame( + surfaceId: capturedSurfaceId, + image: presented.image, + backingScale: presented.backingScale, + windowOcclusionVisible: window?.occlusionState.contains(.visible) ?? false, + appActive: NSApp.isActive, + windowFrame: window?.frame + ))) + } + if !accepted { + finish(.failure( + code: "unavailable", + message: "Tokened render unavailable (no live realized renderer, or another capture is pending)" + )) + } + } + + /// Resamples in the capture's own color space so pixel values stay + /// Display P3 and only resolution changes. + private nonisolated static func v2ResampleDisplayP3(_ image: CGImage, factor: Double) -> CGImage? { + let width = max(1, Int((Double(image.width) * factor).rounded())) + let height = max(1, Int((Double(image.height) * factor).rounded())) + guard let colorSpace = CGColorSpace(name: CGColorSpace.displayP3), + let context = CGContext( + data: nil, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: 0, + space: colorSpace, + bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue + | CGBitmapInfo.byteOrder32Little.rawValue + ) else { return nil } + context.interpolationQuality = .high + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) + return context.makeImage() + } + + /// Writes through ImageIO so the PNG embeds the image's Display P3 ICC + /// profile (NSBitmapImageRep would re-tag through its own colorspace). + private nonisolated static func v2WritePNG(_ image: CGImage, to url: URL) -> Bool { + guard let destination = CGImageDestinationCreateWithURL( + url as CFURL, + UTType.png.identifier as CFString, + 1, + nil + ) else { return false } + CGImageDestinationAddImage(destination, image, nil) + return CGImageDestinationFinalize(destination) + } +} +#endif diff --git a/Sources/TerminalController.swift b/Sources/TerminalController.swift index 14575bb9ffe7..330da5f320c2 100644 --- a/Sources/TerminalController.swift +++ b/Sources/TerminalController.swift @@ -1483,6 +1483,8 @@ class TerminalController { case "sidebar.custom.open": return v2Result(id: request.id, v2CustomSidebarOpen(params: request.params)) #if DEBUG + case "debug.surface.screenshot": + return v2Result(id: request.id, v2DebugSurfaceScreenshot(params: request.params)) case "debug.sidebar.simulate_drag": return v2Result(id: request.id, v2DebugSidebarSimulateDrag(params: request.params)) case "debug.window.screenshot": @@ -1548,7 +1550,8 @@ class TerminalController { // instead of the internal-error backstop below. if request.method == "debug.sidebar.simulate_drag" || request.method == "debug.window.screenshot" - || request.method == "debug.mobile.transport.disconnect" { + || request.method == "debug.mobile.transport.disconnect" + || request.method == "debug.surface.screenshot" { return v2Error(id: request.id, code: "method_not_found", message: "Unknown method") } #endif @@ -13192,7 +13195,7 @@ class TerminalController { return nil } - private func resolveSurfaceId(from arg: String, tab: Workspace) -> UUID? { + func resolveSurfaceId(from arg: String, tab: Workspace) -> UUID? { if let uuid = UUID(uuidString: arg), tab.panels[uuid] != nil || tab.remoteTmuxControlPane(surfaceID: uuid) != nil { return uuid diff --git a/cmux.xcodeproj/project.pbxproj b/cmux.xcodeproj/project.pbxproj index 897ca9adec85..c98854bfa2fc 100644 --- a/cmux.xcodeproj/project.pbxproj +++ b/cmux.xcodeproj/project.pbxproj @@ -2256,6 +2256,7 @@ C0DE71B10000000000000001 /* AppDelegate+AgentChatNotifications.swift in Sources C79470010000000000000003 /* TerminalController+SocketClientCapability.swift in Sources */ = {isa = PBXBuildFile; fileRef = C79470010000000000000004 /* TerminalController+SocketClientCapability.swift */; }; A79840030000000000000002 /* TerminalController+SocketConfiguration.swift in Sources */ = {isa = PBXBuildFile; fileRef = A79840030000000000000001 /* TerminalController+SocketConfiguration.swift */; }; C79470020000000000000001 /* TerminalController+SocketListenerRearm.swift in Sources */ = {isa = PBXBuildFile; fileRef = C79470020000000000000002 /* TerminalController+SocketListenerRearm.swift */; }; + 9B7C50A10000000000000001 /* TerminalController+SurfaceScreenshot.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9B7C50A10000000000000002 /* TerminalController+SurfaceScreenshot.swift */; }; D6212D0C00000000000000D3 /* TerminalController+WindowDockBrowserRouting.swift in Sources */ = {isa = PBXBuildFile; fileRef = D6212D0C00000000000000D4 /* TerminalController+WindowDockBrowserRouting.swift */; }; 9065A0010000000000000001 /* TerminalController+WindowScreenshotCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9065A0010000000000000002 /* TerminalController+WindowScreenshotCapture.swift */; }; C0DE00000000000000000C84 /* TerminalController+WorkspaceCreate.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE00000000000000000C83 /* TerminalController+WorkspaceCreate.swift */; }; @@ -4890,6 +4891,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = C79470010000000000000004 /* TerminalController+SocketClientCapability.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TerminalController+SocketClientCapability.swift"; sourceTree = ""; }; A79840030000000000000001 /* TerminalController+SocketConfiguration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TerminalController+SocketConfiguration.swift"; sourceTree = ""; }; C79470020000000000000002 /* TerminalController+SocketListenerRearm.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TerminalController+SocketListenerRearm.swift"; sourceTree = ""; }; + 9B7C50A10000000000000002 /* TerminalController+SurfaceScreenshot.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TerminalController+SurfaceScreenshot.swift"; sourceTree = ""; }; D6212D0C00000000000000D4 /* TerminalController+WindowDockBrowserRouting.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TerminalController+WindowDockBrowserRouting.swift"; sourceTree = ""; }; 9065A0010000000000000002 /* TerminalController+WindowScreenshotCapture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TerminalController+WindowScreenshotCapture.swift"; sourceTree = ""; }; C0DE00000000000000000C83 /* TerminalController+WorkspaceCreate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TerminalController+WorkspaceCreate.swift"; sourceTree = ""; }; @@ -7248,6 +7250,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = C57B00040000000000000002 /* TerminalController+CustomSidebarCommands.swift */, CC0033E15A1B2C3D4E5F0004 /* TerminalController+CommentsCommands.swift */, 8054A0040000000000000004 /* TerminalController+BrowserAutomationRecovery.swift */, + 9B7C50A10000000000000002 /* TerminalController+SurfaceScreenshot.swift */, C57B00050000000000000002 /* CmuxExtensionSidebarSelection+CustomSidebarName.swift */, 59065015952E1B5BE5E23519 /* Mobile */, 31B04B98376C9BA8B9E44DCB /* TerminalViewportUITestRecorder.swift */, @@ -9971,6 +9974,7 @@ C0DE71B10000000000000002 /* AppDelegate+AgentChatNotifications.swift */ = {isa = C79470010000000000000003 /* TerminalController+SocketClientCapability.swift in Sources */, A79840030000000000000002 /* TerminalController+SocketConfiguration.swift in Sources */, C79470020000000000000001 /* TerminalController+SocketListenerRearm.swift in Sources */, + 9B7C50A10000000000000001 /* TerminalController+SurfaceScreenshot.swift in Sources */, D6212D0C00000000000000D3 /* TerminalController+WindowDockBrowserRouting.swift in Sources */, 9065A0010000000000000001 /* TerminalController+WindowScreenshotCapture.swift in Sources */, C0DE00000000000000000C84 /* TerminalController+WorkspaceCreate.swift in Sources */, diff --git a/ghostty b/ghostty index 11aa609d75de..c0de0687f6a0 160000 --- a/ghostty +++ b/ghostty @@ -1 +1 @@ -Subproject commit 11aa609d75dec882ef2f83171e2cbe887aeddbc5 +Subproject commit c0de0687f6a008bd648c6855cc9c39b6921b3475 diff --git a/scripts/ghosttykit-checksums.txt b/scripts/ghosttykit-checksums.txt index 9a06b17a26f7..dd1cb7acf2da 100644 --- a/scripts/ghosttykit-checksums.txt +++ b/scripts/ghosttykit-checksums.txt @@ -114,3 +114,4 @@ da1ddcf41f6fd763c39bde4c69d1ac7323cb9bd0 51bb73625dd8e53a98675fb75dc573931ab3b65 f0f8273b700e754b3f0052cea033789aab902fd1 4ea556c6203c4757096093247ebbae85fc4650ac87c00841508783933bff77c3 19d03fa4d0161e60e02de2e42601992be0c001c3 d2842bb7778a4e8d5a5a5f57ce6a85508630e3184ba46c1ca1ae5cbe1655472f 11aa609d75dec882ef2f83171e2cbe887aeddbc5 1a4acbcc9e0e5b20c0b4dad6660d0c08546a5d36192053834df960144fa8fdb9 +c0de0687f6a008bd648c6855cc9c39b6921b3475 0f54d361d85cc883497058e9bbd251b713182862fc1a07575623233a014fa5ee diff --git a/tests_v2/cmux.py b/tests_v2/cmux.py index 2253e4acec8c..58661ef8e044 100755 --- a/tests_v2/cmux.py +++ b/tests_v2/cmux.py @@ -1081,6 +1081,29 @@ def screenshot(self, label: str = "") -> dict: params["label"] = label return dict(self._call("debug.window.screenshot", params) or {}) + def surface_screenshot( + self, + panel: Union[str, int], + scale: Optional[float] = None, + label: str = "", + path: str = "", + timeout_s: float = 20.0, + ) -> dict: + """Ground-truth capture of one terminal surface via the renderer's own + presented IOSurface (debug.surface.screenshot). Works while the window + is occluded, on another Space, or never ordered front; never focuses or + reorders anything. Native backing scale (2x on Retina) by default; + pixels are Display P3 (the PNG is tagged accordingly).""" + sid = self._resolve_surface_id(panel) + params: Dict[str, Any] = {"surface_id": sid} + if scale is not None: + params["scale"] = scale + if label: + params["label"] = label + if path: + params["path"] = path + return dict(self._call("debug.surface.screenshot", params, timeout_s=timeout_s) or {}) + def main() -> None: import argparse diff --git a/tests_v2/test_offscreen_surface_screenshot.py b/tests_v2/test_offscreen_surface_screenshot.py new file mode 100644 index 000000000000..1082ee640104 --- /dev/null +++ b/tests_v2/test_offscreen_surface_screenshot.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +""" +Proof for `debug.surface.screenshot`: focus-free, occlusion-proof, +native-resolution ground-truth capture of a terminal surface. + +Drives a tagged cmux dev instance over its debug socket only. It launches +NOTHING except a tiny borderless overlay window (swift-interpreted helper) +used to fully occlude the tagged cmux window; the overlay ignores mouse +events, never activates, and is killed on exit. + +Checks, in order: + 1. writes a distinctive pattern (red/green/blue background bands plus a + text sentinel) to the focused pane and waits for it via read_text + 2. captures once to learn the window frame, then covers that frame with + the overlay and waits until the RPC itself reports + window_occlusion_visible == false (the compositor's own occlusion state) + 3. captures while fully occluded AND while the app is not active, asserts + the capture is native 2x (Retina), decodes the PNG (pure stdlib), and + verifies the pattern bands are present + 4. captures twice and asserts the pixels are byte-identical (determinism) + 5. captures with scale=1 and asserts logical-size output + +Usage: + python3 tests_v2/test_offscreen_surface_screenshot.py --socket /tmp/cmux-debug-.sock + +Refuses to run against /tmp/cmux-debug.sock (the user's own instance). +""" + +import argparse +import os +import struct +import subprocess +import sys +import tempfile +import time +import zlib +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) +from cmux import cmux, cmuxError + +SENTINEL = "OSCAP-SENTINEL-7391" + +OVERLAY_SWIFT = r''' +import AppKit + +let values = CommandLine.arguments.dropFirst().compactMap(Double.init) +guard values.count == 4 else { fatalError("usage: overlay x y w h") } +let app = NSApplication.shared +app.setActivationPolicy(.accessory) +let frame = NSRect(x: values[0], y: values[1], width: values[2], height: values[3]) +let window = NSWindow( + contentRect: frame, + styleMask: [.borderless], + backing: .buffered, + defer: false +) +window.level = .floating +window.backgroundColor = .black +window.isOpaque = true +window.ignoresMouseEvents = true +window.collectionBehavior = [.canJoinAllSpaces, .stationary] +window.orderFrontRegardless() +print("OVERLAY-READY") +FileHandle.standardOutput.synchronizeFile() +app.run() +''' + + +def decode_png_rgba(path: str): + """Minimal PNG decoder (stdlib only): 8-bit RGB/RGBA, non-interlaced. + Returns (width, height, rows) where rows[y] is a bytearray of RGBA.""" + data = Path(path).read_bytes() + assert data[:8] == b"\x89PNG\r\n\x1a\n", "not a PNG" + pos = 8 + width = height = None + bit_depth = color_type = None + idat = bytearray() + while pos < len(data): + (length,) = struct.unpack(">I", data[pos:pos + 4]) + ctype = data[pos + 4:pos + 8] + chunk = data[pos + 8:pos + 8 + length] + pos += 12 + length + if ctype == b"IHDR": + width, height, bit_depth, color_type, _, _, interlace = struct.unpack( + ">IIBBBBB", chunk + ) + assert bit_depth == 8, f"unsupported bit depth {bit_depth}" + assert color_type in (2, 6), f"unsupported color type {color_type}" + assert interlace == 0, "interlaced PNG unsupported" + elif ctype == b"IDAT": + idat.extend(chunk) + elif ctype == b"IEND": + break + raw = zlib.decompress(bytes(idat)) + channels = 4 if color_type == 6 else 3 + stride = width * channels + rows = [] + prev = bytearray(stride) + offset = 0 + for _ in range(height): + filter_type = raw[offset] + offset += 1 + line = bytearray(raw[offset:offset + stride]) + offset += stride + if filter_type == 1: # Sub + for i in range(channels, stride): + line[i] = (line[i] + line[i - channels]) & 0xFF + elif filter_type == 2: # Up + for i in range(stride): + line[i] = (line[i] + prev[i]) & 0xFF + elif filter_type == 3: # Average + for i in range(stride): + left = line[i - channels] if i >= channels else 0 + line[i] = (line[i] + ((left + prev[i]) >> 1)) & 0xFF + elif filter_type == 4: # Paeth + for i in range(stride): + left = line[i - channels] if i >= channels else 0 + up = prev[i] + ul = prev[i - channels] if i >= channels else 0 + p = left + up - ul + pa, pb, pc = abs(p - left), abs(p - up), abs(p - ul) + if pa <= pb and pa <= pc: + pred = left + elif pb <= pc: + pred = up + else: + pred = ul + line[i] = (line[i] + pred) & 0xFF + prev = line + if channels == 3: + rgba = bytearray() + for i in range(0, stride, 3): + rgba.extend(line[i:i + 3]) + rgba.append(255) + rows.append(rgba) + else: + rows.append(line) + return width, height, rows + + +def count_band_pixels(rows, classify): + return sum( + 1 + for line in rows + for i in range(0, len(line), 4) + if classify(line[i], line[i + 1], line[i + 2]) + ) + + +def is_red(r, g, b): + return r > 170 and g < 130 and b < 120 + + +def is_green(r, g, b): + return g > 150 and r < 160 and b < 120 + + +def is_blue(r, g, b): + return b > 150 and r < 120 and g < 130 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--socket", required=True, help="Tagged debug socket path") + parser.add_argument( + "--out-dir", + default="", + help="Directory for evidence PNGs (default: mkdtemp)", + ) + args = parser.parse_args() + + if os.path.realpath(args.socket) == "/tmp/cmux-debug.sock": + print("FAIL: refusing to run against the user's default socket", file=sys.stderr) + return 2 + + out_dir = Path(args.out_dir) if args.out_dir else Path( + tempfile.mkdtemp(prefix="oscap-proof-") + ) + out_dir.mkdir(parents=True, exist_ok=True) + + c = cmux(args.socket) + c.connect() + overlay = None + panel = None + overlay_src = out_dir / "overlay.swift" + failures = [] + + def check(name, ok, detail=""): + status = "ok" if ok else "FAIL" + print(f" [{status}] {name}" + (f" ({detail})" if detail else "")) + if not ok: + failures.append(name) + + try: + surfaces = c.list_surfaces() + if not surfaces: + print("FAIL: no surfaces in the current workspace", file=sys.stderr) + return 2 + panel = surfaces[0][1] + + # 1. Distinctive pattern: 4 rows each of red/green/blue background + # bands (truecolor SGR over spaces) plus a text sentinel; hide the + # cursor so repeated captures are pixel-identical. + print("== writing pattern") + # NOTE: the client's send unescaper turns \\n into a raw newline and + # passes unknown escapes (\\e) through, so printf-level newlines are + # written as \\\\n to survive as printf's own \n escape. The trailing + # foreground sleep holds the shell so no prompt redraw or cursor blink + # can perturb the pixels between captures (released in cleanup). + script = ( + "clear; " + "for i in 1 2 3 4; do printf '\\e[48;2;255;0;0m%80s\\e[0m\\\\n' ''; done; " + "for i in 1 2 3 4; do printf '\\e[48;2;0;255;0m%80s\\e[0m\\\\n' ''; done; " + "for i in 1 2 3 4; do printf '\\e[48;2;0;0;255m%80s\\e[0m\\\\n' ''; done; " + f"printf '{SENTINEL}\\\\n'; printf '\\e[?25l'; sleep 300" + ) + c.send_surface(panel, script + "\\n") + deadline = time.time() + 15 + while time.time() < deadline: + if SENTINEL in c.read_terminal_text(panel): + break + time.sleep(0.2) + else: + print("FAIL: sentinel never appeared in read_text", file=sys.stderr) + return 2 + + # 2. Learn the window frame from a first capture. + print("== baseline capture (window frame discovery)") + baseline = c.surface_screenshot( + panel, path=str(out_dir / "baseline.png") + ) + frame = baseline.get("window_frame") or {} + if not frame: + print("FAIL: capture returned no window_frame", file=sys.stderr) + return 2 + print(f" window_frame={frame} native_scale={baseline.get('native_scale')}") + + # 3. Fully occlude that frame with a floating overlay (margin on every + # side), then wait for the compositor to report the window occluded. + print("== occluding window with overlay") + overlay_src.write_text(OVERLAY_SWIFT) + margin = 40 + overlay = subprocess.Popen( + [ + "swift", str(overlay_src), + str(frame["x"] - margin), str(frame["y"] - margin), + str(frame["width"] + 2 * margin), str(frame["height"] + 2 * margin), + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + assert overlay.stdout is not None + ready_line = overlay.stdout.readline().strip() + if ready_line != "OVERLAY-READY": + print(f"FAIL: overlay helper did not start ({ready_line!r})", file=sys.stderr) + return 2 + + occluded = None + deadline = time.time() + 20 + while time.time() < deadline: + probe = c.surface_screenshot( + panel, path=str(out_dir / "occluded.png") + ) + if not probe.get("window_occlusion_visible", True): + occluded = probe + break + time.sleep(0.5) + if occluded is None: + print( + "FAIL: window never reported occluded; is something moving the overlay?", + file=sys.stderr, + ) + return 2 + + # 4. The occluded, non-frontmost capture is the proof. + print("== verifying occluded capture") + check( + "window fully occluded at capture time", + occluded.get("window_occlusion_visible") is False, + ) + check("app not active at capture time", occluded.get("app_active") is False) + check( + "capture is native 2x", + float(occluded.get("native_scale", 0)) == 2.0 + and float(occluded.get("scale", 0)) == 2.0, + f"scale={occluded.get('scale')}", + ) + check( + "pixel dims are 2x logical dims", + occluded.get("width_px") + == round(occluded.get("points_width", 0) * 2) + and occluded.get("height_px") + == round(occluded.get("points_height", 0) * 2), + f"{occluded.get('width_px')}x{occluded.get('height_px')} px vs " + f"{occluded.get('points_width')}x{occluded.get('points_height')} pt", + ) + check( + "color space tagged display-p3", + occluded.get("color_space") == "display-p3", + ) + + width, height, rows = decode_png_rgba(occluded["path"]) + check( + "decoded PNG matches reported dims", + width == occluded.get("width_px") and height == occluded.get("height_px"), + ) + # Each band is 4 rows x 80 cols of solid background. Expect a + # substantial pixel count per color (thousands of device pixels). + min_band = 80 * 4 * 40 # well under the true count, well over noise + red = count_band_pixels(rows, is_red) + green = count_band_pixels(rows, is_green) + blue = count_band_pixels(rows, is_blue) + check("red band present", red > min_band, f"{red} px") + check("green band present", green > min_band, f"{green} px") + check("blue band present", blue > min_band, f"{blue} px") + + # 5. Determinism: two occluded captures decode to identical pixels. + print("== verifying determinism") + again = c.surface_screenshot(panel, path=str(out_dir / "occluded-2.png")) + _, _, rows2 = decode_png_rgba(again["path"]) + check( + "repeat capture is pixel-identical", + rows == rows2, + ) + + # 6. Explicit --scale 1 produces logical-resolution output. + print("== verifying --scale 1") + one = c.surface_screenshot( + panel, scale=1, path=str(out_dir / "occluded-1x.png") + ) + check( + "scale=1 halves pixel dims", + one.get("width_px") == round(one.get("points_width", 0)) + and one.get("height_px") == round(one.get("points_height", 0)), + f"{one.get('width_px')}x{one.get('height_px')} px", + ) + finally: + if overlay is not None: + overlay.terminate() + try: + overlay.wait(timeout=5) + except subprocess.TimeoutExpired: + overlay.kill() + # Release the pane's hold-open sleep and restore the cursor. + if panel is not None: + try: + c.send_key_surface(panel, "ctrl-c") + c.send_surface(panel, "printf '\\e[?25h'\\n") + except Exception: + pass + c.close() + + print() + print(f"evidence: {out_dir}") + if failures: + print(f"FAIL: {len(failures)} check(s) failed: {', '.join(failures)}") + return 1 + print("PASS: occluded, non-frontmost, native-2x, Display P3, deterministic") + return 0 + + +if __name__ == "__main__": + sys.exit(main())