Let third-party dictation tools insert into terminals

Sources/GhosttyNSView+Accessibility.swift

@@ -0,0 +1,164 @@
+import AppKit
+import CmuxTerminal
+
+extension GhosttyNSView {
+    // MARK: - Accessibility
+
+    /// Expose the terminal surface as an editable accessibility element.
+    /// Voice input tools frequently target AX text areas for text insertion.
+    override func isAccessibilityElement() -> Bool {
+        true
+    }
+
+    override func accessibilityRole() -> NSAccessibility.Role? {
+        .textArea
+    }
+
+    override func accessibilityHelp() -> String? {
+        "Terminal content area"
+    }
+
+    /// The active screen's text. Dictation tools compare this before and
+    /// after an insertion to confirm the text landed.
+    override func accessibilityValue() -> Any? {
+        accessibilityTextValue()
+    }
+
+    override func accessibilityNumberOfCharacters() -> Int {
+        (accessibilityTextValue() as NSString).length
+    }
+
+    override func accessibilityVisibleCharacterRange() -> NSRange {
+        NSRange(location: 0, length: accessibilityNumberOfCharacters())
+    }
+
+    override func accessibilityLine(for index: Int) -> Int {
+        let content = accessibilityTextValue() as NSString
+        let end = min(max(index, 0), content.length)
+        var line = 0
+        for offset in 0..<end where content.character(at: offset) == 0x0A {
+            line += 1
+        }
+        return line
+    }
+
+    override func accessibilityString(for range: NSRange) -> String? {
+        let content = accessibilityTextValue() as NSString
+        guard range.location != NSNotFound,
+              range.location >= 0,
+              range.length >= 0,
+              NSMaxRange(range) <= content.length else { return nil }
+        return content.substring(with: range)
+    }
+
+    override func setAccessibilityValue(_ value: Any?) {
+        guard let content = Self.accessibilityCommittedString(value) else { return }
+        let inject = {
+            // Diff against what clients recently read, on the main thread
+            // where the snapshot lives.
+            let inserted = self.terminalAccessibilityText.insertedText(settingValue: content)
+            self.insertAccessibilityCommittedText(inserted)
+        }
+        if Thread.isMainThread {
+            inject()
+        } else {
+            DispatchQueue.main.async(execute: inject)
+        }
+    }
+
+    /// Clients that insert at the caret set the selected text instead of the
+    /// whole value. A terminal only accepts input at its cursor, so the text
+    /// is committed there, the same as `setAccessibilityValue(_:)`.
+    override func setAccessibilitySelectedText(_ text: String?) {
+        guard let text else { return }
+        let inject = {
+            self.insertAccessibilityCommittedText(text)
+        }
+        if Thread.isMainThread {
+            inject()
+        } else {
+            DispatchQueue.main.async(execute: inject)
+        }
+    }
+
+    override func isAccessibilitySelectorAllowed(_ selector: Selector) -> Bool {
+        switch selector {
+        case #selector(setAccessibilityValue(_:)), #selector(setAccessibilitySelectedText(_:)):
+            return true
+        default:
+            return super.isAccessibilitySelectorAllowed(selector)
+        }
+    }
+
+    override func accessibilitySelectedTextRange() -> NSRange {
+        selectedRange()
+    }
+
+    override func accessibilitySelectedText() -> String? {
+        guard let snapshot = readSelectionSnapshot() else { return nil }
+        return snapshot.string.isEmpty ? nil : snapshot.string
+    }
+
+    private func accessibilityTextValue() -> String {
+        terminalAccessibilityText.value {
+            terminalSurface?.readText(region: .active)
+        }
+    }
+
+    private static func accessibilityCommittedString(_ value: Any?) -> String? {
+        switch value {
+        case let v as NSAttributedString:
+            return v.string
+        case let v as String:
+            return v
+        default:
+            return nil
+        }
+    }
+
+    /// Commits text from an accessibility client into the terminal.
+    ///
+    /// Single-line text keeps typed-input semantics, including a trailing
+    /// newline sent as Return, so autosuggestions and a dictation tool's
+    /// "press enter" keep working. Text with a line break before its end goes
+    /// through the paste path instead, so a bracketed-paste-aware shell or
+    /// agent receives one multi-line block rather than running each line as
+    /// its own command. A trailing newline after that block still submits.
+    private func insertAccessibilityCommittedText(_ content: String) {
+        guard !content.isEmpty else { return }
+
+#if DEBUG
+        cmuxDebugLog("ime.ax.insert len=\(content.count)")
+#endif
+
+        let (body, lineBreaks) = TerminalAccessibilityText.splitTrailingLineBreaks(content)
+        // A cold surface queues the paste but can send the Return at once, so
+        // multi-line text takes the paste path only on a live runtime.
+        if TerminalAccessibilityText.containsLineBreak(body),
+           let terminalSurface,
+           terminalSurface.hasLiveSurface {
+            unmarkText()
+            let payload = TerminalAccessibilityText.pastePayload(
+                Self.sanitizeExternalCommittedText(body)
+            )
+            if !payload.isEmpty {
+                terminalSurface.sendText(payload)
+            }
+            if !lineBreaks.isEmpty {
+                withExternalCommittedText {
+                    insertText(lineBreaks, replacementRange: NSRange(location: NSNotFound, length: 0))
+                }
+            }
+        } else {
+            withExternalCommittedText {
+                insertText(content, replacementRange: NSRange(location: NSNotFound, length: 0))
+            }
+        }
+
+        terminalAccessibilityText.scheduleValueChanged(for: self) { [weak self] in
+            self?.releaseAccessibilityValueChangedFrameDemand()
+        }
+        terminalAccessibilityText.invalidate()
+        retainAccessibilityValueChangedFrameDemand()
+    }
+}

Sources/GhosttyNSView+ForeignCommandChord.swift

@@ -0,0 +1,39 @@
+import AppKit
+
+extension GhosttyNSView {
+    /// Whether another process posted `event` rather than the keyboard or cmux.
+    ///
+    /// Hardware events carry source PID 0, and events cmux synthesizes carry
+    /// its own PID. Anything else was injected by another app, for example a
+    /// dictation tool's global hotkey.
+    static func isKeyEventPostedByAnotherProcess(
+        _ event: NSEvent,
+        currentProcessID: pid_t = ProcessInfo.processInfo.processIdentifier
+    ) -> Bool {
+        guard let sourceProcessID = event.cgEvent?.getIntegerValueField(.eventSourceUnixProcessID),
+              sourceProcessID != 0 else { return false }
+        return sourceProcessID != Int64(currentProcessID)
+    }
+
+    /// Whether to swallow a Command+Option chord that missed both the menu
+    /// and every Ghostty binding, instead of typing it into the terminal.
+    ///
+    /// Dictation tools can post their own hotkey (Cmd+Option+C while Option
+    /// is held as push-to-talk) into the focused app. Nothing in cmux handles
+    /// that chord, so it would reach the program as a stray character. Only
+    /// chords injected by another process are dropped; the same chord from
+    /// the keyboard still reaches the terminal. Command chords without Option
+    /// are left alone so remote-control and automation tools keep working.
+    func shouldDropForeignUnboundCommandChord(_ event: NSEvent) -> Bool {
+        let flags = event.modifierFlags.intersection(.deviceIndependentFlagsMask)
+        guard flags.contains(.command), flags.contains(.option),
+              Self.isKeyEventPostedByAnotherProcess(event) else { return false }
+#if DEBUG
+        cmuxDebugLog(
+            "key.foreignCommandChord.drop keyCode=\(event.keyCode) " +
+            "mods=\(flags.rawValue)"
+        )
+#endif
+        return true
+    }
+}

Sources/GhosttyTerminalView.swift

@@ -3812,6 +3812,7 @@ class GhosttyNSView: NSView, NSUserInterfaceValidations {
     private var _renderedFrameFlushScheduled = false
     private var _pendingRenderedFrameDeliveryReasons:
         TerminalRenderedFrameDeliveryReasons = []
+    private var accessibilityValueChangedFrameDemandRelease: (any RenderDemandRetention)?
     /// Pane-local frame demand lets a terminal-specific consumer observe a
     /// late render without enabling notifications on every terminal surface.
     let localRenderedFrameNotificationDemand = RenderDemandCounter()
@@ -3820,6 +3821,7 @@ class GhosttyNSView: NSView, NSUserInterfaceValidations {
     private let keyboardCopyModeRenderedFrameDemand = RenderDemandCounter()
     nonisolated let selectionAccessibilitySignal = TerminalSelectionAccessibilitySignal()
     private var selectionAccessibilityNotifier: TerminalSelectionAccessibilityNotifier?
+    let terminalAccessibilityText = TerminalAccessibilityText()
     var cellSize: CGSize = .zero
     private var lastKnownMousePointInView: NSPoint?
     private let commandClickReleaseRouter = TerminalCommandClickReleaseRouter()
@@ -3977,6 +3979,16 @@ class GhosttyNSView: NSView, NSUserInterfaceValidations {
         _scrollbarLock.unlock()
     }
 
+    func retainAccessibilityValueChangedFrameDemand() {
+        guard accessibilityValueChangedFrameDemandRelease == nil else { return }
+        accessibilityValueChangedFrameDemandRelease = localRenderedFrameNotificationDemand.retain()
+    }
+
+    func releaseAccessibilityValueChangedFrameDemand() {
+        accessibilityValueChangedFrameDemandRelease?.release()
+        accessibilityValueChangedFrameDemandRelease = nil
+    }
+
     func enqueueRenderedFrameUpdate(
         reasons: TerminalRenderedFrameDeliveryReasons
     ) {
@@ -3998,6 +4010,13 @@ class GhosttyNSView: NSView, NSUserInterfaceValidations {
         let reasons = _pendingRenderedFrameDeliveryReasons
         _pendingRenderedFrameDeliveryReasons = []
 
+        if reasons.contains(.notification),
+           terminalAccessibilityText.screenDidChange(read: { [weak self] in
+               self?.terminalSurface?.readText(region: .active)
+           }) {
+            releaseAccessibilityValueChangedFrameDemand()
+        }
+
         if reasons.contains(.keyboardCopyModeCursor),
            keyboardCopyModeActive,
            let surface {
@@ -6600,73 +6619,12 @@ class GhosttyNSView: NSView, NSUserInterfaceValidations {
         }
     }
 
-    // MARK: - Accessibility
-
-    /// Expose the terminal surface as an editable accessibility element.
-    /// Voice input tools frequently target AX text areas for text insertion.
-    override func isAccessibilityElement() -> Bool {
-        true
-    }
-
-    override func accessibilityRole() -> NSAccessibility.Role? {
-        .textArea
-    }
-
-    override func accessibilityHelp() -> String? {
-        "Terminal content area"
-    }
-
-    override func accessibilityValue() -> Any? {
-        // We don't keep a full terminal text snapshot in this layer.
-        // Expose selected text when available; otherwise provide an empty value
-        // so AX clients still treat this as an editable text area.
-        accessibilitySelectedText() ?? ""
-    }
-
-    override func setAccessibilityValue(_ value: Any?) {
-        let content: String
-        switch value {
-        case let v as NSAttributedString:
-            content = v.string
-        case let v as String:
-            content = v
-        default:
-            return
-        }
-
-        guard !content.isEmpty else { return }
-
-#if DEBUG
-        cmuxDebugLog("ime.ax.setValue len=\(content.count)")
-#endif
-
-        let inject = {
-            self.withExternalCommittedText {
-                self.insertText(content, replacementRange: NSRange(location: NSNotFound, length: 0))
-            }
-        }
-        if Thread.isMainThread {
-            inject()
-        } else {
-            DispatchQueue.main.async(execute: inject)
-        }
-    }
-
-    private func withExternalCommittedText<T>(_ body: () -> T) -> T {
+    func withExternalCommittedText<T>(_ body: () -> T) -> T {
         externalCommittedTextDepth += 1
         defer { externalCommittedTextDepth -= 1 }
         return body()
     }
 
-    override func accessibilitySelectedTextRange() -> NSRange {
-        selectedRange()
-    }
-
-    override func accessibilitySelectedText() -> String? {
-        guard let snapshot = readSelectionSnapshot() else { return nil }
-        return snapshot.string.isEmpty ? nil : snapshot.string
-    }
-
     func readSelectionSnapshot(surface explicitSurface: ghostty_surface_t? = nil) -> SelectionSnapshot? {
         guard let surface = explicitSurface ?? self.surface else { return nil }
 
@@ -7029,7 +6987,11 @@ class GhosttyNSView: NSView, NSUserInterfaceValidations {
                 return false
             }
 
-            if !shouldRetryMainMenu { lastPerformKeyEvent = nil; keyDown(with: event); return true }
+            if !shouldRetryMainMenu {
+                lastPerformKeyEvent = nil
+                if !shouldDropForeignUnboundCommandChord(event) { keyDown(with: event) }
+                return true
+            }
             if let lastPerformKeyEvent {
                 self.lastPerformKeyEvent = nil
                 if lastPerformKeyEvent == event.timestamp {