diff --git a/native/MuesliNative/Sources/MuesliCore/DictationStore.swift b/native/MuesliNative/Sources/MuesliCore/DictationStore.swift index 7e99205e2..c5e5cc308 100644 --- a/native/MuesliNative/Sources/MuesliCore/DictationStore.swift +++ b/native/MuesliNative/Sources/MuesliCore/DictationStore.swift @@ -26,6 +26,7 @@ public struct MeetingThreadNavigation: Equatable, Sendable { public final class DictationStore { private static let targetApplicationBackfillMigration = "dictation_target_application_from_app_context_v1" + private static let quillStatisticsBackfillMigration = "quill_statistics_spoken_instruction_v1" public static let defaultTombstoneRetentionInterval: TimeInterval = 30 * 24 * 60 * 60 private static let iso8601Formatter = ISO8601DateFormatter() @@ -336,6 +337,7 @@ public final class DictationStore { let _ = sqlite3_exec(db, "CREATE INDEX IF NOT EXISTS idx_dictations_sync_dirty ON dictations(updated_at DESC) WHERE sync_dirty = 1", nil, nil, nil) let _ = sqlite3_exec(db, "CREATE INDEX IF NOT EXISTS idx_meetings_sync_dirty ON meetings(updated_at DESC) WHERE sync_dirty = 1", nil, nil, nil) try migrateInsightsCache(db: db) + try backfillQuillStatisticsIfNeeded(db: db) try repairLegacyMacOriginSources(db: db) _ = try purgeSoftDeletedTextRecords(olderThan: Self.defaultTombstoneRetentionInterval, db: db) } @@ -426,6 +428,9 @@ public final class DictationStore { durationSeconds: Double, targetAppName: String? = nil, targetAppBundleID: String? = nil, + finalStatus: String = "done", + finalMessage: String? = nil, + additionalTraceEvents: [ComputerUseTraceEvent] = [], startedAt: Date, endedAt: Date ) throws -> Int64 { @@ -435,6 +440,7 @@ public final class DictationStore { do { let dictationID = try insertDictation( text: outputText, + statisticsText: instruction, durationSeconds: durationSeconds, source: "quil", targetAppName: targetAppName, @@ -445,8 +451,8 @@ public final class DictationStore { ) try insertComputerUseTrace( dictationID: dictationID, - finalStatus: "done", - finalMessage: "\(backend) · \(model)", + finalStatus: finalStatus, + finalMessage: finalMessage ?? "\(backend) · \(model)", events: [ originalText.isEmpty ? ComputerUseTraceEvent( @@ -469,7 +475,7 @@ public final class DictationStore { title: "Model", body: "\(backend) · \(model)" ), - ], + ] + additionalTraceEvents, db: db ) try exec("COMMIT", db: db) @@ -482,6 +488,7 @@ public final class DictationStore { private func insertDictation( text: String, + statisticsText: String? = nil, durationSeconds: Double, appContext: String = "", source: String, @@ -511,7 +518,7 @@ public final class DictationStore { sqlite3_bind_double(statement, 2, durationSeconds) sqlite3_bind_text(statement, 3, (text as NSString).utf8String, -1, nil) sqlite3_bind_text(statement, 4, (appContext as NSString).utf8String, -1, nil) - sqlite3_bind_int(statement, 5, Int32(Self.countWords(in: text))) + sqlite3_bind_int(statement, 5, Int32(Self.countWords(in: statisticsText ?? text))) sqlite3_bind_text(statement, 6, (source as NSString).utf8String, -1, nil) bindOptionalText(targetAppName, at: 7, statement: statement) bindOptionalText(targetAppBundleID, at: 8, statement: statement) @@ -1904,7 +1911,7 @@ public final class DictationStore { private static let insightsCacheBatchSize = 64 private func reconcileInsightsCache(db: OpaquePointer?, calendar: Calendar) throws { - let signature = "2|\(calendar.timeZone.identifier)" + let signature = "3|\(calendar.timeZone.identifier)" if try insightsCacheMeta("signature", db: db) != signature { try resetInsightsCache(signature: signature, db: db) } @@ -2012,22 +2019,45 @@ public final class DictationStore { } private func insightsSourceText(_ source: InsightsCacheSource, db: OpaquePointer?) throws -> String? { - let sql: String if source.kind == "meeting" { - sql = """ + let sql = """ SELECT CASE WHEN meeting_status = 'note_only' THEN COALESCE(manual_notes, '') ELSE COALESCE(raw_transcript, '') END FROM meetings WHERE id = ? AND updated_at = ? """ - } else { - sql = "SELECT COALESCE(raw_text, '') FROM dictations WHERE id = ? AND updated_at = ?" + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { throw lastError(db) } + defer { sqlite3_finalize(statement) } + sqlite3_bind_int64(statement, 1, source.id) + sqlite3_bind_double(statement, 2, source.updatedAt) + guard sqlite3_step(statement) == SQLITE_ROW else { return nil } + return stringColumn(statement, index: 0) } + + let sql = """ + SELECT COALESCE(d.source, ''), COALESCE(d.raw_text, ''), t.trace_json + FROM dictations d + LEFT JOIN computer_use_traces t ON t.dictation_id = d.id + WHERE d.id = ? AND d.updated_at = ? + """ var statement: OpaquePointer? guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { throw lastError(db) } defer { sqlite3_finalize(statement) } sqlite3_bind_int64(statement, 1, source.id) sqlite3_bind_double(statement, 2, source.updatedAt) guard sqlite3_step(statement) == SQLITE_ROW else { return nil } - return stringColumn(statement, index: 0) + + let dictationSource = stringColumn(statement, index: 0) + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased() + guard dictationSource == "quil" else { + return stringColumn(statement, index: 1) + } + guard let traceJSON = optionalStringColumn(statement, index: 2), + let data = traceJSON.data(using: .utf8), + let events = try? JSONDecoder().decode([ComputerUseTraceEvent].self, from: data) else { + return "" + } + return events.first(where: { $0.kind == "quil_instruction" })?.body ?? "" } @discardableResult @@ -5087,6 +5117,100 @@ public final class DictationStore { return (conditions, boundValues) } + /// Quill originally stored the generated output's word count. Analytics should + /// instead describe what the user spoke, while the full output remains history. + private func backfillQuillStatisticsIfNeeded(db: OpaquePointer?) throws { + guard !(try localMigrationCompleted(Self.quillStatisticsBackfillMigration, db: db)) else { + return + } + + try exec("BEGIN IMMEDIATE TRANSACTION", db: db) + do { + if !(try localMigrationCompleted(Self.quillStatisticsBackfillMigration, db: db)) { + let selectSQL = """ + SELECT d.id, t.trace_json + FROM dictations d + LEFT JOIN computer_use_traces t ON t.dictation_id = d.id + WHERE LOWER(TRIM(COALESCE(d.source, ''))) = 'quil' + """ + var select: OpaquePointer? + guard sqlite3_prepare_v2(db, selectSQL, -1, &select, nil) == SQLITE_OK else { + throw lastError(db) + } + defer { sqlite3_finalize(select) } + + let updateSQL = """ + UPDATE dictations + SET word_count = ?, + updated_at = MAX(updated_at, ?), + sync_dirty = 1 + WHERE id = ? + """ + var update: OpaquePointer? + guard sqlite3_prepare_v2(db, updateSQL, -1, &update, nil) == SQLITE_OK else { + throw lastError(db) + } + defer { sqlite3_finalize(update) } + + let migrationTime = Date().timeIntervalSince1970 + while sqlite3_step(select) == SQLITE_ROW { + let dictationID = sqlite3_column_int64(select, 0) + guard let traceJSON = optionalStringColumn(select, index: 1), + let data = traceJSON.data(using: .utf8), + let events = try? JSONDecoder().decode([ComputerUseTraceEvent].self, from: data), + let instruction = events.first(where: { $0.kind == "quil_instruction" })?.body, + !instruction.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + else { + // Computer-use traces are device-local. A synced Quill row can + // legitimately have only the already-correct prompt word count. + continue + } + + sqlite3_reset(update) + sqlite3_clear_bindings(update) + sqlite3_bind_int(update, 1, Int32(Self.countWords(in: instruction))) + sqlite3_bind_double(update, 2, migrationTime) + sqlite3_bind_int64(update, 3, dictationID) + guard sqlite3_step(update) == SQLITE_DONE else { throw lastError(db) } + } + + let markSQL = """ + INSERT INTO local_migrations (identifier, completed_at) + VALUES (?, CAST(strftime('%s', 'now') AS REAL)) + """ + var mark: OpaquePointer? + guard sqlite3_prepare_v2(db, markSQL, -1, &mark, nil) == SQLITE_OK else { + throw lastError(db) + } + defer { sqlite3_finalize(mark) } + sqlite3_bind_text( + mark, + 1, + (Self.quillStatisticsBackfillMigration as NSString).utf8String, + -1, + nil + ) + guard sqlite3_step(mark) == SQLITE_DONE else { throw lastError(db) } + } + try exec("COMMIT", db: db) + } catch { + _ = sqlite3_exec(db, "ROLLBACK", nil, nil, nil) + throw error + } + } + + private func localMigrationCompleted(_ identifier: String, db: OpaquePointer?) throws -> Bool { + let sql = "SELECT EXISTS(SELECT 1 FROM local_migrations WHERE identifier = ?)" + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { + throw lastError(db) + } + defer { sqlite3_finalize(statement) } + sqlite3_bind_text(statement, 1, (identifier as NSString).utf8String, -1, nil) + guard sqlite3_step(statement) == SQLITE_ROW else { throw lastError(db) } + return sqlite3_column_int(statement, 0) != 0 + } + private func backfillLegacyTargetApplicationsIfNeeded(db: OpaquePointer?) throws { guard !(try targetApplicationBackfillCompleted(db: db)) else { return } diff --git a/native/MuesliNative/Sources/MuesliNativeApp/MuesliController.swift b/native/MuesliNative/Sources/MuesliNativeApp/MuesliController.swift index 08ac8b2f5..231220fb2 100644 --- a/native/MuesliNative/Sources/MuesliNativeApp/MuesliController.swift +++ b/native/MuesliNative/Sources/MuesliNativeApp/MuesliController.swift @@ -8116,11 +8116,16 @@ public final class MuesliController: NSObject { ) return } + var pasteLifecycleEvents: [PasteController.LifecycleEvent] = [] PasteController.paste( text: replacement, requireStagedClipboardOwnership: true, targetApplicationProvider: { snapshot.application }, shouldDispatchPaste: { snapshot.isTargetStillFocused() }, + dispatchStrategy: DictationContextCapture.isBrowserApplication(snapshot.application) + ? .targetApplicationPasteCommand + : .keyboardShortcut, + retainStagedTextOnFailure: true, onPasteDispatched: { // The post-dictation correction monitor cannot distinguish a // user edit from Quill's deliberate rewrite. Once Quill actually @@ -8130,19 +8135,48 @@ public final class MuesliController: NSObject { }, onPasteFinished: { target in guard self.quilTaskID == taskID else { return } - if target == nil { - self.presentQuilFailure(QuilTransformationError.selectionChanged) + let usedTargetPasteCommand = pasteLifecycleEvents.contains( + .targetPasteCommandDispatched + ) + let retainedForManualPaste = pasteLifecycleEvents.contains( + .clipboardRetainedForManualPaste + ) + let deliveryStatus: String + let deliveryMessage: String? + let deliveryTraceBody: String + let userMessage: String? + if target != nil { + deliveryStatus = "done" + deliveryMessage = nil + deliveryTraceBody = usedTargetPasteCommand + ? "Pasted through the target application's standard Paste command" + : "Paste keyboard command dispatched to the target application" + userMessage = nil + } else if retainedForManualPaste { + deliveryStatus = "needs_attention" + deliveryMessage = "Generated text is ready for manual paste" + deliveryTraceBody = "Automatic paste was not accepted; generated text was retained on the clipboard" + userMessage = "Generated — press ⌘V to paste" } else { - let saved = self.persistQuilTransformation( - outputText: replacement, - originalText: snapshot.text, - instruction: instruction, - backend: backend, - model: model, - duration: duration, - startedAt: startedAt, - application: snapshot.application - ) + deliveryStatus = "needs_attention" + deliveryMessage = "Automatic paste could not be completed" + deliveryTraceBody = "Automatic paste was not completed and the clipboard changed before fallback could be retained" + userMessage = "Generated, but automatic paste failed; output saved in history" + } + let saved = self.persistQuilTransformation( + outputText: replacement, + originalText: snapshot.text, + instruction: instruction, + backend: backend, + model: model, + duration: duration, + startedAt: startedAt, + application: snapshot.application, + deliveryStatus: deliveryStatus, + deliveryMessage: deliveryMessage, + deliveryTraceBody: deliveryTraceBody + ) + if target != nil { TelemetryDeck.signal("quil.completed", parameters: [ "backend": backend.backend, "input_chars": String(snapshot.text.count), @@ -8152,7 +8186,19 @@ public final class MuesliController: NSObject { taskID: taskID, message: saved ? nil : "Reformatted, but could not save Quill history" ) + } else { + TelemetryDeck.signal("quil.paste_fallback", parameters: [ + "backend": backend.backend, + "clipboard_retained": String(retainedForManualPaste), + ]) + let message = saved + ? userMessage + : "Generated, but paste and Quill history both failed" + self.finishQuilTask(taskID: taskID, message: message) } + }, + onLifecycleEvent: { event in + pasteLifecycleEvents.append(event) } ) } @@ -8204,9 +8250,19 @@ public final class MuesliController: NSObject { model: String, duration: TimeInterval, startedAt: Date, - application: NSRunningApplication + application: NSRunningApplication, + deliveryStatus: String = "done", + deliveryMessage: String? = nil, + deliveryTraceBody: String? = nil ) -> Bool { do { + let additionalTraceEvents = deliveryTraceBody.map { + [ComputerUseTraceEvent( + kind: "quil_delivery", + title: "Delivery", + body: $0 + )] + } ?? [] _ = try dictationStore.insertQuilDictation( outputText: outputText, originalText: originalText, @@ -8216,6 +8272,9 @@ public final class MuesliController: NSObject { durationSeconds: duration, targetAppName: application.localizedName, targetAppBundleID: application.bundleIdentifier, + finalStatus: deliveryStatus, + finalMessage: deliveryMessage, + additionalTraceEvents: additionalTraceEvents, startedAt: startedAt, endedAt: Date() ) diff --git a/native/MuesliNative/Sources/MuesliNativeApp/PasteController.swift b/native/MuesliNative/Sources/MuesliNativeApp/PasteController.swift index 824c4e158..fbee60788 100644 --- a/native/MuesliNative/Sources/MuesliNativeApp/PasteController.swift +++ b/native/MuesliNative/Sources/MuesliNativeApp/PasteController.swift @@ -4,10 +4,18 @@ import Foundation import MuesliCore enum PasteController { + enum DispatchStrategy: Sendable { + case keyboardShortcut + case targetApplicationPasteCommand + } + enum LifecycleEvent: String, CaseIterable, Sendable { case clipboardStaged = "clipboard_staged" case clipboardStageFailed = "clipboard_stage_failed" case targetSnapshotted = "target_snapshotted" + case targetPasteCommandDispatched = "target_paste_command_dispatched" + case targetPasteCommandUnavailable = "target_paste_command_unavailable" + case targetPasteCommandRejected = "target_paste_command_rejected" case pasteDispatched = "paste_dispatched" case pasteDispatchFailed = "paste_dispatch_failed" case pasteDispatchCancelled = "paste_dispatch_cancelled" @@ -15,11 +23,17 @@ enum PasteController { case clipboardRestoreScheduled = "clipboard_restore_scheduled" case clipboardRestored = "clipboard_restored" case clipboardRestoreSkipped = "clipboard_restore_skipped" + case clipboardRetainedForManualPaste = "clipboard_retained_for_manual_paste" } /// How long to wait after simulating Cmd+V before restoring the clipboard. /// The receiving app must have consumed the paste data within this window. private static let clipboardRestoreDelay: TimeInterval = 0.5 + /// Accessibility calls cross a process boundary and can block their caller while + /// the target app is busy. Keep both each request and the full menu walk bounded; + /// an unavailable command falls back to leaving Quill output on the clipboard. + private static let targetPasteAXMaximumRequestTimeout: Float = 0.1 + private static let targetPasteAXTraversalBudget: TimeInterval = 0.35 private static let physicalKeyMap: [Character: (CGKeyCode, CGEventFlags)] = [ "a": (0, []), "b": (11, []), "c": (8, []), "d": (2, []), "e": (14, []), "f": (3, []), "g": (5, []), "h": (4, []), "i": (34, []), "j": (38, []), @@ -48,12 +62,12 @@ enum PasteController { /// Paste text into the active app via clipboard, then restore the original clipboard contents. /// - /// Flow: save clipboard → write text → Cmd+V → restore clipboard after delay. + /// Flow: save clipboard → write text → dispatch Paste → restore clipboard after delay. /// If the clipboard cannot be saved (e.g. lazy-provided data), falls back to a simple /// paste without restoration. - /// For nonempty text, completion receives the target app only when the keyboard events - /// were posted. When staged-clipboard ownership is required, a failed write or intervening - /// clipboard change also completes with `nil` attribution and skips Cmd+V. + /// For nonempty text, completion receives the target app only when the selected Paste + /// command was accepted. When staged-clipboard ownership is required, a failed write or + /// intervening clipboard change also completes with `nil` attribution and skips dispatch. @MainActor static func paste( text: String, @@ -63,6 +77,11 @@ enum PasteController { NSWorkspace.shared.frontmostApplication }, shouldDispatchPaste: @escaping @MainActor () -> Bool = { true }, + dispatchStrategy: DispatchStrategy = .keyboardShortcut, + retainStagedTextOnFailure: Bool = false, + targetPasteAction: @escaping @MainActor (NSRunningApplication) -> Bool? = { + PasteController.performTargetPasteCommand(in: $0) + }, simulatePasteAction: @escaping @MainActor () -> Bool = PasteController.simulatePaste, onPasteDispatched: @escaping @MainActor () -> Void = {}, onPasteFinished: @escaping @MainActor (NSRunningApplication?) -> Void = { _ in }, @@ -80,10 +99,25 @@ enum PasteController { onLifecycleEvent(didStageText ? .clipboardStaged : .clipboardStageFailed) DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { - // Snapshot immediately before Cmd+V so attribution and the paste event + // Snapshot immediately before Paste dispatch so attribution and the command // refer to the same frontmost application. let targetApplication = targetApplicationProvider() onLifecycleEvent(.targetSnapshotted) + + @MainActor + func settleFailedDispatch() { + if retainStagedTextOnFailure, pasteboard.changeCount == pasteChangeCount { + onLifecycleEvent(.clipboardRetainedForManualPaste) + } else if pasteboard.changeCount == pasteChangeCount { + restoreClipboard(pasteboard, from: savedItems) + onLifecycleEvent(.clipboardRestored) + } else { + onLifecycleEvent(.clipboardRestoreSkipped) + } + onPasteFinished(nil) + onClipboardSettled() + } + if requireStagedClipboardOwnership { guard didStageText else { // Restore only when Muesli still owns the cleared pasteboard. If another @@ -107,20 +141,39 @@ enum PasteController { } guard shouldDispatchPaste() else { onLifecycleEvent(.pasteDispatchCancelled) - if pasteboard.changeCount == pasteChangeCount { - restoreClipboard(pasteboard, from: savedItems) - onLifecycleEvent(.clipboardRestored) - } else { - onLifecycleEvent(.clipboardRestoreSkipped) - } - onPasteFinished(nil) - onClipboardSettled() + settleFailedDispatch() return } - let didDispatchPaste = simulatePasteAction() + + let didDispatchPaste: Bool + switch dispatchStrategy { + case .keyboardShortcut: + didDispatchPaste = simulatePasteAction() + case .targetApplicationPasteCommand: + guard let targetApplication else { + onLifecycleEvent(.targetPasteCommandUnavailable) + onLifecycleEvent(.pasteDispatchFailed) + settleFailedDispatch() + return + } + switch targetPasteAction(targetApplication) { + case true: + onLifecycleEvent(.targetPasteCommandDispatched) + didDispatchPaste = true + case false: + onLifecycleEvent(.targetPasteCommandRejected) + didDispatchPaste = false + case nil: + onLifecycleEvent(.targetPasteCommandUnavailable) + didDispatchPaste = false + } + } onLifecycleEvent(didDispatchPaste ? .pasteDispatched : .pasteDispatchFailed) if didDispatchPaste { onPasteDispatched() + } else { + settleFailedDispatch() + return } // Arm restoration before completion bookkeeping. The dictation completion callback @@ -241,6 +294,148 @@ enum PasteController { return true } + /// Invokes the target process's standard Cmd+V menu item through Accessibility. + /// + /// `true` means the target app accepted AXPress, `false` means the command was + /// found but disabled/rejected, and `nil` means the app did not expose a + /// standard Paste command. Resolving by shortcut metadata avoids depending on + /// localized menu titles such as "Paste". + private static func performTargetPasteCommand(in application: NSRunningApplication) -> Bool? { + guard AXIsProcessTrusted() else { return nil } + let deadline = Date().addingTimeInterval(targetPasteAXTraversalBudget) + let appElement = AXUIElementCreateApplication(application.processIdentifier) + guard let menuBar = axElement( + appElement, + attribute: kAXMenuBarAttribute as String, + deadline: deadline + ), + let pasteItem = standardPasteMenuItem( + in: menuBar, + maxDepth: 4, + deadline: deadline, + visited: [] + ), + configureTargetPasteTimeout(for: pasteItem, deadline: deadline) + else { return nil } + // Menu enabled state is lazily validated by AppKit/Electron and can be stale + // while the menu is closed. AXPress is the authoritative acceptance signal. + return AXUIElementPerformAction(pasteItem, kAXPressAction as CFString) == .success + } + + private static func standardPasteMenuItem( + in element: AXUIElement, + maxDepth: Int, + deadline: Date, + visited: Set + ) -> AXUIElement? { + guard maxDepth >= 0, + Date() < deadline, + !visited.contains(element) else { return nil } + var visited = visited + visited.insert(element) + + guard let role = axString( + element, + attribute: kAXRoleAttribute as String, + deadline: deadline + ), + let commandCharacter = axString( + element, + attribute: kAXMenuItemCmdCharAttribute as String, + deadline: deadline + ) else { return nil } + let commandModifiers = axInt( + element, + attribute: kAXMenuItemCmdModifiersAttribute as String, + deadline: deadline + ) + if role == (kAXMenuItemRole as String), + commandCharacter.caseInsensitiveCompare("V") == .orderedSame, + commandModifiers == 0 { + return element + } + + guard let children = axChildren(element, deadline: deadline) else { return nil } + for child in children { + guard Date() < deadline else { return nil } + if let match = standardPasteMenuItem( + in: child, + maxDepth: maxDepth - 1, + deadline: deadline, + visited: visited + ) { + return match + } + } + return nil + } + + static func targetPasteAXTimeout(until deadline: Date, now: Date = Date()) -> Float? { + let remaining = Float(deadline.timeIntervalSince(now)) + guard remaining > 0 else { return nil } + return min(targetPasteAXMaximumRequestTimeout, remaining) + } + + private static func configureTargetPasteTimeout( + for element: AXUIElement, + deadline: Date + ) -> Bool { + guard let timeout = targetPasteAXTimeout(until: deadline) else { return false } + return AXUIElementSetMessagingTimeout(element, timeout) == .success + && Date() < deadline + } + + private static func axElement( + _ element: AXUIElement, + attribute: String, + deadline: Date + ) -> AXUIElement? { + guard configureTargetPasteTimeout(for: element, deadline: deadline) else { return nil } + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, + let value, + CFGetTypeID(value) == AXUIElementGetTypeID() + else { return nil } + return (value as! AXUIElement) + } + + private static func axChildren(_ element: AXUIElement, deadline: Date) -> [AXUIElement]? { + guard configureTargetPasteTimeout(for: element, deadline: deadline) else { return nil } + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue( + element, + kAXChildrenAttribute as CFString, + &value + ) == .success, + let children = value as? [AXUIElement] else { return [] } + return children + } + + private static func axString( + _ element: AXUIElement, + attribute: String, + deadline: Date + ) -> String? { + guard configureTargetPasteTimeout(for: element, deadline: deadline) else { return nil } + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else { + return "" + } + return value as? String ?? "" + } + + private static func axInt( + _ element: AXUIElement, + attribute: String, + deadline: Date + ) -> Int? { + guard configureTargetPasteTimeout(for: element, deadline: deadline) else { return nil } + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, + let number = value as? NSNumber else { return nil } + return number.intValue + } + private static func postPhysicalKey(source: CGEventSource, keyCode: CGKeyCode, flags: CGEventFlags) { guard let keyDown = CGEvent(keyboardEventSource: source, virtualKey: keyCode, keyDown: true), let keyUp = CGEvent(keyboardEventSource: source, virtualKey: keyCode, keyDown: false) diff --git a/native/MuesliNative/Tests/MuesliTests/DictationStoreTests.swift b/native/MuesliNative/Tests/MuesliTests/DictationStoreTests.swift index 33898c8ad..20492b51b 100644 --- a/native/MuesliNative/Tests/MuesliTests/DictationStoreTests.swift +++ b/native/MuesliNative/Tests/MuesliTests/DictationStoreTests.swift @@ -3386,6 +3386,7 @@ struct DictationStoreTests { let row = try #require(try store.dictation(id: dictationID)) #expect(row.source == "quil") #expect(row.rawText == "- First point\n- Second point") + #expect(row.wordCount == 5) #expect(row.targetAppName == "Notes") #expect(row.computerUseTrace?.events.map(\.title) == [ "Original highlighted text", @@ -3407,6 +3408,95 @@ struct DictationStoreTests { #expect(try store.searchDictations(query: "bullet points").map(\.id).contains(dictationID)) } + @Test("Quill statistics count only the spoken rewrite instruction") + func quillStatisticsCountSpokenInstruction() throws { + let store = try makeStore() + let now = Date(timeIntervalSince1970: 1_777_000_050) + try store.insertDictation( + text: "ordinary dictation", + durationSeconds: 60, + startedAt: now.addingTimeInterval(-60), + endedAt: now + ) + try store.insertQuilDictation( + outputText: "generated output has many words that must never count", + originalText: "highlighted source text must not count either", + instruction: "Rewrite this politely", + backend: "gemma4LiteRT", + model: "gemma-4-e4b-it", + durationSeconds: 60, + startedAt: now.addingTimeInterval(-60), + endedAt: now + ) + + let stats = try store.dictationStats() + #expect(stats.totalWords == 5) + #expect(stats.totalSessions == 2) + #expect(stats.averageWordsPerSession == 2.5) + #expect(stats.averageWPM == 2.5) + } + + @Test("migration repairs existing Quill output word counts") + func migrationRepairsExistingQuillWordCounts() throws { + let store = try makeStore() + let now = Date(timeIntervalSince1970: 1_777_000_075) + let dictationID = try store.insertQuilDictation( + outputText: "a deliberately verbose generated response that should not count", + originalText: "source", + instruction: "Make concise", + backend: "local", + model: "qwen35-0.8b", + durationSeconds: 1, + startedAt: now.addingTimeInterval(-1), + endedAt: now + ) + + var db: OpaquePointer? + #expect(sqlite3_open(store.databasePath().path, &db) == SQLITE_OK) + #expect(sqlite3_exec(db, "UPDATE dictations SET word_count = 99 WHERE id = \(dictationID)", nil, nil, nil) == SQLITE_OK) + #expect(sqlite3_exec(db, "DELETE FROM local_migrations WHERE identifier = 'quill_statistics_spoken_instruction_v1'", nil, nil, nil) == SQLITE_OK) + sqlite3_close(db) + + try store.migrateIfNeeded() + + #expect(try store.dictation(id: dictationID)?.wordCount == 2) + #expect(try store.dictationStats().totalWords == 2) + } + + @Test("migration preserves synced Quill counts when the local trace is absent") + func migrationPreservesTraceFreeSyncedQuillWordCounts() throws { + let store = try makeStore() + let now = Date(timeIntervalSince1970: 1_777_000_090) + let dictationID = try store.insertQuilDictation( + outputText: "Generated output that is not available as analytics context", + originalText: "source", + instruction: "Rewrite with warmth", + backend: "gemma4LiteRT", + model: "gemma-4-e4b-it", + durationSeconds: 1, + startedAt: now.addingTimeInterval(-1), + endedAt: now + ) + + var db: OpaquePointer? + #expect(sqlite3_open(store.databasePath().path, &db) == SQLITE_OK) + #expect(sqlite3_exec( + db, + "UPDATE dictations SET word_count = 3, cloud_record_name = 'quill-sync-test', sync_dirty = 0 WHERE id = \(dictationID)", + nil, + nil, + nil + ) == SQLITE_OK) + #expect(sqlite3_exec(db, "DELETE FROM computer_use_traces WHERE dictation_id = \(dictationID)", nil, nil, nil) == SQLITE_OK) + #expect(sqlite3_exec(db, "DELETE FROM local_migrations WHERE identifier = 'quill_statistics_spoken_instruction_v1'", nil, nil, nil) == SQLITE_OK) + sqlite3_close(db) + + try store.migrateIfNeeded() + + #expect(try store.dictation(id: dictationID)?.wordCount == 3) + #expect(try store.textRecordsNeedingSync().isEmpty) + } + @Test("Quill generation at cursor persists its mode and spoken instruction") func quilGenerationAtCursorHydrates() throws { let store = try makeStore() @@ -3440,6 +3530,42 @@ struct DictationStoreTests { #expect(try store.searchDictations(query: "friendly reminder").map(\.id).contains(dictationID)) } + @Test("Quill persists generated output when automatic paste needs attention") + func quilPasteFallbackHydrates() throws { + let store = try makeStore() + let now = Date(timeIntervalSince1970: 1_777_000_200) + let dictationID = try store.insertQuilDictation( + outputText: "Generated text ready to paste", + originalText: "", + instruction: "Draft a short update", + backend: "gemma4-litert", + model: "gemma-4-e4b-it", + durationSeconds: 1.2, + targetAppName: "Google Chrome", + targetAppBundleID: "com.google.Chrome", + finalStatus: "needs_attention", + finalMessage: "Generated text is ready for manual paste", + additionalTraceEvents: [ + ComputerUseTraceEvent( + kind: "quil_delivery", + title: "Delivery", + body: "Automatic paste was not accepted; generated text was retained on the clipboard" + ), + ], + startedAt: now.addingTimeInterval(-1.2), + endedAt: now + ) + + let row = try #require(try store.dictation(id: dictationID)) + let trace = try #require(row.computerUseTrace) + #expect(row.rawText == "Generated text ready to paste") + #expect(trace.finalStatus == "needs_attention") + #expect(trace.finalMessage == "Generated text is ready for manual paste") + #expect(trace.events.last?.kind == "quil_delivery") + #expect(trace.events.last?.title == "Delivery") + #expect(trace.events.last?.body.contains("retained on the clipboard") == true) + } + @Test("insertComputerUseTrace replaces existing trace atomically") func insertComputerUseTraceReplacesExistingTrace() throws { let store = try makeStore() diff --git a/native/MuesliNative/Tests/MuesliTests/InsightsTests.swift b/native/MuesliNative/Tests/MuesliTests/InsightsTests.swift index 934bc7185..4f47c8705 100644 --- a/native/MuesliNative/Tests/MuesliTests/InsightsTests.swift +++ b/native/MuesliNative/Tests/MuesliTests/InsightsTests.swift @@ -81,6 +81,39 @@ struct InsightsTests { #expect(snapshot.selected.averageWPM == 2) } + @Test("Quill insights use the spoken prompt and exclude model output") + func quillInsightsUseSpokenPrompt() throws { + let store = try makeStore() + let now = Date(timeIntervalSince1970: 1_784_092_800) + try store.insertDictation( + text: "spokenanchor", + durationSeconds: 60, + startedAt: now.addingTimeInterval(-60), + endedAt: now + ) + try store.insertQuilDictation( + outputText: "generatedartifact generatedartifact generatedartifact", + originalText: "highlightartifact highlightartifact", + instruction: "rewriteprompt concisely", + backend: "gemma4LiteRT", + model: "gemma-4-e4b-it", + durationSeconds: 60, + startedAt: now.addingTimeInterval(-60), + endedAt: now + ) + + let snapshot = try store.insightsSnapshot(range: .allTime, now: now) + let words = Set(snapshot.dictationWords.map(\.word)) + + #expect(snapshot.lifetime.dictationWords == 3) + #expect(snapshot.lifetime.dictationSessions == 2) + #expect(snapshot.lifetime.averageWPM == 1.5) + #expect(words.contains("spokenanchor")) + #expect(words.contains("rewriteprompt")) + #expect(!words.contains("generatedartifact")) + #expect(!words.contains("highlightartifact")) + } + @Test("selected average pace uses the caller's local-day boundary") func selectedAveragePaceUsesLocalDayBoundary() throws { let store = try makeStore() diff --git a/native/MuesliNative/Tests/MuesliTests/PasteControllerTests.swift b/native/MuesliNative/Tests/MuesliTests/PasteControllerTests.swift index 92f0afb87..8f6397225 100644 --- a/native/MuesliNative/Tests/MuesliTests/PasteControllerTests.swift +++ b/native/MuesliNative/Tests/MuesliTests/PasteControllerTests.swift @@ -157,6 +157,33 @@ struct PasteControllerTests { #expect(pasteboard.string(forType: .string) == "original") } + @Test("Quill cancellation retains generated text for manual paste") + func pasteCancellationRetainsClipboardFallback() async { + let pasteboard = makePasteboard() + pasteboard.clearContents() + pasteboard.setString("original", forType: .string) + var lifecycleEvents: [PasteController.LifecycleEvent] = [] + + let result = await withCheckedContinuation { continuation in + PasteController.paste( + text: "Quill output", + pasteboard: pasteboard, + requireStagedClipboardOwnership: true, + shouldDispatchPaste: { false }, + retainStagedTextOnFailure: true, + onPasteFinished: { application in + continuation.resume(returning: application) + }, + onLifecycleEvent: { lifecycleEvents.append($0) } + ) + } + + #expect(result == nil) + #expect(lifecycleEvents.contains(.pasteDispatchCancelled)) + #expect(lifecycleEvents.contains(.clipboardRetainedForManualPaste)) + #expect(pasteboard.string(forType: .string) == "Quill output") + } + @Test("paste reports the application snapshotted at Cmd+V dispatch") func pasteReportsApplicationAtCommandDispatch() async { let pasteboard = makePasteboard() @@ -235,6 +262,107 @@ struct PasteControllerTests { _ = await waitForClipboardString(in: failedPasteboard, expected: nil) } + @Test("target application Paste command bypasses the global keyboard shortcut") + func targetApplicationPasteCommandDispatchesDirectly() async { + let pasteboard = makePasteboard() + pasteboard.clearContents() + pasteboard.setString("original", forType: .string) + let expectedApplication = NSRunningApplication.current + var didPostKeyboardShortcut = false + + let result = await withCheckedContinuation { continuation in + var lifecycleEvents: [PasteController.LifecycleEvent] = [] + PasteController.paste( + text: "Quill output", + pasteboard: pasteboard, + targetApplicationProvider: { expectedApplication }, + dispatchStrategy: .targetApplicationPasteCommand, + targetPasteAction: { application in + #expect(application.processIdentifier == expectedApplication.processIdentifier) + return true + }, + simulatePasteAction: { + didPostKeyboardShortcut = true + return true + }, + onPasteFinished: { application in + continuation.resume(returning: ( + application?.processIdentifier, + lifecycleEvents + )) + }, + onLifecycleEvent: { lifecycleEvents.append($0) } + ) + } + + #expect(result.0 == expectedApplication.processIdentifier) + #expect(!didPostKeyboardShortcut) + #expect(result.1.contains(.targetPasteCommandDispatched)) + #expect(result.1.contains(.pasteDispatched)) + _ = await waitForClipboardString(in: pasteboard, expected: "original") + } + + @Test("target Paste Accessibility requests use only the remaining traversal budget") + func targetPasteAXTimeoutUsesRemainingBudget() { + let now = Date(timeIntervalSince1970: 1_777_000_000) + + #expect(PasteController.targetPasteAXTimeout( + until: now.addingTimeInterval(1), + now: now + ) == 0.1) + let nearlyExpired = PasteController.targetPasteAXTimeout( + until: now.addingTimeInterval(0.025), + now: now + ) + #expect(nearlyExpired != nil) + #expect(abs((nearlyExpired ?? 0) - 0.025) < 0.000_001) + #expect(PasteController.targetPasteAXTimeout(until: now, now: now) == nil) + #expect(PasteController.targetPasteAXTimeout( + until: now.addingTimeInterval(-1), + now: now + ) == nil) + } + + @Test("rejected target Paste command retains Quill output for manual paste") + func rejectedTargetPasteCommandRetainsClipboardFallback() async throws { + let pasteboard = makePasteboard() + pasteboard.clearContents() + pasteboard.setString("original", forType: .string) + var didPostKeyboardShortcut = false + + let result = await withCheckedContinuation { continuation in + var lifecycleEvents: [PasteController.LifecycleEvent] = [] + PasteController.paste( + text: "Quill output", + pasteboard: pasteboard, + targetApplicationProvider: { NSRunningApplication.current }, + dispatchStrategy: .targetApplicationPasteCommand, + retainStagedTextOnFailure: true, + targetPasteAction: { _ in false }, + simulatePasteAction: { + didPostKeyboardShortcut = true + return true + }, + onPasteFinished: { application in + continuation.resume(returning: (application, lifecycleEvents)) + }, + onLifecycleEvent: { lifecycleEvents.append($0) } + ) + } + + #expect(result.0 == nil) + #expect(!didPostKeyboardShortcut) + #expect(result.1 == [ + .clipboardStaged, + .targetSnapshotted, + .targetPasteCommandRejected, + .pasteDispatchFailed, + .clipboardRetainedForManualPaste, + ]) + try await Task.sleep(nanoseconds: 700_000_000) + #expect(pasteboard.string(forType: .string) == "Quill output") + } + @Test("paste arms restoration before completion bookkeeping and settles afterward") func pasteRestorationOwnsCriticalPath() async { let pasteboard = makePasteboard() @@ -279,6 +407,9 @@ struct PasteControllerTests { "clipboard_staged", "clipboard_stage_failed", "target_snapshotted", + "target_paste_command_dispatched", + "target_paste_command_unavailable", + "target_paste_command_rejected", "paste_dispatched", "paste_dispatch_failed", "paste_dispatch_cancelled", @@ -286,6 +417,7 @@ struct PasteControllerTests { "clipboard_restore_scheduled", "clipboard_restored", "clipboard_restore_skipped", + "clipboard_retained_for_manual_paste", ]) }