From 851878767b33329275d751662cf3ba1cf605a896 Mon Sep 17 00:00:00 2001 From: Peyton-Spencer Date: Sat, 29 Aug 2026 00:38:01 -0400 Subject: [PATCH 1/8] feat(desktop): send Discord replies with macOS Accessibility --- .gitignore | 1 + .../native/discord-accessibility/main.swift | 351 ++++++++++++++++++ .../build-discord-accessibility-helper.mjs | 48 +++ .../DiscordAccessibilityTransport.test.ts | 94 +++++ .../discord/DiscordAccessibilityTransport.ts | 287 ++++++++++++++ apps/desktop/src/ipc/DesktopIpcHandlers.ts | 8 + apps/desktop/src/ipc/channels.ts | 3 + .../src/ipc/methods/discordAccessibility.ts | 65 ++++ apps/desktop/src/preload.ts | 8 + apps/desktop/vite.config.ts | 5 +- apps/web/src/components/inbox/InboxPage.tsx | 124 ++++++- apps/web/src/localApi.ts | 3 + docs/ditto-desktop-v2-plan.md | 1 + .../discord-accessibility-transport.md | 70 ++++ docs/user/discord-accessibility-replies.md | 27 ++ packages/contracts/src/channels.ts | 71 ++++ packages/contracts/src/ipc.ts | 16 + scripts/build-desktop-artifact.test.ts | 5 + scripts/build-desktop-artifact.ts | 25 ++ 19 files changed, 1205 insertions(+), 7 deletions(-) create mode 100644 apps/desktop/native/discord-accessibility/main.swift create mode 100644 apps/desktop/scripts/build-discord-accessibility-helper.mjs create mode 100644 apps/desktop/src/discord/DiscordAccessibilityTransport.test.ts create mode 100644 apps/desktop/src/discord/DiscordAccessibilityTransport.ts create mode 100644 apps/desktop/src/ipc/methods/discordAccessibility.ts create mode 100644 docs/internals/discord-accessibility-transport.md create mode 100644 docs/user/discord-accessibility-replies.md diff --git a/.gitignore b/.gitignore index 57262578a786..b309891a44af 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ squashfs-root/ .gstack/ .plans/ dist-electron/ +apps/desktop/.native/ .electron-runtime/ .showcase/ apps/mobile/.showcase/ diff --git a/apps/desktop/native/discord-accessibility/main.swift b/apps/desktop/native/discord-accessibility/main.swift new file mode 100644 index 000000000000..25c81bd1a4bc --- /dev/null +++ b/apps/desktop/native/discord-accessibility/main.swift @@ -0,0 +1,351 @@ +import AppKit +import ApplicationServices +import Darwin +import Foundation + +private let discordBundleIDs: Set = [ + "com.hnc.Discord", + "com.hnc.DiscordPTB", + "com.hnc.DiscordCanary", +] + +private struct HelperCommand: Decodable { + let command: String + let prompt: Bool? + let actionId: String? + let origin: String? + let mode: String? + let deepLink: String? + let expectedTitle: String? + let text: String? + let timeoutMs: Int? +} + +private struct StatusResponse: Encodable { + let available: Bool + let permission: String + let detail: String +} + +private struct ReplyResponse: Encodable { + let actionId: String + let origin: String + let mode: String + let outcome: String + let permission: String + let startedAt: String + let completedAt: String + let detail: String + let sent: Bool + let draftPrepared: Bool + let duplicate: Bool +} + +private let isoFormatter = ISO8601DateFormatter() + +private func emit(_ value: T) -> Never { + do { + let data = try JSONEncoder().encode(value) + FileHandle.standardOutput.write(data) + exit(EXIT_SUCCESS) + } catch { + FileHandle.standardError.write(Data("Unable to encode helper result.\n".utf8)) + exit(EXIT_FAILURE) + } +} + +private func trusted(prompt: Bool) -> Bool { + if prompt { + let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary + return AXIsProcessTrustedWithOptions(options) + } + return AXIsProcessTrusted() +} + +private func copyAttribute(_ element: AXUIElement, _ attribute: CFString) -> CFTypeRef? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, attribute, &value) == .success else { return nil } + return value +} + +private func stringAttribute(_ element: AXUIElement, _ attribute: CFString) -> String? { + copyAttribute(element, attribute) as? String +} + +private func boolAttribute(_ element: AXUIElement, _ attribute: CFString) -> Bool { + (copyAttribute(element, attribute) as? Bool) ?? false +} + +private func childElements(_ element: AXUIElement) -> [AXUIElement] { + (copyAttribute(element, kAXChildrenAttribute as CFString) as? [AXUIElement]) ?? [] +} + +private func normalizeTitle(_ value: String) -> String { + var normalized = value.folding(options: [.caseInsensitive, .diacriticInsensitive], locale: .current) + normalized = normalized.replacingOccurrences(of: "message to ", with: "") + normalized = normalized.replacingOccurrences(of: "message ", with: "") + normalized = normalized.replacingOccurrences(of: "@", with: "") + normalized = normalized.replacingOccurrences(of: "#", with: "") + normalized = normalized.unicodeScalars.map { scalar in + CharacterSet.alphanumerics.contains(scalar) ? Character(String(scalar)) : " " + }.reduce(into: "") { $0.append($1) } + return normalized.split(whereSeparator: { $0.isWhitespace }).joined(separator: " ") +} + +private struct ComposerMatch { + let element: AXUIElement + let descriptor: String +} + +private func matchingComposers(in root: AXUIElement, expectedTitle: String) -> [ComposerMatch] { + let expected = normalizeTitle(expectedTitle) + var queue: [(AXUIElement, Int)] = [(root, 0)] + var visited = 0 + var matches: [ComposerMatch] = [] + + while !queue.isEmpty && visited < 3_000 { + let (element, depth) = queue.removeFirst() + visited += 1 + let role = stringAttribute(element, kAXRoleAttribute as CFString) ?? "" + if role == (kAXTextAreaRole as String) || role == (kAXTextFieldRole as String) { + let descriptors = [ + stringAttribute(element, kAXPlaceholderValueAttribute as CFString), + stringAttribute(element, kAXDescriptionAttribute as CFString), + stringAttribute(element, kAXHelpAttribute as CFString), + stringAttribute(element, kAXTitleAttribute as CFString), + ].compactMap { $0 } + if let descriptor = descriptors.first(where: { normalizeTitle($0) == expected }) { + matches.append(ComposerMatch(element: element, descriptor: descriptor)) + } + } + if depth < 14 { + queue.append(contentsOf: childElements(element).map { ($0, depth + 1) }) + } + } + return matches +} + +private func discordApplication(deadline: Date) -> NSRunningApplication? { + while Date() < deadline { + if let app = NSWorkspace.shared.runningApplications.first(where: { + guard let bundleIdentifier = $0.bundleIdentifier else { return false } + return discordBundleIDs.contains(bundleIdentifier) + }) { + return app + } + RunLoop.current.run(until: Date().addingTimeInterval(0.05)) + } + return nil +} + +private func verifiedComposer(app: NSRunningApplication, expectedTitle: String, deadline: Date) -> ComposerMatch? { + let appElement = AXUIElementCreateApplication(app.processIdentifier) + while Date() < deadline { + let windows = (copyAttribute(appElement, kAXWindowsAttribute as CFString) as? [AXUIElement]) ?? [] + let matches = windows.flatMap { matchingComposers(in: $0, expectedTitle: expectedTitle) } + if matches.count == 1 { + return matches[0] + } + RunLoop.current.run(until: Date().addingTimeInterval(0.08)) + } + return nil +} + +private func supportsConfirm(_ element: AXUIElement) -> Bool { + var names: CFArray? + guard AXUIElementCopyActionNames(element, &names) == .success, + let actions = names as? [String] + else { return false } + return actions.contains(kAXConfirmAction as String) +} + +private func performSend(_ composer: AXUIElement, application: AXUIElement) -> Bool { + if supportsConfirm(composer) { + return AXUIElementPerformAction(composer, kAXConfirmAction as CFString) == .success + } + guard AXUIElementSetAttributeValue( + composer, + kAXFocusedAttribute as CFString, + true as CFTypeRef + ) == .success, + boolAttribute(composer, kAXFocusedAttribute as CFString) + else { return false } + let returnKeyCode: CGKeyCode = 36 + typealias PostKeyboardEvent = @convention(c) ( + AXUIElement, + UInt16, + CGKeyCode, + UInt8 + ) -> AXError + guard let symbol = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "AXUIElementPostKeyboardEvent") + else { return false } + let postKeyboardEvent = unsafeBitCast(symbol, to: PostKeyboardEvent.self) + guard postKeyboardEvent(application, 0, returnKeyCode, 1) == .success else { + return false + } + return postKeyboardEvent(application, 0, returnKeyCode, 0) == .success +} + +private func composerValue(_ element: AXUIElement) -> String { + stringAttribute(element, kAXValueAttribute as CFString) ?? "" +} + +private func restore(_ application: NSRunningApplication?) { + guard let application, !application.isTerminated else { return } + application.activate(options: [.activateIgnoringOtherApps]) +} + +private func execute(_ command: HelperCommand) -> ReplyResponse { + let startedAt = isoFormatter.string(from: Date()) + let actionId = command.actionId ?? "invalid" + let origin = command.origin ?? "local_desktop" + let mode = command.mode ?? "prepare" + func result( + _ outcome: String, + _ detail: String, + sent: Bool = false, + draftPrepared: Bool = false, + permission: String = "granted" + ) -> ReplyResponse { + ReplyResponse( + actionId: actionId, + origin: origin, + mode: mode, + outcome: outcome, + permission: permission, + startedAt: startedAt, + completedAt: isoFormatter.string(from: Date()), + detail: detail, + sent: sent, + draftPrepared: draftPrepared, + duplicate: false + ) + } + + guard trusted(prompt: false) else { + return result( + "permission_required", + "Allow Ditto's Discord helper in System Settings > Privacy & Security > Accessibility.", + permission: "not_granted" + ) + } + guard let deepLink = command.deepLink, + let url = URL(string: deepLink), + url.scheme == "discord", + url.host == "-", + url.user == nil, + url.password == nil, + url.port == nil, + url.query == nil, + url.fragment == nil, + let expectedTitle = command.expectedTitle, + !normalizeTitle(expectedTitle).isEmpty, + let text = command.text, + !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, + text.count <= 4_000 + else { + return result("target_not_verified", "The Discord target or draft did not pass validation.") + } + let path = url.pathComponents.filter { $0 != "/" } + let snowflake = try? NSRegularExpression(pattern: "^[0-9]{15,24}$") + func isSnowflake(_ value: String) -> Bool { + guard let snowflake else { return false } + return snowflake.firstMatch(in: value, range: NSRange(value.startIndex..., in: value)) != nil + } + guard path.count == 3, + path[0] == "channels", + (path[1] == "@me" || isSnowflake(path[1])), + isSnowflake(path[2]) + else { + return result("target_not_verified", "The Discord deep link was rejected.") + } + + let previousApplication = NSWorkspace.shared.frontmostApplication + defer { restore(previousApplication) } + guard NSWorkspace.shared.open(url) else { + return result("discord_unavailable", "Discord could not open the requested conversation.") + } + let timeout = min(max(command.timeoutMs ?? 10_000, 1_000), 15_000) + let deadline = Date().addingTimeInterval(Double(timeout) / 1_000) + guard let discord = discordApplication(deadline: deadline) else { + return result("discord_unavailable", "Discord did not become available before the action timed out.") + } + guard let match = verifiedComposer(app: discord, expectedTitle: expectedTitle, deadline: deadline) else { + return result( + "target_not_verified", + "Discord opened, but Ditto could not verify the exact conversation composer. Nothing was typed." + ) + } + let discordElement = AXUIElementCreateApplication(discord.processIdentifier) + + guard AXUIElementSetAttributeValue( + match.element, + kAXValueAttribute as CFString, + text as CFTypeRef + ) == .success, + composerValue(match.element) == text + else { + return result("composer_not_found", "The verified Discord composer did not accept the draft.") + } + + if mode == "prepare" { + return result( + "draft_prepared", + "Draft prepared in the verified Discord conversation. Open Discord and press Enter to send.", + draftPrepared: true + ) + } + guard mode == "send" else { + return result( + "draft_prepared", + "The verified Discord draft was left ready instead of being sent.", + draftPrepared: true + ) + } + guard performSend(match.element, application: discordElement) else { + return result( + "draft_prepared", + "Discord rejected the Accessibility send action. The verified draft was left ready instead.", + draftPrepared: true + ) + } + let confirmationDeadline = min(deadline, Date().addingTimeInterval(1.5)) + while Date() < confirmationDeadline { + if composerValue(match.element).isEmpty { + return result( + "sent", + "Discord cleared the verified composer after its Accessibility confirm action.", + sent: true + ) + } + RunLoop.current.run(until: Date().addingTimeInterval(0.05)) + } + return result( + "send_not_confirmed", + "Discord did not clear the composer, so Ditto cannot confirm that the message was sent.", + draftPrepared: true + ) +} + +let inputData = FileHandle.standardInput.readDataToEndOfFile() +guard let command = try? JSONDecoder().decode(HelperCommand.self, from: inputData) else { + FileHandle.standardError.write(Data("Invalid helper command.\n".utf8)) + exit(EXIT_FAILURE) +} + +switch command.command { +case "status": + let isTrusted = trusted(prompt: command.prompt ?? false) + emit(StatusResponse( + available: true, + permission: isTrusted ? "granted" : "not_granted", + detail: isTrusted + ? "Ditto can prepare and send explicit Discord replies." + : "Allow Ditto's Discord helper in System Settings > Privacy & Security > Accessibility." + )) +case "execute": + emit(execute(command)) +default: + FileHandle.standardError.write(Data("Unsupported helper command.\n".utf8)) + exit(EXIT_FAILURE) +} diff --git a/apps/desktop/scripts/build-discord-accessibility-helper.mjs b/apps/desktop/scripts/build-discord-accessibility-helper.mjs new file mode 100644 index 000000000000..966e6971f91a --- /dev/null +++ b/apps/desktop/scripts/build-discord-accessibility-helper.mjs @@ -0,0 +1,48 @@ +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; + +const desktopRoot = NodePath.resolve( + NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), + "..", +); +// oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone native build script has no Effect runtime. +if (process.platform !== "darwin") process.exit(0); + +const archArgumentIndex = process.argv.indexOf("--arch"); +const outputArgumentIndex = process.argv.indexOf("--output"); +// oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone native build script has no Effect runtime. +const arch = archArgumentIndex >= 0 ? process.argv[archArgumentIndex + 1] : process.arch; +const output = + outputArgumentIndex >= 0 + ? NodePath.resolve(process.argv[outputArgumentIndex + 1]) + : NodePath.join(desktopRoot, ".native", "ditto-discord-ax"); +if (arch !== "arm64" && arch !== "x64") { + throw new Error(`Unsupported macOS helper architecture: ${String(arch)}`); +} + +NodeFS.mkdirSync(NodePath.dirname(output), { recursive: true }); +const target = `${arch === "x64" ? "x86_64" : "arm64"}-apple-macos13.0`; +const source = NodePath.join(desktopRoot, "native", "discord-accessibility", "main.swift"); +const result = NodeChildProcess.spawnSync( + "xcrun", + [ + "swiftc", + "-O", + "-target", + target, + "-framework", + "AppKit", + "-framework", + "ApplicationServices", + source, + "-o", + output, + ], + { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, +); +if (result.status !== 0) { + throw new Error(result.stderr || `swiftc exited ${String(result.status)}`); +} +NodeFS.chmodSync(output, 0o755); diff --git a/apps/desktop/src/discord/DiscordAccessibilityTransport.test.ts b/apps/desktop/src/discord/DiscordAccessibilityTransport.test.ts new file mode 100644 index 000000000000..999f806e5ab1 --- /dev/null +++ b/apps/desktop/src/discord/DiscordAccessibilityTransport.test.ts @@ -0,0 +1,94 @@ +import type { DiscordAccessibilityReplyInput } from "@t3tools/contracts"; +import { assert, describe, it } from "@effect/vitest"; + +import { + DiscordAccessibilityTransport, + type DiscordAccessibilityHelperRunner, +} from "./DiscordAccessibilityTransport.ts"; + +const input = { + actionId: "reply-action-123", + origin: "local_desktop", + requestedAt: "2026-08-29T12:00:00.000Z", + accountId: "discord-local", + conversationId: "1531175553309343915", + containerId: "1073292100218654821", + conversationTitle: "general", + text: "hello from Ditto", + mode: "send", +} as DiscordAccessibilityReplyInput; + +describe("DiscordAccessibilityTransport", () => { + it("validates a Discord target before invoking the native helper", async () => { + let calls = 0; + const runner: DiscordAccessibilityHelperRunner = { + run: async () => { + calls += 1; + return {}; + }, + }; + const transport = new DiscordAccessibilityTransport("darwin", runner); + const result = await transport.execute({ ...input, conversationId: "../../settings" } as never); + assert.equal(result.outcome, "failed"); + assert.equal(calls, 0); + }); + + it("forwards only the validated deep link and returns an audited receipt", async () => { + let deepLink = ""; + const runner: DiscordAccessibilityHelperRunner = { + run: async (command) => { + if (command.command !== "execute") throw new Error("unexpected command"); + deepLink = command.deepLink; + return { + actionId: input.actionId, + origin: input.origin, + mode: input.mode, + outcome: "sent", + permission: "granted", + startedAt: "2026-08-29T12:00:01.000Z", + completedAt: "2026-08-29T12:00:02.000Z", + detail: "Sent after Discord cleared the verified composer.", + sent: true, + draftPrepared: false, + duplicate: false, + }; + }, + }; + const result = await new DiscordAccessibilityTransport("darwin", runner).execute(input); + assert.equal(deepLink, "discord://-/channels/1073292100218654821/1531175553309343915"); + assert.isTrue(result.sent); + }); + + it("deduplicates a completed explicit action id", async () => { + let calls = 0; + const runner: DiscordAccessibilityHelperRunner = { + run: async () => { + calls += 1; + return { + actionId: input.actionId, + origin: input.origin, + mode: input.mode, + outcome: "draft_prepared", + permission: "granted", + startedAt: "2026-08-29T12:00:01.000Z", + completedAt: "2026-08-29T12:00:02.000Z", + detail: "Draft prepared.", + sent: false, + draftPrepared: true, + duplicate: false, + }; + }, + }; + const transport = new DiscordAccessibilityTransport("darwin", runner); + await transport.execute(input); + const duplicate = await transport.execute(input); + assert.equal(calls, 1); + assert.isTrue(duplicate.duplicate); + }); + + it("does not expose the transport on non-macOS hosts", async () => { + const runner: DiscordAccessibilityHelperRunner = { run: async () => ({}) }; + const result = await new DiscordAccessibilityTransport("linux", runner).execute(input); + assert.equal(result.outcome, "unsupported"); + }); +}); diff --git a/apps/desktop/src/discord/DiscordAccessibilityTransport.ts b/apps/desktop/src/discord/DiscordAccessibilityTransport.ts new file mode 100644 index 000000000000..f170e7b6ec82 --- /dev/null +++ b/apps/desktop/src/discord/DiscordAccessibilityTransport.ts @@ -0,0 +1,287 @@ +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off globalDate:off - This adapter owns one bounded native helper subprocess; its Promise interface is intentionally mockable without leaking process services into IPC contracts. +import type { + DiscordAccessibilityReplyInput, + DiscordAccessibilityReplyOutcome, + DiscordAccessibilityReplyResult, + DiscordAccessibilityStatus, +} from "@t3tools/contracts"; +import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; + +const DISCORD_SNOWFLAKE = /^\d{15,24}$/; +const ACTION_TIMEOUT_MS = 10_000; +const STATUS_TIMEOUT_MS = 3_000; +const MAX_RECEIPTS = 256; + +type HelperCommand = + | { readonly command: "status"; readonly prompt: boolean } + | { + readonly command: "execute"; + readonly actionId: string; + readonly origin: DiscordAccessibilityReplyInput["origin"]; + readonly mode: DiscordAccessibilityReplyInput["mode"]; + readonly deepLink: string; + readonly expectedTitle: string; + readonly text: string; + readonly timeoutMs: number; + }; + +export interface DiscordAccessibilityHelperRunner { + run( + command: HelperCommand, + timeoutMs: number, + onSpawn?: (cancel: () => void) => void, + ): Promise; +} + +function discordDeepLink(input: DiscordAccessibilityReplyInput): string | null { + if (!DISCORD_SNOWFLAKE.test(input.conversationId)) return null; + const scope = input.containerId ?? "@me"; + if (scope !== "@me" && !DISCORD_SNOWFLAKE.test(scope)) return null; + return `discord://-/channels/${scope}/${input.conversationId}`; +} + +function fallbackResult( + input: DiscordAccessibilityReplyInput, + outcome: DiscordAccessibilityReplyOutcome, + detail: string, + startedAt = new Date().toISOString(), +): DiscordAccessibilityReplyResult { + return { + actionId: input.actionId, + origin: input.origin, + mode: input.mode, + outcome, + permission: outcome === "permission_required" ? "not_granted" : "unavailable", + startedAt, + completedAt: new Date().toISOString(), + detail, + sent: false, + draftPrepared: false, + duplicate: false, + }; +} + +function isReplyResult( + value: unknown, + input: DiscordAccessibilityReplyInput, +): value is DiscordAccessibilityReplyResult { + if (typeof value !== "object" || value === null) return false; + const row = value as Record; + return ( + row.actionId === input.actionId && + row.origin === input.origin && + row.mode === input.mode && + typeof row.outcome === "string" && + typeof row.permission === "string" && + typeof row.startedAt === "string" && + typeof row.completedAt === "string" && + typeof row.detail === "string" && + typeof row.sent === "boolean" && + typeof row.draftPrepared === "boolean" && + typeof row.duplicate === "boolean" + ); +} + +function isStatus(value: unknown): value is DiscordAccessibilityStatus { + if (typeof value !== "object" || value === null) return false; + const row = value as Record; + return ( + typeof row.available === "boolean" && + (row.permission === "granted" || + row.permission === "not_granted" || + row.permission === "unavailable") && + typeof row.detail === "string" + ); +} + +export class NativeDiscordAccessibilityHelper implements DiscordAccessibilityHelperRunner { + readonly helperPath: string; + + constructor(helperPath: string) { + this.helperPath = helperPath; + } + + run( + command: HelperCommand, + timeoutMs: number, + onSpawn?: (cancel: () => void) => void, + ): Promise { + return new Promise((resolve, reject) => { + if (!NodeFS.existsSync(this.helperPath)) { + reject(new Error("The Discord Accessibility helper is not bundled in this build.")); + return; + } + + const child = NodeChildProcess.spawn(this.helperPath, [], { + stdio: ["pipe", "pipe", "pipe"], + windowsHide: true, + }); + let stdout = ""; + let stderr = ""; + let settled = false; + const finish = (callback: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + callback(); + }; + const cancel = () => { + if (!settled) child.kill("SIGTERM"); + }; + onSpawn?.(cancel); + const timer = setTimeout(() => { + child.kill("SIGTERM"); + finish(() => reject(new Error("Discord Accessibility action timed out."))); + }, timeoutMs); + + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + child.once("error", (cause) => finish(() => reject(cause))); + child.once("exit", (code, signal) => { + finish(() => { + if (signal === "SIGTERM") { + reject(new Error("Discord Accessibility action was cancelled.")); + return; + } + if (code !== 0) { + reject( + new Error(stderr.trim() || `Discord Accessibility helper exited ${String(code)}.`), + ); + return; + } + try { + resolve(JSON.parse(stdout)); + } catch { + reject(new Error("Discord Accessibility helper returned an invalid response.")); + } + }); + }); + child.stdin.end(JSON.stringify(command)); + }); + } +} + +export class DiscordAccessibilityTransport { + readonly #inFlight = new Map void>(); + readonly #receipts = new Map(); + readonly platform: NodeJS.Platform; + readonly runner: DiscordAccessibilityHelperRunner; + + constructor(platform: NodeJS.Platform, runner: DiscordAccessibilityHelperRunner) { + this.platform = platform; + this.runner = runner; + } + + async status(prompt = false): Promise { + if (this.platform !== "darwin") { + return { + available: false, + permission: "unavailable", + detail: "Discord Accessibility replies are available on macOS only.", + }; + } + try { + const value = await this.runner.run({ command: "status", prompt }, STATUS_TIMEOUT_MS); + return isStatus(value) + ? value + : { + available: false, + permission: "unavailable", + detail: "Discord Accessibility helper returned an invalid status.", + }; + } catch (cause) { + return { + available: false, + permission: "unavailable", + detail: cause instanceof Error ? cause.message : String(cause), + }; + } + } + + async execute(input: DiscordAccessibilityReplyInput): Promise { + const previous = this.#receipts.get(input.actionId); + if (previous) return { ...previous, duplicate: true }; + const startedAt = new Date().toISOString(); + if (this.platform !== "darwin") { + return this.#remember( + fallbackResult( + input, + "unsupported", + "Discord Accessibility replies require macOS.", + startedAt, + ), + ); + } + const deepLink = discordDeepLink(input); + if (deepLink === null || input.text.trim().length === 0) { + return this.#remember( + fallbackResult(input, "failed", "The Discord target or message is invalid.", startedAt), + ); + } + if (this.#inFlight.has(input.actionId)) { + return fallbackResult(input, "failed", "This reply action is already running.", startedAt); + } + + try { + const value = await this.runner.run( + { + command: "execute", + actionId: input.actionId, + origin: input.origin, + mode: input.mode, + deepLink, + expectedTitle: input.conversationTitle, + text: input.text, + timeoutMs: ACTION_TIMEOUT_MS, + }, + ACTION_TIMEOUT_MS + 1_000, + (cancel) => this.#inFlight.set(input.actionId, cancel), + ); + if (!isReplyResult(value, input)) { + return this.#remember( + fallbackResult( + input, + "failed", + "Discord Accessibility helper returned an invalid receipt.", + startedAt, + ), + ); + } + return this.#remember(value); + } catch (cause) { + const detail = cause instanceof Error ? cause.message : String(cause); + const outcome = detail.includes("cancelled") + ? "cancelled" + : detail.includes("timed out") + ? "timed_out" + : "failed"; + return this.#remember(fallbackResult(input, outcome, detail, startedAt)); + } finally { + this.#inFlight.delete(input.actionId); + } + } + + cancel(actionId: string): boolean { + const cancel = this.#inFlight.get(actionId); + if (!cancel) return false; + cancel(); + return true; + } + + #remember(result: DiscordAccessibilityReplyResult): DiscordAccessibilityReplyResult { + this.#receipts.set(result.actionId, result); + while (this.#receipts.size > MAX_RECEIPTS) { + const oldest = this.#receipts.keys().next().value; + if (oldest === undefined) break; + this.#receipts.delete(oldest); + } + return result; + } +} diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index 8e8317db7971..df8a7561e72d 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -45,6 +45,11 @@ import { showContextMenu, } from "./methods/window.ts"; import * as PreviewIpc from "./methods/preview.ts"; +import { + discordAccessibilityCancel, + discordAccessibilityExecute, + discordAccessibilityStatus, +} from "./methods/discordAccessibility.ts"; import { getWslState, setWslBackendEnabled, setWslDistro, setWslOnly } from "./methods/wsl.ts"; export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers")(function* () { @@ -88,6 +93,9 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(setTheme); yield* ipc.handle(showContextMenu); yield* ipc.handle(openExternal); + yield* ipc.handle(discordAccessibilityStatus); + yield* ipc.handle(discordAccessibilityExecute); + yield* ipc.handle(discordAccessibilityCancel); yield* ipc.handle(probeRemoteEditors); yield* ipc.handle(getUpdateState); yield* ipc.handle(setUpdateChannel); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index c4ef82ec8cb7..d771c83998a6 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -4,6 +4,9 @@ export const PICK_THEME_FILES_CHANNEL = "desktop:pick-theme-files"; export const SET_THEME_CHANNEL = "desktop:set-theme"; export const CONTEXT_MENU_CHANNEL = "desktop:context-menu"; export const OPEN_EXTERNAL_CHANNEL = "desktop:open-external"; +export const DISCORD_ACCESSIBILITY_STATUS_CHANNEL = "desktop:discord-accessibility-status"; +export const DISCORD_ACCESSIBILITY_EXECUTE_CHANNEL = "desktop:discord-accessibility-execute"; +export const DISCORD_ACCESSIBILITY_CANCEL_CHANNEL = "desktop:discord-accessibility-cancel"; export const PROBE_REMOTE_EDITORS_CHANNEL = "desktop:probe-remote-editors"; export const MENU_ACTION_CHANNEL = "desktop:menu-action"; export const QUIT_SHORTCUT_CHANNEL = "desktop:quit-shortcut"; diff --git a/apps/desktop/src/ipc/methods/discordAccessibility.ts b/apps/desktop/src/ipc/methods/discordAccessibility.ts new file mode 100644 index 000000000000..e5518945ce92 --- /dev/null +++ b/apps/desktop/src/ipc/methods/discordAccessibility.ts @@ -0,0 +1,65 @@ +import { + DiscordAccessibilityReplyInput, + DiscordAccessibilityReplyResult, + DiscordAccessibilityStatus, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import * as DesktopEnvironment from "../../app/DesktopEnvironment.ts"; +import { + DiscordAccessibilityTransport, + NativeDiscordAccessibilityHelper, +} from "../../discord/DiscordAccessibilityTransport.ts"; +import * as IpcChannels from "../channels.ts"; +import * as DesktopIpc from "../DesktopIpc.ts"; + +let cachedTransport: + | { readonly key: string; readonly transport: DiscordAccessibilityTransport } + | undefined; + +function resolveTransport( + environment: DesktopEnvironment.DesktopEnvironment["Service"], +): DiscordAccessibilityTransport { + const helperPath = environment.isPackaged + ? environment.path.join(environment.resourcesPath, "discord-accessibility", "ditto-discord-ax") + : environment.path.join(environment.rootDir, "apps", "desktop", ".native", "ditto-discord-ax"); + const key = `${environment.platform}:${helperPath}`; + if (cachedTransport?.key === key) return cachedTransport.transport; + const transport = new DiscordAccessibilityTransport( + environment.platform, + new NativeDiscordAccessibilityHelper(helperPath), + ); + cachedTransport = { key, transport }; + return transport; +} + +export const discordAccessibilityStatus = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.DISCORD_ACCESSIBILITY_STATUS_CHANNEL, + payload: Schema.Boolean, + result: DiscordAccessibilityStatus, + handler: Effect.fn("desktop.ipc.discordAccessibility.status")(function* (prompt) { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + return yield* Effect.promise(() => resolveTransport(environment).status(prompt)); + }), +}); + +export const discordAccessibilityExecute = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.DISCORD_ACCESSIBILITY_EXECUTE_CHANNEL, + payload: DiscordAccessibilityReplyInput, + result: DiscordAccessibilityReplyResult, + handler: Effect.fn("desktop.ipc.discordAccessibility.execute")(function* (input) { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + return yield* Effect.promise(() => resolveTransport(environment).execute(input)); + }), +}); + +export const discordAccessibilityCancel = DesktopIpc.makeIpcMethod({ + channel: IpcChannels.DISCORD_ACCESSIBILITY_CANCEL_CHANNEL, + payload: Schema.String.check(Schema.isMinLength(8), Schema.isMaxLength(128)), + result: Schema.Boolean, + handler: Effect.fn("desktop.ipc.discordAccessibility.cancel")(function* (actionId) { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + return resolveTransport(environment).cancel(actionId); + }), +}); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index d1313ff2e767..cf98de207cac 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -115,6 +115,14 @@ contextBridge.exposeInMainWorld("desktopBridge", { ...(position === undefined ? {} : { position }), }), openExternal: (url: string) => ipcRenderer.invoke(IpcChannels.OPEN_EXTERNAL_CHANNEL, url), + discordAccessibility: { + status: (prompt = false) => + ipcRenderer.invoke(IpcChannels.DISCORD_ACCESSIBILITY_STATUS_CHANNEL, prompt), + execute: (input) => + ipcRenderer.invoke(IpcChannels.DISCORD_ACCESSIBILITY_EXECUTE_CHANNEL, input), + cancel: (actionId) => + ipcRenderer.invoke(IpcChannels.DISCORD_ACCESSIBILITY_CANCEL_CHANNEL, actionId), + }, probeRemoteEditors: () => ipcRenderer.invoke(IpcChannels.PROBE_REMOTE_EDITORS_CHANNEL, undefined), onMenuAction: (listener) => { const wrappedListener = (_event: Electron.IpcRendererEvent, action: unknown) => { diff --git a/apps/desktop/vite.config.ts b/apps/desktop/vite.config.ts index 9f25204f1630..2de28733e80e 100644 --- a/apps/desktop/vite.config.ts +++ b/apps/desktop/vite.config.ts @@ -14,13 +14,14 @@ export default defineConfig({ run: { tasks: { build: { - command: "node scripts/build-preview-annotation-css.mjs && vp pack", + command: + "node scripts/build-discord-accessibility-helper.mjs && node scripts/build-preview-annotation-css.mjs && vp pack", dependsOn: ["t3#build"], cache: false, }, dev: { command: - "node scripts/build-preview-annotation-css.mjs && cross-env T3CODE_DESKTOP_DEV=1 vp pack --watch", + "node scripts/build-discord-accessibility-helper.mjs && node scripts/build-preview-annotation-css.mjs && cross-env T3CODE_DESKTOP_DEV=1 vp pack --watch", dependsOn: ["t3#build"], cache: false, }, diff --git a/apps/web/src/components/inbox/InboxPage.tsx b/apps/web/src/components/inbox/InboxPage.tsx index 5f0f8f8e6b92..c6584bb9373f 100644 --- a/apps/web/src/components/inbox/InboxPage.tsx +++ b/apps/web/src/components/inbox/InboxPage.tsx @@ -4,6 +4,8 @@ import type { ChannelConversation, ChannelMessage, ConnectedChannelAccount, + DiscordAccessibilityReplyResult, + DiscordAccessibilityStatus, EnvironmentId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -254,6 +256,8 @@ function MessagePanel({ (capability) => capability.operation === "message.send" && capability.availability === "available", ); + const canReplyThroughDiscord = + account.service === "discord" && readLocalApi()?.discordAccessibility !== undefined; const canLoadOlder = messages.length >= messageLimit && messageLimit < 5_000; const loadOlder = useCallback(() => { if (!canLoadOlder || loadingOlder) return; @@ -267,7 +271,11 @@ function MessagePanel({

{conversation.title}

- {account.service === "discord" ? "Device cache · read only" : "Messages on this Mac"} + {account.service === "discord" + ? canReplyThroughDiscord + ? "Device cache · replies through Discord" + : "Device cache · read only" + : "Messages on this Mac"}

@@ -544,12 +552,68 @@ function Composer({ const [text, setText] = useState(""); const [sending, setSending] = useState(false); const [error, setError] = useState(null); + const [accessibilityStatus, setAccessibilityStatus] = useState( + null, + ); + const [accessibilityResult, setAccessibilityResult] = + useState(null); + const [activeActionId, setActiveActionId] = useState(null); + const accessibility = + account.service === "discord" ? readLocalApi()?.discordAccessibility : undefined; + const canReply = canSend || accessibility !== undefined; + + useEffect(() => { + if (!accessibility) return; + let active = true; + void accessibility.status(false).then((status) => { + if (active) setAccessibilityStatus(status); + }); + return () => { + active = false; + }; + }, [accessibility]); const submit = async () => { const content = text.trim(); - if (!content || !canSend) return; + if (!content || !canReply) return; setSending(true); setError(null); + setAccessibilityResult(null); + if (!canSend && accessibility) { + let permission = accessibilityStatus; + if (permission?.permission !== "granted") { + permission = await accessibility.status(true); + setAccessibilityStatus(permission); + } + if (permission.permission !== "granted") { + setError(permission.detail); + setSending(false); + return; + } + const actionId = randomUUID(); + setActiveActionId(actionId); + const replyResult = await accessibility.execute({ + actionId, + origin: "local_desktop", + requestedAt: new Date().toISOString(), + accountId: account.accountId, + conversationId: conversation.conversationId, + ...(conversation.containerId ? { containerId: conversation.containerId } : {}), + conversationTitle: conversation.title, + text: content, + mode: "send", + }); + setActiveActionId(null); + setAccessibilityResult(replyResult); + if (replyResult.sent) { + setText(""); + onSent(); + } else if (!replyResult.draftPrepared) { + setError(replyResult.detail); + } + setSending(false); + return; + } const result = await send({ environmentId, input: { @@ -570,7 +634,7 @@ function Composer({ return (
- {canSend ? ( + {canReply ? (