diff --git a/libs/cua-driver/swift/Package.swift b/libs/cua-driver/swift/Package.swift index ee39eaaee7..c548050842 100644 --- a/libs/cua-driver/swift/Package.swift +++ b/libs/cua-driver/swift/Package.swift @@ -43,5 +43,13 @@ let package = Package( name: "FocusStealPreventerTests", dependencies: ["CuaDriverCore"] ), + .testTarget( + name: "HealthReportTests", + dependencies: [ + "CuaDriverCore", + "CuaDriverServer", + .product(name: "MCP", package: "swift-sdk"), + ] + ), ] ) diff --git a/libs/cua-driver/swift/Sources/CuaDriverServer/ToolRegistry.swift b/libs/cua-driver/swift/Sources/CuaDriverServer/ToolRegistry.swift index 6113c345f7..0a55a6592d 100644 --- a/libs/cua-driver/swift/Sources/CuaDriverServer/ToolRegistry.swift +++ b/libs/cua-driver/swift/Sources/CuaDriverServer/ToolRegistry.swift @@ -234,6 +234,7 @@ public struct ToolRegistry: Sendable { LaunchAppTool.handler, GetScreenSizeTool.handler, CheckPermissionsTool.handler, + HealthReportTool.handler, ScreenshotTool.handler, GetCursorPositionTool.handler, diff --git a/libs/cua-driver/swift/Sources/CuaDriverServer/Tools/HealthReportTool.swift b/libs/cua-driver/swift/Sources/CuaDriverServer/Tools/HealthReportTool.swift new file mode 100644 index 0000000000..6a28547507 --- /dev/null +++ b/libs/cua-driver/swift/Sources/CuaDriverServer/Tools/HealthReportTool.swift @@ -0,0 +1,634 @@ +import AppKit +import ApplicationServices +import CuaDriverCore +import Foundation +import MCP +import ScreenCaptureKit + +/// `health_report` — single-call end-to-end driver diagnostics. +/// +/// The point of this tool is to let downstream consumers (Hermes Agent +/// and similar) ship one stable diagnostic call and never have to know +/// cua-driver internals: tool names, TCC field names, bundle IDs, per- +/// platform check matrix. The tool owns the health model; consumers stay +/// thin and cua-driver evolves freely. +/// +/// Output shape is the stable contract — documented inline in the schema +/// text. `schema_version: "1"` is the commitment; future breaking changes +/// go to `"2"`. +/// +/// Reuses `Permissions.currentStatus()` for the TCC checks rather than +/// duplicating the probe logic. The TCC checks therefore inherit the +/// same caveat as `check_permissions`: in-process TCC reads reflect the +/// calling process's responsibility chain. When called via stdio MCP +/// from a terminal, that's the terminal — not CuaDriver.app. The data +/// payload for each TCC check surfaces this attribution so consumers +/// don't get misled. +public enum HealthReportTool { + + // MARK: - Canonical check names (also the values of `output.checks[].name`) + + static let nameBinaryVersion = "binary_version" + static let namePlatformSupported = "platform_supported" + static let nameTccAccessibility = "tcc_accessibility" + static let nameTccScreenRecording = "tcc_screen_recording" + static let nameBundleIdentity = "bundle_identity" + static let nameAxCapability = "ax_capability" + static let nameScreenCaptureCapability = "screen_capture_capability" + static let nameSessionActive = "session_active" + + /// Checks whose failure marks the whole report as `failed` (vs `degraded`). + /// Binary and platform are non-negotiable; everything else is degraded. + static let coreChecks: Set = [ + nameBinaryVersion, + namePlatformSupported, + nameSessionActive, + ] + + /// All check names, in the canonical macOS run order. + static let allMacOSChecks: [String] = [ + nameBinaryVersion, + namePlatformSupported, + nameSessionActive, + nameBundleIdentity, + nameTccAccessibility, + nameTccScreenRecording, + nameAxCapability, + nameScreenCaptureCapability, + ] + + public static let handler = ToolHandler( + tool: Tool( + name: "health_report", + description: """ + Single-call end-to-end driver diagnostics. Designed to let + downstream consumers (Hermes Agent and similar) ship one + stable call instead of stitching together check_permissions, + doctor, version, bundle attribution, and a screenshot probe. + cua-driver owns the health model; consumers stay thin. + + Input — all optional: + { + "include": ["", ...], // run only these + "skip": ["", ...] // skip these + } + If both are given, `include` wins. + + Canonical check names (macOS): + binary_version, platform_supported, session_active, + bundle_identity, tcc_accessibility, tcc_screen_recording, + ax_capability, screen_capture_capability + + Output — stable contract, schema_version="1": + { + "schema_version": "1", + "platform": "darwin" | "win32" | "linux", + "driver_version": "", + "overall": "ok" | "degraded" | "failed", + "checks": [ + { + "name": "", + "status": "pass" | "fail" | "skip", + "message": "", + "hint": "", + "data": { /* check-specific structured fields */ } + }, + ... + ] + } + + `overall` rules: + - `ok` — every non-skipped check passes + - `degraded` — at least one non-core check fails + (binary is still usable) + - `failed` — any core check fails + (binary_version, platform_supported, + session_active) + + Stability: schema_version="1" is the contract. Future + breaking changes will be `"2"`. Adding new check names + under the same schema_version is non-breaking; consumers + must tolerate unknown check names. + + TCC caveat: in-process TCC reads reflect the calling + process's responsibility chain. When called via stdio MCP + spawned from a terminal, that's the terminal — not + CuaDriver.app. Each TCC check's `data` block exposes the + runtime bundle identifier so consumers can detect the + mismatch. + """, + inputSchema: [ + "type": "object", + "properties": [ + "include": [ + "type": "array", + "items": ["type": "string"], + "description": + "Only run these checks (canonical names). Wins over `skip`.", + ], + "skip": [ + "type": "array", + "items": ["type": "string"], + "description": + "Skip these checks (canonical names). Ignored when `include` is set.", + ], + ], + "additionalProperties": false, + ], + annotations: .init( + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false + ) + ), + invoke: { arguments in + let include = parseStringSet(arguments?["include"]) + let skip = parseStringSet(arguments?["skip"]) + + let toRun = selectChecks( + allChecks: allMacOSChecks, include: include, skip: skip + ) + + // Drive the probes. Skipped checks still appear in the + // output with status=skip so consumers see a complete map + // of which checks were considered. + var checks: [CheckEntry] = [] + for name in allMacOSChecks { + if !toRun.contains(name) { + checks.append( + CheckEntry( + name: name, + status: .skip, + message: "Skipped by include/skip filter.", + hint: nil, + data: nil + ) + ) + continue + } + let entry = await runCheck(name: name) + checks.append(entry) + } + + let overall = computeOverall(checks: checks) + let report = Report( + schemaVersion: "1", + platform: "darwin", + driverVersion: CuaDriverCore.version, + overall: overall, + checks: checks + ) + + let textContent: Tool.Content = .text( + text: textSummary(report: report), annotations: nil, _meta: nil + ) + if let result = try? CallTool.Result( + content: [textContent], + structuredContent: report + ) { + return result + } + return CallTool.Result(content: [textContent]) + } + ) + + // MARK: - Output model + + struct Report: Codable, Sendable { + let schemaVersion: String + let platform: String + let driverVersion: String + let overall: Overall + let checks: [CheckEntry] + + private enum CodingKeys: String, CodingKey { + case schemaVersion = "schema_version" + case platform + case driverVersion = "driver_version" + case overall + case checks + } + } + + enum Overall: String, Codable, Sendable { + case ok + case degraded + case failed + } + + enum Status: String, Codable, Sendable { + case pass + case fail + case skip + } + + struct CheckEntry: Codable, Sendable { + let name: String + let status: Status + let message: String + let hint: String? + let data: CheckData? + } + + /// Per-check structured data. Kept as a small, fixed set of optional + /// fields rather than a free-form dictionary so the JSON shape is + /// statically known and Codable. Adding new fields here is + /// non-breaking (consumers ignore unknown fields); removing fields + /// would break clients and is gated by `schema_version`. + struct CheckData: Codable, Sendable { + var bundleIdentifier: String? + var executablePath: String? + var osVersion: String? + var architecture: String? + var displayCount: Int? + var errorDetail: String? + + private enum CodingKeys: String, CodingKey { + case bundleIdentifier = "bundle_identifier" + case executablePath = "executable_path" + case osVersion = "os_version" + case architecture + case displayCount = "display_count" + case errorDetail = "error_detail" + } + + var isEmpty: Bool { + bundleIdentifier == nil && executablePath == nil + && osVersion == nil && architecture == nil + && displayCount == nil && errorDetail == nil + } + } + + // MARK: - Check dispatch + + static func runCheck(name: String) async -> CheckEntry { + switch name { + case nameBinaryVersion: + return checkBinaryVersion() + case namePlatformSupported: + return checkPlatformSupported() + case nameSessionActive: + return checkSessionActive() + case nameBundleIdentity: + return checkBundleIdentity() + case nameTccAccessibility: + return await checkTccAccessibility() + case nameTccScreenRecording: + return await checkTccScreenRecording() + case nameAxCapability: + return checkAxCapability() + case nameScreenCaptureCapability: + return await checkScreenCaptureCapability() + default: + // Should never happen — `selectChecks` only forwards canonical + // names. Still, surface unknowns rather than crashing. + return CheckEntry( + name: name, + status: .skip, + message: "Unknown check name (not implemented on this platform).", + hint: nil, + data: nil + ) + } + } + + // MARK: - Individual checks + + static func checkBinaryVersion() -> CheckEntry { + // CuaDriverCore.version is a compiled-in constant; failure here + // would mean a corrupted binary. Always pass. + return CheckEntry( + name: nameBinaryVersion, + status: .pass, + message: "cua-driver \(CuaDriverCore.version)", + hint: nil, + data: nil + ) + } + + static func checkPlatformSupported() -> CheckEntry { + // The Swift binary is macOS-only (Package.swift pins .macOS(.v14)), + // so reaching this code already implies a supported platform. + let osVersion = ProcessInfo.processInfo.operatingSystemVersionString + let arch = uname_m() + return CheckEntry( + name: namePlatformSupported, + status: .pass, + message: "macOS — \(osVersion) (\(arch))", + hint: nil, + data: CheckData( + bundleIdentifier: nil, + executablePath: nil, + osVersion: osVersion, + architecture: arch, + displayCount: nil, + errorDetail: nil + ) + ) + } + + static func checkSessionActive() -> CheckEntry { + // We are servicing this very MCP call, so by construction the + // session is up. The check exists so consumers can have a + // canonical "is the server reachable?" signal in a fixed shape. + return CheckEntry( + name: nameSessionActive, + status: .pass, + message: "MCP session is active.", + hint: nil, + data: nil + ) + } + + static func checkBundleIdentity() -> CheckEntry { + let bid = Bundle.main.bundleIdentifier ?? "" + let exe = Bundle.main.executablePath ?? "" + // We treat presence of `com.trycua.driver` as the canonical + // pass — that's the bundle whose TCC grants matter. Anything + // else (running from a shell-spawned binary, from Xcode test + // host, etc.) is a `fail` with a hint pointing at the daemon + // launch path. It's `degraded`, not `failed`, in overall terms. + let isCorrect = bid == "com.trycua.driver" + let status: Status = isCorrect ? .pass : .fail + let message: String + let hint: String? + if isCorrect { + message = "Bundle is com.trycua.driver." + hint = nil + } else if bid.isEmpty { + message = "Process has no CFBundleIdentifier." + hint = """ + Run the binary inside CuaDriver.app so TCC grants attribute correctly. \ + Start the daemon with `open -n -g -a CuaDriver --args serve` and \ + connect via `cua-driver mcp`. + """ + } else { + message = "Bundle is \(bid), not com.trycua.driver." + hint = """ + TCC grants will be attributed to \(bid), not the cua-driver daemon. \ + Run via `cua-driver mcp` (auto-relaunches inside CuaDriver.app) or \ + start the daemon manually: `open -n -g -a CuaDriver --args serve`. + """ + } + return CheckEntry( + name: nameBundleIdentity, + status: status, + message: message, + hint: hint, + data: CheckData( + bundleIdentifier: bid.isEmpty ? nil : bid, + executablePath: exe.isEmpty ? nil : exe, + osVersion: nil, + architecture: nil, + displayCount: nil, + errorDetail: nil + ) + ) + } + + static func checkTccAccessibility() async -> CheckEntry { + // Reuse the shared probe — do not duplicate AXIsProcessTrusted. + let status = await Permissions.currentStatus() + let bid = Bundle.main.bundleIdentifier ?? "" + if status.accessibility { + return CheckEntry( + name: nameTccAccessibility, + status: .pass, + message: "Accessibility is granted.", + hint: nil, + data: CheckData( + bundleIdentifier: bid.isEmpty ? nil : bid, + executablePath: nil, + osVersion: nil, + architecture: nil, + displayCount: nil, + errorDetail: nil + ) + ) + } + return CheckEntry( + name: nameTccAccessibility, + status: .fail, + message: "Accessibility is NOT granted for this process.", + hint: """ + Grant Accessibility to CuaDriver.app in System Settings → Privacy & Security → \ + Accessibility. If the process bundle is not com.trycua.driver (see bundle_identity), \ + the grant must target the responsible app — restart via `cua-driver mcp` to relaunch \ + inside CuaDriver.app. + """, + data: CheckData( + bundleIdentifier: bid.isEmpty ? nil : bid, + executablePath: nil, + osVersion: nil, + architecture: nil, + displayCount: nil, + errorDetail: nil + ) + ) + } + + static func checkTccScreenRecording() async -> CheckEntry { + let status = await Permissions.currentStatus() + let bid = Bundle.main.bundleIdentifier ?? "" + if status.screenRecording { + return CheckEntry( + name: nameTccScreenRecording, + status: .pass, + message: "Screen Recording is granted.", + hint: nil, + data: CheckData( + bundleIdentifier: bid.isEmpty ? nil : bid, + executablePath: nil, + osVersion: nil, + architecture: nil, + displayCount: nil, + errorDetail: nil + ) + ) + } + return CheckEntry( + name: nameTccScreenRecording, + status: .fail, + message: "Screen Recording is NOT granted for this process.", + hint: """ + Grant Screen Recording to CuaDriver.app in System Settings → Privacy & Security → \ + Screen Recording. The grant is attributed to the responsible process — see \ + bundle_identity to confirm the right binary is being prompted. + """, + data: CheckData( + bundleIdentifier: bid.isEmpty ? nil : bid, + executablePath: nil, + osVersion: nil, + architecture: nil, + displayCount: nil, + errorDetail: nil + ) + ) + } + + static func checkAxCapability() -> CheckEntry { + // The AX trust gate is the same kAXIsProcessTrusted probe + // `tcc_accessibility` runs, but ax_capability is the consumer- + // facing "can I actually drive UI" signal. We separate them so + // a future change (e.g. capability degraded without TCC denial) + // doesn't break either contract. + let trusted = AXIsProcessTrusted() + if trusted { + return CheckEntry( + name: nameAxCapability, + status: .pass, + message: "AX is trusted and reachable.", + hint: nil, + data: nil + ) + } + return CheckEntry( + name: nameAxCapability, + status: .fail, + message: "AX is not trusted; UI inspection and event posting will fail.", + hint: """ + Resolve tcc_accessibility first — AX capability follows directly from the \ + Accessibility TCC grant. + """, + data: nil + ) + } + + static func checkScreenCaptureCapability() async -> CheckEntry { + // Live ScreenCaptureKit probe: enumerate shareable content. + // Nothing hits disk; we never start a stream. This is the same + // probe `Permissions.probeScreenRecording` uses but explicit + // about being a capability check, not a TCC-grant check. + // Surfaces the actual display count so consumers can detect a + // headless / no-display situation distinct from a denied grant. + do { + let content = try await SCShareableContent.excludingDesktopWindows( + false, onScreenWindowsOnly: true + ) + let count = content.displays.count + return CheckEntry( + name: nameScreenCaptureCapability, + status: .pass, + message: "ScreenCaptureKit reachable; \(count) display(s) shareable.", + hint: nil, + data: CheckData( + bundleIdentifier: nil, + executablePath: nil, + osVersion: nil, + architecture: nil, + displayCount: count, + errorDetail: nil + ) + ) + } catch { + return CheckEntry( + name: nameScreenCaptureCapability, + status: .fail, + message: "ScreenCaptureKit probe failed.", + hint: """ + Confirm tcc_screen_recording is granted. If it is, this is a SCK regression \ + (see #1467) — set capture_mode to `ax` to skip screen capture entirely. + """, + data: CheckData( + bundleIdentifier: nil, + executablePath: nil, + osVersion: nil, + architecture: nil, + displayCount: nil, + errorDetail: String(describing: error) + ) + ) + } + } + + // MARK: - Overall rollup + + static func computeOverall(checks: [CheckEntry]) -> Overall { + var anyFail = false + var anyCoreFail = false + for entry in checks { + guard entry.status == .fail else { continue } + anyFail = true + if coreChecks.contains(entry.name) { + anyCoreFail = true + } + } + if anyCoreFail { return .failed } + if anyFail { return .degraded } + return .ok + } + + // MARK: - Argument parsing helpers + + /// Pull a `Set` out of an MCP `Value` array argument. Missing + /// / nil / non-array values yield an empty set, treated as "no + /// filter applied" by `selectChecks`. + static func parseStringSet(_ value: Value?) -> Set { + guard let value, case let .array(items) = value else { return [] } + var out: Set = [] + for item in items { + if case let .string(s) = item { out.insert(s) } + } + return out + } + + /// Decide which checks to actually run. Rules: + /// - `include` wins if non-empty: run exactly the canonical names + /// that intersect `include`. Unknown names in `include` are + /// silently ignored (forward-compat — consumer may know a name + /// from a newer driver build). + /// - else `skip` filters out from the full canonical list. + /// - else run everything. + static func selectChecks( + allChecks: [String], include: Set, skip: Set + ) -> Set { + if !include.isEmpty { + return Set(allChecks).intersection(include) + } + if !skip.isEmpty { + return Set(allChecks).subtracting(skip) + } + return Set(allChecks) + } + + // MARK: - Text summary + + /// Compact human-readable summary, mirroring the doctor command's + /// vibe but driven off the structured output. The structuredContent + /// payload is the authoritative format; this text content is for + /// humans squinting at MCP traces. + static func textSummary(report: Report) -> String { + var lines: [String] = [] + let overallIcon: String + switch report.overall { + case .ok: overallIcon = "✅" + case .degraded: overallIcon = "⚠️" + case .failed: overallIcon = "❌" + } + lines.append( + "\(overallIcon) cua-driver \(report.driverVersion) on \(report.platform) — \(report.overall.rawValue)" + ) + for entry in report.checks { + let icon: String + switch entry.status { + case .pass: icon = "✅" + case .fail: icon = "❌" + case .skip: icon = "⏭" + } + lines.append(" \(icon) \(entry.name): \(entry.message)") + } + return lines.joined(separator: "\n") + } + + // MARK: - Misc + + private static func uname_m() -> String { + var info = utsname() + uname(&info) + return withUnsafeBytes(of: &info.machine) { bytes in + let str = bytes.bindMemory(to: CChar.self) + return String(cString: str.baseAddress!) + } + } +} diff --git a/libs/cua-driver/swift/Tests/HealthReportTests/HealthReportTests.swift b/libs/cua-driver/swift/Tests/HealthReportTests/HealthReportTests.swift new file mode 100644 index 0000000000..6e425ee0e9 --- /dev/null +++ b/libs/cua-driver/swift/Tests/HealthReportTests/HealthReportTests.swift @@ -0,0 +1,293 @@ +import Foundation +import MCP +import XCTest + +@testable import CuaDriverCore +@testable import CuaDriverServer + +/// Unit tests for the `health_report` MCP tool. +/// +/// We can't exercise every probe in a pure unit test — TCC grants and +/// ScreenCaptureKit reachability depend on the host environment — but +/// the contract pieces we CAN nail down deterministically are: +/// +/// 1. The tool is registered in `ToolRegistry.default` and advertises +/// the documented schema version `"1"` in its description. +/// 2. `selectChecks` correctly handles `include`/`skip` (include wins, +/// unknowns are ignored, empty filters run everything). +/// 3. `computeOverall` rolls per-check status up into `ok` / `degraded` +/// / `failed` exactly as documented (core-fail vs non-core-fail). +/// 4. End-to-end invocation produces a Codable `Report` that +/// round-trips through JSON and matches the documented schema +/// shape (top-level keys, per-check keys). +/// 5. A `skip` filter is honored end-to-end: skipped checks appear in +/// the output with status `"skip"`. +/// 6. A simulated TCC denial (achieved by selecting only the +/// bundle_identity check inside an XCTest host whose bundle id is +/// NOT com.trycua.driver) produces an entry with `status: "fail"`, +/// a `message`, a `hint`, and a `data.bundle_identifier` field — +/// our documented fail-mode shape. +/// +/// The point of (6) is that XCTest test hosts run under a bundle id +/// like `com.apple.dt.xctest.tool` or the package binary's path, so +/// `bundle_identity` reliably fails in the test environment — a free +/// fail-mode fixture without mocking. +final class HealthReportTests: XCTestCase { + + // MARK: - Registry / schema contract + + func testToolIsRegistered() { + let registry = ToolRegistry.default + XCTAssertNotNil( + registry.handlers["health_report"], + "health_report must be advertised through the default ToolRegistry" + ) + } + + func testToolDescriptionCommitsToSchemaVersion1() { + // The PR description and downstream consumers (Hermes Agent) + // bake `schema_version: "1"` into their expectations. A future + // change that drops this commitment from the schema text without + // bumping the version itself would silently break consumers — + // this test fails loudly the moment that drift starts. + let registry = ToolRegistry.default + let handler = registry.handlers["health_report"] + XCTAssertNotNil(handler) + let description = handler?.tool.description ?? "" + XCTAssertTrue( + description.contains("schema_version=\"1\"") + || description.contains("schema_version: \"1\"") + || description.contains("schema_version=1"), + "schema_version=1 must be documented in the tool description" + ) + } + + // MARK: - selectChecks + + func testSelectChecksIncludeWinsOverSkip() { + let chosen = HealthReportTool.selectChecks( + allChecks: HealthReportTool.allMacOSChecks, + include: [HealthReportTool.nameBinaryVersion], + skip: [HealthReportTool.nameBinaryVersion] + ) + // include is explicit; skip is ignored. + XCTAssertEqual(chosen, [HealthReportTool.nameBinaryVersion]) + } + + func testSelectChecksIncludeFiltersOutUnknownsSilently() { + let chosen = HealthReportTool.selectChecks( + allChecks: HealthReportTool.allMacOSChecks, + include: ["binary_version", "tomorrows_check_name"], + skip: [] + ) + XCTAssertEqual(chosen, ["binary_version"]) + } + + func testSelectChecksEmptyFiltersRunsEverything() { + let chosen = HealthReportTool.selectChecks( + allChecks: HealthReportTool.allMacOSChecks, + include: [], + skip: [] + ) + XCTAssertEqual(chosen, Set(HealthReportTool.allMacOSChecks)) + } + + func testSelectChecksSkipRemovesNamed() { + let chosen = HealthReportTool.selectChecks( + allChecks: HealthReportTool.allMacOSChecks, + include: [], + skip: [HealthReportTool.nameTccAccessibility] + ) + XCTAssertFalse(chosen.contains(HealthReportTool.nameTccAccessibility)) + XCTAssertTrue(chosen.contains(HealthReportTool.nameBinaryVersion)) + } + + // MARK: - computeOverall + + private func entry(_ name: String, _ status: HealthReportTool.Status) + -> HealthReportTool.CheckEntry + { + HealthReportTool.CheckEntry( + name: name, status: status, message: "m", hint: nil, data: nil + ) + } + + func testComputeOverallAllPassIsOk() { + let checks = HealthReportTool.allMacOSChecks.map { entry($0, .pass) } + XCTAssertEqual(HealthReportTool.computeOverall(checks: checks), .ok) + } + + func testComputeOverallAllSkipIsOk() { + let checks = HealthReportTool.allMacOSChecks.map { entry($0, .skip) } + XCTAssertEqual(HealthReportTool.computeOverall(checks: checks), .ok) + } + + func testComputeOverallNonCoreFailIsDegraded() { + // bundle_identity is non-core: failing it should produce + // `degraded`, not `failed`. + var checks: [HealthReportTool.CheckEntry] = [] + for name in HealthReportTool.allMacOSChecks { + checks.append( + entry(name, name == HealthReportTool.nameBundleIdentity ? .fail : .pass) + ) + } + XCTAssertEqual(HealthReportTool.computeOverall(checks: checks), .degraded) + } + + func testComputeOverallCoreFailIsFailed() { + var checks: [HealthReportTool.CheckEntry] = [] + for name in HealthReportTool.allMacOSChecks { + checks.append( + entry(name, name == HealthReportTool.nameBinaryVersion ? .fail : .pass) + ) + } + XCTAssertEqual(HealthReportTool.computeOverall(checks: checks), .failed) + } + + // MARK: - End-to-end invocation (Codable round-trip) + + func testInvokeWithNoArgsProducesValidJSONSchema() async throws { + let registry = ToolRegistry.default + let result = try await registry.call("health_report", arguments: nil) + + // Extract the first text payload to land a basic sanity check. + var foundText = false + for content in result.content { + if case .text = content { foundText = true } + } + XCTAssertTrue(foundText, "response must carry at least one text block for humans") + + // Re-encode the inner Report directly so we can assert the + // exact JSON shape. We don't try to read structuredContent off + // the result — that's an internal MCP detail; the public + // contract is the JSON shape, which we compute identically. + // Run every check (no filter) to exercise the longest output. + let report = await Self.buildReportNoFilter() + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + let data = try encoder.encode(report) + let raw = + try JSONSerialization.jsonObject(with: data) as? [String: Any] ?? [:] + + // Top-level keys exactly match the documented schema. + XCTAssertEqual(raw["schema_version"] as? String, "1") + XCTAssertEqual(raw["platform"] as? String, "darwin") + XCTAssertNotNil(raw["driver_version"] as? String) + XCTAssertNotNil(raw["overall"] as? String) + let checks = raw["checks"] as? [[String: Any]] + XCTAssertNotNil(checks) + XCTAssertFalse(checks?.isEmpty ?? true) + + // Every check must carry `name`, `status`, `message` — the + // mandatory triple. `hint` and `data` are optional. + for check in checks ?? [] { + XCTAssertNotNil(check["name"] as? String) + XCTAssertNotNil(check["status"] as? String) + XCTAssertNotNil(check["message"] as? String) + // Status must be one of the documented enum values. + let status = check["status"] as? String ?? "" + XCTAssertTrue( + ["pass", "fail", "skip"].contains(status), + "unexpected status value: \(status)" + ) + } + } + + func testInvokeWithSkipHonorsFilter() async throws { + let registry = ToolRegistry.default + let args: [String: Value] = [ + "skip": .array([.string(HealthReportTool.nameTccAccessibility)]) + ] + _ = try await registry.call("health_report", arguments: args) + // Re-derive the report directly so we can assert per-check + // status; the registry path gives us the same logic. + let report = await Self.buildReport(include: [], skip: [HealthReportTool.nameTccAccessibility]) + let tccEntry = report.checks.first { + $0.name == HealthReportTool.nameTccAccessibility + } + XCTAssertEqual(tccEntry?.status, .skip) + XCTAssertTrue(tccEntry?.message.contains("Skipped") ?? false) + } + + // MARK: - Fail mode: bundle_identity in the test host + + func testBundleIdentityFailModeShape() async { + // XCTest hosts run under a bundle id other than com.trycua.driver, + // so `checkBundleIdentity` reliably returns `.fail` here. This + // is the documented fail-mode shape consumers will see in the + // wild whenever cua-driver is launched outside CuaDriver.app — + // we want it to always have a message, a hint, and a data + // payload with `bundle_identifier`. + let entry = HealthReportTool.checkBundleIdentity() + XCTAssertEqual( + entry.status, .fail, + "expected bundle_identity to fail in the XCTest host environment; " + + "if this asserts on a future Swift Package Manager change that makes " + + "the test bundle id be com.trycua.driver, the test is fine to drop." + ) + XCTAssertFalse(entry.message.isEmpty) + XCTAssertNotNil(entry.hint, "fail entries must carry a remediation hint") + XCTAssertNotNil(entry.data, "fail entries must carry diagnostic data") + // bundle_identifier surfaces the actual runtime CFBundleIdentifier + // — the field downstream consumers key off to detect attribution + // drift. It can be absent only when the process has none. + // In SPM test runs the bundle id is non-empty. + XCTAssertNotNil(entry.data?.bundleIdentifier) + } + + // MARK: - Argument parsing edge cases + + func testParseStringSetIgnoresNonStringItems() { + let value: Value = .array([.string("a"), .int(7), .bool(true), .string("b")]) + let parsed = HealthReportTool.parseStringSet(value) + XCTAssertEqual(parsed, ["a", "b"]) + } + + func testParseStringSetHandlesNilAndScalars() { + XCTAssertEqual(HealthReportTool.parseStringSet(nil), []) + XCTAssertEqual(HealthReportTool.parseStringSet(.string("x")), []) + XCTAssertEqual(HealthReportTool.parseStringSet(.int(3)), []) + } + + // MARK: - Helpers + + /// Re-build the same `Report` value the tool produces, with an + /// optional include / skip filter applied. Used by tests that need + /// to assert per-check status without parsing MCP `Value` content. + private static func buildReport( + include: Set, skip: Set + ) async -> HealthReportTool.Report { + let toRun = HealthReportTool.selectChecks( + allChecks: HealthReportTool.allMacOSChecks, + include: include, + skip: skip + ) + var checks: [HealthReportTool.CheckEntry] = [] + for name in HealthReportTool.allMacOSChecks { + if !toRun.contains(name) { + checks.append( + HealthReportTool.CheckEntry( + name: name, + status: .skip, + message: "Skipped by include/skip filter.", + hint: nil, + data: nil + ) + ) + continue + } + checks.append(await HealthReportTool.runCheck(name: name)) + } + return HealthReportTool.Report( + schemaVersion: "1", + platform: "darwin", + driverVersion: CuaDriverCore.version, + overall: HealthReportTool.computeOverall(checks: checks), + checks: checks + ) + } + + private static func buildReportNoFilter() async -> HealthReportTool.Report { + await buildReport(include: [], skip: []) + } +} diff --git a/libs/cua-driver/swift/Tests/integration/test_health_report_mcp.py b/libs/cua-driver/swift/Tests/integration/test_health_report_mcp.py new file mode 100644 index 0000000000..baf94ec715 --- /dev/null +++ b/libs/cua-driver/swift/Tests/integration/test_health_report_mcp.py @@ -0,0 +1,228 @@ +"""Integration test: `health_report` MCP tool over the stdio protocol. + +Boots the real `cua-driver mcp` server, performs the JSON-RPC handshake, +asks `tools/list` for the tool catalog, and exercises the tool with +several argument shapes: + + 1. `tools/list` advertises `health_report` (no breaking schema drift — + the consumer-facing contract is that the tool *exists* and ships + `schema_version="1"` in its description). + + 2. Calling with no arguments produces a well-formed report with the + documented top-level keys and per-check key shape. + + 3. The `skip` filter is honored end-to-end: a skipped check appears in + the output with `status: "skip"`. + + 4. The `include` filter narrows the report to a single check. + + 5. A documented fail mode is observable end-to-end: when called via + `cua-driver mcp` from a terminal (which is what an XCTest/CI host + looks like), the `bundle_identity` check fails because the stdio + process is NOT attributed to com.trycua.driver. The schema + guarantees `status="fail"` entries carry a `hint` and surface the + runtime bundle identifier under `data.bundle_identifier` — both + are asserted here. + +Run with the same harness as the other integration tests: + scripts/test.sh test_health_report_mcp +""" + +from __future__ import annotations + +import json +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from driver_client import DriverClient, default_binary_path + + +REQUIRED_TOP_LEVEL_KEYS = { + "schema_version", + "platform", + "driver_version", + "overall", + "checks", +} + +REQUIRED_CHECK_KEYS = {"name", "status", "message"} +ALLOWED_STATUS_VALUES = {"pass", "fail", "skip"} +ALLOWED_OVERALL_VALUES = {"ok", "degraded", "failed"} + + +def _structured_or_parsed(result: dict) -> dict: + """Pull the JSON report out of an MCP tool result. + + Tools that emit `structuredContent` (the canonical path here) return + it as a top-level field. We also tolerate a text-content fallback — + if structured emission ever breaks, the text block still carries the + JSON and we want the test to fail loudly with a clear message rather + than crash on a KeyError. Mirrors the same defensive read pattern + used by `driver_client.frontmost_bundle_id`. + """ + if "structuredContent" in result: + return result["structuredContent"] + # Defensive fallback — accept the first text block as JSON. + for content in result.get("content", []): + if content.get("type") == "text": + try: + return json.loads(content.get("text", "")) + except json.JSONDecodeError: + continue + raise AssertionError( + "health_report response carried neither structuredContent nor a JSON text block: " + + json.dumps(result) + ) + + +class HealthReportMCPTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.binary = default_binary_path() + + # ── tools/list ─────────────────────────────────────────────────── + + def test_tools_list_advertises_health_report(self) -> None: + with DriverClient(self.binary) as client: + tools = client.list_tools() + names = {t["name"] for t in tools} + self.assertIn( + "health_report", + names, + "tools/list must advertise health_report so consumers can discover it", + ) + descriptor = next(t for t in tools if t["name"] == "health_report") + # The schema_version commitment lives in the description text — + # that's the consumer-facing stability contract. + description = descriptor.get("description", "") + self.assertIn( + "schema_version", + description, + "tool description must call out the stable schema_version contract", + ) + + # ── tools/call with no args ────────────────────────────────────── + + def test_call_with_no_args_returns_schema_shape(self) -> None: + with DriverClient(self.binary) as client: + result = client.call_tool("health_report") + report = _structured_or_parsed(result) + + # Top-level keys exactly match the documented schema. + self.assertEqual( + REQUIRED_TOP_LEVEL_KEYS, + REQUIRED_TOP_LEVEL_KEYS & set(report.keys()), + f"missing top-level keys; got {set(report.keys())!r}", + ) + self.assertEqual(report["schema_version"], "1") + self.assertEqual(report["platform"], "darwin") + self.assertIsInstance(report["driver_version"], str) + self.assertIn(report["overall"], ALLOWED_OVERALL_VALUES) + + checks = report["checks"] + self.assertIsInstance(checks, list) + self.assertGreater(len(checks), 0) + + for check in checks: + self.assertTrue( + REQUIRED_CHECK_KEYS.issubset(check.keys()), + f"check missing required keys: {check!r}", + ) + self.assertIn(check["status"], ALLOWED_STATUS_VALUES) + # hint is only required when status == fail. + if check["status"] == "fail": + self.assertIn( + "hint", + check, + f"fail entries must carry a remediation hint: {check!r}", + ) + self.assertTrue( + isinstance(check["hint"], str) and check["hint"].strip(), + f"hint must be a non-empty string: {check!r}", + ) + + # ── skip filter ────────────────────────────────────────────────── + + def test_skip_filter_marks_check_as_skip(self) -> None: + with DriverClient(self.binary) as client: + result = client.call_tool( + "health_report", + {"skip": ["tcc_accessibility"]}, + ) + report = _structured_or_parsed(result) + names_to_status = {c["name"]: c["status"] for c in report["checks"]} + self.assertEqual(names_to_status.get("tcc_accessibility"), "skip") + + # ── include filter ────────────────────────────────────────────── + + def test_include_filter_runs_only_named_check(self) -> None: + with DriverClient(self.binary) as client: + result = client.call_tool( + "health_report", + {"include": ["binary_version"]}, + ) + report = _structured_or_parsed(result) + run_names = { + c["name"] for c in report["checks"] if c["status"] != "skip" + } + # Only `binary_version` should have been actually run; the rest + # show up with status: skip. + self.assertEqual(run_names, {"binary_version"}) + + # ── documented fail mode ──────────────────────────────────────── + + def test_bundle_identity_fail_mode_under_stdio_mcp(self) -> None: + """The stdio MCP process spawned by an IDE/CI shell runs outside + CuaDriver.app, so its CFBundleIdentifier is NOT com.trycua.driver. + That's the canonical "wrong attribution" fail mode the tool is + designed to surface — schema-wise: status=fail + non-empty + message + non-empty hint + data.bundle_identifier present so + consumers can detect the drift without parsing the message text. + """ + with DriverClient(self.binary) as client: + result = client.call_tool( + "health_report", + {"include": ["bundle_identity"]}, + ) + report = _structured_or_parsed(result) + bundle_check = next( + c for c in report["checks"] if c["name"] == "bundle_identity" + ) + # If the binary IS reached via the .app bundle in this harness + # the check legitimately passes; tolerate both — but exercise + # the fail-mode contract whenever the check fails. + if bundle_check["status"] == "fail": + self.assertIn("hint", bundle_check) + self.assertTrue(bundle_check["hint"].strip()) + data = bundle_check.get("data") or {} + # bundle_identifier surfaces the runtime CFBundleIdentifier + # so consumers can detect drift without parsing the message. + # Field is optional only when the process literally has no + # bundle id; SPM-built binaries always have one. + self.assertIn( + "bundle_identifier", + data, + "fail mode must surface bundle_identifier in `data`", + ) + + # ── invocation surface itself doesn't error ───────────────────── + + def test_call_does_not_set_isError(self) -> None: + """`health_report` is the consumer's "what's broken?" probe — it + must never itself report `isError: true`, even when every check + fails. Otherwise consumers can't tell apart "the tool itself is + broken" from "the driver has degraded health" — defeating the + whole point of the tool. + """ + with DriverClient(self.binary) as client: + result = client.call_tool("health_report") + self.assertFalse( + result.get("isError", False), + f"health_report must never set isError; got {result!r}", + ) + + +if __name__ == "__main__": + unittest.main()