feat: computer controller overhaul, adding peekaboo

crates/goose-cli/src/cli.rs

@@ -6,6 +6,8 @@ use goose::config::Config;
 use goose::posthog::get_telemetry_choice;
 use goose::recipe::Recipe;
 use goose_mcp::mcp_server_runner::{serve, McpCommand};
+#[cfg(target_os = "macos")]
+use goose_mcp::PeekabooServer;
 use goose_mcp::{
     AutoVisualiserRouter, ComputerControllerServer, DeveloperServer, MemoryServer, TutorialServer,
 };
@@ -979,6 +981,12 @@ async fn handle_mcp_command(server: McpCommand) -> Result<()> {
         McpCommand::Memory => serve(MemoryServer::new()).await?,
         McpCommand::Tutorial => serve(TutorialServer::new()).await?,
         McpCommand::Developer => serve(DeveloperServer::new()).await?,
+        #[cfg(target_os = "macos")]
+        McpCommand::Peekaboo => serve(PeekabooServer::new()).await?,
+        #[cfg(not(target_os = "macos"))]
+        McpCommand::Peekaboo => {
+            anyhow::bail!("Peekaboo is only available on macOS");
+        }
     }
     Ok(())
 }

crates/goose-cli/src/commands/configure.rs

@@ -980,6 +980,11 @@ fn configure_builtin_extension() -> anyhow::Result<()> {
             "Tutorial",
             "Access interactive tutorials and guides",
         ),
+        (
+            "peekaboo",
+            "Peekaboo",
+            "macOS UI automation via annotated screenshots (auto-installs via Homebrew)",
+        ),
     ];
 
     let mut select = cliclack::select("Which built-in extension would you like to enable?");

crates/goose-mcp/Cargo.toml

@@ -56,6 +56,7 @@ libc = "0.2"
 # ~1000 downloads). Pinned to exact version to prevent supply chain attacks.
 mpatch = "=0.2.0"
 tokio-util = { workspace = true }
+shell-words = "1.1.1"
 
 [dev-dependencies]
 serial_test = { workspace = true }

crates/goose-mcp/src/computercontroller/platform/mod.rs

@@ -14,6 +14,7 @@ pub use self::macos::MacOSAutomation;
 #[cfg(any(target_os = "linux", target_os = "android"))]
 pub use self::linux::LinuxAutomation;
 
+#[allow(dead_code)]
 pub trait SystemAutomation: Send + Sync {
     fn execute_system_script(&self, script: &str) -> std::io::Result<String>;
     fn get_shell_command(&self) -> (&'static str, &'static str); // (shell, arg)

crates/goose-mcp/src/lib.rs

@@ -14,6 +14,8 @@
 pub mod developer;
 pub mod mcp_server_runner;
 mod memory;
+#[cfg(target_os = "macos")]
+pub mod peekaboo;
 pub mod subprocess;
 pub mod tutorial;
 
@@ -22,6 +24,8 @@
 pub use developer::rmcp_developer::DeveloperServer;
 pub use developer::rmcp_developer::WORKING_DIR_PLACEHOLDER;
 pub use memory::MemoryServer;
+#[cfg(target_os = "macos")]
+pub use peekaboo::PeekabooServer;
 pub use tutorial::TutorialServer;
 
 /// Type definition for a function that spawns and serves a builtin extension server
@@ -54,11 +58,21 @@
 }
 
 pub static BUILTIN_EXTENSIONS: Lazy<HashMap<&'static str, SpawnServerFn>> = Lazy::new(|| {
-    HashMap::from([
+    let mut map = HashMap::from([
         builtin!(developer, DeveloperServer),
         builtin!(autovisualiser, AutoVisualiserRouter),
         builtin!(computercontroller, ComputerControllerServer),
         builtin!(memory, MemoryServer),
         builtin!(tutorial, TutorialServer),
-    ])
+    ]);
+
+    #[cfg(target_os = "macos")]
+    {
+        map.insert(
+            builtin!(peekaboo, PeekabooServer).0,
+            builtin!(peekaboo, PeekabooServer).1,
+        );
+    }
+
+    map
 });

crates/goose-mcp/src/mcp_server_runner.rs

@@ -9,6 +9,7 @@ pub enum McpCommand {
     ComputerController,
     Developer,
     Memory,
+    Peekaboo,
     Tutorial,
 }
 
@@ -21,6 +22,7 @@ impl FromStr for McpCommand {
             "computercontroller" => Ok(McpCommand::ComputerController),
             "developer" => Ok(McpCommand::Developer),
             "memory" => Ok(McpCommand::Memory),
+            "peekaboo" => Ok(McpCommand::Peekaboo),
             "tutorial" => Ok(McpCommand::Tutorial),
             _ => Err(format!("Invalid command: {}", s)),
         }
@@ -34,6 +36,7 @@ impl McpCommand {
             McpCommand::ComputerController => "computercontroller",
             McpCommand::Developer => "developer",
             McpCommand::Memory => "memory",
+            McpCommand::Peekaboo => "peekaboo",
             McpCommand::Tutorial => "tutorial",
         }
     }