From 0efec18b0b7a32432c2ef6523c42acae545e08d9 Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Thu, 7 Aug 2025 11:38:27 -0400 Subject: [PATCH 01/17] Something Claude started --- crates/goose/tests/mcp_integration_test.rs | 211 +++++++++++++++++++++ crates/goose/tests/test_mcp_server.py | 83 ++++++++ 2 files changed, 294 insertions(+) create mode 100644 crates/goose/tests/mcp_integration_test.rs create mode 100644 crates/goose/tests/test_mcp_server.py diff --git a/crates/goose/tests/mcp_integration_test.rs b/crates/goose/tests/mcp_integration_test.rs new file mode 100644 index 000000000000..2cc32d8f1fe9 --- /dev/null +++ b/crates/goose/tests/mcp_integration_test.rs @@ -0,0 +1,211 @@ +use std::path::PathBuf; +use std::time::Duration; +use tokio_util::sync::CancellationToken; +use serde_json::json; + +use goose::agents::extension_manager::ExtensionManager; +use goose::agents::extension::{ExtensionConfig, Envs}; +use mcp_core::ToolCall; +use rmcp::model::Content; + +#[tokio::test] +async fn test_extension_manager_with_python_mcp_server() { + // Get the path to the test MCP server + let mut test_server_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + test_server_path.push("tests"); + test_server_path.push("test_mcp_server.py"); + + // Ensure the test server exists + assert!(test_server_path.exists(), "Test MCP server not found at {:?}", test_server_path); + + // Create extension manager + let mut extension_manager = ExtensionManager::new(); + + // Create a stdio extension config for the Python MCP server + let extension_config = ExtensionConfig::Stdio { + name: "test_python_mcp".to_string(), + description: Some("Test Python MCP Server".to_string()), + cmd: "python3".to_string(), + args: vec![test_server_path.to_string_lossy().to_string()], + envs: Envs::default(), + env_keys: vec![], + timeout: Some(30), // 30 seconds timeout + bundled: Some(false), + }; + + // Add the extension + let result = extension_manager.add_extension(extension_config).await; + assert!(result.is_ok(), "Failed to add extension: {:?}", result); + + // Wait a moment for the connection to stabilize + tokio::time::sleep(Duration::from_millis(500)).await; + + // Get the list of tools + let tools = extension_manager.get_prefixed_tools(None).await; + assert!(tools.is_ok(), "Failed to get tools: {:?}", tools); + + let tools = tools.unwrap(); + assert!(!tools.is_empty(), "No tools found"); + + // Verify we have the expected tools with proper prefixes + let tool_names: Vec<&str> = tools.iter().map(|t| t.name.as_ref()).collect(); + println!("Available tools: {:?}", tool_names); + + assert!(tool_names.iter().any(|name| name.contains("echo")), + "Echo tool not found in: {:?}", tool_names); + assert!(tool_names.iter().any(|name| name.contains("add_numbers")), + "Add numbers tool not found in: {:?}", tool_names); +} + +#[tokio::test] +async fn test_tool_call_dispatch_with_python_mcp_server() { + // Get the path to the test MCP server + let mut test_server_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + test_server_path.push("tests"); + test_server_path.push("test_mcp_server.py"); + + // Create extension manager + let mut extension_manager = ExtensionManager::new(); + + // Create a stdio extension config for the Python MCP server + let extension_config = ExtensionConfig::Stdio { + name: "test_python_mcp".to_string(), + description: Some("Test Python MCP Server".to_string()), + cmd: "python3".to_string(), + args: vec![test_server_path.to_string_lossy().to_string()], + envs: Envs::default(), + env_keys: vec![], + timeout: Some(30), // 30 seconds timeout + bundled: Some(false), + }; + + // Add the extension + let result = extension_manager.add_extension(extension_config).await; + assert!(result.is_ok(), "Failed to add extension: {:?}", result); + + // Wait a moment for the connection to stabilize + tokio::time::sleep(Duration::from_millis(500)).await; + + // Test echo tool call + let echo_tool_call = ToolCall { + name: "test_python_mcp__echo".to_string(), + arguments: json!({ + "message": "Hello, MCP World!" + }), + }; + + let result = extension_manager + .dispatch_tool_call(echo_tool_call, CancellationToken::default()) + .await; + + assert!(result.is_ok(), "Failed to dispatch echo tool call"); + + let tool_result = result.unwrap(); + let content = tool_result.result.await; + assert!(content.is_ok(), "Tool execution failed: {:?}", content); + + let content = content.unwrap(); + assert!(!content.is_empty(), "Tool result is empty"); + + // Check that the echo response contains our message + let content_text = match &content[0] { + Content::Text { text } => text, + _ => panic!("Expected text content"), + }; + assert!(content_text.contains("Hello, MCP World!"), + "Echo response doesn't contain expected message: {}", content_text); + + // Test add_numbers tool call + let add_tool_call = ToolCall { + name: "test_python_mcp__add_numbers".to_string(), + arguments: json!({ + "a": 5, + "b": 3 + }), + }; + + let result = extension_manager + .dispatch_tool_call(add_tool_call, CancellationToken::default()) + .await; + + assert!(result.is_ok(), "Failed to dispatch add_numbers tool call"); + + let tool_result = result.unwrap(); + let content = tool_result.result.await; + assert!(content.is_ok(), "Add numbers tool execution failed: {:?}", content); + + let content = content.unwrap(); + assert!(!content.is_empty(), "Add numbers tool result is empty"); + + // Check that the addition result is correct + let content_text = match &content[0] { + Content::Text { text } => text, + _ => panic!("Expected text content"), + }; + assert!(content_text.contains("8"), + "Add numbers response doesn't contain expected result: {}", content_text); +} + +#[tokio::test] +async fn test_invalid_tool_call_with_python_mcp_server() { + // Get the path to the test MCP server + let mut test_server_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + test_server_path.push("tests"); + test_server_path.push("test_mcp_server.py"); + + // Create extension manager + let mut extension_manager = ExtensionManager::new(); + + // Create a stdio extension config for the Python MCP server + let extension_config = ExtensionConfig::Stdio { + name: "test_python_mcp".to_string(), + description: Some("Test Python MCP Server".to_string()), + cmd: "python3".to_string(), + args: vec![test_server_path.to_string_lossy().to_string()], + envs: Envs::default(), + env_keys: vec![], + timeout: Some(30), // 30 seconds timeout + bundled: Some(false), + }; + + // Add the extension + let result = extension_manager.add_extension(extension_config).await; + assert!(result.is_ok(), "Failed to add extension: {:?}", result); + + // Wait a moment for the connection to stabilize + tokio::time::sleep(Duration::from_millis(500)).await; + + // Test calling a tool that doesn't exist + let invalid_tool_call = ToolCall { + name: "test_python_mcp__nonexistent_tool".to_string(), + arguments: json!({}), + }; + + let result = extension_manager + .dispatch_tool_call(invalid_tool_call, CancellationToken::default()) + .await; + + assert!(result.is_ok(), "Tool dispatch should succeed even if tool execution fails"); + + let tool_result = result.unwrap(); + let content = tool_result.result.await; + assert!(content.is_err(), "Expected tool execution to fail for nonexistent tool"); + + // Test calling echo tool with missing parameters + let invalid_params_tool_call = ToolCall { + name: "test_python_mcp__echo".to_string(), + arguments: json!({}), // Missing required 'message' parameter + }; + + let result = extension_manager + .dispatch_tool_call(invalid_params_tool_call, CancellationToken::default()) + .await; + + assert!(result.is_ok(), "Tool dispatch should succeed"); + + let tool_result = result.unwrap(); + let content = tool_result.result.await; + // This might succeed or fail depending on how the Python server handles missing params + // The important thing is that the dispatch mechanism works + println!("Result with missing params: {:?}", content); +} \ No newline at end of file diff --git a/crates/goose/tests/test_mcp_server.py b/crates/goose/tests/test_mcp_server.py new file mode 100644 index 000000000000..259df16dbe0e --- /dev/null +++ b/crates/goose/tests/test_mcp_server.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +import asyncio +import sys +from mcp.server.models import InitializationOptions +from mcp.server import NotificationOptions, Server +from mcp.server.session import ServerSession +from mcp.types import Tool, TextContent, CallToolResult +import mcp.server.stdio +import json + +server = Server("test-mcp-server") + +@server.list_tools() +async def handle_list_tools() -> list[Tool]: + """List available tools.""" + return [ + Tool( + name="echo", + description="Echo back the input message", + inputSchema={ + "type": "object", + "properties": { + "message": { + "type": "string", + "description": "The message to echo back" + } + }, + "required": ["message"] + } + ), + Tool( + name="add_numbers", + description="Add two numbers together", + inputSchema={ + "type": "object", + "properties": { + "a": { + "type": "number", + "description": "First number" + }, + "b": { + "type": "number", + "description": "Second number" + } + }, + "required": ["a", "b"] + } + ) + ] + +@server.call_tool() +async def handle_call_tool(name: str, arguments: dict) -> list[TextContent]: + """Handle tool calls.""" + if name == "echo": + message = arguments.get("message", "") + return [TextContent(type="text", text=f"Echo: {message}")] + elif name == "add_numbers": + a = arguments.get("a", 0) + b = arguments.get("b", 0) + result = a + b + return [TextContent(type="text", text=f"Result: {result}")] + else: + raise ValueError(f"Unknown tool: {name}") + +async def main(): + # Run the server using stdin/stdout + async with mcp.server.stdio.stdio_server() as (read_stream, write_stream): + await server.run( + read_stream, + write_stream, + InitializationOptions( + server_name="test-mcp-server", + server_version="1.0.0", + capabilities=server.get_capabilities( + notification_options=NotificationOptions(), + experimental_capabilities={}, + ), + ), + ) + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file From 7a5c4d222a5cb4525cc6329c04eab90c7afaf381 Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Thu, 7 Aug 2025 16:08:51 -0400 Subject: [PATCH 02/17] stdio logger --- crates/goose/src/bin/stdio_logger.rs | 141 +++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 crates/goose/src/bin/stdio_logger.rs diff --git a/crates/goose/src/bin/stdio_logger.rs b/crates/goose/src/bin/stdio_logger.rs new file mode 100644 index 000000000000..0f382bfc994d --- /dev/null +++ b/crates/goose/src/bin/stdio_logger.rs @@ -0,0 +1,141 @@ +use std::env; +use std::fs::{File, OpenOptions}; +use std::io::{self, BufRead, BufReader, Write}; +use std::process::{ChildStdin, Command, Stdio}; +use std::thread::{self, JoinHandle}; + +// Generic function to handle output streams (stdout/stderr) +fn handle_output_stream( + reader: R, + mut log_file: File, + mut output_writer: Box, +) -> JoinHandle<()> { + thread::spawn(move || { + for line in reader.lines() { + match line { + Ok(line) => { + // Log the output + if let Err(e) = writeln!(log_file, "{}", line) { + eprintln!("Error writing to log file: {}", e); + } + log_file.flush().ok(); + + // Forward to output + if writeln!(output_writer, "{}", line).is_err() { + break; + } + } + Err(_) => break, + } + } + }) +} + +// Handle stdin separately since it has different logic +fn handle_stdin_stream(mut child_stdin: ChildStdin, mut log_file: File) -> JoinHandle<()> { + thread::spawn(move || { + let stdin = io::stdin(); + + for line in stdin.lock().lines() { + match line { + Ok(line) => { + // Log the input + if let Err(e) = writeln!(log_file, "{}", line) { + eprintln!("Error writing to stdin.log: {}", e); + } + log_file.flush().ok(); + + // Forward to child process + if writeln!(child_stdin, "{}", line).is_err() { + break; // Child process closed stdin + } + } + Err(_) => break, + } + } + }) +} + +fn main() -> io::Result<()> { + let args: Vec = env::args().collect(); + + if args.len() < 2 { + eprintln!("Usage: {} [args...]", args[0]); + eprintln!("Example: {{}} ls -la"); + std::process::exit(1); + } + + // Extract command and arguments + let cmd = &args[1]; + let cmd_args = &args[2..]; + + // Create log files + let stdin_log = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open("stdin.log")?; + + let stdout_log = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open("stdout.log")?; + + let stderr_log = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open("stderr.log")?; + + // Spawn the child process + let mut child = Command::new(cmd) + .args(cmd_args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| { + eprintln!("Failed to execute command '{}': {}", cmd, e); + e + })?; + + // Get handles to child's stdio + let child_stdin = child.stdin.take().unwrap(); + let child_stdout = child.stdout.take().unwrap(); + let child_stderr = child.stderr.take().unwrap(); + + // Start I/O handling threads + let stdin_handle = handle_stdin_stream(child_stdin, stdin_log); + let stdout_handle = handle_output_stream( + BufReader::new(child_stdout), + stdout_log, + Box::new(io::stdout()), + ); + let stderr_handle = handle_output_stream( + BufReader::new(child_stderr), + stderr_log, + Box::new(io::stderr()), + ); + + // Wait for the child process to complete + let exit_status = child.wait()?; + + // Wait for all I/O threads to finish processing + stdin_handle.join().ok(); + stdout_handle.join().ok(); + stderr_handle.join().ok(); + + // Print completion message + println!( + "\nCommand completed with exit code: {:?}", + exit_status.code() + ); + println!("Logs written to:"); + println!(" - stdin.log (input sent to the command)"); + println!(" - stdout.log (standard output from the command)"); + println!(" - stderr.log (error output from the command)"); + + // Exit with the same code as the child process + std::process::exit(exit_status.code().unwrap_or(1)); +} From 899cbee51414cdd47896ec3a62560336863a711f Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Thu, 7 Aug 2025 16:11:24 -0400 Subject: [PATCH 03/17] de-llmify --- crates/goose/src/bin/stdio_logger.rs | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/crates/goose/src/bin/stdio_logger.rs b/crates/goose/src/bin/stdio_logger.rs index 0f382bfc994d..2a70b4ad4f4b 100644 --- a/crates/goose/src/bin/stdio_logger.rs +++ b/crates/goose/src/bin/stdio_logger.rs @@ -4,7 +4,6 @@ use std::io::{self, BufRead, BufReader, Write}; use std::process::{ChildStdin, Command, Stdio}; use std::thread::{self, JoinHandle}; -// Generic function to handle output streams (stdout/stderr) fn handle_output_stream( reader: R, mut log_file: File, @@ -14,13 +13,11 @@ fn handle_output_stream( for line in reader.lines() { match line { Ok(line) => { - // Log the output if let Err(e) = writeln!(log_file, "{}", line) { eprintln!("Error writing to log file: {}", e); } log_file.flush().ok(); - // Forward to output if writeln!(output_writer, "{}", line).is_err() { break; } @@ -31,7 +28,6 @@ fn handle_output_stream( }) } -// Handle stdin separately since it has different logic fn handle_stdin_stream(mut child_stdin: ChildStdin, mut log_file: File) -> JoinHandle<()> { thread::spawn(move || { let stdin = io::stdin(); @@ -39,15 +35,13 @@ fn handle_stdin_stream(mut child_stdin: ChildStdin, mut log_file: File) -> JoinH for line in stdin.lock().lines() { match line { Ok(line) => { - // Log the input if let Err(e) = writeln!(log_file, "{}", line) { eprintln!("Error writing to stdin.log: {}", e); } log_file.flush().ok(); - // Forward to child process if writeln!(child_stdin, "{}", line).is_err() { - break; // Child process closed stdin + break; } } Err(_) => break, @@ -65,11 +59,9 @@ fn main() -> io::Result<()> { std::process::exit(1); } - // Extract command and arguments let cmd = &args[1]; let cmd_args = &args[2..]; - // Create log files let stdin_log = OpenOptions::new() .create(true) .write(true) @@ -88,7 +80,6 @@ fn main() -> io::Result<()> { .truncate(true) .open("stderr.log")?; - // Spawn the child process let mut child = Command::new(cmd) .args(cmd_args) .stdin(Stdio::piped()) @@ -100,12 +91,10 @@ fn main() -> io::Result<()> { e })?; - // Get handles to child's stdio let child_stdin = child.stdin.take().unwrap(); let child_stdout = child.stdout.take().unwrap(); let child_stderr = child.stderr.take().unwrap(); - // Start I/O handling threads let stdin_handle = handle_stdin_stream(child_stdin, stdin_log); let stdout_handle = handle_output_stream( BufReader::new(child_stdout), @@ -118,24 +107,11 @@ fn main() -> io::Result<()> { Box::new(io::stderr()), ); - // Wait for the child process to complete let exit_status = child.wait()?; - // Wait for all I/O threads to finish processing stdin_handle.join().ok(); stdout_handle.join().ok(); stderr_handle.join().ok(); - // Print completion message - println!( - "\nCommand completed with exit code: {:?}", - exit_status.code() - ); - println!("Logs written to:"); - println!(" - stdin.log (input sent to the command)"); - println!(" - stdout.log (standard output from the command)"); - println!(" - stderr.log (error output from the command)"); - - // Exit with the same code as the child process std::process::exit(exit_status.code().unwrap_or(1)); } From c9680408c6ec66cb376c2b7a91aab6175c297053 Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Thu, 7 Aug 2025 16:23:35 -0400 Subject: [PATCH 04/17] Combined log --- crates/goose/src/bin/stdio_logger.rs | 64 ++++++++++++++++------------ 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/crates/goose/src/bin/stdio_logger.rs b/crates/goose/src/bin/stdio_logger.rs index 2a70b4ad4f4b..dc503fc43113 100644 --- a/crates/goose/src/bin/stdio_logger.rs +++ b/crates/goose/src/bin/stdio_logger.rs @@ -1,22 +1,28 @@ use std::env; -use std::fs::{File, OpenOptions}; +use std::fs::OpenOptions; use std::io::{self, BufRead, BufReader, Write}; use std::process::{ChildStdin, Command, Stdio}; +use std::sync::mpsc; use std::thread::{self, JoinHandle}; +#[derive(Debug, Clone)] +enum StreamType { + Stdin, + Stdout, + Stderr, +} + fn handle_output_stream( reader: R, - mut log_file: File, + sender: mpsc::Sender<(StreamType, String)>, + stream_type: StreamType, mut output_writer: Box, ) -> JoinHandle<()> { thread::spawn(move || { for line in reader.lines() { match line { Ok(line) => { - if let Err(e) = writeln!(log_file, "{}", line) { - eprintln!("Error writing to log file: {}", e); - } - log_file.flush().ok(); + let _ = sender.send((stream_type.clone(), line.clone())); if writeln!(output_writer, "{}", line).is_err() { break; @@ -28,17 +34,14 @@ fn handle_output_stream( }) } -fn handle_stdin_stream(mut child_stdin: ChildStdin, mut log_file: File) -> JoinHandle<()> { +fn handle_stdin_stream(mut child_stdin: ChildStdin, sender: mpsc::Sender<(StreamType, String)>) -> JoinHandle<()> { thread::spawn(move || { let stdin = io::stdin(); for line in stdin.lock().lines() { match line { Ok(line) => { - if let Err(e) = writeln!(log_file, "{}", line) { - eprintln!("Error writing to stdin.log: {}", e); - } - log_file.flush().ok(); + let _ = sender.send((StreamType::Stdin, line.clone())); if writeln!(child_stdin, "{}", line).is_err() { break; @@ -62,23 +65,13 @@ fn main() -> io::Result<()> { let cmd = &args[1]; let cmd_args = &args[2..]; - let stdin_log = OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open("stdin.log")?; + let (tx, rx) = mpsc::channel(); - let stdout_log = OpenOptions::new() + let log_file = OpenOptions::new() .create(true) .write(true) .truncate(true) - .open("stdout.log")?; - - let stderr_log = OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open("stderr.log")?; + .open("stdio.log")?; let mut child = Command::new(cmd) .args(cmd_args) @@ -95,18 +88,35 @@ fn main() -> io::Result<()> { let child_stdout = child.stdout.take().unwrap(); let child_stderr = child.stderr.take().unwrap(); - let stdin_handle = handle_stdin_stream(child_stdin, stdin_log); + let stdin_handle = handle_stdin_stream(child_stdin, tx.clone()); let stdout_handle = handle_output_stream( BufReader::new(child_stdout), - stdout_log, + tx.clone(), + StreamType::Stdout, Box::new(io::stdout()), ); let stderr_handle = handle_output_stream( BufReader::new(child_stderr), - stderr_log, + tx.clone(), + StreamType::Stderr, Box::new(io::stderr()), ); + thread::spawn(move || { + let mut log_file = log_file; + for (stream_type, line) in rx { + let prefix = match stream_type { + StreamType::Stdin => "STDIN", + StreamType::Stdout => "STDOUT", + StreamType::Stderr => "STDERR", + }; + if let Err(e) = writeln!(log_file, "{}: {}", prefix, line) { + eprintln!("Error writing to log file: {}", e); + } + log_file.flush().ok(); + } + }); + let exit_status = child.wait()?; stdin_handle.join().ok(); From 0de839687f3ceb15126588fcdac1f9518e9cb8da Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Thu, 7 Aug 2025 17:03:33 -0400 Subject: [PATCH 05/17] Replay for integration test --- crates/goose/src/bin/stdio_logger.rs | 16 +- crates/goose/src/bin/stdio_replayer.rs | 95 +++++ crates/goose/tests/mcp_integration_test.rs | 470 ++++++++++++--------- 3 files changed, 386 insertions(+), 195 deletions(-) create mode 100644 crates/goose/src/bin/stdio_replayer.rs diff --git a/crates/goose/src/bin/stdio_logger.rs b/crates/goose/src/bin/stdio_logger.rs index dc503fc43113..e3bae3ad00c2 100644 --- a/crates/goose/src/bin/stdio_logger.rs +++ b/crates/goose/src/bin/stdio_logger.rs @@ -34,7 +34,10 @@ fn handle_output_stream( }) } -fn handle_stdin_stream(mut child_stdin: ChildStdin, sender: mpsc::Sender<(StreamType, String)>) -> JoinHandle<()> { +fn handle_stdin_stream( + mut child_stdin: ChildStdin, + sender: mpsc::Sender<(StreamType, String)>, +) -> JoinHandle<()> { thread::spawn(move || { let stdin = io::stdin(); @@ -56,14 +59,15 @@ fn handle_stdin_stream(mut child_stdin: ChildStdin, sender: mpsc::Sender<(Stream fn main() -> io::Result<()> { let args: Vec = env::args().collect(); - if args.len() < 2 { - eprintln!("Usage: {} [args...]", args[0]); + if args.len() < 3 { + eprintln!("Usage: {} [args...]", args[0]); eprintln!("Example: {{}} ls -la"); std::process::exit(1); } - let cmd = &args[1]; - let cmd_args = &args[2..]; + let log_file_path = &args[1]; + let cmd = &args[2]; + let cmd_args = &args[3..]; let (tx, rx) = mpsc::channel(); @@ -71,7 +75,7 @@ fn main() -> io::Result<()> { .create(true) .write(true) .truncate(true) - .open("stdio.log")?; + .open(log_file_path)?; let mut child = Command::new(cmd) .args(cmd_args) diff --git a/crates/goose/src/bin/stdio_replayer.rs b/crates/goose/src/bin/stdio_replayer.rs new file mode 100644 index 000000000000..cc6380d00fcd --- /dev/null +++ b/crates/goose/src/bin/stdio_replayer.rs @@ -0,0 +1,95 @@ +use std::env; +use std::fs::File; +use std::io::{self, BufRead, BufReader, Write}; +use std::process; + +#[derive(Debug, Clone)] +enum StreamType { + Stdin, + Stdout, + Stderr, +} + +#[derive(Debug, Clone)] +struct LogEntry { + stream_type: StreamType, + content: String, +} + +fn parse_log_line(line: &str) -> Option { + if let Some(colon_pos) = line.find(": ") { + let (prefix, content) = line.split_at(colon_pos); + let content = &content[2..]; // Skip ": " + + let stream_type = match prefix { + "STDIN" => StreamType::Stdin, + "STDOUT" => StreamType::Stdout, + "STDERR" => StreamType::Stderr, + _ => return None, + }; + + Some(LogEntry { + stream_type, + content: content.to_string(), + }) + } else { + None + } +} + +fn load_log_file(file_path: &str) -> io::Result> { + let file = File::open(file_path)?; + let reader = BufReader::new(file); + let mut entries = Vec::new(); + + for line in reader.lines() { + let line = line?; + if let Some(entry) = parse_log_line(&line) { + entries.push(entry); + } + } + + Ok(entries) +} + +fn main() -> io::Result<()> { + let args: Vec = env::args().collect(); + + if args.len() != 2 { + eprintln!("Usage: {} ", args[0]); + process::exit(1); + } + + let log_file_path = &args[1]; + let entries = load_log_file(log_file_path)?; + + let stdin = io::stdin(); + let mut stdout = io::stdout(); + let mut stderr = io::stderr(); + + for entry in entries { + match entry.stream_type { + StreamType::Stdout => { + writeln!(stdout, "{}", entry.content)?; + stdout.flush()?; + } + StreamType::Stderr => { + writeln!(stderr, "{}", entry.content)?; + stderr.flush()?; + } + StreamType::Stdin => { + // Wait for matching input + let mut input = String::new(); + stdin.read_line(&mut input)?; + input = input.trim_end_matches('\n').to_string(); + + if input != entry.content { + eprintln!("Expected: '{}', got: '{}'", entry.content, input); + process::exit(1); + } + } + } + } + + Ok(()) +} \ No newline at end of file diff --git a/crates/goose/tests/mcp_integration_test.rs b/crates/goose/tests/mcp_integration_test.rs index 2cc32d8f1fe9..53fd649521c4 100644 --- a/crates/goose/tests/mcp_integration_test.rs +++ b/crates/goose/tests/mcp_integration_test.rs @@ -1,211 +1,303 @@ +use serde_json::json; use std::path::PathBuf; use std::time::Duration; use tokio_util::sync::CancellationToken; -use serde_json::json; +use goose::agents::extension::{Envs, ExtensionConfig}; use goose::agents::extension_manager::ExtensionManager; -use goose::agents::extension::{ExtensionConfig, Envs}; use mcp_core::ToolCall; use rmcp::model::Content; -#[tokio::test] -async fn test_extension_manager_with_python_mcp_server() { - // Get the path to the test MCP server - let mut test_server_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - test_server_path.push("tests"); - test_server_path.push("test_mcp_server.py"); - - // Ensure the test server exists - assert!(test_server_path.exists(), "Test MCP server not found at {:?}", test_server_path); - - // Create extension manager - let mut extension_manager = ExtensionManager::new(); - - // Create a stdio extension config for the Python MCP server - let extension_config = ExtensionConfig::Stdio { - name: "test_python_mcp".to_string(), - description: Some("Test Python MCP Server".to_string()), - cmd: "python3".to_string(), - args: vec![test_server_path.to_string_lossy().to_string()], - envs: Envs::default(), - env_keys: vec![], - timeout: Some(30), // 30 seconds timeout - bundled: Some(false), - }; - - // Add the extension - let result = extension_manager.add_extension(extension_config).await; - assert!(result.is_ok(), "Failed to add extension: {:?}", result); - - // Wait a moment for the connection to stabilize - tokio::time::sleep(Duration::from_millis(500)).await; - - // Get the list of tools - let tools = extension_manager.get_prefixed_tools(None).await; - assert!(tools.is_ok(), "Failed to get tools: {:?}", tools); - - let tools = tools.unwrap(); - assert!(!tools.is_empty(), "No tools found"); - - // Verify we have the expected tools with proper prefixes - let tool_names: Vec<&str> = tools.iter().map(|t| t.name.as_ref()).collect(); - println!("Available tools: {:?}", tool_names); - - assert!(tool_names.iter().any(|name| name.contains("echo")), - "Echo tool not found in: {:?}", tool_names); - assert!(tool_names.iter().any(|name| name.contains("add_numbers")), - "Add numbers tool not found in: {:?}", tool_names); -} +// #[tokio::test] +// async fn test_extension_manager_with_python_mcp_server() { +// // Get the path to the test MCP server +// let mut test_server_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); +// test_server_path.push("tests"); +// test_server_path.push("test_mcp_server.py"); -#[tokio::test] -async fn test_tool_call_dispatch_with_python_mcp_server() { - // Get the path to the test MCP server - let mut test_server_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - test_server_path.push("tests"); - test_server_path.push("test_mcp_server.py"); - - // Create extension manager - let mut extension_manager = ExtensionManager::new(); - - // Create a stdio extension config for the Python MCP server - let extension_config = ExtensionConfig::Stdio { - name: "test_python_mcp".to_string(), - description: Some("Test Python MCP Server".to_string()), - cmd: "python3".to_string(), - args: vec![test_server_path.to_string_lossy().to_string()], - envs: Envs::default(), - env_keys: vec![], - timeout: Some(30), // 30 seconds timeout - bundled: Some(false), - }; - - // Add the extension - let result = extension_manager.add_extension(extension_config).await; - assert!(result.is_ok(), "Failed to add extension: {:?}", result); - - // Wait a moment for the connection to stabilize - tokio::time::sleep(Duration::from_millis(500)).await; - - // Test echo tool call - let echo_tool_call = ToolCall { - name: "test_python_mcp__echo".to_string(), - arguments: json!({ - "message": "Hello, MCP World!" - }), - }; - - let result = extension_manager - .dispatch_tool_call(echo_tool_call, CancellationToken::default()) - .await; - - assert!(result.is_ok(), "Failed to dispatch echo tool call"); - - let tool_result = result.unwrap(); - let content = tool_result.result.await; - assert!(content.is_ok(), "Tool execution failed: {:?}", content); - - let content = content.unwrap(); - assert!(!content.is_empty(), "Tool result is empty"); - - // Check that the echo response contains our message - let content_text = match &content[0] { - Content::Text { text } => text, - _ => panic!("Expected text content"), - }; - assert!(content_text.contains("Hello, MCP World!"), - "Echo response doesn't contain expected message: {}", content_text); - - // Test add_numbers tool call - let add_tool_call = ToolCall { - name: "test_python_mcp__add_numbers".to_string(), - arguments: json!({ - "a": 5, - "b": 3 - }), - }; - - let result = extension_manager - .dispatch_tool_call(add_tool_call, CancellationToken::default()) - .await; - - assert!(result.is_ok(), "Failed to dispatch add_numbers tool call"); - - let tool_result = result.unwrap(); - let content = tool_result.result.await; - assert!(content.is_ok(), "Add numbers tool execution failed: {:?}", content); - - let content = content.unwrap(); - assert!(!content.is_empty(), "Add numbers tool result is empty"); - - // Check that the addition result is correct - let content_text = match &content[0] { - Content::Text { text } => text, - _ => panic!("Expected text content"), - }; - assert!(content_text.contains("8"), - "Add numbers response doesn't contain expected result: {}", content_text); +// // Ensure the test server exists +// assert!( +// test_server_path.exists(), +// "Test MCP server not found at {:?}", +// test_server_path +// ); + +// // Create extension manager +// let mut extension_manager = ExtensionManager::new(); + +// // Create a stdio extension config for the Python MCP server +// let extension_config = ExtensionConfig::Stdio { +// name: "test_python_mcp".to_string(), +// description: Some("Test Python MCP Server".to_string()), +// cmd: "python3".to_string(), +// args: vec![test_server_path.to_string_lossy().to_string()], +// envs: Envs::default(), +// env_keys: vec![], +// timeout: Some(30), // 30 seconds timeout +// bundled: Some(false), +// }; + +// // Add the extension +// let result = extension_manager.add_extension(extension_config).await; +// assert!(result.is_ok(), "Failed to add extension: {:?}", result); + +// // Wait a moment for the connection to stabilize +// tokio::time::sleep(Duration::from_millis(500)).await; + +// // Get the list of tools +// let tools = extension_manager.get_prefixed_tools(None).await; +// assert!(tools.is_ok(), "Failed to get tools: {:?}", tools); + +// let tools = tools.unwrap(); +// assert!(!tools.is_empty(), "No tools found"); + +// // Verify we have the expected tools with proper prefixes +// let tool_names: Vec<&str> = tools.iter().map(|t| t.name.as_ref()).collect(); +// println!("Available tools: {:?}", tool_names); + +// assert!( +// tool_names.iter().any(|name| name.contains("echo")), +// "Echo tool not found in: {:?}", +// tool_names +// ); +// assert!( +// tool_names.iter().any(|name| name.contains("add_numbers")), +// "Add numbers tool not found in: {:?}", +// tool_names +// ); +// } + +// #[tokio::test] +// async fn test_tool_call_dispatch_with_python_mcp_server() { +// // Get the path to the test MCP server +// let mut test_server_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); +// test_server_path.push("tests"); +// test_server_path.push("test_mcp_server.py"); + +// // Create extension manager +// let mut extension_manager = ExtensionManager::new(); + +// // Create a stdio extension config for the Python MCP server +// let extension_config = ExtensionConfig::Stdio { +// name: "test_python_mcp".to_string(), +// description: Some("Test Python MCP Server".to_string()), +// cmd: "python3".to_string(), +// args: vec![test_server_path.to_string_lossy().to_string()], +// envs: Envs::default(), +// env_keys: vec![], +// timeout: Some(30), // 30 seconds timeout +// bundled: Some(false), +// }; + +// // Add the extension +// let result = extension_manager.add_extension(extension_config).await; +// assert!(result.is_ok(), "Failed to add extension: {:?}", result); + +// // Wait a moment for the connection to stabilize +// tokio::time::sleep(Duration::from_millis(500)).await; + +// // Test echo tool call +// let echo_tool_call = ToolCall { +// name: "test_python_mcp__echo".to_string(), +// arguments: json!({ +// "message": "Hello, MCP World!" +// }), +// }; + +// let result = extension_manager +// .dispatch_tool_call(echo_tool_call, CancellationToken::default()) +// .await; + +// assert!(result.is_ok(), "Failed to dispatch echo tool call"); + +// let tool_result = result.unwrap(); +// let content = tool_result.result.await; +// assert!(content.is_ok(), "Tool execution failed: {:?}", content); + +// let content = content.unwrap(); +// assert!(!content.is_empty(), "Tool result is empty"); + +// // Check that the echo response contains our message +// let content_text = match &content[0] { +// Content::Text { text } => text, +// _ => panic!("Expected text content"), +// }; +// assert!( +// content_text.contains("Hello, MCP World!"), +// "Echo response doesn't contain expected message: {}", +// content_text +// ); + +// // Test add_numbers tool call +// let add_tool_call = ToolCall { +// name: "test_python_mcp__add_numbers".to_string(), +// arguments: json!({ +// "a": 5, +// "b": 3 +// }), +// }; + +// let result = extension_manager +// .dispatch_tool_call(add_tool_call, CancellationToken::default()) +// .await; + +// assert!(result.is_ok(), "Failed to dispatch add_numbers tool call"); + +// let tool_result = result.unwrap(); +// let content = tool_result.result.await; +// assert!( +// content.is_ok(), +// "Add numbers tool execution failed: {:?}", +// content +// ); + +// let content = content.unwrap(); +// assert!(!content.is_empty(), "Add numbers tool result is empty"); + +// // Check that the addition result is correct +// let content_text = match &content[0] { +// Content::Text { text } => text, +// _ => panic!("Expected text content"), +// }; +// assert!( +// content_text.contains("8"), +// "Add numbers response doesn't contain expected result: {}", +// content_text +// ); +// } + +// #[tokio::test] +// async fn test_invalid_tool_call_with_python_mcp_server() { +// // Get the path to the test MCP server +// let mut test_server_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); +// test_server_path.push("tests"); +// test_server_path.push("test_mcp_server.py"); + +// // Create extension manager +// let mut extension_manager = ExtensionManager::new(); + +// // Create a stdio extension config for the Python MCP server +// let extension_config = ExtensionConfig::Stdio { +// name: "test_python_mcp".to_string(), +// description: Some("Test Python MCP Server".to_string()), +// cmd: "python3".to_string(), +// args: vec![test_server_path.to_string_lossy().to_string()], +// envs: Envs::default(), +// env_keys: vec![], +// timeout: Some(30), // 30 seconds timeout +// bundled: Some(false), +// }; + +// // Add the extension +// let result = extension_manager.add_extension(extension_config).await; +// assert!(result.is_ok(), "Failed to add extension: {:?}", result); + +// // Wait a moment for the connection to stabilize +// tokio::time::sleep(Duration::from_millis(500)).await; + +// // Test calling a tool that doesn't exist +// let invalid_tool_call = ToolCall { +// name: "test_python_mcp__nonexistent_tool".to_string(), +// arguments: json!({}), +// }; + +// let result = extension_manager +// .dispatch_tool_call(invalid_tool_call, CancellationToken::default()) +// .await; + +// assert!( +// result.is_ok(), +// "Tool dispatch should succeed even if tool execution fails" +// ); + +// let tool_result = result.unwrap(); +// let content = tool_result.result.await; +// assert!( +// content.is_err(), +// "Expected tool execution to fail for nonexistent tool" +// ); + +// // Test calling echo tool with missing parameters +// let invalid_params_tool_call = ToolCall { +// name: "test_python_mcp__echo".to_string(), +// arguments: json!({}), // Missing required 'message' parameter +// }; + +// let result = extension_manager +// .dispatch_tool_call(invalid_params_tool_call, CancellationToken::default()) +// .await; + +// assert!(result.is_ok(), "Tool dispatch should succeed"); + +// let tool_result = result.unwrap(); +// let content = tool_result.result.await; +// // This might succeed or fail depending on how the Python server handles missing params +// // The important thing is that the dispatch mechanism works +// println!("Result with missing params: {:?}", content); +// } + +enum TestMode { + Record, + Replay, } #[tokio::test] -async fn test_invalid_tool_call_with_python_mcp_server() { - // Get the path to the test MCP server - let mut test_server_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - test_server_path.push("tests"); - test_server_path.push("test_mcp_server.py"); - - // Create extension manager +async fn test_replayed_session() { let mut extension_manager = ExtensionManager::new(); - - // Create a stdio extension config for the Python MCP server - let extension_config = ExtensionConfig::Stdio { - name: "test_python_mcp".to_string(), - description: Some("Test Python MCP Server".to_string()), - cmd: "python3".to_string(), - args: vec![test_server_path.to_string_lossy().to_string()], - envs: Envs::default(), - env_keys: vec![], - timeout: Some(30), // 30 seconds timeout - bundled: Some(false), + + let mode = TestMode::Replay; + + let extension_config = match mode { + TestMode::Record => ExtensionConfig::Stdio { + name: "test".to_string(), + description: Some("Test".to_string()), + cmd: "/Users/jackamadeo/development/goose/target/debug/stdio_logger".to_string(), + args: vec![ + "/Users/jackamadeo/development/goose/test-mcp-log.log", + "uv", + "--project", + "/Users/jackamadeo/development/stream-mcp", + "run", + "/Users/jackamadeo/development/stream-mcp/stream_mcp.py", + ] + .into_iter() + .map(str::to_string) + .collect(), + envs: Envs::default(), + env_keys: vec![], + timeout: Some(30), + bundled: Some(false), + }, + TestMode::Replay => ExtensionConfig::Stdio { + name: "test".to_string(), + description: Some("Test".to_string()), + cmd: "/Users/jackamadeo/development/goose/target/debug/stdio_replayer".to_string(), + args: vec!["/Users/jackamadeo/development/goose/test-mcp-log.log".to_string()], + envs: Envs::default(), + env_keys: vec![], + timeout: Some(30), + bundled: Some(false), + }, }; - - // Add the extension + let result = extension_manager.add_extension(extension_config).await; assert!(result.is_ok(), "Failed to add extension: {:?}", result); - - // Wait a moment for the connection to stabilize - tokio::time::sleep(Duration::from_millis(500)).await; - - // Test calling a tool that doesn't exist - let invalid_tool_call = ToolCall { - name: "test_python_mcp__nonexistent_tool".to_string(), + + let tool_call = ToolCall { + name: "test__count".to_string(), arguments: json!({}), }; - - let result = extension_manager - .dispatch_tool_call(invalid_tool_call, CancellationToken::default()) - .await; - - assert!(result.is_ok(), "Tool dispatch should succeed even if tool execution fails"); - - let tool_result = result.unwrap(); - let content = tool_result.result.await; - assert!(content.is_err(), "Expected tool execution to fail for nonexistent tool"); - - // Test calling echo tool with missing parameters - let invalid_params_tool_call = ToolCall { - name: "test_python_mcp__echo".to_string(), - arguments: json!({}), // Missing required 'message' parameter - }; - + let result = extension_manager - .dispatch_tool_call(invalid_params_tool_call, CancellationToken::default()) + .dispatch_tool_call(tool_call, CancellationToken::default()) .await; - - assert!(result.is_ok(), "Tool dispatch should succeed"); - + + assert!( + result.is_ok(), + "Tool dispatch should succeed even if tool execution fails" + ); + let tool_result = result.unwrap(); let content = tool_result.result.await; - // This might succeed or fail depending on how the Python server handles missing params - // The important thing is that the dispatch mechanism works - println!("Result with missing params: {:?}", content); -} \ No newline at end of file + assert!(content.is_ok(), "Expected tool execution to succeed"); +} From d652df8a1041c44cca1771883b939c12a9f68b0d Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Thu, 7 Aug 2025 23:27:33 -0400 Subject: [PATCH 06/17] Replay test --- Cargo.lock | 5 + .../src/bin/stdio_logger.rs | 0 .../src/bin/stdio_replayer.rs | 30 +- crates/goose/Cargo.toml | 1 + crates/goose/tests/mcp_integration_test.rs | 346 ++++-------------- ...x-y@modelcontextprotocol_server-everything | 6 + 6 files changed, 91 insertions(+), 297 deletions(-) rename crates/{goose => goose-test}/src/bin/stdio_logger.rs (100%) rename crates/{goose => goose-test}/src/bin/stdio_replayer.rs (91%) create mode 100644 crates/goose/tests/mcp_replays/npx-y@modelcontextprotocol_server-everything diff --git a/Cargo.lock b/Cargo.lock index 7150160c69a3..a432099cc224 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3333,6 +3333,7 @@ dependencies = [ "sha2", "temp-env", "tempfile", + "test-case", "thiserror 1.0.69", "tiktoken-rs", "tokio", @@ -3514,6 +3515,10 @@ dependencies = [ "utoipa", ] +[[package]] +name = "goose-test" +version = "1.1.0" + [[package]] name = "grep-cli" version = "0.1.11" diff --git a/crates/goose/src/bin/stdio_logger.rs b/crates/goose-test/src/bin/stdio_logger.rs similarity index 100% rename from crates/goose/src/bin/stdio_logger.rs rename to crates/goose-test/src/bin/stdio_logger.rs diff --git a/crates/goose/src/bin/stdio_replayer.rs b/crates/goose-test/src/bin/stdio_replayer.rs similarity index 91% rename from crates/goose/src/bin/stdio_replayer.rs rename to crates/goose-test/src/bin/stdio_replayer.rs index cc6380d00fcd..8f45c218f5ab 100644 --- a/crates/goose/src/bin/stdio_replayer.rs +++ b/crates/goose-test/src/bin/stdio_replayer.rs @@ -17,56 +17,54 @@ struct LogEntry { } fn parse_log_line(line: &str) -> Option { - if let Some(colon_pos) = line.find(": ") { - let (prefix, content) = line.split_at(colon_pos); + line.find(": ").and_then(|pos| { + let (prefix, content) = line.split_at(pos); let content = &content[2..]; // Skip ": " - + let stream_type = match prefix { "STDIN" => StreamType::Stdin, "STDOUT" => StreamType::Stdout, "STDERR" => StreamType::Stderr, _ => return None, }; - + Some(LogEntry { stream_type, content: content.to_string(), }) - } else { - None - } + }) } fn load_log_file(file_path: &str) -> io::Result> { let file = File::open(file_path)?; let reader = BufReader::new(file); let mut entries = Vec::new(); - + for line in reader.lines() { let line = line?; if let Some(entry) = parse_log_line(&line) { entries.push(entry); } } - + Ok(entries) } fn main() -> io::Result<()> { let args: Vec = env::args().collect(); - + if args.len() != 2 { eprintln!("Usage: {} ", args[0]); process::exit(1); } - + let log_file_path = &args[1]; let entries = load_log_file(log_file_path)?; - + let stdin = io::stdin(); let mut stdout = io::stdout(); let mut stderr = io::stderr(); - + for entry in entries { match entry.stream_type { StreamType::Stdout => { @@ -82,7 +80,7 @@ fn main() -> io::Result<()> { let mut input = String::new(); stdin.read_line(&mut input)?; input = input.trim_end_matches('\n').to_string(); - + if input != entry.content { eprintln!("Expected: '{}', got: '{}'", entry.content, input); process::exit(1); @@ -90,6 +88,6 @@ fn main() -> io::Result<()> { } } } - + Ok(()) -} \ No newline at end of file +} diff --git a/crates/goose/Cargo.toml b/crates/goose/Cargo.toml index b8fad75b5ef3..cbcda3aab241 100644 --- a/crates/goose/Cargo.toml +++ b/crates/goose/Cargo.toml @@ -112,6 +112,7 @@ tokio = { version = "1.43", features = ["full"] } temp-env = "0.3.6" dotenvy = "0.15.7" ctor = "0.2.9" +test-case = "3.3" [[example]] name = "agent" diff --git a/crates/goose/tests/mcp_integration_test.rs b/crates/goose/tests/mcp_integration_test.rs index 53fd649521c4..4fb7c8b84ecc 100644 --- a/crates/goose/tests/mcp_integration_test.rs +++ b/crates/goose/tests/mcp_integration_test.rs @@ -1,303 +1,87 @@ -use serde_json::json; +use std::env; use std::path::PathBuf; -use std::time::Duration; + +use serde_json::json; use tokio_util::sync::CancellationToken; use goose::agents::extension::{Envs, ExtensionConfig}; use goose::agents::extension_manager::ExtensionManager; use mcp_core::ToolCall; -use rmcp::model::Content; - -// #[tokio::test] -// async fn test_extension_manager_with_python_mcp_server() { -// // Get the path to the test MCP server -// let mut test_server_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); -// test_server_path.push("tests"); -// test_server_path.push("test_mcp_server.py"); - -// // Ensure the test server exists -// assert!( -// test_server_path.exists(), -// "Test MCP server not found at {:?}", -// test_server_path -// ); - -// // Create extension manager -// let mut extension_manager = ExtensionManager::new(); - -// // Create a stdio extension config for the Python MCP server -// let extension_config = ExtensionConfig::Stdio { -// name: "test_python_mcp".to_string(), -// description: Some("Test Python MCP Server".to_string()), -// cmd: "python3".to_string(), -// args: vec![test_server_path.to_string_lossy().to_string()], -// envs: Envs::default(), -// env_keys: vec![], -// timeout: Some(30), // 30 seconds timeout -// bundled: Some(false), -// }; - -// // Add the extension -// let result = extension_manager.add_extension(extension_config).await; -// assert!(result.is_ok(), "Failed to add extension: {:?}", result); - -// // Wait a moment for the connection to stabilize -// tokio::time::sleep(Duration::from_millis(500)).await; - -// // Get the list of tools -// let tools = extension_manager.get_prefixed_tools(None).await; -// assert!(tools.is_ok(), "Failed to get tools: {:?}", tools); - -// let tools = tools.unwrap(); -// assert!(!tools.is_empty(), "No tools found"); - -// // Verify we have the expected tools with proper prefixes -// let tool_names: Vec<&str> = tools.iter().map(|t| t.name.as_ref()).collect(); -// println!("Available tools: {:?}", tool_names); - -// assert!( -// tool_names.iter().any(|name| name.contains("echo")), -// "Echo tool not found in: {:?}", -// tool_names -// ); -// assert!( -// tool_names.iter().any(|name| name.contains("add_numbers")), -// "Add numbers tool not found in: {:?}", -// tool_names -// ); -// } - -// #[tokio::test] -// async fn test_tool_call_dispatch_with_python_mcp_server() { -// // Get the path to the test MCP server -// let mut test_server_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); -// test_server_path.push("tests"); -// test_server_path.push("test_mcp_server.py"); - -// // Create extension manager -// let mut extension_manager = ExtensionManager::new(); - -// // Create a stdio extension config for the Python MCP server -// let extension_config = ExtensionConfig::Stdio { -// name: "test_python_mcp".to_string(), -// description: Some("Test Python MCP Server".to_string()), -// cmd: "python3".to_string(), -// args: vec![test_server_path.to_string_lossy().to_string()], -// envs: Envs::default(), -// env_keys: vec![], -// timeout: Some(30), // 30 seconds timeout -// bundled: Some(false), -// }; - -// // Add the extension -// let result = extension_manager.add_extension(extension_config).await; -// assert!(result.is_ok(), "Failed to add extension: {:?}", result); - -// // Wait a moment for the connection to stabilize -// tokio::time::sleep(Duration::from_millis(500)).await; - -// // Test echo tool call -// let echo_tool_call = ToolCall { -// name: "test_python_mcp__echo".to_string(), -// arguments: json!({ -// "message": "Hello, MCP World!" -// }), -// }; - -// let result = extension_manager -// .dispatch_tool_call(echo_tool_call, CancellationToken::default()) -// .await; - -// assert!(result.is_ok(), "Failed to dispatch echo tool call"); - -// let tool_result = result.unwrap(); -// let content = tool_result.result.await; -// assert!(content.is_ok(), "Tool execution failed: {:?}", content); - -// let content = content.unwrap(); -// assert!(!content.is_empty(), "Tool result is empty"); - -// // Check that the echo response contains our message -// let content_text = match &content[0] { -// Content::Text { text } => text, -// _ => panic!("Expected text content"), -// }; -// assert!( -// content_text.contains("Hello, MCP World!"), -// "Echo response doesn't contain expected message: {}", -// content_text -// ); - -// // Test add_numbers tool call -// let add_tool_call = ToolCall { -// name: "test_python_mcp__add_numbers".to_string(), -// arguments: json!({ -// "a": 5, -// "b": 3 -// }), -// }; - -// let result = extension_manager -// .dispatch_tool_call(add_tool_call, CancellationToken::default()) -// .await; -// assert!(result.is_ok(), "Failed to dispatch add_numbers tool call"); - -// let tool_result = result.unwrap(); -// let content = tool_result.result.await; -// assert!( -// content.is_ok(), -// "Add numbers tool execution failed: {:?}", -// content -// ); - -// let content = content.unwrap(); -// assert!(!content.is_empty(), "Add numbers tool result is empty"); - -// // Check that the addition result is correct -// let content_text = match &content[0] { -// Content::Text { text } => text, -// _ => panic!("Expected text content"), -// }; -// assert!( -// content_text.contains("8"), -// "Add numbers response doesn't contain expected result: {}", -// content_text -// ); -// } - -// #[tokio::test] -// async fn test_invalid_tool_call_with_python_mcp_server() { -// // Get the path to the test MCP server -// let mut test_server_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); -// test_server_path.push("tests"); -// test_server_path.push("test_mcp_server.py"); - -// // Create extension manager -// let mut extension_manager = ExtensionManager::new(); - -// // Create a stdio extension config for the Python MCP server -// let extension_config = ExtensionConfig::Stdio { -// name: "test_python_mcp".to_string(), -// description: Some("Test Python MCP Server".to_string()), -// cmd: "python3".to_string(), -// args: vec![test_server_path.to_string_lossy().to_string()], -// envs: Envs::default(), -// env_keys: vec![], -// timeout: Some(30), // 30 seconds timeout -// bundled: Some(false), -// }; - -// // Add the extension -// let result = extension_manager.add_extension(extension_config).await; -// assert!(result.is_ok(), "Failed to add extension: {:?}", result); - -// // Wait a moment for the connection to stabilize -// tokio::time::sleep(Duration::from_millis(500)).await; - -// // Test calling a tool that doesn't exist -// let invalid_tool_call = ToolCall { -// name: "test_python_mcp__nonexistent_tool".to_string(), -// arguments: json!({}), -// }; - -// let result = extension_manager -// .dispatch_tool_call(invalid_tool_call, CancellationToken::default()) -// .await; - -// assert!( -// result.is_ok(), -// "Tool dispatch should succeed even if tool execution fails" -// ); - -// let tool_result = result.unwrap(); -// let content = tool_result.result.await; -// assert!( -// content.is_err(), -// "Expected tool execution to fail for nonexistent tool" -// ); - -// // Test calling echo tool with missing parameters -// let invalid_params_tool_call = ToolCall { -// name: "test_python_mcp__echo".to_string(), -// arguments: json!({}), // Missing required 'message' parameter -// }; - -// let result = extension_manager -// .dispatch_tool_call(invalid_params_tool_call, CancellationToken::default()) -// .await; - -// assert!(result.is_ok(), "Tool dispatch should succeed"); - -// let tool_result = result.unwrap(); -// let content = tool_result.result.await; -// // This might succeed or fail depending on how the Python server handles missing params -// // The important thing is that the dispatch mechanism works -// println!("Result with missing params: {:?}", content); -// } +use test_case::test_case; enum TestMode { Record, Replay, } -#[tokio::test] -async fn test_replayed_session() { - let mut extension_manager = ExtensionManager::new(); +const LOGGER_BINARY: &str = "stdio_logger"; +const REPLAY_BINARY: &str = "stdio_replayer"; - let mode = TestMode::Replay; +#[test_case( + vec!["npx", "-y", "@modelcontextprotocol/server-everything"], + vec![ + ToolCall::new("echo", json!({"message": "Hello, world!"})), + ] +)] +#[tokio::test] +async fn test_replayed_session(command: Vec<&str>, tool_calls: Vec) { + let replay_file_name = command + .iter() + .map(|s| s.replace("/", "_")) + .collect::>() + .join(""); + let mut replay_file_path = + PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("should find the project root")); + replay_file_path.push("tests"); + replay_file_path.push("mcp_replays"); + replay_file_path.push(replay_file_name); + + let mode = if env::var("GOOSE_RECORD_MCP").is_ok() { + TestMode::Record + } else { + assert!(replay_file_path.exists(), "replay file doesn't exist"); + TestMode::Replay + }; - let extension_config = match mode { - TestMode::Record => ExtensionConfig::Stdio { - name: "test".to_string(), - description: Some("Test".to_string()), - cmd: "/Users/jackamadeo/development/goose/target/debug/stdio_logger".to_string(), - args: vec![ - "/Users/jackamadeo/development/goose/test-mcp-log.log", - "uv", - "--project", - "/Users/jackamadeo/development/stream-mcp", - "run", - "/Users/jackamadeo/development/stream-mcp/stream_mcp.py", - ] - .into_iter() - .map(str::to_string) - .collect(), - envs: Envs::default(), - env_keys: vec![], - timeout: Some(30), - bundled: Some(false), - }, - TestMode::Replay => ExtensionConfig::Stdio { - name: "test".to_string(), - description: Some("Test".to_string()), - cmd: "/Users/jackamadeo/development/goose/target/debug/stdio_replayer".to_string(), - args: vec!["/Users/jackamadeo/development/goose/test-mcp-log.log".to_string()], - envs: Envs::default(), - env_keys: vec![], - timeout: Some(30), - bundled: Some(false), - }, + let bin = match mode { + TestMode::Record => LOGGER_BINARY, + TestMode::Replay => REPLAY_BINARY, + }; + let cmd = "cargo".to_string(); + let mut args = vec!["run", "--quiet", "-p", "goose-test", "--bin", bin, "--"] + .into_iter() + .map(str::to_string) + .collect::>(); + + args.push(replay_file_path.to_string_lossy().to_string()); + + if matches!(mode, TestMode::Record) { + args.extend(command.into_iter().map(str::to_string)); + } + + let extension_config = ExtensionConfig::Stdio { + name: "test".to_string(), + description: Some("Test".to_string()), + cmd, + args, + envs: Envs::default(), + env_keys: vec![], + timeout: Some(30), + bundled: Some(false), }; + let mut extension_manager = ExtensionManager::new(); let result = extension_manager.add_extension(extension_config).await; assert!(result.is_ok(), "Failed to add extension: {:?}", result); - let tool_call = ToolCall { - name: "test__count".to_string(), - arguments: json!({}), - }; - - let result = extension_manager - .dispatch_tool_call(tool_call, CancellationToken::default()) - .await; - - assert!( - result.is_ok(), - "Tool dispatch should succeed even if tool execution fails" - ); + for tool_call in tool_calls { + let tool_call = ToolCall::new(format!("test__{}", tool_call.name), tool_call.arguments); + let result = extension_manager + .dispatch_tool_call(tool_call, CancellationToken::default()) + .await; - let tool_result = result.unwrap(); - let content = tool_result.result.await; - assert!(content.is_ok(), "Expected tool execution to succeed"); + let tool_result = result.expect("tool dispatch should succeed"); + tool_result.result.await.expect("should get a result"); + } } diff --git a/crates/goose/tests/mcp_replays/npx-y@modelcontextprotocol_server-everything b/crates/goose/tests/mcp_replays/npx-y@modelcontextprotocol_server-everything new file mode 100644 index 000000000000..20fcf5b9f8aa --- /dev/null +++ b/crates/goose/tests/mcp_replays/npx-y@modelcontextprotocol_server-everything @@ -0,0 +1,6 @@ +STDIN: {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"goose","version":"0.1.0"}}} +STDERR: Starting default (STDIO) server... +STDOUT: {"result":{"protocolVersion":"2025-03-26","capabilities":{"prompts":{},"resources":{"subscribe":true},"tools":{},"logging":{},"completions":{},"elicitation":{}},"serverInfo":{"name":"example-servers/everything","title":"Everything Example Server","version":"1.0.0"},"instructions":"Testing and demonstration server for MCP protocol features.\n\n## Resources\n\nResources 1-100 follow pattern: even IDs contain text, odd IDs contain binary data. Resources are paginated at 10 items per page with cursor-based navigation.\n\n## Key dependencies\n\nProgress notifications require `_meta.progressToken` in tool calls. Resource subscriptions generate updates every 10 seconds.\n\n## Performance characteristics\n\nServer generates automatic log messages every 20 seconds (filtered by current log level), stderr notifications every 30 seconds, and resource update notifications every 10 seconds for active subscriptions. Log level changes affect message filtering in real-time.\n\n## Multi-modal testing\n\n`complex_prompt` includes both text arguments and image content for testing client multi-modal handling. `resource_prompt` embeds actual resource content for testing resource reference resolution.\n\nArgument completion is available for prompt parameters and resource IDs. Resource templates enable dynamic URI construction via `test://static/resource/{id}` pattern.\n\n## Easter egg\n\nIf asked about server instructions, respond with \"🎉 Server instructions are working! This response proves the client properly passed server instructions to the LLM. This demonstrates MCP's instructions feature in action.\"\n"},"jsonrpc":"2.0","id":0} +STDIN: {"jsonrpc":"2.0","method":"notifications/initialized"} +STDIN: {"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"_meta":{"progressToken":0},"name":"echo","arguments":{"message":"Hello, world!"}}} +STDOUT: {"result":{"content":[{"type":"text","text":"Echo: Hello, world!"}]},"jsonrpc":"2.0","id":1} From 8e0654ef95eeb4b3c5e1f8797fd71dc9bb3038af Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Thu, 7 Aug 2025 23:27:45 -0400 Subject: [PATCH 07/17] don't need that --- crates/goose/tests/test_mcp_server.py | 83 --------------------------- 1 file changed, 83 deletions(-) delete mode 100644 crates/goose/tests/test_mcp_server.py diff --git a/crates/goose/tests/test_mcp_server.py b/crates/goose/tests/test_mcp_server.py deleted file mode 100644 index 259df16dbe0e..000000000000 --- a/crates/goose/tests/test_mcp_server.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 - -import asyncio -import sys -from mcp.server.models import InitializationOptions -from mcp.server import NotificationOptions, Server -from mcp.server.session import ServerSession -from mcp.types import Tool, TextContent, CallToolResult -import mcp.server.stdio -import json - -server = Server("test-mcp-server") - -@server.list_tools() -async def handle_list_tools() -> list[Tool]: - """List available tools.""" - return [ - Tool( - name="echo", - description="Echo back the input message", - inputSchema={ - "type": "object", - "properties": { - "message": { - "type": "string", - "description": "The message to echo back" - } - }, - "required": ["message"] - } - ), - Tool( - name="add_numbers", - description="Add two numbers together", - inputSchema={ - "type": "object", - "properties": { - "a": { - "type": "number", - "description": "First number" - }, - "b": { - "type": "number", - "description": "Second number" - } - }, - "required": ["a", "b"] - } - ) - ] - -@server.call_tool() -async def handle_call_tool(name: str, arguments: dict) -> list[TextContent]: - """Handle tool calls.""" - if name == "echo": - message = arguments.get("message", "") - return [TextContent(type="text", text=f"Echo: {message}")] - elif name == "add_numbers": - a = arguments.get("a", 0) - b = arguments.get("b", 0) - result = a + b - return [TextContent(type="text", text=f"Result: {result}")] - else: - raise ValueError(f"Unknown tool: {name}") - -async def main(): - # Run the server using stdin/stdout - async with mcp.server.stdio.stdio_server() as (read_stream, write_stream): - await server.run( - read_stream, - write_stream, - InitializationOptions( - server_name="test-mcp-server", - server_version="1.0.0", - capabilities=server.get_capabilities( - notification_options=NotificationOptions(), - experimental_capabilities={}, - ), - ), - ) - -if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file From 6971f8cc023974674b6cdc193e3d221a3f2e3dcd Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Thu, 7 Aug 2025 23:28:18 -0400 Subject: [PATCH 08/17] Testing crate cargo.toml --- crates/goose-test/Cargo.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 crates/goose-test/Cargo.toml diff --git a/crates/goose-test/Cargo.toml b/crates/goose-test/Cargo.toml new file mode 100644 index 000000000000..45d1855abf8b --- /dev/null +++ b/crates/goose-test/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "goose-test" +edition.workspace = true +version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +description.workspace = true + +[lints] +workspace = true + +[[bin]] +name = "stdio_logger" +path = "src/bin/stdio_logger.rs" + +[[bin]] +name = "stdio_replayer" +path = "src/bin/stdio_replayer.rs" \ No newline at end of file From acd3adf44686d713ee0ba698cdd9c1662eddf080 Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Thu, 7 Aug 2025 23:55:37 -0400 Subject: [PATCH 09/17] Also check the results --- crates/goose/tests/mcp_integration_test.rs | 29 +++++++++++++++++-- ...x-y@modelcontextprotocol_server-everything | 11 +++++++ ...extprotocol_server-everything.results.json | 26 +++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 crates/goose/tests/mcp_replays/npx-y@modelcontextprotocol_server-everything.results.json diff --git a/crates/goose/tests/mcp_integration_test.rs b/crates/goose/tests/mcp_integration_test.rs index 4fb7c8b84ecc..85731c368971 100644 --- a/crates/goose/tests/mcp_integration_test.rs +++ b/crates/goose/tests/mcp_integration_test.rs @@ -1,6 +1,8 @@ use std::env; +use std::fs::File; use std::path::PathBuf; +use rmcp::model::Content; use serde_json::json; use tokio_util::sync::CancellationToken; @@ -22,6 +24,9 @@ const REPLAY_BINARY: &str = "stdio_replayer"; vec!["npx", "-y", "@modelcontextprotocol/server-everything"], vec![ ToolCall::new("echo", json!({"message": "Hello, world!"})), + ToolCall::new("add", json!({"a": 1, "b": 2})), + ToolCall::new("longRunningOperation", json!({"duration": 1, "steps": 5})), + ToolCall::new("structuredContent", json!({"location": "11238"})), ] )] #[tokio::test] @@ -35,7 +40,7 @@ async fn test_replayed_session(command: Vec<&str>, tool_calls: Vec) { PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("should find the project root")); replay_file_path.push("tests"); replay_file_path.push("mcp_replays"); - replay_file_path.push(replay_file_name); + replay_file_path.push(&replay_file_name); let mode = if env::var("GOOSE_RECORD_MCP").is_ok() { TestMode::Record @@ -75,6 +80,7 @@ async fn test_replayed_session(command: Vec<&str>, tool_calls: Vec) { let result = extension_manager.add_extension(extension_config).await; assert!(result.is_ok(), "Failed to add extension: {:?}", result); + let mut results = Vec::new(); for tool_call in tool_calls { let tool_call = ToolCall::new(format!("test__{}", tool_call.name), tool_call.arguments); let result = extension_manager @@ -82,6 +88,25 @@ async fn test_replayed_session(command: Vec<&str>, tool_calls: Vec) { .await; let tool_result = result.expect("tool dispatch should succeed"); - tool_result.result.await.expect("should get a result"); + results.push(tool_result.result.await.expect("should get a result")); } + + let mut results_path = replay_file_path.clone(); + results_path.pop(); + results_path.push(format!("{}.results.json", &replay_file_name)); + + match mode { + TestMode::Record => serde_json::to_writer_pretty( + File::create(results_path).expect("could not reate results file"), + &results, + ) + .expect("could not write results"), + TestMode::Replay => assert_eq!( + serde_json::from_reader::<_, Vec>>( + File::open(results_path).expect("could not read results file") + ) + .expect("could not deserialize results"), + results + ), + }; } diff --git a/crates/goose/tests/mcp_replays/npx-y@modelcontextprotocol_server-everything b/crates/goose/tests/mcp_replays/npx-y@modelcontextprotocol_server-everything index 20fcf5b9f8aa..4829ad6cc94c 100644 --- a/crates/goose/tests/mcp_replays/npx-y@modelcontextprotocol_server-everything +++ b/crates/goose/tests/mcp_replays/npx-y@modelcontextprotocol_server-everything @@ -4,3 +4,14 @@ STDOUT: {"result":{"protocolVersion":"2025-03-26","capabilities":{"prompts":{}," STDIN: {"jsonrpc":"2.0","method":"notifications/initialized"} STDIN: {"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"_meta":{"progressToken":0},"name":"echo","arguments":{"message":"Hello, world!"}}} STDOUT: {"result":{"content":[{"type":"text","text":"Echo: Hello, world!"}]},"jsonrpc":"2.0","id":1} +STDIN: {"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"_meta":{"progressToken":1},"name":"add","arguments":{"a":1,"b":2}}} +STDOUT: {"result":{"content":[{"type":"text","text":"The sum of 1 and 2 is 3."}]},"jsonrpc":"2.0","id":2} +STDIN: {"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"_meta":{"progressToken":2},"name":"longRunningOperation","arguments":{"duration":1,"steps":5}}} +STDOUT: {"method":"notifications/progress","params":{"progress":1,"total":5,"progressToken":2},"jsonrpc":"2.0"} +STDOUT: {"method":"notifications/progress","params":{"progress":2,"total":5,"progressToken":2},"jsonrpc":"2.0"} +STDOUT: {"method":"notifications/progress","params":{"progress":3,"total":5,"progressToken":2},"jsonrpc":"2.0"} +STDOUT: {"method":"notifications/progress","params":{"progress":4,"total":5,"progressToken":2},"jsonrpc":"2.0"} +STDOUT: {"method":"notifications/progress","params":{"progress":5,"total":5,"progressToken":2},"jsonrpc":"2.0"} +STDOUT: {"result":{"content":[{"type":"text","text":"Long running operation completed. Duration: 1 seconds, Steps: 5."}]},"jsonrpc":"2.0","id":3} +STDIN: {"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"_meta":{"progressToken":3},"name":"structuredContent","arguments":{"location":"11238"}}} +STDOUT: {"result":{"content":[{"type":"text","text":"{\"temperature\":22.5,\"conditions\":\"Partly cloudy\",\"humidity\":65}"}],"structuredContent":{"temperature":22.5,"conditions":"Partly cloudy","humidity":65}},"jsonrpc":"2.0","id":4} diff --git a/crates/goose/tests/mcp_replays/npx-y@modelcontextprotocol_server-everything.results.json b/crates/goose/tests/mcp_replays/npx-y@modelcontextprotocol_server-everything.results.json new file mode 100644 index 000000000000..7d4a3b268c6a --- /dev/null +++ b/crates/goose/tests/mcp_replays/npx-y@modelcontextprotocol_server-everything.results.json @@ -0,0 +1,26 @@ +[ + [ + { + "type": "text", + "text": "Echo: Hello, world!" + } + ], + [ + { + "type": "text", + "text": "The sum of 1 and 2 is 3." + } + ], + [ + { + "type": "text", + "text": "Long running operation completed. Duration: 1 seconds, Steps: 5." + } + ], + [ + { + "type": "text", + "text": "{\"temperature\":22.5,\"conditions\":\"Partly cloudy\",\"humidity\":65}" + } + ] +] \ No newline at end of file From e7ab49cd7b1aa76cb5639fd09377700247c63533 Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Fri, 8 Aug 2025 14:25:30 -0400 Subject: [PATCH 10/17] Add a Github test, plus env var support --- crates/goose/tests/mcp_integration_test.rs | 34 +++++++++++++++++-- .../goose/tests/mcp_replays/github-mcp-server | 8 +++++ .../tests/mcp_replays/github-mcp-serverstdio | 6 ++++ .../github-mcp-serverstdio.results.json | 8 +++++ 4 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 crates/goose/tests/mcp_replays/github-mcp-server create mode 100644 crates/goose/tests/mcp_replays/github-mcp-serverstdio create mode 100644 crates/goose/tests/mcp_replays/github-mcp-serverstdio.results.json diff --git a/crates/goose/tests/mcp_integration_test.rs b/crates/goose/tests/mcp_integration_test.rs index 85731c368971..c718680a853f 100644 --- a/crates/goose/tests/mcp_integration_test.rs +++ b/crates/goose/tests/mcp_integration_test.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::env; use std::fs::File; use std::path::PathBuf; @@ -27,10 +28,22 @@ const REPLAY_BINARY: &str = "stdio_replayer"; ToolCall::new("add", json!({"a": 1, "b": 2})), ToolCall::new("longRunningOperation", json!({"duration": 1, "steps": 5})), ToolCall::new("structuredContent", json!({"location": "11238"})), - ] + ], + vec![] +)] +#[test_case( + vec!["github-mcp-server", "stdio"], + vec![ + ToolCall::new("get_pull_request", json!({"owner": "block", "repo": "goose", "pullNumber": 3939})), + ], + vec!["GITHUB_PERSONAL_ACCESS_TOKEN"] )] #[tokio::test] -async fn test_replayed_session(command: Vec<&str>, tool_calls: Vec) { +async fn test_replayed_session( + command: Vec<&str>, + tool_calls: Vec, + required_envs: Vec<&str>, +) { let replay_file_name = command .iter() .map(|s| s.replace("/", "_")) @@ -61,16 +74,31 @@ async fn test_replayed_session(command: Vec<&str>, tool_calls: Vec) { args.push(replay_file_path.to_string_lossy().to_string()); + let mut env = HashMap::new(); + if matches!(mode, TestMode::Record) { args.extend(command.into_iter().map(str::to_string)); + + for key in required_envs { + match env::var(key) { + Ok(v) => { + env.insert(key.to_string(), v); + } + Err(_) => { + eprintln!("skipping due to missing required env variable: {}", key); + return; + } + } + } } + let envs = Envs::new(env); let extension_config = ExtensionConfig::Stdio { name: "test".to_string(), description: Some("Test".to_string()), cmd, args, - envs: Envs::default(), + envs, env_keys: vec![], timeout: Some(30), bundled: Some(false), diff --git a/crates/goose/tests/mcp_replays/github-mcp-server b/crates/goose/tests/mcp_replays/github-mcp-server new file mode 100644 index 000000000000..e87bf1172ffb --- /dev/null +++ b/crates/goose/tests/mcp_replays/github-mcp-server @@ -0,0 +1,8 @@ +STDIN: {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"goose","version":"0.1.0"}}} +STDOUT: A GitHub MCP server that handles various tools and resources. +STDOUT: +STDOUT: Usage: +STDOUT: server [command] +STDOUT: +STDOUT: Available Commands: +STDOUT \ No newline at end of file diff --git a/crates/goose/tests/mcp_replays/github-mcp-serverstdio b/crates/goose/tests/mcp_replays/github-mcp-serverstdio new file mode 100644 index 000000000000..7b05a497ad7e --- /dev/null +++ b/crates/goose/tests/mcp_replays/github-mcp-serverstdio @@ -0,0 +1,6 @@ +STDIN: {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"goose","version":"0.1.0"}}} +STDERR: GitHub MCP Server running on stdio +STDOUT: {"jsonrpc":"2.0","id":0,"result":{"protocolVersion":"2025-03-26","capabilities":{"logging":{},"prompts":{},"resources":{"subscribe":true,"listChanged":true},"tools":{"listChanged":true}},"serverInfo":{"name":"github-mcp-server","version":"version"}}} +STDIN: {"jsonrpc":"2.0","method":"notifications/initialized"} +STDIN: {"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"_meta":{"progressToken":0},"name":"get_pull_request","arguments":{"owner":"block","pullNumber":3939,"repo":"goose"}}} +STDOUT: {"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"{\"id\":2729516918,\"number\":3939,\"state\":\"open\",\"locked\":false,\"title\":\"MCP session replay integration test\",\"body\":\"Lets you set up an MCP server (stdio only for now), some interactions (tool calling only for now) and record the whole thing to a session replay file. Then commit the replay and result files and the test will run the interaction through the extension manager.\\r\\n\\r\\nThis works by using two new binaries included in this PR: one that sits in front of a stdio server, writing all i/o to a replay file, and one that reads said replay file and mimics the behavior of the recorded session.\",\"created_at\":\"2025-08-08T04:01:56Z\",\"updated_at\":\"2025-08-08T15:58:02Z\",\"user\":{\"login\":\"jamadeo\",\"id\":5307860,\"node_id\":\"MDQ6VXNlcjUzMDc4NjA=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/5307860?v=4\",\"html_url\":\"https://github.com/jamadeo\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false,\"url\":\"https://api.github.com/users/jamadeo\",\"events_url\":\"https://api.github.com/users/jamadeo/events{/privacy}\",\"following_url\":\"https://api.github.com/users/jamadeo/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/jamadeo/followers\",\"gists_url\":\"https://api.github.com/users/jamadeo/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/jamadeo/orgs\",\"received_events_url\":\"https://api.github.com/users/jamadeo/received_events\",\"repos_url\":\"https://api.github.com/users/jamadeo/repos\",\"starred_url\":\"https://api.github.com/users/jamadeo/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/jamadeo/subscriptions\"},\"draft\":false,\"url\":\"https://api.github.com/repos/block/goose/pulls/3939\",\"html_url\":\"https://github.com/block/goose/pull/3939\",\"issue_url\":\"https://api.github.com/repos/block/goose/issues/3939\",\"statuses_url\":\"https://api.github.com/repos/block/goose/statuses/acd3adf44686d713ee0ba698cdd9c1662eddf080\",\"diff_url\":\"https://github.com/block/goose/pull/3939.diff\",\"patch_url\":\"https://github.com/block/goose/pull/3939.patch\",\"commits_url\":\"https://api.github.com/repos/block/goose/pulls/3939/commits\",\"comments_url\":\"https://api.github.com/repos/block/goose/issues/3939/comments\",\"review_comments_url\":\"https://api.github.com/repos/block/goose/pulls/3939/comments\",\"review_comment_url\":\"https://api.github.com/repos/block/goose/pulls/comments{/number}\",\"author_association\":\"MEMBER\",\"node_id\":\"PR_kwDOMneZ986isR92\",\"requested_reviewers\":[{\"login\":\"alexhancock\",\"id\":427516,\"node_id\":\"MDQ6VXNlcjQyNzUxNg==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/427516?v=4\",\"html_url\":\"https://github.com/alexhancock\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false,\"url\":\"https://api.github.com/users/alexhancock\",\"events_url\":\"https://api.github.com/users/alexhancock/events{/privacy}\",\"following_url\":\"https://api.github.com/users/alexhancock/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/alexhancock/followers\",\"gists_url\":\"https://api.github.com/users/alexhancock/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/alexhancock/orgs\",\"received_events_url\":\"https://api.github.com/users/alexhancock/received_events\",\"repos_url\":\"https://api.github.com/users/alexhancock/repos\",\"starred_url\":\"https://api.github.com/users/alexhancock/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/alexhancock/subscriptions\"},{\"login\":\"DOsinga\",\"id\":952558,\"node_id\":\"MDQ6VXNlcjk1MjU1OA==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/952558?v=4\",\"html_url\":\"https://github.com/DOsinga\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false,\"url\":\"https://api.github.com/users/DOsinga\",\"events_url\":\"https://api.github.com/users/DOsinga/events{/privacy}\",\"following_url\":\"https://api.github.com/users/DOsinga/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/DOsinga/followers\",\"gists_url\":\"https://api.github.com/users/DOsinga/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/DOsinga/orgs\",\"received_events_url\":\"https://api.github.com/users/DOsinga/received_events\",\"repos_url\":\"https://api.github.com/users/DOsinga/repos\",\"starred_url\":\"https://api.github.com/users/DOsinga/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/DOsinga/subscriptions\"}],\"merged\":false,\"mergeable\":true,\"mergeable_state\":\"blocked\",\"rebaseable\":true,\"merge_commit_sha\":\"a90c4d7dae18d9235a1fda84f6d3f647c61277df\",\"comments\":0,\"commits\":9,\"additions\":404,\"deletions\":0,\"changed_files\":8,\"maintainer_can_modify\":false,\"review_comments\":0,\"_links\":{\"self\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/3939\"},\"html\":{\"href\":\"https://github.com/block/goose/pull/3939\"},\"issue\":{\"href\":\"https://api.github.com/repos/block/goose/issues/3939\"},\"comments\":{\"href\":\"https://api.github.com/repos/block/goose/issues/3939/comments\"},\"review_comments\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/3939/comments\"},\"review_comment\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/comments{/number}\"},\"commits\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/3939/commits\"},\"statuses\":{\"href\":\"https://api.github.com/repos/block/goose/statuses/acd3adf44686d713ee0ba698cdd9c1662eddf080\"}},\"head\":{\"label\":\"block:jackamadeo/mcp-integration-test\",\"ref\":\"jackamadeo/mcp-integration-test\",\"sha\":\"acd3adf44686d713ee0ba698cdd9c1662eddf080\",\"repo\":{\"id\":846698999,\"node_id\":\"R_kgDOMneZ9w\",\"owner\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"},\"name\":\"goose\",\"full_name\":\"block/goose\",\"description\":\"an open source, extensible AI agent that goes beyond code suggestions - install, execute, edit, and test with any LLM\",\"homepage\":\"https://block.github.io/goose/\",\"default_branch\":\"main\",\"created_at\":\"2024-08-23T19:03:36Z\",\"pushed_at\":\"2025-08-08T18:20:36Z\",\"updated_at\":\"2025-08-08T17:37:14Z\",\"html_url\":\"https://github.com/block/goose\",\"clone_url\":\"https://github.com/block/goose.git\",\"git_url\":\"git://github.com/block/goose.git\",\"ssh_url\":\"git@github.com:block/goose.git\",\"svn_url\":\"https://github.com/block/goose\",\"language\":\"Rust\",\"fork\":false,\"forks_count\":1564,\"open_issues_count\":298,\"open_issues\":298,\"stargazers_count\":18401,\"watchers_count\":18401,\"watchers\":18401,\"size\":1325658,\"allow_forking\":true,\"web_commit_signoff_required\":false,\"archived\":false,\"disabled\":false,\"license\":{\"key\":\"apache-2.0\",\"name\":\"Apache License 2.0\",\"url\":\"https://api.github.com/licenses/apache-2.0\",\"spdx_id\":\"Apache-2.0\"},\"private\":false,\"has_issues\":true,\"has_wiki\":true,\"has_pages\":true,\"has_projects\":true,\"has_downloads\":true,\"has_discussions\":true,\"is_template\":false,\"url\":\"https://api.github.com/repos/block/goose\",\"archive_url\":\"https://api.github.com/repos/block/goose/{archive_format}{/ref}\",\"assignees_url\":\"https://api.github.com/repos/block/goose/assignees{/user}\",\"blobs_url\":\"https://api.github.com/repos/block/goose/git/blobs{/sha}\",\"branches_url\":\"https://api.github.com/repos/block/goose/branches{/branch}\",\"collaborators_url\":\"https://api.github.com/repos/block/goose/collaborators{/collaborator}\",\"comments_url\":\"https://api.github.com/repos/block/goose/comments{/number}\",\"commits_url\":\"https://api.github.com/repos/block/goose/commits{/sha}\",\"compare_url\":\"https://api.github.com/repos/block/goose/compare/{base}...{head}\",\"contents_url\":\"https://api.github.com/repos/block/goose/contents/{+path}\",\"contributors_url\":\"https://api.github.com/repos/block/goose/contributors\",\"deployments_url\":\"https://api.github.com/repos/block/goose/deployments\",\"downloads_url\":\"https://api.github.com/repos/block/goose/downloads\",\"events_url\":\"https://api.github.com/repos/block/goose/events\",\"forks_url\":\"https://api.github.com/repos/block/goose/forks\",\"git_commits_url\":\"https://api.github.com/repos/block/goose/git/commits{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/block/goose/git/refs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/block/goose/git/tags{/sha}\",\"hooks_url\":\"https://api.github.com/repos/block/goose/hooks\",\"issue_comment_url\":\"https://api.github.com/repos/block/goose/issues/comments{/number}\",\"issue_events_url\":\"https://api.github.com/repos/block/goose/issues/events{/number}\",\"issues_url\":\"https://api.github.com/repos/block/goose/issues{/number}\",\"keys_url\":\"https://api.github.com/repos/block/goose/keys{/key_id}\",\"labels_url\":\"https://api.github.com/repos/block/goose/labels{/name}\",\"languages_url\":\"https://api.github.com/repos/block/goose/languages\",\"merges_url\":\"https://api.github.com/repos/block/goose/merges\",\"milestones_url\":\"https://api.github.com/repos/block/goose/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/block/goose/notifications{?since,all,participating}\",\"pulls_url\":\"https://api.github.com/repos/block/goose/pulls{/number}\",\"releases_url\":\"https://api.github.com/repos/block/goose/releases{/id}\",\"stargazers_url\":\"https://api.github.com/repos/block/goose/stargazers\",\"statuses_url\":\"https://api.github.com/repos/block/goose/statuses/{sha}\",\"subscribers_url\":\"https://api.github.com/repos/block/goose/subscribers\",\"subscription_url\":\"https://api.github.com/repos/block/goose/subscription\",\"tags_url\":\"https://api.github.com/repos/block/goose/tags\",\"trees_url\":\"https://api.github.com/repos/block/goose/git/trees{/sha}\",\"teams_url\":\"https://api.github.com/repos/block/goose/teams\",\"visibility\":\"public\"},\"user\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"}},\"base\":{\"label\":\"block:main\",\"ref\":\"main\",\"sha\":\"b38aa1668a0b594e95a8e517265dae7c69397ace\",\"repo\":{\"id\":846698999,\"node_id\":\"R_kgDOMneZ9w\",\"owner\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"},\"name\":\"goose\",\"full_name\":\"block/goose\",\"description\":\"an open source, extensible AI agent that goes beyond code suggestions - install, execute, edit, and test with any LLM\",\"homepage\":\"https://block.github.io/goose/\",\"default_branch\":\"main\",\"created_at\":\"2024-08-23T19:03:36Z\",\"pushed_at\":\"2025-08-08T18:20:36Z\",\"updated_at\":\"2025-08-08T17:37:14Z\",\"html_url\":\"https://github.com/block/goose\",\"clone_url\":\"https://github.com/block/goose.git\",\"git_url\":\"git://github.com/block/goose.git\",\"ssh_url\":\"git@github.com:block/goose.git\",\"svn_url\":\"https://github.com/block/goose\",\"language\":\"Rust\",\"fork\":false,\"forks_count\":1564,\"open_issues_count\":298,\"open_issues\":298,\"stargazers_count\":18401,\"watchers_count\":18401,\"watchers\":18401,\"size\":1325658,\"allow_forking\":true,\"web_commit_signoff_required\":false,\"archived\":false,\"disabled\":false,\"license\":{\"key\":\"apache-2.0\",\"name\":\"Apache License 2.0\",\"url\":\"https://api.github.com/licenses/apache-2.0\",\"spdx_id\":\"Apache-2.0\"},\"private\":false,\"has_issues\":true,\"has_wiki\":true,\"has_pages\":true,\"has_projects\":true,\"has_downloads\":true,\"has_discussions\":true,\"is_template\":false,\"url\":\"https://api.github.com/repos/block/goose\",\"archive_url\":\"https://api.github.com/repos/block/goose/{archive_format}{/ref}\",\"assignees_url\":\"https://api.github.com/repos/block/goose/assignees{/user}\",\"blobs_url\":\"https://api.github.com/repos/block/goose/git/blobs{/sha}\",\"branches_url\":\"https://api.github.com/repos/block/goose/branches{/branch}\",\"collaborators_url\":\"https://api.github.com/repos/block/goose/collaborators{/collaborator}\",\"comments_url\":\"https://api.github.com/repos/block/goose/comments{/number}\",\"commits_url\":\"https://api.github.com/repos/block/goose/commits{/sha}\",\"compare_url\":\"https://api.github.com/repos/block/goose/compare/{base}...{head}\",\"contents_url\":\"https://api.github.com/repos/block/goose/contents/{+path}\",\"contributors_url\":\"https://api.github.com/repos/block/goose/contributors\",\"deployments_url\":\"https://api.github.com/repos/block/goose/deployments\",\"downloads_url\":\"https://api.github.com/repos/block/goose/downloads\",\"events_url\":\"https://api.github.com/repos/block/goose/events\",\"forks_url\":\"https://api.github.com/repos/block/goose/forks\",\"git_commits_url\":\"https://api.github.com/repos/block/goose/git/commits{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/block/goose/git/refs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/block/goose/git/tags{/sha}\",\"hooks_url\":\"https://api.github.com/repos/block/goose/hooks\",\"issue_comment_url\":\"https://api.github.com/repos/block/goose/issues/comments{/number}\",\"issue_events_url\":\"https://api.github.com/repos/block/goose/issues/events{/number}\",\"issues_url\":\"https://api.github.com/repos/block/goose/issues{/number}\",\"keys_url\":\"https://api.github.com/repos/block/goose/keys{/key_id}\",\"labels_url\":\"https://api.github.com/repos/block/goose/labels{/name}\",\"languages_url\":\"https://api.github.com/repos/block/goose/languages\",\"merges_url\":\"https://api.github.com/repos/block/goose/merges\",\"milestones_url\":\"https://api.github.com/repos/block/goose/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/block/goose/notifications{?since,all,participating}\",\"pulls_url\":\"https://api.github.com/repos/block/goose/pulls{/number}\",\"releases_url\":\"https://api.github.com/repos/block/goose/releases{/id}\",\"stargazers_url\":\"https://api.github.com/repos/block/goose/stargazers\",\"statuses_url\":\"https://api.github.com/repos/block/goose/statuses/{sha}\",\"subscribers_url\":\"https://api.github.com/repos/block/goose/subscribers\",\"subscription_url\":\"https://api.github.com/repos/block/goose/subscription\",\"tags_url\":\"https://api.github.com/repos/block/goose/tags\",\"trees_url\":\"https://api.github.com/repos/block/goose/git/trees{/sha}\",\"teams_url\":\"https://api.github.com/repos/block/goose/teams\",\"visibility\":\"public\"},\"user\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"}}}"}]}} diff --git a/crates/goose/tests/mcp_replays/github-mcp-serverstdio.results.json b/crates/goose/tests/mcp_replays/github-mcp-serverstdio.results.json new file mode 100644 index 000000000000..089c4c50fcfa --- /dev/null +++ b/crates/goose/tests/mcp_replays/github-mcp-serverstdio.results.json @@ -0,0 +1,8 @@ +[ + [ + { + "type": "text", + "text": "{\"id\":2729516918,\"number\":3939,\"state\":\"open\",\"locked\":false,\"title\":\"MCP session replay integration test\",\"body\":\"Lets you set up an MCP server (stdio only for now), some interactions (tool calling only for now) and record the whole thing to a session replay file. Then commit the replay and result files and the test will run the interaction through the extension manager.\\r\\n\\r\\nThis works by using two new binaries included in this PR: one that sits in front of a stdio server, writing all i/o to a replay file, and one that reads said replay file and mimics the behavior of the recorded session.\",\"created_at\":\"2025-08-08T04:01:56Z\",\"updated_at\":\"2025-08-08T15:58:02Z\",\"user\":{\"login\":\"jamadeo\",\"id\":5307860,\"node_id\":\"MDQ6VXNlcjUzMDc4NjA=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/5307860?v=4\",\"html_url\":\"https://github.com/jamadeo\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false,\"url\":\"https://api.github.com/users/jamadeo\",\"events_url\":\"https://api.github.com/users/jamadeo/events{/privacy}\",\"following_url\":\"https://api.github.com/users/jamadeo/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/jamadeo/followers\",\"gists_url\":\"https://api.github.com/users/jamadeo/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/jamadeo/orgs\",\"received_events_url\":\"https://api.github.com/users/jamadeo/received_events\",\"repos_url\":\"https://api.github.com/users/jamadeo/repos\",\"starred_url\":\"https://api.github.com/users/jamadeo/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/jamadeo/subscriptions\"},\"draft\":false,\"url\":\"https://api.github.com/repos/block/goose/pulls/3939\",\"html_url\":\"https://github.com/block/goose/pull/3939\",\"issue_url\":\"https://api.github.com/repos/block/goose/issues/3939\",\"statuses_url\":\"https://api.github.com/repos/block/goose/statuses/acd3adf44686d713ee0ba698cdd9c1662eddf080\",\"diff_url\":\"https://github.com/block/goose/pull/3939.diff\",\"patch_url\":\"https://github.com/block/goose/pull/3939.patch\",\"commits_url\":\"https://api.github.com/repos/block/goose/pulls/3939/commits\",\"comments_url\":\"https://api.github.com/repos/block/goose/issues/3939/comments\",\"review_comments_url\":\"https://api.github.com/repos/block/goose/pulls/3939/comments\",\"review_comment_url\":\"https://api.github.com/repos/block/goose/pulls/comments{/number}\",\"author_association\":\"MEMBER\",\"node_id\":\"PR_kwDOMneZ986isR92\",\"requested_reviewers\":[{\"login\":\"alexhancock\",\"id\":427516,\"node_id\":\"MDQ6VXNlcjQyNzUxNg==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/427516?v=4\",\"html_url\":\"https://github.com/alexhancock\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false,\"url\":\"https://api.github.com/users/alexhancock\",\"events_url\":\"https://api.github.com/users/alexhancock/events{/privacy}\",\"following_url\":\"https://api.github.com/users/alexhancock/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/alexhancock/followers\",\"gists_url\":\"https://api.github.com/users/alexhancock/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/alexhancock/orgs\",\"received_events_url\":\"https://api.github.com/users/alexhancock/received_events\",\"repos_url\":\"https://api.github.com/users/alexhancock/repos\",\"starred_url\":\"https://api.github.com/users/alexhancock/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/alexhancock/subscriptions\"},{\"login\":\"DOsinga\",\"id\":952558,\"node_id\":\"MDQ6VXNlcjk1MjU1OA==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/952558?v=4\",\"html_url\":\"https://github.com/DOsinga\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false,\"url\":\"https://api.github.com/users/DOsinga\",\"events_url\":\"https://api.github.com/users/DOsinga/events{/privacy}\",\"following_url\":\"https://api.github.com/users/DOsinga/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/DOsinga/followers\",\"gists_url\":\"https://api.github.com/users/DOsinga/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/DOsinga/orgs\",\"received_events_url\":\"https://api.github.com/users/DOsinga/received_events\",\"repos_url\":\"https://api.github.com/users/DOsinga/repos\",\"starred_url\":\"https://api.github.com/users/DOsinga/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/DOsinga/subscriptions\"}],\"merged\":false,\"mergeable\":true,\"mergeable_state\":\"blocked\",\"rebaseable\":true,\"merge_commit_sha\":\"a90c4d7dae18d9235a1fda84f6d3f647c61277df\",\"comments\":0,\"commits\":9,\"additions\":404,\"deletions\":0,\"changed_files\":8,\"maintainer_can_modify\":false,\"review_comments\":0,\"_links\":{\"self\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/3939\"},\"html\":{\"href\":\"https://github.com/block/goose/pull/3939\"},\"issue\":{\"href\":\"https://api.github.com/repos/block/goose/issues/3939\"},\"comments\":{\"href\":\"https://api.github.com/repos/block/goose/issues/3939/comments\"},\"review_comments\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/3939/comments\"},\"review_comment\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/comments{/number}\"},\"commits\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/3939/commits\"},\"statuses\":{\"href\":\"https://api.github.com/repos/block/goose/statuses/acd3adf44686d713ee0ba698cdd9c1662eddf080\"}},\"head\":{\"label\":\"block:jackamadeo/mcp-integration-test\",\"ref\":\"jackamadeo/mcp-integration-test\",\"sha\":\"acd3adf44686d713ee0ba698cdd9c1662eddf080\",\"repo\":{\"id\":846698999,\"node_id\":\"R_kgDOMneZ9w\",\"owner\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"},\"name\":\"goose\",\"full_name\":\"block/goose\",\"description\":\"an open source, extensible AI agent that goes beyond code suggestions - install, execute, edit, and test with any LLM\",\"homepage\":\"https://block.github.io/goose/\",\"default_branch\":\"main\",\"created_at\":\"2024-08-23T19:03:36Z\",\"pushed_at\":\"2025-08-08T18:20:36Z\",\"updated_at\":\"2025-08-08T17:37:14Z\",\"html_url\":\"https://github.com/block/goose\",\"clone_url\":\"https://github.com/block/goose.git\",\"git_url\":\"git://github.com/block/goose.git\",\"ssh_url\":\"git@github.com:block/goose.git\",\"svn_url\":\"https://github.com/block/goose\",\"language\":\"Rust\",\"fork\":false,\"forks_count\":1564,\"open_issues_count\":298,\"open_issues\":298,\"stargazers_count\":18401,\"watchers_count\":18401,\"watchers\":18401,\"size\":1325658,\"allow_forking\":true,\"web_commit_signoff_required\":false,\"archived\":false,\"disabled\":false,\"license\":{\"key\":\"apache-2.0\",\"name\":\"Apache License 2.0\",\"url\":\"https://api.github.com/licenses/apache-2.0\",\"spdx_id\":\"Apache-2.0\"},\"private\":false,\"has_issues\":true,\"has_wiki\":true,\"has_pages\":true,\"has_projects\":true,\"has_downloads\":true,\"has_discussions\":true,\"is_template\":false,\"url\":\"https://api.github.com/repos/block/goose\",\"archive_url\":\"https://api.github.com/repos/block/goose/{archive_format}{/ref}\",\"assignees_url\":\"https://api.github.com/repos/block/goose/assignees{/user}\",\"blobs_url\":\"https://api.github.com/repos/block/goose/git/blobs{/sha}\",\"branches_url\":\"https://api.github.com/repos/block/goose/branches{/branch}\",\"collaborators_url\":\"https://api.github.com/repos/block/goose/collaborators{/collaborator}\",\"comments_url\":\"https://api.github.com/repos/block/goose/comments{/number}\",\"commits_url\":\"https://api.github.com/repos/block/goose/commits{/sha}\",\"compare_url\":\"https://api.github.com/repos/block/goose/compare/{base}...{head}\",\"contents_url\":\"https://api.github.com/repos/block/goose/contents/{+path}\",\"contributors_url\":\"https://api.github.com/repos/block/goose/contributors\",\"deployments_url\":\"https://api.github.com/repos/block/goose/deployments\",\"downloads_url\":\"https://api.github.com/repos/block/goose/downloads\",\"events_url\":\"https://api.github.com/repos/block/goose/events\",\"forks_url\":\"https://api.github.com/repos/block/goose/forks\",\"git_commits_url\":\"https://api.github.com/repos/block/goose/git/commits{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/block/goose/git/refs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/block/goose/git/tags{/sha}\",\"hooks_url\":\"https://api.github.com/repos/block/goose/hooks\",\"issue_comment_url\":\"https://api.github.com/repos/block/goose/issues/comments{/number}\",\"issue_events_url\":\"https://api.github.com/repos/block/goose/issues/events{/number}\",\"issues_url\":\"https://api.github.com/repos/block/goose/issues{/number}\",\"keys_url\":\"https://api.github.com/repos/block/goose/keys{/key_id}\",\"labels_url\":\"https://api.github.com/repos/block/goose/labels{/name}\",\"languages_url\":\"https://api.github.com/repos/block/goose/languages\",\"merges_url\":\"https://api.github.com/repos/block/goose/merges\",\"milestones_url\":\"https://api.github.com/repos/block/goose/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/block/goose/notifications{?since,all,participating}\",\"pulls_url\":\"https://api.github.com/repos/block/goose/pulls{/number}\",\"releases_url\":\"https://api.github.com/repos/block/goose/releases{/id}\",\"stargazers_url\":\"https://api.github.com/repos/block/goose/stargazers\",\"statuses_url\":\"https://api.github.com/repos/block/goose/statuses/{sha}\",\"subscribers_url\":\"https://api.github.com/repos/block/goose/subscribers\",\"subscription_url\":\"https://api.github.com/repos/block/goose/subscription\",\"tags_url\":\"https://api.github.com/repos/block/goose/tags\",\"trees_url\":\"https://api.github.com/repos/block/goose/git/trees{/sha}\",\"teams_url\":\"https://api.github.com/repos/block/goose/teams\",\"visibility\":\"public\"},\"user\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"}},\"base\":{\"label\":\"block:main\",\"ref\":\"main\",\"sha\":\"b38aa1668a0b594e95a8e517265dae7c69397ace\",\"repo\":{\"id\":846698999,\"node_id\":\"R_kgDOMneZ9w\",\"owner\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"},\"name\":\"goose\",\"full_name\":\"block/goose\",\"description\":\"an open source, extensible AI agent that goes beyond code suggestions - install, execute, edit, and test with any LLM\",\"homepage\":\"https://block.github.io/goose/\",\"default_branch\":\"main\",\"created_at\":\"2024-08-23T19:03:36Z\",\"pushed_at\":\"2025-08-08T18:20:36Z\",\"updated_at\":\"2025-08-08T17:37:14Z\",\"html_url\":\"https://github.com/block/goose\",\"clone_url\":\"https://github.com/block/goose.git\",\"git_url\":\"git://github.com/block/goose.git\",\"ssh_url\":\"git@github.com:block/goose.git\",\"svn_url\":\"https://github.com/block/goose\",\"language\":\"Rust\",\"fork\":false,\"forks_count\":1564,\"open_issues_count\":298,\"open_issues\":298,\"stargazers_count\":18401,\"watchers_count\":18401,\"watchers\":18401,\"size\":1325658,\"allow_forking\":true,\"web_commit_signoff_required\":false,\"archived\":false,\"disabled\":false,\"license\":{\"key\":\"apache-2.0\",\"name\":\"Apache License 2.0\",\"url\":\"https://api.github.com/licenses/apache-2.0\",\"spdx_id\":\"Apache-2.0\"},\"private\":false,\"has_issues\":true,\"has_wiki\":true,\"has_pages\":true,\"has_projects\":true,\"has_downloads\":true,\"has_discussions\":true,\"is_template\":false,\"url\":\"https://api.github.com/repos/block/goose\",\"archive_url\":\"https://api.github.com/repos/block/goose/{archive_format}{/ref}\",\"assignees_url\":\"https://api.github.com/repos/block/goose/assignees{/user}\",\"blobs_url\":\"https://api.github.com/repos/block/goose/git/blobs{/sha}\",\"branches_url\":\"https://api.github.com/repos/block/goose/branches{/branch}\",\"collaborators_url\":\"https://api.github.com/repos/block/goose/collaborators{/collaborator}\",\"comments_url\":\"https://api.github.com/repos/block/goose/comments{/number}\",\"commits_url\":\"https://api.github.com/repos/block/goose/commits{/sha}\",\"compare_url\":\"https://api.github.com/repos/block/goose/compare/{base}...{head}\",\"contents_url\":\"https://api.github.com/repos/block/goose/contents/{+path}\",\"contributors_url\":\"https://api.github.com/repos/block/goose/contributors\",\"deployments_url\":\"https://api.github.com/repos/block/goose/deployments\",\"downloads_url\":\"https://api.github.com/repos/block/goose/downloads\",\"events_url\":\"https://api.github.com/repos/block/goose/events\",\"forks_url\":\"https://api.github.com/repos/block/goose/forks\",\"git_commits_url\":\"https://api.github.com/repos/block/goose/git/commits{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/block/goose/git/refs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/block/goose/git/tags{/sha}\",\"hooks_url\":\"https://api.github.com/repos/block/goose/hooks\",\"issue_comment_url\":\"https://api.github.com/repos/block/goose/issues/comments{/number}\",\"issue_events_url\":\"https://api.github.com/repos/block/goose/issues/events{/number}\",\"issues_url\":\"https://api.github.com/repos/block/goose/issues{/number}\",\"keys_url\":\"https://api.github.com/repos/block/goose/keys{/key_id}\",\"labels_url\":\"https://api.github.com/repos/block/goose/labels{/name}\",\"languages_url\":\"https://api.github.com/repos/block/goose/languages\",\"merges_url\":\"https://api.github.com/repos/block/goose/merges\",\"milestones_url\":\"https://api.github.com/repos/block/goose/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/block/goose/notifications{?since,all,participating}\",\"pulls_url\":\"https://api.github.com/repos/block/goose/pulls{/number}\",\"releases_url\":\"https://api.github.com/repos/block/goose/releases{/id}\",\"stargazers_url\":\"https://api.github.com/repos/block/goose/stargazers\",\"statuses_url\":\"https://api.github.com/repos/block/goose/statuses/{sha}\",\"subscribers_url\":\"https://api.github.com/repos/block/goose/subscribers\",\"subscription_url\":\"https://api.github.com/repos/block/goose/subscription\",\"tags_url\":\"https://api.github.com/repos/block/goose/tags\",\"trees_url\":\"https://api.github.com/repos/block/goose/git/trees{/sha}\",\"teams_url\":\"https://api.github.com/repos/block/goose/teams\",\"visibility\":\"public\"},\"user\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"}}}" + } + ] +] \ No newline at end of file From 9d094cd3954e5c4a8bc06653a353e7434439a885 Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Fri, 8 Aug 2025 14:25:38 -0400 Subject: [PATCH 11/17] Add a just recipe --- Justfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Justfile b/Justfile index e956a1133d47..d5f32e85d962 100644 --- a/Justfile +++ b/Justfile @@ -458,3 +458,9 @@ win-total-rls *allparam: just win-bld-rls{{allparam}} just win-run-rls +build-test-tools: + cargo build -p goose-test + +record-mcp-tests: build-test-tools + GOOSE_RECORD_MCP=1 cargo test --package goose --test mcp_integration_test + git add crates/goose/tests/mcp_replays/ \ No newline at end of file From 32ad62aa477a568d6f0ca3f1cf31214ea845fc0c Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Fri, 8 Aug 2025 14:30:08 -0400 Subject: [PATCH 12/17] Get some content that won't change --- crates/goose/tests/mcp_integration_test.rs | 7 ++++++- crates/goose/tests/mcp_replays/github-mcp-serverstdio | 4 ++-- .../mcp_replays/github-mcp-serverstdio.results.json | 10 +++++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/goose/tests/mcp_integration_test.rs b/crates/goose/tests/mcp_integration_test.rs index c718680a853f..8c1604fed6d9 100644 --- a/crates/goose/tests/mcp_integration_test.rs +++ b/crates/goose/tests/mcp_integration_test.rs @@ -34,7 +34,12 @@ const REPLAY_BINARY: &str = "stdio_replayer"; #[test_case( vec!["github-mcp-server", "stdio"], vec![ - ToolCall::new("get_pull_request", json!({"owner": "block", "repo": "goose", "pullNumber": 3939})), + ToolCall::new("get_file_contents", json!({ + "owner": "block", + "repo": "goose", + "path": "README.md", + "sha": "48c1ec8afdb7d4d5b4f6e67e623926c884034776" + })), ], vec!["GITHUB_PERSONAL_ACCESS_TOKEN"] )] diff --git a/crates/goose/tests/mcp_replays/github-mcp-serverstdio b/crates/goose/tests/mcp_replays/github-mcp-serverstdio index 7b05a497ad7e..a0cfa3e7933c 100644 --- a/crates/goose/tests/mcp_replays/github-mcp-serverstdio +++ b/crates/goose/tests/mcp_replays/github-mcp-serverstdio @@ -2,5 +2,5 @@ STDIN: {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"protocolVersion" STDERR: GitHub MCP Server running on stdio STDOUT: {"jsonrpc":"2.0","id":0,"result":{"protocolVersion":"2025-03-26","capabilities":{"logging":{},"prompts":{},"resources":{"subscribe":true,"listChanged":true},"tools":{"listChanged":true}},"serverInfo":{"name":"github-mcp-server","version":"version"}}} STDIN: {"jsonrpc":"2.0","method":"notifications/initialized"} -STDIN: {"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"_meta":{"progressToken":0},"name":"get_pull_request","arguments":{"owner":"block","pullNumber":3939,"repo":"goose"}}} -STDOUT: {"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"{\"id\":2729516918,\"number\":3939,\"state\":\"open\",\"locked\":false,\"title\":\"MCP session replay integration test\",\"body\":\"Lets you set up an MCP server (stdio only for now), some interactions (tool calling only for now) and record the whole thing to a session replay file. Then commit the replay and result files and the test will run the interaction through the extension manager.\\r\\n\\r\\nThis works by using two new binaries included in this PR: one that sits in front of a stdio server, writing all i/o to a replay file, and one that reads said replay file and mimics the behavior of the recorded session.\",\"created_at\":\"2025-08-08T04:01:56Z\",\"updated_at\":\"2025-08-08T15:58:02Z\",\"user\":{\"login\":\"jamadeo\",\"id\":5307860,\"node_id\":\"MDQ6VXNlcjUzMDc4NjA=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/5307860?v=4\",\"html_url\":\"https://github.com/jamadeo\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false,\"url\":\"https://api.github.com/users/jamadeo\",\"events_url\":\"https://api.github.com/users/jamadeo/events{/privacy}\",\"following_url\":\"https://api.github.com/users/jamadeo/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/jamadeo/followers\",\"gists_url\":\"https://api.github.com/users/jamadeo/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/jamadeo/orgs\",\"received_events_url\":\"https://api.github.com/users/jamadeo/received_events\",\"repos_url\":\"https://api.github.com/users/jamadeo/repos\",\"starred_url\":\"https://api.github.com/users/jamadeo/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/jamadeo/subscriptions\"},\"draft\":false,\"url\":\"https://api.github.com/repos/block/goose/pulls/3939\",\"html_url\":\"https://github.com/block/goose/pull/3939\",\"issue_url\":\"https://api.github.com/repos/block/goose/issues/3939\",\"statuses_url\":\"https://api.github.com/repos/block/goose/statuses/acd3adf44686d713ee0ba698cdd9c1662eddf080\",\"diff_url\":\"https://github.com/block/goose/pull/3939.diff\",\"patch_url\":\"https://github.com/block/goose/pull/3939.patch\",\"commits_url\":\"https://api.github.com/repos/block/goose/pulls/3939/commits\",\"comments_url\":\"https://api.github.com/repos/block/goose/issues/3939/comments\",\"review_comments_url\":\"https://api.github.com/repos/block/goose/pulls/3939/comments\",\"review_comment_url\":\"https://api.github.com/repos/block/goose/pulls/comments{/number}\",\"author_association\":\"MEMBER\",\"node_id\":\"PR_kwDOMneZ986isR92\",\"requested_reviewers\":[{\"login\":\"alexhancock\",\"id\":427516,\"node_id\":\"MDQ6VXNlcjQyNzUxNg==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/427516?v=4\",\"html_url\":\"https://github.com/alexhancock\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false,\"url\":\"https://api.github.com/users/alexhancock\",\"events_url\":\"https://api.github.com/users/alexhancock/events{/privacy}\",\"following_url\":\"https://api.github.com/users/alexhancock/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/alexhancock/followers\",\"gists_url\":\"https://api.github.com/users/alexhancock/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/alexhancock/orgs\",\"received_events_url\":\"https://api.github.com/users/alexhancock/received_events\",\"repos_url\":\"https://api.github.com/users/alexhancock/repos\",\"starred_url\":\"https://api.github.com/users/alexhancock/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/alexhancock/subscriptions\"},{\"login\":\"DOsinga\",\"id\":952558,\"node_id\":\"MDQ6VXNlcjk1MjU1OA==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/952558?v=4\",\"html_url\":\"https://github.com/DOsinga\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false,\"url\":\"https://api.github.com/users/DOsinga\",\"events_url\":\"https://api.github.com/users/DOsinga/events{/privacy}\",\"following_url\":\"https://api.github.com/users/DOsinga/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/DOsinga/followers\",\"gists_url\":\"https://api.github.com/users/DOsinga/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/DOsinga/orgs\",\"received_events_url\":\"https://api.github.com/users/DOsinga/received_events\",\"repos_url\":\"https://api.github.com/users/DOsinga/repos\",\"starred_url\":\"https://api.github.com/users/DOsinga/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/DOsinga/subscriptions\"}],\"merged\":false,\"mergeable\":true,\"mergeable_state\":\"blocked\",\"rebaseable\":true,\"merge_commit_sha\":\"a90c4d7dae18d9235a1fda84f6d3f647c61277df\",\"comments\":0,\"commits\":9,\"additions\":404,\"deletions\":0,\"changed_files\":8,\"maintainer_can_modify\":false,\"review_comments\":0,\"_links\":{\"self\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/3939\"},\"html\":{\"href\":\"https://github.com/block/goose/pull/3939\"},\"issue\":{\"href\":\"https://api.github.com/repos/block/goose/issues/3939\"},\"comments\":{\"href\":\"https://api.github.com/repos/block/goose/issues/3939/comments\"},\"review_comments\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/3939/comments\"},\"review_comment\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/comments{/number}\"},\"commits\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/3939/commits\"},\"statuses\":{\"href\":\"https://api.github.com/repos/block/goose/statuses/acd3adf44686d713ee0ba698cdd9c1662eddf080\"}},\"head\":{\"label\":\"block:jackamadeo/mcp-integration-test\",\"ref\":\"jackamadeo/mcp-integration-test\",\"sha\":\"acd3adf44686d713ee0ba698cdd9c1662eddf080\",\"repo\":{\"id\":846698999,\"node_id\":\"R_kgDOMneZ9w\",\"owner\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"},\"name\":\"goose\",\"full_name\":\"block/goose\",\"description\":\"an open source, extensible AI agent that goes beyond code suggestions - install, execute, edit, and test with any LLM\",\"homepage\":\"https://block.github.io/goose/\",\"default_branch\":\"main\",\"created_at\":\"2024-08-23T19:03:36Z\",\"pushed_at\":\"2025-08-08T18:20:36Z\",\"updated_at\":\"2025-08-08T17:37:14Z\",\"html_url\":\"https://github.com/block/goose\",\"clone_url\":\"https://github.com/block/goose.git\",\"git_url\":\"git://github.com/block/goose.git\",\"ssh_url\":\"git@github.com:block/goose.git\",\"svn_url\":\"https://github.com/block/goose\",\"language\":\"Rust\",\"fork\":false,\"forks_count\":1564,\"open_issues_count\":298,\"open_issues\":298,\"stargazers_count\":18401,\"watchers_count\":18401,\"watchers\":18401,\"size\":1325658,\"allow_forking\":true,\"web_commit_signoff_required\":false,\"archived\":false,\"disabled\":false,\"license\":{\"key\":\"apache-2.0\",\"name\":\"Apache License 2.0\",\"url\":\"https://api.github.com/licenses/apache-2.0\",\"spdx_id\":\"Apache-2.0\"},\"private\":false,\"has_issues\":true,\"has_wiki\":true,\"has_pages\":true,\"has_projects\":true,\"has_downloads\":true,\"has_discussions\":true,\"is_template\":false,\"url\":\"https://api.github.com/repos/block/goose\",\"archive_url\":\"https://api.github.com/repos/block/goose/{archive_format}{/ref}\",\"assignees_url\":\"https://api.github.com/repos/block/goose/assignees{/user}\",\"blobs_url\":\"https://api.github.com/repos/block/goose/git/blobs{/sha}\",\"branches_url\":\"https://api.github.com/repos/block/goose/branches{/branch}\",\"collaborators_url\":\"https://api.github.com/repos/block/goose/collaborators{/collaborator}\",\"comments_url\":\"https://api.github.com/repos/block/goose/comments{/number}\",\"commits_url\":\"https://api.github.com/repos/block/goose/commits{/sha}\",\"compare_url\":\"https://api.github.com/repos/block/goose/compare/{base}...{head}\",\"contents_url\":\"https://api.github.com/repos/block/goose/contents/{+path}\",\"contributors_url\":\"https://api.github.com/repos/block/goose/contributors\",\"deployments_url\":\"https://api.github.com/repos/block/goose/deployments\",\"downloads_url\":\"https://api.github.com/repos/block/goose/downloads\",\"events_url\":\"https://api.github.com/repos/block/goose/events\",\"forks_url\":\"https://api.github.com/repos/block/goose/forks\",\"git_commits_url\":\"https://api.github.com/repos/block/goose/git/commits{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/block/goose/git/refs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/block/goose/git/tags{/sha}\",\"hooks_url\":\"https://api.github.com/repos/block/goose/hooks\",\"issue_comment_url\":\"https://api.github.com/repos/block/goose/issues/comments{/number}\",\"issue_events_url\":\"https://api.github.com/repos/block/goose/issues/events{/number}\",\"issues_url\":\"https://api.github.com/repos/block/goose/issues{/number}\",\"keys_url\":\"https://api.github.com/repos/block/goose/keys{/key_id}\",\"labels_url\":\"https://api.github.com/repos/block/goose/labels{/name}\",\"languages_url\":\"https://api.github.com/repos/block/goose/languages\",\"merges_url\":\"https://api.github.com/repos/block/goose/merges\",\"milestones_url\":\"https://api.github.com/repos/block/goose/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/block/goose/notifications{?since,all,participating}\",\"pulls_url\":\"https://api.github.com/repos/block/goose/pulls{/number}\",\"releases_url\":\"https://api.github.com/repos/block/goose/releases{/id}\",\"stargazers_url\":\"https://api.github.com/repos/block/goose/stargazers\",\"statuses_url\":\"https://api.github.com/repos/block/goose/statuses/{sha}\",\"subscribers_url\":\"https://api.github.com/repos/block/goose/subscribers\",\"subscription_url\":\"https://api.github.com/repos/block/goose/subscription\",\"tags_url\":\"https://api.github.com/repos/block/goose/tags\",\"trees_url\":\"https://api.github.com/repos/block/goose/git/trees{/sha}\",\"teams_url\":\"https://api.github.com/repos/block/goose/teams\",\"visibility\":\"public\"},\"user\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"}},\"base\":{\"label\":\"block:main\",\"ref\":\"main\",\"sha\":\"b38aa1668a0b594e95a8e517265dae7c69397ace\",\"repo\":{\"id\":846698999,\"node_id\":\"R_kgDOMneZ9w\",\"owner\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"},\"name\":\"goose\",\"full_name\":\"block/goose\",\"description\":\"an open source, extensible AI agent that goes beyond code suggestions - install, execute, edit, and test with any LLM\",\"homepage\":\"https://block.github.io/goose/\",\"default_branch\":\"main\",\"created_at\":\"2024-08-23T19:03:36Z\",\"pushed_at\":\"2025-08-08T18:20:36Z\",\"updated_at\":\"2025-08-08T17:37:14Z\",\"html_url\":\"https://github.com/block/goose\",\"clone_url\":\"https://github.com/block/goose.git\",\"git_url\":\"git://github.com/block/goose.git\",\"ssh_url\":\"git@github.com:block/goose.git\",\"svn_url\":\"https://github.com/block/goose\",\"language\":\"Rust\",\"fork\":false,\"forks_count\":1564,\"open_issues_count\":298,\"open_issues\":298,\"stargazers_count\":18401,\"watchers_count\":18401,\"watchers\":18401,\"size\":1325658,\"allow_forking\":true,\"web_commit_signoff_required\":false,\"archived\":false,\"disabled\":false,\"license\":{\"key\":\"apache-2.0\",\"name\":\"Apache License 2.0\",\"url\":\"https://api.github.com/licenses/apache-2.0\",\"spdx_id\":\"Apache-2.0\"},\"private\":false,\"has_issues\":true,\"has_wiki\":true,\"has_pages\":true,\"has_projects\":true,\"has_downloads\":true,\"has_discussions\":true,\"is_template\":false,\"url\":\"https://api.github.com/repos/block/goose\",\"archive_url\":\"https://api.github.com/repos/block/goose/{archive_format}{/ref}\",\"assignees_url\":\"https://api.github.com/repos/block/goose/assignees{/user}\",\"blobs_url\":\"https://api.github.com/repos/block/goose/git/blobs{/sha}\",\"branches_url\":\"https://api.github.com/repos/block/goose/branches{/branch}\",\"collaborators_url\":\"https://api.github.com/repos/block/goose/collaborators{/collaborator}\",\"comments_url\":\"https://api.github.com/repos/block/goose/comments{/number}\",\"commits_url\":\"https://api.github.com/repos/block/goose/commits{/sha}\",\"compare_url\":\"https://api.github.com/repos/block/goose/compare/{base}...{head}\",\"contents_url\":\"https://api.github.com/repos/block/goose/contents/{+path}\",\"contributors_url\":\"https://api.github.com/repos/block/goose/contributors\",\"deployments_url\":\"https://api.github.com/repos/block/goose/deployments\",\"downloads_url\":\"https://api.github.com/repos/block/goose/downloads\",\"events_url\":\"https://api.github.com/repos/block/goose/events\",\"forks_url\":\"https://api.github.com/repos/block/goose/forks\",\"git_commits_url\":\"https://api.github.com/repos/block/goose/git/commits{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/block/goose/git/refs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/block/goose/git/tags{/sha}\",\"hooks_url\":\"https://api.github.com/repos/block/goose/hooks\",\"issue_comment_url\":\"https://api.github.com/repos/block/goose/issues/comments{/number}\",\"issue_events_url\":\"https://api.github.com/repos/block/goose/issues/events{/number}\",\"issues_url\":\"https://api.github.com/repos/block/goose/issues{/number}\",\"keys_url\":\"https://api.github.com/repos/block/goose/keys{/key_id}\",\"labels_url\":\"https://api.github.com/repos/block/goose/labels{/name}\",\"languages_url\":\"https://api.github.com/repos/block/goose/languages\",\"merges_url\":\"https://api.github.com/repos/block/goose/merges\",\"milestones_url\":\"https://api.github.com/repos/block/goose/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/block/goose/notifications{?since,all,participating}\",\"pulls_url\":\"https://api.github.com/repos/block/goose/pulls{/number}\",\"releases_url\":\"https://api.github.com/repos/block/goose/releases{/id}\",\"stargazers_url\":\"https://api.github.com/repos/block/goose/stargazers\",\"statuses_url\":\"https://api.github.com/repos/block/goose/statuses/{sha}\",\"subscribers_url\":\"https://api.github.com/repos/block/goose/subscribers\",\"subscription_url\":\"https://api.github.com/repos/block/goose/subscription\",\"tags_url\":\"https://api.github.com/repos/block/goose/tags\",\"trees_url\":\"https://api.github.com/repos/block/goose/git/trees{/sha}\",\"teams_url\":\"https://api.github.com/repos/block/goose/teams\",\"visibility\":\"public\"},\"user\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"}}}"}]}} +STDIN: {"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"_meta":{"progressToken":0},"name":"get_file_contents","arguments":{"owner":"block","path":"README.md","repo":"goose","sha":"48c1ec8afdb7d4d5b4f6e67e623926c884034776"}}} +STDOUT: {"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"successfully downloaded text file"},{"type":"resource","resource":{"uri":"repo://block/goose/sha/48c1ec8afdb7d4d5b4f6e67e623926c884034776/contents/README.md","mimeType":"text/plain; charset=utf-8","text":"\u003cdiv align=\"center\"\u003e\n\n# codename goose\n\n_a local, extensible, open source AI agent that automates engineering tasks_\n\n\u003cp align=\"center\"\u003e\n \u003ca href=\"https://opensource.org/licenses/Apache-2.0\"\u003e\n \u003cimg src=\"https://img.shields.io/badge/License-Apache_2.0-blue.svg\"\u003e\n \u003c/a\u003e\n \u003ca href=\"https://discord.gg/7GaTvbDwga\"\u003e\n \u003cimg src=\"https://img.shields.io/discord/1287729918100246654?logo=discord\u0026logoColor=white\u0026label=Join+Us\u0026color=blueviolet\" alt=\"Discord\"\u003e\n \u003c/a\u003e\n \u003ca href=\"https://github.com/block/goose/actions/workflows/ci.yml\"\u003e\n \u003cimg src=\"https://img.shields.io/github/actions/workflow/status/block/goose/ci.yml?branch=main\" alt=\"CI\"\u003e\n \u003c/a\u003e\n\u003c/p\u003e\n\u003c/div\u003e\n\ngoose is your on-machine AI agent, capable of automating complex development tasks from start to finish. More than just code suggestions, goose can build entire projects from scratch, write and execute code, debug failures, orchestrate workflows, and interact with external APIs - _autonomously_.\n\nWhether you're prototyping an idea, refining existing code, or managing intricate engineering pipelines, goose adapts to your workflow and executes tasks with precision.\n\nDesigned for maximum flexibility, goose works with any LLM and supports multi-model configuration to optimize performance and cost, seamlessly integrates with MCP servers, and is available as both a desktop app as well as CLI - making it the ultimate AI assistant for developers who want to move faster and focus on innovation.\n\n# Quick Links\n- [Quickstart](https://block.github.io/goose/docs/quickstart)\n- [Installation](https://block.github.io/goose/docs/getting-started/installation)\n- [Tutorials](https://block.github.io/goose/docs/category/tutorials)\n- [Documentation](https://block.github.io/goose/docs/category/getting-started)\n\n\n# Goose Around with Us\n- [Discord](https://discord.gg/block-opensource)\n- [YouTube](https://www.youtube.com/@blockopensource)\n- [LinkedIn](https://www.linkedin.com/company/block-opensource)\n- [Twitter/X](https://x.com/blockopensource)\n- [Bluesky](https://bsky.app/profile/opensource.block.xyz)\n- [Nostr](https://njump.me/opensource@block.xyz)\n"}}]}} diff --git a/crates/goose/tests/mcp_replays/github-mcp-serverstdio.results.json b/crates/goose/tests/mcp_replays/github-mcp-serverstdio.results.json index 089c4c50fcfa..268ac74a2250 100644 --- a/crates/goose/tests/mcp_replays/github-mcp-serverstdio.results.json +++ b/crates/goose/tests/mcp_replays/github-mcp-serverstdio.results.json @@ -2,7 +2,15 @@ [ { "type": "text", - "text": "{\"id\":2729516918,\"number\":3939,\"state\":\"open\",\"locked\":false,\"title\":\"MCP session replay integration test\",\"body\":\"Lets you set up an MCP server (stdio only for now), some interactions (tool calling only for now) and record the whole thing to a session replay file. Then commit the replay and result files and the test will run the interaction through the extension manager.\\r\\n\\r\\nThis works by using two new binaries included in this PR: one that sits in front of a stdio server, writing all i/o to a replay file, and one that reads said replay file and mimics the behavior of the recorded session.\",\"created_at\":\"2025-08-08T04:01:56Z\",\"updated_at\":\"2025-08-08T15:58:02Z\",\"user\":{\"login\":\"jamadeo\",\"id\":5307860,\"node_id\":\"MDQ6VXNlcjUzMDc4NjA=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/5307860?v=4\",\"html_url\":\"https://github.com/jamadeo\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false,\"url\":\"https://api.github.com/users/jamadeo\",\"events_url\":\"https://api.github.com/users/jamadeo/events{/privacy}\",\"following_url\":\"https://api.github.com/users/jamadeo/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/jamadeo/followers\",\"gists_url\":\"https://api.github.com/users/jamadeo/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/jamadeo/orgs\",\"received_events_url\":\"https://api.github.com/users/jamadeo/received_events\",\"repos_url\":\"https://api.github.com/users/jamadeo/repos\",\"starred_url\":\"https://api.github.com/users/jamadeo/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/jamadeo/subscriptions\"},\"draft\":false,\"url\":\"https://api.github.com/repos/block/goose/pulls/3939\",\"html_url\":\"https://github.com/block/goose/pull/3939\",\"issue_url\":\"https://api.github.com/repos/block/goose/issues/3939\",\"statuses_url\":\"https://api.github.com/repos/block/goose/statuses/acd3adf44686d713ee0ba698cdd9c1662eddf080\",\"diff_url\":\"https://github.com/block/goose/pull/3939.diff\",\"patch_url\":\"https://github.com/block/goose/pull/3939.patch\",\"commits_url\":\"https://api.github.com/repos/block/goose/pulls/3939/commits\",\"comments_url\":\"https://api.github.com/repos/block/goose/issues/3939/comments\",\"review_comments_url\":\"https://api.github.com/repos/block/goose/pulls/3939/comments\",\"review_comment_url\":\"https://api.github.com/repos/block/goose/pulls/comments{/number}\",\"author_association\":\"MEMBER\",\"node_id\":\"PR_kwDOMneZ986isR92\",\"requested_reviewers\":[{\"login\":\"alexhancock\",\"id\":427516,\"node_id\":\"MDQ6VXNlcjQyNzUxNg==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/427516?v=4\",\"html_url\":\"https://github.com/alexhancock\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false,\"url\":\"https://api.github.com/users/alexhancock\",\"events_url\":\"https://api.github.com/users/alexhancock/events{/privacy}\",\"following_url\":\"https://api.github.com/users/alexhancock/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/alexhancock/followers\",\"gists_url\":\"https://api.github.com/users/alexhancock/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/alexhancock/orgs\",\"received_events_url\":\"https://api.github.com/users/alexhancock/received_events\",\"repos_url\":\"https://api.github.com/users/alexhancock/repos\",\"starred_url\":\"https://api.github.com/users/alexhancock/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/alexhancock/subscriptions\"},{\"login\":\"DOsinga\",\"id\":952558,\"node_id\":\"MDQ6VXNlcjk1MjU1OA==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/952558?v=4\",\"html_url\":\"https://github.com/DOsinga\",\"gravatar_id\":\"\",\"type\":\"User\",\"site_admin\":false,\"url\":\"https://api.github.com/users/DOsinga\",\"events_url\":\"https://api.github.com/users/DOsinga/events{/privacy}\",\"following_url\":\"https://api.github.com/users/DOsinga/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/DOsinga/followers\",\"gists_url\":\"https://api.github.com/users/DOsinga/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/DOsinga/orgs\",\"received_events_url\":\"https://api.github.com/users/DOsinga/received_events\",\"repos_url\":\"https://api.github.com/users/DOsinga/repos\",\"starred_url\":\"https://api.github.com/users/DOsinga/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/DOsinga/subscriptions\"}],\"merged\":false,\"mergeable\":true,\"mergeable_state\":\"blocked\",\"rebaseable\":true,\"merge_commit_sha\":\"a90c4d7dae18d9235a1fda84f6d3f647c61277df\",\"comments\":0,\"commits\":9,\"additions\":404,\"deletions\":0,\"changed_files\":8,\"maintainer_can_modify\":false,\"review_comments\":0,\"_links\":{\"self\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/3939\"},\"html\":{\"href\":\"https://github.com/block/goose/pull/3939\"},\"issue\":{\"href\":\"https://api.github.com/repos/block/goose/issues/3939\"},\"comments\":{\"href\":\"https://api.github.com/repos/block/goose/issues/3939/comments\"},\"review_comments\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/3939/comments\"},\"review_comment\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/comments{/number}\"},\"commits\":{\"href\":\"https://api.github.com/repos/block/goose/pulls/3939/commits\"},\"statuses\":{\"href\":\"https://api.github.com/repos/block/goose/statuses/acd3adf44686d713ee0ba698cdd9c1662eddf080\"}},\"head\":{\"label\":\"block:jackamadeo/mcp-integration-test\",\"ref\":\"jackamadeo/mcp-integration-test\",\"sha\":\"acd3adf44686d713ee0ba698cdd9c1662eddf080\",\"repo\":{\"id\":846698999,\"node_id\":\"R_kgDOMneZ9w\",\"owner\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"},\"name\":\"goose\",\"full_name\":\"block/goose\",\"description\":\"an open source, extensible AI agent that goes beyond code suggestions - install, execute, edit, and test with any LLM\",\"homepage\":\"https://block.github.io/goose/\",\"default_branch\":\"main\",\"created_at\":\"2024-08-23T19:03:36Z\",\"pushed_at\":\"2025-08-08T18:20:36Z\",\"updated_at\":\"2025-08-08T17:37:14Z\",\"html_url\":\"https://github.com/block/goose\",\"clone_url\":\"https://github.com/block/goose.git\",\"git_url\":\"git://github.com/block/goose.git\",\"ssh_url\":\"git@github.com:block/goose.git\",\"svn_url\":\"https://github.com/block/goose\",\"language\":\"Rust\",\"fork\":false,\"forks_count\":1564,\"open_issues_count\":298,\"open_issues\":298,\"stargazers_count\":18401,\"watchers_count\":18401,\"watchers\":18401,\"size\":1325658,\"allow_forking\":true,\"web_commit_signoff_required\":false,\"archived\":false,\"disabled\":false,\"license\":{\"key\":\"apache-2.0\",\"name\":\"Apache License 2.0\",\"url\":\"https://api.github.com/licenses/apache-2.0\",\"spdx_id\":\"Apache-2.0\"},\"private\":false,\"has_issues\":true,\"has_wiki\":true,\"has_pages\":true,\"has_projects\":true,\"has_downloads\":true,\"has_discussions\":true,\"is_template\":false,\"url\":\"https://api.github.com/repos/block/goose\",\"archive_url\":\"https://api.github.com/repos/block/goose/{archive_format}{/ref}\",\"assignees_url\":\"https://api.github.com/repos/block/goose/assignees{/user}\",\"blobs_url\":\"https://api.github.com/repos/block/goose/git/blobs{/sha}\",\"branches_url\":\"https://api.github.com/repos/block/goose/branches{/branch}\",\"collaborators_url\":\"https://api.github.com/repos/block/goose/collaborators{/collaborator}\",\"comments_url\":\"https://api.github.com/repos/block/goose/comments{/number}\",\"commits_url\":\"https://api.github.com/repos/block/goose/commits{/sha}\",\"compare_url\":\"https://api.github.com/repos/block/goose/compare/{base}...{head}\",\"contents_url\":\"https://api.github.com/repos/block/goose/contents/{+path}\",\"contributors_url\":\"https://api.github.com/repos/block/goose/contributors\",\"deployments_url\":\"https://api.github.com/repos/block/goose/deployments\",\"downloads_url\":\"https://api.github.com/repos/block/goose/downloads\",\"events_url\":\"https://api.github.com/repos/block/goose/events\",\"forks_url\":\"https://api.github.com/repos/block/goose/forks\",\"git_commits_url\":\"https://api.github.com/repos/block/goose/git/commits{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/block/goose/git/refs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/block/goose/git/tags{/sha}\",\"hooks_url\":\"https://api.github.com/repos/block/goose/hooks\",\"issue_comment_url\":\"https://api.github.com/repos/block/goose/issues/comments{/number}\",\"issue_events_url\":\"https://api.github.com/repos/block/goose/issues/events{/number}\",\"issues_url\":\"https://api.github.com/repos/block/goose/issues{/number}\",\"keys_url\":\"https://api.github.com/repos/block/goose/keys{/key_id}\",\"labels_url\":\"https://api.github.com/repos/block/goose/labels{/name}\",\"languages_url\":\"https://api.github.com/repos/block/goose/languages\",\"merges_url\":\"https://api.github.com/repos/block/goose/merges\",\"milestones_url\":\"https://api.github.com/repos/block/goose/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/block/goose/notifications{?since,all,participating}\",\"pulls_url\":\"https://api.github.com/repos/block/goose/pulls{/number}\",\"releases_url\":\"https://api.github.com/repos/block/goose/releases{/id}\",\"stargazers_url\":\"https://api.github.com/repos/block/goose/stargazers\",\"statuses_url\":\"https://api.github.com/repos/block/goose/statuses/{sha}\",\"subscribers_url\":\"https://api.github.com/repos/block/goose/subscribers\",\"subscription_url\":\"https://api.github.com/repos/block/goose/subscription\",\"tags_url\":\"https://api.github.com/repos/block/goose/tags\",\"trees_url\":\"https://api.github.com/repos/block/goose/git/trees{/sha}\",\"teams_url\":\"https://api.github.com/repos/block/goose/teams\",\"visibility\":\"public\"},\"user\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"}},\"base\":{\"label\":\"block:main\",\"ref\":\"main\",\"sha\":\"b38aa1668a0b594e95a8e517265dae7c69397ace\",\"repo\":{\"id\":846698999,\"node_id\":\"R_kgDOMneZ9w\",\"owner\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"},\"name\":\"goose\",\"full_name\":\"block/goose\",\"description\":\"an open source, extensible AI agent that goes beyond code suggestions - install, execute, edit, and test with any LLM\",\"homepage\":\"https://block.github.io/goose/\",\"default_branch\":\"main\",\"created_at\":\"2024-08-23T19:03:36Z\",\"pushed_at\":\"2025-08-08T18:20:36Z\",\"updated_at\":\"2025-08-08T17:37:14Z\",\"html_url\":\"https://github.com/block/goose\",\"clone_url\":\"https://github.com/block/goose.git\",\"git_url\":\"git://github.com/block/goose.git\",\"ssh_url\":\"git@github.com:block/goose.git\",\"svn_url\":\"https://github.com/block/goose\",\"language\":\"Rust\",\"fork\":false,\"forks_count\":1564,\"open_issues_count\":298,\"open_issues\":298,\"stargazers_count\":18401,\"watchers_count\":18401,\"watchers\":18401,\"size\":1325658,\"allow_forking\":true,\"web_commit_signoff_required\":false,\"archived\":false,\"disabled\":false,\"license\":{\"key\":\"apache-2.0\",\"name\":\"Apache License 2.0\",\"url\":\"https://api.github.com/licenses/apache-2.0\",\"spdx_id\":\"Apache-2.0\"},\"private\":false,\"has_issues\":true,\"has_wiki\":true,\"has_pages\":true,\"has_projects\":true,\"has_downloads\":true,\"has_discussions\":true,\"is_template\":false,\"url\":\"https://api.github.com/repos/block/goose\",\"archive_url\":\"https://api.github.com/repos/block/goose/{archive_format}{/ref}\",\"assignees_url\":\"https://api.github.com/repos/block/goose/assignees{/user}\",\"blobs_url\":\"https://api.github.com/repos/block/goose/git/blobs{/sha}\",\"branches_url\":\"https://api.github.com/repos/block/goose/branches{/branch}\",\"collaborators_url\":\"https://api.github.com/repos/block/goose/collaborators{/collaborator}\",\"comments_url\":\"https://api.github.com/repos/block/goose/comments{/number}\",\"commits_url\":\"https://api.github.com/repos/block/goose/commits{/sha}\",\"compare_url\":\"https://api.github.com/repos/block/goose/compare/{base}...{head}\",\"contents_url\":\"https://api.github.com/repos/block/goose/contents/{+path}\",\"contributors_url\":\"https://api.github.com/repos/block/goose/contributors\",\"deployments_url\":\"https://api.github.com/repos/block/goose/deployments\",\"downloads_url\":\"https://api.github.com/repos/block/goose/downloads\",\"events_url\":\"https://api.github.com/repos/block/goose/events\",\"forks_url\":\"https://api.github.com/repos/block/goose/forks\",\"git_commits_url\":\"https://api.github.com/repos/block/goose/git/commits{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/block/goose/git/refs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/block/goose/git/tags{/sha}\",\"hooks_url\":\"https://api.github.com/repos/block/goose/hooks\",\"issue_comment_url\":\"https://api.github.com/repos/block/goose/issues/comments{/number}\",\"issue_events_url\":\"https://api.github.com/repos/block/goose/issues/events{/number}\",\"issues_url\":\"https://api.github.com/repos/block/goose/issues{/number}\",\"keys_url\":\"https://api.github.com/repos/block/goose/keys{/key_id}\",\"labels_url\":\"https://api.github.com/repos/block/goose/labels{/name}\",\"languages_url\":\"https://api.github.com/repos/block/goose/languages\",\"merges_url\":\"https://api.github.com/repos/block/goose/merges\",\"milestones_url\":\"https://api.github.com/repos/block/goose/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/block/goose/notifications{?since,all,participating}\",\"pulls_url\":\"https://api.github.com/repos/block/goose/pulls{/number}\",\"releases_url\":\"https://api.github.com/repos/block/goose/releases{/id}\",\"stargazers_url\":\"https://api.github.com/repos/block/goose/stargazers\",\"statuses_url\":\"https://api.github.com/repos/block/goose/statuses/{sha}\",\"subscribers_url\":\"https://api.github.com/repos/block/goose/subscribers\",\"subscription_url\":\"https://api.github.com/repos/block/goose/subscription\",\"tags_url\":\"https://api.github.com/repos/block/goose/tags\",\"trees_url\":\"https://api.github.com/repos/block/goose/git/trees{/sha}\",\"teams_url\":\"https://api.github.com/repos/block/goose/teams\",\"visibility\":\"public\"},\"user\":{\"login\":\"block\",\"id\":185116535,\"node_id\":\"O_kgDOCwindw\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/185116535?v=4\",\"html_url\":\"https://github.com/block\",\"gravatar_id\":\"\",\"type\":\"Organization\",\"site_admin\":false,\"url\":\"https://api.github.com/users/block\",\"events_url\":\"https://api.github.com/users/block/events{/privacy}\",\"following_url\":\"https://api.github.com/users/block/following{/other_user}\",\"followers_url\":\"https://api.github.com/users/block/followers\",\"gists_url\":\"https://api.github.com/users/block/gists{/gist_id}\",\"organizations_url\":\"https://api.github.com/users/block/orgs\",\"received_events_url\":\"https://api.github.com/users/block/received_events\",\"repos_url\":\"https://api.github.com/users/block/repos\",\"starred_url\":\"https://api.github.com/users/block/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/block/subscriptions\"}}}" + "text": "successfully downloaded text file" + }, + { + "type": "resource", + "resource": { + "uri": "repo://block/goose/sha/48c1ec8afdb7d4d5b4f6e67e623926c884034776/contents/README.md", + "mimeType": "text/plain; charset=utf-8", + "text": "
\n\n# codename goose\n\n_a local, extensible, open source AI agent that automates engineering tasks_\n\n

\n \n \n \n \n \"Discord\"\n \n \n \"CI\"\n \n

\n
\n\ngoose is your on-machine AI agent, capable of automating complex development tasks from start to finish. More than just code suggestions, goose can build entire projects from scratch, write and execute code, debug failures, orchestrate workflows, and interact with external APIs - _autonomously_.\n\nWhether you're prototyping an idea, refining existing code, or managing intricate engineering pipelines, goose adapts to your workflow and executes tasks with precision.\n\nDesigned for maximum flexibility, goose works with any LLM and supports multi-model configuration to optimize performance and cost, seamlessly integrates with MCP servers, and is available as both a desktop app as well as CLI - making it the ultimate AI assistant for developers who want to move faster and focus on innovation.\n\n# Quick Links\n- [Quickstart](https://block.github.io/goose/docs/quickstart)\n- [Installation](https://block.github.io/goose/docs/getting-started/installation)\n- [Tutorials](https://block.github.io/goose/docs/category/tutorials)\n- [Documentation](https://block.github.io/goose/docs/category/getting-started)\n\n\n# Goose Around with Us\n- [Discord](https://discord.gg/block-opensource)\n- [YouTube](https://www.youtube.com/@blockopensource)\n- [LinkedIn](https://www.linkedin.com/company/block-opensource)\n- [Twitter/X](https://x.com/blockopensource)\n- [Bluesky](https://bsky.app/profile/opensource.block.xyz)\n- [Nostr](https://njump.me/opensource@block.xyz)\n" + } } ] ] \ No newline at end of file From 8aa3a87a42248008b16aadb51b6bd900445d0606 Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Fri, 8 Aug 2025 15:26:54 -0400 Subject: [PATCH 13/17] Print the mismatch --- .gitignore | 3 + crates/goose-test/src/bin/stdio_replayer.rs | 8 ++- crates/goose/tests/mcp_integration_test.rs | 67 +++++++++++-------- .../goose/tests/mcp_replays/github-mcp-server | 8 --- 4 files changed, 49 insertions(+), 37 deletions(-) delete mode 100644 crates/goose/tests/mcp_replays/github-mcp-server diff --git a/.gitignore b/.gitignore index f9edb850c007..2696337838b1 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,6 @@ do_not_version/ /.env /working_dir + +# Error log artifacts from mcp replay tests +crates/goose/tests/mcp_replays/*errors.txt \ No newline at end of file diff --git a/crates/goose-test/src/bin/stdio_replayer.rs b/crates/goose-test/src/bin/stdio_replayer.rs index 8f45c218f5ab..6c05e97da68b 100644 --- a/crates/goose-test/src/bin/stdio_replayer.rs +++ b/crates/goose-test/src/bin/stdio_replayer.rs @@ -1,3 +1,4 @@ +use core::error; use std::env; use std::fs::File; use std::io::{self, BufRead, BufReader, Write}; @@ -60,6 +61,7 @@ fn main() -> io::Result<()> { let log_file_path = &args[1]; let entries = load_log_file(log_file_path)?; + let errors_file = File::create(format!("{}.errors.txt", log_file_path))?; let stdin = io::stdin(); let mut stdout = io::stdout(); @@ -82,7 +84,11 @@ fn main() -> io::Result<()> { input = input.trim_end_matches('\n').to_string(); if input != entry.content { - eprintln!("Expected: '{}', got: '{}'", entry.content, input); + writeln!( + &errors_file, + "Expected:\n\t'{}'\n\ngot:\n\t'{}'", + entry.content, input + )?; process::exit(1); } } diff --git a/crates/goose/tests/mcp_integration_test.rs b/crates/goose/tests/mcp_integration_test.rs index 8c1604fed6d9..8f08d4ad1763 100644 --- a/crates/goose/tests/mcp_integration_test.rs +++ b/crates/goose/tests/mcp_integration_test.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; -use std::env; use std::fs::File; use std::path::PathBuf; +use std::{env, fs}; use rmcp::model::Content; use serde_json::json; @@ -110,36 +110,47 @@ async fn test_replayed_session( }; let mut extension_manager = ExtensionManager::new(); + let result = extension_manager.add_extension(extension_config).await; assert!(result.is_ok(), "Failed to add extension: {:?}", result); - let mut results = Vec::new(); - for tool_call in tool_calls { - let tool_call = ToolCall::new(format!("test__{}", tool_call.name), tool_call.arguments); - let result = extension_manager - .dispatch_tool_call(tool_call, CancellationToken::default()) - .await; + let result = (async || -> Result<(), Box> { + let mut results = Vec::new(); + for tool_call in tool_calls { + let tool_call = ToolCall::new(format!("test__{}", tool_call.name), tool_call.arguments); + let result = extension_manager + .dispatch_tool_call(tool_call, CancellationToken::default()) + .await; - let tool_result = result.expect("tool dispatch should succeed"); - results.push(tool_result.result.await.expect("should get a result")); - } + let tool_result = result?; + results.push(tool_result.result.await?); + } - let mut results_path = replay_file_path.clone(); - results_path.pop(); - results_path.push(format!("{}.results.json", &replay_file_name)); - - match mode { - TestMode::Record => serde_json::to_writer_pretty( - File::create(results_path).expect("could not reate results file"), - &results, - ) - .expect("could not write results"), - TestMode::Replay => assert_eq!( - serde_json::from_reader::<_, Vec>>( - File::open(results_path).expect("could not read results file") - ) - .expect("could not deserialize results"), - results - ), - }; + let mut results_path = replay_file_path.clone(); + results_path.pop(); + results_path.push(format!("{}.results.json", &replay_file_name)); + + match mode { + TestMode::Record => { + serde_json::to_writer_pretty(File::create(results_path)?, &results)? + } + TestMode::Replay => assert_eq!( + serde_json::from_reader::<_, Vec>>(File::open(results_path)?)?, + results + ), + }; + + Ok(()) + })() + .await; + + if let Err(err) = result { + let errors = + fs::read_to_string(format!("{}.errors.txt", replay_file_path.to_string_lossy())) + .expect("could not read errors"); + eprintln!("errors from {}", replay_file_path.to_string_lossy()); + eprintln!("{}", errors); + eprintln!(); + panic!("Test failed: {:?}", err); + } } diff --git a/crates/goose/tests/mcp_replays/github-mcp-server b/crates/goose/tests/mcp_replays/github-mcp-server deleted file mode 100644 index e87bf1172ffb..000000000000 --- a/crates/goose/tests/mcp_replays/github-mcp-server +++ /dev/null @@ -1,8 +0,0 @@ -STDIN: {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"goose","version":"0.1.0"}}} -STDOUT: A GitHub MCP server that handles various tools and resources. -STDOUT: -STDOUT: Usage: -STDOUT: server [command] -STDOUT: -STDOUT: Available Commands: -STDOUT \ No newline at end of file From e5ca8e6bf6c33c223908e0b492f76dfb145d0873 Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Fri, 8 Aug 2025 15:27:36 -0400 Subject: [PATCH 14/17] unused import --- crates/goose-test/src/bin/stdio_replayer.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/goose-test/src/bin/stdio_replayer.rs b/crates/goose-test/src/bin/stdio_replayer.rs index 6c05e97da68b..d785594bd325 100644 --- a/crates/goose-test/src/bin/stdio_replayer.rs +++ b/crates/goose-test/src/bin/stdio_replayer.rs @@ -1,4 +1,3 @@ -use core::error; use std::env; use std::fs::File; use std::io::{self, BufRead, BufReader, Write}; From 5c99dd63817734b16d4d4ccb974667899f6622c2 Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Fri, 8 Aug 2025 17:50:33 -0400 Subject: [PATCH 15/17] Compare the objects --- Cargo.lock | 7 +++++-- crates/goose-test/Cargo.toml | 5 ++++- crates/goose-test/src/bin/stdio_replayer.rs | 11 ++++++++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a432099cc224..cfa964a0521d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3518,6 +3518,9 @@ dependencies = [ [[package]] name = "goose-test" version = "1.1.0" +dependencies = [ + "serde_json", +] [[package]] name = "grep-cli" @@ -7453,9 +7456,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.140" +version = "1.0.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" dependencies = [ "indexmap 2.7.1", "itoa", diff --git a/crates/goose-test/Cargo.toml b/crates/goose-test/Cargo.toml index 45d1855abf8b..f13751a349b0 100644 --- a/crates/goose-test/Cargo.toml +++ b/crates/goose-test/Cargo.toml @@ -16,4 +16,7 @@ path = "src/bin/stdio_logger.rs" [[bin]] name = "stdio_replayer" -path = "src/bin/stdio_replayer.rs" \ No newline at end of file +path = "src/bin/stdio_replayer.rs" + +[dependencies] +serde_json = "1.0.142" diff --git a/crates/goose-test/src/bin/stdio_replayer.rs b/crates/goose-test/src/bin/stdio_replayer.rs index d785594bd325..eda40176d60c 100644 --- a/crates/goose-test/src/bin/stdio_replayer.rs +++ b/crates/goose-test/src/bin/stdio_replayer.rs @@ -3,6 +3,8 @@ use std::fs::File; use std::io::{self, BufRead, BufReader, Write}; use std::process; +use serde_json::Value; + #[derive(Debug, Clone)] enum StreamType { Stdin, @@ -82,11 +84,14 @@ fn main() -> io::Result<()> { stdin.read_line(&mut input)?; input = input.trim_end_matches('\n').to_string(); - if input != entry.content { + let input_value: Value = serde_json::from_str::(&input)?; + let entry_value: Value = serde_json::from_str::(&entry.content)?; + if input_value != entry_value { writeln!( &errors_file, - "Expected:\n\t'{}'\n\ngot:\n\t'{}'", - entry.content, input + "expected:\n{}\ngot:\n{}", + serde_json::to_string(&input_value)?, + serde_json::to_string(&entry_value)? )?; process::exit(1); } From e7f79b380d57862f5039c49a5e31f0fd77907389 Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Fri, 8 Aug 2025 18:50:34 -0400 Subject: [PATCH 16/17] Something with uvx --- crates/goose/tests/mcp_integration_test.rs | 9 +++++++++ crates/goose/tests/mcp_replays/uvxmcp-server-fetch | 5 +++++ .../tests/mcp_replays/uvxmcp-server-fetch.results.json | 8 ++++++++ 3 files changed, 22 insertions(+) create mode 100644 crates/goose/tests/mcp_replays/uvxmcp-server-fetch create mode 100644 crates/goose/tests/mcp_replays/uvxmcp-server-fetch.results.json diff --git a/crates/goose/tests/mcp_integration_test.rs b/crates/goose/tests/mcp_integration_test.rs index 8f08d4ad1763..4a0687f00282 100644 --- a/crates/goose/tests/mcp_integration_test.rs +++ b/crates/goose/tests/mcp_integration_test.rs @@ -43,6 +43,15 @@ const REPLAY_BINARY: &str = "stdio_replayer"; ], vec!["GITHUB_PERSONAL_ACCESS_TOKEN"] )] +#[test_case( + vec!["uvx", "mcp-server-fetch"], + vec![ + ToolCall::new("fetch", json!({ + "url": "https://example.com", + })), + ], + vec![] +)] #[tokio::test] async fn test_replayed_session( command: Vec<&str>, diff --git a/crates/goose/tests/mcp_replays/uvxmcp-server-fetch b/crates/goose/tests/mcp_replays/uvxmcp-server-fetch new file mode 100644 index 000000000000..6faab7145677 --- /dev/null +++ b/crates/goose/tests/mcp_replays/uvxmcp-server-fetch @@ -0,0 +1,5 @@ +STDIN: {"jsonrpc":"2.0","id":0,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"goose","version":"0.1.0"}}} +STDOUT: {"jsonrpc":"2.0","id":0,"result":{"protocolVersion":"2025-03-26","capabilities":{"experimental":{},"prompts":{"listChanged":false},"tools":{"listChanged":false}},"serverInfo":{"name":"mcp-fetch","version":"1.12.4"}}} +STDIN: {"jsonrpc":"2.0","method":"notifications/initialized"} +STDIN: {"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"_meta":{"progressToken":0},"name":"fetch","arguments":{"url":"https://example.com"}}} +STDOUT: {"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"Contents of https://example.com/:\nThis domain is for use in illustrative examples in documents. You may use this\ndomain in literature without prior coordination or asking for permission.\n\n[More information...](https://www.iana.org/domains/example)"}],"isError":false}} diff --git a/crates/goose/tests/mcp_replays/uvxmcp-server-fetch.results.json b/crates/goose/tests/mcp_replays/uvxmcp-server-fetch.results.json new file mode 100644 index 000000000000..09ad07d3b91a --- /dev/null +++ b/crates/goose/tests/mcp_replays/uvxmcp-server-fetch.results.json @@ -0,0 +1,8 @@ +[ + [ + { + "type": "text", + "text": "Contents of https://example.com/:\nThis domain is for use in illustrative examples in documents. You may use this\ndomain in literature without prior coordination or asking for permission.\n\n[More information...](https://www.iana.org/domains/example)" + } + ] +] \ No newline at end of file From 9200ec45e123e7e112e5c1703edd164e0b34c978 Mon Sep 17 00:00:00 2001 From: Jack Amadeo Date: Tue, 12 Aug 2025 13:26:36 -0400 Subject: [PATCH 17/17] One binary with some organziation --- Cargo.lock | 13 +++--- crates/goose-test/Cargo.toml | 9 ++-- crates/goose-test/src/bin/capture.rs | 45 +++++++++++++++++++ crates/goose-test/src/lib.rs | 1 + crates/goose-test/src/mcp/mod.rs | 1 + crates/goose-test/src/mcp/stdio/mod.rs | 2 + .../stdio/playback.rs} | 11 +---- .../stdio_logger.rs => mcp/stdio/record.rs} | 26 +++-------- crates/goose/tests/mcp_integration_test.rs | 33 ++++++++------ 9 files changed, 85 insertions(+), 56 deletions(-) create mode 100644 crates/goose-test/src/bin/capture.rs create mode 100644 crates/goose-test/src/lib.rs create mode 100644 crates/goose-test/src/mcp/mod.rs create mode 100644 crates/goose-test/src/mcp/stdio/mod.rs rename crates/goose-test/src/{bin/stdio_replayer.rs => mcp/stdio/playback.rs} (91%) rename crates/goose-test/src/{bin/stdio_logger.rs => mcp/stdio/record.rs} (83%) diff --git a/Cargo.lock b/Cargo.lock index cfa964a0521d..571368e56474 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1591,9 +1591,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.31" +version = "4.5.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" +checksum = "1c1f056bae57e3e54c3375c41ff79619ddd13460a17d7438712bd0d83fda4ff8" dependencies = [ "clap_builder", "clap_derive", @@ -1601,9 +1601,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.31" +version = "4.5.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" +checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8" dependencies = [ "anstream", "anstyle", @@ -1614,9 +1614,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.28" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" +checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -3519,6 +3519,7 @@ dependencies = [ name = "goose-test" version = "1.1.0" dependencies = [ + "clap", "serde_json", ] diff --git a/crates/goose-test/Cargo.toml b/crates/goose-test/Cargo.toml index f13751a349b0..f42dbfd5d5a7 100644 --- a/crates/goose-test/Cargo.toml +++ b/crates/goose-test/Cargo.toml @@ -11,12 +11,9 @@ description.workspace = true workspace = true [[bin]] -name = "stdio_logger" -path = "src/bin/stdio_logger.rs" - -[[bin]] -name = "stdio_replayer" -path = "src/bin/stdio_replayer.rs" +name = "capture" +path = "src/bin/capture.rs" [dependencies] +clap = { version = "4.5.44", features = ["derive"] } serde_json = "1.0.142" diff --git a/crates/goose-test/src/bin/capture.rs b/crates/goose-test/src/bin/capture.rs new file mode 100644 index 000000000000..6dd58d5e329f --- /dev/null +++ b/crates/goose-test/src/bin/capture.rs @@ -0,0 +1,45 @@ +use std::io; + +use clap::{Parser, Subcommand, ValueEnum}; + +use goose_test::mcp::stdio::playback::playback; +use goose_test::mcp::stdio::record::record; + +#[derive(Parser)] +struct Cli { + #[arg(value_enum)] + transport: Transport, + #[command(subcommand)] + mode: Mode, +} + +#[derive(ValueEnum, Clone, Debug)] +enum Transport { + Stdio, +} + +#[derive(Subcommand, Clone, Debug)] +enum Mode { + Record { + file: String, + command: String, + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + args: Vec, + }, + Playback { + file: String, + }, +} + +fn main() -> io::Result<()> { + let cli = Cli::parse(); + + match cli.mode { + Mode::Record { + file, + command, + args, + } => record(&file, &command, &args), + Mode::Playback { file } => playback(&file), + } +} diff --git a/crates/goose-test/src/lib.rs b/crates/goose-test/src/lib.rs new file mode 100644 index 000000000000..e7fd74de9ef2 --- /dev/null +++ b/crates/goose-test/src/lib.rs @@ -0,0 +1 @@ +pub mod mcp; diff --git a/crates/goose-test/src/mcp/mod.rs b/crates/goose-test/src/mcp/mod.rs new file mode 100644 index 000000000000..b1d7f04bbe6a --- /dev/null +++ b/crates/goose-test/src/mcp/mod.rs @@ -0,0 +1 @@ +pub mod stdio; diff --git a/crates/goose-test/src/mcp/stdio/mod.rs b/crates/goose-test/src/mcp/stdio/mod.rs new file mode 100644 index 000000000000..f6d7f7bb1bdc --- /dev/null +++ b/crates/goose-test/src/mcp/stdio/mod.rs @@ -0,0 +1,2 @@ +pub mod playback; +pub mod record; diff --git a/crates/goose-test/src/bin/stdio_replayer.rs b/crates/goose-test/src/mcp/stdio/playback.rs similarity index 91% rename from crates/goose-test/src/bin/stdio_replayer.rs rename to crates/goose-test/src/mcp/stdio/playback.rs index eda40176d60c..82f3897ecf1e 100644 --- a/crates/goose-test/src/bin/stdio_replayer.rs +++ b/crates/goose-test/src/mcp/stdio/playback.rs @@ -1,4 +1,3 @@ -use std::env; use std::fs::File; use std::io::{self, BufRead, BufReader, Write}; use std::process; @@ -52,15 +51,7 @@ fn load_log_file(file_path: &str) -> io::Result> { Ok(entries) } -fn main() -> io::Result<()> { - let args: Vec = env::args().collect(); - - if args.len() != 2 { - eprintln!("Usage: {} ", args[0]); - process::exit(1); - } - - let log_file_path = &args[1]; +pub fn playback(log_file_path: &String) -> io::Result<()> { let entries = load_log_file(log_file_path)?; let errors_file = File::create(format!("{}.errors.txt", log_file_path))?; diff --git a/crates/goose-test/src/bin/stdio_logger.rs b/crates/goose-test/src/mcp/stdio/record.rs similarity index 83% rename from crates/goose-test/src/bin/stdio_logger.rs rename to crates/goose-test/src/mcp/stdio/record.rs index e3bae3ad00c2..6d6f349c8e27 100644 --- a/crates/goose-test/src/bin/stdio_logger.rs +++ b/crates/goose-test/src/mcp/stdio/record.rs @@ -1,4 +1,3 @@ -use std::env; use std::fs::OpenOptions; use std::io::{self, BufRead, BufReader, Write}; use std::process::{ChildStdin, Command, Stdio}; @@ -56,19 +55,7 @@ fn handle_stdin_stream( }) } -fn main() -> io::Result<()> { - let args: Vec = env::args().collect(); - - if args.len() < 3 { - eprintln!("Usage: {} [args...]", args[0]); - eprintln!("Example: {{}} ls -la"); - std::process::exit(1); - } - - let log_file_path = &args[1]; - let cmd = &args[2]; - let cmd_args = &args[3..]; - +pub fn record(log_file_path: &String, cmd: &String, cmd_args: &[String]) -> io::Result<()> { let (tx, rx) = mpsc::channel(); let log_file = OpenOptions::new() @@ -78,15 +65,12 @@ fn main() -> io::Result<()> { .open(log_file_path)?; let mut child = Command::new(cmd) - .args(cmd_args) + .args(cmd_args.iter()) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() - .map_err(|e| { - eprintln!("Failed to execute command '{}': {}", cmd, e); - e - })?; + .inspect_err(|e| eprintln!("Failed to execute command '{}': {}", &cmd, e))?; let child_stdin = child.stdin.take().unwrap(); let child_stdout = child.stdout.take().unwrap(); @@ -121,11 +105,11 @@ fn main() -> io::Result<()> { } }); - let exit_status = child.wait()?; + child.wait()?; stdin_handle.join().ok(); stdout_handle.join().ok(); stderr_handle.join().ok(); - std::process::exit(exit_status.code().unwrap_or(1)); + Ok(()) } diff --git a/crates/goose/tests/mcp_integration_test.rs b/crates/goose/tests/mcp_integration_test.rs index 4a0687f00282..36cd190fe577 100644 --- a/crates/goose/tests/mcp_integration_test.rs +++ b/crates/goose/tests/mcp_integration_test.rs @@ -15,12 +15,9 @@ use test_case::test_case; enum TestMode { Record, - Replay, + Playback, } -const LOGGER_BINARY: &str = "stdio_logger"; -const REPLAY_BINARY: &str = "stdio_replayer"; - #[test_case( vec!["npx", "-y", "@modelcontextprotocol/server-everything"], vec![ @@ -73,18 +70,28 @@ async fn test_replayed_session( TestMode::Record } else { assert!(replay_file_path.exists(), "replay file doesn't exist"); - TestMode::Replay + TestMode::Playback }; - let bin = match mode { - TestMode::Record => LOGGER_BINARY, - TestMode::Replay => REPLAY_BINARY, + let mode_arg = match mode { + TestMode::Record => "record", + TestMode::Playback => "playback", }; let cmd = "cargo".to_string(); - let mut args = vec!["run", "--quiet", "-p", "goose-test", "--bin", bin, "--"] - .into_iter() - .map(str::to_string) - .collect::>(); + let mut args = vec![ + "run", + "--quiet", + "-p", + "goose-test", + "--bin", + "capture", + "--", + "stdio", + mode_arg, + ] + .into_iter() + .map(str::to_string) + .collect::>(); args.push(replay_file_path.to_string_lossy().to_string()); @@ -143,7 +150,7 @@ async fn test_replayed_session( TestMode::Record => { serde_json::to_writer_pretty(File::create(results_path)?, &results)? } - TestMode::Replay => assert_eq!( + TestMode::Playback => assert_eq!( serde_json::from_reader::<_, Vec>>(File::open(results_path)?)?, results ),