Skip to content

fix: optimise reading large file content #3767

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
michaelneale merged 10 commits into from
Aug 5, 2025
Merged

fix: optimise reading large file content #3767

Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
34ddd59
logic and test to not ingest large files
michaelneale Aug 1, 2025
5714a08
Update mod.rs
michaelneale Aug 1, 2025
d78b407
Merge branch 'main' into micn/large-file-read
michaelneale Aug 4, 2025
357f4a2
refactoring for new clippy rules
michaelneale Aug 5, 2025
e107800
fmt
michaelneale Aug 5, 2025
06c25ff
human written comment
michaelneale Aug 5, 2025
b9cf946
Merge branch 'main' into micn/large-file-read
michaelneale Aug 5, 2025
2b7c23a
Merge branch 'main' into micn/large-file-read
michaelneale Aug 5, 2025
cf87a41
some leftover mistakes carried over, cleaning
michaelneale Aug 5, 2025
51584f5
Merge branch 'main' into micn/large-file-read
michaelneale Aug 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
230 changes: 230 additions & 0 deletions crates/goose-mcp/src/developer/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1011,6 +1011,16 @@ impl DeveloperRouter {
let lines: Vec<&str> = content.lines().collect();
let total_lines = lines.len();

// Check if file has more than 2000 lines and no view_range is provided
if view_range.is_none() && total_lines > 2000 {
Comment thread
michaelneale marked this conversation as resolved.
Outdated
return Err(ToolError::ExecutionError(format!(
"File '{}' is {} lines long, recommended to read in with view_range (or searching) to get bite size content. If you do wish to read all the file, please pass in view_range with [1, {}] to read it all at once",
path.display(),
total_lines,
total_lines
)));
}

// Handle view_range if provided, otherwise show all lines
let (start_idx, end_idx) = if let Some((start_line, end_line)) = view_range {
// Convert 1-indexed line numbers to 0-indexed
Expand Down Expand Up @@ -3233,6 +3243,226 @@ mod tests {
temp_dir.close().unwrap();
}

#[tokio::test]
#[serial]
async fn test_text_editor_view_large_file_without_range() {
let router = get_router().await;

let temp_dir = tempfile::tempdir().unwrap();
let file_path = temp_dir.path().join("large_file.txt");
let file_path_str = file_path.to_str().unwrap();
std::env::set_current_dir(&temp_dir).unwrap();

// Create a file with more than 2000 lines
let mut content = String::new();
for i in 1..=2001 {
Comment thread
michaelneale marked this conversation as resolved.
Outdated
content.push_str(&format!("Line {}\n", i));
}

router
.call_tool(
"text_editor",
json!({
"command": "write",
"path": file_path_str,
"file_text": content
}),
dummy_sender(),
)
.await
.unwrap();

// Test viewing without view_range - should trigger the error
let result = router
.call_tool(
"text_editor",
json!({
"command": "view",
"path": file_path_str
}),
dummy_sender(),
)
.await;

assert!(result.is_err());
let err = result.err().unwrap();
assert!(matches!(err, ToolError::ExecutionError(_)));
assert!(err.to_string().contains("2001 lines long"));
assert!(err
.to_string()
.contains("recommended to read in with view_range"));
assert!(err
.to_string()
.contains("please pass in view_range with [1, 2001]"));

// Test viewing with view_range - should work
let result = router
.call_tool(
"text_editor",
json!({
"command": "view",
"path": file_path_str,
"view_range": [1, 100]
}),
dummy_sender(),
)
.await;

assert!(result.is_ok());
let view_result = result.unwrap();
let text = view_result
.iter()
.find(|c| {
c.audience()
.is_some_and(|roles| roles.contains(&Role::User))
})
.unwrap()
.as_text()
.unwrap();

// Should contain lines 1-100
assert!(text.text.contains("1: Line 1"));
assert!(text.text.contains("100: Line 100"));
assert!(!text.text.contains("101: Line 101"));

// Test viewing with explicit full range - should work
let result = router
.call_tool(
"text_editor",
json!({
"command": "view",
"path": file_path_str,
"view_range": [1, 2001]
}),
dummy_sender(),
)
.await;

assert!(result.is_ok());

temp_dir.close().unwrap();
}

#[tokio::test]
#[serial]
async fn test_text_editor_view_file_with_exactly_2000_lines() {
Comment thread
michaelneale marked this conversation as resolved.
let router = get_router().await;

let temp_dir = tempfile::tempdir().unwrap();
let file_path = temp_dir.path().join("file_2000.txt");
let file_path_str = file_path.to_str().unwrap();
std::env::set_current_dir(&temp_dir).unwrap();

// Create a file with exactly 2000 lines (should not trigger the check)
let mut content = String::new();
for i in 1..=2000 {
content.push_str(&format!("Line {}\n", i));
}

router
.call_tool(
"text_editor",
json!({
"command": "write",
"path": file_path_str,
"file_text": content
}),
dummy_sender(),
)
.await
.unwrap();

// Test viewing without view_range - should work since it's exactly 2000 lines
let result = router
.call_tool(
"text_editor",
json!({
"command": "view",
"path": file_path_str
}),
dummy_sender(),
)
.await;

assert!(result.is_ok());
let view_result = result.unwrap();
let text = view_result
.iter()
.find(|c| {
c.audience()
.is_some_and(|roles| roles.contains(&Role::User))
})
.unwrap()
.as_text()
.unwrap();

// Should contain all lines
assert!(text.text.contains("1: Line 1"));
assert!(text.text.contains("2000: Line 2000"));

temp_dir.close().unwrap();
}

#[tokio::test]
#[serial]
async fn test_text_editor_view_small_file_without_range() {
let router = get_router().await;

let temp_dir = tempfile::tempdir().unwrap();
let file_path = temp_dir.path().join("small_file.txt");
let file_path_str = file_path.to_str().unwrap();
std::env::set_current_dir(&temp_dir).unwrap();

// Create a file with less than 2000 lines
let mut content = String::new();
for i in 1..=100 {
content.push_str(&format!("Line {}\n", i));
}

router
.call_tool(
"text_editor",
json!({
"command": "write",
"path": file_path_str,
"file_text": content
}),
dummy_sender(),
)
.await
.unwrap();

// Test viewing without view_range - should work fine
let result = router
.call_tool(
"text_editor",
json!({
"command": "view",
"path": file_path_str
}),
dummy_sender(),
)
.await;

assert!(result.is_ok());
let view_result = result.unwrap();
let text = view_result
.iter()
.find(|c| {
c.audience()
.is_some_and(|roles| roles.contains(&Role::User))
})
.unwrap()
.as_text()
.unwrap();

// Should contain all lines
assert!(text.text.contains("1: Line 1"));
assert!(text.text.contains("100: Line 100"));

temp_dir.close().unwrap();
}

#[tokio::test]
#[serial]
async fn test_bash_output_truncation() {
Expand Down
Loading