Skip to content

Better image compression #6746

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DOsinga merged 2 commits into from
Jan 27, 2026
Merged

Better image compression #6746

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3cc1474
Better image compression
Jan 27, 2026
02d354e
Update doc
Jan 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/goose-mcp/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ once_cell = "1.20.2"
ignore = { workspace = true }
lopdf = "0.36.0"
docx-rs = "0.4.7"
image = "0.24.9"
image = { version = "0.24.9", features = ["jpeg"] }
umya-spreadsheet = "2.2.3"
which = {workspace = true}
lru = "0.16"
Expand Down
106 changes: 54 additions & 52 deletions crates/goose-mcp/src/developer/rmcp_developer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -618,7 +618,7 @@ impl DeveloperServer {
) -> Result<CallToolResult, ErrorData> {
let params = params.0;

let mut image = if let Some(window_title) = &params.window_title {
let image = if let Some(window_title) = &params.window_title {
// Try to find and capture the specified window
let windows = Window::all().map_err(|_| {
ErrorData::new(
Expand Down Expand Up @@ -679,29 +679,8 @@ impl DeveloperServer {
})?
};

// Resize the image to a reasonable width while maintaining aspect ratio
let max_width = 768;
if image.width() > max_width {
let scale = max_width as f32 / image.width() as f32;
let new_height = (image.height() as f32 * scale) as u32;
image = xcap::image::imageops::resize(
&image,
max_width,
new_height,
xcap::image::imageops::FilterType::Lanczos3,
);
}

let mut bytes: Vec<u8> = Vec::new();
image
.write_to(&mut Cursor::new(&mut bytes), xcap::image::ImageFormat::Png)
.map_err(|e| {
ErrorData::new(
ErrorCode::INTERNAL_ERROR,
format!("Failed to write image buffer {}", e),
None,
)
})?;
let dynamic_image = xcap::image::DynamicImage::ImageRgba8(image);
let (bytes, mime_type) = Self::prepare_image_for_llm(dynamic_image)?;

// Convert to base64
let data = base64::prelude::BASE64_STANDARD.encode(bytes);
Expand All @@ -710,7 +689,7 @@ impl DeveloperServer {
// one text for Assistant, one image with priority 0.0
Ok(CallToolResult::success(vec![
Content::text("Screenshot captured").with_audience(vec![Role::Assistant]),
Content::image(data, "image/png").with_priority(0.0),
Content::image(data, &mime_type).with_priority(0.0),
]))
}

Expand Down Expand Up @@ -1155,14 +1134,14 @@ impl DeveloperServer {
/// Process an image file from disk.
///
/// The image will be:
/// 1. Resized if larger than max width while maintaining aspect ratio
/// 2. Converted to PNG format
/// 1. Resized to max 1024px on either dimension while maintaining aspect ratio
/// 2. Converted to JPEG format (85% quality)
/// 3. Returned as base64 encoded data
///
/// This allows processing image files for use in the conversation.
/// This allows processing image files for use in the conversation with optimized file sizes.
#[tool(
name = "image_processor",
description = "Process an image file from disk. Resizes if needed, converts to PNG, and returns as base64 data."
description = "Process an image file from disk. Resizes to max 1024px, converts to JPEG (85% quality), and returns as base64 data for optimized LLM consumption."
)]
pub async fn image_processor(
&self,
Expand Down Expand Up @@ -1234,42 +1213,65 @@ impl DeveloperServer {
)
})?;

// Resize if necessary (same logic as screen_capture)
let mut processed_image = image;
let max_width = 768;
if processed_image.width() > max_width {
let scale = max_width as f32 / processed_image.width() as f32;
let new_height = (processed_image.height() as f32 * scale) as u32;
processed_image = xcap::image::DynamicImage::ImageRgba8(xcap::image::imageops::resize(
&processed_image,
max_width,
let (bytes, mime_type) = Self::prepare_image_for_llm(image)?;

let data = base64::prelude::BASE64_STANDARD.encode(bytes);

Ok(CallToolResult::success(vec![
Content::text(format!(
"Successfully processed image from {}",
path.display()
))
.with_audience(vec![Role::Assistant]),
Content::image(data, &mime_type).with_priority(0.0),
]))
}

fn prepare_image_for_llm(
mut image: xcap::image::DynamicImage,
) -> Result<(Vec<u8>, String), ErrorData> {
let max_dimension = 1024;
let (width, height) = (image.width(), image.height());

if width > max_dimension || height > max_dimension {
let (new_width, new_height) = if width > height {
let scale = max_dimension as f32 / width as f32;
(max_dimension, (height as f32 * scale) as u32)
} else {
let scale = max_dimension as f32 / height as f32;
((width as f32 * scale) as u32, max_dimension)
};

image = xcap::image::DynamicImage::ImageRgba8(xcap::image::imageops::resize(
&image,
new_width,
new_height,
xcap::image::imageops::FilterType::Lanczos3,
));
}

// Convert to PNG and encode as base64
let rgb_image = image.to_rgb8();
let (img_width, img_height) = rgb_image.dimensions();

let mut bytes: Vec<u8> = Vec::new();
processed_image
.write_to(&mut Cursor::new(&mut bytes), xcap::image::ImageFormat::Png)
let mut cursor = Cursor::new(&mut bytes);

image::codecs::jpeg::JpegEncoder::new_with_quality(&mut cursor, 85)
.encode(
rgb_image.as_raw(),
img_width,
img_height,
image::ColorType::Rgb8,
Comment on lines +1259 to +1264
Copy link

Copilot AI Jan 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missing import statement for the image crate. The code uses image::codecs::jpeg::JpegEncoder and image::ColorType::Rgb8 but there's no use image::... import at the top of the file. Add use image; to the imports section (around line 1-30) to fix the compilation error.

Copilot uses AI. Check for mistakes.
)
.map_err(|e| {
ErrorData::new(
ErrorCode::INTERNAL_ERROR,
format!("Failed to write image buffer: {}", e),
format!("Failed to encode image as JPEG: {}", e),
None,
)
})?;

let data = base64::prelude::BASE64_STANDARD.encode(bytes);

Ok(CallToolResult::success(vec![
Content::text(format!(
"Successfully processed image from {}",
path.display()
))
.with_audience(vec![Role::Assistant]),
Content::image(data, "image/png").with_priority(0.0),
]))
Ok((bytes, "image/jpeg".to_string()))
}

// Helper method to resolve and validate file paths
Expand Down
Loading