Feat: Let providers configure a fast model for summarization

crates/goose/src/context_mgmt/summarize.rs

@@ -44,7 +44,7 @@ pub async fn summarize_messages(
 
     // Send the request to the provider and fetch the response
     let (mut response, mut provider_usage) = provider
-        .complete(&system_prompt, &summarization_request, &[])
+        .complete_fast(&system_prompt, &summarization_request, &[])
         .await?;
 
     // Set role to user as it will be used in following conversation as user content

crates/goose/src/model.rs

@@ -72,6 +72,7 @@ pub struct ModelConfig {
     pub max_tokens: Option<i32>,
     pub toolshim: bool,
     pub toolshim_model: Option<String>,
+    pub fast_model: Option<String>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -101,6 +102,7 @@ impl ModelConfig {
             max_tokens: None,
             toolshim,
             toolshim_model,
+            fast_model: None,
         })
     }
 

crates/goose/src/providers/base.rs

@@ -337,6 +337,32 @@ pub trait Provider: Send + Sync {
         tools: &[Tool],
     ) -> Result<(Message, ProviderUsage), ProviderError>;
 
+    /// Generate the next message using a fast/cheaper model when available
+    ///
+    /// Default implementation just calls regular complete() for providers that don't support fast models
+    ///
+    /// # Arguments
+    /// * `system` - The system prompt that guides the model's behavior
+    /// * `messages` - The conversation history as a sequence of messages
+    /// * `tools` - Optional list of tools the model can use
+    ///
+    /// # Returns
+    /// A tuple containing the model's response message and provider usage statistics
+    ///
+    /// # Errors
+    /// ProviderError
+    ///   - It's important to raise ContextLengthExceeded correctly since agent handles it
+    async fn complete_fast(
+        &self,
+        system: &str,
+        messages: &[Message],
+        tools: &[Tool],
+    ) -> Result<(Message, ProviderUsage), ProviderError> {
+        // Default implementation: just call regular complete
+        // Providers that support fast models should override this
+        self.complete(system, messages, tools).await
+    }
+
     /// Get the model config from the provider
     fn get_model_config(&self) -> ModelConfig;
 
@@ -418,7 +444,7 @@ pub trait Provider: Send + Sync {
         let prompt = self.create_session_name_prompt(&context);
         let message = Message::user().with_text(&prompt);
         let result = self
-            .complete(
+            .complete_fast(
                 "Reply with only a description in four words or less",
                 &[message],
                 &[],

crates/goose/src/providers/formats/databricks.rs

@@ -1045,6 +1045,7 @@ mod tests {
             max_tokens: Some(1024),
             toolshim: false,
             toolshim_model: None,
+            fast_model: None,
         };
         let request = create_request(&model_config, "system", &[], &[], &ImageFormat::OpenAi)?;
         let obj = request.as_object().unwrap();
@@ -1076,6 +1077,7 @@ mod tests {
             max_tokens: Some(1024),
             toolshim: false,
             toolshim_model: None,
+            fast_model: None,
         };
         let request = create_request(&model_config, "system", &[], &[], &ImageFormat::OpenAi)?;
         let obj = request.as_object().unwrap();
@@ -1108,6 +1110,7 @@ mod tests {
             max_tokens: Some(1024),
             toolshim: false,
             toolshim_model: None,
+            fast_model: None,
         };
         let request = create_request(&model_config, "system", &[], &[], &ImageFormat::OpenAi)?;
         let obj = request.as_object().unwrap();

crates/goose/src/providers/formats/openai.rs

@@ -1077,6 +1077,7 @@ mod tests {
             max_tokens: Some(1024),
             toolshim: false,
             toolshim_model: None,
+            fast_model: None,
         };
         let request = create_request(&model_config, "system", &[], &[], &ImageFormat::OpenAi)?;
         let obj = request.as_object().unwrap();
@@ -1108,6 +1109,7 @@ mod tests {
             max_tokens: Some(1024),
             toolshim: false,
             toolshim_model: None,
+            fast_model: None,
         };
         let request = create_request(&model_config, "system", &[], &[], &ImageFormat::OpenAi)?;
         let obj = request.as_object().unwrap();
@@ -1140,6 +1142,7 @@ mod tests {
             max_tokens: Some(1024),
             toolshim: false,
             toolshim_model: None,
+            fast_model: None,
         };
         let request = create_request(&model_config, "system", &[], &[], &ImageFormat::OpenAi)?;
         let obj = request.as_object().unwrap();

crates/goose/src/providers/openai.rs

@@ -28,6 +28,8 @@ use crate::providers::base::MessageStream;
 use crate::providers::formats::openai::response_to_streaming_message;
 use rmcp::model::Tool;
 
+const OPEN_AI_FAST_MODEL: &str = "gpt-4o-mini";
+
 pub const OPEN_AI_DEFAULT_MODEL: &str = "gpt-4o";
 pub const OPEN_AI_KNOWN_MODELS: &[(&str, usize)] = &[
     ("gpt-4o", 128_000),
@@ -160,6 +162,31 @@ impl OpenAiProvider {
             .await?;
         handle_response_openai_compat(response).await
     }
+
+    // Core completion logic that takes a model config
+    async fn complete_with_model(
+        &self,
+        model_config: &ModelConfig,
+        system: &str,
+        messages: &[Message],
+        tools: &[Tool],
+    ) -> Result<(Message, ProviderUsage), ProviderError> {
+        let payload = create_request(model_config, system, messages, tools, &ImageFormat::OpenAi)?;
+
+        let json_response = self.post(&payload).await?;
+
+        let message = response_to_message(&json_response)?;
+        let usage = json_response
+            .get("usage")
+            .map(get_usage)
+            .unwrap_or_else(|| {
+                tracing::debug!("Failed to get usage data");
+                Usage::default()
+            });
+        let model = get_model(&json_response);
+        emit_debug_trace(model_config, &payload, &json_response, &usage);
+        Ok((message, ProviderUsage::new(model, usage)))
+    }
 }
 
 #[async_trait]
@@ -202,21 +229,23 @@ impl Provider for OpenAiProvider {
         messages: &[Message],
         tools: &[Tool],
     ) -> Result<(Message, ProviderUsage), ProviderError> {
-        let payload = create_request(&self.model, system, messages, tools, &ImageFormat::OpenAi)?;
+        // Thin wrapper that calls complete_with_model with the configured model
+        self.complete_with_model(&self.model, system, messages, tools)
+            .await
+    }
 
-        let json_response = self.post(&payload).await?;
+    async fn complete_fast(
+        &self,
+        system: &str,
+        messages: &[Message],
+        tools: &[Tool],
+    ) -> Result<(Message, ProviderUsage), ProviderError> {
+        // Use the fast model (gpt-4o-mini) for fast completions
+        let mut fast_config = self.model.clone();
+        fast_config.model_name = OPEN_AI_FAST_MODEL.to_string();
 
-        let message = response_to_message(&json_response)?;
-        let usage = json_response
-            .get("usage")
-            .map(get_usage)
-            .unwrap_or_else(|| {
-                tracing::debug!("Failed to get usage data");
-                Usage::default()
-            });
-        let model = get_model(&json_response);
-        emit_debug_trace(&self.model, &payload, &json_response, &usage);
-        Ok((message, ProviderUsage::new(model, usage)))
+        self.complete_with_model(&fast_config, system, messages, tools)
+            .await
     }
 
     async fn fetch_supported_models(&self) -> Result<Option<Vec<String>>, ProviderError> {