feat: enable native tool calling for gemini provider (#9343)

Co-authored-by: daniel-lxs <[email protected]>

Author: hannesrudolph

packages/types/src/providers/gemini.ts

@@ -10,6 +10,7 @@ export const geminiModels = {
 		maxTokens: 65_536,
 		contextWindow: 1_048_576,
 		supportsImages: true,
+		supportsNativeTools: true,
 		supportsPromptCache: true,
 		supportsReasoningEffort: ["low", "high"],
 		reasoningEffort: "low",
@@ -35,6 +36,7 @@ export const geminiModels = {
 		maxTokens: 64_000,
 		contextWindow: 1_048_576,
 		supportsImages: true,
+		supportsNativeTools: true,
 		supportsPromptCache: true,
 		inputPrice: 2.5, // This is the pricing for prompts above 200k tokens.
 		outputPrice: 15,
@@ -62,6 +64,7 @@ export const geminiModels = {
 		maxTokens: 65_535,
 		contextWindow: 1_048_576,
 		supportsImages: true,
+		supportsNativeTools: true,
 		supportsPromptCache: true,
 		inputPrice: 2.5, // This is the pricing for prompts above 200k tokens.
 		outputPrice: 15,
@@ -88,6 +91,7 @@ export const geminiModels = {
 		maxTokens: 65_535,
 		contextWindow: 1_048_576,
 		supportsImages: true,
+		supportsNativeTools: true,
 		supportsPromptCache: true,
 		inputPrice: 2.5, // This is the pricing for prompts above 200k tokens.
 		outputPrice: 15,
@@ -112,6 +116,7 @@ export const geminiModels = {
 		maxTokens: 65_535,
 		contextWindow: 1_048_576,
 		supportsImages: true,
+		supportsNativeTools: true,
 		supportsPromptCache: true,
 		inputPrice: 2.5, // This is the pricing for prompts above 200k tokens.
 		outputPrice: 15,
@@ -140,6 +145,7 @@ export const geminiModels = {
 		maxTokens: 65_536,
 		contextWindow: 1_048_576,
 		supportsImages: true,
+		supportsNativeTools: true,
 		supportsPromptCache: true,
 		inputPrice: 0.3,
 		outputPrice: 2.5,
@@ -152,6 +158,7 @@ export const geminiModels = {
 		maxTokens: 65_536,
 		contextWindow: 1_048_576,
 		supportsImages: true,
+		supportsNativeTools: true,
 		supportsPromptCache: true,
 		inputPrice: 0.3,
 		outputPrice: 2.5,
@@ -164,6 +171,7 @@ export const geminiModels = {
 		maxTokens: 64_000,
 		contextWindow: 1_048_576,
 		supportsImages: true,
+		supportsNativeTools: true,
 		supportsPromptCache: true,
 		inputPrice: 0.3,
 		outputPrice: 2.5,
@@ -178,6 +186,7 @@ export const geminiModels = {
 		maxTokens: 65_536,
 		contextWindow: 1_048_576,
 		supportsImages: true,
+		supportsNativeTools: true,
 		supportsPromptCache: true,
 		inputPrice: 0.1,
 		outputPrice: 0.4,
@@ -190,6 +199,7 @@ export const geminiModels = {
 		maxTokens: 65_536,
 		contextWindow: 1_048_576,
 		supportsImages: true,
+		supportsNativeTools: true,
 		supportsPromptCache: true,
 		inputPrice: 0.1,
 		outputPrice: 0.4,

src/api/providers/gemini.ts

@@ -5,6 +5,8 @@ import {
 	type GenerateContentParameters,
 	type GenerateContentConfig,
 	type GroundingMetadata,
+	FunctionCallingConfigMode,
+	Content,
 } from "@google/genai"
 import type { JWTInput } from "google-auth-library"
 
@@ -101,17 +103,46 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 			return true
 		})
 
-		const contents = geminiMessages.map((message) =>
-			convertAnthropicMessageToGemini(message, { includeThoughtSignatures }),
-		)
+		// Build a map of tool IDs to names from previous messages
+		// This is needed because Anthropic's tool_result blocks only contain the ID,
+		// but Gemini requires the name in functionResponse
+		const toolIdToName = new Map<string, string>()
+		for (const message of messages) {
+			if (Array.isArray(message.content)) {
+				for (const block of message.content) {
+					if (block.type === "tool_use") {
+						toolIdToName.set(block.id, block.name)
+					}
+				}
+			}
+		}
+
+		const contents = geminiMessages
+			.map((message) => convertAnthropicMessageToGemini(message, { includeThoughtSignatures, toolIdToName }))
+			.flat()
 
 		const tools: GenerateContentConfig["tools"] = []
-		if (this.options.enableUrlContext) {
-			tools.push({ urlContext: {} })
-		}
 
-		if (this.options.enableGrounding) {
-			tools.push({ googleSearch: {} })
+		// Google built-in tools (Grounding, URL Context) are currently mutually exclusive
+		// with function declarations in the Gemini API. If native function calling is
+		// used (Agent tools), we must prioritize it and skip built-in tools to avoid
+		// "Tool use with function calling is unsupported" (HTTP 400) errors.
+		if (metadata?.tools && metadata.tools.length > 0) {
+			tools.push({
+				functionDeclarations: metadata.tools.map((tool) => ({
+					name: (tool as any).function.name,
+					description: (tool as any).function.description,
+					parametersJsonSchema: (tool as any).function.parameters,
+				})),
+			})
+		} else {
+			if (this.options.enableUrlContext) {
+				tools.push({ urlContext: {} })
+			}
+
+			if (this.options.enableGrounding) {
+				tools.push({ googleSearch: {} })
+			}
 		}
 
 		// Determine temperature respecting model capabilities and defaults:
@@ -133,6 +164,34 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 			...(tools.length > 0 ? { tools } : {}),
 		}
 
+		if (metadata?.tool_choice) {
+			const choice = metadata.tool_choice
+			let mode: FunctionCallingConfigMode
+			let allowedFunctionNames: string[] | undefined
+
+			if (choice === "auto") {
+				mode = FunctionCallingConfigMode.AUTO
+			} else if (choice === "none") {
+				mode = FunctionCallingConfigMode.NONE
+			} else if (choice === "required") {
+				// "required" means the model must call at least one tool; Gemini uses ANY for this.
+				mode = FunctionCallingConfigMode.ANY
+			} else if (typeof choice === "object" && "function" in choice && choice.type === "function") {
+				mode = FunctionCallingConfigMode.ANY
+				allowedFunctionNames = [choice.function.name]
+			} else {
+				// Fall back to AUTO for unknown values to avoid unintentionally broadening tool access.
+				mode = FunctionCallingConfigMode.AUTO
+			}
+
+			config.toolConfig = {
+				functionCallingConfig: {
+					mode,
+					...(allowedFunctionNames ? { allowedFunctionNames } : {}),
+				},
+			}
+		}
+
 		const params: GenerateContentParameters = { model, contents, config }
 		try {
 			const result = await this.client.models.generateContentStream(params)
@@ -141,6 +200,8 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 			let pendingGroundingMetadata: GroundingMetadata | undefined
 			let finalResponse: { responseId?: string } | undefined
 
+			let toolCallCounter = 0
+
 			for await (const chunk of result) {
 				// Track the final structured response (per SDK pattern: candidate.finishReason)
 				if (chunk.candidates && chunk.candidates[0]?.finishReason) {
@@ -159,6 +220,7 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 							thought?: boolean
 							text?: string
 							thoughtSignature?: string
+							functionCall?: { name: string; args: Record<string, unknown> }
 						}>) {
 							// Capture thought signatures so they can be persisted into API history.
 							const thoughtSignature = part.thoughtSignature
@@ -173,6 +235,14 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 								if (part.text) {
 									yield { type: "reasoning", text: part.text }
 								}
+							} else if (part.functionCall) {
+								const callId = `${part.functionCall.name}-${toolCallCounter++}`
+								yield {
+									type: "tool_call",
+									id: callId,
+									name: part.functionCall.name,
+									arguments: JSON.stringify(part.functionCall.args),
+								}
 							} else {
 								// This is regular content
 								if (part.text) {
@@ -350,12 +420,7 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 			const countTokensRequest = {
 				model,
 				// Token counting does not need encrypted continuation; always drop thoughtSignature.
-				contents: [
-					{
-						role: "user",
-						parts: convertAnthropicContentToGemini(content, { includeThoughtSignatures: false }),
-					},
-				],
+				contents: convertAnthropicContentToGemini(content, { includeThoughtSignatures: false }),
 			}
 
 			const response = await this.client.models.countTokens(countTokensRequest)