From 5c1ffee046b8c07a4d5e850cbbba33f5797f7e9c Mon Sep 17 00:00:00 2001 From: akshaydeo Date: Wed, 19 Aug 2026 00:42:59 +0530 Subject: [PATCH] gemini tool call fixes --- core/providers/gemini/responses.go | 4 +- .../gemini/thinkinglevelsupport_test.go | 215 +++++++++++++ core/providers/gemini/utils.go | 146 +++++++-- tests/cmd/e2eseed/go.mod | 2 +- tests/cmd/seed/go.mod | 2 +- tests/cmd/seedvks/go.mod | 2 +- .../e2e/api/collections/provider-harness.json | 286 ++++++++++++++++++ 7 files changed, 632 insertions(+), 25 deletions(-) create mode 100644 core/providers/gemini/thinkinglevelsupport_test.go diff --git a/core/providers/gemini/responses.go b/core/providers/gemini/responses.go index 8cd9270056e..d16de2ff1ef 100644 --- a/core/providers/gemini/responses.go +++ b/core/providers/gemini/responses.go @@ -3788,7 +3788,9 @@ func (r *GeminiGenerationRequest) convertParamsToGenerationConfigResponses(param // User provided effort only (no max_tokens) if supportsLevel { // Gemini 3.0+ - use thinkingLevel (more native) - config.ThinkingConfig.ThinkingLevel = schemas.Ptr(effortToThinkingLevel(*params.Reasoning.Effort, capModel)) + if level := effortToThinkingLevel(*params.Reasoning.Effort, capModel); level != "" { + config.ThinkingConfig.ThinkingLevel = schemas.Ptr(level) + } } else { maxTokens := providerUtils.GetMaxOutputTokensOrDefault(capModel, DefaultCompletionMaxTokens) if config.MaxOutputTokens > 0 { diff --git a/core/providers/gemini/thinkinglevelsupport_test.go b/core/providers/gemini/thinkinglevelsupport_test.go new file mode 100644 index 00000000000..b6f5b1712f8 --- /dev/null +++ b/core/providers/gemini/thinkinglevelsupport_test.go @@ -0,0 +1,215 @@ +package gemini_test + +import ( + "testing" + + "github.com/maximhq/bifrost/core/providers/gemini" + "github.com/maximhq/bifrost/core/schemas" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Gemini 3 accepts thinkingLevel, but the set of levels each model accepts is +// per-model, and Gemini 3 has no "off" switch. Bifrost used to branch only on +// whether the model name contained "pro", which emitted levels the model rejects +// (e.g. "minimal" to gemini-3.7-flash) and disabled thinking entirely via +// thinkingBudget:0 - and a Gemini 3 model with thinking disabled stops calling +// functions, which reads as "MCP tools exposed but never invoked". +// +// Levels per model: https://ai.google.dev/gemini-api/docs/thinking#thinking-levels +// ThinkingLevel enum: https://ai.google.dev/api/generate-content#ThinkingLevel +// Thinking improves Gemini 3 function calling: +// +// https://ai.google.dev/gemini-api/docs/function-calling#thinking +func chatReqWithReasoning(model string, reasoning *schemas.ChatReasoning) *schemas.BifrostChatRequest { + props := schemas.NewOrderedMap() + props.Set("path", map[string]interface{}{"type": "string"}) + return &schemas.BifrostChatRequest{ + Model: model, + Input: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: schemas.Ptr("read /tmp/x")}}, + }, + Params: &schemas.ChatParameters{ + Reasoning: reasoning, + Tools: []schemas.ChatTool{ + {Type: "function", Function: &schemas.ChatToolFunction{ + Name: "mcp_fs_read_file", + Parameters: &schemas.ToolFunctionParameters{Type: "object", Properties: props, Required: []string{"path"}}, + }}, + }, + }, + } +} + +// assertFunctionDeclarationsSurvived pins the other half of the bug: the reasoning +// controls are only worth clamping if the tools they affect actually reach Gemini. +// Asserting thinkingConfig alone would still pass if a later change dropped +// functionDeclarations, which is the exact failure this folder exists to prevent. +func assertFunctionDeclarationsSurvived(t *testing.T, out *gemini.GeminiGenerationRequest) { + t.Helper() + var names []string + for _, tool := range out.Tools { + for _, fd := range tool.FunctionDeclarations { + names = append(names, fd.Name) + } + } + require.NotEmpty(t, names, "function declarations must reach Gemini, otherwise the model cannot call the tool") + assert.Contains(t, names, "mcp_fs_read_file") +} + +func TestGeminiThinkingLevelClampedToModelSupport(t *testing.T) { + cases := []struct { + name string + model string + effort string + want string + }{ + // gemini-3.7-flash supports low, medium, high - NOT minimal. + {"3.7-flash minimal clamps up to low", "gemini-3.7-flash", "minimal", "low"}, + {"3.7-flash low", "gemini-3.7-flash", "low", "low"}, + {"3.7-flash medium", "gemini-3.7-flash", "medium", "medium"}, + {"3.7-flash high", "gemini-3.7-flash", "high", "high"}, + + // gemini-3-pro-preview supports low and high only. + {"3-pro minimal clamps to low", "gemini-3-pro-preview", "minimal", "low"}, + {"3-pro medium clamps to high", "gemini-3-pro-preview", "medium", "high"}, + + // gemini-3.6-flash supports the full set including minimal. + {"3.6-flash keeps minimal", "gemini-3.6-flash", "minimal", "minimal"}, + + // gemini-3-flash-preview supports the full set including minimal. + {"3-flash-preview keeps minimal", "gemini-3-flash-preview", "minimal", "minimal"}, + + // gemini-3.1-flash-lite-image is the one documented Gemini 3 model with NO "low" + // rung at all - Google lists exactly "minimal, high" for it. It is the reason the + // defaultGemini3ThinkingLevels comment cannot claim every documented Gemini 3 + // model accepts "low". Source: https://ai.google.dev/gemini-api/docs/thinking + {"3.1-flash-lite-image keeps minimal", "gemini-3.1-flash-lite-image", "minimal", "minimal"}, + {"3.1-flash-lite-image low clamps down to minimal", "gemini-3.1-flash-lite-image", "low", "minimal"}, + {"3.1-flash-lite-image medium clamps up to high", "gemini-3.1-flash-lite-image", "medium", "high"}, + {"3.1-flash-lite-image high", "gemini-3.1-flash-lite-image", "high", "high"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + out, err := gemini.ToGeminiChatCompletionRequest(nil, chatReqWithReasoning(tc.model, + &schemas.ChatReasoning{Effort: schemas.Ptr(tc.effort)})) + require.NoError(t, err) + require.NotNil(t, out.GenerationConfig.ThinkingConfig, "thinkingConfig must be set") + require.NotNil(t, out.GenerationConfig.ThinkingConfig.ThinkingLevel, "thinkingLevel must be set on Gemini 3") + assert.Equal(t, tc.want, *out.GenerationConfig.ThinkingConfig.ThinkingLevel) + assert.Nil(t, out.GenerationConfig.ThinkingConfig.ThinkingBudget, + "Gemini 3 is controlled by thinkingLevel, thinkingBudget must not be sent alongside it") + assertFunctionDeclarationsSurvived(t, out) + }) + } +} + +// Gemini 3 cannot turn thinking off. Sending thinkingBudget:0 both uses the wrong +// control surface and kills function calling. "none" must land on the model's +// lowest supported level instead. +func TestGeminiEffortNoneDoesNotDisableThinkingOnGemini3(t *testing.T) { + cases := []struct { + model string + want string + }{ + {"gemini-3.7-flash", "low"}, // floor is low + {"gemini-3.6-flash", "minimal"}, // floor is minimal + {"gemini-3-pro-preview", "low"}, // floor is low + {"gemini-3-flash-preview", "minimal"}, + } + + for _, tc := range cases { + t.Run(tc.model, func(t *testing.T) { + out, err := gemini.ToGeminiChatCompletionRequest(nil, chatReqWithReasoning(tc.model, + &schemas.ChatReasoning{Effort: schemas.Ptr("none")})) + require.NoError(t, err) + require.NotNil(t, out.GenerationConfig.ThinkingConfig) + require.NotNil(t, out.GenerationConfig.ThinkingConfig.ThinkingLevel) + assert.Equal(t, tc.want, *out.GenerationConfig.ThinkingConfig.ThinkingLevel) + assert.Nil(t, out.GenerationConfig.ThinkingConfig.ThinkingBudget, + "thinkingBudget:0 disables thinking and breaks Gemini 3 function calling") + assertFunctionDeclarationsSurvived(t, out) + }) + } +} + +// Same for an explicit zero budget: on Gemini 3 it must not disable thinking. +func TestGeminiZeroBudgetDoesNotDisableThinkingOnGemini3(t *testing.T) { + out, err := gemini.ToGeminiChatCompletionRequest(nil, chatReqWithReasoning("gemini-3.7-flash", + &schemas.ChatReasoning{MaxTokens: schemas.Ptr(0)})) + require.NoError(t, err) + require.NotNil(t, out.GenerationConfig.ThinkingConfig) + require.NotNil(t, out.GenerationConfig.ThinkingConfig.ThinkingLevel) + assert.Equal(t, "low", *out.GenerationConfig.ThinkingConfig.ThinkingLevel) + assert.Nil(t, out.GenerationConfig.ThinkingConfig.ThinkingBudget) + assertFunctionDeclarationsSurvived(t, out) +} + +// Gemini 2.5 must keep its existing budget-based behaviour untouched. +func TestGeminiThinkingUnchangedForGemini25(t *testing.T) { + t.Run("effort maps to budget", func(t *testing.T) { + out, err := gemini.ToGeminiChatCompletionRequest(nil, chatReqWithReasoning("gemini-2.5-pro", + &schemas.ChatReasoning{Effort: schemas.Ptr("medium")})) + require.NoError(t, err) + require.NotNil(t, out.GenerationConfig.ThinkingConfig) + assert.Nil(t, out.GenerationConfig.ThinkingConfig.ThinkingLevel, + "thinkingLevel on a pre-3.0 model is a hard API error") + require.NotNil(t, out.GenerationConfig.ThinkingConfig.ThinkingBudget) + }) + + t.Run("gemini-2.5-pro cannot disable thinking", func(t *testing.T) { + out, err := gemini.ToGeminiChatCompletionRequest(nil, chatReqWithReasoning("gemini-2.5-pro", + &schemas.ChatReasoning{Effort: schemas.Ptr("none")})) + require.NoError(t, err) + assert.Nil(t, out.GenerationConfig.ThinkingConfig) + }) + + t.Run("gemini-2.5-flash can disable thinking with budget 0", func(t *testing.T) { + out, err := gemini.ToGeminiChatCompletionRequest(nil, chatReqWithReasoning("gemini-2.5-flash", + &schemas.ChatReasoning{Effort: schemas.Ptr("none")})) + require.NoError(t, err) + require.NotNil(t, out.GenerationConfig.ThinkingConfig) + require.NotNil(t, out.GenerationConfig.ThinkingConfig.ThinkingBudget) + assert.Equal(t, int32(0), *out.GenerationConfig.ThinkingConfig.ThinkingBudget) + }) +} + +// The Responses path shares the same gating and must behave identically. +func TestGeminiThinkingLevelClampedOnResponsesPath(t *testing.T) { + req := &schemas.BifrostResponsesRequest{ + Model: "gemini-3.7-flash", + Input: []schemas.ResponsesMessage{ + {Role: schemas.Ptr(schemas.ResponsesInputMessageRoleUser), + Content: &schemas.ResponsesMessageContent{ContentStr: schemas.Ptr("read /tmp/x")}}, + }, + Params: &schemas.ResponsesParameters{ + Reasoning: &schemas.ResponsesParametersReasoning{Effort: schemas.Ptr("minimal")}, + }, + } + out, err := gemini.ToGeminiResponsesRequest(nil, req) + require.NoError(t, err) + require.NotNil(t, out.GenerationConfig.ThinkingConfig) + require.NotNil(t, out.GenerationConfig.ThinkingConfig.ThinkingLevel) + assert.Equal(t, "low", *out.GenerationConfig.ThinkingConfig.ThinkingLevel) +} + +// Pins the claim defaultGemini3ThinkingLevels' comment makes about its own fallback. +// +// The comment used to justify the fallback with "low is the only rung every documented +// Gemini 3 model accepts", which the support table itself contradicts: its first entry, +// gemini-3.1-flash-lite-image, accepts only "minimal" and "high". This asserts that +// counterexample directly, so the justification cannot silently drift back to a claim the +// table disproves. Source: https://ai.google.dev/gemini-api/docs/thinking +func TestNotEveryDocumentedGemini3ModelAcceptsLow(t *testing.T) { + out, err := gemini.ToGeminiChatCompletionRequest(nil, chatReqWithReasoning( + "gemini-3.1-flash-lite-image", &schemas.ChatReasoning{Effort: schemas.Ptr("low")})) + require.NoError(t, err) + require.NotNil(t, out.GenerationConfig.ThinkingConfig) + require.NotNil(t, out.GenerationConfig.ThinkingConfig.ThinkingLevel) + + assert.NotEqual(t, "low", *out.GenerationConfig.ThinkingConfig.ThinkingLevel, + `gemini-3.1-flash-lite-image has no "low" rung, so "low" must never reach the wire for it`) + assert.Equal(t, "minimal", *out.GenerationConfig.ThinkingConfig.ThinkingLevel, + `"minimal" is the nearest rung below "low" that this model implements`) +} diff --git a/core/providers/gemini/utils.go b/core/providers/gemini/utils.go index d15a108c3c6..b0c661255b8 100644 --- a/core/providers/gemini/utils.go +++ b/core/providers/gemini/utils.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "net/http" + "slices" "sort" "strings" "time" @@ -172,7 +173,118 @@ func canDisableThinkingWithBudget(model string) bool { return !strings.Contains(strings.ToLower(model), "gemini-2.5-pro") } +// geminiThinkingLevels is the thinkingLevel ladder ordered from least to most +// thinking. Source: https://ai.google.dev/api/generate-content#ThinkingLevel +var geminiThinkingLevels = []string{"minimal", "low", "medium", "high"} + +// geminiThinkingLevelSupport records which rungs of that ladder each model actually +// implements. The sets are not uniform across the Gemini 3 family - gemini-3.7-flash +// has no "minimal", gemini-3-pro-preview has neither "minimal" nor "medium" - and the +// API rejects a level the model does not implement, so an effort has to be clamped per +// model rather than per family. +// Source: https://ai.google.dev/gemini-api/docs/thinking#thinking-levels +// Matching is first-prefix-wins, so longer prefixes are listed before the shorter +// prefixes they would otherwise be shadowed by. +var geminiThinkingLevelSupport = []struct { + prefix string + levels []string +}{ + {"gemini-3.1-flash-lite-image", []string{"minimal", "high"}}, + {"gemini-3.7-flash", []string{"low", "medium", "high"}}, + {"gemini-3.6-flash", []string{"minimal", "low", "medium", "high"}}, + {"gemini-3.5-flash-lite", []string{"minimal", "low", "medium", "high"}}, + {"gemini-3.5-flash", []string{"minimal", "low", "medium", "high"}}, + {"gemini-3.1-pro", []string{"low", "medium", "high"}}, + {"gemini-3-flash", []string{"minimal", "low", "medium", "high"}}, + {"gemini-3-pro", []string{"low", "high"}}, +} + +// defaultGemini3ThinkingLevels is the fallback for a Gemini 3 model not yet in the table. +// +// It omits "minimal" because that is the rung the text models most often lack: of the +// documented Gemini 3 text models, gemini-3.7-flash, gemini-3.1-pro-preview and +// gemini-3-pro-preview all reject it, so defaulting to it would send an unreleased model +// a level it is more likely than not to refuse. +// +// This is a heuristic, not a guarantee -- there is no rung every documented Gemini 3 model +// accepts. gemini-3.1-flash-lite-image implements only "minimal" and "high", so even "low" +// is not universal, which is why that model has its own entry above rather than relying on +// this fallback. Any model whose set genuinely differs needs an explicit entry too. +// TestNotEveryDocumentedGemini3ModelAcceptsLow pins that counterexample. +// Source: https://ai.google.dev/gemini-api/docs/thinking#thinking-levels +var defaultGemini3ThinkingLevels = []string{"low", "medium", "high"} + +// supportedThinkingLevels returns the thinkingLevel values model accepts. +func supportedThinkingLevels(model string) []string { + modelLower := strings.ToLower(model) + for _, entry := range geminiThinkingLevelSupport { + if strings.Contains(modelLower, entry.prefix) { + return entry.levels + } + } + return defaultGemini3ThinkingLevels +} + +// lowestThinkingLevel returns the least amount of thinking model can be asked for. +// Gemini 3 has no "off" switch, so this is the floor a "none" effort lands on. +func lowestThinkingLevel(model string) string { + levels := supportedThinkingLevels(model) + if len(levels) == 0 { + return "low" + } + return levels[0] +} + +// clampThinkingLevel snaps a requested level onto the nearest rung model implements. +// Ties break upward so a clamp never silently spends less reasoning than asked for. +func clampThinkingLevel(level string, model string) string { + supported := supportedThinkingLevels(model) + if slices.Contains(supported, level) { + return level + } + want := slices.Index(geminiThinkingLevels, level) + if want < 0 { + return level + } + best := "" + bestDistance := 0 + for _, candidate := range supported { + idx := slices.Index(geminiThinkingLevels, candidate) + if idx < 0 { + continue + } + distance := idx - want + if distance < 0 { + distance = -distance + } + if best == "" || distance < bestDistance || (distance == bestDistance && idx > want) { + best = candidate + bestDistance = distance + } + } + if best == "" { + return level + } + return best +} + func setThinkingBudgetZeroIfSupported(config *GenerationConfig, model string) { + // Gemini 3 cannot turn thinking off. Depth is controlled by thinkingLevel and the + // floor is the model's lowest supported rung, so a "none" effort clamps to that rung + // instead of zeroing the budget. Sending thinkingBudget:0 here used the pre-3.0 + // control surface and suppressed the internal reasoning Gemini 3 leans on to pick + // functions, which surfaced as tools being advertised but never called. + // Docs: https://ai.google.dev/gemini-api/docs/thinking#thinking-levels + // https://ai.google.dev/gemini-api/docs/function-calling#thinking + if isGemini3Plus(model) { + if config.ThinkingConfig == nil { + config.ThinkingConfig = &GenerationConfigThinkingConfig{} + } + config.ThinkingConfig.IncludeThoughts = false + config.ThinkingConfig.ThinkingBudget = nil + config.ThinkingConfig.ThinkingLevel = schemas.Ptr(lowestThinkingLevel(model)) + return + } if !canDisableThinkingWithBudget(model) { config.ThinkingConfig = nil return @@ -184,35 +296,26 @@ func setThinkingBudgetZeroIfSupported(config *GenerationConfig, model string) { config.ThinkingConfig.ThinkingBudget = schemas.Ptr(int32(0)) } -// effortToThinkingLevel converts reasoning effort to Gemini ThinkingLevel string -// Pro models only support "low" or "high" -// Other models support "minimal", "low", "medium", and "high" +// effortToThinkingLevel converts reasoning effort to a Gemini ThinkingLevel string, +// clamped to the levels the target model implements. Returns "" for "none", which +// callers handle through setThinkingBudgetZeroIfSupported instead. func effortToThinkingLevel(effort string, model string) string { - isPro := strings.Contains(strings.ToLower(model), "pro") - + var desired string switch effort { case "none": return "" // Empty string for no thinking case "minimal": - if isPro { - return "low" // Pro models don't support minimal, use low - } - return "minimal" + desired = "minimal" case "low": - return "low" + desired = "low" case "medium": - if isPro { - return "high" // Pro models don't support medium, use high - } - return "medium" + desired = "medium" case "high", "xhigh", "max": - return "high" + desired = "high" default: - if isPro { - return "high" - } - return "medium" + desired = "medium" } + return clampThinkingLevel(desired, model) } func getThinkingBudgetRange(model string, defaultMaxTokens int) thinkingBudgetRange { @@ -1232,8 +1335,9 @@ func convertParamsToGenerationConfig(params *schemas.ChatParameters, responseMod // User provided effort only (no max_tokens) if supportsLevel { // Gemini 3.0+ - use thinkingLevel (more native) - level := effortToThinkingLevel(*params.Reasoning.Effort, model) - config.ThinkingConfig.ThinkingLevel = &level + if level := effortToThinkingLevel(*params.Reasoning.Effort, model); level != "" { + config.ThinkingConfig.ThinkingLevel = &level + } } else { maxTokens := providerUtils.GetMaxOutputTokensOrDefault(model, DefaultCompletionMaxTokens) if config.MaxOutputTokens > 0 { diff --git a/tests/cmd/e2eseed/go.mod b/tests/cmd/e2eseed/go.mod index 1fe1bee5377..0e20e3fccac 100644 --- a/tests/cmd/e2eseed/go.mod +++ b/tests/cmd/e2eseed/go.mod @@ -86,7 +86,7 @@ require ( github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.32 // indirect - github.com/maximhq/bifrost/core v1.7.9 // indirect + github.com/maximhq/bifrost/core v1.7.10 // indirect github.com/maximhq/bifrost/framework v1.3.16 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect diff --git a/tests/cmd/seed/go.mod b/tests/cmd/seed/go.mod index 26af7665c49..8e50343193e 100644 --- a/tests/cmd/seed/go.mod +++ b/tests/cmd/seed/go.mod @@ -8,7 +8,7 @@ replace ( ) require ( - github.com/maximhq/bifrost/core v1.7.9 + github.com/maximhq/bifrost/core v1.7.10 github.com/maximhq/bifrost/framework v1.3.16 gorm.io/driver/postgres v1.6.0 gorm.io/driver/sqlite v1.6.0 diff --git a/tests/cmd/seedvks/go.mod b/tests/cmd/seedvks/go.mod index ebf1826bd1f..f9c403dcec5 100644 --- a/tests/cmd/seedvks/go.mod +++ b/tests/cmd/seedvks/go.mod @@ -9,7 +9,7 @@ replace ( require ( github.com/google/uuid v1.6.0 - github.com/maximhq/bifrost/core v1.7.9 + github.com/maximhq/bifrost/core v1.7.10 github.com/maximhq/bifrost/framework v1.3.16 gorm.io/driver/postgres v1.6.0 gorm.io/gorm v1.31.1 diff --git a/tests/e2e/api/collections/provider-harness.json b/tests/e2e/api/collections/provider-harness.json index c8aea8e6b1d..34babe8adcf 100644 --- a/tests/e2e/api/collections/provider-harness.json +++ b/tests/e2e/api/collections/provider-harness.json @@ -131329,6 +131329,292 @@ ] } ] + }, + { + "name": "54. Gemini 3 thinkingLevel per-model clamp + thinking floor", + "description": "Gemini 3 controls reasoning depth with generationConfig.thinkingConfig.thinkingLevel, and the set of levels is per-model rather than per-family: gemini-3.7-flash accepts low/medium/high but NOT minimal, while gemini-3.6-flash accepts all four. Gemini 3 also has no \"off\" rung - thinkingBudget is the pre-3.0 control surface, and zeroing it suppresses the internal reasoning Gemini 3 relies on to select functions, which surfaced as \"MCP tools advertised but never invoked\" after moving from gemini-2.5-pro. Bifrost previously branched only on whether the model name contained \"pro\", which emitted \"minimal\" to models that reject it and silently upgraded \"medium\" to \"high\" on gemini-3.1-pro. These cases assert the OUTBOUND payload via x-bf-send-back-raw-request rather than inferring from model behaviour, so they are deterministic. Levels per model: https://ai.google.dev/gemini-api/docs/thinking#thinking-levels ThinkingLevel enum: https://ai.google.dev/api/generate-content#ThinkingLevel Thinking improves Gemini 3 function calling: https://ai.google.dev/gemini-api/docs/function-calling#thinking", + "item": [ + { + "name": "gemini-3.7-flash reasoning_effort minimal clamps thinkingLevel to low (chat completions)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gemini/gemini-3.7-flash\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"What is the weather in Tokyo? Use the get_weather tool.\"\n }\n ],\n \"reasoning_effort\": \"minimal\",\n \"tools\": [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather for a city\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"city\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"city\"\n ]\n }\n }\n }\n ],\n \"tool_choice\": \"required\",\n \"max_completion_tokens\": 256\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }", + "pm.test('gemini-3.7-flash minimal clamp: 2xx', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "if (pm.response.code >= 400) { return; }", + "var body = pm.response.json() || {};", + "var ef = body.extra_fields || {};", + "var rr = ef.raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "pm.test('gemini-3.7-flash minimal clamp: raw_request captured', function () {", + " pm.expect(rr, 'raw_request missing - x-bf-send-back-raw-request not honored').to.be.an('object');", + "});", + "if (!rr || typeof rr !== 'object') { return; }", + "var tc = (rr.generationConfig || {}).thinkingConfig || {};", + "var tcs = JSON.stringify(tc);", + "pm.test('gemini-3.7-flash: thinkingLevel clamped to low (model rejects minimal)', function () {", + " pm.expect(tc.thinkingLevel, 'thinkingConfig: ' + tcs).to.eql('low');", + "});", + "pm.test('gemini-3.7-flash: minimal never reaches the wire', function () {", + " pm.expect(tc.thinkingLevel, 'thinkingConfig: ' + tcs).to.not.eql('minimal');", + "});", + "pm.test('gemini-3.7-flash: thinkingBudget not sent alongside thinkingLevel', function () {", + " pm.expect(tc.thinkingBudget === undefined || tc.thinkingBudget === null, 'thinkingConfig: ' + tcs).to.be.true;", + "});", + "var toolCalls = [];", + "(body.choices || []).forEach(function (ch) {", + " var m = ch.message || ch.delta || {};", + " (m.tool_calls || []).forEach(function (t) { toolCalls.push(t); });", + "});", + "pm.test('gemini-3.7-flash minimal clamp: function tool actually invoked', function () {", + " pm.expect(toolCalls.length, 'no tool_calls returned: ' + pm.response.text()).to.be.above(0);", + "});" + ] + } + } + ] + }, + { + "name": "gemini-3.7-flash reasoning_effort none keeps thinking on and still calls tools (chat completions)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gemini/gemini-3.7-flash\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"What is the weather in Tokyo? Use the get_weather tool.\"\n }\n ],\n \"reasoning_effort\": \"none\",\n \"tools\": [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather for a city\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"city\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"city\"\n ]\n }\n }\n }\n ],\n \"tool_choice\": \"required\",\n \"max_completion_tokens\": 256\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }", + "pm.test('gemini-3.7-flash effort none: 2xx', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "if (pm.response.code >= 400) { return; }", + "var body = pm.response.json() || {};", + "var ef = body.extra_fields || {};", + "var rr = ef.raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "pm.test('gemini-3.7-flash effort none: raw_request captured', function () {", + " pm.expect(rr, 'raw_request missing - x-bf-send-back-raw-request not honored').to.be.an('object');", + "});", + "if (!rr || typeof rr !== 'object') { return; }", + "var tc = (rr.generationConfig || {}).thinkingConfig || {};", + "var tcs = JSON.stringify(tc);", + "pm.test('gemini-3.7-flash: thinking not disabled via thinkingBudget 0', function () {", + " pm.expect(tc.thinkingBudget, 'thinkingConfig: ' + tcs).to.not.eql(0);", + "});", + "pm.test('gemini-3.7-flash: effort none floors at the lowest supported level', function () {", + " pm.expect(tc.thinkingLevel, 'thinkingConfig: ' + tcs).to.eql('low');", + "});", + "var toolCalls = [];", + "(body.choices || []).forEach(function (ch) {", + " var m = ch.message || ch.delta || {};", + " (m.tool_calls || []).forEach(function (t) { toolCalls.push(t); });", + "});", + "pm.test('gemini-3.7-flash: function tool still invoked with effort none', function () {", + " pm.expect(toolCalls.length, 'no tool_calls returned: ' + pm.response.text()).to.be.above(0);", + "});" + ] + } + } + ] + }, + { + "name": "gemini-3.7-flash reasoning effort minimal clamps to low on responses", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gemini/gemini-3.7-flash\",\n \"input\": \"What is the weather in Tokyo? Use the get_weather tool.\",\n \"reasoning\": {\n \"effort\": \"minimal\"\n },\n \"tools\": [\n {\n \"type\": \"function\",\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather for a city\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"city\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"city\"\n ]\n }\n }\n ],\n \"tool_choice\": \"required\",\n \"max_output_tokens\": 256\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }", + "pm.test('gemini-3.7-flash responses minimal clamp: 2xx', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "if (pm.response.code >= 400) { return; }", + "var body = pm.response.json() || {};", + "var ef = body.extra_fields || {};", + "var rr = ef.raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "pm.test('gemini-3.7-flash responses minimal clamp: raw_request captured', function () {", + " pm.expect(rr, 'raw_request missing - x-bf-send-back-raw-request not honored').to.be.an('object');", + "});", + "if (!rr || typeof rr !== 'object') { return; }", + "var tc = (rr.generationConfig || {}).thinkingConfig || {};", + "var tcs = JSON.stringify(tc);", + "pm.test('gemini-3.7-flash responses: thinkingLevel clamped to low', function () {", + " pm.expect(tc.thinkingLevel, 'thinkingConfig: ' + tcs).to.eql('low');", + "});", + "pm.test('gemini-3.7-flash responses: minimal never reaches the wire', function () {", + " pm.expect(tc.thinkingLevel, 'thinkingConfig: ' + tcs).to.not.eql('minimal');", + "});", + "var fnCalls = [];", + "(body.output || []).forEach(function (o) {", + " if (o && o.type === 'function_call') { fnCalls.push(o); }", + "});", + "pm.test('gemini-3.7-flash responses minimal clamp: function tool actually invoked', function () {", + " pm.expect(fnCalls.length, 'no function_call output returned: ' + pm.response.text()).to.be.above(0);", + "});" + ] + } + } + ] + }, + { + "name": "gemini-3.6-flash keeps thinkingLevel minimal (clamp must not over-correct)", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-bf-send-back-raw-request", + "value": "true" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gemini/gemini-3.6-flash\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"What is the weather in Tokyo? Use the get_weather tool.\"\n }\n ],\n \"reasoning_effort\": \"minimal\",\n \"tools\": [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather for a city\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"city\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"city\"\n ]\n }\n }\n }\n ],\n \"tool_choice\": \"required\",\n \"max_completion_tokens\": 256\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + }, + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "if ([401, 403, 429, 500, 502, 503, 504].indexOf(pm.response.code) !== -1) { return; }", + "pm.test('gemini-3.6-flash minimal preserved: 2xx', function () {", + " pm.expect(pm.response.code, 'failed: ' + pm.response.text()).to.be.below(400);", + "});", + "if (pm.response.code >= 400) { return; }", + "var body = pm.response.json() || {};", + "var ef = body.extra_fields || {};", + "var rr = ef.raw_request;", + "if (typeof rr === 'string') { try { rr = JSON.parse(rr); } catch (e) { rr = null; } }", + "pm.test('gemini-3.6-flash minimal preserved: raw_request captured', function () {", + " pm.expect(rr, 'raw_request missing - x-bf-send-back-raw-request not honored').to.be.an('object');", + "});", + "if (!rr || typeof rr !== 'object') { return; }", + "var tc = (rr.generationConfig || {}).thinkingConfig || {};", + "var tcs = JSON.stringify(tc);", + "// Negative control: gemini-3.6-flash DOES support minimal, so a fix that blanket-maps", + "// minimal -> low for every Gemini 3 model must fail here.", + "pm.test('gemini-3.6-flash: minimal preserved (model supports it)', function () {", + " pm.expect(tc.thinkingLevel, 'thinkingConfig: ' + tcs).to.eql('minimal');", + "});", + "var toolCalls = [];", + "(body.choices || []).forEach(function (ch) {", + " var m = ch.message || ch.delta || {};", + " (m.tool_calls || []).forEach(function (t) { toolCalls.push(t); });", + "});", + "pm.test('gemini-3.6-flash minimal preserved: function tool actually invoked', function () {", + " pm.expect(toolCalls.length, 'no tool_calls returned: ' + pm.response.text()).to.be.above(0);", + "});" + ] + } + } + ] + } + ] } ] } \ No newline at end of file