Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 86 additions & 0 deletions framework/modelcatalog/datasheet/costonlyparams_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package datasheet

import (
"encoding/json"
"slices"
"testing"

"github.com/maximhq/bifrost/core/schemas"
)

// Regression test for issue #6276: a datasheet row that declares ONLY pricing (no
// capability flags, no model_parameters) must not produce a supported-parameters
// allowlist. The default "reasoning_with_tool_calls" marker used to be added even to
// capability-empty rows, turning them into a one-element allowlist; the compat plugin
// (should_drop_params) then stripped tools / tool_choice / temperature / everything
// else from requests to that model.
func TestCostOnlyRowProducesNoParamAllowlist(t *testing.T) {
// Hosted-datasheet shape reported in the issue for gemini-3.6-flash:
// pricing keys only, none of which map to ModelCapabilities fields.
costOnlyRow := json.RawMessage(`{
"input_cost_per_token_batches": 0.000000075,
"output_cost_per_token_batches": 0.0000003
}`)

t.Run("extractSupportedParams on a capability-empty row", func(t *testing.T) {
var caps schemas.ModelCapabilities
if err := json.Unmarshal(costOnlyRow, &caps); err != nil {
t.Fatalf("unmarshal cost-only row: %v", err)
}
if !IsEmptyModelCapabilities(&caps) {
t.Fatalf("precondition: cost-only row should parse to empty ModelCapabilities")
}
if got := extractSupportedParams(&caps); len(got) != 0 {
t.Errorf("extractSupportedParams(cost-only row) = %v, want empty — a row declaring no request parameters must not synthesize an allowlist", got)
}
})

t.Run("applyModelParameters end-to-end", func(t *testing.T) {
s := &Store{}
// A second, genuinely populated row keeps applied > 0 so the index swap
// happens, mirroring the real datasheet where thousands of populated
// rows sit alongside the cost-only gemini-3.6-flash row.
s.applyModelParameters(map[string]json.RawMessage{
"gemini-3.6-flash": costOnlyRow,
"gpt-4o": json.RawMessage(`{"supports_function_calling":true,"supports_tool_choice":true}`),
})

if got := s.GetSupportedParameters("gemini-3.6-flash"); got != nil {
t.Errorf("GetSupportedParameters(gemini-3.6-flash) = %v, want nil (unknown) — compat treats any non-nil list as a complete allowlist and drops tools/tool_choice", got)
}
})

t.Run("populated row keeps the default marker", func(t *testing.T) {
// Backward-compat contract from #4630: rows declaring real capabilities but
// lacking the reasoning_with_tool_calls flag still get the default marker.
var caps schemas.ModelCapabilities
if err := json.Unmarshal(json.RawMessage(`{"supports_function_calling":true}`), &caps); err != nil {
t.Fatalf("unmarshal populated row: %v", err)
}
got := extractSupportedParams(&caps)
if !slices.Contains(got, "reasoning_with_tool_calls") {
t.Errorf("extractSupportedParams(populated row) = %v, want it to include the default reasoning_with_tool_calls marker", got)
}
if !slices.Contains(got, "tools") {
t.Errorf("extractSupportedParams(populated row) = %v, want it to include tools", got)
}
})

t.Run("explicit flag is honored regardless of other capabilities", func(t *testing.T) {
var explicitTrue schemas.ModelCapabilities
if err := json.Unmarshal(json.RawMessage(`{"supports_reasoning_with_tool_calls":true}`), &explicitTrue); err != nil {
t.Fatalf("unmarshal explicit-true row: %v", err)
}
if got := extractSupportedParams(&explicitTrue); !slices.Contains(got, "reasoning_with_tool_calls") {
t.Errorf("extractSupportedParams(explicit true) = %v, want reasoning_with_tool_calls present", got)
}

var explicitFalse schemas.ModelCapabilities
if err := json.Unmarshal(json.RawMessage(`{"supports_function_calling":true,"supports_reasoning_with_tool_calls":false}`), &explicitFalse); err != nil {
t.Fatalf("unmarshal explicit-false row: %v", err)
}
if got := extractSupportedParams(&explicitFalse); slices.Contains(got, "reasoning_with_tool_calls") {
t.Errorf("extractSupportedParams(explicit false) = %v, want reasoning_with_tool_calls absent", got)
}
})
}
17 changes: 14 additions & 3 deletions framework/modelcatalog/datasheet/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -515,9 +515,6 @@ func extractSupportedParams(parsed *schemas.ModelCapabilities) []string {
if parsed.SupportsReasoning != nil && *parsed.SupportsReasoning {
addParam("reasoning")
}
if parsed.SupportsReasoningWithToolCalls == nil || *parsed.SupportsReasoningWithToolCalls {
addParam("reasoning_with_tool_calls")
}
if parsed.SupportsNoneReasoningEffort != nil && *parsed.SupportsNoneReasoningEffort {
addParam("supports_none_reasoning_effort")
}
Expand All @@ -539,6 +536,20 @@ func extractSupportedParams(parsed *schemas.ModelCapabilities) []string {
addParam("web_search_options")
}

// reasoning_with_tool_calls defaults on for rows that lack the flag, so models
// missing it don't get reasoning stripped when tools are present. Restrict the
// default to rows that declared at least one capability: for a capability-empty
// (e.g. cost-only) row the marker alone would otherwise become a one-element
// allowlist, and compat with should_drop_params would strip every other
// parameter (tools, tool_choice, ...) from requests.
if parsed.SupportsReasoningWithToolCalls != nil {
if *parsed.SupportsReasoningWithToolCalls {
addParam("reasoning_with_tool_calls")
}
} else if len(supported) > 0 {

@akshaydeo akshaydeo Aug 23, 2026 •

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

1. supportedParams is also the qualified-key suffix index

params.go:378 iterates s.supportedParams to resolve a bare model name to provider-qualified datasheet keys, and the comment at params.go:235 names that second role explicitly: supportedParams feeds "the compat parameter allowlist and the qualified-key suffix index". Before this change the unconditional marker put essentially every row into that map. After it, a row that is not capability-empty but produces no parameter name drops out of it.

Reproduced on both sides with a scratch probe on a {"mode":"chat","supports_cache_point":true,"supports_system_messages":true,"beta_headers":{...}} row keyed vertex_ai/curated-model:

base 0356a56: supportedParams keys=[vertex_ai/curated-model gpt-4o]
              candidates for bare "curated-model" = [curated-model vertex_ai/curated-model]
this branch:  supportedParams keys=[gpt-4o]
              candidates for bare "curated-model" = [curated-model]

Failure scenario: a caller sends gen4.5, ResolveModelParameters and LoadModelCapabilities walk modelParameterCandidates, and runwayml/gen4.5 is no longer offered, so the row is not found. Impact today is small, since the 18 affected rows carry only mode, but the coupling is invisible from the changed line and it gets worse the first time a curated row carries server_tools, beta_headers or supports_cache_point without any of the eleven parameter-mapping flags.

Suggested fix, as a follow-up: give the suffix lookup its own key set populated for every parsed row in applyModelParameters, so name resolution and the compat allowlist stop sharing one map.

2. Capability absent versus capability explicitly false

The contract downstream is two-state: nil means "unknown, do not drop", non-nil means "authoritative allowlist" (plugins/compat/main.go:147). A row that says {"supports_function_calling": false} and nothing else is making a statement about the parameter surface. On dev it produced a one-element allowlist, so tools was dropped. On this branch it produces an empty list, is not indexed, GetSupportedParameters returns nil, and compat skips dropping entirely, so tools now reaches a model whose row explicitly says it has no function calling. That is the same conflation this PR is fixing, pointed the other way.

Does this flip behaviour for existing datasheet rows? For the public feed, no: all 13 explicit-false rows also declare true flags, so they keep a non-empty allowlist and behave exactly as before. The rows that do flip are the 5 capability-empty ones (the intended fix) and the 18 mode-only ones (finding 1). But model_parameters_url is user-configurable, and hand-written self-hosted feeds are exactly where a bare "supports X: false" row shows up.

The discriminator that separates all three cases cleanly is not "any capability field" and not "any parameter produced", but "did the row say anything about the request-parameter surface":

declaredParamSurface := len(parsed.ModelParameters) > 0 ||
	parsed.SupportsAssistantPrefill != nil ||
	parsed.SupportsFunctionCalling != nil ||
	parsed.SupportsParallelFunctionCalling != nil ||
	parsed.SupportsToolChoice != nil ||
	parsed.SupportsReasoning != nil ||
	parsed.SupportsResponseSchema != nil ||
	parsed.SupportsNoneReasoningEffort != nil ||
	parsed.SupportsServiceTier != nil ||
	parsed.SupportsPromptCaching != nil ||
	parsed.SupportsWebSearch != nil

Then } else if declaredParamSurface {. Cost-only, deprecation-only and mode-only rows say nothing about parameters, so they stay unknown, which is the fix you want. Explicit-false rows keep their authoritative allowlist. Populated rows keep the #4630 default. If you take this, finding 3 resolves too, because the comment then describes the code.

3 and 4: one-line comment nits

The comment at line 542 promises a capability check and the code at line 555 does a parameter-name check. Either adjust the wording or adopt the predicate above. And since the gate reads the accumulated supported, it silently depends on staying the last block in extractSupportedParams. A short "keep this block last" note would save the next person a debugging session.

addParam("reasoning_with_tool_calls")
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

return supported
}

Expand Down