diff --git a/framework/modelcatalog/datasheet/costonlyparams_test.go b/framework/modelcatalog/datasheet/costonlyparams_test.go new file mode 100644 index 00000000000..c9b1bf58d77 --- /dev/null +++ b/framework/modelcatalog/datasheet/costonlyparams_test.go @@ -0,0 +1,119 @@ +package datasheet + +import ( + "encoding/json" + "slices" + "testing" + + "github.com/maximhq/bifrost/core/schemas" +) + +// Regression test for issue #6276: a datasheet row that declares ONLY pricing (no +// capability flags, no model_parameters) must not produce a supported-parameters +// allowlist. The default "reasoning_with_tool_calls" marker used to be added even to +// capability-empty rows, turning them into a one-element allowlist; the compat plugin +// (should_drop_params) then stripped tools / tool_choice / temperature / everything +// else from requests to that model. +func TestCostOnlyRowProducesNoParamAllowlist(t *testing.T) { + // Hosted-datasheet shape reported in the issue: pricing keys only, none of + // which map to ModelCapabilities fields. The issue's original row + // (gemini-3.6-flash) has since been fixed upstream; claude-opus-4-7-20260416 + // ({"deprecation_date": "2027-04-16"}) still reproduces on the live feed and + // parses to the same empty ModelCapabilities. + costOnlyRow := json.RawMessage(`{ + "input_cost_per_token_batches": 0.000000075, + "output_cost_per_token_batches": 0.0000003 + }`) + + t.Run("extractSupportedParams on a capability-empty row", func(t *testing.T) { + var caps schemas.ModelCapabilities + if err := json.Unmarshal(costOnlyRow, &caps); err != nil { + t.Fatalf("unmarshal cost-only row: %v", err) + } + if !IsEmptyModelCapabilities(&caps) { + t.Fatalf("precondition: cost-only row should parse to empty ModelCapabilities") + } + if got := extractSupportedParams(&caps); len(got) != 0 { + t.Errorf("extractSupportedParams(cost-only row) = %v, want empty — a row declaring no request parameters must not synthesize an allowlist", got) + } + }) + + t.Run("applyModelParameters end-to-end", func(t *testing.T) { + s := &Store{} + // A second, genuinely populated row keeps applied > 0 so the index swap + // happens, mirroring the real datasheet where thousands of populated + // rows sit alongside the cost-only gemini-3.6-flash row. + s.applyModelParameters(map[string]json.RawMessage{ + "claude-opus-4-7-20260416": costOnlyRow, + "gpt-4o": json.RawMessage(`{"supports_function_calling":true,"supports_tool_choice":true}`), + }) + + if got := s.GetSupportedParameters("claude-opus-4-7-20260416"); got != nil { + t.Errorf("GetSupportedParameters(claude-opus-4-7-20260416) = %v, want nil (unknown) — compat treats any non-nil list as a complete allowlist and drops tools/tool_choice", got) + } + }) + + t.Run("explicit-false-only row keeps an authoritative allowlist", func(t *testing.T) { + // A row saying only {"supports_function_calling": false} IS a statement + // about the parameter surface: it must keep a non-nil allowlist (with the + // default marker) so compat still drops tools, instead of degrading to + // "unknown, do not drop". + var caps schemas.ModelCapabilities + if err := json.Unmarshal(json.RawMessage(`{"supports_function_calling":false}`), &caps); err != nil { + t.Fatalf("unmarshal explicit-false-only row: %v", err) + } + got := extractSupportedParams(&caps) + if len(got) == 0 { + t.Fatalf("extractSupportedParams(explicit-false-only row) = empty, want the default reasoning_with_tool_calls marker so the allowlist stays authoritative") + } + if !slices.Contains(got, "reasoning_with_tool_calls") { + t.Errorf("extractSupportedParams(explicit-false-only row) = %v, want reasoning_with_tool_calls present", got) + } + if slices.Contains(got, "tools") { + t.Errorf("extractSupportedParams(explicit-false-only row) = %v, want tools absent", got) + } + }) + + t.Run("populated row keeps the default marker", func(t *testing.T) { + // Backward-compat contract from #4630: rows declaring real capabilities but + // lacking the reasoning_with_tool_calls flag still get the default marker. + var caps schemas.ModelCapabilities + if err := json.Unmarshal(json.RawMessage(`{"supports_function_calling":true}`), &caps); err != nil { + t.Fatalf("unmarshal populated row: %v", err) + } + got := extractSupportedParams(&caps) + if !slices.Contains(got, "reasoning_with_tool_calls") { + t.Errorf("extractSupportedParams(populated row) = %v, want it to include the default reasoning_with_tool_calls marker", got) + } + if !slices.Contains(got, "tools") { + t.Errorf("extractSupportedParams(populated row) = %v, want it to include tools", got) + } + }) + + t.Run("explicit flag is honored regardless of other capabilities", func(t *testing.T) { + var explicitTrue schemas.ModelCapabilities + if err := json.Unmarshal(json.RawMessage(`{"supports_reasoning_with_tool_calls":true}`), &explicitTrue); err != nil { + t.Fatalf("unmarshal explicit-true row: %v", err) + } + if got := extractSupportedParams(&explicitTrue); !slices.Contains(got, "reasoning_with_tool_calls") { + t.Errorf("extractSupportedParams(explicit true) = %v, want reasoning_with_tool_calls present", got) + } + + var explicitFalse schemas.ModelCapabilities + if err := json.Unmarshal(json.RawMessage(`{"supports_function_calling":true,"supports_reasoning_with_tool_calls":false}`), &explicitFalse); err != nil { + t.Fatalf("unmarshal explicit-false row: %v", err) + } + if got := extractSupportedParams(&explicitFalse); slices.Contains(got, "reasoning_with_tool_calls") { + t.Errorf("extractSupportedParams(explicit false) = %v, want reasoning_with_tool_calls absent", got) + } + + // Explicit false must also override a marker sourced from model_parameters ids. + var falseWithParamID schemas.ModelCapabilities + if err := json.Unmarshal(json.RawMessage(`{"model_parameters":[{"id":"reasoning_with_tool_calls"}],"supports_reasoning_with_tool_calls":false}`), &falseWithParamID); err != nil { + t.Fatalf("unmarshal explicit-false row with param id: %v", err) + } + if got := extractSupportedParams(&falseWithParamID); slices.Contains(got, "reasoning_with_tool_calls") { + t.Errorf("extractSupportedParams(explicit false + model_parameters id) = %v, want reasoning_with_tool_calls absent", got) + } + }) +} diff --git a/framework/modelcatalog/datasheet/types.go b/framework/modelcatalog/datasheet/types.go index a1aacfc525b..c720f21d101 100644 --- a/framework/modelcatalog/datasheet/types.go +++ b/framework/modelcatalog/datasheet/types.go @@ -515,9 +515,6 @@ func extractSupportedParams(parsed *schemas.ModelCapabilities) []string { if parsed.SupportsReasoning != nil && *parsed.SupportsReasoning { addParam("reasoning") } - if parsed.SupportsReasoningWithToolCalls == nil || *parsed.SupportsReasoningWithToolCalls { - addParam("reasoning_with_tool_calls") - } if parsed.SupportsNoneReasoningEffort != nil && *parsed.SupportsNoneReasoningEffort { addParam("supports_none_reasoning_effort") } @@ -539,6 +536,46 @@ func extractSupportedParams(parsed *schemas.ModelCapabilities) []string { addParam("web_search_options") } + // declaredParamSurface reports whether the row said anything at all about the + // request-parameter surface, including explicit "supports X: false". That is a + // different set from "produced a parameter name above": a row carrying only + // explicit-false flags produces no names but IS making an authoritative + // statement, and must keep a (possibly empty-of-tools) allowlist rather than + // degrade to "unknown, do not drop". + declaredParamSurface := len(parsed.ModelParameters) > 0 || + parsed.SupportsAssistantPrefill != nil || + parsed.SupportsFunctionCalling != nil || + parsed.SupportsParallelFunctionCalling != nil || + parsed.SupportsToolChoice != nil || + parsed.SupportsReasoning != nil || + parsed.SupportsResponseSchema != nil || + parsed.SupportsNoneReasoningEffort != nil || + parsed.SupportsServiceTier != nil || + parsed.SupportsPromptCaching != nil || + parsed.SupportsWebSearch != nil + + // reasoning_with_tool_calls defaults on for rows that lack the flag, so models + // missing it don't get reasoning stripped when tools are present. Restrict the + // default to rows that declared a parameter surface: for a row that said + // nothing about parameters (cost-only, deprecation-only, mode-only) the marker + // alone would otherwise become a one-element allowlist, and compat with + // should_drop_params would strip every other parameter (tools, tool_choice, + // ...) from requests. This block must stay last in the function: the explicit + // false below removes the marker regardless of how it was added above. + if parsed.SupportsReasoningWithToolCalls != nil { + if *parsed.SupportsReasoningWithToolCalls { + addParam("reasoning_with_tool_calls") + } else { + // Explicit false wins even when a model_parameters id already added + // the marker via the default case above. + supported = slices.DeleteFunc(supported, func(p string) bool { + return p == "reasoning_with_tool_calls" + }) + } + } else if declaredParamSurface { + addParam("reasoning_with_tool_calls") + } + return supported }