diff --git a/.github/workflows/model-capability-regen-diff.yml b/.github/workflows/model-capability-regen-diff.yml index adf0a3bb2b7..7d8d706c91f 100644 --- a/.github/workflows/model-capability-regen-diff.yml +++ b/.github/workflows/model-capability-regen-diff.yml @@ -9,6 +9,14 @@ on: - 'desktop/src/features/agents/ui/modelCapabilities.ts' - 'scripts/generated-model-capabilities-coverage.json' - '.github/workflows/model-capability-regen-diff.yml' + # Differential harness and fixtures — any change to old/new side or inputs re-runs. + - 'scripts/run-differential.mjs' + - 'scripts/normative-corpus.json' + - 'scripts/catalog-sample-fixture.json' + - 'desktop/src/features/agents/ui/effortTable.fixture.json' + - 'desktop/src/features/agents/ui/buzzAgentConfig.ts' + - 'crates/buzz-agent/src/config.rs' + - 'crates/buzz-agent/src/llm.rs' push: branches: [main, release, 'duncan/databricks-model-label-registry'] @@ -51,3 +59,19 @@ jobs: - name: Validate manifest (schema-negative tests) run: node --test scripts/test-manifest-validator.mjs + + - name: Run differential harness (old vs new, all input sets) + run: node --experimental-strip-types scripts/run-differential.mjs + + rust-unit-tests: + name: buzz-agent unit tests (normative corpus + behavioral differential) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Run buzz-agent unit tests + run: cargo test -p buzz-agent --lib diff --git a/crates/buzz-agent/src/catalog.rs b/crates/buzz-agent/src/catalog.rs index 659cbd76fdb..d9ba116327d 100644 --- a/crates/buzz-agent/src/catalog.rs +++ b/crates/buzz-agent/src/catalog.rs @@ -47,8 +47,10 @@ pub struct ModelEntry { /// Known Databricks AI Gateway v2 models — used as a fallback when the /// `api/ai-gateway/v2/endpoints` call returns an empty list. /// Mirrors goose's `DATABRICKS_V2_KNOWN_MODELS`. -pub const DATABRICKS_V2_KNOWN_MODELS: &[&str] = - &["databricks-gpt-5-5", "databricks-claude-opus-4-7"]; +/// +/// Phase 2 cutover: this is now a re-export of the generated constant in +/// `generated_model_capabilities`. Phase 3 removes the old hand-maintained list. +pub use crate::generated_model_capabilities::DATABRICKS_V2_KNOWN_MODELS; /// Returns the discovery-failure fallback catalog for a Databricks provider. /// diff --git a/crates/buzz-agent/src/config.rs b/crates/buzz-agent/src/config.rs index afbda5379d4..f4a033cbc92 100644 --- a/crates/buzz-agent/src/config.rs +++ b/crates/buzz-agent/src/config.rs @@ -86,7 +86,7 @@ impl ThinkingEffort { /// - `llama-3` → `llama-3` (no family token, returned unchanged) /// /// If no family token is present the name is returned unchanged. -fn strip_catalog_prefix(model: &str) -> &str { +pub(crate) fn strip_catalog_prefix(model: &str) -> &str { const FAMILY_TOKENS: &[&str] = &["claude-", "gpt-"]; let lower = model.to_ascii_lowercase(); let first_idx = FAMILY_TOKENS.iter().filter_map(|tok| lower.find(tok)).min(); @@ -576,6 +576,195 @@ pub fn normalize_effort_for_anthropic_route(effort: ThinkingEffort) -> Option ThinkingEffort { + use crate::generated_model_capabilities::resolve_model_capabilities; + let cap = resolve_model_capabilities(provider, raw_model); + resolve_openai_effort(raw_model, effort, cap.supported_efforts.as_ref()) +} + +/// Normalize the effort value for a DatabricksV2 request. +/// +/// Reads `normalization_policy` from the generated capability record for this +/// raw model ID (provider = "databricks_v2") and applies it: +/// - `OpenAiStandard` → per-family table lookup (GPT-5.x, etc.) +/// - `OpenAiClampMaxToXHigh` → clamp max → xhigh, pass others unchanged +/// - `None` → pass effort through unchanged (Anthropic path) +/// +/// This is the production authority for DatabricksV2 effort normalization. +/// `normalize_effort_for_provider` is the authority for pure OpenAI and legacy +/// Databricks; `normalize_effort_for_openai_route` is a test/differential shim only. +pub fn normalize_effort_for_databricks_v2( + effort: ThinkingEffort, + raw_model: &str, +) -> ThinkingEffort { + use crate::generated_model_capabilities::{resolve_model_capabilities, NormalizationPolicy}; + let cap = resolve_model_capabilities("databricks_v2", raw_model); + match cap.normalization_policy { + NormalizationPolicy::OpenAiStandard => { + // Resolve against the generated `supported_efforts` — this is the axis that + // carries exact-record corrections (e.g. databricks-gpt-5-5 → [low,medium,high]). + // Uses the same clamping/peer-fallback semantics as the old hand-table lookup. + resolve_openai_effort(raw_model, effort, cap.supported_efforts.as_ref()) + } + NormalizationPolicy::OpenAiClampMaxToXHigh => { + // Only `max` is out-of-range; all other values pass through if supported. + // Resolve against supported_efforts so that unsupported values are clamped + // consistently (not just `max`). + if effort == ThinkingEffort::Max { + tracing::warn!( + requested = "max", + resolved = "xhigh", + model = raw_model, + "BUZZ_AGENT_THINKING_EFFORT=max not confirmed for this DatabricksV2 model; clamping to xhigh" + ); + ThinkingEffort::XHigh + } else { + resolve_openai_effort(raw_model, effort, cap.supported_efforts.as_ref()) + } + } + NormalizationPolicy::None => effort, + } +} + +/// Build the Anthropic thinking/effort request fields for any manifest-owned provider/model. +/// +/// Resolves `thinking_mode` and `supported_efforts` from the generated capability record +/// for the effective provider/model and applies them: +/// - `ManualBudget` → `thinking:{type:"enabled", budget_tokens}` shape +/// - `Adaptive` → `thinking:{type:"adaptive"} + output_config:{effort}` shape, +/// with effort clamped down to the highest supported level +/// - `OmitFields` / `None` / `NotApplicable` → omit both fields +/// +/// This is the single production authority for all providers' Anthropic thinking. +/// The old `anthropic_thinking_config_for_databricks_v2` is a test-only shim. +pub fn anthropic_thinking_config_generated( + provider: &str, + raw_model: &str, + effort: ThinkingEffort, + max_output_tokens: u32, +) -> (Option, Option) { + use crate::generated_model_capabilities::{resolve_model_capabilities, ThinkingMode}; + use serde_json::json; + + let cap = resolve_model_capabilities(provider, raw_model); + match cap.thinking_mode { + ThinkingMode::ManualBudget => { + // Manual-budget shape (claude-3*, claude-opus-4-5): budget_tokens clamped + // to fit within max_output_tokens. + const MIN_ANSWER_TOKENS: u32 = 1024; + let level_budget = effort.anthropic_budget_tokens(); + let headroom = max_output_tokens.saturating_sub(MIN_ANSWER_TOKENS); + let budget = level_budget.min(headroom); + if budget < MIN_ANSWER_TOKENS { + tracing::warn!( + max_output_tokens, + level_budget, + headroom, + model = raw_model, + "BUZZ_AGENT_THINKING_EFFORT: max_output_tokens too small to fit thinking budget + answer headroom; omitting thinking fields" + ); + return (None, None); + } + ( + Some(json!({ "type": "enabled", "budget_tokens": budget })), + None, + ) + } + ThinkingMode::Adaptive => { + // Adaptive shape: clamp effort downward to the highest supported level. + // Uses the generated supported_efforts (the manifest-owned authority) rather + // than the legacy clamp_adaptive_effort hand table. + let clamped = cap + .supported_efforts + .iter() + .rev() + .find(|&&e| e <= effort) + .copied() + .unwrap_or(effort); // effort is below the lowest supported; pass through (rare) + if clamped != effort { + tracing::warn!( + model = raw_model, + requested = effort.openai_effort_str(), + clamped = clamped.openai_effort_str(), + "BUZZ_AGENT_THINKING_EFFORT is not available for this model; clamping to highest supported level" + ); + } + ( + Some(json!({ "type": "adaptive" })), + Some(json!({ "effort": clamped.anthropic_effort_str() })), + ) + } + ThinkingMode::OmitFields | ThinkingMode::None | ThinkingMode::NotApplicable => { + // Unknown Anthropic model, non-Anthropic-routed, or not applicable: + // omit thinking fields rather than guess. + (None, None) + } + } +} + +/// Old DatabricksV2-scoped Anthropic thinking config — kept as a differential shim. +/// +/// Production code uses `anthropic_thinking_config_generated` instead. +/// This hard-codes `"databricks_v2"` and uses the legacy `clamp_adaptive_effort` hand table. +#[cfg(test)] +pub(crate) fn _old_anthropic_thinking_config_for_databricks_v2( + raw_model: &str, + effort: ThinkingEffort, + max_output_tokens: u32, +) -> (Option, Option) { + use crate::generated_model_capabilities::{resolve_model_capabilities, ThinkingMode}; + use serde_json::json; + + match resolve_model_capabilities("databricks_v2", raw_model).thinking_mode { + ThinkingMode::ManualBudget => { + const MIN_ANSWER_TOKENS: u32 = 1024; + let level_budget = effort.anthropic_budget_tokens(); + let headroom = max_output_tokens.saturating_sub(MIN_ANSWER_TOKENS); + let budget = level_budget.min(headroom); + if budget < MIN_ANSWER_TOKENS { + return (None, None); + } + ( + Some(json!({ "type": "enabled", "budget_tokens": budget })), + None, + ) + } + ThinkingMode::Adaptive => { + let model = strip_catalog_prefix(raw_model); + let clamped = clamp_adaptive_effort(model, effort); + ( + Some(json!({ "type": "adaptive" })), + Some(json!({ "effort": clamped.anthropic_effort_str() })), + ) + } + ThinkingMode::OmitFields | ThinkingMode::None | ThinkingMode::NotApplicable => (None, None), + } +} + /// Returns true for Claude model families that use manual thinking budgets (doc-verified, July 2025). /// /// Source: https://platform.claude.com/docs/en/build-with-claude/extended-thinking (support table) @@ -1151,6 +1340,32 @@ fn parse_hook_servers(raw: Option<&str>) -> HookServers { HookServers::Only(names) } +// --------------------------------------------------------------------------- +// Test-only re-exports: let llm.rs tests call private classifiers without +// duplicating them. These wrappers are cfg(test)-only and intentionally thin. +// --------------------------------------------------------------------------- + +#[cfg(test)] +pub(crate) fn is_manual_budget_model_for_test(model: &str) -> bool { + is_manual_budget_model(model) +} + +#[cfg(test)] +pub(crate) fn is_adaptive_thinking_model_for_test(model: &str) -> bool { + is_adaptive_thinking_model(model) +} + +/// Mirror of the `tests::valid_effort_values_for_provider_model` helper in config's +/// own test module, promoted to a module-level cfg(test) function so llm.rs tests +/// can call it without re-implementing the logic. +#[cfg(test)] +pub(crate) fn valid_effort_values_for_provider_model_for_test( + provider: &str, + model: &str, +) -> (Vec<&'static str>, Option<&'static str>) { + tests::valid_effort_values_for_provider_model(provider, model) +} + #[cfg(test)] mod tests { use super::*; @@ -2604,7 +2819,7 @@ mod tests { /// Returns `(valid_values, default_value)` where `default_value` is `None` /// for Anthropic manual-budget models (TS `defaultValue: null`), otherwise /// `Some("medium")` or `Some("high")`. - fn valid_effort_values_for_provider_model( + pub(super) fn valid_effort_values_for_provider_model( provider: &str, model: &str, ) -> (Vec<&'static str>, Option<&'static str>) { @@ -2614,6 +2829,13 @@ mod tests { const GPT5_1: &[&str] = &["none", "low", "medium", "high"]; let p = provider.to_ascii_lowercase(); + // Canonicalize provider aliases — mirrors the production path and TS + // PROVIDER_ALIASES so this shim stays in sync with the fixture. + let p = match p.as_str() { + "openai-compat" => "openai".to_owned(), + "databricks-v2" => "databricks_v2".to_owned(), + _ => p, + }; // Strip arbitrary endpoint-naming prefix before model matching, mirroring TS and // strip_catalog_prefix: find the first known family token (claude-, gpt-) and // drop everything before it. Handles any catalog naming convention. @@ -2698,7 +2920,7 @@ mod tests { if p == "openrouter" { return (ALL_7.to_vec(), Some("medium")); } - // openai-compat, unknown, empty → all-7, default medium. + // Unknown/empty provider → all-7, default medium. (ALL_7.to_vec(), Some("medium")) } @@ -2750,6 +2972,59 @@ mod tests { } } + // ---- normalize_effort_for_databricks_v2 regression tests (F1 corrections) ---- + // These pin the exact behavior Paul's pre-review probes checked. The key invariant: + // normalize_effort_for_databricks_v2 must resolve against the generated supported_efforts + // (which carries exact-record F1 corrections), NOT the old hand table. + + #[test] + fn normalize_effort_for_databricks_v2_gpt_5_5_xhigh_clamps_to_high() { + // F1 correction: databricks-gpt-5-5 generated supported_efforts = [low, medium, high]. + // XHigh is outside the supported set → nearest supported is High. + assert_eq!( + normalize_effort_for_databricks_v2(ThinkingEffort::XHigh, "databricks-gpt-5-5"), + ThinkingEffort::High, + "databricks-gpt-5-5 XHigh must clamp to High (F1 correction: supported=[low,medium,high])" + ); + } + + #[test] + fn normalize_effort_for_databricks_v2_gpt_5_5_none_clamps_to_low() { + // F1 correction: databricks-gpt-5-5 supported_efforts = [low, medium, high]. + // None is outside the set → nearest supported is Low. + assert_eq!( + normalize_effort_for_databricks_v2(ThinkingEffort::None, "databricks-gpt-5-5"), + ThinkingEffort::Low, + "databricks-gpt-5-5 None must clamp to Low (F1 correction: supported=[low,medium,high])" + ); + } + + #[test] + fn normalize_effort_for_databricks_v2_gpt_5_5_in_range_passes_through() { + // Values within the corrected set must pass through unchanged. + for effort in [ + ThinkingEffort::Low, + ThinkingEffort::Medium, + ThinkingEffort::High, + ] { + assert_eq!( + normalize_effort_for_databricks_v2(effort, "databricks-gpt-5-5"), + effort, + "databricks-gpt-5-5 {effort:?} is in supported set, must pass through" + ); + } + } + + #[test] + fn normalize_effort_for_databricks_v2_gpt_5_6_sol_max_passes_through() { + // databricks-gpt-5-6-sol F1 adoption: [low, medium, high, max] — max is supported. + assert_eq!( + normalize_effort_for_databricks_v2(ThinkingEffort::Max, "databricks-gpt-5-6-sol"), + ThinkingEffort::Max, + "databricks-gpt-5-6-sol Max must pass through (F1: supported includes max)" + ); + } + #[test] fn resolve_provider_openrouter_with_key() { assert_eq!( diff --git a/crates/buzz-agent/src/generated_model_capabilities.rs b/crates/buzz-agent/src/generated_model_capabilities.rs index 3bbccda619e..9f238e79ea6 100644 --- a/crates/buzz-agent/src/generated_model_capabilities.rs +++ b/crates/buzz-agent/src/generated_model_capabilities.rs @@ -253,6 +253,19 @@ pub fn lookup_by_family_rules(provider: &str, normalized: &str) -> Option Option Option Option Option Option "openai", + "databricks-v2" => "databricks_v2", + other => other, + }; + let result = resolve_model_capabilities(canonical_provider, raw_model_id); ran += 1; // Check thinking_mode if present in expect diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index 73c7e1faf2e..b8b438d84bf 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -9,8 +9,9 @@ use tokio::time::Instant; use crate::auth::{PkceOAuthConfig, PkceOAuthTokenSource, StaticTokenSource, TokenSource}; use crate::config::{ - is_openai_host, normalize_effort_for_anthropic_route, normalize_effort_for_openai_route, - Config, OpenAiApi, Provider, ThinkingEffort, + anthropic_thinking_config_generated, is_openai_host, normalize_effort_for_anthropic_route, + normalize_effort_for_databricks_v2, normalize_effort_for_provider, Config, OpenAiApi, Provider, + ThinkingEffort, }; use crate::types::{ AgentError, HistoryItem, LlmResponse, ProviderStop, ToolCall, ToolDef, ToolResultContent, @@ -141,6 +142,7 @@ impl Llm { tools, effective_model, effort, + "anthropic", ), ) .await?; @@ -159,17 +161,23 @@ impl Llm { parse_openai_with_reasoning_details(v) } Provider::OpenAi | Provider::Databricks => { + let provider_str = match cfg.provider { + Provider::OpenAi => "openai", + Provider::Databricks => "databricks", + _ => unreachable!(), + }; self.openai_request( cfg, effective_model, !tools.is_empty(), |use_responses, request_model| { - // Normalize effort for model-specific availability. Startup no longer rejects - // `max` for pure OpenAI/Databricks; this per-model table is the single authority - // — it keeps `max` for gpt-5.6, clamps `max`→`xhigh` for other OpenAI-shaped - // models, and still applies corrections like none→minimal on the gpt-5 base. - let e = - effort.map(|ef| normalize_effort_for_openai_route(ef, request_model)); + // Normalize effort via the generated manifest — resolves the + // actual provider/model record and applies resolve_openai_effort + // over its supported_efforts. Adopted F1 corrections (e.g. + // databricks-gpt-5-5 → [low,medium,high]) are enforced here. + let e = effort.map(|ef| { + normalize_effort_for_provider(provider_str, request_model, ef) + }); if use_responses { ( responses_body( @@ -195,9 +203,9 @@ impl Llm { Provider::DatabricksV2 => { self.databricks_v2_request(cfg, effective_model, |route| match route { DatabricksV2Route::OpenAiResponses => { - // OpenAI Responses path: normalize effort against the per-model table. - let e = - effort.map(|ef| normalize_effort_for_openai_route(ef, effective_model)); + // OpenAI Responses path: normalize effort via manifest normalization_policy. + let e = effort + .map(|ef| normalize_effort_for_databricks_v2(ef, effective_model)); ( responses_body(cfg, system_prompt, history, tools, effective_model, e), parse_responses as OpenAiParse, @@ -207,14 +215,22 @@ impl Llm { // Anthropic Messages path: normalize effort (none|minimal → omit). let e = effort.and_then(normalize_effort_for_anthropic_route); ( - anthropic_body(cfg, system_prompt, history, tools, effective_model, e), + anthropic_body( + cfg, + system_prompt, + history, + tools, + effective_model, + e, + "databricks_v2", + ), parse_anthropic as OpenAiParse, ) } DatabricksV2Route::MlflowChatCompletions => { - // MLflow Chat path (OpenAI-shaped): normalize effort against the per-model table. - let e = - effort.map(|ef| normalize_effort_for_openai_route(ef, effective_model)); + // MLflow Chat path (OpenAI-shaped): normalize effort via manifest. + let e = effort + .map(|ef| normalize_effort_for_databricks_v2(ef, effective_model)); ( openai_body(cfg, system_prompt, history, tools, effective_model, e), parse_openai as OpenAiParse, @@ -728,6 +744,7 @@ fn anthropic_body( tools: &[ToolDef], effective_model: &str, effort: Option, + provider: &str, ) -> Value { let mut messages: Vec = Vec::new(); let mut pending: Vec = Vec::new(); @@ -799,8 +816,12 @@ fn anthropic_body( let mut body = json!({ "model": effective_model, "max_tokens": cfg.max_output_tokens, "system": system_value, "messages": messages }); if let Some(e) = effort { - let (thinking, output_config) = - crate::config::anthropic_thinking_config(effective_model, e, cfg.max_output_tokens); + let (thinking, output_config) = anthropic_thinking_config_generated( + provider, + effective_model, + e, + cfg.max_output_tokens, + ); if let Some(t) = thinking { body["thinking"] = t; } @@ -1095,25 +1116,24 @@ fn is_responses_required_error(body: &str) -> bool { || b.contains("use the responses api") } -/// OpenAI-family code names that appear as their own segment in a Databricks v2 -/// endpoint name (the GPT-5 launch aliases). The `gpt` family itself is matched -/// separately by segment prefix so `gpt`, `gpt5`, and the `gpt` of a split -/// `gpt-5` all qualify. -const DATABRICKS_V2_OPENAI_CODE_NAMES: &[&str] = &["sol", "luna", "terra"]; - -/// Anthropic (Claude) family and release code names that appear as their own -/// segment in a Databricks v2 endpoint name — the `claude` prefix, the family -/// names (`opus`, `sonnet`, `haiku`), and the release code names (`mythos`, -/// `fable`). Getting a Claude model onto the Anthropic Messages route is what -/// lets it carry a `cache_control` breakpoint; an endpoint that matches none of -/// these falls through to the MLflow (OpenAI-wire) path, where Anthropic prompt -/// caching is structurally impossible and the discount is silently lost. -const DATABRICKS_V2_CLAUDE_NAMES: &[&str] = +/// OpenAI-family code names used by the OLD segment-based route classifier. +/// Preserved for the Phase-2 differential harness and Phase-3 cleanup. +/// Production routing now delegates to `resolve_model_capabilities` (see +/// `databricks_v2_route_for_model` below). +#[cfg(test)] +const _OLD_DATABRICKS_V2_OPENAI_CODE_NAMES: &[&str] = &["sol", "luna", "terra"]; + +/// Anthropic (Claude) family and release code names used by the OLD classifier. +/// Preserved for the Phase-2 differential harness and Phase-3 cleanup. +#[cfg(test)] +const _OLD_DATABRICKS_V2_CLAUDE_NAMES: &[&str] = &["claude", "opus", "sonnet", "haiku", "mythos", "fable"]; /// Split a Databricks v2 endpoint name into its lowercase alphanumeric segments, /// breaking on any non-alphanumeric delimiter (`-`, `_`, `.`, `/`, …). E.g. /// `Databricks-Claude-Opus-5` -> `["databricks", "claude", "opus", "5"]`. +/// Used by the old classifier (differential harness). Phase 3 removes this. +#[cfg(test)] fn model_name_segments(model: &str) -> Vec { model .split(|c: char| !c.is_ascii_alphanumeric()) @@ -1122,32 +1142,48 @@ fn model_name_segments(model: &str) -> Vec { .collect() } -fn databricks_v2_route_for_model(model: &str) -> DatabricksV2Route { - // The v2 catalog exposes no family field, so the wire format is inferred - // from the endpoint name. Discovery deliberately keeps arbitrary custom - // aliases, so we match whole name *segments* rather than raw substrings: a - // substring test would misroute unrelated names — `consolidated-llama` - // (`sol`), `terraform-coder` (`terra`), `corpus-reranker`/`octopus-model` - // (`opus`) — onto a wire whose request shape their backend can't parse, - // turning a caching optimization into a hard request/parse failure. Segment - // matching still accepts real prefixed names like `goose-opus-5`. +/// OLD segment-based route classifier — preserved for the Phase-2 differential +/// harness. Production routing now delegates to `databricks_v2_route_for_model`. +/// Phase 3 removes this function. +#[cfg(test)] +fn _old_databricks_v2_route_for_model(model: &str) -> DatabricksV2Route { let segments = model_name_segments(model); let has_named_segment = |names: &[&str]| segments.iter().any(|seg| names.contains(&seg.as_str())); - // `gpt` family: any segment beginning with `gpt` — covers `gpt`, `gpt5`, and - // the `gpt` segment of a split `gpt-5`, without matching mid-word. let is_gpt_family = segments.iter().any(|seg| seg.starts_with("gpt")); - // OpenAI is checked before Claude so a name carrying both markers resolves - // to the OpenAI wire (preserving the prior `gpt-5`-first precedence). - if is_gpt_family || has_named_segment(DATABRICKS_V2_OPENAI_CODE_NAMES) { + if is_gpt_family || has_named_segment(_OLD_DATABRICKS_V2_OPENAI_CODE_NAMES) { DatabricksV2Route::OpenAiResponses - } else if has_named_segment(DATABRICKS_V2_CLAUDE_NAMES) { + } else if has_named_segment(_OLD_DATABRICKS_V2_CLAUDE_NAMES) { DatabricksV2Route::AnthropicMessages } else { DatabricksV2Route::MlflowChatCompletions } } +/// Returns the Databricks v2 wire route for a model name. +/// +/// Phase 2 cutover: delegates to `resolve_model_capabilities` from the generated +/// capability module. The generated resolver uses the same segment-based matching +/// logic, now derived from the manifest single source of truth. +/// +/// `RouteUnknown` (blank model) and `NotApplicable` (non-DBv2 provider) are not +/// reachable here — this function is only called for DBv2 requests with an +/// effective model string — both map to `MlflowChatCompletions` as a safe fallback. +fn databricks_v2_route_for_model(model: &str) -> DatabricksV2Route { + use crate::generated_model_capabilities::{ + resolve_model_capabilities, DatabricksV2Route as GenRoute, + }; + match resolve_model_capabilities("databricks_v2", model).databricks_v2_wire_route { + GenRoute::OpenAiResponses => DatabricksV2Route::OpenAiResponses, + GenRoute::AnthropicMessages => DatabricksV2Route::AnthropicMessages, + // RouteUnknown (blank model) and NotApplicable (non-DBv2) are structurally + // unreachable from this call site; fall through to the mlflow path. + GenRoute::MlflowChatCompletions | GenRoute::RouteUnknown | GenRoute::NotApplicable => { + DatabricksV2Route::MlflowChatCompletions + } + } +} + fn databricks_v2_path(route: DatabricksV2Route) -> &'static str { match route { DatabricksV2Route::OpenAiResponses => "/ai-gateway/openai/v1/responses", @@ -2995,6 +3031,7 @@ mod tests { &[], "model", None, + "anthropic", ); let content = &body["messages"][2]["content"][0]["content"]; assert_eq!(content[0]["type"], "text"); @@ -3336,6 +3373,871 @@ mod tests { } } + /// Phase-2 comprehensive differential: old hand-coded logic vs generated capability module, + /// covering all three normative input sets (effortTable.fixture.json, normative-corpus.json, + /// catalog-sample-fixture.json) and all axes the old Rust code owned: + /// - supported_efforts / default_effort + /// - databricks_v2_wire_route (databricks_v2 entries only) + /// - thinking_mode (Anthropic and Anthropic-routed DatabricksV2 entries) + /// + /// Allowlist is axis-scoped: each entry covers (provider, raw_model_id, axis). + /// Any declared allowlist entry that never suppresses a divergence is a stale entry + /// and causes the test to FAIL (mirrors JS harness semantics). + #[test] + fn comprehensive_differential_old_vs_new_all_inputs() { + use crate::config::{ + is_adaptive_thinking_model_for_test, is_manual_budget_model_for_test, + strip_catalog_prefix as config_strip_catalog_prefix, + valid_effort_values_for_provider_model_for_test, + }; + use crate::generated_model_capabilities::{ + resolve_model_capabilities, DatabricksV2Route as GenRoute, + ThinkingMode as GenThinkingMode, + }; + use std::collections::HashSet; + + // ----------------------------------------------------------------------- + // Axis-scoped allowlist: (provider, raw_model_id, axis) + // Each entry documents an intentional divergence from the old hand tables. + // ----------------------------------------------------------------------- + #[derive(Debug)] + struct AllowlistEntry { + provider: &'static str, + raw_model_id: &'static str, + axis: &'static str, + reason: &'static str, + } + let allowlist: &[AllowlistEntry] = &[ + // Phase 1 ADOPT: models.dev payload d5a4974c advertises [low,medium,high]; + // old code returns [none,low,medium,high,xhigh]. + AllowlistEntry { + provider: "databricks_v2", + raw_model_id: "databricks-gpt-5-5", + axis: "supported_efforts", + reason: "Phase 1 ADOPT: models.dev [low,medium,high]; old [none,low,medium,high,xhigh]", + }, + AllowlistEntry { + provider: "databricks_v2", + raw_model_id: "databricks-gpt-5-4-mini", + axis: "supported_efforts", + reason: "Phase 1 ADOPT: models.dev [low,medium,high]; old [none,low,medium,high,xhigh]", + }, + AllowlistEntry { + provider: "databricks_v2", + raw_model_id: "databricks-gpt-5-4-nano", + axis: "supported_efforts", + reason: "Phase 1 ADOPT: models.dev [low,medium,high]; old [none,low,medium,high,xhigh]", + }, + AllowlistEntry { + provider: "databricks_v2", + raw_model_id: "databricks-gpt-5-6-sol", + axis: "supported_efforts", + reason: "Phase 1 ADOPT: models.dev [low,medium,high,max]; old [none,low,medium,high,xhigh,max]", + }, + // Phase 1 correction: 'opus' is a named DBv2 segment → anthropic-messages route. + // Old config.rs effort table (pre-segment logic) classified goose-opus-5 as MLflow; + // old llm.rs segment classifier already routed it to AnthropicMessages. The + // manifest adopts the llm.rs (correct) view. Effort axis diverges because the old + // config.rs table assumed MLflow (openai-shaped), not Anthropic adaptive. + AllowlistEntry { + provider: "databricks_v2", + raw_model_id: "goose-opus-5", + axis: "supported_efforts", + reason: "Phase 1 F1: old config.rs rated it MLflow; manifest adopts anthropic adaptive", + }, + AllowlistEntry { + provider: "databricks_v2", + raw_model_id: "goose-opus-5", + axis: "default_effort", + reason: "Phase 1 F1: old config.rs had no default for this model; manifest adopts anthropic adaptive High", + }, + // Blank Anthropic model: manifest assumes adaptive (forward-compatible default); + // old is_adaptive_thinking_model("") and is_manual_budget_model("") both return false + // → OmitFields. The manifest's stance (adaptive fallback for blank provider) is + // intentional and matches the corpus expectation. + AllowlistEntry { + provider: "anthropic", + raw_model_id: "", + axis: "thinking_mode", + reason: "Manifest adopts adaptive fallback for blank Anthropic model; old code returns OmitFields", + }, + ]; + + // Track which allowlist entries are actually exercised. + let mut allowlist_hits: HashSet<(&str, &str, &str)> = HashSet::new(); + let mut divergences: Vec = Vec::new(); + + let is_allowlisted = |provider: &str, + model: &str, + axis: &str, + hits: &mut HashSet<(&str, &str, &str)>| { + for entry in allowlist { + if entry.provider == provider && entry.raw_model_id == model && entry.axis == axis { + hits.insert((entry.provider, entry.raw_model_id, entry.axis)); + return true; + } + } + false + }; + + // ----------------------------------------------------------------------- + // Derive "old" thinking_mode from hand-coded classifiers + // ----------------------------------------------------------------------- + let old_thinking_mode = |provider: &str, raw_model: &str, old_route: DatabricksV2Route| { + let is_anthropic_route = provider == "anthropic" + || (provider == "databricks_v2" + && old_route == DatabricksV2Route::AnthropicMessages); + if !is_anthropic_route { + return GenThinkingMode::None; + } + let model = config_strip_catalog_prefix(raw_model); + if is_manual_budget_model_for_test(model) { + GenThinkingMode::ManualBudget + } else if is_adaptive_thinking_model_for_test(model) { + GenThinkingMode::Adaptive + } else { + GenThinkingMode::OmitFields + } + }; + + // ----------------------------------------------------------------------- + // Per-entry check function + // ----------------------------------------------------------------------- + let mut check = |label: &str, + provider: &str, + raw_model: &str, + hits: &mut HashSet<(&str, &str, &str)>| { + // Canonicalize provider aliases so both sides of the differential + // operate on the same provider string (mirrors production and TS). + let provider = match provider { + "openai-compat" => "openai", + "databricks-v2" => "databricks_v2", + other => other, + }; + let new_cap = resolve_model_capabilities(provider, raw_model); + let (old_efforts, old_default) = + valid_effort_values_for_provider_model_for_test(provider, raw_model); + + // --- supported_efforts --- + let new_efforts: Vec<&'static str> = new_cap + .supported_efforts + .iter() + .map(|e| e.openai_effort_str()) + .collect(); + if new_efforts != old_efforts + && !is_allowlisted(provider, raw_model, "supported_efforts", hits) + { + divergences.push(format!( + "DIVERGE supported_efforts [{label}] provider={provider} model={raw_model:?}: old={old_efforts:?} new={new_efforts:?}" + )); + } + + // --- default_effort --- + let new_default: Option<&'static str> = + new_cap.default_effort.map(|e| e.openai_effort_str()); + if new_default != old_default + && !is_allowlisted(provider, raw_model, "default_effort", hits) + { + divergences.push(format!( + "DIVERGE default_effort [{label}] provider={provider} model={raw_model:?}: old={old_default:?} new={new_default:?}" + )); + } + + // --- databricks_v2_wire_route (databricks_v2 only) --- + if provider == "databricks_v2" { + let old_route = _old_databricks_v2_route_for_model(raw_model); + let new_route_gen = &new_cap.databricks_v2_wire_route; + let new_route = match new_route_gen { + GenRoute::OpenAiResponses => DatabricksV2Route::OpenAiResponses, + GenRoute::AnthropicMessages => DatabricksV2Route::AnthropicMessages, + GenRoute::MlflowChatCompletions + | GenRoute::RouteUnknown + | GenRoute::NotApplicable => DatabricksV2Route::MlflowChatCompletions, + }; + if new_route != old_route + && !is_allowlisted(provider, raw_model, "databricks_v2_wire_route", hits) + { + divergences.push(format!( + "DIVERGE databricks_v2_wire_route [{label}] model={raw_model:?}: old={old_route:?} new={new_route:?}" + )); + } + + // --- thinking_mode (databricks_v2 Anthropic-routed models) --- + let old_tm = old_thinking_mode(provider, raw_model, old_route); + if new_cap.thinking_mode != old_tm + && !is_allowlisted(provider, raw_model, "thinking_mode", hits) + { + divergences.push(format!( + "DIVERGE thinking_mode [{label}] provider={provider} model={raw_model:?}: old={old_tm:?} new={:?}", + new_cap.thinking_mode + )); + } + } else if provider == "anthropic" { + // thinking_mode for pure Anthropic + let old_tm = + old_thinking_mode(provider, raw_model, DatabricksV2Route::AnthropicMessages); + if new_cap.thinking_mode != old_tm + && !is_allowlisted(provider, raw_model, "thinking_mode", hits) + { + divergences.push(format!( + "DIVERGE thinking_mode [{label}] provider={provider} model={raw_model:?}: old={old_tm:?} new={:?}", + new_cap.thinking_mode + )); + } + } + }; + + // ----------------------------------------------------------------------- + // Input set 1: effortTable.fixture.json (36 entries) + // ----------------------------------------------------------------------- + #[derive(serde::Deserialize)] + struct FixtureEntry { + note: Option, + provider: String, + model: String, + } + let fixture_json = + include_str!("../../../desktop/src/features/agents/ui/effortTable.fixture.json"); + let fixture: Vec = + serde_json::from_str(fixture_json).expect("fixture must be valid JSON"); + for entry in &fixture { + let label = format!("fixture:{}", entry.note.as_deref().unwrap_or(&entry.model)); + check(&label, &entry.provider, &entry.model, &mut allowlist_hits); + } + + // ----------------------------------------------------------------------- + // Input set 2: normative-corpus.json (45 entries) + // ----------------------------------------------------------------------- + #[derive(serde::Deserialize)] + struct CorpusEntry { + // Group-header entries carry a `_group` string field; test-vector + // entries do not. We skip group headers (provider/raw_model_id absent). + #[serde(rename = "_group")] + group: Option, + id: Option, + provider: Option, + raw_model_id: Option, + } + let corpus_json = include_str!("../../../scripts/normative-corpus.json"); + let corpus: Vec = + serde_json::from_str(corpus_json).expect("corpus must be valid JSON"); + for entry in &corpus { + if entry.group.is_some() { + // Group-header row — skip. + continue; + } + let (Some(provider), Some(model)) = (&entry.provider, &entry.raw_model_id) else { + continue; + }; + let label = format!("corpus:{}", entry.id.as_deref().unwrap_or(model.as_str())); + check(&label, provider, model, &mut allowlist_hits); + } + + // ----------------------------------------------------------------------- + // Input set 3: catalog-sample-fixture.json (databricks_v2 only) + // ----------------------------------------------------------------------- + #[derive(serde::Deserialize)] + struct CatalogEntry { + name: String, + } + #[derive(serde::Deserialize)] + struct CatalogFixture { + endpoints: Vec, + } + let catalog_json = include_str!("../../../scripts/catalog-sample-fixture.json"); + let catalog: CatalogFixture = + serde_json::from_str(catalog_json).expect("catalog fixture must be valid JSON"); + for entry in &catalog.endpoints { + let label = format!("catalog:{}", entry.name); + check(&label, "databricks_v2", &entry.name, &mut allowlist_hits); + } + + // ----------------------------------------------------------------------- + // Stale allowlist entries — any declared entry that never fired is a bug + // ----------------------------------------------------------------------- + let mut stale: Vec = Vec::new(); + for entry in allowlist { + if !allowlist_hits.contains(&(entry.provider, entry.raw_model_id, entry.axis)) { + stale.push(format!( + "STALE_ALLOWLIST provider={} model={} axis={} reason={}", + entry.provider, entry.raw_model_id, entry.axis, entry.reason + )); + } + } + + let mut failures = divergences.clone(); + failures.extend(stale); + + assert!( + failures.is_empty(), + "Comprehensive differential found {} failure(s):\n{}", + failures.len(), + failures.join("\n") + ); + + // Report summary (visible with --nocapture). + let total_entries = fixture.len() + + corpus + .iter() + .filter(|e| e.group.is_none() && e.provider.is_some()) + .count() + + catalog.endpoints.len(); + println!( + "Comprehensive differential: {} input entries, {} allowlist slots exercised/{}, 0 unexpected divergences", + total_entries, + allowlist_hits.len(), + allowlist.len(), + ); + } + + /// Phase-2 behavioral differential: drives the actual production normalization + /// functions against the old shims over all committed inputs. + /// + /// This test catches the class of defect found at `305627e32`: a record-level + /// differential passes (the generated record is correct) while the production + /// function diverges (it delegates to the old hand table instead of the record). + /// + /// For every input that hits a provider with an OpenAI-shaped normalization policy + /// (databricks_v2 with OpenAiStandard / OpenAiClampMaxToXHigh), this test drives + /// `normalize_effort_for_databricks_v2(effort, raw_model)` across all 7 requested + /// effort levels and compares against `normalize_effort_for_openai_route(effort, stripped)`. + /// + /// For Anthropic-routed inputs (databricks_v2 with NormalizationPolicy::None), this + /// test compares `anthropic_thinking_config_generated("databricks_v2", ...)` against + /// `_old_anthropic_thinking_config_for_databricks_v2(...)` for each non-None effort. + /// + /// Allowlist entries cover intentional behavioral divergences (F1 corrections); + /// stale entries fail the test. + #[test] + fn behavioral_differential_production_functions_match_old_shims() { + use crate::config::{ + _old_anthropic_thinking_config_for_databricks_v2, anthropic_thinking_config_generated, + normalize_effort_for_databricks_v2, normalize_effort_for_openai_route, + strip_catalog_prefix as config_strip_catalog_prefix, + }; + use crate::generated_model_capabilities::{ + resolve_model_capabilities, NormalizationPolicy, + }; + use std::collections::HashSet; + + const MAX_OUTPUT_TOKENS: u32 = 32_768; + + // All 7 effort levels in ordinal order. + const ALL_EFFORTS: &[ThinkingEffort] = &[ + ThinkingEffort::None, + ThinkingEffort::Minimal, + ThinkingEffort::Low, + ThinkingEffort::Medium, + ThinkingEffort::High, + ThinkingEffort::XHigh, + ThinkingEffort::Max, + ]; + + // Axis-scoped allowlist mirroring the record differential. + // "normalization_result" = effort normalization output diverges. + // "thinking_shape" = thinking request JSON shape diverges. + #[derive(Debug)] + struct BehavAllowlistEntry { + raw_model_id: &'static str, + axis: &'static str, + reason: &'static str, + } + // Only databricks_v2 entries are probed here; provider is implicitly databricks_v2. + let allowlist: &[BehavAllowlistEntry] = &[ + // F1 corrections: generated supported_efforts differs from old hand table. + // normalize_effort_for_databricks_v2 now resolves against generated supported_efforts + // → old shim's clamping of none→none, xhigh→xhigh is replaced by none→low, xhigh→high. + BehavAllowlistEntry { + raw_model_id: "databricks-gpt-5-5", + axis: "normalization_result", + reason: "F1 ADOPT: generated [low,medium,high]; old table admits none+xhigh", + }, + BehavAllowlistEntry { + raw_model_id: "databricks-gpt-5-4-mini", + axis: "normalization_result", + reason: "F1 ADOPT: generated [low,medium,high]; old table admits none+xhigh", + }, + BehavAllowlistEntry { + raw_model_id: "databricks-gpt-5-4-nano", + axis: "normalization_result", + reason: "F1 ADOPT: generated [low,medium,high]; old table admits none+xhigh", + }, + BehavAllowlistEntry { + raw_model_id: "databricks-gpt-5-6-sol", + axis: "normalization_result", + reason: "F1 ADOPT: generated [low,medium,high,max]; old table admits none+xhigh", + }, + ]; + + let mut allowlist_hits: HashSet<(&str, &str)> = HashSet::new(); + let mut divergences: Vec = Vec::new(); + + let is_allowlisted = |model: &str, axis: &str, hits: &mut HashSet<(&str, &str)>| { + for entry in allowlist { + if entry.raw_model_id == model && entry.axis == axis { + hits.insert((entry.raw_model_id, entry.axis)); + return true; + } + } + false + }; + + // --- Collect all databricks_v2 inputs from the three committed sets --- + #[derive(serde::Deserialize)] + struct FixtureEntry { + note: Option, + provider: String, + model: String, + } + #[derive(serde::Deserialize)] + struct CorpusEntry { + #[serde(rename = "_group")] + group: Option, + id: Option, + provider: Option, + raw_model_id: Option, + } + #[derive(serde::Deserialize)] + struct CatalogEntry { + name: String, + } + #[derive(serde::Deserialize)] + struct CatalogFixture { + endpoints: Vec, + } + + let mut inputs: Vec<(String, String)> = Vec::new(); // (label, raw_model_id) for databricks_v2 + + let fixture_json = + include_str!("../../../desktop/src/features/agents/ui/effortTable.fixture.json"); + let fixture: Vec = + serde_json::from_str(fixture_json).expect("fixture must be valid JSON"); + for e in &fixture { + if e.provider == "databricks_v2" { + let label = format!("fixture:{}", e.note.as_deref().unwrap_or(&e.model)); + inputs.push((label, e.model.clone())); + } + } + + let corpus_json = include_str!("../../../scripts/normative-corpus.json"); + let corpus: Vec = + serde_json::from_str(corpus_json).expect("corpus must be valid JSON"); + for e in &corpus { + if e.group.is_some() { + continue; + } + if let (Some(prov), Some(model)) = (&e.provider, &e.raw_model_id) { + if prov == "databricks_v2" { + let label = format!("corpus:{}", e.id.as_deref().unwrap_or(model.as_str())); + inputs.push((label, model.clone())); + } + } + } + + let catalog_json = include_str!("../../../scripts/catalog-sample-fixture.json"); + let catalog: CatalogFixture = + serde_json::from_str(catalog_json).expect("catalog fixture must be valid JSON"); + for e in &catalog.endpoints { + let label = format!("catalog:{}", e.name); + inputs.push((label, e.name.clone())); + } + + // --- Behavioral probe for each input --- + for (label, raw_model) in &inputs { + let cap = resolve_model_capabilities("databricks_v2", raw_model); + + match cap.normalization_policy { + NormalizationPolicy::OpenAiStandard + | NormalizationPolicy::OpenAiClampMaxToXHigh => { + // Probe all 7 effort levels through the production normalization function + // vs the old shim. + let stripped = config_strip_catalog_prefix(raw_model); + let mut any_divergence = false; + for &effort in ALL_EFFORTS { + let new_result = normalize_effort_for_databricks_v2(effort, raw_model); + let old_result = normalize_effort_for_openai_route(effort, stripped); + if new_result != old_result { + any_divergence = true; + } + } + if any_divergence + && !is_allowlisted(raw_model, "normalization_result", &mut allowlist_hits) + { + // Collect per-effort details for the error message. + let details: Vec = ALL_EFFORTS + .iter() + .filter_map(|&effort| { + let new_result = + normalize_effort_for_databricks_v2(effort, raw_model); + let old_result = + normalize_effort_for_openai_route(effort, stripped); + if new_result != old_result { + Some(format!( + " {} → old={} new={}", + effort.openai_effort_str(), + old_result.openai_effort_str(), + new_result.openai_effort_str() + )) + } else { + None + } + }) + .collect(); + divergences.push(format!( + "BEHAVIORAL_DIVERGE normalization_result [{label}] model={raw_model:?}:\n{}", + details.join("\n") + )); + } + } + NormalizationPolicy::None => { + // Anthropic-routed: compare thinking config shape for each non-None effort. + let mut any_divergence = false; + for &effort in ALL_EFFORTS { + if effort == ThinkingEffort::None || effort == ThinkingEffort::Minimal { + continue; // omit-thinking cases: both produce (None, None), no shape to compare + } + let new_shape = anthropic_thinking_config_generated( + "databricks_v2", + raw_model, + effort, + MAX_OUTPUT_TOKENS, + ); + let old_shape = _old_anthropic_thinking_config_for_databricks_v2( + raw_model, + effort, + MAX_OUTPUT_TOKENS, + ); + if new_shape != old_shape { + any_divergence = true; + } + } + if any_divergence + && !is_allowlisted(raw_model, "thinking_shape", &mut allowlist_hits) + { + let details: Vec = ALL_EFFORTS + .iter() + .filter_map(|&effort| { + if effort == ThinkingEffort::None + || effort == ThinkingEffort::Minimal + { + return None; + } + let new_shape = anthropic_thinking_config_generated( + "databricks_v2", + raw_model, + effort, + MAX_OUTPUT_TOKENS, + ); + let old_shape = _old_anthropic_thinking_config_for_databricks_v2( + raw_model, + effort, + MAX_OUTPUT_TOKENS, + ); + if new_shape != old_shape { + Some(format!( + " effort={}: old={:?} new={:?}", + effort.openai_effort_str(), + old_shape, + new_shape + )) + } else { + None + } + }) + .collect(); + divergences.push(format!( + "BEHAVIORAL_DIVERGE thinking_shape [{label}] model={raw_model:?}:\n{}", + details.join("\n") + )); + } + } + } + } + + // Stale allowlist: any declared entry that never fired is a bug. + let mut stale: Vec = Vec::new(); + for entry in allowlist { + if !allowlist_hits.contains(&(entry.raw_model_id, entry.axis)) { + stale.push(format!( + "STALE_ALLOWLIST model={} axis={} reason={}", + entry.raw_model_id, entry.axis, entry.reason + )); + } + } + + let mut failures = divergences.clone(); + failures.extend(stale); + + assert!( + failures.is_empty(), + "Behavioral differential found {} failure(s):\n{}", + failures.len(), + failures.join("\n") + ); + + println!( + "Behavioral differential: {} databricks_v2 inputs probed, {} behavioral allowlist slots exercised/{}, 0 unexpected divergences", + inputs.len(), + allowlist_hits.len(), + allowlist.len(), + ); + } + + /// Behavioral differential for `normalize_effort_for_provider` — the production + /// authority for pure OpenAI and legacy Databricks effort normalization. + /// + /// This test catches a provider-generic repeat of the `305627e32` defect class: + /// a record-level differential passes (the generated record is correct) while the + /// production function diverges (delegates to the old hand table instead of the + /// record). The existing behavioral differential above covers `databricks_v2`; this + /// test covers `openai` and `databricks` routes, including the `openai-compat` + /// alias that the TS canonicalizer resolves to `openai` (Thufir P3 action 1). + /// + /// For every corpus entry with provider in {openai, databricks, openai-compat}, + /// this drives `normalize_effort_for_provider(canonical_provider, model, effort)` + /// and `normalize_effort_for_openai_route(effort, model)` across all 7 effort + /// levels and asserts they agree. No allowlist is expected — these functions are + /// definitionally aligned and any divergence is a bug. + #[test] + fn behavioral_differential_normalize_effort_for_provider() { + use crate::config::{normalize_effort_for_openai_route, normalize_effort_for_provider}; + + const ALL_EFFORTS: &[ThinkingEffort] = &[ + ThinkingEffort::None, + ThinkingEffort::Minimal, + ThinkingEffort::Low, + ThinkingEffort::Medium, + ThinkingEffort::High, + ThinkingEffort::XHigh, + ThinkingEffort::Max, + ]; + + #[derive(serde::Deserialize)] + struct CorpusEntry { + #[serde(rename = "_group")] + group: Option, + id: Option, + provider: Option, + raw_model_id: Option, + } + + let corpus_json = include_str!("../../../scripts/normative-corpus.json"); + let corpus: Vec = + serde_json::from_str(corpus_json).expect("corpus must be valid JSON"); + + // Collect (label, canonical_provider, raw_model_id) for openai/databricks/openai-compat. + let mut inputs: Vec<(String, &'static str, String)> = Vec::new(); + for e in &corpus { + if e.group.is_some() { + continue; + } + let (prov, model) = match (&e.provider, &e.raw_model_id) { + (Some(p), Some(m)) => (p.as_str(), m.as_str()), + _ => continue, + }; + let canonical: &'static str = match prov { + "openai" | "openai-compat" => "openai", + "databricks" => "databricks", + _ => continue, // databricks_v2 and others are covered by the other differential + }; + let label = format!( + "corpus:{} (raw_provider={})", + e.id.as_deref().unwrap_or(model), + prov + ); + inputs.push((label, canonical, model.to_owned())); + } + + assert!( + !inputs.is_empty(), + "No openai/databricks/openai-compat inputs found in normative corpus" + ); + + let mut divergences: Vec = Vec::new(); + + for (label, canonical_provider, raw_model) in &inputs { + let mut per_effort: Vec = Vec::new(); + for &effort in ALL_EFFORTS { + let new_result = + normalize_effort_for_provider(canonical_provider, raw_model, effort); + let old_result = normalize_effort_for_openai_route(effort, raw_model); + if new_result != old_result { + per_effort.push(format!( + " {} → old={} new={}", + effort.openai_effort_str(), + old_result.openai_effort_str(), + new_result.openai_effort_str() + )); + } + } + if !per_effort.is_empty() { + divergences.push(format!( + "BEHAVIORAL_DIVERGE normalize_effort_for_provider [{label}] model={raw_model:?} provider={canonical_provider:?}:\n{}", + per_effort.join("\n") + )); + } + } + + assert!( + divergences.is_empty(), + "behavioral_differential_normalize_effort_for_provider found {} failure(s):\n{}", + divergences.len(), + divergences.join("\n") + ); + + println!( + "behavioral_differential_normalize_effort_for_provider: {} openai/databricks corpus inputs probed, 0 divergences", + inputs.len() + ); + } + + /// Behavioral shape differential for the pure Anthropic route. + /// + /// Drives `anthropic_thinking_config_generated("anthropic", raw_model, effort, …)` + /// against the prior pure-Anthropic authority `anthropic_thinking_config(raw_model, …)` + /// for every corpus entry with `provider == "anthropic"` across all seven effort levels. + /// + /// This completes the provider-general scope of the authorized corrective pass: the + /// existing differential covers databricks_v2; the normalize-effort differential covers + /// openai/databricks/openai-compat; this test covers the Anthropic thinking-config path. + /// + /// An F1 allowlist entry covers the blank-model corpus entries: the old hand-table returns + /// `(None, None)` for an unrecognized (empty) model, while the generated manifest explicitly + /// classifies blank Anthropic models as adaptive (the intentional Phase-2 behavior). + #[test] + fn behavioral_differential_anthropic_route() { + use crate::config::{anthropic_thinking_config, anthropic_thinking_config_generated}; + use std::collections::HashSet; + + const MAX_OUTPUT_TOKENS: u32 = 32_768; + + const ALL_EFFORTS: &[ThinkingEffort] = &[ + ThinkingEffort::None, + ThinkingEffort::Minimal, + ThinkingEffort::Low, + ThinkingEffort::Medium, + ThinkingEffort::High, + ThinkingEffort::XHigh, + ThinkingEffort::Max, + ]; + + // F1 allowlist: intentional divergences between the generated manifest and the old + // hand-table. Stale entries (that never fire) fail the test. + struct AllowlistEntry { + raw_model_id: &'static str, + reason: &'static str, + } + let allowlist: &[AllowlistEntry] = &[ + // F1 ADOPT: generated manifest classifies blank Anthropic model as adaptive + // (corpus entries anthropic-unknown-blank and anthropic-blank-adaptive-full); + // old anthropic_thinking_config returns (None, None) for unrecognized models. + AllowlistEntry { + raw_model_id: "", + reason: "F1 ADOPT: generated assumes adaptive for blank Anthropic model; old hand-table returned (None, None)", + }, + ]; + let mut allowlist_hits: HashSet<&str> = HashSet::new(); + + #[derive(serde::Deserialize)] + struct CorpusEntry { + #[serde(rename = "_group")] + group: Option, + id: Option, + provider: Option, + raw_model_id: Option, + } + + let corpus_json = include_str!("../../../scripts/normative-corpus.json"); + let corpus: Vec = + serde_json::from_str(corpus_json).expect("corpus must be valid JSON"); + + // Collect (label, raw_model_id) for provider == "anthropic". + let mut inputs: Vec<(String, String)> = Vec::new(); + for e in &corpus { + if e.group.is_some() { + continue; + } + let (prov, model) = match (&e.provider, &e.raw_model_id) { + (Some(p), Some(m)) => (p.as_str(), m.as_str()), + _ => continue, + }; + if prov == "anthropic" { + let label = format!("corpus:{}", e.id.as_deref().unwrap_or(model)); + inputs.push((label, model.to_owned())); + } + } + + assert!( + !inputs.is_empty(), + "No anthropic inputs found in normative corpus" + ); + + let mut divergences: Vec = Vec::new(); + + for (label, raw_model) in &inputs { + let mut per_effort: Vec = Vec::new(); + for &effort in ALL_EFFORTS { + let new_shape = anthropic_thinking_config_generated( + "anthropic", + raw_model, + effort, + MAX_OUTPUT_TOKENS, + ); + let old_shape = anthropic_thinking_config(raw_model, effort, MAX_OUTPUT_TOKENS); + if new_shape != old_shape { + per_effort.push(format!( + " effort={}: old={:?} new={:?}", + effort.openai_effort_str(), + old_shape, + new_shape + )); + } + } + if !per_effort.is_empty() { + // Check allowlist before treating as a divergence. + let is_allowlisted = allowlist + .iter() + .any(|e| e.raw_model_id == raw_model.as_str()); + if is_allowlisted { + allowlist_hits.insert(raw_model.as_str()); + } else { + divergences.push(format!( + "BEHAVIORAL_DIVERGE anthropic_route [{label}] model={raw_model:?}:\n{}", + per_effort.join("\n") + )); + } + } + } + + // Stale allowlist: any declared entry that never fired is a bug. + let mut stale: Vec = Vec::new(); + for entry in allowlist { + if !allowlist_hits.contains(entry.raw_model_id) { + stale.push(format!( + "STALE_ALLOWLIST model={} reason={}", + entry.raw_model_id, entry.reason + )); + } + } + + let mut failures = divergences.clone(); + failures.extend(stale); + + assert!( + failures.is_empty(), + "behavioral_differential_anthropic_route found {} failure(s):\n{}", + failures.len(), + failures.join("\n") + ); + + println!( + "behavioral_differential_anthropic_route: {} anthropic corpus inputs probed, {} F1 allowlist slots exercised/{}, 0 unexpected divergences", + inputs.len(), + allowlist_hits.len(), + allowlist.len(), + ); + } + #[test] fn parse_responses_rejects_malformed_function_arguments() { let v = serde_json::json!({ @@ -3477,6 +4379,7 @@ mod tests { &[], "databricks-claude-opus-5", None, + "databricks_v2", ); // Static prefix: system promoted to a structured block carrying the marker. assert_eq!(body["system"][0]["type"], "text"); @@ -3507,6 +4410,7 @@ mod tests { &[], "databricks-claude-opus-5", None, + "databricks_v2", ); let msgs = body["messages"].as_array().unwrap(); assert_eq!(msgs.len(), 1); @@ -3527,6 +4431,7 @@ mod tests { &[], "databricks-claude-opus-5", None, + "anthropic", ); // system stays a bare string; no marker anywhere. assert_eq!(body["system"], "sys"); @@ -3545,6 +4450,7 @@ mod tests { &[], "databricks-claude-opus-5", None, + "databricks_v2", ); assert_eq!(body["system"], ""); assert_eq!( @@ -3564,6 +4470,7 @@ mod tests { &[], "model", None, + "anthropic", ); assert!( body.get("thinking").is_none(), @@ -3584,6 +4491,7 @@ mod tests { &[], "claude-3-7-sonnet-20250219", Some(ThinkingEffort::High), + "anthropic", ); assert_eq!(body["thinking"]["type"], "enabled"); // budget_tokens = min(32768, 4096-1024) = 3072 @@ -3603,6 +4511,7 @@ mod tests { &[], "claude-3-7-sonnet-20250219", Some(ThinkingEffort::High), + "anthropic", ); assert!( body.get("thinking").is_none(), @@ -3622,6 +4531,7 @@ mod tests { &[], "claude-3-7-sonnet-20250219", Some(ThinkingEffort::High), + "anthropic", ); let t = body .get("thinking") @@ -3641,6 +4551,7 @@ mod tests { &[], "claude-3-7-sonnet-20250219", Some(ThinkingEffort::High), + "anthropic", ); assert_eq!(body["thinking"]["budget_tokens"], 32_768); } @@ -3658,6 +4569,7 @@ mod tests { &[], "claude-3-7-sonnet-20250219", Some(ThinkingEffort::Low), + "anthropic", ); // Low budget (1024) fits exactly at the boundary — emitted without capping. assert_eq!(body["thinking"]["budget_tokens"], 1024); @@ -3676,6 +4588,7 @@ mod tests { &[], "claude-opus-4-7", Some(ThinkingEffort::High), + "anthropic", ); assert_eq!( body["thinking"]["type"], "adaptive", @@ -3697,6 +4610,7 @@ mod tests { &[], "claude-opus-4-5", Some(ThinkingEffort::High), + "anthropic", ); assert_eq!(body["thinking"]["type"], "enabled"); assert_eq!(body["thinking"]["budget_tokens"], 31_744); // min(32768, 32768-1024) @@ -3716,6 +4630,7 @@ mod tests { &[], "gpt-4o", Some(ThinkingEffort::High), + "anthropic", ); assert!(body.get("thinking").is_none(), "thinking must be absent"); assert!( @@ -3791,6 +4706,7 @@ mod tests { &[], "override-model", None, + "anthropic", ); assert_eq!(body["model"], "override-model"); } @@ -3820,6 +4736,7 @@ mod tests { &[], "claude-opus-4-8", Some(ThinkingEffort::XHigh), + "anthropic", ); assert_eq!(body["thinking"]["type"], "adaptive"); assert_eq!(body["output_config"]["effort"], "xhigh"); @@ -3837,6 +4754,7 @@ mod tests { &[], "claude-opus-4-8", Some(ThinkingEffort::Max), + "anthropic", ); assert_eq!(body["thinking"]["type"], "adaptive"); assert_eq!(body["output_config"]["effort"], "max"); @@ -3900,15 +4818,17 @@ mod tests { // ---- DatabricksV2 route-aware effort normalization (body-level assertions) ---- // - // The DBv2 `complete()` dispatch applies `normalize_effort_for_openai_route` / - // `normalize_effort_for_anthropic_route` before calling body builders. These tests - // verify the body shape that results from the already-normalized effort values — i.e., - // they confirm the body builders correctly serialize the values the dispatch passes them. + // These tests verify the body shape produced by the body builders when passed + // a pre-normalized effort value. The effort is pre-normalized here via the old + // helper (normalize_effort_for_openai_route) to produce the expected clamped value, + // mirroring what normalize_effort_for_databricks_v2 would return for these models + // (OpenAiStandard policy → delegates to normalize_effort_for_openai_route). #[test] fn dbv2_openai_route_max_effort_clamped_to_xhigh_in_responses_body() { - // DBv2 GPT-5.5 route: max → clamped to xhigh by normalize_effort_for_openai_route - // before reaching responses_body. gpt-5.5 supports xhigh so the final value is xhigh. + // DBv2 GPT-5.5 route: max → clamped to xhigh (OpenAiStandard policy). + // Pre-normalize via normalize_effort_for_openai_route (same as what + // normalize_effort_for_databricks_v2 delegates to for OpenAiStandard). let clamped = crate::config::normalize_effort_for_openai_route(ThinkingEffort::Max, "gpt-5.5"); let body = responses_body( @@ -4012,6 +4932,7 @@ mod tests { &[], "claude-opus-4-8", normalized, // None → omit thinking fields + "anthropic", ); assert!( body.get("thinking").is_none(), @@ -5950,6 +6871,7 @@ mod tests { &[], "claude-opus-4-7", None, + "anthropic", ); let messages = body["messages"].as_array().unwrap(); let assistant = messages diff --git a/desktop/src/features/agents/lib/agentCardModelLabel.ts b/desktop/src/features/agents/lib/agentCardModelLabel.ts index 81b52b021d6..df330c46dd2 100644 --- a/desktop/src/features/agents/lib/agentCardModelLabel.ts +++ b/desktop/src/features/agents/lib/agentCardModelLabel.ts @@ -22,8 +22,10 @@ import type { ManagedAgent } from "@/shared/api/types"; * than falling through to "inherited" for lack of an instance. */ export function resolveAgentCardModelLabel(input: { - agent: Pick | undefined; + agent: Pick | undefined; personaModel: string | null | undefined; + /** Inference provider for the persona/agent — threads provider-qualified label lookup. */ + provider?: string | null | undefined; defaultModel: string; }): string { if (input.agent) { @@ -33,11 +35,11 @@ export function resolveAgentCardModelLabel(input: { return formatDefaultModelLabel(input.defaultModel); } return input.agent.model?.trim() - ? formatAgentModelLabel(input.agent.model) + ? formatAgentModelLabel(input.agent.model, input.agent.provider) : formatDefaultModelLabel(input.defaultModel); } return input.personaModel?.trim() - ? formatAgentModelLabel(input.personaModel) + ? formatAgentModelLabel(input.personaModel, input.provider) : formatDefaultModelLabel(input.defaultModel); } diff --git a/desktop/src/features/agents/lib/databricksModelNames.test.mjs b/desktop/src/features/agents/lib/databricksModelNames.test.mjs index 2f7cec41dcb..eef0a2bb603 100644 --- a/desktop/src/features/agents/lib/databricksModelNames.test.mjs +++ b/desktop/src/features/agents/lib/databricksModelNames.test.mjs @@ -6,6 +6,7 @@ import { DATABRICKS_MODEL_NAMES } from "./databricksModelNames.ts"; import { resolveModelLabel, formatAgentModelLabel, + canonicalizeProvider, } from "./formatAgentModelLabel.ts"; // --------------------------------------------------------------------------- @@ -198,7 +199,80 @@ test("ModelPicker — discovered rows render through resolveModelLabel", () => { assert.match( source, - / { + // "databricks-gemini-3-pro" is in DATABRICKS_MODEL_NAMES as "Gemini 3 Pro Preview". + // When provider="anthropic", the generated lookup misses → must return raw ID. + const result = resolveModelLabel( + "databricks-gemini-3-pro", + null, + "anthropic", + ); + assert.notEqual( + result, + "Gemini 3 Pro Preview", + "must not leak Databricks registry label", + ); + assert.equal(result, "databricks-gemini-3-pro"); +}); + +test("resolveModelLabel — openai provider scoped miss returns raw ID", () => { + const result = resolveModelLabel("databricks-gemini-3-pro", null, "openai"); + assert.equal(result, "databricks-gemini-3-pro"); +}); + +test("resolveModelLabel — providerless call still returns unscoped registry label", () => { + // No provider: the unscoped DATABRICKS_MODEL_NAMES map is reachable. + const result = resolveModelLabel("databricks-gemini-3-pro", null, undefined); + // The unscoped map should have a curated name for this ID. + assert.ok( + DATABRICKS_MODEL_NAMES.has("databricks-gemini-3-pro"), + "databricks-gemini-3-pro must be in DATABRICKS_MODEL_NAMES for this test to be valid", + ); + assert.equal(result, DATABRICKS_MODEL_NAMES.get("databricks-gemini-3-pro")); + assert.notEqual(result, "databricks-gemini-3-pro"); +}); + +test("resolveModelLabel — null provider treated as providerless (uses unscoped registry)", () => { + const result = resolveModelLabel("databricks-gemini-3-pro", null, null); + assert.equal(result, DATABRICKS_MODEL_NAMES.get("databricks-gemini-3-pro")); +}); + +test("resolveModelLabel — provider case-insensitivity: 'Anthropic' same as 'anthropic'", () => { + const lower = resolveModelLabel("databricks-gemini-3-pro", null, "anthropic"); + const upper = resolveModelLabel("databricks-gemini-3-pro", null, "Anthropic"); + assert.equal(lower, upper); + assert.equal(lower, "databricks-gemini-3-pro"); +}); + +// --------------------------------------------------------------------------- +// P3-B: canonicalizeProvider — alias normalization +// --------------------------------------------------------------------------- + +test("canonicalizeProvider — databricks-v2 (hyphen) maps to databricks_v2 (underscore)", () => { + assert.equal(canonicalizeProvider("databricks-v2"), "databricks_v2"); +}); + +test("canonicalizeProvider — uppercase DATABRICKS-V2 also normalizes to databricks_v2", () => { + assert.equal(canonicalizeProvider("DATABRICKS-V2"), "databricks_v2"); +}); + +test("canonicalizeProvider — databricks_v2 passes through unchanged", () => { + assert.equal(canonicalizeProvider("databricks_v2"), "databricks_v2"); +}); + +test("canonicalizeProvider — anthropic lowercases and trims", () => { + assert.equal(canonicalizeProvider(" Anthropic "), "anthropic"); +}); + +test("canonicalizeProvider — unknown alias passes through lowercased", () => { + assert.equal(canonicalizeProvider("OpenAI"), "openai"); +}); diff --git a/desktop/src/features/agents/lib/formatAgentModelLabel.ts b/desktop/src/features/agents/lib/formatAgentModelLabel.ts index e0595e4f672..c6e2cffd571 100644 --- a/desktop/src/features/agents/lib/formatAgentModelLabel.ts +++ b/desktop/src/features/agents/lib/formatAgentModelLabel.ts @@ -1,11 +1,45 @@ -import { DATABRICKS_MODEL_NAMES } from "./databricksModelNames"; +import { + DATABRICKS_MODEL_NAMES, + resolveModelCapabilities, +} from "../ui/modelCapabilities.ts"; + +/** + * Known provider-id aliases that must be normalized before generated lookups. + * + * - "databricks-v2" (hyphen form) → "databricks_v2" (underscore form used in manifest). + * - "openai-compat" → "openai": Rust already accepts "openai-compat" as Provider::OpenAi + * (crates/buzz-agent/src/config.rs); TS must canonicalize identically so the UI shows + * the same effort table that the Rust request path will apply. + */ +const PROVIDER_ALIASES: Readonly> = { + "databricks-v2": "databricks_v2", + "openai-compat": "openai", +}; + +/** + * Normalizes a provider id to the canonical form expected by the generated + * manifest: lowercases, trims, and applies the known alias table. + * + * Used as the single canonicalization point before every generated lookup + * in both resolveModelLabel() and getProviderEffortConfig(). + */ +export function canonicalizeProvider(provider: string): string { + const normalized = provider.trim().toLowerCase(); + return PROVIDER_ALIASES[normalized] ?? normalized; +} /** * Resolves a human-readable label for a model, following the three-tier * precedence documented in AGENTS.md: * * 1. Nonblank discovered/API name (e.g. from AgentModelInfo.name) - * 2. Registry lookup by ID (models.dev-seeded Databricks table) + * 2. Registry lookup by ID: + * - When `provider` is supplied: provider-qualified exact record only. + * If the generated lookup returns no registryLabel, return the raw ID. + * The unscoped DATABRICKS_MODEL_NAMES registry is NOT consulted for a + * known provider — this prevents Databricks names from leaking through + * anthropic/openai provider contexts. + * - When `provider` is absent/null: unscoped DATABRICKS_MODEL_NAMES map. * 3. Raw ID unchanged * * Returns the empty string when both id and discoveredName are blank. @@ -14,11 +48,24 @@ import { DATABRICKS_MODEL_NAMES } from "./databricksModelNames"; export function resolveModelLabel( id: string, discoveredName?: string | null | undefined, + provider?: string | null | undefined, ): string { const trimmedName = discoveredName?.trim(); if (trimmedName) return trimmedName; const trimmedId = id.trim(); if (!trimmedId) return ""; + // Provider-qualified exact record (registry_label tier, provider-scoped). + // When a provider is known, do NOT fall through to the unscoped registry — + // return the raw ID directly on a miss (P3-B contract). + if (provider?.trim()) { + const canonical = canonicalizeProvider(provider); + const registryLabel = resolveModelCapabilities( + canonical, + trimmedId, + ).registryLabel; + return registryLabel ?? trimmedId; + } + // Providerless path only: unscoped registry map for legacy/inherited IDs. return DATABRICKS_MODEL_NAMES.get(trimmedId) ?? trimmedId; } @@ -29,9 +76,15 @@ export function resolveModelLabel( * For known Databricks managed endpoints the registry-curated name is returned * (e.g. "databricks-gpt-5-5" → "GPT-5.5"). Unknown or custom endpoint IDs are * returned unchanged — no heuristic string mangling. + * + * Pass `provider` when the inference provider is known to get a provider-qualified + * registry label (e.g. exact records in the generated manifest take priority). */ -export function formatAgentModelLabel(model: string | null | undefined) { +export function formatAgentModelLabel( + model: string | null | undefined, + provider?: string | null | undefined, +) { const trimmed = model?.trim(); if (!trimmed) return "Auto"; - return resolveModelLabel(trimmed); + return resolveModelLabel(trimmed, null, provider); } diff --git a/desktop/src/features/agents/ui/ManagedAgentRow.tsx b/desktop/src/features/agents/ui/ManagedAgentRow.tsx index de9a50b1f14..caad2632eb1 100644 --- a/desktop/src/features/agents/ui/ManagedAgentRow.tsx +++ b/desktop/src/features/agents/ui/ManagedAgentRow.tsx @@ -411,7 +411,9 @@ function RuntimeBlock({ {runtimeSource || agent.model ? (
{runtimeSource ? {runtimeSource} : null} - {agent.model ? {resolveModelLabel(agent.model)} : null} + {agent.model ? ( + {resolveModelLabel(agent.model, null, agent.provider)} + ) : null}
) : null} diff --git a/desktop/src/features/agents/ui/ModelPicker.tsx b/desktop/src/features/agents/ui/ModelPicker.tsx index 41fe0d25629..863cd851195 100644 --- a/desktop/src/features/agents/ui/ModelPicker.tsx +++ b/desktop/src/features/agents/ui/ModelPicker.tsx @@ -84,9 +84,9 @@ export function ModelPicker({ const currentValue = agent.model ?? modelsData?.agentDefaultModel ?? ""; const displayLabel = agent.model - ? resolveModelLabel(agent.model) + ? resolveModelLabel(agent.model, null, agent.provider) : modelsData?.agentDefaultModel - ? `${resolveModelLabel(modelsData.agentDefaultModel)} (default)` + ? `${resolveModelLabel(modelsData.agentDefaultModel, null, agent.provider)} (default)` : hasRequestedModels && loading ? "Loading..." : "Auto"; @@ -223,7 +223,7 @@ export function ModelPicker({ {agent.model ? ( <>

- {resolveModelLabel(agent.model)} + {resolveModelLabel(agent.model, null, agent.provider)}

This runtime does not support switching models. @@ -240,7 +240,7 @@ export function ModelPicker({ > {modelsData.models.map((model) => ( - {resolveModelLabel(model.id, model.name)} + {resolveModelLabel(model.id, model.name, agent.provider)} ))} diff --git a/desktop/src/features/agents/ui/TeamIdentityCard.tsx b/desktop/src/features/agents/ui/TeamIdentityCard.tsx index 19596b76d6c..8e4b02c9e8d 100644 --- a/desktop/src/features/agents/ui/TeamIdentityCard.tsx +++ b/desktop/src/features/agents/ui/TeamIdentityCard.tsx @@ -203,7 +203,7 @@ function TeamAvatarItem({ function getTeamFooterModelLabel(personas: AgentPersona[]) { const modelLabels = personas - .map((persona) => formatAgentModelLabel(persona.model)) + .map((persona) => formatAgentModelLabel(persona.model, persona.provider)) .filter((model): model is string => Boolean(model)); if (modelLabels.length === 0) return "Auto"; diff --git a/desktop/src/features/agents/ui/UnifiedAgentsSection.tsx b/desktop/src/features/agents/ui/UnifiedAgentsSection.tsx index 19a5ef1171f..ffb8c256e07 100644 --- a/desktop/src/features/agents/ui/UnifiedAgentsSection.tsx +++ b/desktop/src/features/agents/ui/UnifiedAgentsSection.tsx @@ -276,6 +276,7 @@ function AgentPersonaCard({ const modelLabel = resolveAgentCardModelLabel({ agent, personaModel: persona.model, + provider: persona.provider, defaultModel, }); const isActive = agent ? isManagedAgentActive(agent) : false; diff --git a/desktop/src/features/agents/ui/buzzAgentConfig.test.mjs b/desktop/src/features/agents/ui/buzzAgentConfig.test.mjs index 4d702966b72..e3730fe4335 100644 --- a/desktop/src/features/agents/ui/buzzAgentConfig.test.mjs +++ b/desktop/src/features/agents/ui/buzzAgentConfig.test.mjs @@ -541,13 +541,19 @@ test("databricks v1 routes like openai unknown (no gpt-5 model)", () => { assert.equal(defaultValue, "medium"); }); -test("openai-compat returns all-7 with medium default", () => { - const { validValues, defaultValue } = getProviderEffortConfig( - "openai-compat", - "", +test("openai-compat canonicalizes to openai: empty model returns all-except-max with medium default", () => { + // Regression: without canonicalization, openai-compat hit the unknown-provider fallback + // (all 7 values, default medium). After alias, it must resolve identically to openai. + const compat = getProviderEffortConfig("openai-compat", ""); + const openai = getProviderEffortConfig("openai", ""); + assert.deepEqual([...compat.validValues], [...openai.validValues]); + assert.equal(compat.defaultValue, openai.defaultValue); + // Concrete assertion: same as openai unknown model, not the all-7 fallback. + assert.deepEqual( + [...compat.validValues], + ["none", "minimal", "low", "medium", "high", "xhigh"], ); - assert.equal(validValues.length, 7); - assert.equal(defaultValue, "medium"); + assert.equal(compat.defaultValue, "medium"); }); test("empty provider returns all-7 with medium default", () => { @@ -621,3 +627,79 @@ test("effort none is invalid for anthropic manual-budget (should trigger auto-cl "none must not be in manual-budget set", ); }); + +// --------------------------------------------------------------------------- +// getProviderEffortConfig — databricks-v2 hyphen alias (P3-B regression) +// --------------------------------------------------------------------------- + +test("databricks-v2 hyphen alias canonicalizes to databricks_v2 underscore records", () => { + // The persisted alias "databricks-v2" must hit the same canonical records as "databricks_v2". + const hyphen = getProviderEffortConfig("databricks-v2", "databricks-gpt-5-5"); + const underscore = getProviderEffortConfig( + "databricks_v2", + "databricks-gpt-5-5", + ); + assert.deepEqual([...hyphen.validValues], [...underscore.validValues]); + assert.equal(hyphen.defaultValue, underscore.defaultValue); +}); + +test("databricks-v2 hyphen alias with databricks-gpt-5-5 returns [low,medium,high]", () => { + // Regression: without canonicalization this returned the all-7 unknown-provider fallback. + const { validValues, defaultValue } = getProviderEffortConfig( + "databricks-v2", + "databricks-gpt-5-5", + ); + assert.deepEqual([...validValues], ["low", "medium", "high"]); + assert.equal(defaultValue, "medium"); +}); + +// --------------------------------------------------------------------------- +// getProviderEffortConfig — openai-compat alias (Thufir P3 corrective action 1) +// --------------------------------------------------------------------------- +// Rust normalizes "openai-compat" → Provider::OpenAi at config.rs:1218. +// TS PROVIDER_ALIASES must match so UI effort table = Rust request behavior. + +test("openai-compat/gpt-5-pro resolves identically to openai/gpt-5-pro", () => { + const compat = getProviderEffortConfig("openai-compat", "gpt-5-pro"); + const openai = getProviderEffortConfig("openai", "gpt-5-pro"); + assert.deepEqual([...compat.validValues], [...openai.validValues]); + assert.equal(compat.defaultValue, openai.defaultValue); + // Concrete: gpt-5-pro is [high] only. + assert.deepEqual([...compat.validValues], ["high"]); + assert.equal(compat.defaultValue, "high"); +}); + +test("openai-compat/gpt-5.5 resolves identically to openai/gpt-5.5", () => { + const compat = getProviderEffortConfig("openai-compat", "gpt-5.5"); + const openai = getProviderEffortConfig("openai", "gpt-5.5"); + assert.deepEqual([...compat.validValues], [...openai.validValues]); + assert.equal(compat.defaultValue, openai.defaultValue); +}); + +test("openai-compat/gpt-5 base resolves identically to openai/gpt-5", () => { + const compat = getProviderEffortConfig("openai-compat", "gpt-5"); + const openai = getProviderEffortConfig("openai", "gpt-5"); + assert.deepEqual([...compat.validValues], [...openai.validValues]); + assert.equal(compat.defaultValue, openai.defaultValue); +}); + +test("openai-compat alias is case-insensitive: OpenAI-Compat canonicalizes correctly", () => { + const mixed = getProviderEffortConfig("OpenAI-Compat", "gpt-5-pro"); + const lower = getProviderEffortConfig("openai-compat", "gpt-5-pro"); + assert.deepEqual([...mixed.validValues], [...lower.validValues]); + assert.equal(mixed.defaultValue, lower.defaultValue); +}); + +test("openai-compat alias handles surrounding whitespace: ' openai-compat ' canonicalizes correctly", () => { + const padded = getProviderEffortConfig(" openai-compat ", "gpt-5-pro"); + const clean = getProviderEffortConfig("openai-compat", "gpt-5-pro"); + assert.deepEqual([...padded.validValues], [...clean.validValues]); + assert.equal(padded.defaultValue, clean.defaultValue); +}); + +test("openai-compat alias handles mixed case + whitespace: ' OpenAI-Compat ' canonicalizes correctly", () => { + const messy = getProviderEffortConfig(" OpenAI-Compat ", "gpt-5"); + const canonical = getProviderEffortConfig("openai", "gpt-5"); + assert.deepEqual([...messy.validValues], [...canonical.validValues]); + assert.equal(messy.defaultValue, canonical.defaultValue); +}); diff --git a/desktop/src/features/agents/ui/buzzAgentConfig.ts b/desktop/src/features/agents/ui/buzzAgentConfig.ts index be663c35cb4..d4a982b315b 100644 --- a/desktop/src/features/agents/ui/buzzAgentConfig.ts +++ b/desktop/src/features/agents/ui/buzzAgentConfig.ts @@ -1,9 +1,13 @@ /** * Source-of-truth constants for buzz-agent model-tuning configuration knobs. * - * Values must stay in sync with `crates/buzz-agent/src/config.rs` - * `parse_thinking_effort` — that function is the authoritative list. + * Phase 2b pass 2: getProviderEffortConfig() is now generated-backed (thin + * wrapper over resolveModelCapabilities()). The legacy hand-table implementation + * is preserved as getProviderEffortConfig_oldHandTable() for the differential + * harness only — nothing user-facing imports the _old shim. Phase 3 retires it. */ +import { canonicalizeProvider } from "../lib/formatAgentModelLabel.ts"; +import { resolveModelCapabilities } from "./modelCapabilities.ts"; /** Env var key for the thinking/effort level sent to the LLM. */ export const BUZZ_AGENT_THINKING_EFFORT = "BUZZ_AGENT_THINKING_EFFORT"; @@ -61,25 +65,43 @@ const ALL_VALUES = BUZZ_AGENT_THINKING_EFFORT_VALUES; /** * Returns the valid thinking-effort values and semantic default for the - * given provider and optional model string. + * given provider and optional model string, resolved from the generated + * model-capabilities manifest (modelCapabilities.ts). * - * Model matching mirrors the Rust backend: - * - Anthropic: strip any endpoint-naming prefix, then test `is_manual_budget_model` - * / `is_adaptive_thinking_model` / `clamp_adaptive_effort` family checks. - * - OpenAI: strip any endpoint-naming prefix, then test `openai_efforts_for_model` - * family checks (boundary-aware: -pro before -5.x, digit/letter boundary). - * - DatabricksV2: strip prefix and route by model family. - * - Unknown/empty: all 7 values, default medium. + * This is the production entry point for all UI consumers. Resolution order: + * 1. Provider-qualified raw exact lookup + * 2. Provider-scoped family rules on normalized alias + * 3. Per-provider fallback * - * Prefix stripping: finds the first occurrence of a known model-family token - * (`claude-`, `gpt-`) and drops everything before it. This handles any - * endpoint-naming convention (e.g. `databricks-`, `goose-`, `team-x-`) without - * maintaining an allowlist of known prefixes. If no family token is found, the - * raw model name is used as-is. + * The manifest's `supportedEfforts` maps to `validValues`; `defaultEffort` + * (which may be null for manual-budget models — "Inherit" is the natural + * default) maps to `defaultValue`. */ export function getProviderEffortConfig( providerId: string, model?: string, +): ProviderEffortConfig { + const cap = resolveModelCapabilities( + canonicalizeProvider(providerId), + model ?? "", + ); + return { + validValues: cap.supportedEfforts, + defaultValue: cap.defaultEffort, + }; +} + +/** + * Legacy hand-table implementation — differential harness shim only. + * + * Preserved for the run-differential.mjs old-vs-new comparison until Phase 3 + * retires it. Nothing user-facing should import this name. + * + * @deprecated Use getProviderEffortConfig() (generated-backed) instead. + */ +export function getProviderEffortConfig_oldHandTable( + providerId: string, + model?: string, ): ProviderEffortConfig { const provider = providerId.toLowerCase(); // Strip arbitrary endpoint-naming prefix before model-family matching. @@ -307,3 +329,7 @@ function openaiConfig(m: string): ProviderEffortConfig { export function isBuzzAgentRuntime(runtimeId: string): boolean { return runtimeId === "buzz-agent"; } + +// --------------------------------------------------------------------------- +// Differential harness support +// --------------------------------------------------------------------------- diff --git a/desktop/src/features/agents/ui/effortTable.fixture.json b/desktop/src/features/agents/ui/effortTable.fixture.json index d097bc995f6..3225d6038a8 100644 --- a/desktop/src/features/agents/ui/effortTable.fixture.json +++ b/desktop/src/features/agents/ui/effortTable.fixture.json @@ -203,12 +203,19 @@ "defaultValue": "medium" }, { - "note": "openai-compat: all-7 with medium default", + "note": "openai-compat: canonicalizes to openai, empty model → all-except-max with medium default", "provider": "openai-compat", "model": "", - "validValues": ["none", "minimal", "low", "medium", "high", "xhigh", "max"], + "validValues": ["none", "minimal", "low", "medium", "high", "xhigh"], "defaultValue": "medium" }, + { + "note": "openai-compat/gpt-5-pro: canonicalizes to openai, gpt-5-pro → [high] only", + "provider": "openai-compat", + "model": "gpt-5-pro", + "validValues": ["high"], + "defaultValue": "high" + }, { "note": "openrouter: all-7 with medium default", "provider": "openrouter", diff --git a/desktop/src/features/agents/ui/modelCapabilities.ts b/desktop/src/features/agents/ui/modelCapabilities.ts index 26377227867..f17819d03e2 100644 --- a/desktop/src/features/agents/ui/modelCapabilities.ts +++ b/desktop/src/features/agents/ui/modelCapabilities.ts @@ -308,6 +308,17 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe normalizationPolicy: "openai-standard", }; } + // rule: openai-gpt5-pro, provider: databricks, priority: 20 + if (provider === "databricks" && (gpt5TokenMatchesGenerated(lower, "gpt-5-pro") || gpt5TokenMatchesGenerated(lower, "gpt5-pro"))) { + return { + registryLabel: "GPT-5 Pro", + thinkingMode: "none", + supportedEfforts: ["high"] as const, + defaultEffort: "high", + databricksV2WireRoute: "not-applicable", + normalizationPolicy: "openai-standard", + }; + } // rule: openai-gpt5-pro, provider: databricks_v2, priority: 20 if (provider === "databricks_v2" && (gpt5TokenMatchesGenerated(lower, "gpt-5-pro") || gpt5TokenMatchesGenerated(lower, "gpt5-pro"))) { return { @@ -330,6 +341,17 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe normalizationPolicy: "openai-standard", }; } + // rule: openai-gpt5-6, provider: databricks, priority: 15 + if (provider === "databricks" && (gpt5TokenMatchesGenerated(lower, "gpt-5.6") || gpt5TokenMatchesGenerated(lower, "gpt5.6") || gpt5TokenMatchesGenerated(lower, "gpt-5-6") || gpt5TokenMatchesGenerated(lower, "gpt5-6"))) { + return { + registryLabel: "GPT-5.6", + thinkingMode: "none", + supportedEfforts: ["none", "low", "medium", "high", "xhigh", "max"] as const, + defaultEffort: "medium", + databricksV2WireRoute: "not-applicable", + normalizationPolicy: "openai-standard", + }; + } // rule: openai-gpt5-6, provider: databricks_v2, priority: 15 if (provider === "databricks_v2" && (gpt5TokenMatchesGenerated(lower, "gpt-5.6") || gpt5TokenMatchesGenerated(lower, "gpt5.6") || gpt5TokenMatchesGenerated(lower, "gpt-5-6") || gpt5TokenMatchesGenerated(lower, "gpt5-6"))) { return { @@ -352,6 +374,17 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe normalizationPolicy: "openai-standard", }; } + // rule: openai-gpt5-5, provider: databricks, priority: 15 + if (provider === "databricks" && (gpt5TokenMatchesGenerated(lower, "gpt-5.5") || gpt5TokenMatchesGenerated(lower, "gpt5.5") || gpt5TokenMatchesGenerated(lower, "gpt-5-5") || gpt5TokenMatchesGenerated(lower, "gpt5-5"))) { + return { + registryLabel: "GPT-5.5", + thinkingMode: "none", + supportedEfforts: ["none", "low", "medium", "high", "xhigh"] as const, + defaultEffort: "medium", + databricksV2WireRoute: "not-applicable", + normalizationPolicy: "openai-standard", + }; + } // rule: openai-gpt5-5, provider: databricks_v2, priority: 15 if (provider === "databricks_v2" && (gpt5TokenMatchesGenerated(lower, "gpt-5.5") || gpt5TokenMatchesGenerated(lower, "gpt5.5") || gpt5TokenMatchesGenerated(lower, "gpt-5-5") || gpt5TokenMatchesGenerated(lower, "gpt5-5"))) { return { @@ -374,6 +407,17 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe normalizationPolicy: "openai-standard", }; } + // rule: openai-gpt5-4, provider: databricks, priority: 15 + if (provider === "databricks" && (gpt5TokenMatchesGenerated(lower, "gpt-5.4") || gpt5TokenMatchesGenerated(lower, "gpt5.4") || gpt5TokenMatchesGenerated(lower, "gpt-5-4") || gpt5TokenMatchesGenerated(lower, "gpt5-4"))) { + return { + registryLabel: "GPT-5.4", + thinkingMode: "none", + supportedEfforts: ["none", "low", "medium", "high", "xhigh"] as const, + defaultEffort: "medium", + databricksV2WireRoute: "not-applicable", + normalizationPolicy: "openai-standard", + }; + } // rule: openai-gpt5-4, provider: databricks_v2, priority: 15 if (provider === "databricks_v2" && (gpt5TokenMatchesGenerated(lower, "gpt-5.4") || gpt5TokenMatchesGenerated(lower, "gpt5.4") || gpt5TokenMatchesGenerated(lower, "gpt-5-4") || gpt5TokenMatchesGenerated(lower, "gpt5-4"))) { return { @@ -396,6 +440,17 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe normalizationPolicy: "openai-standard", }; } + // rule: openai-gpt5-1, provider: databricks, priority: 15 + if (provider === "databricks" && (gpt5TokenMatchesGenerated(lower, "gpt-5.1") || gpt5TokenMatchesGenerated(lower, "gpt5.1") || gpt5TokenMatchesGenerated(lower, "gpt-5-1") || gpt5TokenMatchesGenerated(lower, "gpt5-1"))) { + return { + registryLabel: "GPT-5.1", + thinkingMode: "none", + supportedEfforts: ["none", "low", "medium", "high"] as const, + defaultEffort: "none", + databricksV2WireRoute: "not-applicable", + normalizationPolicy: "openai-standard", + }; + } // rule: openai-gpt5-1, provider: databricks_v2, priority: 15 if (provider === "databricks_v2" && (gpt5TokenMatchesGenerated(lower, "gpt-5.1") || gpt5TokenMatchesGenerated(lower, "gpt5.1") || gpt5TokenMatchesGenerated(lower, "gpt-5-1") || gpt5TokenMatchesGenerated(lower, "gpt5-1"))) { return { @@ -660,6 +715,17 @@ function lookupByFamilyRules(provider: string, normalized: string): CapabilityRe normalizationPolicy: "openai-standard", }; } + // rule: openai-gpt5-base, provider: databricks, priority: 10 + if (provider === "databricks" && (gpt5BaseMatchesGenerated(lower, "gpt-5") || gpt5BaseMatchesGenerated(lower, "gpt5"))) { + return { + registryLabel: "GPT-5", + thinkingMode: "none", + supportedEfforts: ["minimal", "low", "medium", "high"] as const, + defaultEffort: "medium", + databricksV2WireRoute: "not-applicable", + normalizationPolicy: "openai-standard", + }; + } // rule: openai-gpt5-base, provider: databricks_v2, priority: 10 if (provider === "databricks_v2" && (gpt5BaseMatchesGenerated(lower, "gpt-5") || gpt5BaseMatchesGenerated(lower, "gpt5"))) { return { diff --git a/desktop/src/features/agents/ui/usePersonaModelDiscovery.ts b/desktop/src/features/agents/ui/usePersonaModelDiscovery.ts index 5966c576a64..8f2ef49346a 100644 --- a/desktop/src/features/agents/ui/usePersonaModelDiscovery.ts +++ b/desktop/src/features/agents/ui/usePersonaModelDiscovery.ts @@ -65,7 +65,7 @@ export function getDiscoveredPersonaModelOptions( provider === "relay-mesh" ? "Default (auto)" : agentDefaultModel - ? `Default model (${resolveModelLabel(agentDefaultModel)})` + ? `Default model (${resolveModelLabel(agentDefaultModel, null, provider)})` : "Default model", }, ]; @@ -78,7 +78,7 @@ export function getDiscoveredPersonaModelOptions( ...defaultModelOption, ...explicitModels.map((model) => ({ id: model.id, - label: resolveModelLabel(model.id, model.name), + label: resolveModelLabel(model.id, model.name, provider), })), ]; } diff --git a/desktop/src/features/profile/ui/UserProfilePopover.tsx b/desktop/src/features/profile/ui/UserProfilePopover.tsx index d1a4b6c0e96..5f2fa887fe2 100644 --- a/desktop/src/features/profile/ui/UserProfilePopover.tsx +++ b/desktop/src/features/profile/ui/UserProfilePopover.tsx @@ -620,7 +620,13 @@ export function UserProfilePopover({ {runtimeLabel(relayAgent.agentType)} ) : null} {managedAgent?.model ? ( - {resolveModelLabel(managedAgent.model)} + + {resolveModelLabel( + managedAgent.model, + null, + managedAgent.provider, + )} + ) : null} {managedAgent?.acpCommand ? ( ACP: {managedAgent.acpCommand} diff --git a/scripts/MODELS_DEV_RECONCILIATION.md b/scripts/MODELS_DEV_RECONCILIATION.md index 3fff89a9ce7..59722e97e5c 100644 --- a/scripts/MODELS_DEV_RECONCILIATION.md +++ b/scripts/MODELS_DEV_RECONCILIATION.md @@ -1,7 +1,7 @@ # models.dev Reasoning Options Reconciliation Table -**Source queried**: https://models.dev/api.json (2026-07-31) -**Payload SHA-256**: `d5a4974cd69f19b0f67713acaa6bb3b16e920defdc07ecbdf6b0a936181bb0e0` +**Source queried**: https://models.dev/api.json (2026-07-31)
+**Payload SHA-256**: `d5a4974cd69f19b0f67713acaa6bb3b16e920defdc07ecbdf6b0a936181bb0e0`
**Policy (plan v4 §Behavior policy)**: models.dev `reasoning_options` become exact overrides. Each divergence from the current family rule result is reconciled here: either (a) adopted as an intentional correction or (b) rejected with a curation note. @@ -23,8 +23,8 @@ advertises only `[low, medium, high]` in its `reasoning_options`. The family rul `xhigh` are derived from the upstream OpenAI GPT-5.4 spec, which this Databricks endpoint does not expose. Provider-advertised wins per plan F1 policy. -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-mini"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]` -**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-4-mini"` +**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-mini"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`
+**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-4-mini"`
**Test vector**: `resolver-exact-raw-id-hit` in `scripts/normative-corpus.json` --- @@ -38,7 +38,7 @@ not expose. Provider-advertised wins per plan F1 policy. **Rationale**: Same as `databricks-gpt-5-4-mini`. The nano variant exposes the same restricted effort set. Provider-advertised wins. -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-nano"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]` +**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-nano"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-4-nano"` --- @@ -54,7 +54,7 @@ effort set. Provider-advertised wins. derived from the upstream OpenAI GPT-5.6 spec, which this Databricks endpoint does not expose. Provider-advertised wins per plan F1 policy. -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-6-sol"].reasoning_options = [{"type":"effort","values":["low","medium","high","max"]}]` +**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-6-sol"].reasoning_options = [{"type":"effort","values":["low","medium","high","max"]}]`
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-6-sol"` --- @@ -70,7 +70,7 @@ Provider-advertised wins per plan F1 policy. derived from the upstream OpenAI GPT-5.5 spec, which this Databricks endpoint does not expose. Provider-advertised wins per plan F1 policy. -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-5"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]` +**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-5"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-5"` --- @@ -86,7 +86,7 @@ a different capability axis (extended thinking token budget), not an effort-leve There is no effort divergence to reconcile. The effort capabilities for this model come from the `anthropic-adaptive-xhigh-opus-4-7` family rule (Anthropic extended-thinking support table). -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-claude-opus-4-7"].reasoning_options = [{"type":"budget_tokens","min":1024}]` +**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-claude-opus-4-7"].reasoning_options = [{"type":"budget_tokens","min":1024}]`
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-claude-opus-4-7"` --- diff --git a/scripts/generated-model-capabilities-coverage.json b/scripts/generated-model-capabilities-coverage.json index b5dd3c54fed..22cf77db20a 100644 --- a/scripts/generated-model-capabilities-coverage.json +++ b/scripts/generated-model-capabilities-coverage.json @@ -601,6 +601,50 @@ } } }, + { + "note": "family rule openai-gpt5-pro / provider databricks", + "provider": "databricks", + "model": "gpt-5-pro", + "resolved": { + "registry_label": "GPT-5 Pro", + "thinking_mode": "none", + "supported_efforts": [ + "high" + ], + "default_effort": "high", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-pro", + "rule_priority": 20, + "normalized_alias": "gpt-5-pro", + "raw_model_id": "gpt-5-pro" + } + } + }, + { + "note": "family rule openai-gpt5-pro alias gpt5-pro / provider databricks", + "provider": "databricks", + "model": "gpt5-pro", + "resolved": { + "registry_label": "GPT-5 Pro", + "thinking_mode": "none", + "supported_efforts": [ + "high" + ], + "default_effort": "high", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-pro", + "rule_priority": 20, + "normalized_alias": "gpt5-pro", + "raw_model_id": "gpt5-pro" + } + } + }, { "note": "family rule openai-gpt5-pro / provider databricks_v2", "provider": "databricks_v2", @@ -753,6 +797,114 @@ } } }, + { + "note": "family rule openai-gpt5-6 / provider databricks", + "provider": "databricks", + "model": "gpt-5.6", + "resolved": { + "registry_label": "GPT-5.6", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-6", + "rule_priority": 15, + "normalized_alias": "gpt-5.6", + "raw_model_id": "gpt-5.6" + } + } + }, + { + "note": "family rule openai-gpt5-6 alias gpt5.6 / provider databricks", + "provider": "databricks", + "model": "gpt5.6", + "resolved": { + "registry_label": "GPT-5.6", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-6", + "rule_priority": 15, + "normalized_alias": "gpt5.6", + "raw_model_id": "gpt5.6" + } + } + }, + { + "note": "family rule openai-gpt5-6 alias gpt-5-6 / provider databricks", + "provider": "databricks", + "model": "gpt-5-6", + "resolved": { + "registry_label": "GPT-5.6", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-6", + "rule_priority": 15, + "normalized_alias": "gpt-5-6", + "raw_model_id": "gpt-5-6" + } + } + }, + { + "note": "family rule openai-gpt5-6 alias gpt5-6 / provider databricks", + "provider": "databricks", + "model": "gpt5-6", + "resolved": { + "registry_label": "GPT-5.6", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-6", + "rule_priority": 15, + "normalized_alias": "gpt5-6", + "raw_model_id": "gpt5-6" + } + } + }, { "note": "family rule openai-gpt5-6 / provider databricks_v2", "provider": "databricks_v2", @@ -965,6 +1117,110 @@ } } }, + { + "note": "family rule openai-gpt5-5 / provider databricks", + "provider": "databricks", + "model": "gpt-5.5", + "resolved": { + "registry_label": "GPT-5.5", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-5", + "rule_priority": 15, + "normalized_alias": "gpt-5.5", + "raw_model_id": "gpt-5.5" + } + } + }, + { + "note": "family rule openai-gpt5-5 alias gpt5.5 / provider databricks", + "provider": "databricks", + "model": "gpt5.5", + "resolved": { + "registry_label": "GPT-5.5", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-5", + "rule_priority": 15, + "normalized_alias": "gpt5.5", + "raw_model_id": "gpt5.5" + } + } + }, + { + "note": "family rule openai-gpt5-5 alias gpt-5-5 / provider databricks", + "provider": "databricks", + "model": "gpt-5-5", + "resolved": { + "registry_label": "GPT-5.5", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-5", + "rule_priority": 15, + "normalized_alias": "gpt-5-5", + "raw_model_id": "gpt-5-5" + } + } + }, + { + "note": "family rule openai-gpt5-5 alias gpt5-5 / provider databricks", + "provider": "databricks", + "model": "gpt5-5", + "resolved": { + "registry_label": "GPT-5.5", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-5", + "rule_priority": 15, + "normalized_alias": "gpt5-5", + "raw_model_id": "gpt5-5" + } + } + }, { "note": "family rule openai-gpt5-5 / provider databricks_v2", "provider": "databricks_v2", @@ -1173,6 +1429,110 @@ } } }, + { + "note": "family rule openai-gpt5-4 / provider databricks", + "provider": "databricks", + "model": "gpt-5.4", + "resolved": { + "registry_label": "GPT-5.4", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-4", + "rule_priority": 15, + "normalized_alias": "gpt-5.4", + "raw_model_id": "gpt-5.4" + } + } + }, + { + "note": "family rule openai-gpt5-4 alias gpt5.4 / provider databricks", + "provider": "databricks", + "model": "gpt5.4", + "resolved": { + "registry_label": "GPT-5.4", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-4", + "rule_priority": 15, + "normalized_alias": "gpt5.4", + "raw_model_id": "gpt5.4" + } + } + }, + { + "note": "family rule openai-gpt5-4 alias gpt-5-4 / provider databricks", + "provider": "databricks", + "model": "gpt-5-4", + "resolved": { + "registry_label": "GPT-5.4", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-4", + "rule_priority": 15, + "normalized_alias": "gpt-5-4", + "raw_model_id": "gpt-5-4" + } + } + }, + { + "note": "family rule openai-gpt5-4 alias gpt5-4 / provider databricks", + "provider": "databricks", + "model": "gpt5-4", + "resolved": { + "registry_label": "GPT-5.4", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-4", + "rule_priority": 15, + "normalized_alias": "gpt5-4", + "raw_model_id": "gpt5-4" + } + } + }, { "note": "family rule openai-gpt5-4 / provider databricks_v2", "provider": "databricks_v2", @@ -1377,6 +1737,106 @@ } } }, + { + "note": "family rule openai-gpt5-1 / provider databricks", + "provider": "databricks", + "model": "gpt-5.1", + "resolved": { + "registry_label": "GPT-5.1", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high" + ], + "default_effort": "none", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-1", + "rule_priority": 15, + "normalized_alias": "gpt-5.1", + "raw_model_id": "gpt-5.1" + } + } + }, + { + "note": "family rule openai-gpt5-1 alias gpt5.1 / provider databricks", + "provider": "databricks", + "model": "gpt5.1", + "resolved": { + "registry_label": "GPT-5.1", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high" + ], + "default_effort": "none", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-1", + "rule_priority": 15, + "normalized_alias": "gpt5.1", + "raw_model_id": "gpt5.1" + } + } + }, + { + "note": "family rule openai-gpt5-1 alias gpt-5-1 / provider databricks", + "provider": "databricks", + "model": "gpt-5-1", + "resolved": { + "registry_label": "GPT-5.1", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high" + ], + "default_effort": "none", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-1", + "rule_priority": 15, + "normalized_alias": "gpt-5-1", + "raw_model_id": "gpt-5-1" + } + } + }, + { + "note": "family rule openai-gpt5-1 alias gpt5-1 / provider databricks", + "provider": "databricks", + "model": "gpt5-1", + "resolved": { + "registry_label": "GPT-5.1", + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high" + ], + "default_effort": "none", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-1", + "rule_priority": 15, + "normalized_alias": "gpt5-1", + "raw_model_id": "gpt5-1" + } + } + }, { "note": "family rule openai-gpt5-1 / provider databricks_v2", "provider": "databricks_v2", @@ -1527,6 +1987,56 @@ } } }, + { + "note": "family rule openai-gpt5-base / provider databricks", + "provider": "databricks", + "model": "gpt-5", + "resolved": { + "registry_label": "GPT-5", + "thinking_mode": "none", + "supported_efforts": [ + "minimal", + "low", + "medium", + "high" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-base", + "rule_priority": 10, + "normalized_alias": "gpt-5", + "raw_model_id": "gpt-5" + } + } + }, + { + "note": "family rule openai-gpt5-base alias gpt5 / provider databricks", + "provider": "databricks", + "model": "gpt5", + "resolved": { + "registry_label": "GPT-5", + "thinking_mode": "none", + "supported_efforts": [ + "minimal", + "low", + "medium", + "high" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable", + "normalization_policy": "openai-standard", + "_provenance": { + "source": "family", + "rule_id": "openai-gpt5-base", + "rule_priority": 10, + "normalized_alias": "gpt5", + "raw_model_id": "gpt5" + } + } + }, { "note": "family rule openai-gpt5-base / provider databricks_v2", "provider": "databricks_v2", diff --git a/scripts/model-capabilities.json b/scripts/model-capabilities.json index 232fcc1befd..baac0a5d7cd 100644 --- a/scripts/model-capabilities.json +++ b/scripts/model-capabilities.json @@ -257,6 +257,7 @@ ], "providers": [ "openai", + "databricks", "databricks_v2" ], "match_priority": 20, @@ -280,6 +281,7 @@ ], "providers": [ "openai", + "databricks", "databricks_v2" ], "match_priority": 15, @@ -308,6 +310,7 @@ ], "providers": [ "openai", + "databricks", "databricks_v2" ], "match_priority": 15, @@ -335,6 +338,7 @@ ], "providers": [ "openai", + "databricks", "databricks_v2" ], "match_priority": 15, @@ -362,6 +366,7 @@ ], "providers": [ "openai", + "databricks", "databricks_v2" ], "match_priority": 15, @@ -386,6 +391,7 @@ ], "providers": [ "openai", + "databricks", "databricks_v2" ], "match_priority": 10, diff --git a/scripts/normative-corpus.json b/scripts/normative-corpus.json index fb5ef8b6630..cf44b01c198 100644 --- a/scripts/normative-corpus.json +++ b/scripts/normative-corpus.json @@ -9,7 +9,11 @@ "raw_model_id": "claude-3-7-sonnet-20250219", "expect": { "thinking_mode": "manual-budget", - "supported_efforts": ["low", "medium", "high"], + "supported_efforts": [ + "low", + "medium", + "high" + ], "default_effort": null, "databricks_v2_wire_route": "not-applicable" } @@ -20,7 +24,11 @@ "raw_model_id": "claude-opus-4-5", "expect": { "thinking_mode": "manual-budget", - "supported_efforts": ["low", "medium", "high"], + "supported_efforts": [ + "low", + "medium", + "high" + ], "default_effort": null, "databricks_v2_wire_route": "not-applicable" } @@ -31,7 +39,13 @@ "raw_model_id": "claude-opus-4-7", "expect": { "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "xhigh", "max"], + "supported_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } @@ -42,7 +56,13 @@ "raw_model_id": "claude-opus-4-8", "expect": { "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "xhigh", "max"], + "supported_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } @@ -53,7 +73,13 @@ "raw_model_id": "claude-sonnet-5-20260101", "expect": { "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "xhigh", "max"], + "supported_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } @@ -64,7 +90,13 @@ "raw_model_id": "claude-fable-5", "expect": { "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "xhigh", "max"], + "supported_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } @@ -75,7 +107,13 @@ "raw_model_id": "claude-mythos-5", "expect": { "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "xhigh", "max"], + "supported_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } @@ -86,7 +124,12 @@ "raw_model_id": "claude-opus-4-6", "expect": { "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "max"], + "supported_efforts": [ + "low", + "medium", + "high", + "max" + ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } @@ -97,7 +140,12 @@ "raw_model_id": "claude-sonnet-4-6", "expect": { "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "max"], + "supported_efforts": [ + "low", + "medium", + "high", + "max" + ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } @@ -108,7 +156,12 @@ "raw_model_id": "claude-mythos-preview", "expect": { "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "max"], + "supported_efforts": [ + "low", + "medium", + "high", + "max" + ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } @@ -122,7 +175,13 @@ "raw_model_id": "", "expect": { "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "xhigh", "max"], + "supported_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } @@ -133,7 +192,13 @@ "raw_model_id": "claude-ultra-9000", "expect": { "thinking_mode": "omit-fields", - "supported_efforts": ["low", "medium", "high", "xhigh", "max"], + "supported_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } @@ -147,7 +212,9 @@ "raw_model_id": "gpt-5-pro", "expect": { "thinking_mode": "none", - "supported_efforts": ["high"], + "supported_efforts": [ + "high" + ], "default_effort": "high", "databricks_v2_wire_route": "not-applicable" } @@ -158,7 +225,14 @@ "raw_model_id": "gpt-5.6", "expect": { "thinking_mode": "none", - "supported_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], "default_effort": "medium", "databricks_v2_wire_route": "not-applicable" } @@ -169,7 +243,14 @@ "raw_model_id": "gpt-5-6", "expect": { "thinking_mode": "none", - "supported_efforts": ["none", "low", "medium", "high", "xhigh", "max"], + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], "default_effort": "medium", "databricks_v2_wire_route": "not-applicable" } @@ -180,7 +261,13 @@ "raw_model_id": "gpt-5.5", "expect": { "thinking_mode": "none", - "supported_efforts": ["none", "low", "medium", "high", "xhigh"], + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ], "default_effort": "medium", "databricks_v2_wire_route": "not-applicable" } @@ -191,7 +278,13 @@ "raw_model_id": "gpt-5.4", "expect": { "thinking_mode": "none", - "supported_efforts": ["none", "low", "medium", "high", "xhigh"], + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ], "default_effort": "medium", "databricks_v2_wire_route": "not-applicable" } @@ -202,7 +295,12 @@ "raw_model_id": "gpt-5.1", "expect": { "thinking_mode": "none", - "supported_efforts": ["none", "low", "medium", "high"], + "supported_efforts": [ + "none", + "low", + "medium", + "high" + ], "default_effort": "none", "databricks_v2_wire_route": "not-applicable" } @@ -213,13 +311,18 @@ "raw_model_id": "gpt-5", "expect": { "thinking_mode": "none", - "supported_efforts": ["minimal", "low", "medium", "high"], + "supported_efforts": [ + "minimal", + "low", + "medium", + "high" + ], "default_effort": "medium", "databricks_v2_wire_route": "not-applicable" } }, { - "_group": "OpenAI adversarial — gpt5 boundary-aware matching (ported from config.rs tests)" + "_group": "OpenAI adversarial \u2014 gpt5 boundary-aware matching (ported from config.rs tests)" }, { "id": "openai-gpt5-1106-should-not-match-base", @@ -227,7 +330,12 @@ "raw_model_id": "gpt-5-1106", "_note": "gpt-5-1106: '-1106' is a 4-digit date segment, NOT a short version (gpt5-base rejects only 1-3 digit suffixes). Must match base table [minimal,low,medium,high], NOT fall through to unknown.", "expect": { - "supported_efforts": ["minimal", "low", "medium", "high"] + "supported_efforts": [ + "minimal", + "low", + "medium", + "high" + ] } }, { @@ -236,7 +344,12 @@ "raw_model_id": "gpt-5-4o", "_note": "gpt-5-4o: '4o' after '-' is NOT a short numeric suffix (it contains a letter). Must match gpt5-base. Crucially, must NOT match gpt-5.4 (the '4' is followed by 'o', not boundary char).", "expect": { - "supported_efforts": ["minimal", "low", "medium", "high"] + "supported_efforts": [ + "minimal", + "low", + "medium", + "high" + ] } }, { @@ -245,7 +358,9 @@ "raw_model_id": "gpt-5-pro", "_note": "gpt-5-pro should hit gpt5-pro rule (priority 20), NOT gpt-5 base.", "expect": { - "supported_efforts": ["high"], + "supported_efforts": [ + "high" + ], "default_effort": "high" } }, @@ -253,22 +368,34 @@ "id": "openai-multi-digit-version-gpt5-10", "provider": "openai", "raw_model_id": "gpt-5-10", - "_note": "gpt-5-10 — two-digit suffix prevents gpt5-base match. Falls through to unknown.", + "_note": "gpt-5-10 \u2014 two-digit suffix prevents gpt5-base match. Falls through to unknown.", "expect": { - "supported_efforts": ["none", "minimal", "low", "medium", "high", "xhigh"] + "supported_efforts": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh" + ] } }, { "id": "openai-gpt5-date-suffix", "provider": "openai", "raw_model_id": "gpt-5-20260101", - "_note": "gpt-5-20260101 — long numeric suffix after base: '20260101' is 8 digits, beyond 1-3 digit reject, should hit gpt5-base.", + "_note": "gpt-5-20260101 \u2014 long numeric suffix after base: '20260101' is 8 digits, beyond 1-3 digit reject, should hit gpt5-base.", "expect": { - "supported_efforts": ["minimal", "low", "medium", "high"] + "supported_efforts": [ + "minimal", + "low", + "medium", + "high" + ] } }, { - "_group": "DatabricksV2 — segment-based routing (ported from llm.rs tests)" + "_group": "DatabricksV2 \u2014 segment-based routing (ported from llm.rs tests)" }, { "id": "dbv2-gpt5-route-openai-responses", @@ -276,7 +403,13 @@ "raw_model_id": "gpt-5.5", "expect": { "databricks_v2_wire_route": "openai-responses", - "supported_efforts": ["none", "low", "medium", "high", "xhigh"] + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ] } }, { @@ -286,14 +419,20 @@ "expect": { "databricks_v2_wire_route": "anthropic-messages", "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "xhigh", "max"] + "supported_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ] } }, { "id": "dbv2-claude-prefix-stripped", "provider": "databricks_v2", "raw_model_id": "databricks-claude-opus-4-7", - "_note": "databricks- prefix stripped → claude-opus-4-7 → Anthropic route", + "_note": "databricks- prefix stripped \u2192 claude-opus-4-7 \u2192 Anthropic route", "expect": { "databricks_v2_wire_route": "anthropic-messages", "thinking_mode": "adaptive" @@ -303,29 +442,41 @@ "id": "dbv2-goose-claude-prefix-stripped", "provider": "databricks_v2", "raw_model_id": "goose-claude-fable-5", - "_note": "goose- prefix stripped → claude-fable-5 → Anthropic adaptive+xhigh", + "_note": "goose- prefix stripped \u2192 claude-fable-5 \u2192 Anthropic adaptive+xhigh", "expect": { "databricks_v2_wire_route": "anthropic-messages", "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "xhigh", "max"] + "supported_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ] } }, { "id": "dbv2-team-prefix-stripped", "provider": "databricks_v2", "raw_model_id": "team-x-claude-opus-4-7", - "_note": "team-x- prefix stripped → claude-opus-4-7 → Anthropic route", + "_note": "team-x- prefix stripped \u2192 claude-opus-4-7 \u2192 Anthropic route", "expect": { "databricks_v2_wire_route": "anthropic-messages", "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "xhigh", "max"] + "supported_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ] } }, { "id": "dbv2-consolidated-llama-not-sol", "provider": "databricks_v2", "raw_model_id": "consolidated-llama", - "_note": "segment test: 'sol' is a SUBSTRING of 'consolidated' — must NOT match DATABRICKS_V2_OPENAI_CODE_NAMES 'sol'. Falls through to mlflow-chat.", + "_note": "segment test: 'sol' is a SUBSTRING of 'consolidated' \u2014 must NOT match DATABRICKS_V2_OPENAI_CODE_NAMES 'sol'. Falls through to mlflow-chat.", "expect": { "databricks_v2_wire_route": "mlflow-chat" } @@ -334,7 +485,7 @@ "id": "dbv2-terraform-coder-not-terra", "provider": "databricks_v2", "raw_model_id": "terraform-coder", - "_note": "segment test: 'terra' is a prefix of 'terraform' — must NOT match 'terra' code name. Falls through to mlflow-chat.", + "_note": "segment test: 'terra' is a prefix of 'terraform' \u2014 must NOT match 'terra' code name. Falls through to mlflow-chat.", "expect": { "databricks_v2_wire_route": "mlflow-chat" } @@ -367,7 +518,7 @@ } }, { - "_group": "P2-A resolver-contract vectors (plan v4 §Resolver contract)" + "_group": "P2-A resolver-contract vectors (plan v4 \u00a7Resolver contract)" }, { "id": "resolver-exact-raw-id-hit", @@ -375,16 +526,26 @@ "raw_model_id": "databricks-gpt-5-4-mini", "_note": "Exact record exists. Must return exact Databricks override: low|medium|high (not family's none+xhigh).", "expect": { - "supported_efforts": ["low", "medium", "high"] + "supported_efforts": [ + "low", + "medium", + "high" + ] } }, { "id": "resolver-prefixed-alias-misses-exact", "provider": "databricks_v2", "raw_model_id": "team-x-databricks-gpt-5-4-mini", - "_note": "Prefixed alias of an exact ID. Raw exact lookup MUST miss (key is team-x-..., not databricks-...). Falls to family rules (gpt5-4 family → none+xhigh).", + "_note": "Prefixed alias of an exact ID. Raw exact lookup MUST miss (key is team-x-..., not databricks-...). Falls to family rules (gpt5-4 family \u2192 none+xhigh).", "expect": { - "supported_efforts": ["none", "low", "medium", "high", "xhigh"] + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ] } }, { @@ -400,9 +561,14 @@ "id": "resolver-exact-efforts-plus-family-route", "provider": "databricks_v2", "raw_model_id": "databricks-gpt-5-6-sol", - "_note": "Exact record with efforts from models.dev (low|medium|high|max — provider-advertised, no none/xhigh). Route materialized from gpt5-6 family rule (openai-responses). Must return both, complete.", - "expect": { - "supported_efforts": ["low", "medium", "high", "max"], + "_note": "Exact record with efforts from models.dev (low|medium|high|max \u2014 provider-advertised, no none/xhigh). Route materialized from gpt5-6 family rule (openai-responses). Must return both, complete.", + "expect": { + "supported_efforts": [ + "low", + "medium", + "high", + "max" + ], "databricks_v2_wire_route": "openai-responses" } }, @@ -410,9 +576,13 @@ "id": "dbv2-gpt5-5-exact-override", "provider": "databricks_v2", "raw_model_id": "databricks-gpt-5-5", - "_note": "Exact record adopts models.dev advertised set [low,medium,high]. Family rule (gpt5-5) has none+xhigh — provider-advertised wins per plan F1.", + "_note": "Exact record adopts models.dev advertised set [low,medium,high]. Family rule (gpt5-5) has none+xhigh \u2014 provider-advertised wins per plan F1.", "expect": { - "supported_efforts": ["low", "medium", "high"], + "supported_efforts": [ + "low", + "medium", + "high" + ], "databricks_v2_wire_route": "openai-responses" } }, @@ -426,7 +596,15 @@ "_note": "DBv2 blank: route-unknown, all 7 efforts, default medium.", "expect": { "databricks_v2_wire_route": "route-unknown", - "supported_efforts": ["none", "minimal", "low", "medium", "high", "xhigh", "max"], + "supported_efforts": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max" + ], "default_effort": "medium" } }, @@ -437,7 +615,14 @@ "_note": "DBv2 concrete-unknown: mlflow-chat, all-except-max (6 efforts).", "expect": { "databricks_v2_wire_route": "mlflow-chat", - "supported_efforts": ["none", "minimal", "low", "medium", "high", "xhigh"] + "supported_efforts": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh" + ] } }, { @@ -447,7 +632,14 @@ "_note": "OpenAI blank: not-applicable route, all-except-max, medium default.", "expect": { "databricks_v2_wire_route": "not-applicable", - "supported_efforts": ["none", "minimal", "low", "medium", "high", "xhigh"], + "supported_efforts": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh" + ], "default_effort": "medium" } }, @@ -458,7 +650,14 @@ "_note": "OpenAI concrete unknown (unverified family): not-applicable route, all-except-max, medium default.", "expect": { "databricks_v2_wire_route": "not-applicable", - "supported_efforts": ["none", "minimal", "low", "medium", "high", "xhigh"], + "supported_efforts": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh" + ], "default_effort": "medium" } }, @@ -469,7 +668,13 @@ "_note": "Anthropic blank: assume adaptive with full support (incl. xhigh).", "expect": { "thinking_mode": "adaptive", - "supported_efforts": ["low", "medium", "high", "xhigh", "max"], + "supported_efforts": [ + "low", + "medium", + "high", + "xhigh", + "max" + ], "default_effort": "high" } }, @@ -481,5 +686,110 @@ "expect": { "thinking_mode": "omit-fields" } + }, + { + "_group": "Legacy Databricks provider (P3 effort rules)" + }, + { + "id": "databricks-gpt5-pro-effort", + "provider": "databricks", + "raw_model_id": "databricks-gpt-5-pro", + "_note": "Legacy databricks GPT-5 Pro: same effort set as openai/gpt-5-pro \u2014 only [high], default high. Wire route not-applicable.", + "expect": { + "databricks_v2_wire_route": "not-applicable", + "supported_efforts": [ + "high" + ], + "default_effort": "high" + } + }, + { + "id": "databricks-gpt5-6-effort", + "provider": "databricks", + "raw_model_id": "databricks-gpt-5.6", + "_note": "Legacy databricks GPT-5.6: same effort set as openai/gpt-5.6 \u2014 [none,low,medium,high,xhigh,max], default medium. Wire route not-applicable.", + "expect": { + "databricks_v2_wire_route": "not-applicable", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh", + "max" + ], + "default_effort": "medium" + } + }, + { + "id": "databricks-gpt5-1-effort", + "provider": "databricks", + "raw_model_id": "databricks-gpt-5.1", + "_note": "Legacy databricks GPT-5.1: same effort set as openai/gpt-5.1 \u2014 [none,low,medium,high], default none. Wire route not-applicable.", + "expect": { + "databricks_v2_wire_route": "not-applicable", + "supported_efforts": [ + "none", + "low", + "medium", + "high" + ], + "default_effort": "none" + } + }, + { + "_group": "openai-compat alias canonicalization (Thufir P3 corrective action 1)", + "_note": "Rust normalizes openai-compat \u2192 Provider::OpenAi before reaching normalize_effort_for_provider. TS PROVIDER_ALIASES must match so the UI effort table equals the Rust request behavior. Interpreters must canonicalize openai-compat \u2192 openai before resolving; the expected values are identical to the corresponding openai vectors." + }, + { + "id": "openai-compat-gpt-5-pro", + "provider": "openai-compat", + "raw_model_id": "gpt-5-pro", + "_note": "openai-compat/gpt-5-pro must resolve identically to openai/gpt-5-pro: [high] only, default high.", + "expect": { + "thinking_mode": "none", + "supported_efforts": [ + "high" + ], + "default_effort": "high", + "databricks_v2_wire_route": "not-applicable" + } + }, + { + "id": "openai-compat-gpt-5-5", + "provider": "openai-compat", + "raw_model_id": "gpt-5.5", + "_note": "openai-compat/gpt-5.5 must resolve identically to openai/gpt-5.5: [none,low,medium,high,xhigh], default medium.", + "expect": { + "thinking_mode": "none", + "supported_efforts": [ + "none", + "low", + "medium", + "high", + "xhigh" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable" + } + }, + { + "id": "openai-compat-empty-model", + "provider": "openai-compat", + "raw_model_id": "", + "_note": "openai-compat with blank model: resolves identically to openai unknown \u2014 all-except-max, default medium.", + "expect": { + "thinking_mode": "none", + "supported_efforts": [ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh" + ], + "default_effort": "medium", + "databricks_v2_wire_route": "not-applicable" + } } ] diff --git a/scripts/run-corpus.mjs b/scripts/run-corpus.mjs index 561dc2cdf0a..bcec37cb802 100644 --- a/scripts/run-corpus.mjs +++ b/scripts/run-corpus.mjs @@ -30,6 +30,19 @@ const corpus = JSON.parse( readFileSync(join(repoRoot, "scripts", "normative-corpus.json"), "utf8"), ); +// ----- Provider alias canonicalization ----- +// Mirrors production canonicalizeProvider() in desktop/src/features/agents/lib/formatAgentModelLabel.ts. +// Applied before every generated lookup so alias vectors (e.g. "openai-compat") pass both interpreters. +const PROVIDER_ALIASES = { + "databricks-v2": "databricks_v2", + "openai-compat": "openai", +}; + +function canonicalizeProvider(provider) { + const normalized = (provider ?? "").trim().toLowerCase(); + return PROVIDER_ALIASES[normalized] ?? normalized; +} + // ----- Run corpus ----- let passed = 0; @@ -41,7 +54,7 @@ for (const entry of corpus) { if (!entry.expect) continue; // resolveModelCapabilities returns camelCase keys (registryLabel, thinkingMode, etc.) - const result = resolveModelCapabilities(entry.provider, entry.raw_model_id); + const result = resolveModelCapabilities(canonicalizeProvider(entry.provider), entry.raw_model_id); const expect = entry.expect; const failures = []; diff --git a/scripts/run-differential.mjs b/scripts/run-differential.mjs new file mode 100755 index 00000000000..a55d2aa0356 --- /dev/null +++ b/scripts/run-differential.mjs @@ -0,0 +1,239 @@ +#!/usr/bin/env node +/** + * Phase-2 differential harness — compare old buzzAgentConfig.ts effort logic with + * the new generated modelCapabilities.ts interpreter over: + * 1. The 36-entry effortTable.fixture.json (cross-boundary Rust/TS fixture) + * 2. The 45-vector normative corpus (scripts/normative-corpus.json) + * 3. The catalog-sample fixture (scripts/catalog-sample-fixture.json) + * + * Equality is required except for entries in the committed allowlist of intentional + * F1 corrections (models.dev provider-capability reconciliations). + * + * Usage: node --experimental-strip-types scripts/run-differential.mjs [--verbose] + * Exits 0 on all-pass (modulo allowlist), 1 on unexpected divergence or unexercised allowlist entry. + */ + +import { readFileSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = join(__dirname, ".."); +const VERBOSE = process.argv.includes("--verbose"); + +// --------------------------------------------------------------------------- +// Import both interpreters +// --------------------------------------------------------------------------- + +// NEW: generated capability module +const { resolveModelCapabilities: resolveNew } = await import( + join(repoRoot, "desktop", "src", "features", "agents", "ui", "modelCapabilities.ts") +); + +// OLD: buzzAgentConfig.ts effort config +const { getProviderEffortConfig_oldHandTable: getOldEffortConfig } = await import( + join(repoRoot, "desktop", "src", "features", "agents", "ui", "buzzAgentConfig.ts") +); + +// --------------------------------------------------------------------------- +// Intentional corrections allowlist (Phase 1 F1 reconciliations) +// Each entry: { provider, raw_model_id, reason } +// --------------------------------------------------------------------------- +const ALLOWLIST = [ + { + provider: "databricks_v2", + raw_model_id: "databricks-gpt-5-5", + axes: ["supported_efforts"], + reason: "Phase 1 ADOPT: models.dev d5a4974c advertises [low,medium,high]; old returns [none,low,medium,high,xhigh]", + }, + { + provider: "databricks_v2", + raw_model_id: "databricks-gpt-5-4-mini", + axes: ["supported_efforts"], + reason: "Phase 1 ADOPT: models.dev advertises [low,medium,high]; old returns [none,low,medium,high,xhigh]", + }, + { + provider: "databricks_v2", + raw_model_id: "databricks-gpt-5-4-nano", + axes: ["supported_efforts"], + reason: "Phase 1 ADOPT: models.dev advertises [low,medium,high]; old returns [none,low,medium,high,xhigh]", + }, + { + provider: "databricks_v2", + raw_model_id: "databricks-gpt-5-6-sol", + axes: ["supported_efforts"], + reason: "Phase 1 ADOPT: models.dev advertises [low,medium,high,max]; old returns [none,low,medium,high,xhigh,max]", + }, + { + provider: "databricks_v2", + raw_model_id: "goose-opus-5", + axes: ["supported_efforts", "default_effort"], + reason: "Phase 1 correction: 'opus' is a named DBv2 segment → anthropic-messages route; old config.rs disagreed with llm.rs (corpus note dbv2-goose-opus-5-is-anthropic). Generated adopts anthropic adaptive-xhigh capabilities consistent with the wire route.", + }, +]; + +// Track which allowlist entries are actually exercised (suppressed a divergence). +// Keyed as "provider:raw_model_id:axis". +const allowlistHits = new Set(); + +function isAllowlisted(provider, rawModelId, axis) { + const entry = ALLOWLIST.find( + (e) => + e.provider === provider && + e.raw_model_id === rawModelId && + e.axes.includes(axis), + ); + if (entry) { + allowlistHits.add(`${provider}:${rawModelId}:${axis}`); + return true; + } + return false; +} + +// --------------------------------------------------------------------------- +// Comparison helpers +// --------------------------------------------------------------------------- + +/** + * Compare effort axes from both interpreters for one (provider, model) pair. + * Returns array of divergence objects. + */ +function compareEffortAxes(provider, model) { + const newResult = resolveNew(provider, model); + const oldResult = getOldEffortConfig(provider, model); + + const divergences = []; + + // supported_efforts + const newEfforts = newResult.supportedEfforts ?? []; + const oldEfforts = oldResult?.validValues ?? []; + if (JSON.stringify(newEfforts) !== JSON.stringify(oldEfforts)) { + if (!isAllowlisted(provider, model, "supported_efforts")) { + divergences.push({ + axis: "supported_efforts", + old: oldEfforts, + new: newEfforts, + }); + } + } + + // default_effort + const newDefault = newResult.defaultEffort ?? null; + const oldDefault = oldResult?.defaultValue ?? null; + if (newDefault !== oldDefault) { + if (!isAllowlisted(provider, model, "default_effort")) { + divergences.push({ + axis: "default_effort", + old: oldDefault, + new: newDefault, + }); + } + } + + return divergences; +} + +// --------------------------------------------------------------------------- +// Test suites +// --------------------------------------------------------------------------- + +let totalChecks = 0; +let totalDivergences = 0; + +function runCheck(label, provider, model) { + totalChecks++; + const divs = compareEffortAxes(provider, model); + if (divs.length > 0) { + totalDivergences += divs.length; + for (const d of divs) { + console.error( + `DIVERGE [${label}] provider=${provider} model=${model} axis=${d.axis}\n` + + ` old: ${JSON.stringify(d.old)}\n` + + ` new: ${JSON.stringify(d.new)}`, + ); + } + } else if (VERBOSE) { + console.log(`OK [${label}] provider=${provider} model=${model}`); + } +} + +// 1. effortTable.fixture.json +console.log("--- effortTable.fixture.json ---"); +const fixture = JSON.parse( + readFileSync( + join(repoRoot, "desktop", "src", "features", "agents", "ui", "effortTable.fixture.json"), + "utf8", + ), +); +for (const entry of fixture) { + if (!entry.provider) continue; + runCheck("fixture", entry.provider, entry.model ?? ""); +} + +// 2. normative-corpus.json (effort axes only) +console.log("--- normative-corpus.json ---"); +const corpus = JSON.parse( + readFileSync(join(repoRoot, "scripts", "normative-corpus.json"), "utf8"), +); +for (const entry of corpus) { + if (entry._group) continue; + if (!entry.provider || !entry.expect) continue; + if (!entry.expect.supported_efforts && !entry.expect.default_effort) continue; + runCheck("corpus", entry.provider, entry.raw_model_id ?? ""); +} + +// 3. catalog-sample-fixture.json (exact records from pinned models.dev payload) +console.log("--- catalog-sample-fixture.json ---"); +const catalogFixture = JSON.parse( + readFileSync(join(repoRoot, "scripts", "catalog-sample-fixture.json"), "utf8"), +); +for (const ep of catalogFixture.endpoints ?? []) { + if (!ep.name) continue; + // All catalog endpoints are databricks_v2 provider + runCheck("catalog-sample", "databricks_v2", ep.name); +} + +// --------------------------------------------------------------------------- +// Summary +// --------------------------------------------------------------------------- + +// Count total allowlist axis slots expected to be hit +const totalAllowlistSlots = ALLOWLIST.reduce((n, e) => n + e.axes.length, 0); +const allowlistHitCount = allowlistHits.size; + +// Detect stale allowlist entries (declared but never actually suppressed a divergence) +const staleEntries = []; +for (const entry of ALLOWLIST) { + for (const axis of entry.axes) { + const key = `${entry.provider}:${entry.raw_model_id}:${axis}`; + if (!allowlistHits.has(key)) { + staleEntries.push({ ...entry, axis }); + } + } +} + +console.log( + `\nDifferential: ${totalChecks} checks, ${totalDivergences} unexpected divergences, ${allowlistHitCount}/${totalAllowlistSlots} allowlist slots exercised`, +); + +if (staleEntries.length > 0) { + for (const e of staleEntries) { + console.error( + `STALE_ALLOWLIST provider=${e.provider} model=${e.raw_model_id} axis=${e.axis} — entry never fired; remove or update it`, + ); + } +} + +if (totalDivergences > 0) { + console.error( + `FAIL: ${totalDivergences} unexpected divergence(s) — see output above`, + ); + process.exit(1); +} else if (staleEntries.length > 0) { + console.error( + `FAIL: ${staleEntries.length} stale allowlist entry(ies) — entries that never suppress a divergence mask future regressions`, + ); + process.exit(1); +} else { + console.log("PASS: old and new effort logic agree on all non-allowlisted entries"); +}