diff --git a/.env.example b/.env.example index b571c064b..03ba1959f 100644 --- a/.env.example +++ b/.env.example @@ -144,8 +144,25 @@ GROQ_API_KEY= # Groq free (chat fallback + reasoning) # arbitrary free-model router -- that billed customers the priced model's cost # while an undisclosed free model actually answered. deploy-demo-box.yml's # "Assert model catalog prices agree..." step fails the deploy if these ever -# drift from the catalog again. Failures still cascade via LiteLLM fallbacks -# to the Groq routes below. +# drift from the catalog again. +# +# SUPERSEDED IN PART, 2026-08-22 catalog restructure. hive-default and hive-auto +# both moved off OpenRouter onto Groq gpt-oss models, and their routes were +# retired in favour of route-groq-default and route-groq-auto, whose upstream +# model is written literally in deploy/litellm/config.yaml and is owned by +# provider_routes.provider_model through the config sync (issue #713). No new +# env var was added for them on purpose: another os.environ indirection would +# rebuild exactly the drift that #689 and #965 were. +# +# * OPENROUTER_DEFAULT_MODEL is now UNUSED. Nothing reads it. Left in place +# rather than deleted only because several CI workflows still pass it +# through; treat it as dead and do not wire anything new to it. +# * OPENROUTER_AUTO_MODEL is still LIVE, but for route-doc-vlm alone, which +# is the only vision-capable route left after hive-auto moved to the +# text-only gpt-oss-120b. It must stay a multimodal slug. +# +# The LiteLLM chat fallbacks that used to cascade these to the Groq routes are +# also gone; a fallback answers from a model the alias was not priced against. OPENROUTER_DEFAULT_MODEL=openrouter/openai/gpt-4o-mini OPENROUTER_AUTO_MODEL=openrouter/openai/gpt-4.1-mini # route-openrouter-fast-fallback is disabled (health_state='disabled') and is diff --git a/apps/control-plane/internal/routing/catalog_alias_pricing_test.go b/apps/control-plane/internal/routing/catalog_alias_pricing_test.go new file mode 100644 index 000000000..c0a939d03 --- /dev/null +++ b/apps/control-plane/internal/routing/catalog_alias_pricing_test.go @@ -0,0 +1,712 @@ +package routing + +import ( + "encoding/json" + "fmt" + "math/big" + "os" + "regexp" + "strings" + "testing" +) + +// Offline guards over the catalog restructure migration. +// +// Why these exist: nothing in the running system re-derives a credit figure, so +// a wrong one is a silent mispricing that ships. Issues #689 and #965 were both +// that shape. CI holds no provider API keys, so the rates are pinned in a +// committed snapshot taken when the migration was written. +// +// Why they parse rather than grep: the first version of this file asserted that +// a credit figure appeared SOMEWHERE in the migration text. Review proved three +// mispricings that passed it. Every one of them is now a positional assertion +// against a named column of a named row: +// +// - hive-default's reprice changed to hive-medium's figures. Both numbers were +// already elsewhere in the file, so a 100 percent overcharge on the +// no-model-specified default alias stayed green. +// - input and output swapped inside one alias tuple. Both numbers present, green. +// - a route repointed at a different upstream model with the price untouched, +// a 2x undercharge, green, because provider_model was checked against the +// rate snapshot but never against the SQL. +// +// Presence of digits in a file is not the assertion. A value in a column of a +// row is. +// +// KNOWN LIMIT, stated rather than left to be discovered. This whole file is +// pinned to ONE migration filename. The next migration that reprices these +// aliases inherits none of these guards, and will pass this suite while doing +// anything it likes to the catalog. Two ways out when that day comes: key the +// pricing checks off every migration that writes input_price_credits, or move +// the assertions to the database level where catalog_pricing_integration_test.go +// already lives. Neither is done here, because a guard that scans every +// migration has to cope with the historical ones that predate the DERIVE +// convention entirely. +const ( + pricingMigrationRelPath = "supabase/migrations/20260822_02_catalog_alias_restructure.sql" + providerRatesRelPath = "testdata/provider_rates_2026-08-22.json" + + // creditsPerUSD mirrors apps/control-plane/internal/payments/types.go. + // marginNum/marginDen express 1.4 exactly. Integers only, fed to math/big, + // so no float64 touches a money figure. + creditsPerUSD = 100000 + marginNum = 14 + marginDen = 10 +) + +// deriveField maps the DERIVE table's field name to the model_aliases column +// the figure must actually land in. +var deriveField = map[string]string{ + "in": "input_price_credits", + "out": "output_price_credits", + "cache_read": "cache_read_price_credits", +} + +type deriveRow struct { + Alias string + RouteID string + ProviderModel string + Field string + USD string + Credits string +} + +type providerRate struct { + ProviderModel string `json:"provider_model"` + In string `json:"usd_in_per_million"` + Out string `json:"usd_out_per_million"` + CacheRead *string `json:"usd_cache_read_per_million"` +} + +type providerRatesFixture struct { + FetchedUTC string `json:"fetched_utc"` + Models []providerRate `json:"models"` +} + +// decimalRe constrains a DERIVE rate to a plain decimal. big.Rat.SetString also +// accepts ratio, exponent and hex-float forms, so "1/0.5" or "0x1p-3" would +// otherwise parse as a valid provider rate. +var decimalRe = regexp.MustCompile(`^[0-9]+(\.[0-9]+)?$`) + +// isoDateRe pins fetched_utc to a full calendar date, which is what the two +// substring checks in loadProviderRates need in order to compare anything. +var isoDateRe = regexp.MustCompile(`^[0-9]{4}-[0-9]{2}-[0-9]{2}$`) + +// parseRate turns a documented provider rate into an exact rational, rejecting +// anything that is not a plain positive decimal. +// +// big.Rat parses a decimal string without loss, so "0.01278" is 1278/100000 and +// not the nearest float64 to it. That is the one rate in this migration whose +// product is fractional, so it is the row a float would round differently. +// +// It returns an error rather than calling t.Fatalf, because the callers loop +// over every DERIVE row with t.Errorf and continue: one malformed rate must not +// abort the run and hide every remaining row, which is the difference between +// a review that reports one problem and one that reports all of them. +func parseRate(usd string) (*big.Rat, error) { + if !decimalRe.MatchString(usd) { + return nil, fmt.Errorf("rate %q is not a plain decimal; big.Rat also accepts ratio, exponent and hex-float forms, which are not prices", usd) + } + rate, ok := new(big.Rat).SetString(usd) + if !ok { + return nil, fmt.Errorf("rate %q is not a valid decimal", usd) + } + // A zero or negative rate is a mispricing, not a derivation. Without this, + // a DERIVE row claiming 0 USD and 0 credits satisfies every assertion here + // as a correctly derived free model. + if rate.Sign() <= 0 { + return nil, fmt.Errorf("rate %q is zero or negative; that is a mispricing, not a rate", usd) + } + return rate, nil +} + +// expectedCredits computes ceil(usd * MARGIN * CREDITS_PER_USD) exactly. +func expectedCredits(rate *big.Rat) *big.Int { + product := new(big.Rat).Mul(rate, big.NewRat(marginNum*creditsPerUSD, marginDen)) + quotient, remainder := new(big.Int).QuoRem(product.Num(), product.Denom(), new(big.Int)) + if remainder.Sign() != 0 { + quotient.Add(quotient, big.NewInt(1)) + } + return quotient +} + +func readPricingMigration(t *testing.T) string { + t.Helper() + return readRepoFile(t, pricingMigrationRelPath) +} + +// migrationSQL is the migration with all commentary removed. Every structural +// assertion runs on this, never on the raw text: the migration's header +// discusses `lifecycle = 'deprecated'` and `delete from public.model_aliases` +// at length, and a guard matching raw text is satisfied or tripped by prose. +func migrationSQL(t *testing.T) string { + t.Helper() + return stripSQLComments(readPricingMigration(t)) +} + +func loadProviderRates(t *testing.T) map[string]providerRate { + t.Helper() + + // Go runs tests with the working directory set to the package directory, so + // the testdata path is relative and the package stays movable. + body, err := os.ReadFile(providerRatesRelPath) + if err != nil { + t.Fatalf("read provider rate snapshot: %v", err) + } + + var fixture providerRatesFixture + if err := json.Unmarshal(body, &fixture); err != nil { + t.Fatalf("parse provider rate snapshot: %v", err) + } + if len(fixture.Models) == 0 { + t.Fatal("provider rate snapshot is empty; it is the only offline record of what these prices were derived from") + } + + // The snapshot date must match the migration it backs. Repricing against a + // stale snapshot is the failure this catches, deterministically and with no + // wall-clock comparison. + // + // The shape is checked before the two comparisons, because strings.Contains + // reports true for an empty substring. A snapshot that omits fetched_utc, or + // sets it to "", satisfies both checks below while comparing nothing, and so + // does a one-character value such as "2", which occurs in both of these + // paths anyway. That is a guard structurally incapable of failing, which is + // worse than no guard at all, because it reads as coverage. + if !isoDateRe.MatchString(fixture.FetchedUTC) { + t.Fatalf("provider rate snapshot has fetched_utc %q; want a full YYYY-MM-DD date. An empty or one-character value passes both substring checks below without comparing anything.", fixture.FetchedUTC) + } + if !strings.Contains(providerRatesRelPath, fixture.FetchedUTC) { + t.Errorf("snapshot fetched_utc %q does not match its own filename %q; the rates and the file have drifted apart", fixture.FetchedUTC, providerRatesRelPath) + } + if !strings.Contains(pricingMigrationRelPath, strings.ReplaceAll(fixture.FetchedUTC, "-", "")) { + t.Errorf("snapshot fetched_utc %q does not match the migration date in %q; these prices were derived from rates fetched for a different migration", fixture.FetchedUTC, pricingMigrationRelPath) + } + + byModel := make(map[string]providerRate, len(fixture.Models)) + for _, m := range fixture.Models { + // A duplicate carrying two different rates is exactly the drift this + // fixture exists to detect, and a map would silently last-wins it. + if _, dup := byModel[m.ProviderModel]; dup { + t.Fatalf("provider rate snapshot lists %q twice; one of the two rates is wrong and a map would silently pick one", m.ProviderModel) + } + byModel[m.ProviderModel] = m + } + return byModel +} + +var deriveLineRe = regexp.MustCompile(`(?m)^--\s*DERIVE\|([^\n]*)$`) + +// disableRe matches a statement that retires a route. Compiled once at package +// level rather than per call. +var disableRe = regexp.MustCompile(`(?is)^update\s+public\.provider_routes\s+set\s+health_state\s*=\s*'disabled'`) + +func parseDeriveRows(t *testing.T, migration string) []deriveRow { + t.Helper() + + matches := deriveLineRe.FindAllStringSubmatch(migration, -1) + if len(matches) == 0 { + t.Fatalf("%s declares no '-- DERIVE|' rows; every price in this repo must show its derivation", pricingMigrationRelPath) + } + + var rows []deriveRow + for _, m := range matches { + fields := strings.Split(m[1], "|") + for i := range fields { + fields[i] = strings.TrimSpace(fields[i]) + } + if len(fields) > 0 && strings.EqualFold(fields[0], "alias_id") { + continue // column header + } + if len(fields) != 6 { + t.Fatalf("malformed DERIVE row %q: want 6 pipe-separated fields (alias|route|provider_model|field|usd|credits), got %d", m[1], len(fields)) + } + if _, ok := deriveField[fields[3]]; !ok { + t.Fatalf("DERIVE row %q has unknown field %q; want in, out or cache_read", m[1], fields[3]) + } + rows = append(rows, deriveRow{ + Alias: fields[0], + RouteID: fields[1], + ProviderModel: fields[2], + Field: fields[3], + USD: fields[4], + Credits: fields[5], + }) + } + if len(rows) == 0 { + t.Fatal("DERIVE table has a header but no data rows") + } + return rows +} + +// pricedAliases returns every alias whose input or output price this migration +// WRITES, whether by INSERT or by UPDATE, mapped to the column values written. +// +// Covering the UPDATE side is load bearing. hive-default and hive-auto are +// repriced rather than inserted, which put them outside every guard keyed on +// the INSERT block, and they are the two highest-value rows in the file. +func pricedAliases(t *testing.T, sql string) map[string]map[string]string { + t.Helper() + + out := map[string]map[string]string{} + for _, row := range insertRows(sql, "public.model_aliases") { + alias := row["alias_id"] + if alias == "" { + t.Fatalf("a model_aliases INSERT tuple has no alias_id: %#v", row) + } + out[alias] = row + } + for alias, assigns := range updateAssignments(sql, "public.model_aliases", "alias_id") { + // Either price counts. Testing input alone let an + // `UPDATE ... SET output_price_credits = N` through: the alias landed in + // neither TestEveryPricedAliasHasCompleteDerivation nor + // TestDeclaredCreditsLandOnTheirOwnAliasRow, so an output reprice with no + // DERIVE row behind it passed the whole suite. On an alias this migration + // also INSERTs, the merge below is what binds the later value to the + // assertion; without it the INSERT's superseded figure is the one checked, + // and the figure customers are actually charged goes unexamined. + _, writesIn := assigns["input_price_credits"] + _, writesOut := assigns["output_price_credits"] + if !writesIn && !writesOut { + continue // e.g. the hive-fast lifecycle marker, which touches no price + } + if existing, ok := out[alias]; ok { + for k, v := range assigns { + existing[k] = v + } + continue + } + out[alias] = assigns + } + if len(out) == 0 { + t.Fatal("parsed no priced aliases out of the migration; the parser and the file have diverged and every guard below is checking nothing") + } + return out +} + +// TestCatalogAliasPricesMatchProviderRates checks the DERIVE table's own +// arithmetic against the committed rate snapshot. +func TestCatalogAliasPricesMatchProviderRates(t *testing.T) { + migration := readPricingMigration(t) + rates := loadProviderRates(t) + + for _, row := range parseDeriveRows(t, migration) { + rate, ok := rates[row.ProviderModel] + if !ok { + t.Errorf("alias %s routes to provider_model %q, which is absent from the verified provider snapshot; either the model id is wrong (a dropped tilde or a bad date suffix looks exactly like this) or the snapshot needs re-fetching", + row.Alias, row.ProviderModel) + continue + } + + var snapshotUSD string + switch row.Field { + case "in": + snapshotUSD = rate.In + case "out": + snapshotUSD = rate.Out + case "cache_read": + if rate.CacheRead == nil { + t.Errorf("alias %s documents a cache_read price for %q, but the provider publishes no cache-read rate for that model", row.Alias, row.ProviderModel) + continue + } + snapshotUSD = *rate.CacheRead + } + + // Compared as rationals, not as strings: "0.3" and "0.30" are the same + // rate, and a string comparison would report a false red on a purely + // cosmetic difference in how one of the two files writes it. + documented, err := parseRate(row.USD) + if err != nil { + t.Errorf("alias %s %s: migration's documented %v", row.Alias, row.Field, err) + continue + } + snapshot, err := parseRate(snapshotUSD) + if err != nil { + t.Errorf("alias %s %s: snapshot's %v", row.Alias, row.Field, err) + continue + } + if documented.Cmp(snapshot) != 0 { + t.Errorf("alias %s %s: migration documents the provider rate as $%s per million, snapshot recorded $%s", + row.Alias, row.Field, row.USD, snapshotUSD) + continue + } + + want := expectedCredits(snapshot) + if want.String() != row.Credits { + t.Errorf("alias %s %s: ceil($%s * 1.4 * 100000) = %s credits, migration claims %s", + row.Alias, row.Field, snapshotUSD, want.String(), row.Credits) + } + } +} + +// TestDeclaredCreditsLandOnTheirOwnAliasRow is THE money-path guard. It binds +// each DERIVE figure to the specific column of the specific alias row, so a +// figure that is right in the comment and wrong in the SQL, or right for one +// alias and applied to another, fails. +func TestDeclaredCreditsLandOnTheirOwnAliasRow(t *testing.T) { + migration := readPricingMigration(t) + sql := stripSQLComments(migration) + priced := pricedAliases(t, sql) + + for _, row := range parseDeriveRows(t, migration) { + aliasRow, ok := priced[row.Alias] + if !ok { + t.Errorf("DERIVE documents a price for alias %s, but the migration writes no price for it at all", row.Alias) + continue + } + column := deriveField[row.Field] + got, present := aliasRow[column] + if !present { + t.Errorf("alias %s: DERIVE declares %s = %s credits, but the migration never writes column %s for that alias", + row.Alias, row.Field, row.Credits, column) + continue + } + if got != row.Credits { + t.Errorf("alias %s: DERIVE declares %s = %s credits, but the migration writes %s = %s. The comment and the SQL disagree, and the SQL is what customers are charged.", + row.Alias, row.Field, row.Credits, column, got) + } + } +} + +// TestEveryPricedAliasHasCompleteDerivation is the floor under parseDeriveRows. +// Without it the checked set can silently shrink: deleting the two hive-default +// DERIVE lines left every other assertion green, and hive-default is the alias +// every request naming no model lands on. +func TestEveryPricedAliasHasCompleteDerivation(t *testing.T) { + migration := readPricingMigration(t) + sql := stripSQLComments(migration) + + documented := map[string]map[string]bool{} + for _, row := range parseDeriveRows(t, migration) { + if documented[row.Alias] == nil { + documented[row.Alias] = map[string]bool{} + } + documented[row.Alias][row.Field] = true + } + + for alias, row := range pricedAliases(t, sql) { + for _, field := range []string{"in", "out"} { + if !documented[alias][field] { + t.Errorf("alias %s has its %s price written by this migration (%s) but no '-- DERIVE|' row documenting where that figure came from", + alias, field, row[deriveField[field]]) + } + } + } +} + +// TestDerivedRouteMatchesProviderRoutes binds the DERIVE table's route and +// upstream model to the provider_routes SQL. Without it, repointing a route at +// a different model while leaving its price alone passes everything: the +// provider_model was checked against the rate snapshot but never against the +// migration, which is issue #689 and #965 in miniature. +func TestDerivedRouteMatchesProviderRoutes(t *testing.T) { + migration := readPricingMigration(t) + sql := stripSQLComments(migration) + + routes := map[string]map[string]string{} + for _, r := range insertRows(sql, "public.provider_routes") { + routes[r["route_id"]] = r + } + if len(routes) == 0 { + t.Fatal("parsed no provider_routes rows; the parser and the migration have diverged") + } + + for _, row := range parseDeriveRows(t, migration) { + route, ok := routes[row.RouteID] + if !ok { + t.Errorf("alias %s is priced against route %s, which this migration never inserts", row.Alias, row.RouteID) + continue + } + if route["alias_id"] != row.Alias { + t.Errorf("route %s is priced under alias %s but provider_routes attaches it to %s", row.RouteID, row.Alias, route["alias_id"]) + } + if route["provider_model"] != row.ProviderModel { + t.Errorf("alias %s is priced against upstream model %q, but route %s actually calls %q. The price and the model have drifted apart, which is a silent over or undercharge.", + row.Alias, row.ProviderModel, row.RouteID, route["provider_model"]) + } + // provider must agree with the model's own prefix, or the config sync + // picks the wrong api_base and API key for it. + if wantProvider := strings.SplitN(row.ProviderModel, "/", 2)[0]; route["provider"] != wantProvider { + t.Errorf("route %s calls %q but declares provider %q; the sync would send this model, and that provider's key, to the wrong endpoint", + row.RouteID, row.ProviderModel, route["provider"]) + } + } +} + +// TestOneEnabledRoutePerAliasInSQL enforces the owner's one-alias-one-price rule +// against provider_routes itself rather than against the comment table. A second +// enabled route makes an alias's cost depend on which route won, which is not +// priceable at the alias level. +// +// This reads only this migration, so it cannot see routes added elsewhere. The +// database-level invariant across the whole catalog is +// TestSeededAliasHasExactlyOneEnabledRoute in catalog_pricing_integration_test.go, +// which needs ROUTING_TEST_DB_URL. Do not read this offline guard as that one. +func TestOneEnabledRoutePerAliasInSQL(t *testing.T) { + sql := migrationSQL(t) + + enabledByAlias := map[string][]string{} + for _, r := range insertRows(sql, "public.provider_routes") { + if strings.EqualFold(r["health_state"], "disabled") || strings.EqualFold(r["health_state"], "eol") { + continue + } + enabledByAlias[r["alias_id"]] = append(enabledByAlias[r["alias_id"]], r["route_id"]) + } + if len(enabledByAlias) == 0 { + t.Fatal("parsed no enabled provider_routes rows; this guard is checking nothing") + } + + for alias, routes := range enabledByAlias { + if len(routes) != 1 { + t.Errorf("alias %s is given %d enabled routes by this migration (%s); the one-alias-one-price rule allows exactly one", + alias, len(routes), strings.Join(routes, ", ")) + } + } +} + +// TestInsertedAliasesArePinnedToTheirRoute checks the policy row for aliases +// this migration creates. A repointed alias keeps the policy an earlier +// migration gave it, so it is covered by TestRetiredRoutesAreDisabledAndRepointed +// instead. +func TestInsertedAliasesArePinnedToTheirRoute(t *testing.T) { + sql := migrationSQL(t) + + routeOf := map[string]string{} + for _, r := range insertRows(sql, "public.provider_routes") { + if !strings.EqualFold(r["health_state"], "disabled") { + routeOf[r["alias_id"]] = r["route_id"] + } + } + + policies := map[string]string{} + for _, p := range insertRows(sql, "public.alias_route_policies") { + policies[p["alias_id"]] = p["fallback_order"] + } + + for _, row := range insertRows(sql, "public.model_aliases") { + alias := row["alias_id"] + order, ok := policies[alias] + if !ok { + t.Errorf("alias %s is inserted with no alias_route_policies row", alias) + continue + } + want := `["` + routeOf[alias] + `"]` + if strings.ReplaceAll(order, " ", "") != want { + t.Errorf("alias %s has fallback_order %s; the one-alias-one-price rule requires exactly %s", alias, order, want) + } + } +} + +// retiredRoutes are the OpenRouter routes hive-default and hive-auto used to +// take, mapped to the Groq route each alias moves onto. +var retiredRoutes = map[string]struct{ alias, replacement string }{ + "route-openrouter-default": {"hive-default", "route-groq-default"}, + "route-openrouter-auto": {"hive-auto", "route-groq-auto"}, +} + +// TestRetiredRoutesAreDisabledAndRepointed guards the move that has to happen +// as a whole for hive-default and hive-auto. +// +// The old route must be DISABLED rather than repointed in place, because +// litellmconfig's mergeParams deliberately preserves every litellm_params key +// the database does not own. The OpenRouter-specific extra_body block those two +// entries carry would otherwise stay attached to a route now pointing at Groq +// and be sent to Groq on every request to the default model, with no sync able +// to remove it. Disabling the route id makes the merge drop the stale entry. +func TestRetiredRoutesAreDisabledAndRepointed(t *testing.T) { + sql := migrationSQL(t) + + // Per statement, not over a concatenation of all of them: a file disabling + // route A in one statement and merely mentioning route B in another must + // not satisfy the guard for B. + var disablers []string + for _, stmt := range splitStatements(sql) { + if disableRe.MatchString(strings.TrimSpace(stmt)) { + disablers = append(disablers, stmt) + } + } + if len(disablers) == 0 { + t.Fatal("no statement disables any provider_route; the two OpenRouter routes must be retired, not left enabled alongside their replacements") + } + + policyUpdates := updateAssignments(sql, "public.alias_route_policies", "alias_id") + + for old, move := range retiredRoutes { + disabled := false + for _, stmt := range disablers { + if strings.Contains(stmt, "'"+old+"'") { + disabled = true + break + } + } + if !disabled { + t.Errorf("route %s is still enabled: it must be disabled when %s moves to %s, or the alias has two enabled routes and stops being priceable", + old, move.alias, move.replacement) + } + + order := strings.ReplaceAll(policyUpdates[move.alias]["fallback_order"], " ", "") + if want := `["` + move.replacement + `"]`; order != want { + t.Errorf("%s needs its alias_route_policies fallback_order updated to %s, found %q; otherwise the policy keeps naming the retired route %s", + move.alias, want, order, old) + } + + // The stale entry must not be resurrected by an in-place repoint. + // Accept both the `= 'x'` and the `IN ('x', ...)` forms, since this + // migration's own retirement statement uses IN and a repoint written + // that way would otherwise walk past the check. + for _, stmt := range splitStatements(sql) { + lower := strings.ToLower(strings.TrimSpace(stmt)) + if !strings.HasPrefix(lower, "update public.provider_routes") { + continue + } + if !strings.Contains(lower, "provider_model") { + continue + } + if strings.Contains(stmt, "'"+old+"'") { + t.Errorf("route %s is repointed in place; it must be disabled instead, or its OpenRouter extra_body block survives the config sync and is sent to Groq", old) + } + } + } +} + +// soleCarrierFlags are capability flags held by exactly ONE route in the whole +// catalog before this migration, route-openrouter-auto, granted by +// 20260414_01_provider_capabilities_media_columns.sql and never granted since. +var soleCarrierFlags = []string{ + "supports_batch", + "supports_image_generation", + "supports_image_edit", +} + +// TestDisablingASoleCapabilityCarrierHandsItsFlagsOn is a regression guard for a +// real defect an earlier revision of this migration shipped. +// +// Disabling route-openrouter-auto removes the only route carrying these three +// flags. SelectRoute skips disabled candidates and then hard filters on each +// flag, and both batchstore/submitter.go and local_executor_adapters.go send +// NeedBatch for EVERY batch, so /v1/batches, /v1/images/generations and +// /v1/images/edits each find zero eligible routes for every alias in the system. +// It fails closed, so nothing reports it. +func TestDisablingASoleCapabilityCarrierHandsItsFlagsOn(t *testing.T) { + sql := migrationSQL(t) + + stillDisabled := false + for _, stmt := range splitStatements(sql) { + // disableRe, not a literal substring: `health_state='disabled'`, or the + // assignment broken across lines, would otherwise leave stillDisabled + // false and send this guard to the t.Skip below, while the migration went + // on disabling the route and stripping its three flags unobserved. + if disableRe.MatchString(strings.TrimSpace(stmt)) && strings.Contains(stmt, "'route-openrouter-auto'") { + stillDisabled = true + break + } + } + if !stillDisabled { + t.Skip("route-openrouter-auto is not disabled by this migration, so its capabilities are not at risk") + } + + caps := insertRows(sql, "public.provider_capabilities") + if len(caps) == 0 { + t.Fatal("route-openrouter-auto is disabled but this migration inserts no provider_capabilities rows to hand its flags to") + } + + for _, flag := range soleCarrierFlags { + granted := false + for _, row := range caps { + if strings.EqualFold(row[flag], "true") { + granted = true + break + } + } + if !granted { + t.Errorf("this migration disables route-openrouter-auto, the only route in the catalog carrying %s, and grants that flag to no replacement route. Every endpoint gated on it would find zero eligible routes for every alias.", flag) + } + } +} + +// TestRepointedAliasesKeepTheirCapabilities is the sibling of the guard above +// for the per-alias flags. hive-default and hive-auto both had +// supports_reasoning true on their OpenRouter routes. Because every alias here +// is pinned to exactly ONE route, a narrower replacement does not quietly +// withhold a feature: matchesRequestedCapabilities drops the only candidate and +// SelectRoute returns ErrRouteNotEligible, which writeRoutingError maps to 422. +// So a chat completion carrying reasoning_effort would start failing outright. +func TestRepointedAliasesKeepTheirCapabilities(t *testing.T) { + sql := migrationSQL(t) + + caps := map[string]map[string]string{} + for _, row := range insertRows(sql, "public.provider_capabilities") { + caps[row["route_id"]] = row + } + + for _, move := range retiredRoutes { + row, ok := caps[move.replacement] + if !ok { + t.Errorf("replacement route %s has no provider_capabilities row, so it takes the column defaults of false for everything", move.replacement) + continue + } + if !strings.EqualFold(row["supports_reasoning"], "true") { + t.Errorf("route %s must keep supports_reasoning = true: %s is pinned to this single route, its previous OpenRouter route had the flag, and gpt-oss does expose reasoning effort. With one candidate an under-claim is not a withheld feature, it is a 422 on every request carrying reasoning_effort.", + move.replacement, move.alias) + } + } +} + +// TestHiveFastStaysInvocableAfterDeprecation guards the back-compat promise. +// hive-fast is the model id stored in existing Open WebUI conversations and in +// live API clients, so the restructure must not remove it, must not stop it +// resolving, and must not reprice it. +// +// Every check here is statement scoped. An earlier version used `.*?` under +// `(?s)`, which let the pieces come from three different statements: it could +// report hive-fast marked hidden when a DIFFERENT alias had been hidden, which +// is a false green on a back-compat guard. +func TestHiveFastStaysInvocableAfterDeprecation(t *testing.T) { + sql := migrationSQL(t) + + for _, stmt := range splitStatements(sql) { + lower := strings.ToLower(strings.TrimSpace(stmt)) + if strings.HasPrefix(lower, "delete from public.model_aliases") || strings.HasPrefix(lower, "delete from model_aliases") { + t.Error("the restructure must not DELETE from model_aliases: hive-fast is stored per-chat in existing Open WebUI conversations and would 404 for every one of them") + } + } + + updates := updateAssignments(sql, "public.model_aliases", "alias_id") + hiveFast, ok := updates["hive-fast"] + if !ok { + t.Fatal("hive-fast must be marked with lifecycle = 'hidden'; that is the deprecation marker this restructure promised, and no UPDATE targets it") + } + + if hiveFast["lifecycle"] != "hidden" { + t.Errorf("hive-fast's lifecycle is set to %q; the deprecation marker must be 'hidden'. model_aliases' CHECK constraint permits only stable, preview and hidden, so a literal 'deprecated' would abort the migration on apply.", hiveFast["lifecycle"]) + } + if v, set := hiveFast["visibility"]; set && (v == "internal" || v == "restricted") { + t.Errorf("hive-fast must keep visibility 'public', found %q: AliasVisibleToTenant fails closed on anything but public and preview, which would block invocation, not merely hide the alias from the picker", v) + } + for _, priceCol := range []string{"input_price_credits", "output_price_credits"} { + if v, set := hiveFast[priceCol]; set { + t.Errorf("hive-fast must not be repriced by this migration, but %s is set to %s; it has to keep charging exactly what it charged before", priceCol, v) + } + } +} + +// TestNewAliasesReachDefaultTierKeys is the inert-change guard. An alias can be +// inserted, priced, routed and wired into LiteLLM and still be invisible to +// every customer, because api_key_policies.allowed_group_names defaults to +// ["default"] and a key never sees an alias outside its groups. That gap has +// been patched by hand twice already, by 20260717_01 and 20260717_02. +func TestNewAliasesReachDefaultTierKeys(t *testing.T) { + sql := migrationSQL(t) + + inDefault := map[string]bool{} + for _, row := range insertRows(sql, "public.model_policy_group_members") { + if row["group_name"] == "default" { + inDefault[row["alias_id"]] = true + } + } + + for _, row := range insertRows(sql, "public.model_aliases") { + if alias := row["alias_id"]; !inDefault[alias] { + t.Errorf("alias %s is never added to the 'default' model policy group, so no default-tier API key can call it; the alias would ship inert", alias) + } + } +} diff --git a/apps/control-plane/internal/routing/sanitize.go b/apps/control-plane/internal/routing/sanitize.go index 9977b112b..a716a9dc2 100644 --- a/apps/control-plane/internal/routing/sanitize.go +++ b/apps/control-plane/internal/routing/sanitize.go @@ -10,6 +10,23 @@ func SanitizeProviderMessage(alias string, raw string) string { providerReplacement = trimmedAlias } + // NOTE, established while reviewing PR #1007: this function has NO + // production caller. It is referenced only by its own tests. The live + // customer-facing provider-blindness boundaries are elsewhere, and both + // scrub generically rather than from a hardcoded list, so neither needs + // updating when a route is added: + // + // * apps/edge-api/internal/errors/provider_blind.go, on every inference, + // audio, images, RAG and chat dispatch error path. Its routeSlugRegex + // is (?i)\broute-[a-z0-9][a-z0-9._/-]*\b, which already covers any + // route id, and its providerModelRegex already covers any + // provider-prefixed model string. + // * apps/control-plane/internal/batchstore/executor/dispatcher.go's + // SanitizeMessage, on batch output files. That one strips provider + // WORDS only and has no route-slug pattern, so it is the real gap. + // + // Do not add route ids to the list below and assume the job is done. An + // earlier revision of #1007 did exactly that and shipped nothing. message := strings.NewReplacer( "route-openrouter-default", resourceReplacement, "route-openrouter-auto", resourceReplacement, diff --git a/apps/control-plane/internal/routing/sqlparse_test.go b/apps/control-plane/internal/routing/sqlparse_test.go new file mode 100644 index 000000000..7501c348d --- /dev/null +++ b/apps/control-plane/internal/routing/sqlparse_test.go @@ -0,0 +1,383 @@ +package routing + +import ( + "regexp" + "strings" + "testing" +) + +// Minimal structural reader for the migration files these guards check. +// +// It exists because the first version of catalog_alias_pricing_test.go asserted +// on the migration as raw text, and review proved three separate mispricings +// could pass it: a credit figure only had to appear SOMEWHERE in the file, so +// an alias could be given another alias's price, two figures inside one tuple +// could be swapped, and a route could be repointed at a different upstream +// model with the price left alone. Presence in a file is not the assertion; a +// value sitting in a named column of a named row is. That needs parsing. +// +// Deliberately not a SQL parser. It understands exactly the shapes these +// migrations use: single-statement-per-semicolon, INSERT ... VALUES tuples, and +// UPDATE ... SET ... WHERE. It is quote-aware so a comma or a semicolon inside a +// string literal cannot split a statement or a field. + +// stripSQLComments removes line comments and block comments while preserving +// string literals. Every structural regex and parser in these tests runs on the +// result, because matching over raw text lets a migration's own prose satisfy or +// trip a guard: this file's header discusses `lifecycle = 'deprecated'` and +// `delete from public.model_aliases` at length, and both were matchable. +func stripSQLComments(sql string) string { + var out strings.Builder + out.Grow(len(sql)) + + inSingle, inLine, inBlock := false, false, false + for i := 0; i < len(sql); i++ { + c := sql[i] + + switch { + case inLine: + if c == '\n' { + inLine = false + out.WriteByte(c) + } + continue + case inBlock: + if c == '*' && i+1 < len(sql) && sql[i+1] == '/' { + inBlock = false + i++ + } + continue + case inSingle: + out.WriteByte(c) + if c == '\'' { + // '' is an escaped quote inside a literal, not a terminator. + if i+1 < len(sql) && sql[i+1] == '\'' { + out.WriteByte(sql[i+1]) + i++ + continue + } + inSingle = false + } + continue + } + + if c == '\'' { + inSingle = true + out.WriteByte(c) + continue + } + if c == '-' && i+1 < len(sql) && sql[i+1] == '-' { + inLine = true + continue + } + if c == '/' && i+1 < len(sql) && sql[i+1] == '*' { + inBlock = true + i++ + continue + } + out.WriteByte(c) + } + return out.String() +} + +// splitStatements splits on top-level semicolons, quote and paren aware. +func splitStatements(sql string) []string { + var stmts []string + var cur strings.Builder + inSingle := false + depth := 0 + + for i := 0; i < len(sql); i++ { + c := sql[i] + if inSingle { + cur.WriteByte(c) + if c == '\'' { + if i+1 < len(sql) && sql[i+1] == '\'' { + cur.WriteByte(sql[i+1]) + i++ + continue + } + inSingle = false + } + continue + } + switch c { + case '\'': + inSingle = true + cur.WriteByte(c) + case '(': + depth++ + cur.WriteByte(c) + case ')': + depth-- + cur.WriteByte(c) + case ';': + if depth == 0 { + if s := strings.TrimSpace(cur.String()); s != "" { + stmts = append(stmts, s) + } + cur.Reset() + continue + } + cur.WriteByte(c) + default: + cur.WriteByte(c) + } + } + if s := strings.TrimSpace(cur.String()); s != "" { + stmts = append(stmts, s) + } + return stmts +} + +// topLevelGroups returns the contents of each top-level parenthesised group in +// s, in order, quote aware. +func topLevelGroups(s string) []string { + var groups []string + var cur strings.Builder + inSingle := false + depth := 0 + + for i := 0; i < len(s); i++ { + c := s[i] + if inSingle { + cur.WriteByte(c) + if c == '\'' { + if i+1 < len(s) && s[i+1] == '\'' { + cur.WriteByte(s[i+1]) + i++ + continue + } + inSingle = false + } + continue + } + switch c { + case '\'': + inSingle = true + if depth > 0 { + cur.WriteByte(c) + } + case '(': + depth++ + if depth == 1 { + cur.Reset() + continue + } + cur.WriteByte(c) + case ')': + depth-- + if depth == 0 { + groups = append(groups, cur.String()) + cur.Reset() + continue + } + cur.WriteByte(c) + default: + if depth > 0 { + cur.WriteByte(c) + } + } + } + return groups +} + +// splitFields splits a tuple body on top-level commas, quote and paren aware. +func splitFields(tuple string) []string { + var fields []string + var cur strings.Builder + inSingle := false + depth := 0 + + flush := func() { + fields = append(fields, strings.TrimSpace(cur.String())) + cur.Reset() + } + + for i := 0; i < len(tuple); i++ { + c := tuple[i] + if inSingle { + cur.WriteByte(c) + if c == '\'' { + if i+1 < len(tuple) && tuple[i+1] == '\'' { + cur.WriteByte(tuple[i+1]) + i++ + continue + } + inSingle = false + } + continue + } + switch c { + case '\'': + inSingle = true + cur.WriteByte(c) + case '(', '[': + depth++ + cur.WriteByte(c) + case ')', ']': + depth-- + cur.WriteByte(c) + case ',': + if depth == 0 { + flush() + continue + } + cur.WriteByte(c) + default: + cur.WriteByte(c) + } + } + if strings.TrimSpace(cur.String()) != "" || len(fields) > 0 { + flush() + } + return fields +} + +// unquote strips one layer of single quotes and any ::cast suffix. +func unquote(v string) string { + v = strings.TrimSpace(v) + if i := strings.LastIndex(v, "::"); i > 0 && strings.HasPrefix(v, "'") { + v = strings.TrimSpace(v[:i]) + } + if len(v) >= 2 && strings.HasPrefix(v, "'") && strings.HasSuffix(v, "'") { + return strings.ReplaceAll(v[1:len(v)-1], "''", "'") + } + return v +} + +// insertRows returns every VALUES tuple of every INSERT INTO statement +// in sql, as column-name keyed maps. ALL matching statements are read, not the +// first: this migration carries two provider_capabilities INSERTs, and an +// earlier guard that read only the first reported a correct migration as broken. +func insertRows(sql, table string) []map[string]string { + var rows []map[string]string + + for _, stmt := range splitStatements(sql) { + lower := strings.ToLower(stmt) + if !strings.Contains(lower, "insert into "+table) { + continue + } + valuesAt := strings.Index(lower, " values") + if valuesAt < 0 { + continue + } + + colGroups := topLevelGroups(stmt[:valuesAt]) + if len(colGroups) == 0 { + continue + } + var cols []string + for _, c := range splitFields(colGroups[0]) { + cols = append(cols, strings.ToLower(strings.TrimSpace(c))) + } + + for _, tuple := range topLevelGroups(stmt[valuesAt:]) { + fields := splitFields(tuple) + if len(fields) != len(cols) { + continue + } + row := make(map[string]string, len(cols)) + for i, c := range cols { + row[c] = unquote(fields[i]) + } + rows = append(rows, row) + } + } + return rows +} + +// updateAssignments returns, for each UPDATE of
whose WHERE clause pins +// a single = 'value', that value mapped to its SET assignments. +func updateAssignments(sql, table, keyCol string) map[string]map[string]string { + out := map[string]map[string]string{} + + for _, stmt := range splitStatements(sql) { + lower := strings.ToLower(stmt) + if !strings.HasPrefix(lower, "update "+table) { + continue + } + setAt := strings.Index(lower, " set ") + whereAt := strings.Index(lower, " where ") + if setAt < 0 || whereAt < 0 || whereAt < setAt { + continue + } + + // Take the first ` = ''` in the WHERE clause. Anything + // after it (an `AND price <> ...` re-runnability guard, for instance) + // is not part of the key, which an earlier hand-rolled version of this + // swallowed whole. + keyMatch := regexp.MustCompile(`(?i)\b` + regexp.QuoteMeta(keyCol) + `\s*=\s*'([^']*)'`).FindStringSubmatch(stmt[whereAt:]) + if keyMatch == nil { + continue + } + keyVal := keyMatch[1] + + assigns := map[string]string{} + for _, part := range splitFields(stmt[setAt+5 : whereAt]) { + eq := strings.Index(part, "=") + if eq < 0 { + continue + } + assigns[strings.ToLower(strings.TrimSpace(part[:eq]))] = unquote(part[eq+1:]) + } + if len(assigns) == 0 { + continue + } + if existing, ok := out[keyVal]; ok { + for k, v := range assigns { + existing[k] = v + } + continue + } + out[keyVal] = assigns + } + return out +} + +// TestSQLParseHelpers is a self-test. These helpers are the foundation every +// money-path assertion in catalog_alias_pricing_test.go now stands on, so a bug +// here would quietly weaken all of them at once. +func TestSQLParseHelpers(t *testing.T) { + src := ` +-- a comment mentioning lifecycle = 'deprecated' and delete from public.model_aliases +insert into public.model_aliases (alias_id, summary, input_price_credits) values + ('a-one', 'text; with, punctuation', 10), + ('a-two', 'badges', 20) -- trailing comment +on conflict (alias_id) do nothing; + +update public.model_aliases + set input_price_credits = 30, lifecycle = 'hidden' + where alias_id = 'a-three' and lifecycle <> 'hidden'; +` + stripped := stripSQLComments(src) + if strings.Contains(stripped, "deprecated") || strings.Contains(stripped, "trailing comment") { + t.Fatalf("stripSQLComments left comment text behind: %q", stripped) + } + if !strings.Contains(stripped, "text; with, punctuation") { + t.Fatal("stripSQLComments damaged a string literal") + } + + rows := insertRows(stripped, "public.model_aliases") + if len(rows) != 2 { + t.Fatalf("want 2 inserted rows, got %d: %#v", len(rows), rows) + } + if rows[0]["alias_id"] != "a-one" || rows[0]["input_price_credits"] != "10" { + t.Errorf("row 0 parsed wrong: %#v", rows[0]) + } + if rows[0]["summary"] != "text; with, punctuation" { + t.Errorf("a semicolon and comma inside a literal split the statement: %#v", rows[0]) + } + if rows[1]["alias_id"] != "a-two" || rows[1]["input_price_credits"] != "20" { + t.Errorf("row 1 parsed wrong: %#v", rows[1]) + } + + ups := updateAssignments(stripped, "public.model_aliases", "alias_id") + got, ok := ups["a-three"] + if !ok { + t.Fatalf("update not parsed: %#v", ups) + } + if got["input_price_credits"] != "30" || got["lifecycle"] != "hidden" { + t.Errorf("update assignments parsed wrong: %#v", got) + } +} diff --git a/apps/control-plane/internal/routing/testdata/provider_rates_2026-08-22.json b/apps/control-plane/internal/routing/testdata/provider_rates_2026-08-22.json new file mode 100644 index 000000000..f9745dbd9 --- /dev/null +++ b/apps/control-plane/internal/routing/testdata/provider_rates_2026-08-22.json @@ -0,0 +1,57 @@ +{ + "_comment": [ + "Point-in-time snapshot of the upstream provider list rates every alias added by", + "supabase/migrations/20260822_02_catalog_alias_restructure.sql is priced against.", + "Fetched 2026-08-22. Committed so TestCatalogAliasPricesMatchProviderRates can check", + "the migration's arithmetic offline: CI has no provider API keys, and a test that", + "needs one is a test that silently skips.", + "", + "WHAT THIS FIXTURE CAN AND CANNOT CATCH.", + "Catches: a credit figure that does not follow from the rate beside it, a rate typo,", + "a provider_model string that is not one of the models actually verified (a dropped", + "tilde, a case change, a wrong date suffix), and an alias added to the migration with", + "no documented derivation at all.", + "Does NOT catch: a model the provider decommissions AFTER this date, or a rate the", + "provider changes after this date. That was issue #965 exactly, and only a live query", + "against the provider catalog catches it. Re-fetch and re-snapshot when repricing.", + "", + "usd_*_per_million are USD per 1,000,000 tokens, as strings so no float rounding", + "enters the fixture. null cache_read means the provider publishes no cache-read rate", + "for that model, which is not the same as a rate of zero." + ], + "fetched_utc": "2026-08-22", + "models": [ + { + "provider_model": "groq/openai/gpt-oss-20b", + "usd_in_per_million": "0.075", + "usd_out_per_million": "0.30", + "usd_cache_read_per_million": null, + "source": "https://console.groq.com/docs/models", + "note": "Unchanged from the rate 20260801_01 and 20260818_01 both derived hive-fast from." + }, + { + "provider_model": "groq/openai/gpt-oss-120b", + "usd_in_per_million": "0.15", + "usd_out_per_million": "0.60", + "usd_cache_read_per_million": null, + "source": "https://console.groq.com/docs/models", + "note": "Second Groq-owned source agreeing: https://console.groq.com/docs/compound/systems/compound-mini lists GPT-OSS 120B at $0.15 input / $0.60 output in its underlying-model cost breakdown. https://groq.com/pricing, the source 20260801_01 and 20260818_01 used, now 301-redirects to the marketing homepage and carries no rate card." + }, + { + "provider_model": "openrouter/~deepseek/deepseek-v4-flash-latest", + "usd_in_per_million": "0.0639", + "usd_out_per_million": "0.1278", + "usd_cache_read_per_million": "0.01278", + "source": "https://openrouter.ai/api/v1/models", + "note": "The leading tilde is part of the real OpenRouter model id, not a typo. Verified live: the API returns id '~deepseek/deepseek-v4-flash-latest' with tokenizer 'Router', distinct from the pinned 'deepseek/deepseek-v4-flash-0731' (0.08/0.18) and 'deepseek/deepseek-v4-flash' (0.05586/0.11172). Being a -latest router alias, the model it resolves to can change under us and so can its rate; see the migration header." + }, + { + "provider_model": "openrouter/deepseek/deepseek-v4-pro-0813", + "usd_in_per_million": "1.122", + "usd_out_per_million": "3.366", + "usd_cache_read_per_million": "0.0374", + "source": "https://openrouter.ai/api/v1/models", + "note": "Date-pinned build, so unlike the flash router alias this id resolves to one fixed model." + } + ] +} diff --git a/deploy/litellm/config.yaml b/deploy/litellm/config.yaml index 5b24b7082..fa4b973ea 100644 --- a/deploy/litellm/config.yaml +++ b/deploy/litellm/config.yaml @@ -12,6 +12,22 @@ # Demo budget posture — paid-route count # ────────────────────────────────────── +# RECOUNTED 2026-08-22 by the catalog restructure. Every chat route except the +# two DeepSeek ones is now Groq, which is free to us at present; the aliases +# hive-default, hive-auto, hive-small, hive-medium and the deprecated hive-fast +# all resolve to one of two Groq gpt-oss models. The remaining OpenRouter spend +# is: route-deepseek-v4-flash, route-deepseek-v4-pro, route-doc-vlm (still +# OPENROUTER_AUTO_MODEL, the only vision-capable route left) and +# route-openrouter-embedding-fallback. So FOUR paid routes, down from a mix +# where the default and vision chat paths were both paid. +# +# The concentration risk is the flip side and is called out here so a reader +# meets it in the same paragraph as the saving: with the chat surface almost +# entirely on one provider, and with no fallbacks left (see litellm_settings +# below), a Groq outage takes chat down entirely rather than degrading it. +# +# The historical note below is kept for the reasoning it records. +# # CORRECTED by issue #689 (2026-08). This block used to claim only ONE route # was paid, on the theory that OPENROUTER_DEFAULT_MODEL / OPENROUTER_AUTO_MODEL # default to `openrouter/free` in .env.example, so the flagship and vision @@ -38,55 +54,117 @@ # openai/ generic adapter), so there is no verified free embedding slug to swap # the fallback to; it stays paid by necessity and is documented here. model_list: - # ── 1. Flagship chat (paid; priced against gpt-4o-mini, hive-default) ─────── - # OPENROUTER_DEFAULT_MODEL defaults to a paid slug (see .env.example), pinned - # to the exact model hive-default's catalog price is derived from. It used - # to default to the `openrouter/free` pool, an undisclosed free model billed - # at the paid price -- that mismatch was issue #689. See PR #99 and git - # history for the earlier flaky-usage-tokens root-cause debug notes on why a - # pinned slug beats the free pool regardless of price. - - model_name: route-openrouter-default + # ── 1. Default chat (Groq gpt-oss-20b, hive-default) ──────────────────────── + # Was route-openrouter-default on OPENROUTER_DEFAULT_MODEL. The catalog + # restructure of 2026-08-22 moved hive-default onto Groq on cost grounds + # (OpenRouter is paid out of pocket, Groq is currently free to us) and gave + # it a new route id rather than repointing the old one. The old id is + # retired as `disabled` in provider_routes, so the config sync drops its + # entry from the live gateway; a new id is what lets that happen, because + # the OpenRouter-only `extra_body.provider` block above would otherwise have + # survived the field-level merge and been sent to Groq forever. + # + # The model is written literally, not through an env var. The upstream model + # for a DB-managed route comes from provider_routes.provider_model via + # POST /internal/litellm/sync (issue #713), and adding a fresh + # os.environ/... indirection would just rebuild the two-mechanisms-one- + # checked drift that issues #689 and #965 were. Same shape as route-groq-stt + # and route-groq-tts below. This literal is only the first-boot seed. + - model_name: route-groq-default litellm_params: - model: os.environ/OPENROUTER_DEFAULT_MODEL - api_key: os.environ/OPENROUTER_API_KEY - extra_body: - provider: - allow_fallbacks: false - sort: throughput + model: groq/openai/gpt-oss-20b + api_key: os.environ/GROQ_API_KEY - # ── 2. Fast / cheap chat (Groq free tier) ─────────────────────────────────── - # Groq free tier: moonshotai/kimi-k2-instruct — fast inference, free quota, - # ideal for lightweight everyday prompts and cost-sensitive paths. - # Falls back to the flagship route automatically via litellm_settings.fallbacks - # if Groq's free quota is exhausted or the service is unavailable. + # ── 2. Deprecated fast chat (hive-fast) ───────────────────────────────────── + # Serves Groq openai/gpt-oss-20b, via GROQ_FAST_MODEL, whose default is + # pinned to exactly that in .env.example (reverted there by issue #965 after + # Groq decommissioned llama-3.1-8b-instant). The older comment here named + # moonshotai/kimi-k2-instruct, which this route has not served for some time. + # + # hive-fast is DEPRECATED as of the 2026-08-22 catalog restructure. It is + # kept, priced and routed identically to hive-small, because the model id is + # persisted per-conversation in existing Open WebUI chats and in live API + # clients. Do not retire this route without a migration path for those. + # + # It no longer falls back to anything. The previous comment claimed an + # automatic cascade to the flagship route via litellm_settings.fallbacks; + # those chat fallbacks are gone, deliberately, because a fallback answers + # from a model the alias was not priced against. See litellm_settings below. - model_name: route-groq-fast litellm_params: model: os.environ/GROQ_FAST_MODEL api_key: os.environ/GROQ_API_KEY - # ── 3. Vision (paid; priced against gpt-4.1-mini, hive-auto) ──────────────── - # OPENROUTER_AUTO_MODEL defaults to a paid, vision-capable slug pinned to the - # exact model hive-auto's catalog price is derived from (same #689 fix as - # route-openrouter-default above; this route used to default to the free - # pool too). Used for multimodal / image-analysis requests routed by the - # edge-api capability selector. - - model_name: route-openrouter-auto + # ── 3. Larger chat (Groq gpt-oss-120b, hive-auto) ─────────────────────────── + # Was route-openrouter-auto on OPENROUTER_AUTO_MODEL, retired for the same + # reason and in the same way as route-openrouter-default above. + # + # NOTE this route is no longer vision-capable. gpt-oss-120b is text-only, + # where gpt-4.1-mini was multimodal, so state the consequence plainly rather + # than pointing somewhere reassuring: after this change there is NO + # customer-reachable vision path through the gateway. hive-auto was the last + # one. + # + # route-doc-vlm (section 4) is not a migration path for that traffic. It has + # no provider_routes row, which is why the config sync leaves it alone, and + # SelectRoute only ever returns routes joined to an alias, so nothing a + # customer sends in the OpenAI-compatible `model` field can select it. Its + # only consumer is apps/agent-engine/internal/docvlm, which addresses this + # LiteLLM model name directly and never goes through the catalog. Do not + # point a customer at it. + # + # hive-auto also no longer performs any automatic model selection; the name + # is kept for back-compat only. + - model_name: route-groq-auto litellm_params: - model: os.environ/OPENROUTER_AUTO_MODEL + model: groq/openai/gpt-oss-120b + api_key: os.environ/GROQ_API_KEY + + # ── 3b. Catalog restructure routes (2026-08-22) ───────────────────────────── + # hive-small and hive-medium are the provider-blind names for the two Groq + # gpt-oss models; hive-default and hive-auto above resolve to the same two + # models through their own route rows, per the one-alias-one-enabled-route + # rule. The two DeepSeek routes are the only paid OpenRouter chat routes + # left after this change. + # + # The tilde in ~deepseek/deepseek-v4-flash-latest is part of the real + # OpenRouter model id. Removing it silently selects a different, differently + # priced model, so do not "tidy" it away. + - model_name: route-groq-small + litellm_params: + model: groq/openai/gpt-oss-20b + api_key: os.environ/GROQ_API_KEY + - model_name: route-groq-medium + litellm_params: + model: groq/openai/gpt-oss-120b + api_key: os.environ/GROQ_API_KEY + - model_name: route-deepseek-v4-flash + litellm_params: + model: openrouter/~deepseek/deepseek-v4-flash-latest + api_key: os.environ/OPENROUTER_API_KEY + - model_name: route-deepseek-v4-pro + litellm_params: + model: openrouter/deepseek/deepseek-v4-pro-0813 api_key: os.environ/OPENROUTER_API_KEY - extra_body: - provider: - allow_fallbacks: false - sort: throughput - # ── 4. Doc-layout vision (paid, same model + env var as route-openrouter-auto) ─ + # ── 4. Doc-layout vision (paid, OpenRouter, the last multimodal route) ────── # Blueprint Wave 3, Step 3.2 (issue #300): the doc-layout knowledge-work-pack # skill (apps/agent-engine/internal/docvlm) sends contract/PDF page images - # here for structured layout extraction. Named as its own route (rather than - # reusing route-openrouter-auto directly) so the doc-layout skill's model - # choice can be tuned or swapped independently of the general vision route - # edge-api's capability selector uses; today both share OPENROUTER_AUTO_MODEL - # and so both point at the same paid vision model. + # here for structured layout extraction. + # + # UPDATED 2026-08-22. This section used to describe the route as sharing a + # model and an env var with route-openrouter-auto, and as being separable + # from "the general vision route edge-api's capability selector uses". + # Neither that route nor that selector path exists any more: the catalog + # restructure retired route-openrouter-auto and moved hive-auto to a + # text-only Groq model. + # + # What is left is this: route-doc-vlm is the ONLY multimodal route in the + # file, it is reachable only by agent-engine calling this LiteLLM model name + # directly, and it is NOT selectable by any customer request, because it has + # no provider_routes row and therefore no alias. OPENROUTER_AUTO_MODEL now + # exists for this route alone, so it must stay pointed at a vision-capable + # slug even though no alias is priced against it any more. - model_name: route-doc-vlm litellm_params: model: os.environ/OPENROUTER_AUTO_MODEL @@ -182,11 +260,29 @@ litellm_settings: cooldown_time: 30 drop_params: true fallbacks: - - route-openrouter-default: [route-groq-fast] - - route-openrouter-auto: [route-groq-fast] + # The two chat fallbacks that used to live here are GONE, deliberately. + # They read: + # - route-openrouter-default: [route-groq-fast] + # - route-openrouter-auto: [route-groq-fast] + # Both named routes are retired by the 2026-08-22 catalog restructure, so + # they would be inert anyway, but they should not simply be re-pointed at + # the replacement routes either. A gateway fallback serves a DIFFERENT + # upstream model than the alias is priced against: route-openrouter-auto + # was priced on gpt-4.1-mini and would silently answer from gpt-oss-20b. + # That is the one-alias-one-price rule being broken one layer below the + # database, where the catalog cannot see it. The owner's rule is that an + # alias maps to exactly one model, so a chat request now fails rather than + # being quietly served by a model the customer was not charged for. + # + # Consequence, stated plainly: there is no absorption if Groq has an + # outage. That is the accepted trade-off of one model, one price. + # # Embedding cascade: Nemotron-Embed VL (free pool) → Qwen3 Embedding # (paid, larger context). NVIDIA NIM is no longer in the chain after - # the 2026-05-18 retrieval-model EOL wave. + # the 2026-05-18 retrieval-model EOL wave. Kept, because both legs are + # embedding routes under one alias whose price is not per-token in the + # same sense, and RAG ingestion failing outright is worse than a + # second-choice embedding model. - route-openrouter-embedding: [route-openrouter-embedding-fallback] files_settings: diff --git a/docs/proof/catalog-alias-restructure-2026-08-22/README.md b/docs/proof/catalog-alias-restructure-2026-08-22/README.md new file mode 100644 index 000000000..ce590bbbc --- /dev/null +++ b/docs/proof/catalog-alias-restructure-2026-08-22/README.md @@ -0,0 +1,57 @@ +# Catalog alias restructure, verification evidence + +PR #1007. Captured 2026-08-22. + +## Read this first: what kind of evidence this is + +**This is database-level evidence, not a live UI capture.** It is not a screenshot of the chat model selector, and it should not be read as one. + +The change is a data change with no front-end code in it. Its user-visible surfaces are `GET /v1/models` and the Open WebUI model picker, both of which render whatever the catalog contains. A live capture of either requires the migration applied against a running stack. + +That was not possible when this PR was raised, for a reason outside this change: + +* The demo box has been undeployable since 2026-08-22. `deploy` depends on `migrate`, migrate fails on #994's pg_cron migration, and the pg_cron image that migration needs is built only by the `deploy` job that therefore never runs (#1002). A separate agent is breaking that deadlock. +* Sign-in on the box is separately broken by an unrelated P0, so even a deployed build could not have been driven through the picker. + +A live capture is still owed once #1002 lands and the box takes this migration. Until then this file is the strongest honest evidence available, and it deliberately does not claim to be more. + +**Update, 2026-08-23.** A live capture now exists: `docs/proof/catalog-alias-restructure-live-stack-2026-08-23/`. It is a local full-stack capture, not a demo-box one, and it covers the two gaps named under "Limits of this evidence" below: the LiteLLM config sync and `GET /v1/models` end to end. #1002 is fixed and #1005 merged, so the box is deployable again; a deployed capture is still owed after this merges and deploys. + +## What was actually done + +`catalog-after-migration.txt` is the unedited transcript of: + +1. A throwaway `pgvector/pgvector:pg17` Postgres, created empty. +2. The repo's own `scripts/ci-throwaway-db.sh`, which runs the Supabase role and schema bootstrap and then `scripts/apply-migrations.sh`, the same applier the demo box runs. Not an approximation of the chain, the chain. +3. `psql` queries against the result. + +All 91 migrations executed, including `20260822_02_catalog_alias_restructure.sql`. The applier's own assertion confirms it: `throwaway database ready: 91 of 91 migrations executed`. The statement-level output for this migration is in the transcript, showing the row counts each statement touched. + +CI runs the same thing independently as the required check `apply-migrations.sh against a throwaway Postgres`, which passed on this commit. + +## What the transcript establishes + +* Every alias price matches the derivation recorded in the migration header, applied to a real database rather than asserted against SQL text. +* Every alias except `hive-embedding-default` has exactly one enabled route. See the caveat below. +* `hive-fast` survives with `visibility = 'public'` and `lifecycle = 'hidden'`, at the same price and on the same route as before, so existing conversations and API clients keep resolving it. That is the back-compat requirement, demonstrated rather than asserted. +* `route-openrouter-default` and `route-openrouter-auto` are `disabled`, alongside the previously retired `route-openrouter-fast-fallback`. +* All four new aliases are members of both the `default` and `closed` policy groups, so a default-tier API key can actually reach them. This is the gate that has silently shipped inert aliases twice before, in `20260717_01` and `20260717_02`. +* `tools_supported` is true on all six new and moved routes, rather than falling to the column default of false. + +## Pre-existing condition surfaced, not introduced + +The one-enabled-route query returns `hive-embedding-default` with two enabled routes. **This predates this PR and is not changed by it.** `20260801_01_alias_pricing_correction.sql` found the same thing and explicitly left it alone, recording it as "Reported for a decision rather than changed unilaterally". It is reproduced here because the query is written to check the whole table rather than only the rows this change touches, which is the point of running it that way. + +## Limits of this evidence + +* It does not prove what LiteLLM sends upstream. That depends on the config sync running against a live gateway, which needs the box. +* It does not exercise `GET /v1/models` end to end, so it does not prove the HTTP response shape or the per-tenant visibility filter, only the catalog rows those read from. +* It proves nothing about whether the upstream models still exist at the provider. The migration's prices come from rates fetched live on 2026-08-22, and a model decommissioned after that date would not be visible here. That failure mode is issue #965 and only a live provider query catches it. + +## Reproducing + +```bash +bash scripts/ci-throwaway-db.sh # with PGHOST/PGPORT/PGUSER/PGDATABASE exported +``` + +against any empty `pgvector/pgvector:pg17`, then the queries at the end of the transcript. diff --git a/docs/proof/catalog-alias-restructure-2026-08-22/catalog-after-migration.txt b/docs/proof/catalog-alias-restructure-2026-08-22/catalog-after-migration.txt new file mode 100644 index 000000000..8fcee9d48 --- /dev/null +++ b/docs/proof/catalog-alias-restructure-2026-08-22/catalog-after-migration.txt @@ -0,0 +1,121 @@ +postgres ready +DROP POLICY +COMMIT +::endgroup:: +applied 20260822_01_tenant_email_domains_admin_only.sql +::group::applying 20260822_02_catalog_alias_restructure.sql +BEGIN +INSERT 0 4 +INSERT 0 4 +INSERT 0 4 +INSERT 0 4 +INSERT 0 9 +INSERT 0 2 +INSERT 0 2 +UPDATE 2 +UPDATE 1 +UPDATE 1 +UPDATE 1 +UPDATE 1 +UPDATE 1 +COMMIT +::endgroup:: +applied 20260822_02_catalog_alias_restructure.sql +applied 91 migration(s) +pg_cron: absent on this image, so 20260729_02 created its config table and purge procedure but scheduled no nightly job. Expected here; a throwaway database does not live long enough to run one. +throwaway database ready: 91 of 91 migrations executed + +=============== CUSTOMER-FACING CATALOG AFTER MIGRATION =============== +--- model_aliases visible in /v1/models (visibility public or preview) --- + alias_id | display_name | visibility | lifecycle | in_cr | out_cr | price_unit +------------------------+------------------------+------------+-----------+--------+---------+------------ + deepseek-v4-flash | Deepseek V4 Flash | public | stable | 8946 | 17892 | tokens + deepseek-v4-pro | Deepseek V4 Pro | public | stable | 157080 | 471240 | tokens + hive-auto | Hive Auto | preview | preview | 21000 | 84000 | tokens + hive-default | Hive Default | public | stable | 10500 | 42000 | tokens + hive-embedding-default | Hive Embedding Default | public | stable | 1 | 0 | tokens + hive-fast | Hive Fast | public | hidden | 10500 | 42000 | tokens + hive-medium | Hive Medium | public | stable | 21000 | 84000 | tokens + hive-small | Hive Small | public | stable | 10500 | 42000 | tokens + hive-stt | Hive Voice STT | public | stable | 0 | 4316667 | seconds + hive-tts | Hive Voice TTS | public | stable | 0 | 3080000 | characters +(10 rows) + + +--- enabled route per alias (health_state not disabled/eol) --- + alias_id | route_id | provider | provider_model | health_state +------------------------+-------------------------------------+------------+------------------------------------------------------+-------------- + deepseek-v4-flash | route-deepseek-v4-flash | openrouter | openrouter/~deepseek/deepseek-v4-flash-latest | healthy + deepseek-v4-pro | route-deepseek-v4-pro | openrouter | openrouter/deepseek/deepseek-v4-pro-0813 | healthy + hive-auto | route-groq-auto | groq | groq/openai/gpt-oss-120b | healthy + hive-default | route-groq-default | groq | groq/openai/gpt-oss-20b | healthy + hive-embedding-default | route-openrouter-embedding | openrouter | openrouter/nvidia/llama-nemotron-embed-vl-1b-v2:free | healthy + hive-embedding-default | route-openrouter-embedding-fallback | openrouter | openrouter/qwen/qwen3-embedding-8b | healthy + hive-fast | route-groq-fast | groq | groq/openai/gpt-oss-20b | healthy + hive-medium | route-groq-medium | groq | groq/openai/gpt-oss-120b | healthy + hive-small | route-groq-small | groq | groq/openai/gpt-oss-20b | healthy + hive-stt | route-groq-stt | groq | groq/whisper-large-v3 | healthy + hive-tts | route-groq-tts | groq | groq/canopylabs/orpheus-v1-english | healthy +(11 rows) + + +--- INVARIANT: any alias whose enabled-route count is not exactly 1 --- + alias_id | enabled_routes +------------------------+---------------- + hive-embedding-default | 2 +(1 row) + + +--- retired routes (should list the two OpenRouter ones plus the old fast fallback) --- + route_id | alias_id | provider | health_state +--------------------------------+--------------+------------+-------------- + route-openrouter-auto | hive-auto | openrouter | disabled + route-openrouter-default | hive-default | openrouter | disabled + route-openrouter-fast-fallback | hive-fast | openrouter | disabled +(3 rows) + + +--- default-tier reachability for the four new aliases --- + alias_id | groups +-------------------+------------------------ + deepseek-v4-flash | closed,default + deepseek-v4-pro | closed,default,premium + hive-medium | closed,default + hive-small | closed,default +(4 rows) + + +--- tools_supported on the new and moved routes --- + route_id | tools_supported | supports_reasoning | supports_cache_read +-------------------------+-----------------+--------------------+--------------------- + route-deepseek-v4-flash | t | t | t + route-deepseek-v4-pro | t | t | t + route-groq-auto | t | t | f + route-groq-default | t | t | f + route-groq-medium | t | t | f + route-groq-small | t | t | f +(6 rows) + + +--- REGRESSION CHECK: enabled routes still serving batch / image endpoints --- + route_id | alias_id | supports_batch | supports_image_generation | supports_image_edit +-----------------+-----------+----------------+---------------------------+--------------------- + route-groq-auto | hive-auto | t | t | t +(1 row) + + +--- these three counts must all be non-zero or an endpoint is dead --- + batch_routes | imagegen_routes | imageedit_routes +--------------+-----------------+------------------ + 1 | 1 | 1 +(1 row) + + +--- policy groups for deepseek-v4-pro --- + group_name | alias_id +------------+----------------- + closed | deepseek-v4-pro + default | deepseek-v4-pro + premium | deepseek-v4-pro +(3 rows) + diff --git a/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/README.md b/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/README.md new file mode 100644 index 000000000..d613edda8 --- /dev/null +++ b/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/README.md @@ -0,0 +1,96 @@ +# Catalog alias restructure, live full-stack capture + +PR #1007. Captured 2026-08-23 against a stack brought up for this capture. + +## What kind of evidence this is + +**A local full-stack capture, not a demo-box capture.** Every service in the +chain was built from this branch and run together on one host, and the +screenshots are of a real browser driving a real Open WebUI against them. It is +not a screenshot of the deployed demo box, and it should not be read as one. + +It is the live counterpart to `docs/proof/catalog-alias-restructure-2026-08-22/`, +which is database-level only and says so. That transcript proves the migration +applies. This one proves a user sees and can use the result, which is the claim +a migration transcript cannot make. + +## Why a local stack rather than the box + +The migration is not on the demo box yet, and staging it there was refused +deliberately: that box is live and its data plane is not a scratch surface. The +hosted Supabase project the shared `.env` points at no longer resolves in DNS, +so there was no shared database to capture against either. A stack built from +the branch, on its own throwaway Postgres, is what remains, and it exercises the +same code paths. + +## The chain that was actually exercised + +1. A throwaway `pgvector/pgvector:pg17` Postgres, created empty, then + `scripts/ci-throwaway-db.sh`: all 91 migrations executed, including + `20260822_02_catalog_alias_restructure.sql`. Its own assertion confirms it, + `throwaway database ready: 91 of 91 migrations executed`. +2. `scripts/ci-seed-api-key.sh` minted one tenant, account, API key, policy and + credit grant on that database. Generated at run time, thrown away with the + stack. +3. `control-plane` and `edge-api` images **rebuilt from this branch** and run + against that database, with `litellm`, `redis`, `open-webui` and + `caddy-owui`, on their own Compose project and their own ports. +4. `POST /internal/litellm/sync` regenerated LiteLLM's `model_list` from + `public.provider_routes.provider_model` and restarted the gateway. +5. A browser signed in to Open WebUI, opened the model picker, selected an + alias and sent a message. + +## What each file shows + +| File | Shows | +|---|---| +| `litellm-sync-result.txt` | The regenerated `model_list` and the model ids the running gateway serves. This is the load-bearing one: it is the config sync, not the checked-in seed. | +| `edge-api-v1-models.json.txt` | `GET /v1/models` end to end through edge-api, which is what the picker reads. | +| `catalog-and-metering.txt` | The catalog rows the stack served from, and the `usage_events` rows the chat turn wrote. | +| `browser-capture-log.txt` | The browser transcript: URLs visited, console output, and the model names the open picker rendered. | + +The two screenshots are attached to the pull request as release assets rather +than committed here, per `.claude/skills/pr-visual-proof.md`: a +`raw.githubusercontent.com` URL pinned to this branch would 404 the moment the +branch is deleted on merge. + +## What it establishes + +* All six restructured aliases reach a user. The picker rendered + `deepseek-v4-flash`, `deepseek-v4-pro`, `hive-auto`, `hive-default`, + `hive-medium` and `hive-small`, plus the deprecated `hive-fast`, which is the + back-compat requirement shown rather than asserted. +* The config sync regenerates from the database, so `provider_routes` owns the + upstream model. Every route resolves to the model the migration set: + `route-groq-small`, `route-groq-default` and `route-groq-fast` to + `groq/openai/gpt-oss-20b`; `route-groq-medium` and `route-groq-auto` to + `groq/openai/gpt-oss-120b`; the two DeepSeek routes to their OpenRouter slugs. +* The retired `route-openrouter-default` and `route-openrouter-auto` are absent + from the regenerated config. They are `disabled` in `provider_routes` and the + sync drops them, which is the behaviour the migration depends on. +* A route is not merely listed but live. `hive-small` was selected and answered, + and `usage_events` recorded the turn against `hive-small` with real token + counts and a credit delta. Routes on this project have been silently inert + before; a served answer is what rules that out. + +## Limits of this evidence + +* It is not the demo box. Anything box-specific (its env, its volumes, its + deployed image) is untested here. The deploy that carries this migration is + still owed its own confirmation. +* The embedding, STT and TTS aliases are absent from the picker because Open + WebUI lists chat models only. They were not exercised. +* Open WebUI ran with `ENABLE_LOGIN_FORM` and `ENABLE_SIGNUP` on, for a local + throwaway account. The deployed configuration is OIDC-only and is unchanged + by this; no repository file was edited to obtain the capture and no existing + account was touched. +* It says nothing about whether the upstream models still exist at the provider + beyond the one that answered. That is issue #965. + +## Reproducing + +Bring up a throwaway Postgres, run `scripts/ci-throwaway-db.sh` and +`scripts/ci-seed-api-key.sh` against it, then start `redis`, `litellm`, +`control-plane`, `edge-api`, `open-webui` and `caddy-owui` from +`deploy/docker/docker-compose.yml` with `SUPABASE_DB_URL` pointed at it and the +Supabase JWT variables left blank, and `POST /internal/litellm/sync`. diff --git a/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/browser-capture-log.txt b/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/browser-capture-log.txt new file mode 100644 index 000000000..c6c21a1f3 --- /dev/null +++ b/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/browser-capture-log.txt @@ -0,0 +1,23 @@ +capture started 2026-08-23T03:59:53.441Z +target http://127.0.0.1:13002 +[console:warning] No token found in localStorage, user-join event not emitted +landed on http://127.0.0.1:13002/auth +clicked through the splash panel +no sign-up toggle found; form may already be in sign-up mode +submitted local sign-up form (password generated at run time, not recorded) +after auth, url = http://127.0.0.1:13002/ +captured 00-chat-loaded.png +opened model picker via selector: button#model-selector-0-button +captured 01-model-picker.png +--- text nodes visible in the open picker --- + deepseek-v4-flash + deepseek-v4-pro + hive-auto + hive-default + hive-fast + hive-medium + hive-small +selected hive-small in the picker +sent a prompt through the selected alias +captured 02-completion.png +capture finished 2026-08-23T04:00:37.050Z diff --git a/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/catalog-and-metering.txt b/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/catalog-and-metering.txt new file mode 100644 index 000000000..f6bff6325 --- /dev/null +++ b/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/catalog-and-metering.txt @@ -0,0 +1,36 @@ +public.usage_events after the Open WebUI chat turn in 02-completion.png. +Local throwaway Postgres. captured: 2026-08-23T04:02:10Z + + model_alias | endpoint | status | input_tokens | output_tokens | hive_credit_delta | created_at +-------------+------------------+-------------+--------------+---------------+-------------------+------------------------------- + hive-small | chat_completions | completed | 248 | 85 | 333 | 2026-08-23 04:00:21.856366+00 + hive-small | chat_completions | completed | 0 | 0 | -6 | 2026-08-23 04:00:21.840192+00 + hive-small | chat_completions | dispatching | 0 | 0 | 0 | 2026-08-23 04:00:21.284816+00 + hive-small | chat_completions | completed | 297 | 459 | 756 | 2026-08-23 04:00:21.15038+00 + hive-small | chat_completions | completed | 0 | 0 | -22 | 2026-08-23 04:00:21.128388+00 + hive-small | chat_completions | dispatching | 0 | 0 | 0 | 2026-08-23 04:00:20.105854+00 + hive-small | chat_completions | completed | 0 | 0 | 0 | 2026-08-23 04:00:19.93307+00 + hive-small | chat_completions | completed | 0 | 0 | -1 | 2026-08-23 04:00:19.914845+00 +(8 rows) + + +--- catalog rows this stack was serving from --- + alias_id | display_name | visibility | lifecycle | provider | provider_model | litellm_model_name | health_state +------------------------+------------------------+------------+-----------+------------+---------------------------------------------------------+-------------------------------------+-------------- + deepseek-v4-flash | Deepseek V4 Flash | public | stable | openrouter | openrouter/~deepseek/deepseek-v4-flash-latest | route-deepseek-v4-flash | healthy + deepseek-v4-pro | Deepseek V4 Pro | public | stable | openrouter | openrouter/deepseek/deepseek-v4-pro-0813 | route-deepseek-v4-pro | healthy + hive-auto | Hive Auto | preview | preview | groq | groq/openai/gpt-oss-120b | route-groq-auto | healthy + hive-auto | Hive Auto | preview | preview | openrouter | openrouter/openai/gpt-4.1-mini | route-openrouter-auto | disabled + hive-default | Hive Default | public | stable | groq | groq/openai/gpt-oss-20b | route-groq-default | healthy + hive-default | Hive Default | public | stable | openrouter | openrouter/openai/gpt-4o-mini | route-openrouter-default | disabled + hive-embedding-default | Hive Embedding Default | public | stable | nvidia_nim | nvidia_nim/nvidia/llama-3.2-nemoretriever-300m-embed-v1 | route-nvidia-embedding | eol + hive-embedding-default | Hive Embedding Default | public | stable | openrouter | openrouter/nvidia/llama-nemotron-embed-vl-1b-v2:free | route-openrouter-embedding | healthy + hive-embedding-default | Hive Embedding Default | public | stable | openrouter | openrouter/qwen/qwen3-embedding-8b | route-openrouter-embedding-fallback | healthy + hive-fast | Hive Fast | public | hidden | groq | groq/openai/gpt-oss-20b | route-groq-fast | healthy + hive-fast | Hive Fast | public | hidden | openrouter | openrouter/meta-llama/3.1-8b-instruct | route-openrouter-fast-fallback | disabled + hive-medium | Hive Medium | public | stable | groq | groq/openai/gpt-oss-120b | route-groq-medium | healthy + hive-small | Hive Small | public | stable | groq | groq/openai/gpt-oss-20b | route-groq-small | healthy + hive-stt | Hive Voice STT | public | stable | groq | groq/whisper-large-v3 | route-groq-stt | healthy + hive-tts | Hive Voice TTS | public | stable | groq | groq/canopylabs/orpheus-v1-english | route-groq-tts | healthy +(15 rows) + diff --git a/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/edge-api-v1-models.json.txt b/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/edge-api-v1-models.json.txt new file mode 100644 index 000000000..81becdf59 --- /dev/null +++ b/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/edge-api-v1-models.json.txt @@ -0,0 +1,68 @@ +GET /v1/models (edge-api, authenticated with a locally seeded Hive API key) +host: http://127.0.0.1:18080 captured: 2026-08-23T04:02:09Z + +{ + "data": [ + { + "id": "deepseek-v4-flash", + "object": "model", + "created": 1787456241, + "owned_by": "hive" + }, + { + "id": "deepseek-v4-pro", + "object": "model", + "created": 1787456241, + "owned_by": "hive" + }, + { + "id": "hive-auto", + "object": "model", + "created": 1787456222, + "owned_by": "hive" + }, + { + "id": "hive-default", + "object": "model", + "created": 1787456222, + "owned_by": "hive" + }, + { + "id": "hive-embedding-default", + "object": "model", + "created": 1787456225, + "owned_by": "hive" + }, + { + "id": "hive-fast", + "object": "model", + "created": 1787456222, + "owned_by": "hive" + }, + { + "id": "hive-medium", + "object": "model", + "created": 1787456241, + "owned_by": "hive" + }, + { + "id": "hive-small", + "object": "model", + "created": 1787456241, + "owned_by": "hive" + }, + { + "id": "hive-stt", + "object": "model", + "created": 1787456235, + "owned_by": "hive" + }, + { + "id": "hive-tts", + "object": "model", + "created": 1787456235, + "owned_by": "hive" + } + ], + "object": "list" +} diff --git a/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/litellm-sync-result.txt b/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/litellm-sync-result.txt new file mode 100644 index 000000000..d31507060 --- /dev/null +++ b/docs/proof/catalog-alias-restructure-live-stack-2026-08-23/litellm-sync-result.txt @@ -0,0 +1,46 @@ +LiteLLM gateway, after POST /internal/litellm/sync regenerated its +model_list from public.provider_routes.provider_model. +captured: 2026-08-23T04:02:09Z + +--- /etc/litellm/config.yaml, model_name -> upstream model --- +20: model: openrouter/~deepseek/deepseek-v4-flash-latest +21: model_name: route-deepseek-v4-flash +25: model: openrouter/deepseek/deepseek-v4-pro-0813 +26: model_name: route-deepseek-v4-pro +30: model: groq/openai/gpt-oss-120b +31: model_name: route-groq-auto +35: model: groq/openai/gpt-oss-20b +36: model_name: route-groq-default +40: model: groq/openai/gpt-oss-20b +41: model_name: route-groq-fast +45: model: groq/openai/gpt-oss-120b +46: model_name: route-groq-medium +50: model: groq/openai/gpt-oss-20b +51: model_name: route-groq-small +55: model: groq/whisper-large-v3 +58: model_name: route-groq-stt +62: model: groq/canopylabs/orpheus-v1-english +65: model_name: route-groq-tts +69: model: openai/nvidia/llama-nemotron-embed-vl-1b-v2:free +72: model_name: route-openrouter-embedding +76: model: openai/qwen/qwen3-embedding-8b +79: model_name: route-openrouter-embedding-fallback +86: model: os.environ/OPENROUTER_AUTO_MODEL +87: model_name: route-doc-vlm +91: model: openai/bge-m3 +94: model_name: bge-m3 + +--- GET /v1/models on the running gateway (what it actually serves) --- +route-deepseek-v4-flash +route-deepseek-v4-pro +route-groq-auto +route-groq-default +route-groq-fast +route-groq-medium +route-groq-small +route-groq-stt +route-groq-tts +route-openrouter-embedding +route-openrouter-embedding-fallback +route-doc-vlm +bge-m3 diff --git a/supabase/migrations/20260822_02_catalog_alias_restructure.sql b/supabase/migrations/20260822_02_catalog_alias_restructure.sql new file mode 100644 index 000000000..c791680bc --- /dev/null +++ b/supabase/migrations/20260822_02_catalog_alias_restructure.sql @@ -0,0 +1,653 @@ +-- ============================================================================= +-- Customer-facing catalog restructure (owner directive, 2026-08-22). +-- +-- WHAT CHANGES +-- * hive-small NEW. Same upstream model as hive-fast, and the same +-- INPUT and OUTPUT price. This is the rename half of +-- "hive-fast becomes hive-small", done as an add rather +-- than an in-place rename; see BACK-COMPAT below. +-- "Same price" is stated for input and output only, on +-- purpose. hive-fast still carries cache_read 1 and +-- cache_write 4 from the original 20260331_01 seed, +-- while hive-small is inserted with 0 and 0. Those +-- columns are display-only, precedence.go never reads +-- them, but they ARE published through +-- catalog.CatalogPricing, so the difference is visible. +-- Zero is the correct value for a Groq gpt-oss route, +-- which declares no cache support and has no published +-- cache rate; hive-fast's non-zero pair is the stale +-- one, and it is left alone rather than corrected here +-- so that the deprecated alias is not repriced on any +-- axis by this change. +-- * hive-medium NEW. Groq openai/gpt-oss-120b, the larger sibling of +-- the model hive-small serves. +-- * deepseek-v4-flash NEW. OpenRouter ~deepseek/deepseek-v4-flash-latest. +-- * deepseek-v4-pro NEW. OpenRouter deepseek/deepseek-v4-pro-0813. +-- * hive-default REPOINTED from OpenRouter openai/gpt-4o-mini to Groq +-- openai/gpt-oss-20b, and repriced to that model's rate. +-- * hive-auto REPOINTED from OpenRouter openai/gpt-4.1-mini to Groq +-- openai/gpt-oss-120b, and repriced to that model's rate. +-- * hive-fast Marked deprecated. Nothing else about it moves. +-- +-- WHY THE TWO REPOINTS GO TO GROQ AND NOT TO COMPOUND +-- The original directive asked for hive-default and hive-auto to move to +-- Groq's Compound systems (groq/compound-mini and groq/compound). They do +-- not, because Compound has no per-token price to derive from. Groq's own +-- model table at https://console.groq.com/docs/models shows "--" in the +-- pricing column for both systems, and +-- https://console.groq.com/docs/compound/systems/compound-mini states +-- verbatim "Final pricing depends on which underlying models and tools are +-- used for your specific query", then itemises charges that are not +-- per-token at all: basic web search $5 per 1000 requests, advanced web +-- search $8 per 1000, visit website $1 per 1000, code execution $0.18 per +-- hour. model_aliases stores exactly one input/output credit pair and +-- apps/edge-api/internal/metering/precedence.go charges from it, so any pair +-- written here would be a fixed customer price against a supplier cost that +-- varies per request with tool use. On a representative 2000-in/500-out +-- request the token cost is about $0.0006 while two basic web searches cost +-- $0.010, so the tool fee is the dominant term, not a rounding error. +-- Compound therefore waits on variable-cost settlement, which does not exist +-- yet; hive-auto is its intended home once it does. +-- +-- Both aliases still leave OpenRouter, because OpenRouter costs real money +-- out of pocket while Groq is free to us at present. They move instead to +-- the same two plain per-token Groq models this migration already adds, so +-- the repoint needs no new pricing machinery. +-- +-- BOTH REPOINTED ALIASES GET CHEAPER +-- hive-default 21000 in / 84000 out -> 10500 in / 42000 out +-- hive-auto 56000 in / 224000 out -> 21000 in / 84000 out +-- A price reduction cannot overcharge anyone, so this is safe to apply, but +-- it does change revenue on the highest-volume path in the product. +-- +-- Honesty about what the names now mean: hive-auto performs no automatic +-- routing any more, and hive-default is functionally identical to +-- hive-small. Both are kept as semantic aliases for back-compat and for the +-- meaning of their names, not as distinct capabilities, and their summary +-- text is rewritten below to say exactly that rather than imply routing +-- intelligence that is not there. +-- +-- WHAT DELIBERATELY DOES NOT CHANGE +-- hive-embedding-default, hive-stt and hive-tts are untouched, and so is +-- route-doc-vlm, which has no provider_routes row at all and survives as an +-- operator-managed LiteLLM entry still reading OPENROUTER_AUTO_MODEL. +-- +-- One loose end this migration cannot tidy from the database. The two +-- gateway-level chat fallbacks that named route-openrouter-default and +-- route-openrouter-auto have been REMOVED from deploy/litellm/config.yaml in +-- this same change, so a fresh boot is correct. They were not re-pointed at +-- the replacement routes on purpose: a fallback answers from a different +-- upstream model than the alias was priced against, which breaks the +-- one-alias-one-price rule one layer below the catalog, where the price +-- cannot see it. Do not reintroduce them. +-- +-- What the seed file cannot reach is a box that is already running. The +-- config sync preserves litellm_settings verbatim, so a live volume keeps +-- whatever fallback lines it already has until someone rewrites that block +-- by hand. Those stale entries will name models no longer in model_list, +-- which is inert rather than fatal, so cleaning them up is deliberately out +-- of scope for a catalog change. +-- +-- BACK-COMPAT (non-negotiable) +-- hive-fast is the model id persisted per-conversation in existing Open +-- WebUI chats and in live API clients. It is NOT renamed in place and NOT +-- deleted: hive-small is added alongside it, pointing at the same upstream +-- model at the same price through its own route, and hive-fast keeps +-- visibility 'public' so it stays resolvable. Only its lifecycle moves. +-- +-- The deprecation marker is lifecycle = 'hidden'. It is not 'deprecated' +-- because model_aliases' CHECK constraint +-- (20260331_01_model_catalog.sql) permits only 'stable', 'preview' and +-- 'hidden'; a literal 'deprecated' would abort this migration on apply. +-- Being honest about its reach. No Go code filters on lifecycle: it is +-- selected and returned by catalog/repository.go as a display field and +-- nothing gates on it, so hive-fast still appears in /v1/models and stays +-- invocable, which is the point. The front end DOES read it, though: +-- apps/web-console/components/catalog/model-catalog-table.tsx maps lifecycle +-- to a status badge and has an explicit 'hidden' entry rendering as +-- "Hidden". So the visible effect in the console catalog is that hive-fast +-- is listed, priced and fully callable while its status column reads +-- Hidden. That is a fair rendering of a deprecated alias, but it is not +-- nothing, and a reader should not have to discover it. The alternative, +-- moving its visibility to 'internal', would make catalog.AliasVisibleToTenant +-- fail closed and break every saved chat, because that one predicate governs +-- both the listing and the inference-time entitlement check. +-- +-- ONE ALIAS, ONE ENABLED ROUTE (owner rule) +-- Every alias added here gets exactly one route and an alias_route_policies +-- row in 'pinned' mode whose fallback_order names only that route, matching +-- how 20260717_02 seeded the voice aliases. No fallbacks are added and the +-- existing disabled route-openrouter-fast-fallback is left disabled. +-- +-- FORMULA (identical to 20260801_01, 20260801_13 and 20260818_01) +-- credits_per_million = ceil(provider_list_usd_per_million * MARGIN * CREDITS_PER_USD) +-- MARGIN = 1.4 +-- CREDITS_PER_USD = 100000 (apps/control-plane/internal/payments/types.go) +-- +-- PROVIDER RATES, ALL FETCHED 2026-08-22 +-- Groq openai/gpt-oss-20b $0.075 in / $0.30 out per million +-- Groq openai/gpt-oss-120b $0.15 in / $0.60 out per million +-- source: https://console.groq.com/docs/models. Note for future +-- re-derivations: https://groq.com/pricing, the source 20260801_01 and +-- 20260818_01 both cite, now 301-redirects to the marketing homepage and +-- carries no rate card at all. The second agreeing Groq-owned source used +-- here is https://console.groq.com/docs/compound/systems/compound-mini, +-- whose underlying-model cost breakdown independently lists GPT-OSS 120B +-- at $0.15 input / $0.60 output. The 20b figures are unchanged from what +-- 20260818_01 recorded. +-- OpenRouter ~deepseek/deepseek-v4-flash-latest +-- $0.0639 in / $0.1278 out / $0.01278 cache read per million +-- OpenRouter deepseek/deepseek-v4-pro-0813 +-- $1.122 in / $3.366 out / $0.0374 cache read per million +-- source: https://openrouter.ai/api/v1/models, which quotes USD per token; +-- the figures above are those values multiplied by 1e6. Neither model +-- publishes a cache-WRITE rate, so both cache_write_price_credits are 0. +-- +-- The leading tilde in ~deepseek/deepseek-v4-flash-latest is part of the +-- real OpenRouter model id and must not be "corrected" away. Verified live: +-- the API returns three distinct flash entries and they are not the same +-- model at the same price, so dropping the tilde silently reprices the +-- route: '~deepseek/deepseek-v4-flash-latest' (0.0639/0.1278, tokenizer +-- "Router"), 'deepseek/deepseek-v4-flash-0731' (0.08/0.18) and +-- 'deepseek/deepseek-v4-flash' (0.05586/0.11172). +-- +-- KNOWN RISK on that alias: it is a "-latest" router entry, so the model it +-- resolves to, and therefore its published rate, can change without a +-- migration. The price written here is correct as of the fetch date only. +-- deepseek-v4-pro is date-pinned (-0813) and does not carry this risk. +-- +-- WORKED DERIVATION +-- Machine-checked. The DERIVE rows below are parsed by +-- TestCatalogAliasPricesMatchProviderRates in +-- apps/control-plane/internal/routing/catalog_alias_pricing_test.go, which +-- recomputes every credit figure from the rate beside it in exact math/big +-- rational arithmetic, cross-checks that rate against the committed snapshot +-- in that package's testdata/provider_rates_2026-08-22.json, and fails if +-- any figure here disagrees with the SQL below. Editing a price without +-- editing its rate, or either without re-snapshotting, turns that test red. +-- +-- DERIVE| alias_id | route_id | provider_model | field | usd_per_million | credits +-- DERIVE| hive-small | route-groq-small | groq/openai/gpt-oss-20b | in | 0.075 | 10500 +-- DERIVE| hive-small | route-groq-small | groq/openai/gpt-oss-20b | out | 0.30 | 42000 +-- DERIVE| hive-medium | route-groq-medium | groq/openai/gpt-oss-120b | in | 0.15 | 21000 +-- DERIVE| hive-medium | route-groq-medium | groq/openai/gpt-oss-120b | out | 0.60 | 84000 +-- DERIVE| hive-default | route-groq-default | groq/openai/gpt-oss-20b | in | 0.075 | 10500 +-- DERIVE| hive-default | route-groq-default | groq/openai/gpt-oss-20b | out | 0.30 | 42000 +-- DERIVE| hive-auto | route-groq-auto | groq/openai/gpt-oss-120b | in | 0.15 | 21000 +-- DERIVE| hive-auto | route-groq-auto | groq/openai/gpt-oss-120b | out | 0.60 | 84000 +-- DERIVE| deepseek-v4-flash | route-deepseek-v4-flash | openrouter/~deepseek/deepseek-v4-flash-latest | in | 0.0639 | 8946 +-- DERIVE| deepseek-v4-flash | route-deepseek-v4-flash | openrouter/~deepseek/deepseek-v4-flash-latest | out | 0.1278 | 17892 +-- DERIVE| deepseek-v4-flash | route-deepseek-v4-flash | openrouter/~deepseek/deepseek-v4-flash-latest | cache_read | 0.01278 | 1790 +-- DERIVE| deepseek-v4-pro | route-deepseek-v4-pro | openrouter/deepseek/deepseek-v4-pro-0813 | in | 1.122 | 157080 +-- DERIVE| deepseek-v4-pro | route-deepseek-v4-pro | openrouter/deepseek/deepseek-v4-pro-0813 | out | 3.366 | 471240 +-- DERIVE| deepseek-v4-pro | route-deepseek-v4-pro | openrouter/deepseek/deepseek-v4-pro-0813 | cache_read | 0.0374 | 5236 +-- +-- Longhand, for a reader rather than the parser: +-- hive-small in 0.075 * 1.4 * 100000 = 10500 (exact) +-- hive-small out 0.300 * 1.4 * 100000 = 42000 (exact) +-- hive-medium in 0.150 * 1.4 * 100000 = 21000 (exact) +-- hive-medium out 0.600 * 1.4 * 100000 = 84000 (exact) +-- hive-default in 0.075 * 1.4 * 100000 = 10500 (exact) +-- hive-default out 0.300 * 1.4 * 100000 = 42000 (exact) +-- hive-auto in 0.150 * 1.4 * 100000 = 21000 (exact) +-- hive-auto out 0.600 * 1.4 * 100000 = 84000 (exact) +-- hive-default and hive-auto are written out in full rather than as "same as +-- hive-small" and "same as hive-medium" on purpose: they are independent +-- rows, and a future reader repricing one of them must not silently reprice +-- the other by editing a shared line. +-- deepseek-v4-flash in 0.0639 * 1.4 * 100000 = 8946 (exact) +-- deepseek-v4-flash out 0.1278 * 1.4 * 100000 = 17892 (exact) +-- deepseek-v4-flash cr 0.01278 * 1.4 * 100000 = 1789.2 -> 1790 (CEILED) +-- deepseek-v4-pro in 1.122 * 1.4 * 100000 = 157080 (exact) +-- deepseek-v4-pro out 3.366 * 1.4 * 100000 = 471240 (exact) +-- deepseek-v4-pro cr 0.0374 * 1.4 * 100000 = 5236 (exact) +-- Only deepseek-v4-flash's cache-read product is fractional, so it is the +-- one row where the ceiling actually does something. +-- +-- ALIAS NAMING +-- deepseek-v4-flash and deepseek-v4-pro name their provider's model family, +-- unlike the provider-blind hive-* aliases. That is an explicit owner +-- decision taken after the provider-blind convention in CLAUDE.md was +-- raised, and it is scoped to these two alias names only. The +-- provider-blind rule continues to apply in full to error messages and to +-- routing internals. The display_name strings are the owner's own wording, +-- verbatim. alias_id is the value a client sends in the OpenAI-compatible +-- "model" field, so it is the slug form rather than the display string. +-- +-- PRICE UNIT +-- All four aliases are per-token, so price_unit is left at its column +-- default of 'tokens' (20260801_13_alias_price_unit.sql). +-- +-- CACHE COLUMNS +-- cache_read_price_credits is derived for the two DeepSeek aliases because +-- OpenRouter publishes a cache-read rate for both. It is 0 for the two Groq +-- aliases because Groq publishes no cache-read rate for the gpt-oss family; +-- 0 here means "no rate published", and mirrors that both Groq routes below +-- declare supports_cache_read = false. These columns remain display-only +-- either way: precedence.go does not read them, exactly as recorded in +-- 20260801_01. +-- +-- RE-RUNNABILITY +-- Every INSERT carries ON CONFLICT DO NOTHING and every UPDATE carries a +-- WHERE guard that excludes rows already at the target value, so a second +-- run of this file affects zero rows and errors on nothing. +-- +-- DO NOTHING rather than DO UPDATE on the INSERTS, for the reason 20260717_02 +-- gives: a row already present may have been retuned since, and this +-- migration has no business reverting that. Scoped deliberately to the +-- INSERTs, because the UPDATEs below do exactly that reverting. Replaying +-- this file after someone corrects hive-default's or hive-auto's price, or +-- un-hides hive-fast, puts those rows back to the 2026-08-22 values. That is +-- what a repricing migration is for and 20260818_01 set the same precedent, +-- but it means this file is NOT safe to replay over hand-tuned state. +-- ============================================================================= + +-- One transaction, for the reason 20260818_01 introduced it: these statements +-- establish the alias, its single route, its capabilities, its policy and its +-- group membership together. A request landing on ListRouteCandidates or +-- LoadAliasPricing partway through must not see an alias that exists but has no +-- route, or a route with no price. +BEGIN; + +-- 1. The four new customer-facing aliases. +insert into public.model_aliases ( + alias_id, + owned_by, + display_name, + summary, + visibility, + lifecycle, + capability_badges, + input_price_credits, + output_price_credits, + cache_read_price_credits, + cache_write_price_credits +) values + ( + 'hive-small', + 'hive', + 'Hive Small', + 'Fast, low-cost chat for everyday prompts. Replaces hive-fast, which is deprecated and now resolves to the same model at the same price.', + 'public', + 'stable', + '["stable","chat","responses"]'::jsonb, + 10500, + 42000, + 0, + 0 + ), + ( + 'hive-medium', + 'hive', + 'Hive Medium', + 'Larger general-purpose chat model. Same family as Hive Small, more capacity per request.', + 'public', + 'stable', + '["stable","chat","responses"]'::jsonb, + 21000, + 84000, + 0, + 0 + ), + ( + 'deepseek-v4-flash', + 'hive', + 'Deepseek V4 Flash', + 'Very low-cost long-context chat with tool use and reasoning. Largest context window in the catalog.', + 'public', + 'stable', + '["stable","chat","responses","tools","reasoning"]'::jsonb, + 8946, + 17892, + 1790, + 0 + ), + ( + 'deepseek-v4-pro', + 'hive', + 'Deepseek V4 Pro', + 'Highest-capability long-context chat with tool use and reasoning, for harder work.', + 'public', + 'stable', + '["stable","chat","responses","tools","reasoning"]'::jsonb, + 157080, + 471240, + 5236, + 0 + ) +on conflict (alias_id) do nothing; + +-- 2. Exactly one route per new alias. +-- route-groq-small deliberately points at the same upstream model as the +-- existing route-groq-fast rather than reusing that route: provider_routes +-- is keyed one route to one alias, so hive-small needs its own row for +-- hive-fast to keep working. Two routes naming one upstream model is a +-- shape this migration itself establishes three times over: +-- route-groq-fast, route-groq-small and route-groq-default all call +-- groq/openai/gpt-oss-20b, and route-groq-medium and route-groq-auto both +-- call groq/openai/gpt-oss-120b. (An earlier draft cited +-- route-openrouter-auto and route-doc-vlm sharing a model in +-- deploy/litellm/config.yaml; this change deletes the first of those, so +-- the citation would have sent a reader looking for something gone.) +insert into public.provider_routes ( + route_id, + alias_id, + provider, + provider_model, + litellm_model_name, + price_class, + health_state, + priority +) values + ( + 'route-groq-small', + 'hive-small', + 'groq', + 'groq/openai/gpt-oss-20b', + 'route-groq-small', + 'standard', + 'healthy', + 10 + ), + ( + 'route-groq-medium', + 'hive-medium', + 'groq', + 'groq/openai/gpt-oss-120b', + 'route-groq-medium', + 'standard', + 'healthy', + 10 + ), + ( + 'route-deepseek-v4-flash', + 'deepseek-v4-flash', + 'openrouter', + 'openrouter/~deepseek/deepseek-v4-flash-latest', + 'route-deepseek-v4-flash', + 'budget', + 'healthy', + 10 + ), + ( + 'route-deepseek-v4-pro', + 'deepseek-v4-pro', + 'openrouter', + 'openrouter/deepseek/deepseek-v4-pro-0813', + 'route-deepseek-v4-pro', + 'premium', + 'healthy', + 10 + ) +on conflict (route_id) do nothing; + +-- 3. Capabilities per route. +-- +-- supports_reasoning is TRUE on both Groq rows, and this is worth stating +-- because an earlier revision had it false on the reasoning that an +-- over-claim breaks requests while an under-claim merely withholds a +-- feature. That reasoning is wrong for every alias in this migration. +-- It holds only for a multi-route alias, where a narrower route just loses +-- the ordering contest. Every alias here is pinned to exactly ONE route by +-- step 4, so with a single candidate matchesRequestedCapabilities +-- (routing/service.go) drops it, SelectRoute returns ErrRouteNotEligible +-- and writeRoutingError maps that to 422. An under-claim on a pinned alias +-- is not a withheld feature, it is a failed request. +-- The flag is also simply true, and verified rather than assumed: +-- https://console.groq.com/docs/reasoning (checked 2026-08-22) lists both +-- openai/gpt-oss-20b and openai/gpt-oss-120b as reasoning models and states +-- that reasoning_effort values 'low', 'medium' and 'high' "are only +-- supported by GPT-OSS 20B and GPT-OSS 120B". On our side, +-- apps/edge-api/internal/inference/chat_completions.go sets NeedReasoning +-- whenever a request carries reasoning_effort. +-- route-groq-fast still carries false from its original seed. That is a +-- pre-existing under-claim on the deprecated alias, left alone here rather +-- than widened in a pricing migration. +-- +-- supports_cache_read and supports_cache_write stay false. Groq publishes +-- no cache rate for the gpt-oss family, and unlike reasoning this costs +-- nothing at dispatch: no request path in edge-api ever sets NeedCacheRead +-- or NeedCacheWrite, they are plumbed through the routing API and never +-- populated, so a false flag here cannot turn into a 422. +-- +-- tools_supported is true for all four. 20260612_01 already sets it true +-- for every openrouter and groq route by provider, but that migration ran +-- once over the rows that existed then; new rows would otherwise take the +-- column DEFAULT of false and silently reject tools, tool_choice and +-- response_format (PR #206 routes those on this column). +-- +-- The DeepSeek flags follow the live API's supported_parameters and pricing +-- fields directly: both list tools, tool_choice, response_format, +-- structured_outputs, reasoning and reasoning_effort, and both publish a +-- cache-read but no cache-write rate. One real difference is not +-- representable here: flash lists parallel_tool_calls and pro does not, and +-- provider_capabilities has no parallel-tool-call column. No catalog claim +-- becomes untrue by that omission, the same conclusion 20260818_01 reached +-- about the identical gap. +insert into public.provider_capabilities ( + route_id, + supports_responses, + supports_chat_completions, + supports_completions, + supports_embeddings, + supports_streaming, + supports_reasoning, + supports_cache_read, + supports_cache_write, + tools_supported +) values + ('route-groq-small', true, true, true, false, true, true, false, false, true), + ('route-groq-medium', true, true, true, false, true, true, false, false, true), + ('route-deepseek-v4-flash', true, true, true, false, true, true, true, false, true), + ('route-deepseek-v4-pro', true, true, true, false, true, true, true, false, true) +on conflict (route_id) do nothing; + +-- 4. Pin each alias to its single route. 'pinned' with a one-entry +-- fallback_order is the shape 20260717_02 used for the voice aliases and is +-- what the one-alias-one-enabled-route rule requires. +insert into public.alias_route_policies ( + alias_id, + policy_mode, + allow_price_class_widening, + fallback_order +) values + ('hive-small', 'pinned', false, '["route-groq-small"]'::jsonb), + ('hive-medium', 'pinned', false, '["route-groq-medium"]'::jsonb), + ('deepseek-v4-flash', 'pinned', false, '["route-deepseek-v4-flash"]'::jsonb), + ('deepseek-v4-pro', 'pinned', false, '["route-deepseek-v4-pro"]'::jsonb) +on conflict (alias_id) do nothing; + +-- 5. Group membership. Without this the whole migration is inert for customers: +-- api_key_policies.allowed_group_names defaults to '["default"]', so a +-- default-tier key never sees an alias that is not in the 'default' group, +-- however correctly it is priced and routed. This gap has already had to be +-- patched by hand twice, by 20260717_01 for hive-auto and by 20260717_02 for +-- the voice aliases. 'closed' mirrors how 20260331_03 seeded every chat +-- alias into both groups. +-- deepseek-v4-pro is additionally placed in 'premium'. That group exists +-- for "Premium or higher-cost models" (20260331_03), and after this +-- migration deepseek-v4-pro is by a wide margin the most expensive alias in +-- the catalog at 157080 in and 471240 out, while hive-auto, the alias +-- 'premium' was originally created around, has just been repriced DOWN to +-- 21000 and 84000. Leaving the highest-cost model out of the cost-gating +-- group would quietly break that group's only purpose, and a key scoped to +-- ["premium"] could not reach the premium model at all. It stays in +-- 'default' as well because the owner asked for these aliases to be +-- customer-facing, and prepaid credit balance, not group membership, is +-- what actually bounds spend. +insert into public.model_policy_group_members (group_name, alias_id) values + ('default', 'hive-small'), + ('default', 'hive-medium'), + ('default', 'deepseek-v4-flash'), + ('default', 'deepseek-v4-pro'), + ('closed', 'hive-small'), + ('closed', 'hive-medium'), + ('closed', 'deepseek-v4-flash'), + ('closed', 'deepseek-v4-pro'), + ('premium', 'deepseek-v4-pro') +on conflict (group_name, alias_id) do nothing; + +-- 6. Move hive-default and hive-auto off OpenRouter and onto the two Groq +-- models. Each alias gets a NEW route row and its old OpenRouter route is +-- disabled, so every alias still has exactly one enabled route: four +-- aliases resolving to two distinct upstream models through four separate +-- routes. No route row is shared between aliases. +-- +-- WHY NEW ROUTES RATHER THAN REPOINTING THE EXISTING ROWS IN PLACE. +-- This was the tempting one-line change and it is wrong. LiteLLM's config +-- sync merges FIELD BY FIELD: the database owns only model, api_base and +-- api_key, and every other key already on the entry survives on purpose, so +-- that hand-tuning sticks across syncs (mergeParams in +-- apps/control-plane/internal/litellmconfig/generator.go, issue #707). +-- route-openrouter-default and route-openrouter-auto both carry an +-- OpenRouter-specific extra_body block in deploy/litellm/config.yaml +-- (provider.allow_fallbacks and provider.sort). Repointing those route_ids +-- at Groq would leave that block attached and send OpenRouter's vendor +-- routing object to Groq on every request to the DEFAULT model, and no +-- sync could ever remove it. Retiring the route id instead makes the merge +-- drop the whole stale entry, because a known route_id that is no longer +-- active is deleted from the config rather than updated. Disabling rather +-- than deleting the row follows 20260801_01, which retired +-- route-openrouter-fast-fallback exactly this way: SelectRoute filters +-- 'disabled', the config sync excludes it, and the row stays reversible. +insert into public.provider_routes ( + route_id, + alias_id, + provider, + provider_model, + litellm_model_name, + price_class, + health_state, + priority +) values + ( + 'route-groq-default', + 'hive-default', + 'groq', + 'groq/openai/gpt-oss-20b', + 'route-groq-default', + 'standard', + 'healthy', + 10 + ), + ( + 'route-groq-auto', + 'hive-auto', + 'groq', + 'groq/openai/gpt-oss-120b', + 'route-groq-auto', + 'standard', + 'healthy', + 10 + ) +on conflict (route_id) do nothing; + +-- route-groq-auto additionally inherits three flags that live on NO other +-- route in the catalog: supports_batch, supports_image_generation and +-- supports_image_edit. 20260414_01_provider_capabilities_media_columns.sql +-- granted all five media flags to route-openrouter-auto and to no other +-- row, and 20260717_02 later corrected two of them (tts, stt) back to +-- false. Disabling route-openrouter-auto without carrying the remaining +-- three forward would delete three product surfaces catalog-wide, not just +-- for hive-auto: SelectRoute skips disabled candidates and then hard +-- filters on each flag (service.go), and both batchstore/submitter.go and +-- batchstore/local_executor_adapters.go send NeedBatch = true for EVERY +-- batch, so /v1/batches, /v1/images/generations and /v1/images/edits would +-- each find zero eligible routes for every alias in the system. +-- +-- supports_batch is a true claim: Phase 15 ships a control-plane local +-- batch executor and the route carries executor_kind 'local', so batching +-- does not depend on Groq having a native batch API, which it does not. +-- +-- The two image flags are carried forward as a STATUS QUO PRESERVATION and +-- are not a fresh assertion by this migration. They were already untrue of +-- route-openrouter-auto's previous model: gpt-4.1-mini cannot generate or +-- edit images either. Carrying them keeps this catalog change from +-- silently removing two endpoints as a side effect, which is not what a +-- repricing was asked to do. Correcting them properly needs a real image +-- route with a model that can actually serve one, and that is its own +-- change with its own decision behind it. Tracked separately; do not read +-- these two flags as a claim that gpt-oss-120b does images. +insert into public.provider_capabilities ( + route_id, + supports_responses, + supports_chat_completions, + supports_completions, + supports_embeddings, + supports_streaming, + supports_reasoning, + supports_cache_read, + supports_cache_write, + tools_supported, + supports_batch, + supports_image_generation, + supports_image_edit +) values + ('route-groq-default', true, true, true, false, true, true, false, false, true, false, false, false), + ('route-groq-auto', true, true, true, false, true, true, false, false, true, true, true, true) +on conflict (route_id) do nothing; + +-- 6b. Retire the two OpenRouter routes these aliases used to take. Disabled, +-- not deleted, so the change is reversible and the history survives. +UPDATE public.provider_routes + SET health_state = 'disabled' + WHERE route_id IN ('route-openrouter-default', 'route-openrouter-auto') + AND health_state <> 'disabled'; + +-- 6c. Point each alias's policy at its new route, so no policy row names a +-- route that is no longer selectable. Same step 20260801_01 took after +-- disabling route-openrouter-fast-fallback. +UPDATE public.alias_route_policies + SET fallback_order = '["route-groq-default"]'::jsonb + WHERE alias_id = 'hive-default' + AND fallback_order <> '["route-groq-default"]'::jsonb; + +UPDATE public.alias_route_policies + SET fallback_order = '["route-groq-auto"]'::jsonb + WHERE alias_id = 'hive-auto' + AND fallback_order <> '["route-groq-auto"]'::jsonb; + +-- 7. Reprice the two moved aliases to the rate of the model they now call, +-- and rewrite their summaries so the names do not imply behaviour the +-- aliases no longer have. Both prices go DOWN; see the header table. +-- The cache columns are zeroed on the same pass. They still carry the +-- values the original seed gave them against OpenRouter models +-- (20260331_01: hive-default 2 and 6, hive-auto 1 and 5), and both aliases +-- now sit on Groq routes declaring supports_cache_read = false and +-- supports_cache_write = false. These columns are not internal: +-- catalog.CatalogPricing marshals them into the public catalog response, so +-- leaving them would advertise a cache-read price for two models that do no +-- caching. Zero here means "no rate published", exactly as it does for the +-- four aliases inserted above. +UPDATE public.model_aliases + SET input_price_credits = 10500, + output_price_credits = 42000, + cache_read_price_credits = 0, + cache_write_price_credits = 0, + summary = 'Default alias for requests that name no model. Now resolves to the same fast, low-cost model as Hive Small; kept as a distinct alias for back-compat.', + updated_at = now() + WHERE alias_id = 'hive-default' + AND (input_price_credits <> 10500 OR output_price_credits <> 42000 + OR cache_read_price_credits <> 0 OR cache_write_price_credits <> 0); + +UPDATE public.model_aliases + SET input_price_credits = 21000, + output_price_credits = 84000, + cache_read_price_credits = 0, + cache_write_price_credits = 0, + summary = 'Larger-capacity alias. Performs no automatic routing or model selection; it resolves to the same model as Hive Medium and is kept as a distinct alias for back-compat.', + updated_at = now() + WHERE alias_id = 'hive-auto' + AND (input_price_credits <> 21000 OR output_price_credits <> 84000 + OR cache_read_price_credits <> 0 OR cache_write_price_credits <> 0); + +-- 8. Deprecate hive-fast. Route, price, visibility and group membership are all +-- left exactly as they are, so every existing conversation and API client +-- keeps working unchanged; only the lifecycle marker moves. +UPDATE public.model_aliases + SET lifecycle = 'hidden', + updated_at = now() + WHERE alias_id = 'hive-fast' + AND lifecycle <> 'hidden'; + +COMMIT;