diff --git a/apps/gateway/src/models/models.spec.ts b/apps/gateway/src/models/models.spec.ts index cc8235c889..d6cb4ba1b8 100644 --- a/apps/gateway/src/models/models.spec.ts +++ b/apps/gateway/src/models/models.spec.ts @@ -126,4 +126,60 @@ describe("Models API", () => { "image", ]); }); + + test("GET /v1/models should include stability information for models", async () => { + const res = await app.request("/v1/models"); + expect(res.status).toBe(200); + + const json = await res.json(); + + // Check that the stability field exists in the model schema (models may or may not have it set) + const modelsWithStability = json.data.filter( + (model: any) => model.stability !== undefined, + ); + // At least one model should have stability information (our DeepSeek models) + expect(modelsWithStability.length).toBeGreaterThan(0); + + // Find DeepSeek models to test specific stability flags + const deepSeekR1Distill = json.data.find( + (model: any) => model.id === "deepseek-r1-distill-llama-70b", + ); + const deepSeekV31 = json.data.find( + (model: any) => model.id === "deepseek-v3.1", + ); + + if (deepSeekR1Distill) { + expect(deepSeekR1Distill.stability).toBe("beta"); + } + + // DeepSeek v3.1 should default to stable (undefined in response means stable) + if (deepSeekV31) { + expect( + deepSeekV31.stability === undefined || + deepSeekV31.stability === "stable", + ).toBe(true); + } + }); + + test("GET /v1/models should handle stability field values correctly", async () => { + const res = await app.request("/v1/models"); + expect(res.status).toBe(200); + + const json = await res.json(); + + // Validate that stability field contains only valid values + const validStabilityValues = [ + "stable", + "beta", + "unstable", + "experimental", + undefined, + ]; + + for (const model of json.data) { + if (model.stability !== undefined) { + expect(validStabilityValues).toContain(model.stability); + } + } + }); }); diff --git a/apps/gateway/src/models/models.ts b/apps/gateway/src/models/models.ts index dd282e2bba..4975d655bc 100644 --- a/apps/gateway/src/models/models.ts +++ b/apps/gateway/src/models/models.ts @@ -43,6 +43,9 @@ const modelSchema = z.object({ tools: z.boolean(), parallelToolCalls: z.boolean(), reasoning: z.boolean(), + stability: z + .enum(["stable", "beta", "unstable", "experimental"]) + .optional(), }), ), pricing: z.object({ @@ -62,6 +65,7 @@ const modelSchema = z.object({ free: z.boolean().optional(), deprecated_at: z.string().optional(), deactivated_at: z.string().optional(), + stability: z.enum(["stable", "beta", "unstable", "experimental"]).optional(), }); const listModelsResponseSchema = z.object({ @@ -198,6 +202,7 @@ modelsApi.openapi(listModels, async (c) => { tools: provider.tools || false, parallelToolCalls: provider.parallelToolCalls || false, reasoning: provider.reasoning || false, + stability: provider.stability || model.stability, }; }), pricing: { @@ -222,6 +227,7 @@ modelsApi.openapi(listModels, async (c) => { free: model.free || false, deprecated_at: model.deprecatedAt?.toISOString(), deactivated_at: model.deactivatedAt?.toISOString(), + stability: model.stability, }; }); diff --git a/apps/ui/src/app/models/[name]/page.tsx b/apps/ui/src/app/models/[name]/page.tsx index 78bb64128a..892b0659d1 100644 --- a/apps/ui/src/app/models/[name]/page.tsx +++ b/apps/ui/src/app/models/[name]/page.tsx @@ -1,13 +1,17 @@ import { models as modelDefinitions, providers as providerDefinitions, + type StabilityLevel, + type ModelDefinition, } from "@llmgateway/models"; +import { AlertTriangle } from "lucide-react"; import { notFound } from "next/navigation"; import Footer from "@/components/landing/footer"; import { Navbar } from "@/components/landing/navbar"; import { CopyModelName } from "@/components/models/copy-model-name"; import { ProviderCard } from "@/components/models/provider-card"; +import { Badge } from "@/lib/components/badge"; interface PageProps { params: Promise<{ name: string }>; @@ -17,12 +21,43 @@ export default async function ModelPage({ params }: PageProps) { const { name } = await params; const decodedName = decodeURIComponent(name); - const modelDef = modelDefinitions.find((m) => m.id === decodedName); + const modelDef = modelDefinitions.find( + (m) => m.id === decodedName, + ) as ModelDefinition; if (!modelDef) { notFound(); } + const getStabilityBadgeProps = (stability?: StabilityLevel) => { + switch (stability) { + case "beta": + return { + variant: "secondary" as const, + color: "text-blue-600", + label: "BETA", + }; + case "unstable": + return { + variant: "destructive" as const, + color: "text-red-600", + label: "UNSTABLE", + }; + case "experimental": + return { + variant: "destructive" as const, + color: "text-orange-600", + label: "EXPERIMENTAL", + }; + default: + return null; + } + }; + + const shouldShowStabilityWarning = (stability?: StabilityLevel) => { + return stability && ["unstable", "experimental"].includes(stability); + }; + const modelProviders = modelDef.providers.map((provider) => { const providerInfo = providerDefinitions.find( (p) => p.id === provider.providerId, @@ -39,11 +74,33 @@ export default async function ModelPage({ params }: PageProps) {
-

- {modelDef.id} -

+
+

+ {modelDef.id} +

+ {shouldShowStabilityWarning(modelDef.stability) && ( + + )} +
+ {(() => { + const stabilityProps = getStabilityBadgeProps( + modelDef.stability, + ); + return stabilityProps ? ( + + {stabilityProps.label} + + ) : ( + + STABLE + + ); + })()}
@@ -86,6 +143,26 @@ export default async function ModelPage({ params }: PageProps) { different configurations, pricing, and performance characteristics. Choose the provider that best fits your needs.

+ {shouldShowStabilityWarning(modelDef.stability) && ( +
+
+ +
+

+ {modelDef.stability === "experimental" + ? "Experimental" + : "Unstable"}{" "} + Model Warning +

+

+ This model is marked as {modelDef.stability} and may have + issues with reliability, performance, or consistency. Use + with caution in production environments. +

+
+
+
+ )}
@@ -108,6 +185,7 @@ export default async function ModelPage({ params }: PageProps) { key={provider.providerId} provider={provider} modelName={decodedName} + modelStability={modelDef.stability} /> ))}
diff --git a/apps/ui/src/components/models/all-models.tsx b/apps/ui/src/components/models/all-models.tsx index 840ebf0b1d..58a87d0fa7 100644 --- a/apps/ui/src/components/models/all-models.tsx +++ b/apps/ui/src/components/models/all-models.tsx @@ -19,6 +19,7 @@ import { ArrowDown, Play, ImagePlus, + AlertTriangle, ExternalLink, } from "lucide-react"; import Link from "next/link"; @@ -61,7 +62,11 @@ import { } from "@/lib/components/tooltip"; import { cn, formatContextSize } from "@/lib/utils"; -import type { ModelDefinition, ProviderModelMapping } from "@llmgateway/models"; +import type { + ModelDefinition, + ProviderModelMapping, + StabilityLevel, +} from "@llmgateway/models"; interface ModelWithProviders extends ModelDefinition { providerDetails: Array<{ @@ -369,6 +374,59 @@ export function AllModels({ children }: { children: React.ReactNode }) { ); }; + const getStabilityBadgeProps = (stability?: StabilityLevel) => { + switch (stability) { + case "beta": + return { + variant: "secondary" as const, + color: "text-blue-600", + label: "BETA", + }; + case "unstable": + return { + variant: "destructive" as const, + color: "text-red-600", + label: "UNSTABLE", + }; + case "experimental": + return { + variant: "destructive" as const, + color: "text-orange-600", + label: "EXPERIMENTAL", + }; + default: + return null; + } + }; + + const shouldShowStabilityWarning = (stability?: StabilityLevel) => { + return stability && ["unstable", "experimental"].includes(stability); + }; + + const getMostUnstableStability = (model: any): StabilityLevel | undefined => { + const stabilityLevels: StabilityLevel[] = [ + "experimental", + "unstable", + "beta", + "stable", + ]; + + // Get all stability levels (model-level and provider-level) + const allStabilities = [ + model.stability, + ...model.providers.map((p: any) => p.stability || model.stability), + ].filter(Boolean) as StabilityLevel[]; + + // Return the most unstable level + for (const level of stabilityLevels) { + if (allStabilities.includes(level)) { + return level; + } + } + + return undefined; + }; + const copyToClipboard = async (text: string) => { try { await navigator.clipboard.writeText(text); @@ -756,6 +814,7 @@ export function AllModels({ children }: { children: React.ReactNode }) { Capabilities + Stability Actions @@ -774,6 +833,11 @@ export function AllModels({ children }: { children: React.ReactNode }) {
{model.name || model.id} + {shouldShowStabilityWarning( + getMostUnstableStability(model), + ) && ( + + )} {model.free && ( + + {(() => { + const mostUnstableStability = + getMostUnstableStability(model); + const stabilityProps = getStabilityBadgeProps( + mostUnstableStability, + ); + return stabilityProps ? ( + + {stabilityProps.label} + + ) : ( + + STABLE + + ); + })()} + + {model.name || model.id} + {shouldShowStabilityWarning(model.stability) && ( + + )} {model.free && ( +
+
Stability:
+ {(() => { + const mostUnstableStability = getMostUnstableStability(model); + const stabilityProps = getStabilityBadgeProps( + mostUnstableStability, + ); + return stabilityProps ? ( + + {stabilityProps.label} + + ) : ( + + STABLE + + ); + })()} +
+
{ + switch (stability) { + case "beta": + return { + variant: "secondary" as const, + color: "text-blue-600", + label: "BETA", + }; + case "unstable": + return { + variant: "destructive" as const, + color: "text-red-600", + label: "UNSTABLE", + }; + case "experimental": + return { + variant: "destructive" as const, + color: "text-orange-600", + label: "EXPERIMENTAL", + }; + default: + return null; + } + }; + + const shouldShowStabilityWarning = (stability?: StabilityLevel) => { + return stability && ["unstable", "experimental"].includes(stability); + }; + return (
{(models as readonly ModelDefinition[]).map((model) => ( -
{model.name || model.id}
+
+ {model.name || model.id} + {shouldShowStabilityWarning(model.stability) && ( + + )} +
Providers:
- {model.providers.map((provider) => ( - {provider.providerId} - ))} + {model.providers.map((provider) => { + const providerStability = provider.stability || model.stability; + const stabilityProps = getStabilityBadgeProps(providerStability); + + return ( +
+ {provider.providerId} + {stabilityProps && ( + + {stabilityProps.label} + + )} +
+ ); + })} +
+
+ Stability: + {(() => { + const stabilityProps = getStabilityBadgeProps(model.stability); + return stabilityProps ? ( + + {stabilityProps.label} + + ) : ( + + STABLE + + ); + })()}
{model.providers.map((provider) => ( diff --git a/apps/ui/src/components/models/provider-card.tsx b/apps/ui/src/components/models/provider-card.tsx index 57f49e92c5..ab1b721cb7 100644 --- a/apps/ui/src/components/models/provider-card.tsx +++ b/apps/ui/src/components/models/provider-card.tsx @@ -1,9 +1,10 @@ "use client"; -import { Copy, Check } from "lucide-react"; +import { Copy, Check, AlertTriangle } from "lucide-react"; import { useState } from "react"; import { getProviderIcon } from "@/components/ui/providers-icons"; +import { Badge } from "@/lib/components/badge"; import { Button } from "@/lib/components/button"; import { Card, CardContent } from "@/lib/components/card"; import { formatContextSize } from "@/lib/utils"; @@ -11,6 +12,7 @@ import { formatContextSize } from "@/lib/utils"; import type { ProviderModelMapping, ProviderDefinition, + StabilityLevel, } from "@llmgateway/models"; interface ProviderWithInfo extends ProviderModelMapping { @@ -20,12 +22,47 @@ interface ProviderWithInfo extends ProviderModelMapping { interface ProviderCardProps { provider: ProviderWithInfo; modelName: string; + modelStability?: StabilityLevel; } -export function ProviderCard({ provider, modelName }: ProviderCardProps) { +export function ProviderCard({ + provider, + modelName, + modelStability, +}: ProviderCardProps) { const [copied, setCopied] = useState(false); const providerModelName = `${provider.providerId}/${modelName}`; const ProviderIcon = getProviderIcon(provider.providerId); + const providerStability = provider.stability || modelStability; + + const getStabilityBadgeProps = (stability?: StabilityLevel) => { + switch (stability) { + case "beta": + return { + variant: "secondary" as const, + color: "text-blue-600", + label: "BETA", + }; + case "unstable": + return { + variant: "destructive" as const, + color: "text-red-600", + label: "UNSTABLE", + }; + case "experimental": + return { + variant: "destructive" as const, + color: "text-orange-600", + label: "EXPERIMENTAL", + }; + default: + return null; + } + }; + + const shouldShowStabilityWarning = (stability?: StabilityLevel) => { + return stability && ["unstable", "experimental"].includes(stability); + }; const copyToClipboard = async () => { try { @@ -50,9 +87,26 @@ export function ProviderCard({ provider, modelName }: ProviderCardProps) { )}
-

- {provider.providerInfo?.name || provider.providerId} -

+
+

+ {provider.providerInfo?.name || provider.providerId} +

+ {shouldShowStabilityWarning(providerStability) && ( + + )} + {(() => { + const stabilityProps = + getStabilityBadgeProps(providerStability); + return stabilityProps ? ( + + {stabilityProps.label} + + ) : null; + })()} +
{providerModelName} diff --git a/apps/ui/src/components/playground/model-selector.tsx b/apps/ui/src/components/playground/model-selector.tsx index 1fc1e51ee2..60909de67f 100644 --- a/apps/ui/src/components/playground/model-selector.tsx +++ b/apps/ui/src/components/playground/model-selector.tsx @@ -1,5 +1,10 @@ -import { models, providers, type ModelDefinition } from "@llmgateway/models"; -import { Check, ChevronDown } from "lucide-react"; +import { + models, + providers, + type ModelDefinition, + type StabilityLevel, +} from "@llmgateway/models"; +import { Check, ChevronDown, AlertTriangle } from "lucide-react"; import { Badge } from "@/lib/components/badge"; import { Button } from "@/lib/components/button"; @@ -19,6 +24,7 @@ interface LocalModel { id: string; name?: string; jsonOutput: boolean; + stability?: StabilityLevel; providers: Array<{ providerId: string; modelName: string; @@ -27,6 +33,7 @@ interface LocalModel { imageInputPrice?: number; requestPrice?: number; contextSize?: number; + stability?: StabilityLevel; providerInfo?: { id: string; name: string; @@ -47,6 +54,61 @@ export function ModelSelector({ return providers.find((p) => p.id === providerId); }; + const getStabilityBadgeProps = (stability?: StabilityLevel) => { + switch (stability) { + case "beta": + return { + variant: "secondary" as const, + color: "text-blue-600", + label: "BETA", + }; + case "unstable": + return { + variant: "destructive" as const, + color: "text-red-600", + label: "UNSTABLE", + }; + case "experimental": + return { + variant: "destructive" as const, + color: "text-orange-600", + label: "EXPERIMENTAL", + }; + default: + return null; + } + }; + + const shouldShowStabilityWarning = (stability?: StabilityLevel) => { + return stability && ["unstable", "experimental"].includes(stability); + }; + + const getMostUnstableStability = ( + model: LocalModel, + ): StabilityLevel | undefined => { + const stabilityLevels: StabilityLevel[] = [ + "experimental", + "unstable", + "beta", + "stable", + ]; + + // Get all stability levels (model-level and provider-level) + const allStabilities = [ + model.stability, + ...model.providers.map((p) => p.stability || model.stability), + ].filter(Boolean) as StabilityLevel[]; + + // Return the most unstable level + for (const level of stabilityLevels) { + if (allStabilities.includes(level)) { + return level; + } + } + + return undefined; + }; + // Group by model instead of provider to avoid duplicates const uniqueModels: LocalModel[] = models.map((model) => { const modelProviders = model.providers @@ -64,6 +126,7 @@ export function ModelSelector({ id: typedModel.id, name: typedModel.name, jsonOutput: typedModel.jsonOutput ?? false, + stability: typedModel.stability, providers: modelProviders, }; }); @@ -90,6 +153,10 @@ export function ModelSelector({ {currentModelInfo?.name || currentModelInfo?.id || selectedModel} + {currentModelInfo && + shouldShowStabilityWarning( + getMostUnstableStability(currentModelInfo), + ) && }
@@ -113,6 +180,9 @@ export function ModelSelector({ ))}
{model.name || model.id} + {shouldShowStabilityWarning( + getMostUnstableStability(model), + ) && } {model.id === selectedModel && ( )} @@ -144,6 +214,21 @@ export function ModelSelector({ Stream )} + {(() => { + const mostUnstableStability = + getMostUnstableStability(model); + const stabilityProps = getStabilityBadgeProps( + mostUnstableStability, + ); + return stabilityProps ? ( + + {stabilityProps.label} + + ) : null; + })()}
diff --git a/packages/models/src/models.ts b/packages/models/src/models.ts index 34cc9957b8..158fe4d430 100644 --- a/packages/models/src/models.ts +++ b/packages/models/src/models.ts @@ -88,8 +88,18 @@ export interface ProviderModelMapping { * Test skip/only functionality */ test?: "skip" | "only"; + /** + * Stability level of the model for this specific provider (defaults to model-level stability if not specified) + * - stable: Fully tested and production ready + * - beta: Generally stable but may have minor issues + * - unstable: May have significant issues or frequent changes + * - experimental: Early stage, use with caution + */ + stability?: StabilityLevel; } +export type StabilityLevel = "stable" | "beta" | "unstable" | "experimental"; + export interface ModelDefinition { /** * Unique identifier for the model @@ -131,6 +141,14 @@ export interface ModelDefinition { * Output formats supported by the model (defaults to ['text'] if not specified) */ output?: ("text" | "image")[]; + /** + * Stability level of the model (defaults to 'stable' if not specified) + * - stable: Fully tested and production ready + * - beta: Generally stable but may have minor issues + * - unstable: May have significant issues or frequent changes + * - experimental: Early stage, use with caution + */ + stability?: StabilityLevel; /** * Whether this model supports system role messages (defaults to true if not specified) */ diff --git a/packages/models/src/models/deepseek.ts b/packages/models/src/models/deepseek.ts index 41934262eb..a68b8194bb 100644 --- a/packages/models/src/models/deepseek.ts +++ b/packages/models/src/models/deepseek.ts @@ -112,19 +112,19 @@ export const deepseekModels = [ vision: false, tools: false, }, - // TODO mark as unstable - // { - // providerId: "nebius", - // modelName: "deepseek-ai/DeepSeek-R1-0528", - // inputPrice: 0.8 / 1e6, - // outputPrice: 2.4 / 1e6, - // requestPrice: 0, - // contextSize: 64000, - // maxOutput: undefined, - // streaming: true, - // vision: false, - // tools: false, - // }, + { + providerId: "nebius", + modelName: "deepseek-ai/DeepSeek-R1-0528", + inputPrice: 0.8 / 1e6, + outputPrice: 2.4 / 1e6, + requestPrice: 0, + contextSize: 64000, + maxOutput: undefined, + streaming: true, + vision: false, + tools: false, + stability: "unstable" as const, + }, ], jsonOutput: false, }, @@ -134,6 +134,7 @@ export const deepseekModels = [ family: "deepseek", deprecatedAt: undefined, deactivatedAt: undefined, + stability: "beta" as const, providers: [ { providerId: "groq",