feat(ai): validate catalog generation invariants and fail violating regens

packages/ai/.changes/res-1273-catalog-generation-invariants.md

@@ -0,0 +1,5 @@
+- Fixed GitHub Copilot Grok and MAI-Code models failing on chat completions by routing them through the Copilot Responses API.
+- Raised the Codex subscription (openai-codex) context window for the GPT-5.6 family from 272k to the full 1,050,000 measured on the API side.
+- Added low/high reasoning effort levels for Kimi K3 on effort-capable providers (OpenRouter, Prime Inference, Hugging Face, Fireworks, opencode, Kimi Coding, Vercel); the Moonshot and GitHub Copilot transports cannot send effort, so their rows now advertise no selectable levels instead of a no-op max.
+- Fixed thinkingmachines/Inkling-Small advertising a maxTokens above its context window.
+- Catalog regeneration now validates invariants (maxTokens vs context window, Copilot API classification, Codex context divergence, cross-provider thinking-level consistency) and fails instead of writing a violating catalog.

packages/ai/scripts/generate-models.ts

@@ -4,6 +4,13 @@ import { writeFileSync } from "fs";
 import { dirname, join } from "path";
 import { fileURLToPath } from "url";
 import { getAnthropicCacheCosts } from "../src/cache-pricing.js";
+import { supportsAdaptiveThinking } from "../src/providers/anthropic.js";
+import { getCompat } from "../src/providers/openai-completions.js";
+import {
+	CODEX_SMALLER_WINDOW_VERIFIED,
+	copilotModelApi,
+	validateModelCatalog,
+} from "./validate-model-catalog.js";
 import { COPILOT_CLIENT_HEADERS } from "../src/copilot-client-version.js";
 import { getOpenRouterReasoningCapabilities } from "../src/openrouter-reasoning.js";
 import {
@@ -96,9 +103,9 @@ const DEEPSEEK_V4_THINKING_LEVEL_MAP = {
 const KIMI_K3_THINKING_LEVEL_MAP = {
 	off: null,
 	minimal: null,
-	low: null,
+	low: "low",
 	medium: null,
-	high: null,
+	high: "high",
 	xhigh: null,
 	max: "max",
 } as const;
@@ -1257,17 +1264,8 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 				if (m.tool_call !== true) continue;
 				if (m.status === "deprecated") continue;
 
-				// Copilot proxies Claude via the Anthropic Messages API
-				const isCopilotClaude = modelId.startsWith("claude-");
-				// gpt-5/gpt-6 models require responses API, others use completions
-				const needsResponsesApi =
-					modelId.startsWith("gpt-5") || modelId.startsWith("gpt-6") || modelId.startsWith("oswe");
-
-				const api: Api = isCopilotClaude
-					? "anthropic-messages"
-					: needsResponsesApi
-						? "openai-responses"
-						: "openai-completions";
+				// Unclassified families still ship a row so validation reports them by name instead of silently misrouting.
+				const api: Api = copilotModelApi(modelId) ?? "openai-completions";
 
 				const anthropicCompat =
 					api === "anthropic-messages" ? getAnthropicMessagesCompat("github-copilot", modelId) : undefined;
@@ -1456,6 +1454,14 @@ async function loadModelsDevData(): Promise<Model<any>[]> {
 			}
 		}
 
+		// models.dev rows occasionally carry an output limit above the context window; clamp to keep the pair coherent.
+		for (const model of models) {
+			if (model.maxTokens > model.contextWindow) {
+				console.log(`Clamping ${model.provider}/${model.id} maxTokens ${model.maxTokens} -> ${model.contextWindow}`);
+				model.maxTokens = model.contextWindow;
+			}
+		}
+
 		console.log(`Loaded ${models.length} tool-capable models from models.dev`);
 		return models;
 	} catch (error) {
@@ -2014,6 +2020,14 @@ async function generateModels() {
 			maxTokens: CODEX_MAX_TOKENS,
 		},
 	];
+	// The ChatGPT backend accepts the 1M+ API-side window (#1597) except upstream-verified smaller rows.
+	for (const codexModel of codexModels) {
+		if (CODEX_SMALLER_WINDOW_VERIFIED.has(codexModel.id)) continue;
+		const openaiTwin = allModels.find((m) => m.provider === "openai" && m.id === codexModel.id);
+		if (openaiTwin && openaiTwin.contextWindow >= 1_000_000) {
+			codexModel.contextWindow = openaiTwin.contextWindow;
+		}
+	}
 	allModels.push(...codexModels);
 
 	// Add missing Grok models
@@ -2259,6 +2273,30 @@ async function generateModels() {
 		applyThinkingLevelMetadata(model);
 	}
 
+	// Non-adaptive anthropic-messages rows think via budget tokens, where xhigh/max clamp to high.
+	for (const model of allModels) {
+		if (model.api !== "anthropic-messages" || !model.reasoning || !model.thinkingLevelMap) continue;
+		if (supportsAdaptiveThinking(model.id)) continue;
+		const phantom = ["xhigh", "max"].filter((level) => model.thinkingLevelMap?.[level] != null);
+		if (phantom.length === 0) continue;
+		console.log(`Nulling budget-clamped thinking levels on ${model.provider}/${model.id}: ${phantom.join(", ")}`);
+		model.thinkingLevelMap = { ...model.thinkingLevelMap, ...Object.fromEntries(phantom.map((level) => [level, null])) };
+	}
+
+	// Family maps can land on transports that never send reasoning effort; null them so the UI offers nothing the request drops.
+	for (const model of allModels) {
+		if (model.api !== "openai-completions" || !model.reasoning || !model.thinkingLevelMap) continue;
+		const compat = getCompat(model as Model<"openai-completions">);
+		if (compat.thinkingFormat !== "openai" || compat.supportsReasoningEffort) continue;
+		const selectable = Object.entries(model.thinkingLevelMap).filter(([, mapped]) => mapped !== null);
+		if (selectable.length === 0) continue;
+		console.log(
+			`Nulling unsendable thinking levels on ${model.provider}/${model.id}: ${selectable.map(([level]) => level).join(", ")}`,
+		);
+		// All levels explicitly null: absent keys read as supported at runtime.
+		model.thinkingLevelMap = { off: null, minimal: null, low: null, medium: null, high: null, xhigh: null, max: null };
+	}
+
 	// Group by provider and deduplicate by model ID
 	const providers: Record<string, Record<string, Model<Api>>> = {};
 	for (const model of allModels) {
@@ -2272,6 +2310,14 @@ async function generateModels() {
 		}
 	}
 
+	const violations = validateModelCatalog(providers);
+	if (violations.length > 0) {
+		for (const violation of violations) {
+			console.error(`Catalog validation: ${violation}`);
+		}
+		throw new Error(`Model catalog validation failed with ${violations.length} violation(s); not writing catalog`);
+	}
+
 	// Generate TypeScript file. JSON string literals prevent remote catalog
 	// text from becoming executable source code.
 	const output = renderModelsFile(providers);
@@ -2294,4 +2340,7 @@ async function generateModels() {
 }
 
 // Run the generator
-generateModels().catch(console.error);
+generateModels().catch((error) => {
+	console.error(error);
+	process.exitCode = 1;
+});

packages/ai/scripts/validate-model-catalog.ts

@@ -0,0 +1,132 @@
+import { getSupportedThinkingLevels } from "../src/models.js";
+import { supportsAdaptiveThinking } from "../src/providers/anthropic.js";
+import { getCompat } from "../src/providers/openai-completions.js";
+import type { Api, Model } from "../src/types.js";
+
+/** The subset of a catalog row the invariants read; MODELS rows satisfy it. */
+export interface CatalogRowLike {
+	id: string;
+	api: string;
+	provider: string;
+	contextWindow: number;
+	maxTokens: number;
+	reasoning: boolean;
+	baseUrl?: string;
+	thinkingLevelMap?: Readonly<Record<string, string | null>>;
+	compat?: Readonly<Record<string, unknown>>;
+}
+
+export type CatalogLike = Readonly<Record<string, Readonly<Record<string, CatalogRowLike>>>>;
+
+/** Copilot serves each model family through exactly one endpoint; unclassified ids fail validation. */
+export function copilotModelApi(modelId: string): Api | undefined {
+	if (modelId.startsWith("claude-")) return "anthropic-messages";
+	// Responses-only on Copilot; /chat/completions rejects these families (upstream pi-mono #906).
+	if (
+		modelId.startsWith("gpt-5") ||
+		modelId.startsWith("gpt-6") ||
+		modelId.startsWith("oswe") ||
+		modelId.startsWith("grok-") ||
+		modelId.startsWith("mai-")
+	) {
+		return "openai-responses";
+	}
+	if (modelId.startsWith("gemini-") || modelId.startsWith("kimi-")) return "openai-completions";
+	return undefined;
+}
+
+// ChatGPT-backend window verified smaller than the API side (Codex CLI models.json, rust-v0.153.4).
+export const CODEX_SMALLER_WINDOW_VERIFIED = new Set(["gpt-6-astra"]);
+
+function familyKey(modelId: string): string {
+	const segments = modelId.split("/");
+	return segments[segments.length - 1].toLowerCase();
+}
+
+// Runtime-selectable levels via the UI's own function; compared within one transport, declared maps only, modulo "off".
+function selectableLevels(model: CatalogRowLike): string {
+	return getSupportedThinkingLevels(model as Model<Api>)
+		.filter((level) => level !== "off")
+		.join(",");
+}
+
+// Plain openai-format completions gate reasoning params on compat; other formats use the map as an enable toggle.
+function effortIsSendable(model: CatalogRowLike): boolean {
+	// Model requires baseUrl, so rows without one behave like the empty-string rows: provider-only detection.
+	const compat = getCompat({ ...model, baseUrl: model.baseUrl ?? "" } as Model<"openai-completions">);
+	return compat.thinkingFormat !== "openai" || compat.supportsReasoningEffort;
+}
+
+/** Generation-time catalog invariants; an empty return means the catalog is valid. */
+export function validateModelCatalog(catalog: CatalogLike): string[] {
+	const violations: string[] = [];
+
+	for (const [provider, models] of Object.entries(catalog)) {
+		for (const model of Object.values(models)) {
+			if (model.maxTokens > model.contextWindow) {
+				violations.push(
+					`${provider}/${model.id}: maxTokens ${model.maxTokens} exceeds contextWindow ${model.contextWindow}`,
+				);
+			}
+		}
+	}
+
+	for (const model of Object.values(catalog["github-copilot"] ?? {})) {
+		const expectedApi = copilotModelApi(model.id);
+		if (expectedApi === undefined) {
+			violations.push(
+				`github-copilot/${model.id}: unclassified model family; add it to copilotModelApi in validate-model-catalog.ts`,
+			);
+		} else if (model.api !== expectedApi) {
+			violations.push(`github-copilot/${model.id}: api ${model.api} does not match classification ${expectedApi}`);
+		}
+	}
+
+	for (const model of Object.values(catalog["openai-codex"] ?? {})) {
+		const openaiTwin = catalog.openai?.[model.id];
+		if (!openaiTwin || CODEX_SMALLER_WINDOW_VERIFIED.has(model.id)) continue;
+		const ratio = openaiTwin.contextWindow / model.contextWindow;
+		if (ratio > 2 || ratio < 0.5) {
+			violations.push(
+				`openai-codex/${model.id}: contextWindow ${model.contextWindow} diverges more than 2x from openai/${model.id} (${openaiTwin.contextWindow})`,
+			);
+		}
+	}
+
+	const familyLevels = new Map<string, Map<string, string>>();
+	for (const [provider, models] of Object.entries(catalog)) {
+		for (const model of Object.values(models)) {
+			if (!model.thinkingLevelMap || !model.reasoning) continue;
+			if (model.api === "anthropic-messages" && !supportsAdaptiveThinking(model.id)) {
+				const clamped = ["xhigh", "max"].filter((level) => model.thinkingLevelMap?.[level] != null);
+				if (clamped.length > 0) {
+					violations.push(
+						`${provider}/${model.id}: thinkingLevelMap offers [${clamped.join(",")}] but the budget path serializes them as high`,
+					);
+				}
+			}
+			if (model.api === "openai-completions" && !effortIsSendable(model)) {
+				const levels = selectableLevels(model);
+				if (levels.length > 0) {
+					violations.push(
+						`${provider}/${model.id}: thinkingLevelMap offers [${levels}] but the transport cannot send reasoning effort`,
+					);
+				}
+				continue;
+			}
+			const key = `${familyKey(model.id)} [${model.api}]`;
+			const seen = familyLevels.get(key) ?? new Map<string, string>();
+			seen.set(`${provider}/${model.id}`, selectableLevels(model));
+			familyLevels.set(key, seen);
+		}
+	}
+	for (const [key, seen] of familyLevels) {
+		const distinct = new Set(seen.values());
+		if (distinct.size > 1) {
+			const detail = [...seen.entries()].map(([row, levels]) => `${row}=[${levels}]`).join(", ");
+			violations.push(`${key}: selectable thinking levels disagree across providers: ${detail}`);
+		}
+	}
+
+	return violations;
+}