Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 15 additions & 13 deletions packages/core/src/__tests__/model-metadata.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,15 +80,21 @@ describe('model-metadata vision capability', () => {
assert.equal(resolveModelVisionSupport('openai', granted, 'some-unlisted-model'), true);
});

it('publishes the Kimi K3 Coding Plan limits and sole supported effort', () => {
it('publishes the Kimi Coding Plan K3 limits and effort levels from models.dev', () => {
assert.deepEqual(lookupModelMetadata('kimi-coding-plan', 'k3'), {
displayName: 'Kimi K3',
lifecycle: 'active',
docsUrl: 'https://www.kimi.com/code/docs/en/kimi-code/models.html',
docsUrl: 'https://www.kimi.com/code/docs/en/third-party-tools/other-coding-agents.html',
contextWindow: 1_048_576,
maxOutputTokens: 131_072,
capabilities: { reasoning: true, functionCalling: true, vision: true },
thinkingOptions: { efforts: ['max'] },
capabilities: { vision: true, reasoning: true, functionCalling: true },
thinkingOptions: { efforts: ['low', 'high', 'max'], toggle: true },
modalities: { input: ['text', 'image'], output: ['text'] },
});
// k3-256k joined the snapshot with the same effort set; the wire contract
// test then requires every declared level to actually wire.
assert.deepEqual(lookupModelMetadata('kimi-coding-plan', 'k3-256k').thinkingOptions, {
efforts: ['low', 'high', 'max'],
});
});

Expand All @@ -108,15 +114,11 @@ describe('model-metadata vision capability', () => {
assert.equal(kimi.maxOutputTokens, 32_000);
});

it('reuses MiniMax snapshot facts for the Coding Plan access path', () => {
assert.deepEqual(
lookupModelMetadata('minimax-coding-plan', 'MiniMax-M3'),
lookupModelMetadata('MiniMax', 'MiniMax-M3'),
);
assert.equal(
lookupModelMetadata('minimax-coding-plan', 'MiniMax-M3').capabilities?.vision,
true,
);
it('publishes MiniMax Coding Plan snapshot facts with its own access-path docs', () => {
const metadata = lookupModelMetadata('minimax-coding-plan', 'MiniMax-M3');
assert.equal(metadata.capabilities?.vision, true);
assert.equal(metadata.thinkingOptions?.toggle, true);
assert.equal(metadata.docsUrl, 'https://platform.minimax.io/docs/token-plan/intro');
});

it('uses synchronized facts while preserving access-path overrides', () => {
Expand Down
138 changes: 138 additions & 0 deletions packages/core/src/__tests__/models-dev-sync-contract.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
/**
* models.dev sync contract — every provider type must declare where its
* metadata comes from, and that declaration must be truthful.
*
* This closes the silent-forget gap that kimi-coding-plan and
* stepfun-step-plan fell into:
* - kimi's shape: registry entry exists, upstream provider exists, but
* neither modelsDevId nor a sync-map entry connected them (metadata
* stayed frozen in hand-written overrides).
* - stepfun/minimax's shape: registry entry declared a *neighbour* segment
* id (stepfun / MiniMax) instead of its own, and nothing checked it.
*
* Every registry provider must satisfy exactly one of:
* - have its own snapshot segment (modelsDevId === its segment id), or
* - be a declared access-path alias to another provider's segment
* (MODELS_DEV_SEGMENT_ALIAS below, modelsDevId === the alias target's id),
* or
* - be in the NOT_IN_MODELS_DEV whitelist (upstream has no entry: local
* runtimes, user-configured endpoints, volcengine which models.dev does
* not cover).
*
* A provider in none of the three buckets, or whose modelsDevId does not
* match the bucket's expected id, is the exact bug shape this suite exists
* to catch.
*/

import assert from 'node:assert/strict';
import { describe, it } from 'node:test';
import {
GENERATED_MODELS_DEV_DIRECTORY,
GENERATED_MODELS_DEV_PROVIDER_FACTS,
} from '../model-metadata.generated.js';
import { PROVIDER_REGISTRY, type ProviderType } from '../provider-registry.js';

/**
* Access paths that intentionally reuse another provider's models.dev
* segment. Each entry must stay in sync with the metadata lookup alias
* rules in model-metadata.ts.
*/
const MODELS_DEV_SEGMENT_ALIAS: Readonly<Record<string, ProviderType>> = {
'xai-oauth': 'xai',
'opencode-free': 'opencode',
'claude-subscription': 'anthropic',
'openai-codex': 'openai',
};

/**
* Providers that intentionally have no models.dev segment: local/self-hosted
* runtimes, user-configured endpoints, and volcengine (absent from the
* upstream catalog). A new registry entry must land in one of the other two
* buckets; adding it here requires a reason.
*/
const NOT_IN_MODELS_DEV: Readonly<Record<string, string>> = {
'volcengine-ark': 'models.dev does not cover Volcengine Ark',
'volcengine-coding-plan': 'models.dev does not cover Volcengine Ark',
'volcengine-agent-plan': 'models.dev does not cover Volcengine Ark',
ollama: 'local runtime, not an upstream service',
'lm-studio': 'local runtime, not an upstream service',
localai: 'local runtime, not an upstream service',
'openai-compatible': 'user-configured endpoint, no upstream catalog entry',
'openai-responses-compatible': 'user-configured endpoint, no upstream catalog entry',
'anthropic-compatible': 'user-configured endpoint, no upstream catalog entry',
};

function expectedSegmentId(providerType: ProviderType): string | undefined {
const facts = GENERATED_MODELS_DEV_PROVIDER_FACTS as Partial<
Record<ProviderType, { id: string }>
>;
const own = facts[providerType];
if (own) return own.id;
const alias = MODELS_DEV_SEGMENT_ALIAS[providerType];
if (alias) return facts[alias]!.id;
return undefined;
}

function hostOf(value: string): string | undefined {
try {
return new URL(value).host;
} catch {
return undefined;
}
}

describe('models.dev sync contract', () => {
it('every provider type declares a truthful models.dev segment or is whitelisted', () => {
const gaps: string[] = [];
for (const [providerType, def] of Object.entries(PROVIDER_REGISTRY) as [
ProviderType,
(typeof PROVIDER_REGISTRY)[ProviderType],
][]) {
const expected = expectedSegmentId(providerType);
if (expected !== undefined) {
if (def.modelsDevId !== expected) {
gaps.push(
`${providerType} must declare modelsDevId ${expected} (own segment or alias); got ${def.modelsDevId ?? 'nothing'}`,
);
}
continue;
}
if (!(providerType in NOT_IN_MODELS_DEV)) {
gaps.push(
`${providerType} is neither snapshot-backed nor aliased nor whitelisted — the kimi/stepfun orphan shape`,
);
}
}
assert.deepEqual(gaps, []);
});

it('every snapshot segment is declared by the registry', () => {
const gaps: string[] = [];
for (const providerType of Object.keys(GENERATED_MODELS_DEV_PROVIDER_FACTS)) {
if (!(providerType in PROVIDER_REGISTRY)) {
gaps.push(`snapshot segment ${providerType} is missing from the provider registry`);
}
}
assert.deepEqual(gaps, []);
});

it('a whitelisted provider whose base URL matches a directory provider must declare it', () => {
const gaps: string[] = [];
for (const providerType of Object.keys(NOT_IN_MODELS_DEV)) {
const def = PROVIDER_REGISTRY[providerType as ProviderType];
if (!def?.baseUrl) continue;
const registryHost = hostOf(def.baseUrl);
if (!registryHost) continue;
if (registryHost.startsWith('localhost') || registryHost.startsWith('127.0.0.1')) continue;
const matched = Object.entries(GENERATED_MODELS_DEV_DIRECTORY).find(
([, entry]) => entry.api !== undefined && hostOf(entry.api) === registryHost,
);
if (matched) {
gaps.push(
`${providerType} is whitelisted but base URL host ${registryHost} matches models.dev provider ${matched[0]}`,
);
}
}
assert.deepEqual(gaps, []);
});
});
Loading
Loading