Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions open-sse/config/providers/registry/anthropic/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ export const anthropicProvider: RegistryEntry = {
id: "claude-fable-5",
name: "Claude Fable 5",
contextLength: 1048576,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{
Expand All @@ -39,24 +42,42 @@ export const anthropicProvider: RegistryEntry = {
contextLength: 1000000,
maxOutputTokens: 128000,
supportsXHighEffort: true,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{
id: "claude-opus-4.7",
name: "Claude Opus 4.7",
// Opus 4.7+ rejects non-default temperature/top_p/top_k with a 400 (sampling fixed;
// reasoning via output_config.effort). Mirrors the dashed `claude` registry ids.
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{
id: "claude-opus-4.8",
name: "Claude Opus 4.8",
contextLength: 1048576,
// Opus 4.7+ (incl. 4.8, Fable 5) reject non-default sampling with a 400. Mirrors claude-opus-4.7.
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{ id: "claude-opus-4.6", name: "Claude Opus 4.6" },
{ id: "claude-opus-4.5", name: "Claude Opus 4.5" },
{
id: "claude-opus-4.6",
name: "Claude Opus 4.6",
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "max"],
},
{
id: "claude-opus-4.5",
name: "Claude Opus 4.5",
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high"],
},
{
id: "claude-sonnet-5",
name: "Claude Sonnet 5",
Expand Down
13 changes: 13 additions & 0 deletions open-sse/config/providers/registry/claude/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ export const claudeProvider: RegistryEntry = {
maxOutputTokens: 128000,
// Opus 4.7+/Fable 5 reject non-default temperature/top_p/top_k with a 400 (sampling
// is fixed; reasoning is steered by output_config.effort). Strip them before dispatch.
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{
Expand All @@ -60,13 +63,19 @@ export const claudeProvider: RegistryEntry = {
name: "Claude Opus 4.8",
contextLength: 1000000,
maxOutputTokens: 128000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{
id: "claude-opus-4-7",
name: "Claude Opus 4.7",
contextLength: 1000000,
maxOutputTokens: 128000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{
Expand All @@ -75,13 +84,17 @@ export const claudeProvider: RegistryEntry = {
supportsXHighEffort: false,
contextLength: 1000000,
maxOutputTokens: 128000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "max"],
},
{
id: "claude-opus-4-5-20251101",
name: "Claude Opus 4.5",
supportsXHighEffort: false,
contextLength: 200000,
maxOutputTokens: 64000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high"],
},
{
id: "claude-sonnet-5",
Expand Down
17 changes: 17 additions & 0 deletions open-sse/config/providers/registry/ghe-copilot/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,33 +37,48 @@ export const gheCopilotProvider: RegistryEntry = {
name: "Claude Fable 5",
contextLength: 1000000,
maxOutputTokens: 64000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
},
{
id: "claude-opus-5",
name: "Claude Opus 5",
contextLength: 1000000,
maxOutputTokens: 64000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{
id: "claude-opus-4.8-fast",
name: "Claude Opus 4.8 (fast mode)",
contextLength: 1000000,
maxOutputTokens: 64000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{
id: "claude-opus-4.8",
name: "Claude Opus 4.8",
contextLength: 1000000,
maxOutputTokens: 64000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{
id: "claude-opus-4.7",
name: "Claude Opus 4.7",
contextLength: 1000000,
maxOutputTokens: 64000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
},
{
id: "claude-sonnet-4.6",
Expand All @@ -76,6 +91,8 @@ export const gheCopilotProvider: RegistryEntry = {
name: "Claude Opus 4.5",
contextLength: 200000,
maxOutputTokens: 32000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high"],
},
{
id: "claude-sonnet-5",
Expand Down
19 changes: 19 additions & 0 deletions open-sse/config/providers/registry/github/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,19 @@ export const githubProvider: RegistryEntry = {
targetFormat: "claude",
contextLength: 1000000,
maxOutputTokens: 64000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
},
{
id: "claude-opus-5",
name: "Claude Opus 5",
targetFormat: "claude",
contextLength: 1000000,
maxOutputTokens: 64000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{
Expand All @@ -57,6 +63,9 @@ export const githubProvider: RegistryEntry = {
targetFormat: "claude",
contextLength: 1000000,
maxOutputTokens: 64000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{
Expand All @@ -65,6 +74,9 @@ export const githubProvider: RegistryEntry = {
targetFormat: "claude",
contextLength: 1000000,
maxOutputTokens: 64000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
unsupportedParams: ["temperature", "top_p", "top_k"],
},
{
Expand All @@ -73,13 +85,18 @@ export const githubProvider: RegistryEntry = {
targetFormat: "claude",
contextLength: 1000000,
maxOutputTokens: 64000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "xhigh", "max"],
supportsXHighEffort: true,
},
{
id: "claude-opus-4.6",
name: "Claude Opus 4.6",
targetFormat: "claude",
contextLength: 1000000,
maxOutputTokens: 64000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high", "max"],
},
{
id: "claude-sonnet-4.6",
Expand All @@ -94,6 +111,8 @@ export const githubProvider: RegistryEntry = {
targetFormat: "claude",
contextLength: 200000,
maxOutputTokens: 32000,
supportsReasoning: true,
supportedThinkingEfforts: ["low", "medium", "high"],
},
{
id: "claude-sonnet-5",
Expand Down
124 changes: 124 additions & 0 deletions tests/unit/claude-firstparty-effort-tiers.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import {
getModelsByProviderId,
supportsXHighEffort,
} from "../../open-sse/config/providerModels.ts";
import { getRegistryModelThinkingEfforts } from "../../open-sse/config/providerRegistry.ts";

// Tier lists follow Anthropic's published per-model effort table:
// xhigh exists on Opus 4.7+ / Opus 5 / Fable 5 (added with Opus 4.7); max exists on
// Opus 5/4.8/4.7/4.6 and Fable 5; Opus 4.5 tops out at high.
const FIVE = ["low", "medium", "high", "xhigh", "max"];
const FOUR = ["low", "medium", "high", "max"]; // no xhigh — Opus 4.6
const THREE = ["low", "medium", "high"]; // Opus 4.5

function entry(providerId: string, modelId: string) {
const model = getModelsByProviderId(providerId).find((e) => e.id === modelId);
assert.ok(model, `${providerId}/${modelId} must stay seeded`);
return model;
}

const EXPECTED: Record<string, Array<[string, string[]]>> = {
claude: [
["claude-fable-5", FIVE],
["claude-opus-4-8", FIVE],
["claude-opus-4-7", FIVE],
["claude-opus-4-6", FOUR],
["claude-opus-4-5-20251101", THREE],
],
anthropic: [
["claude-fable-5", FIVE],
["claude-opus-5", FIVE],
["claude-opus-4.8", FIVE],
["claude-opus-4.7", FIVE],
["claude-opus-4.6", FOUR],
["claude-opus-4.5", THREE],
],
github: [
["claude-fable-5", FIVE],
["claude-opus-5", FIVE],
["claude-opus-4.8-fast", FIVE],
["claude-opus-4.8", FIVE],
["claude-opus-4.7", FIVE],
["claude-opus-4.6", FOUR],
["claude-opus-4.5", THREE],
],
"ghe-copilot": [
["claude-fable-5", FIVE],
["claude-opus-5", FIVE],
["claude-opus-4.8-fast", FIVE],
["claude-opus-4.8", FIVE],
["claude-opus-4.7", FIVE],
["claude-opus-4.5", THREE],
],
};

for (const [providerId, models] of Object.entries(EXPECTED)) {
test(`${providerId} registry declares per-model thinking efforts`, () => {
for (const [modelId, tiers] of models) {
const model = entry(providerId, modelId);
assert.deepEqual(model.supportedThinkingEfforts, tiers, `${providerId}/${modelId} tiers`);
assert.equal(model.supportsReasoning, true, `${providerId}/${modelId} reasoning`);
assert.equal(
model.supportsXHighEffort,
tiers.includes("xhigh") ? true : model.supportsXHighEffort,
`${providerId}/${modelId} xhigh flag consistency`
);
// The lookup the catalog uses must resolve through the registry entry.
assert.deepEqual(getRegistryModelThinkingEfforts(providerId, modelId), tiers);
}
});
}

test("xhigh-capable entries declare supportsXHighEffort; 4.6/4.5 must not", () => {
for (const [providerId, models] of Object.entries(EXPECTED)) {
for (const [modelId, tiers] of models) {
const model = entry(providerId, modelId);
if (tiers.includes("xhigh")) {
assert.equal(model.supportsXHighEffort, true, `${providerId}/${modelId}`);
} else {
assert.notEqual(
model.supportsXHighEffort,
true,
`${providerId}/${modelId} must not advertise xhigh`
);
}
}
}
});

// Behavior lock: supportsXHighEffort() defaults to true for unlisted models, so
// Opus 4.6/4.5 correctness on anthropic/github/ghe-copilot depends on the canonical
// fallback reaching the claude registry's explicit false entries (providerModels.ts:372-387).
// If those explicit false flags are ever dropped, xhigh requests would silently route
// to models that reject them — these assertions pin the end-to-end behavior.
test("supportsXHighEffort() end-to-end: 4.6/4.5 resolve false on every registry", () => {
for (const providerId of Object.keys(EXPECTED)) {
for (const modelId of ["claude-opus-4.6", "claude-opus-4.5"]) {
// ghe-copilot has no 4.6 entry; claude registry uses dash ids.
const id =
providerId === "claude"
? modelId
.replace("claude-opus-4.6", "claude-opus-4-6")
.replace("claude-opus-4.5", "claude-opus-4-5-20251101")
: modelId;
if (!getModelsByProviderId(providerId).some((e) => e.id === id)) continue;
assert.equal(supportsXHighEffort(providerId, id), false, `${providerId}/${id}`);
}
}
});

test("supportsXHighEffort() end-to-end: xhigh-capable models resolve true", () => {
const cases: Array<[string, string]> = [
["claude", "claude-opus-4-8"],
["claude", "claude-opus-4-7"],
["anthropic", "claude-opus-4.8"],
["anthropic", "claude-opus-5"],
["github", "claude-opus-4.8-fast"],
["ghe-copilot", "claude-fable-5"],
];
for (const [providerId, modelId] of cases) {
assert.equal(supportsXHighEffort(providerId, modelId), true, `${providerId}/${modelId}`);
}
});
Loading