Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 7 additions & 23 deletions open-sse/mcp-server/schemas/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@
*/

import { z } from "zod";
import {
AUTO_ROUTING_STRATEGY_VALUES,
ROUTING_STRATEGY_VALUES,
} from "../../../src/shared/constants/routingStrategies.ts";

// ============ Shared Types ============

Expand Down Expand Up @@ -109,17 +113,7 @@ export const listCombosOutput = z.object({
priority: z.number(),
})
),
strategy: z.enum([
"priority",
"weighted",
"round-robin",
"context-relay",
"strict-random",
"random",
"least-used",
"cost-optimized",
"auto",
]),
strategy: z.enum(ROUTING_STRATEGY_VALUES),
enabled: z.boolean(),
metrics: z
.object({
Expand Down Expand Up @@ -545,20 +539,10 @@ export const setBudgetGuardTool: McpToolDefinition<
export const setRoutingStrategyInput = z.object({
comboId: z.string().describe("Combo ID or name to update"),
strategy: z
.enum([
"priority",
"weighted",
"round-robin",
"context-relay",
"strict-random",
"random",
"least-used",
"cost-optimized",
"auto",
])
.enum(ROUTING_STRATEGY_VALUES)
.describe("Routing strategy to apply"),
autoRoutingStrategy: z
.enum(["rules", "cost", "eco", "latency", "fast"])
.enum(AUTO_ROUTING_STRATEGY_VALUES)
.optional()
.describe("Optional strategy used by auto mode (only used when strategy='auto')"),
});
Expand Down
29 changes: 13 additions & 16 deletions open-sse/mcp-server/tools/advancedTools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ import {
getComboModelString,
getComboStepTarget,
} from "../../../src/lib/combos/steps.ts";
import type {
AutoRoutingStrategyValue,
RoutingStrategyValue,
} from "../../../src/shared/constants/routingStrategies.ts";
import { normalizeRoutingStrategy } from "../../../src/shared/constants/routingStrategies.ts";

const OMNIROUTE_BASE_URL = resolveOmniRouteBaseUrl();
const OMNIROUTE_API_KEY = process.env.OMNIROUTE_API_KEY || "";
Expand Down Expand Up @@ -372,17 +377,8 @@ export async function handleSetBudgetGuard(args: {

export async function handleSetRoutingStrategy(args: {
comboId: string;
strategy:
| "priority"
| "weighted"
| "round-robin"
| "context-relay"
| "strict-random"
| "random"
| "least-used"
| "cost-optimized"
| "auto";
autoRoutingStrategy?: "rules" | "cost" | "eco" | "latency" | "fast";
strategy: RoutingStrategyValue;
autoRoutingStrategy?: AutoRoutingStrategyValue;
}) {
const start = Date.now();
try {
Expand Down Expand Up @@ -424,8 +420,9 @@ export async function handleSetRoutingStrategy(args: {
Object.keys(toRecord(combo.config)).length > 0 ? combo.config : comboData.config
);

const normalizedStrategy = normalizeRoutingStrategy(args.strategy);
let nextConfig: JsonRecord | undefined = undefined;
if (args.strategy === "auto" && args.autoRoutingStrategy) {
if (normalizedStrategy === "auto" && args.autoRoutingStrategy) {
const currentAutoConfig = toRecord(currentConfig.auto);
nextConfig = {
...currentConfig,
Expand All @@ -436,7 +433,7 @@ export async function handleSetRoutingStrategy(args: {
};
}

const payload: JsonRecord = { strategy: args.strategy };
const payload: JsonRecord = { strategy: normalizedStrategy };
if (nextConfig && Object.keys(nextConfig).length > 0) {
payload.config = nextConfig;
}
Expand All @@ -451,16 +448,16 @@ export async function handleSetRoutingStrategy(args: {
const updatedConfig = toRecord(updatedCombo.config);
const resolvedAutoStrategy =
toString(toRecord(updatedConfig.auto).routingStrategy) ||
(args.strategy === "auto" ? (args.autoRoutingStrategy ?? "rules") : "");
(normalizedStrategy === "auto" ? (args.autoRoutingStrategy ?? "rules") : "");

const result = {
success: true,
combo: {
id: toString(updatedCombo.id, comboId),
name: toString(updatedCombo.name, toString(combo.name, comboId)),
strategy: toString(updatedCombo.strategy, args.strategy),
strategy: toString(updatedCombo.strategy, normalizedStrategy),
autoRoutingStrategy:
toString(updatedCombo.strategy, args.strategy) === "auto" ? resolvedAutoStrategy : null,
toString(updatedCombo.strategy, normalizedStrategy) === "auto" ? resolvedAutoStrategy : null,
},
};

Expand Down
5 changes: 0 additions & 5 deletions open-sse/services/autoCombo/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,6 @@ export { SelfHealingManager, getSelfHealingManager } from "./selfHealing";
export { MODE_PACKS, getModePack, getModePackNames } from "./modePacks";
export {
selectProvider,
createAutoCombo,
getAutoCombo,
updateAutoCombo,
deleteAutoCombo,
listAutoCombos,
type AutoComboConfig,
type SelectionResult,
} from "./engine";
54 changes: 48 additions & 6 deletions open-sse/services/combo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import {
resolveRequestRoutingTags,
type RoutingTagMatchMode,
} from "../../src/domain/tagRouter.ts";
import { normalizeRoutingStrategy } from "../../src/shared/constants/routingStrategies.ts";

// Status codes that should mark round-robin target semaphores as cooling down.
const TRANSIENT_FOR_SEMAPHORE = [429, 502, 503, 504];
Expand Down Expand Up @@ -630,17 +631,25 @@ function sortTargetsByUsage(targets: ResolvedComboTarget[], comboName: string) {
*/
function sortModelsByContextSize(models) {
const withContext = models.map((modelStr) => {
const parsed = parseModel(modelStr);
const provider = parsed.provider || parsed.providerAlias || "unknown";
const model = parsed.model || modelStr;
const limit = getModelContextLimit(provider, model);
return { modelStr, context: limit ?? 0 };
return { modelStr, context: getModelContextLimitForModelString(modelStr) ?? 0 };
});
withContext.sort((a, b) => b.context - a.context);
return withContext.map((e) => e.modelStr);
}

function getModelContextLimitForModelString(modelStr: string) {
const parsed = parseModel(modelStr);
const provider = parsed.provider || parsed.providerAlias || "unknown";
const model = parsed.model || modelStr;
return getModelContextLimit(provider, model);
}

function sortTargetsByContextSize(targets: ResolvedComboTarget[]) {
const hasKnownContext = targets.some(
(target) => getModelContextLimitForModelString(target.modelStr) != null
);
if (!hasKnownContext) return targets;

const orderedModels = sortModelsByContextSize(targets.map((target) => target.modelStr));
const byModel = new Map<string, ResolvedComboTarget[]>();
for (const target of targets) {
Expand All @@ -656,6 +665,34 @@ function sortTargetsByContextSize(targets: ResolvedComboTarget[]) {
.filter((target): target is ResolvedComboTarget => target !== null);
}

function getP2CTargetScore(target: ResolvedComboTarget, metrics: ReturnType<typeof getComboMetrics>): number {
const breakerState = getCircuitBreaker(target.provider)?.getStatus?.()?.state;
if (breakerState === "OPEN") return -Infinity;
const modelMetric = metrics?.byModel?.[target.modelStr] || null;
const successRate = Number(modelMetric?.successRate);
const avgLatency = Number(modelMetric?.avgLatencyMs);
const successScore = Number.isFinite(successRate) ? successRate / 100 : 0.5;
const latencyScore = Number.isFinite(avgLatency) && avgLatency > 0 ? 1 / Math.log10(avgLatency + 10) : 0.25;
const breakerPenalty = breakerState === "HALF_OPEN" ? 0.25 : 0;
return successScore + latencyScore - breakerPenalty;
}

function orderTargetsByPowerOfTwoChoices(targets: ResolvedComboTarget[], comboName: string) {
if (targets.length <= 1) return targets;
const metrics = getComboMetrics(comboName);
const firstIndex = Math.floor(Math.random() * targets.length);
let secondIndex = Math.floor(Math.random() * (targets.length - 1));
if (secondIndex >= firstIndex) secondIndex++;

const first = targets[firstIndex];
const second = targets[secondIndex];
const selectedIndex =
getP2CTargetScore(second, metrics) > getP2CTargetScore(first, metrics)
? secondIndex
: firstIndex;
return [targets[selectedIndex], ...targets.filter((_, index) => index !== selectedIndex)];
}

function toTextContent(content) {
if (typeof content === "string") return content;
if (!Array.isArray(content)) return "";
Expand Down Expand Up @@ -1034,7 +1071,7 @@ export async function handleComboChat({
relayOptions,
signal,
}) {
const strategy = combo.strategy || "priority";
const strategy = normalizeRoutingStrategy(combo.strategy || "priority");
const relayConfig =
strategy === "context-relay" ? resolveContextRelayConfig(relayOptions?.config || null) : null;

Expand Down Expand Up @@ -1427,6 +1464,11 @@ export async function handleComboChat({
} else if (strategy === "random") {
orderedTargets = fisherYatesShuffle([...orderedTargets]);
log.info("COMBO", `Random shuffle: ${orderedTargets.length} targets`);
} else if (strategy === "fill-first") {
log.info("COMBO", `Fill-first ordering: preserving priority order (${orderedTargets.length} targets)`);
} else if (strategy === "p2c") {
orderedTargets = orderTargetsByPowerOfTwoChoices(orderedTargets, combo.name);
log.info("COMBO", `Power-of-two-choices ordering: selected ${orderedTargets[0]?.modelStr}`);
} else if (strategy === "least-used") {
orderedTargets = sortTargetsByUsage(orderedTargets, combo.name);
log.info("COMBO", `Least-used ordering: ${orderedTargets[0]?.modelStr} has fewest requests`);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
import { useState, useEffect } from "react";
import { Card, Button, Input, Toggle } from "@/shared/components";
import { cn } from "@/shared/utils/cn";
import { ROUTING_STRATEGIES } from "@/shared/constants/routingStrategies";
import {
ROUTING_STRATEGIES,
SETTINGS_FALLBACK_STRATEGY_VALUES,
} from "@/shared/constants/routingStrategies";
import { useTranslations } from "next-intl";

const STRATEGY_LABEL_FALLBACKS: Record<string, string> = {
Expand All @@ -15,6 +18,7 @@ const LEGACY_COMBO_RESILIENCE_KEYS = new Set([
"healthCheckEnabled",
"healthCheckTimeoutMs",
]);
const ACCOUNT_FALLBACK_STRATEGIES = new Set<string>(SETTINGS_FALLBACK_STRATEGY_VALUES);

function translateOrFallback(
t: ReturnType<typeof useTranslations>,
Expand Down Expand Up @@ -44,6 +48,17 @@ function sanitizeProviderOverrides(overrides?: Record<string, any> | null) {
);
}

function toGlobalRoutingPatch(strategy: string | undefined, stickyRoundRobinLimit?: number) {
const patch: Record<string, unknown> = {};
if (strategy && ACCOUNT_FALLBACK_STRATEGIES.has(strategy)) {
patch.fallbackStrategy = strategy;
}
if (strategy === "round-robin" && stickyRoundRobinLimit !== undefined) {
patch.stickyRoundRobinLimit = stickyRoundRobinLimit;
}
return patch;
}

export default function ComboDefaultsTab() {
const [comboDefaults, setComboDefaults] = useState<any>({
strategy: "priority",
Expand Down Expand Up @@ -89,8 +104,7 @@ export default function ComboDefaultsTab() {
setComboDefaults((prev) => ({
...prev,
...sanitizeComboRuntimeConfig(comboData.comboDefaults),
strategy:
settingsData.fallbackStrategy ?? comboData.comboDefaults?.strategy ?? prev.strategy,
strategy: comboData.comboDefaults?.strategy ?? settingsData.fallbackStrategy ?? prev.strategy,
stickyRoundRobinLimit:
settingsData.stickyRoundRobinLimit ??
comboData.comboDefaults?.stickyRoundRobinLimit ??
Expand All @@ -109,8 +123,7 @@ export default function ComboDefaultsTab() {
};

const syncGlobalRoutingSettings = async (patch: Record<string, unknown>) => {
const keys = Object.keys(patch);
if (keys.length === 0) return true;
if (Object.keys(patch).length === 0) return;

const res = await fetch("/api/settings", {
method: "PATCH",
Expand All @@ -121,21 +134,13 @@ export default function ComboDefaultsTab() {
if (!res.ok) {
throw new Error("Failed to sync global routing settings");
}

return true;
};

const saveComboDefaults = async () => {
setSaving(true);
try {
const { stickyRoundRobinLimit, ...comboDefaultsPayload } = comboDefaults;
const settingsPatch: Record<string, unknown> = {};
if (comboDefaults.strategy) {
settingsPatch.fallbackStrategy = comboDefaults.strategy;
}
if (comboDefaults.strategy === "round-robin" && stickyRoundRobinLimit !== undefined) {
settingsPatch.stickyRoundRobinLimit = stickyRoundRobinLimit;
}
const settingsPatch = toGlobalRoutingPatch(comboDefaults.strategy, stickyRoundRobinLimit);

const comboDefaultsRes = await fetch("/api/settings/combo-defaults", {
method: "PATCH",
Expand Down Expand Up @@ -243,7 +248,7 @@ export default function ComboDefaultsTab() {
onClick={async () => {
setComboDefaults((prev) => ({ ...prev, strategy: s.value }));
try {
await syncGlobalRoutingSettings({ fallbackStrategy: s.value });
await syncGlobalRoutingSettings(toGlobalRoutingPatch(s.value));
} catch (error) {
console.error("Failed to sync fallback strategy:", error);
showStatus("error", t("errorOccurred"));
Expand Down
14 changes: 13 additions & 1 deletion src/lib/db/jsonMigration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*/

import type Database from "better-sqlite3";
import { normalizeRoutingStrategy } from "@/shared/constants/routingStrategies";

type SqliteDatabase = InstanceType<typeof Database>;

Expand Down Expand Up @@ -172,8 +173,19 @@ export function runJsonMigration(

// 5. Combos
for (const [index, combo] of (data.combos ?? []).entries()) {
const normalizedCombo = {
const config =
combo.config && typeof combo.config === "object" && !Array.isArray(combo.config)
? { ...(combo.config as Record<string, unknown>) }
: combo.config;
if (config && typeof config === "object" && !Array.isArray(config) && "strategy" in config) {
(config as Record<string, unknown>).strategy = normalizeRoutingStrategy(
(config as Record<string, unknown>).strategy
);
}
const normalizedCombo: Record<string, unknown> = {
...combo,
strategy: normalizeRoutingStrategy(combo.strategy),
config,
sortOrder: typeof combo.sortOrder === "number" ? combo.sortOrder : index + 1,
};
insertCombo.run({
Expand Down
Loading