Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 58 additions & 1 deletion open-sse/services/autoCombo/virtualFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import {
type AutoCategory,
type AutoTier,
} from "./suffixComposition";
import { classifyTier } from "../tierResolver";
import type { AutoVariant } from "./autoPrefix";
import { buildFamilyCandidateFilter, type ModelFamily } from "./modelFamily";
import { getHiddenModelsByProvider } from "@/models";
Expand Down Expand Up @@ -560,6 +561,61 @@ export async function prepareVirtualAutoComboInputs(
};
}

/**
* Score candidates at snapshot time using available data (capabilities, tier)
* and the mode-pack's dominant factors. Runtime telemetry (p95 latency, quota
* remaining) is not available during combo creation — this uses static signals only.
*
* Returns a map from modelStr → normalized weight score [0, 1].
*/
export function computeSnapshotWeights(
candidates: readonly VirtualAutoComboCandidate[],
weights: ScoringWeights
): Map<string, number> {
const scores = new Map<string, number>();
for (const c of candidates) {
let score = 0;

// taskFit: reasoning + vision capable models score higher when taskFit is weighted
if (weights.taskFit > 0) {
if (c.resolvedReasoning || c.resolvedSupportsThinking) score += weights.taskFit * 0.6;
if (c.resolvedSupportsVision) score += weights.taskFit * 0.3;
}

// stability: models with richer capabilities are assumed more stable
if (weights.stability > 0) {
const capabilityCount =
Number(c.resolvedReasoning ?? false) +
Number(c.resolvedSupportsThinking ?? false) +
Number(c.resolvedSupportsVision ?? false);
score += weights.stability * Math.min(capabilityCount / 2, 1);
}

// Tier-based scoring (single classifyTier call covers both checks)
let tierInfo: { tier: string } | null = null;
if (weights.tierPriority > 0 || weights.costInv > 0) {
try {
tierInfo = classifyTier(c.provider, c.model);
} catch {
// fall through with zero
}
}
if (tierInfo && weights.tierPriority > 0 && tierInfo.tier === "premium")
score += weights.tierPriority;
if (tierInfo && weights.costInv > 0 && tierInfo.tier === "free") score += weights.costInv;

// latencyInv: all candidates get a base score when latency matters
// (no runtime data at snapshot time, so equal baseline)
if (weights.latencyInv > 0) score += weights.latencyInv * 0.5;

// health + quota: no runtime telemetry at snapshot time → neutral baseline
score += (weights.health + weights.quota) * 0.5;

scores.set(c.modelStr, Math.min(score, 1));
}
return scores;
}

function clonePreparedCandidates(
candidates: readonly VirtualAutoComboCandidate[]
): VirtualAutoComboCandidate[] {
Expand Down Expand Up @@ -728,6 +784,7 @@ export async function createVirtualAutoComboFromPrepared(
}

const providerPool = [...new Set(effectivePool.map((c) => c.provider))];
const snapshotScores = computeSnapshotWeights(effectivePool, weights);

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@diegosouzapw
Need confirmation from product team that this is desired and is a bug that was fixed. This isn't an area I'm familiar with.

const models = effectivePool.map((candidate, index) => ({
id: `virtual-auto-${variant || "default"}-${index + 1}-${candidate.provider}`,
kind: "model" as const,
Expand All @@ -737,7 +794,7 @@ export async function createVirtualAutoComboFromPrepared(
...(candidate.allowedConnectionIds
? { allowedConnectionIds: candidate.allowedConnectionIds }
: {}),
weight: 1,
weight: snapshotScores.get(candidate.modelStr) ?? 1,
label: candidate.provider,
}));
const autoConfig = {
Expand Down
52 changes: 52 additions & 0 deletions scripts/ad-hoc/dump-auto-combos.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/**
* One-shot diagnostic: resolve every built-in auto-combo template and dump the
* resulting candidate pool, weight pack, and config as JSON for inspection.
*
* Run from repo root:
* node --import tsx/esm scripts/ad-hoc/dump-auto-combos.ts > _tasks/research/auto-combos-snapshot.json
*/

const { AUTO_TEMPLATE_VARIANTS, AUTO_SUFFIX_VARIANTS, AUTO_FAMILY_IDS } =
await import("@omniroute/open-sse/services/autoCombo/builtinCatalog");
const { createBuiltinAutoCombo, prepareBuiltinAutoComboInputs } =
await import("@omniroute/open-sse/services/autoCombo/builtinCatalog");

// Prepares the candidate pool once (DB reads: connections, settings, capabilities)
const prepared = await prepareBuiltinAutoComboInputs();

const allTemplates: string[] = [];
allTemplates.push(...Object.keys(AUTO_TEMPLATE_VARIANTS));
allTemplates.push(...AUTO_SUFFIX_VARIANTS);
allTemplates.push(...AUTO_FAMILY_IDS);

const results: Array<{
template: string;
candidateCount: number;
models: string[];
weightPack: Record<string, number>;
explorationRate: number;
}> = [];

for (const name of allTemplates) {
try {
const suffix = name.slice("auto/".length);
const combo = await createBuiltinAutoCombo(name, suffix, prepared as never);
results.push({
template: name,
candidateCount: combo.models.length,
models: combo.models.map((m) => m.model ?? `${m.providerId}/unknown`),
weightPack: combo.weights ?? {},
explorationRate: combo.explorationRate,
});
} catch (err) {
results.push({
template: name,
candidateCount: 0,
models: [],
weightPack: {},
explorationRate: 0,
});
}
}

console.log(JSON.stringify(results, null, 2));
70 changes: 65 additions & 5 deletions src/app/(dashboard)/dashboard/combos/AutoComboCatalog.tsx
Original file line number Diff line number Diff line change
@@ -1,19 +1,66 @@
"use client";

import { useState } from "react";
import { useState, useCallback } from "react";
import { useTranslations } from "next-intl";
import { Card } from "@/shared/components";
import { AUTO_COMBO_TEMPLATES } from "@/domain/assessment/types";
import { AUTO_COMBO_TEMPLATES, type AutoComboTemplate } from "@/domain/assessment/types";

// Informational catalog of zero-config auto-routing combos.
// Auto combos are resolved at request time by the chat handler based on the
// currently connected providers / models — they have no row in the combos
// table, so they were previously invisible in the UI. This panel surfaces
// the static catalog (name, intent, categories, tiers, strategy) so users
// can discover the auto/ prefix without reading source.
export default function AutoComboCatalog() {
// can discover the auto/ prefix without reading source. Duplicate icon lets
// you materialize a snapshot into an editable static combo you can customize.
export default function AutoComboCatalog({
onComboCreated,
}: {
onComboCreated?: (comboId: string) => void;
}) {
const t = useTranslations("combos");
const [open, setOpen] = useState(false);
const [duplicatingName, setDuplicatingName] = useState<string | null>(null);

const handleDuplicateTemplate = useCallback(
async (template: AutoComboTemplate) => {
if (
!confirm(
`${t("duplicateAutoComboConfirm", { name: template.name })}\n\n${t("duplicateAutoComboSnapshotMsg")}`
)
)
return;

setDuplicatingName(template.name);
try {
const res = await fetch("/api/combos/duplicate", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ name: template.name, strategy: template.strategy }),
});

if (!res.ok) {
const data = await res.json().catch(() => ({}));
alert(
`${t("duplicateAutoComboFailedPrefix")} ${data.error || t("duplicateAutoComboUnknownError")}`
);
return;
}

const combo = await res.json();

// Notify parent page to re-fetch combos so the new card renders.
onComboCreated?.(String(combo.id));
} catch (err) {
console.error("Error duplicating auto-combo:", err);
alert(
`${t("duplicateAutoComboFailedPrefix")} ${err instanceof Error ? err.message : t("duplicateAutoComboUnknownError")}`
);
} finally {
setDuplicatingName(null);
}
},
[t, onComboCreated]
);

return (
<Card className="p-4">
Expand Down Expand Up @@ -44,8 +91,21 @@ export default function AutoComboCatalog() {
{AUTO_COMBO_TEMPLATES.map((tpl) => (
<div
key={tpl.name}
className="rounded-lg border border-border bg-bg-subtle p-3 text-xs"
className="rounded-lg border border-border bg-bg-subtle p-3 text-xs relative"
>
<button
onClick={() => handleDuplicateTemplate(tpl)}
disabled={duplicatingName !== null}
className="absolute bottom-1.5 right-1.5 p-0.5 hover:bg-black/5 dark:hover:bg-white/5 rounded text-text-muted hover:text-primary transition-colors"
title={t("duplicateAutoComboTitle", { name: tpl.name })}
>
<span
className={`material-symbols-outlined text-[14px] ${duplicatingName === tpl.name ? "animate-spin" : ""}`}
>
{duplicatingName === tpl.name ? "progress_activity" : "content_copy"}
</span>
</button>

<div className="flex items-center justify-between gap-2">
<code className="font-mono text-sm text-text-main">{tpl.name}</code>
<span className="rounded bg-black/5 px-1.5 py-0.5 text-[10px] uppercase tracking-wide text-text-muted dark:bg-white/5">
Expand Down
Loading
Loading