Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 35 additions & 2 deletions apps/web/src/app/api/openrouter/[...path]/route.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -629,6 +629,34 @@ describe('kilo-auto/efficient classifier billing', () => {
expect(ctx.posthog_distinct_id).toBeUndefined();
});

it('bills classifier cost for the balanced alias using its requested model id', async () => {
mockedFetchEfficientAutoDecision.mockResolvedValue({
decision: {
model: 'anthropic/claude-haiku-4',
taskType: 'implementation',
subtaskType: 'feature_development',
source: 'benchmark',
tableVersion: 'v1',
sticky: false,
},
costUsd: 0.002,
});

const { POST } = await import('./route');
const response = await POST(makeRequest(makeBody('kilo-auto/balanced')) as never);

expect(response.status).toBe(200);
await Promise.resolve();
await Promise.resolve();

expect(mockedFetchEfficientAutoDecision).toHaveBeenCalledWith(
expect.objectContaining({ requestedModel: 'kilo-auto/balanced' })
);
expect(mockedLogMicrodollarUsage).toHaveBeenCalledTimes(1);
const [, ctx] = mockedLogMicrodollarUsage.mock.calls[0];
expect(ctx.requested_model).toBe('kilo-auto/balanced');
});

it('does not bill when classifier cost is 0 (cache hit)', async () => {
mockedFetchEfficientAutoDecision.mockResolvedValue({
decision: {
Expand Down Expand Up @@ -816,20 +844,25 @@ describe('auto-routing shadow classifier', () => {
});
mockedEmitApiMetricsForResponse.mockReturnValue(undefined);
mockedAccountForMicrodollarUsage.mockReturnValue(undefined);
mockedApplyResolvedAutoModel.mockImplementation(async (_opts, request) => {
mockedApplyResolvedAutoModel.mockImplementation(async (opts, request) => {
if (opts.efficientDecision) await opts.efficientDecision();
request.body.model = 'openai/gpt-4o';
return { kind: 'ok', resolved: { model: 'openai/gpt-4o' } };
});
});

it('does not schedule a background classifier request for non-efficient auto models', async () => {
it('routes kilo-auto/balanced through the efficient classifier', async () => {
const { after: mockedAfter } = jest.requireMock<{ after: jest.Mock }>('next/server');
mockedFetchEfficientAutoDecision.mockResolvedValue({ decision: null, costUsd: 0 });

const { POST } = await import('./route');
const response = await POST(makeRequest(makeBody('kilo-auto/balanced')) as never);

expect(response.status).toBe(200);
expect(mockedUpstreamRequest).toHaveBeenCalledTimes(1);
expect(mockedFetchEfficientAutoDecision).toHaveBeenCalledWith(
expect.objectContaining({ requestedModel: 'kilo-auto/balanced' })
);
expect(mockedAfter).not.toHaveBeenCalled();
});
});
12 changes: 7 additions & 5 deletions apps/web/src/app/api/openrouter/[...path]/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ import { isUnavailableModel } from '@/lib/ai-gateway/unavailable-models';
import { isCloudflareIP } from '@/lib/cloudflare-ip';
import {
isKiloAutoModel,
KILO_AUTO_BALANCED_MODEL,
KILO_AUTO_EFFICIENT_MODEL,
ORG_AUTO_MODEL,
} from '@/lib/ai-gateway/auto-model';
Expand Down Expand Up @@ -295,14 +296,15 @@ export async function POST(request: NextRequest): Promise<NextResponseType<unkno
if (isKiloAutoModel(requestedModelLowerCased)) {
autoModel = requestedModelLowerCased;
const efficientDecision =
requestedModelLowerCased === KILO_AUTO_EFFICIENT_MODEL.id
requestedModelLowerCased === KILO_AUTO_EFFICIENT_MODEL.id ||
requestedModelLowerCased === KILO_AUTO_BALANCED_MODEL.id
? async () => {
const { user, authFailedResponse, organizationId } = await authPromise;
// The classifier is a paid call on Kilo's own credential. Skip it
// for unauthenticated requests: kilo-auto/efficient resolves to a
// for unauthenticated requests: auto-routed models resolve to a
// paid model, so an unauthenticated caller is rejected downstream
// regardless, and a null decision simply falls back to balanced.
// This stops anonymous/abusive traffic from repeatedly spending
// This stops anonymous or abusive traffic from repeatedly spending
// Kilo-funded classification with no user to attribute it to.
if (!user || authFailedResponse) return null;
const { settings, plan } = await balanceAndSettingsPromise;
Expand Down Expand Up @@ -520,7 +522,7 @@ export async function POST(request: NextRequest): Promise<NextResponseType<unkno
fraudHeaders,
organizationId,
provider: 'openrouter',
requested_model: KILO_AUTO_EFFICIENT_MODEL.id,
requested_model: requestedModelLowerCased,
promptInfo: {
system_prompt_prefix: '',
system_prompt_length: 0,
Expand Down Expand Up @@ -554,7 +556,7 @@ export async function POST(request: NextRequest): Promise<NextResponseType<unkno
};
await logMicrodollarUsage(classifierStats, classifierContext);
} catch (error) {
console.error('Failed to bill classifier cost for kilo-auto/efficient', error);
console.error('Failed to bill classifier cost for auto routing', error);
}
})()
);
Expand Down
7 changes: 6 additions & 1 deletion apps/web/src/app/api/openrouter/models/route.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,12 @@ describe('GET /api/openrouter/models', () => {
models: ['poolside/laguna-m.1:free'],
},
},
balancedModel,
{
...balancedModel,
autoRouting: {
models: ['google/gemini-2.5-flash', 'openai/gpt-5.4-mini'],
},
},
geminiModel,
gptMiniModel,
poolsideModel,
Expand Down
16 changes: 16 additions & 0 deletions apps/web/src/lib/ai-gateway/auto-model/resolution.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { resolveAutoModel } from './resolution';
import {
BALANCED_QWEN_MODEL,
FRONTIER_MODE_TO_MODEL,
KILO_AUTO_BALANCED_MODEL,
KILO_AUTO_EFFICIENT_MODEL,
ORG_AUTO_MODEL,
} from '@/lib/ai-gateway/auto-model';
Expand All @@ -38,6 +39,21 @@ const sampleDecision: AutoRoutingDecision = {
};

describe('resolveAutoModel — kilo-auto/efficient branch', () => {
it('resolves kilo-auto/balanced as an alias of kilo-auto/efficient', async () => {
const result = await resolveAutoModel(
{
...baseParams,
model: KILO_AUTO_BALANCED_MODEL.id,
apiKind: 'chat_completions',
efficientDecision: async () => sampleDecision,
},
nullUserPromise,
zeroBalancePromise
);

expect(result).toEqual({ kind: 'ok', resolved: { model: sampleDecision.model } });
});

it('resolves to decision.model when the thunk returns a decision', async () => {
const result = await resolveAutoModel(
{
Expand Down
10 changes: 4 additions & 6 deletions apps/web/src/lib/ai-gateway/auto-model/resolution.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,7 @@ type ResolveAutoModelParams = {
sessionId: string | null;
apiKind: GatewayRequest['kind'] | null;
clientIp: string | null;
// Lazily fetches the auto-routing worker's decision; only set for
// kilo-auto/efficient requests (route.ts owns the request-body capture).
// Lazily fetches the auto-routing worker's decision (route.ts owns the request-body capture).
efficientDecision?: () => Promise<AutoRoutingDecision | null>;
organizationContext?: Promise<{
organizationId?: string;
Expand Down Expand Up @@ -308,7 +307,7 @@ export async function resolveAutoModel(
},
};
}
if (model === KILO_AUTO_EFFICIENT_MODEL.id) {
if (model === KILO_AUTO_EFFICIENT_MODEL.id || model === KILO_AUTO_BALANCED_MODEL.id) {
const decision = params.efficientDecision ? await params.efficientDecision() : null;
if (decision && !isVirtualAutoModelId(decision.model)) {
const resolvedFromDecision = resolveEfficientDecisionModel(decision);
Expand All @@ -319,12 +318,11 @@ export async function resolveAutoModel(
// with implicit defaults — same balanced fallback as the no-decision path.
return { kind: 'ok', resolved: BALANCED_QWEN_MODEL };
}
// Static fallback when the worker is slow/unavailable: same model as
// balanced so an efficient request never degrades below balanced.
// Static fallback when the worker is slow or unavailable.
return { kind: 'ok', resolved: BALANCED_QWEN_MODEL };
}
const mode = resolveMode(modeHeader, featureHeader);
if (model === KILO_AUTO_BALANCED_MODEL.id || model === KILO_AUTO_LEGACY_MODEL) {
if (model === KILO_AUTO_LEGACY_MODEL) {
Comment thread
iscekic marked this conversation as resolved.
Resolved
Comment thread
iscekic marked this conversation as resolved.
Resolved
if (mode === 'claw' && featureHeader === 'kiloclaw') {
const user = await userPromise;
if (user && (await userIsWithinFirstKiloClawInstanceWindow({ userId: user.id }))) {
Expand Down
24 changes: 23 additions & 1 deletion apps/web/src/lib/ai-gateway/auto-routing-models.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,29 @@ describe('addAutoRoutingModels', () => {
...efficientModel,
autoRouting: { models: ['google/gemini-2.5-flash', 'openai/gpt-5.4-mini'] },
});
expect(result.slice(1)).toEqual([balancedModel, geminiModel, gptModel]);
expect(result.slice(1)).toEqual([
{
...balancedModel,
autoRouting: { models: ['google/gemini-2.5-flash', 'openai/gpt-5.4-mini'] },
},
geminiModel,
gptModel,
]);
});

test('annotates balanced as an alias of efficient routing', async () => {
const balancedModel = makeModel('kilo-auto/balanced');
const efficientModel = makeModel('kilo-auto/efficient');
const visibleModel = makeModel('google/gemini-2.5-flash');
mockedGetCachedRoutingTable.mockResolvedValue(routingTable([visibleModel.id]));

const result = await addAutoRoutingModels([balancedModel, efficientModel, visibleModel]);

expect(result).toEqual([
{ ...balancedModel, autoRouting: { models: [visibleModel.id] } },
{ ...efficientModel, autoRouting: { models: [visibleModel.id] } },
visibleModel,
]);
});

test('annotates the free auto model from its candidate source', async () => {
Expand Down
8 changes: 7 additions & 1 deletion apps/web/src/lib/ai-gateway/auto-routing-models.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import type { OpenRouterModelsResponse } from '@/lib/organizations/organization-types';
import { KILO_AUTO_EFFICIENT_MODEL, KILO_AUTO_FREE_MODEL } from '@/lib/ai-gateway/auto-model';
import {
KILO_AUTO_BALANCED_MODEL,
KILO_AUTO_EFFICIENT_MODEL,
KILO_AUTO_FREE_MODEL,
} from '@/lib/ai-gateway/auto-model';
import { getAutoFreeCandidates } from '@/lib/ai-gateway/auto-model/resolution';
import { isVirtualAutoModelId } from '@kilocode/auto-routing-contracts';
import { getCachedRoutingTable } from '@/lib/ai-gateway/auto-routing-table-cache';
Expand All @@ -15,6 +19,7 @@ export async function addAutoRoutingModels(
): Promise<OpenRouterModelsResponse['data']> {
const availableModelIds = new Set(models.map(model => model.id));
if (
!availableModelIds.has(KILO_AUTO_BALANCED_MODEL.id) &&
!availableModelIds.has(KILO_AUTO_EFFICIENT_MODEL.id) &&
!availableModelIds.has(KILO_AUTO_FREE_MODEL.id)
) {
Expand All @@ -34,6 +39,7 @@ export async function addAutoRoutingModels(
);
const freeModelIds = visibleConcreteModelIds(autoFreeCandidates, availableModelIds);
const autoRoutingChoices = new Map([
[KILO_AUTO_BALANCED_MODEL.id, efficientModelIds],
[KILO_AUTO_EFFICIENT_MODEL.id, efficientModelIds],
[KILO_AUTO_FREE_MODEL.id, freeModelIds],
]);
Expand Down
2 changes: 1 addition & 1 deletion apps/web/src/lib/ai-gateway/auto-routing-table-cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ const ROUTING_TABLE_TTL_MS = 5 * 60 * 1000;
* listing (org endpoint and the tRPC settings query), so an uncached admin-worker
* round-trip per request is wasteful. `createCachedFetch` also serves the
* last-known-good table when a refresh throws, so a transient worker outage does
* not blank the Auto Efficient choices shown in the UI.
* not blank the Auto Balanced and Auto Efficient choices shown in the UI.
*/
export const getCachedRoutingTable = createCachedFetch(
async () => {
Expand Down
32 changes: 14 additions & 18 deletions docs/adr/0002-auto-routing-efficient.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,18 @@ Accepted
## Context

`kilo-auto/*` virtual models route a request to a concrete model on the user's
behalf. The existing `balanced` tier picks a single fixed default (Qwen). We want
a tier that routes each request to the *cheapest model proven accurate enough for
that request's difficulty*, where "proven" means measured by our own benchmarks
rather than asserted by hand.
behalf. The `balanced` and `efficient` IDs now use the same benchmark-driven
decision engine. The `balanced` ID remains as a compatibility alias.

This requires three capabilities the codebase did not have: a way to benchmark
candidate models reproducibly, a way to turn benchmark results into a routing
decision per request, and a way to bill the routing overhead honestly. The model
must ship hidden so it can be validated on Kilo team traffic before it competes
with `balanced` for real users.
decision per request, and a way to bill the routing overhead honestly. The
`efficient` ID must remain hidden until team validation completes; `balanced` is
the public compatibility alias for the same routing behavior.

## Decision

Introduce a hidden virtual model `kilo-auto/efficient` backed by a
Use the virtual models `kilo-auto/efficient` and `kilo-auto/balanced` with the same
benchmark-driven decision engine. Ownership is split across three components with
strict, one-directional dependencies:

Expand All @@ -32,10 +30,9 @@ strict, one-directional dependencies:
endpoint classifies the request, derives a difficulty tier, and reads (never
writes) the published artifacts to pick a model. Session stickiness lives in a
Durable Object here.
- **`apps/web` gateway** owns *exposure and billing*. It resolves
`kilo-auto/efficient`, blocks on `/decide`, falls back to balanced Qwen, bills
the classifier cost, and hosts the admin panel (proxied to the benchmark worker
with the internal secret).
- **`apps/web` gateway** owns *exposure and billing*. It resolves both public IDs,
blocks on `/decide`, falls back to balanced Qwen, bills the classifier cost, and
hosts the admin panel (proxied to the benchmark worker with the internal secret).

Shared request-classification code (prompt, parsing, taxonomy, tier derivation,
routing-table schema) lives in `packages/auto-routing-contracts` so the benchmark
Expand Down Expand Up @@ -69,9 +66,8 @@ replays the exact code production runs.
6. **One active run per kind.** A partial unique index plus a server-side check
admit at most one `running` classifier and one `running` decider run; a second
start returns 409, not 500. Stale runs are swept to `failed` on run listing.
7. **The model stays hidden** (excluded from `/models`, usable by id) until team
validation graduates it. Graduation criteria live in the rollout section
below, not in code.
7. **The efficient model stays hidden** (excluded from `/models`, usable by id)
until team validation graduates it. The balanced alias remains public.
8. **Token boundary.** The decider CLI authenticates as a real Kilo user via a 6h
token minted by `apps/web`'s internal endpoint (gated by
`INTERNAL_API_SECRET`). The token only ever lives in a child-process env var —
Expand All @@ -81,7 +77,7 @@ replays the exact code production runs.

The classifier LLM runs on Kilo's OpenRouter credential during model resolution,
so its cost is owed regardless of how the request ends. It is billed as a separate
microdollar usage row (`requested_model: kilo-auto/efficient`, model
microdollar usage row (`requested_model` set to the requested auto ID, model
`auto-routing/classifier`) to the authenticated requesting user, scheduled as soon
as auth resolves so it survives every downstream rejection path (abuse block,
provider/api-kind rejection, balance/org checks, upstream 4xx). It is billed even
Expand Down Expand Up @@ -139,8 +135,8 @@ fallbacks never re-anchor the session's model.

### Rollback

`kilo-auto/efficient` is hidden and additive, so rollback is containment, not
revert:
`kilo-auto/efficient` is hidden and additive, while `kilo-auto/balanced` remains
the public compatibility alias. Rollback is containment, not revert:

- **Disable the model**: stop routing to it. Because it is hidden, no `/models`
consumer depends on it; the gateway already serves balanced on any null
Expand Down
Loading