diff --git a/apps/web/src/app/api/openrouter/[...path]/route.test.ts b/apps/web/src/app/api/openrouter/[...path]/route.test.ts index 79a39482d7..85f4ce85f3 100644 --- a/apps/web/src/app/api/openrouter/[...path]/route.test.ts +++ b/apps/web/src/app/api/openrouter/[...path]/route.test.ts @@ -629,6 +629,34 @@ describe('kilo-auto/efficient classifier billing', () => { expect(ctx.posthog_distinct_id).toBeUndefined(); }); + it('bills classifier cost for the balanced alias using its requested model id', async () => { + mockedFetchEfficientAutoDecision.mockResolvedValue({ + decision: { + model: 'anthropic/claude-haiku-4', + taskType: 'implementation', + subtaskType: 'feature_development', + source: 'benchmark', + tableVersion: 'v1', + sticky: false, + }, + costUsd: 0.002, + }); + + const { POST } = await import('./route'); + const response = await POST(makeRequest(makeBody('kilo-auto/balanced')) as never); + + expect(response.status).toBe(200); + await Promise.resolve(); + await Promise.resolve(); + + expect(mockedFetchEfficientAutoDecision).toHaveBeenCalledWith( + expect.objectContaining({ requestedModel: 'kilo-auto/balanced' }) + ); + expect(mockedLogMicrodollarUsage).toHaveBeenCalledTimes(1); + const [, ctx] = mockedLogMicrodollarUsage.mock.calls[0]; + expect(ctx.requested_model).toBe('kilo-auto/balanced'); + }); + it('does not bill when classifier cost is 0 (cache hit)', async () => { mockedFetchEfficientAutoDecision.mockResolvedValue({ decision: { @@ -816,20 +844,25 @@ describe('auto-routing shadow classifier', () => { }); mockedEmitApiMetricsForResponse.mockReturnValue(undefined); mockedAccountForMicrodollarUsage.mockReturnValue(undefined); - mockedApplyResolvedAutoModel.mockImplementation(async (_opts, request) => { + mockedApplyResolvedAutoModel.mockImplementation(async (opts, request) => { + if (opts.efficientDecision) await opts.efficientDecision(); request.body.model = 'openai/gpt-4o'; return { kind: 'ok', resolved: { model: 'openai/gpt-4o' } }; }); }); - it('does not schedule a background classifier request for non-efficient auto models', async () => { + it('routes kilo-auto/balanced through the efficient classifier', async () => { const { after: mockedAfter } = jest.requireMock<{ after: jest.Mock }>('next/server'); + mockedFetchEfficientAutoDecision.mockResolvedValue({ decision: null, costUsd: 0 }); const { POST } = await import('./route'); const response = await POST(makeRequest(makeBody('kilo-auto/balanced')) as never); expect(response.status).toBe(200); expect(mockedUpstreamRequest).toHaveBeenCalledTimes(1); + expect(mockedFetchEfficientAutoDecision).toHaveBeenCalledWith( + expect.objectContaining({ requestedModel: 'kilo-auto/balanced' }) + ); expect(mockedAfter).not.toHaveBeenCalled(); }); }); diff --git a/apps/web/src/app/api/openrouter/[...path]/route.ts b/apps/web/src/app/api/openrouter/[...path]/route.ts index 9f8d15ada9..0720578369 100644 --- a/apps/web/src/app/api/openrouter/[...path]/route.ts +++ b/apps/web/src/app/api/openrouter/[...path]/route.ts @@ -88,6 +88,7 @@ import { isUnavailableModel } from '@/lib/ai-gateway/unavailable-models'; import { isCloudflareIP } from '@/lib/cloudflare-ip'; import { isKiloAutoModel, + KILO_AUTO_BALANCED_MODEL, KILO_AUTO_EFFICIENT_MODEL, ORG_AUTO_MODEL, } from '@/lib/ai-gateway/auto-model'; @@ -295,14 +296,15 @@ export async function POST(request: NextRequest): Promise { const { user, authFailedResponse, organizationId } = await authPromise; // The classifier is a paid call on Kilo's own credential. Skip it - // for unauthenticated requests: kilo-auto/efficient resolves to a + // for unauthenticated requests: auto-routed models resolve to a // paid model, so an unauthenticated caller is rejected downstream // regardless, and a null decision simply falls back to balanced. - // This stops anonymous/abusive traffic from repeatedly spending + // This stops anonymous or abusive traffic from repeatedly spending // Kilo-funded classification with no user to attribute it to. if (!user || authFailedResponse) return null; const { settings, plan } = await balanceAndSettingsPromise; @@ -520,7 +522,7 @@ export async function POST(request: NextRequest): Promise { models: ['poolside/laguna-m.1:free'], }, }, - balancedModel, + { + ...balancedModel, + autoRouting: { + models: ['google/gemini-2.5-flash', 'openai/gpt-5.4-mini'], + }, + }, geminiModel, gptMiniModel, poolsideModel, diff --git a/apps/web/src/app/api/public/leaderboard-model-provider-usage/route.ts b/apps/web/src/app/api/public/leaderboard-model-provider-usage/route.ts index fae96aa2f1..bcc7248e18 100644 --- a/apps/web/src/app/api/public/leaderboard-model-provider-usage/route.ts +++ b/apps/web/src/app/api/public/leaderboard-model-provider-usage/route.ts @@ -27,7 +27,6 @@ where and mu.provider not in ('custom', 'direct-byok') and mu.total_output_tokens > 0 and mu.is_user_byok = false - and mu.requested_model not ilike '%clawsetup%' and mu.requested_model not ilike '%mercury-edit%' group by 1, 2 order by 4 desc; diff --git a/apps/web/src/lib/ai-gateway/auto-model/index.ts b/apps/web/src/lib/ai-gateway/auto-model/index.ts index a34c852eb1..f222f404fd 100644 --- a/apps/web/src/lib/ai-gateway/auto-model/index.ts +++ b/apps/web/src/lib/ai-gateway/auto-model/index.ts @@ -1,7 +1,6 @@ import { z } from 'zod'; import { CLAUDE_OPUS_CURRENT_MODEL_ID, - claude_sonnet_clawsetup_model, CLAUDE_SONNET_CURRENT_MODEL_ID, } from '@/lib/ai-gateway/providers/anthropic.constants'; import type { OpenRouterReasoningConfig } from '@/lib/ai-gateway/providers/openrouter/types'; @@ -34,8 +33,6 @@ export type ResolvedAutoModel = { verbosity?: Verbosity; }; -export const KILO_AUTO_LEGACY_MODEL = 'kilo/auto'; // hardcoded in upstream OpenClaw - export const modeSchema = z.enum([ 'claw', 'plan', @@ -81,12 +78,6 @@ export const FRONTIER_MODE_TO_MODEL: Record = { code: SONNET_FRONTIER, }; -export const BALANCED_CLAW_SETUP_MODEL: ResolvedAutoModel = { - model: claude_sonnet_clawsetup_model.public_id, - reasoning: { enabled: true, effort: 'high' }, - verbosity: 'high', -}; - // INVARIANT: the efficient static fallback must remain image-capable. // The capability-aware routing filter relies on this guarantee to make // image requests succeed even when no benchmark candidate is capable. @@ -195,9 +186,5 @@ export const AUTO_MODELS = [ ]; export function isKiloAutoModel(model: string) { - return ( - AUTO_MODELS.some(m => m.id === model) || - model === ORG_AUTO_MODEL.id || - model === KILO_AUTO_LEGACY_MODEL - ); + return AUTO_MODELS.some(m => m.id === model) || model === ORG_AUTO_MODEL.id; } diff --git a/apps/web/src/lib/ai-gateway/auto-model/resolution.test.ts b/apps/web/src/lib/ai-gateway/auto-model/resolution.test.ts index 96062319e8..de95a89a66 100644 --- a/apps/web/src/lib/ai-gateway/auto-model/resolution.test.ts +++ b/apps/web/src/lib/ai-gateway/auto-model/resolution.test.ts @@ -4,14 +4,11 @@ jest.mock('@/lib/ai-gateway/providers/gateway-models-cache', () => ({ getOpenRouterModelsFromRedis: jest.fn(async () => new Set()), })); -jest.mock('@/lib/kiloclaw/setup-promo', () => ({ - userIsWithinFirstKiloClawInstanceWindow: jest.fn(async () => false), -})); - import { resolveAutoModel } from './resolution'; import { BALANCED_QWEN_MODEL, FRONTIER_MODE_TO_MODEL, + KILO_AUTO_BALANCED_MODEL, KILO_AUTO_EFFICIENT_MODEL, ORG_AUTO_MODEL, } from '@/lib/ai-gateway/auto-model'; @@ -38,6 +35,21 @@ const sampleDecision: AutoRoutingDecision = { }; describe('resolveAutoModel — kilo-auto/efficient branch', () => { + it('resolves kilo-auto/balanced as an alias of kilo-auto/efficient', async () => { + const result = await resolveAutoModel( + { + ...baseParams, + model: KILO_AUTO_BALANCED_MODEL.id, + apiKind: 'chat_completions', + efficientDecision: async () => sampleDecision, + }, + nullUserPromise, + zeroBalancePromise + ); + + expect(result).toEqual({ kind: 'ok', resolved: { model: sampleDecision.model } }); + }); + it('resolves to decision.model when the thunk returns a decision', async () => { const result = await resolveAutoModel( { diff --git a/apps/web/src/lib/ai-gateway/auto-model/resolution.ts b/apps/web/src/lib/ai-gateway/auto-model/resolution.ts index aec50da220..cae60d5981 100644 --- a/apps/web/src/lib/ai-gateway/auto-model/resolution.ts +++ b/apps/web/src/lib/ai-gateway/auto-model/resolution.ts @@ -20,15 +20,12 @@ import { KILO_AUTO_BALANCED_MODEL, KILO_AUTO_EFFICIENT_MODEL, modeSchema, - BALANCED_CLAW_SETUP_MODEL, BALANCED_QWEN_MODEL, FRONTIER_MODE_TO_MODEL, FRONTIER_CODE_MODEL, type ResolvedAutoModel, - KILO_AUTO_LEGACY_MODEL, ORG_AUTO_MODEL, } from '@/lib/ai-gateway/auto-model'; -import { userIsWithinFirstKiloClawInstanceWindow } from '@/lib/kiloclaw/setup-promo'; import { autoFreeModels, findKiloExclusiveModel, @@ -54,8 +51,7 @@ type ResolveAutoModelParams = { sessionId: string | null; apiKind: GatewayRequest['kind'] | null; clientIp: string | null; - // Lazily fetches the auto-routing worker's decision; only set for - // kilo-auto/efficient requests (route.ts owns the request-body capture). + // Lazily fetches the auto-routing worker's decision (route.ts owns the request-body capture). efficientDecision?: () => Promise; organizationContext?: Promise<{ organizationId?: string; @@ -308,7 +304,7 @@ export async function resolveAutoModel( }, }; } - if (model === KILO_AUTO_EFFICIENT_MODEL.id) { + if (model === KILO_AUTO_EFFICIENT_MODEL.id || model === KILO_AUTO_BALANCED_MODEL.id) { const decision = params.efficientDecision ? await params.efficientDecision() : null; if (decision && !isVirtualAutoModelId(decision.model)) { const resolvedFromDecision = resolveEfficientDecisionModel(decision); @@ -319,21 +315,10 @@ export async function resolveAutoModel( // with implicit defaults — same balanced fallback as the no-decision path. return { kind: 'ok', resolved: BALANCED_QWEN_MODEL }; } - // Static fallback when the worker is slow/unavailable: same model as - // balanced so an efficient request never degrades below balanced. + // Static fallback when the worker is slow or unavailable. return { kind: 'ok', resolved: BALANCED_QWEN_MODEL }; } const mode = resolveMode(modeHeader, featureHeader); - if (model === KILO_AUTO_BALANCED_MODEL.id || model === KILO_AUTO_LEGACY_MODEL) { - if (mode === 'claw' && featureHeader === 'kiloclaw') { - const user = await userPromise; - if (user && (await userIsWithinFirstKiloClawInstanceWindow({ userId: user.id }))) { - return { kind: 'ok', resolved: BALANCED_CLAW_SETUP_MODEL }; - } - } - - return { kind: 'ok', resolved: BALANCED_QWEN_MODEL }; - } return { kind: 'ok', resolved: (mode !== null ? FRONTIER_MODE_TO_MODEL[mode] : null) ?? FRONTIER_CODE_MODEL, diff --git a/apps/web/src/lib/ai-gateway/auto-routing-models.test.ts b/apps/web/src/lib/ai-gateway/auto-routing-models.test.ts index 18ad9a5d39..4f184e08ac 100644 --- a/apps/web/src/lib/ai-gateway/auto-routing-models.test.ts +++ b/apps/web/src/lib/ai-gateway/auto-routing-models.test.ts @@ -90,7 +90,29 @@ describe('addAutoRoutingModels', () => { ...efficientModel, autoRouting: { models: ['google/gemini-2.5-flash', 'openai/gpt-5.4-mini'] }, }); - expect(result.slice(1)).toEqual([balancedModel, geminiModel, gptModel]); + expect(result.slice(1)).toEqual([ + { + ...balancedModel, + autoRouting: { models: ['google/gemini-2.5-flash', 'openai/gpt-5.4-mini'] }, + }, + geminiModel, + gptModel, + ]); + }); + + test('annotates balanced as an alias of efficient routing', async () => { + const balancedModel = makeModel('kilo-auto/balanced'); + const efficientModel = makeModel('kilo-auto/efficient'); + const visibleModel = makeModel('google/gemini-2.5-flash'); + mockedGetCachedRoutingTable.mockResolvedValue(routingTable([visibleModel.id])); + + const result = await addAutoRoutingModels([balancedModel, efficientModel, visibleModel]); + + expect(result).toEqual([ + { ...balancedModel, autoRouting: { models: [visibleModel.id] } }, + { ...efficientModel, autoRouting: { models: [visibleModel.id] } }, + visibleModel, + ]); }); test('annotates the free auto model from its candidate source', async () => { diff --git a/apps/web/src/lib/ai-gateway/auto-routing-models.ts b/apps/web/src/lib/ai-gateway/auto-routing-models.ts index ffd2f37445..4ac3cecfce 100644 --- a/apps/web/src/lib/ai-gateway/auto-routing-models.ts +++ b/apps/web/src/lib/ai-gateway/auto-routing-models.ts @@ -1,5 +1,9 @@ import type { OpenRouterModelsResponse } from '@/lib/organizations/organization-types'; -import { KILO_AUTO_EFFICIENT_MODEL, KILO_AUTO_FREE_MODEL } from '@/lib/ai-gateway/auto-model'; +import { + KILO_AUTO_BALANCED_MODEL, + KILO_AUTO_EFFICIENT_MODEL, + KILO_AUTO_FREE_MODEL, +} from '@/lib/ai-gateway/auto-model'; import { getAutoFreeCandidates } from '@/lib/ai-gateway/auto-model/resolution'; import { isVirtualAutoModelId } from '@kilocode/auto-routing-contracts'; import { getCachedRoutingTable } from '@/lib/ai-gateway/auto-routing-table-cache'; @@ -15,6 +19,7 @@ export async function addAutoRoutingModels( ): Promise { const availableModelIds = new Set(models.map(model => model.id)); if ( + !availableModelIds.has(KILO_AUTO_BALANCED_MODEL.id) && !availableModelIds.has(KILO_AUTO_EFFICIENT_MODEL.id) && !availableModelIds.has(KILO_AUTO_FREE_MODEL.id) ) { @@ -34,6 +39,7 @@ export async function addAutoRoutingModels( ); const freeModelIds = visibleConcreteModelIds(autoFreeCandidates, availableModelIds); const autoRoutingChoices = new Map([ + [KILO_AUTO_BALANCED_MODEL.id, efficientModelIds], [KILO_AUTO_EFFICIENT_MODEL.id, efficientModelIds], [KILO_AUTO_FREE_MODEL.id, freeModelIds], ]); diff --git a/apps/web/src/lib/ai-gateway/auto-routing-table-cache.ts b/apps/web/src/lib/ai-gateway/auto-routing-table-cache.ts index 119f85974c..9fed279b73 100644 --- a/apps/web/src/lib/ai-gateway/auto-routing-table-cache.ts +++ b/apps/web/src/lib/ai-gateway/auto-routing-table-cache.ts @@ -8,7 +8,7 @@ const ROUTING_TABLE_TTL_MS = 5 * 60 * 1000; * listing (org endpoint and the tRPC settings query), so an uncached admin-worker * round-trip per request is wasteful. `createCachedFetch` also serves the * last-known-good table when a refresh throws, so a transient worker outage does - * not blank the Auto Efficient choices shown in the UI. + * not blank the Auto Balanced and Auto Efficient choices shown in the UI. */ export const getCachedRoutingTable = createCachedFetch( async () => { diff --git a/apps/web/src/lib/ai-gateway/models.ts b/apps/web/src/lib/ai-gateway/models.ts index ddc832a263..f1eb2fc451 100644 --- a/apps/web/src/lib/ai-gateway/models.ts +++ b/apps/web/src/lib/ai-gateway/models.ts @@ -13,7 +13,6 @@ import { claude_opus_4_7_stealth_model, claude_sonnet_4_6_stealth_model, claude_opus_4_6_stealth_model, - claude_sonnet_clawsetup_model, CLAUDE_SONNET_CURRENT_MODEL_ID, CLAUDE_OPUS_CURRENT_MODEL_ID, } from '@/lib/ai-gateway/providers/anthropic.constants'; @@ -115,7 +114,6 @@ export const kiloExclusiveModels = [ ...deepseekDiscountedModels, qwen36_plus_stealth_model, gpt_5_6_sol_stealth_model, - claude_sonnet_clawsetup_model, claude_opus_4_8_stealth_model, claude_opus_4_7_stealth_model, claude_sonnet_4_6_stealth_model, diff --git a/apps/web/src/lib/ai-gateway/providers/anthropic.constants.ts b/apps/web/src/lib/ai-gateway/providers/anthropic.constants.ts index 90fe96234b..f74bf3f5d5 100644 --- a/apps/web/src/lib/ai-gateway/providers/anthropic.constants.ts +++ b/apps/web/src/lib/ai-gateway/providers/anthropic.constants.ts @@ -102,20 +102,6 @@ export const claude_opus_4_6_stealth_model: KiloExclusiveModel = { inference_provider_restriction: [], }; -export const claude_sonnet_clawsetup_model: KiloExclusiveModel = { - public_id: CLAUDE_SONNET_CURRENT_MODEL_ID + ':clawsetup', - internal_id: CLAUDE_SONNET_CURRENT_MODEL_ID, - display_name: 'Claude Sonnet KiloClaw Setup Promo', - description: 'Claude Sonnet KiloClaw Setup Promo', - status: 'hidden', // only usable through kilo-auto - context_length: 1_000_000, - max_completion_tokens: 128_000, - gateway: 'openrouter', - flags: ['reasoning', 'vision', 'vercel-routing'], - pricing: null, - inference_provider_restriction: [], -}; - export function isClaudeModel(requestedModel: string) { return requestedModel.includes('claude'); } diff --git a/apps/web/src/lib/ai-gateway/providers/vercel/mapModelIdToVercel.test.ts b/apps/web/src/lib/ai-gateway/providers/vercel/mapModelIdToVercel.test.ts index b52c4b8f96..22eb48dff7 100644 --- a/apps/web/src/lib/ai-gateway/providers/vercel/mapModelIdToVercel.test.ts +++ b/apps/web/src/lib/ai-gateway/providers/vercel/mapModelIdToVercel.test.ts @@ -113,12 +113,10 @@ describe('mapModelIdToVercel', () => { }); it('does not use internal_id for exclusives that are not vercel-routed', () => { - // claude_sonnet_clawsetup_model has gateway 'openrouter' and no + // claude_sonnet_4_6_stealth_model has gateway 'martian' and no // 'vercel-routing' flag, so the mapping must pass the public id through // the generic prefix rewrite instead of substituting internal_id. - expect(mapModelIdToVercel('anthropic/claude-sonnet-4.6:clawsetup')).toBe( - 'anthropic/claude-sonnet-4.6:clawsetup' - ); + expect(mapModelIdToVercel('stealth/claude-sonnet-4.6')).toBe('stealth/claude-sonnet-4.6'); }); it('does not use internal_id for disabled exclusives even when vercel-routed', () => { diff --git a/apps/web/src/lib/ai-gateway/unavailable-models.ts b/apps/web/src/lib/ai-gateway/unavailable-models.ts index a502522fe8..f120b7ec8f 100644 --- a/apps/web/src/lib/ai-gateway/unavailable-models.ts +++ b/apps/web/src/lib/ai-gateway/unavailable-models.ts @@ -1,4 +1,3 @@ -import { claude_sonnet_clawsetup_model } from '@/lib/ai-gateway/providers/anthropic.constants'; import { normalizeModelId } from '@/lib/ai-gateway/model-utils'; const unavailableModelIds: ReadonlySet = new Set([ @@ -55,7 +54,6 @@ const unavailableModelIds: ReadonlySet = new Set([ 'z-ai/glm-4.7:free', 'stepfun/step-3.5-flash:free', 'z-ai/glm-5:free', - claude_sonnet_clawsetup_model.public_id, // only usable through kilo-auto ]); export function isUnavailableModel(modelId: string): boolean { diff --git a/apps/web/src/lib/kiloclaw/setup-promo.test.ts b/apps/web/src/lib/kiloclaw/setup-promo.test.ts deleted file mode 100644 index 68f6c22661..0000000000 --- a/apps/web/src/lib/kiloclaw/setup-promo.test.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { beforeEach, describe, expect, it } from '@jest/globals'; -import { kiloclaw_instances } from '@kilocode/db/schema'; - -import { cleanupDbForTest, db } from '@/lib/drizzle'; -import { insertTestUser } from '@/tests/helpers/user.helper'; - -import { userIsWithinFirstKiloClawInstanceWindow } from './setup-promo'; - -const HOUR_MS = 60 * 60 * 1000; - -function hoursAgo(hours: number): string { - return new Date(Date.now() - hours * HOUR_MS).toISOString(); -} - -async function insertInstance(params: { - userId: string; - createdAt: string; - destroyedAt?: string | null; - sandboxSuffix?: string; -}) { - const id = crypto.randomUUID(); - await db.insert(kiloclaw_instances).values({ - id, - user_id: params.userId, - sandbox_id: `sb_${id.replaceAll('-', '')}${params.sandboxSuffix ?? ''}`, - created_at: params.createdAt, - destroyed_at: params.destroyedAt ?? null, - }); - return id; -} - -describe('userIsWithinFirstKiloClawInstanceWindow', () => { - beforeEach(async () => { - await cleanupDbForTest(); - }); - - it('returns false when the user has no instances', async () => { - const user = await insertTestUser(); - - const eligible = await userIsWithinFirstKiloClawInstanceWindow({ userId: user.id }); - - expect(eligible).toBe(false); - }); - - it('returns true for a single active instance created inside the window', async () => { - const user = await insertTestUser(); - await insertInstance({ userId: user.id, createdAt: hoursAgo(0.5) }); - - const eligible = await userIsWithinFirstKiloClawInstanceWindow({ userId: user.id }); - - expect(eligible).toBe(true); - }); - - it('returns false for a single active instance created outside the window', async () => { - const user = await insertTestUser(); - await insertInstance({ userId: user.id, createdAt: hoursAgo(3) }); - - const eligible = await userIsWithinFirstKiloClawInstanceWindow({ userId: user.id }); - - expect(eligible).toBe(false); - }); - - it('returns false when oldest instance is outside the window even if newest is fresh', async () => { - // Core "first instance" semantic: a returning user creating a new instance - // today is NOT eligible because their FIRST instance is old. - const user = await insertTestUser(); - await insertInstance({ - userId: user.id, - createdAt: hoursAgo(48), - sandboxSuffix: '_old', - }); - await insertInstance({ - userId: user.id, - createdAt: hoursAgo(0.25), - sandboxSuffix: '_new', - }); - - const eligible = await userIsWithinFirstKiloClawInstanceWindow({ userId: user.id }); - - expect(eligible).toBe(false); - }); - - it('returns true when all instances are inside the window', async () => { - const user = await insertTestUser(); - await insertInstance({ - userId: user.id, - createdAt: hoursAgo(1.5), - sandboxSuffix: '_a', - }); - await insertInstance({ - userId: user.id, - createdAt: hoursAgo(0.1), - sandboxSuffix: '_b', - }); - - const eligible = await userIsWithinFirstKiloClawInstanceWindow({ userId: user.id }); - - expect(eligible).toBe(true); - }); - - it('counts destroyed instances when computing the first-instance timestamp', async () => { - // Destroyed instances must still count: otherwise a user could destroy their - // first instance and re-qualify for the setup-promo window indefinitely. - const user = await insertTestUser(); - await insertInstance({ - userId: user.id, - createdAt: hoursAgo(48), - destroyedAt: hoursAgo(40), - sandboxSuffix: '_destroyed_old', - }); - await insertInstance({ - userId: user.id, - createdAt: hoursAgo(0.25), - sandboxSuffix: '_active_new', - }); - - const eligible = await userIsWithinFirstKiloClawInstanceWindow({ userId: user.id }); - - expect(eligible).toBe(false); - }); - - it('returns true when only a single destroyed instance exists and it is inside the window', async () => { - const user = await insertTestUser(); - await insertInstance({ - userId: user.id, - createdAt: hoursAgo(0.5), - destroyedAt: hoursAgo(0.1), - }); - - const eligible = await userIsWithinFirstKiloClawInstanceWindow({ userId: user.id }); - - expect(eligible).toBe(true); - }); - - it('honors a custom maxAgeHours', async () => { - const user = await insertTestUser(); - await insertInstance({ userId: user.id, createdAt: hoursAgo(5) }); - - expect(await userIsWithinFirstKiloClawInstanceWindow({ userId: user.id, maxAgeHours: 2 })).toBe( - false - ); - expect(await userIsWithinFirstKiloClawInstanceWindow({ userId: user.id, maxAgeHours: 8 })).toBe( - true - ); - }); - - it('does not see other users instances', async () => { - const target = await insertTestUser(); - const other = await insertTestUser(); - await insertInstance({ userId: other.id, createdAt: hoursAgo(0.1) }); - - const eligible = await userIsWithinFirstKiloClawInstanceWindow({ userId: target.id }); - - expect(eligible).toBe(false); - }); - - it('does not let other users instances rescue an outside-window user', async () => { - const target = await insertTestUser(); - const other = await insertTestUser(); - await insertInstance({ - userId: target.id, - createdAt: hoursAgo(48), - sandboxSuffix: '_target_old', - }); - await insertInstance({ - userId: other.id, - createdAt: hoursAgo(0.1), - sandboxSuffix: '_other_new', - }); - - const eligible = await userIsWithinFirstKiloClawInstanceWindow({ userId: target.id }); - - expect(eligible).toBe(false); - }); -}); diff --git a/apps/web/src/lib/kiloclaw/setup-promo.ts b/apps/web/src/lib/kiloclaw/setup-promo.ts deleted file mode 100644 index f0f55c91d5..0000000000 --- a/apps/web/src/lib/kiloclaw/setup-promo.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { readDb } from '@/lib/drizzle'; -import { kiloclaw_instances } from '@kilocode/db'; -import { eq } from 'drizzle-orm'; - -export async function userIsWithinFirstKiloClawInstanceWindow(params: { - userId: string; - maxAgeHours?: number; -}): Promise { - const maxAgeHours = params.maxAgeHours ?? 2; - // Fetch the user's earliest instance (including destroyed ones — see test - // "counts destroyed instances when computing the first-instance timestamp" - // for rationale) and check the window in JS. Backed by - // IDX_kiloclaw_instances_user_id_created_at so this is a single-row index - // scan rather than a full aggregate over the user's history. - const [row] = await readDb - .select({ created_at: kiloclaw_instances.created_at }) - .from(kiloclaw_instances) - .where(eq(kiloclaw_instances.user_id, params.userId)) - .orderBy(kiloclaw_instances.created_at) - .limit(1); - if (row == null) return false; - const firstInstanceMs = new Date(row.created_at).getTime(); - return firstInstanceMs >= Date.now() - maxAgeHours * 60 * 60 * 1000; -} diff --git a/docs/adr/0002-auto-routing-efficient.md b/docs/adr/0002-auto-routing-efficient.md index 8060347c43..c0c9913129 100644 --- a/docs/adr/0002-auto-routing-efficient.md +++ b/docs/adr/0002-auto-routing-efficient.md @@ -7,20 +7,24 @@ Accepted ## Context `kilo-auto/*` virtual models route a request to a concrete model on the user's -behalf. The existing `balanced` tier picks a single fixed default (Qwen). We want -a tier that routes each request to the *cheapest model proven accurate enough for -that request's difficulty*, where "proven" means measured by our own benchmarks -rather than asserted by hand. +behalf. The `balanced` and `efficient` IDs now use the same benchmark-driven +decision engine. The `balanced` ID remains as a compatibility alias. + +This change retires two legacy behaviors. The `kilo/auto` compatibility ID is +no longer treated as an auto model (upstream OpenClaw stopped hardcoding it), +and the KiloClaw first-instance setup promo (the free `:clawsetup` Sonnet +variant served from the legacy balanced branch) is removed; balanced requests +resolve through the benchmark classifier like efficient ones. This requires three capabilities the codebase did not have: a way to benchmark candidate models reproducibly, a way to turn benchmark results into a routing -decision per request, and a way to bill the routing overhead honestly. The model -must ship hidden so it can be validated on Kilo team traffic before it competes -with `balanced` for real users. +decision per request, and a way to bill the routing overhead honestly. The +`efficient` ID must remain hidden until team validation completes; `balanced` is +the public compatibility alias for the same routing behavior. ## Decision -Introduce a hidden virtual model `kilo-auto/efficient` backed by a +Use the virtual models `kilo-auto/efficient` and `kilo-auto/balanced` with the same benchmark-driven decision engine. Ownership is split across three components with strict, one-directional dependencies: @@ -32,10 +36,9 @@ strict, one-directional dependencies: endpoint classifies the request, derives a difficulty tier, and reads (never writes) the published artifacts to pick a model. Session stickiness lives in a Durable Object here. -- **`apps/web` gateway** owns *exposure and billing*. It resolves - `kilo-auto/efficient`, blocks on `/decide`, falls back to balanced Qwen, bills - the classifier cost, and hosts the admin panel (proxied to the benchmark worker - with the internal secret). +- **`apps/web` gateway** owns *exposure and billing*. It resolves both public IDs, + blocks on `/decide`, falls back to balanced Qwen, bills the classifier cost, and + hosts the admin panel (proxied to the benchmark worker with the internal secret). Shared request-classification code (prompt, parsing, taxonomy, tier derivation, routing-table schema) lives in `packages/auto-routing-contracts` so the benchmark @@ -69,9 +72,8 @@ replays the exact code production runs. 6. **One active run per kind.** A partial unique index plus a server-side check admit at most one `running` classifier and one `running` decider run; a second start returns 409, not 500. Stale runs are swept to `failed` on run listing. -7. **The model stays hidden** (excluded from `/models`, usable by id) until team - validation graduates it. Graduation criteria live in the rollout section - below, not in code. +7. **The efficient model stays hidden** (excluded from `/models`, usable by id) + until team validation graduates it. The balanced alias remains public. 8. **Token boundary.** The decider CLI authenticates as a real Kilo user via a 6h token minted by `apps/web`'s internal endpoint (gated by `INTERNAL_API_SECRET`). The token only ever lives in a child-process env var — @@ -81,7 +83,7 @@ replays the exact code production runs. The classifier LLM runs on Kilo's OpenRouter credential during model resolution, so its cost is owed regardless of how the request ends. It is billed as a separate -microdollar usage row (`requested_model: kilo-auto/efficient`, model +microdollar usage row (`requested_model` set to the requested auto ID, model `auto-routing/classifier`) to the authenticated requesting user, scheduled as soon as auth resolves so it survives every downstream rejection path (abuse block, provider/api-kind rejection, balance/org checks, upstream 4xx). It is billed even @@ -139,8 +141,8 @@ fallbacks never re-anchor the session's model. ### Rollback -`kilo-auto/efficient` is hidden and additive, so rollback is containment, not -revert: +`kilo-auto/efficient` is hidden and additive, while `kilo-auto/balanced` remains +the public compatibility alias. Rollback is containment, not revert: - **Disable the model**: stop routing to it. Because it is hidden, no `/models` consumer depends on it; the gateway already serves balanced on any null diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index e963744702..691adc848c 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -8289,11 +8289,10 @@ export const kiloclaw_instances = pgTable( .on(table.organization_id, table.created_at) .where(sql`${table.organization_id} IS NOT NULL AND ${table.destroyed_at} IS NULL`), // Non-partial index over all rows (including destroyed) so we can answer - // "what is this user's earliest instance" without a sequential scan. Used - // by `userIsWithinFirstKiloClawInstanceWindow` on the AI gateway hot path; - // the existing partial-by-user indexes can't serve it because they exclude - // destroyed rows, and destroyed rows must still count for "first instance" - // semantics. + // "what is this user's earliest instance" and user-instance listings that + // include destroyed rows without a sequential scan. The existing + // partial-by-user indexes can't serve these because they exclude destroyed + // rows. index('IDX_kiloclaw_instances_user_id_created_at').on(table.user_id, table.created_at), // Powers admin "instances on version X" filter; partial since destroyed rows are excluded. index('IDX_kiloclaw_instances_tracked_image_tag')