diff --git a/.changeset/direct-fim-providers.md b/.changeset/direct-fim-providers.md new file mode 100644 index 000000000000..5be3a54c2721 --- /dev/null +++ b/.changeset/direct-fim-providers.md @@ -0,0 +1,5 @@ +--- +"kilo-code": patch +--- + +Add explicit Mistral and Inception autocomplete options that use connected provider API keys for FIM requests. diff --git a/packages/kilo-docs/lychee.toml b/packages/kilo-docs/lychee.toml index 1958c7e5ce55..f3e8de71098c 100644 --- a/packages/kilo-docs/lychee.toml +++ b/packages/kilo-docs/lychee.toml @@ -23,6 +23,8 @@ exclude = [ 'https://console.groq.com/pages/models', '^https?://([A-Za-z0-9-]+\.)?glama\.ai(/|$)', '^https?://(api|console)\.mistral\.ai', + '^https?://codestral\.mistral\.ai/v1/fim/completions/?$', + '^https?://api\.inceptionlabs\.ai/v1/fim/completions/?$', '^https?://(app|console)\.requesty\.ai', '^https?://(cloud|console)\.google\.ai', # API endpoint prefixes are valid but return 4xx to plain GET link checks. diff --git a/packages/kilo-gateway/package.json b/packages/kilo-gateway/package.json index 982e52c55061..8520685d8393 100644 --- a/packages/kilo-gateway/package.json +++ b/packages/kilo-gateway/package.json @@ -17,6 +17,8 @@ ], "exports": { ".": "./src/index.ts", + "./autocomplete": "./src/autocomplete.ts", + "./fim": "./src/fim.ts", "./tui": "./src/tui.ts" }, "files": [ diff --git a/packages/kilo-gateway/src/autocomplete.ts b/packages/kilo-gateway/src/autocomplete.ts new file mode 100644 index 000000000000..561da4b09e36 --- /dev/null +++ b/packages/kilo-gateway/src/autocomplete.ts @@ -0,0 +1,101 @@ +export type AutocompleteProviderID = "kilo" | "mistral" | "inception" +export type DirectAutocompleteProviderID = Exclude + +export interface AutocompleteModelDef { + /** Stable combined value for internal comparisons. */ + readonly id: string + /** Model value stored in settings and sent to the FIM API. */ + readonly modelID: string + /** Human-readable label shown in settings. */ + readonly label: string + /** Provider value stored in settings and used by the selector group. */ + readonly providerID: AutocompleteProviderID + /** Provider display name for status bar / telemetry. */ + readonly provider: string + /** Full model ID sent upstream by the FIM route. */ + readonly requestModel: string + /** Provider key to use for direct BYOK FIM. Empty means Kilo Gateway. */ + readonly directProvider?: DirectAutocompleteProviderID + /** FIM request temperature. */ + readonly temperature: number +} + +const models: AutocompleteModelDef[] = [ + { + id: "kilo/mistralai/codestral-2508", + modelID: "mistralai/codestral-2508", + label: "Codestral", + providerID: "kilo", + provider: "Kilo Gateway", + requestModel: "mistralai/codestral-2508", + temperature: 0.2, + }, + { + id: "kilo/inception/mercury-edit-2", + modelID: "inception/mercury-edit-2", + label: "Mercury Edit 2", + providerID: "kilo", + provider: "Kilo Gateway", + requestModel: "inception/mercury-edit-2", + temperature: 0, + }, + { + id: "mistral/codestral-2508", + modelID: "codestral-2508", + label: "Codestral", + providerID: "mistral", + provider: "Mistral", + requestModel: "codestral-2508", + directProvider: "mistral", + temperature: 0.2, + }, + { + id: "inception/mercury-edit-2", + modelID: "mercury-edit-2", + label: "Mercury Edit 2", + providerID: "inception", + provider: "Inception", + requestModel: "mercury-edit-2", + directProvider: "inception", + temperature: 0, + }, +] + +export const AUTOCOMPLETE_MODELS: readonly AutocompleteModelDef[] = models + +export const DEFAULT_AUTOCOMPLETE_MODEL: AutocompleteModelDef = models[0]! + +const aliases: Record = { + "inception/mercury-edit": "inception/mercury-edit-2", +} + +export function getAutocompleteModel(provider?: string, model?: string): AutocompleteModelDef { + // When provider is unset, always default to Kilo Gateway. Direct-provider + // use must be opted into explicitly via the provider setting — never inferred + // from a model name, since the same plain model id can exist on multiple + // providers and we don't want to silently route legacy settings to BYOK. + const pid = provider ?? "kilo" + const mid = aliases[model ?? ""] ?? model + for (const m of models) { + if (m.providerID === pid && m.modelID === mid) return m + } + return DEFAULT_AUTOCOMPLETE_MODEL +} + +export function getAutocompleteModelById(id: string): AutocompleteModelDef { + for (const m of models) { + if (m.id === id) return m + } + return DEFAULT_AUTOCOMPLETE_MODEL +} + +export function validAutocompleteProvider(value: unknown) { + if (typeof value !== "string") return false + return models.some((m) => m.providerID === value) +} + +export function validAutocompleteModel(value: unknown) { + if (typeof value !== "string") return false + const resolved = aliases[value] ?? value + return models.some((m) => m.modelID === resolved) +} diff --git a/packages/kilo-gateway/src/fim.ts b/packages/kilo-gateway/src/fim.ts new file mode 100644 index 000000000000..2c7783080aea --- /dev/null +++ b/packages/kilo-gateway/src/fim.ts @@ -0,0 +1,34 @@ +import { KILO_API_BASE } from "./api/constants.js" +import { getAutocompleteModel, type DirectAutocompleteProviderID } from "./autocomplete.js" + +export { requestMistralFim } from "./mistral-fim-endpoint.js" + +export const DIRECT_FIM_ENV: Record = { + mistral: ["MISTRAL_API_KEY"], + inception: ["INCEPTION_API_KEY"], +} + +export type FimTarget = + | { provider: "kilo"; model: string; url: string } + | { provider: "inception"; model: string; url: string } + | { provider: "mistral"; model: string } + +const KILO_FIM_URL = KILO_API_BASE + "/api/fim/completions" +const INCEPTION_FIM_URL = "https://api.inceptionlabs.ai/v1/fim/completions" + +function kiloTarget(model?: string): FimTarget { + return { provider: "kilo", model: model ?? "mistralai/codestral-2501", url: KILO_FIM_URL } +} + +export function resolveFimTarget(provider?: string, model?: string): FimTarget { + if (!provider || provider === "kilo") return kiloTarget(model) + + const info = getAutocompleteModel(provider, model) + if (info.directProvider === "mistral") { + return { provider: "mistral", model: info.requestModel } + } + if (info.directProvider === "inception") { + return { provider: "inception", model: info.requestModel, url: INCEPTION_FIM_URL } + } + return kiloTarget(model) +} diff --git a/packages/kilo-gateway/src/index.ts b/packages/kilo-gateway/src/index.ts index 429575d093d1..6a1449d9c195 100644 --- a/packages/kilo-gateway/src/index.ts +++ b/packages/kilo-gateway/src/index.ts @@ -41,6 +41,16 @@ export { type KiloEmbeddingModelCatalog, } from "./api/embedding-models.js" export { resolveKiloGatewayBaseUrl, resolveKiloOpenRouterBaseUrl } from "./api/url.js" +export { + AUTOCOMPLETE_MODELS, + DEFAULT_AUTOCOMPLETE_MODEL, + getAutocompleteModel, + getAutocompleteModelById, + validAutocompleteModel, + validAutocompleteProvider, + type AutocompleteModelDef, + type AutocompleteProviderID, +} from "./autocomplete.js" export { fetchOrganizationModes, clearModesCache, diff --git a/packages/kilo-gateway/src/mistral-fim-endpoint.ts b/packages/kilo-gateway/src/mistral-fim-endpoint.ts new file mode 100644 index 000000000000..ebd5b70fb414 --- /dev/null +++ b/packages/kilo-gateway/src/mistral-fim-endpoint.ts @@ -0,0 +1,30 @@ +export const MISTRAL_FIM_URL = "https://api.mistral.ai/v1/fim/completions" +export const CODESTRAL_FIM_URL = "https://codestral.mistral.ai/v1/fim/completions" + +let preferred: string | undefined + +export function isMistralEndpointMismatch(response: Response) { + return response.status === 401 || response.status === 403 +} + +export function clearMistralFimEndpointCache() { + preferred = undefined +} + +export function getCachedMistralFimEndpoint() { + return preferred +} + +export async function requestMistralFim(request: (url: string) => Promise) { + const firstUrl = preferred ?? MISTRAL_FIM_URL + const secondUrl = firstUrl === MISTRAL_FIM_URL ? CODESTRAL_FIM_URL : MISTRAL_FIM_URL + const first = await request(firstUrl) + + if (first.ok) return first + if (!isMistralEndpointMismatch(first)) return first + + preferred = undefined + const second = await request(secondUrl) + if (second.ok) preferred = secondUrl + return second +} diff --git a/packages/kilo-gateway/src/server/fim.ts b/packages/kilo-gateway/src/server/fim.ts new file mode 100644 index 000000000000..08790784ea03 --- /dev/null +++ b/packages/kilo-gateway/src/server/fim.ts @@ -0,0 +1,120 @@ +import { HEADER_FEATURE } from "../api/constants.js" +import type { DirectAutocompleteProviderID } from "../autocomplete.js" +import { DIRECT_FIM_ENV, requestMistralFim, resolveFimTarget, type FimTarget } from "../fim.js" +import { buildKiloHeaders } from "../headers.js" +import type { AuthStore } from "./handlers.js" + +type Auth = Pick + +const FIM_TIMEOUT_MS = 30_000 + +async function getProxyAuth(Auth: Auth) { + const auth = await Auth.get("kilo") + const token = auth?.type === "api" ? auth.key : auth?.type === "oauth" ? auth.access : undefined + return { + auth, + token, + organizationId: auth?.type === "oauth" ? auth.accountId : undefined, + } +} + +async function getProviderKey(Auth: Auth, provider: DirectAutocompleteProviderID) { + const auth = await Auth.get(provider) + if (auth?.type === "api") return auth.key + return DIRECT_FIM_ENV[provider].map((key) => process.env[key]).find(Boolean) +} + +async function fetchFim( + target: FimTarget, + key: string, + input: { + prefix: string + suffix: string + maxTokens: number + temperature: number + signal: AbortSignal + organizationId?: string + }, +): Promise { + const run = async (url: string) => { + console.info(`[FIM] request provider=${target.provider} model=${target.model} url=${url}`) + return fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${key}`, + ...(target.provider === "kilo" + ? buildKiloHeaders(undefined, { kilocodeOrganizationId: input.organizationId }) + : {}), + ...(target.provider === "kilo" ? { [HEADER_FEATURE]: "autocomplete" } : {}), + }, + signal: input.signal, + body: JSON.stringify({ + model: target.model, + prompt: input.prefix, + suffix: input.suffix, + max_tokens: input.maxTokens, + temperature: input.temperature, + stream: true, + }), + }) + } + + if (target.provider === "mistral") return requestMistralFim(run) + return run(target.url) +} + +export function createFimHandler(Auth: Auth) { + return async (c: any) => { + const { prefix, suffix, provider, model, maxTokens, temperature } = c.req.valid("json") + const target = resolveFimTarget(provider, model) + const fimMaxTokens = maxTokens ?? 256 + const fimTemperature = temperature ?? 0.2 + const proxy = target.provider === "kilo" ? await getProxyAuth(Auth) : undefined + const token = target.provider === "kilo" ? proxy?.token : await getProviderKey(Auth, target.provider) + + if (target.provider === "kilo" && !proxy?.auth) { + return c.json({ error: "Not authenticated with Kilo Gateway" }, 401) + } + + if (target.provider === "kilo" && !token) { + return c.json({ error: "No valid token found" }, 401) + } + + if (!token) { + return c.json({ error: `Missing ${target.provider} provider API key` }, 401) + } + + const signal = AbortSignal.any([c.req.raw.signal, AbortSignal.timeout(FIM_TIMEOUT_MS)]) + + try { + const response = await fetchFim(target, token, { + prefix, + suffix, + maxTokens: fimMaxTokens, + temperature: fimTemperature, + signal, + organizationId: proxy?.organizationId, + }) + + if (!response.ok) { + const text = await response.text() + return c.json({ error: `FIM request failed: ${response.status} ${text}` }, response.status as any) + } + + return new Response(response.body, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }, + }) + } catch (err) { + if (err instanceof DOMException && err.name === "TimeoutError") { + return c.json({ error: "FIM request timed out" }, 504 as any) + } + if (signal.aborted) return c.json({ error: "FIM request canceled" }, 499 as any) + throw err + } + } +} diff --git a/packages/kilo-gateway/src/server/routes.ts b/packages/kilo-gateway/src/server/routes.ts index 78c2bfda6b57..6b2115459156 100644 --- a/packages/kilo-gateway/src/server/routes.ts +++ b/packages/kilo-gateway/src/server/routes.ts @@ -11,6 +11,7 @@ import { KILO_API_BASE, HEADER_FEATURE, HEADER_ORGANIZATIONID } from "../api/con import { buildKiloHeaders } from "../headers.js" import type { ImportDeps, DrizzleDb } from "../cloud-sessions.js" import { fetchCloudSession, fetchCloudSessionForImport, importSessionToDb } from "../cloud-sessions.js" +import { createFimHandler } from "./fim.js" import { GatewayError, UnauthorizedError, @@ -44,8 +45,6 @@ interface KiloRoutesDeps extends ImportDeps { Instances: { disposeAllInstances(): Promise } } -const FIM_TIMEOUT_MS = 30_000 - /** * Create Kilo Gateway routes with OpenCode dependencies injected * @@ -318,74 +317,13 @@ export function createKiloRoutes(deps: KiloRoutesDeps) { z.object({ prefix: z.string(), suffix: z.string(), + provider: z.string().optional(), model: z.string().optional(), maxTokens: z.number().optional(), temperature: z.number().optional(), }), ), - async (c: any) => { - const proxy = await getProxyAuth() - - if (!proxy.auth) { - return c.json({ error: "Not authenticated with Kilo Gateway" }, 401) - } - - if (!proxy.token) { - return c.json({ error: "No valid token found" }, 401) - } - - const { prefix, suffix, model, maxTokens, temperature } = c.req.valid("json") - const fimModel = model ?? "mistralai/codestral-2501" - const fimMaxTokens = maxTokens ?? 256 - const fimTemperature = temperature ?? 0.2 - - const baseApiUrl = KILO_API_BASE + "/api/" - const endpoint = new URL("fim/completions", baseApiUrl) - - const headers = { - "Content-Type": "application/json", - Authorization: `Bearer ${proxy.token}`, - ...buildKiloHeaders(undefined, { kilocodeOrganizationId: proxy.organizationId }), - [HEADER_FEATURE]: "autocomplete", - } - - const signal = AbortSignal.any([c.req.raw.signal, AbortSignal.timeout(FIM_TIMEOUT_MS)]) - - let response: Response - try { - response = await fetch(endpoint, { - method: "POST", - headers, - signal, - body: JSON.stringify({ - model: fimModel, - prompt: prefix, - suffix, - max_tokens: fimMaxTokens, - temperature: fimTemperature, - stream: true, - }), - }) - } catch (err) { - if (err instanceof DOMException && err.name === "TimeoutError") - return c.json({ error: "FIM request timed out" }, 504 as any) - if (signal.aborted) return c.json({ error: "FIM request canceled" }, 499 as any) - throw err - } - - if (!response.ok) { - const text = await response.text() - return c.json({ error: `FIM request failed: ${response.status} ${text}` }, response.status as any) - } - - return new Response(response.body, { - headers: { - "Content-Type": "text/event-stream", - "Cache-Control": "no-cache", - Connection: "keep-alive", - }, - }) - }, + createFimHandler(Auth), ) .post( "/audio/transcriptions", diff --git a/packages/kilo-gateway/test/fim.test.ts b/packages/kilo-gateway/test/fim.test.ts new file mode 100644 index 000000000000..86d7e08b5562 --- /dev/null +++ b/packages/kilo-gateway/test/fim.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, test } from "bun:test" +import { resolveFimTarget } from "../src/fim" + +describe("FIM target resolution", () => { + test("keeps gateway autocomplete models on Kilo Gateway", () => { + expect(resolveFimTarget("kilo", "mistralai/codestral-2508")).toEqual({ + provider: "kilo", + model: "mistralai/codestral-2508", + url: "https://api.kilo.ai/api/fim/completions", + }) + expect(resolveFimTarget("kilo", "inception/mercury-edit-2")).toEqual({ + provider: "kilo", + model: "inception/mercury-edit-2", + url: "https://api.kilo.ai/api/fim/completions", + }) + }) + + test("routes explicit provider autocomplete models directly", () => { + expect(resolveFimTarget("mistral", "codestral-2508")).toEqual({ + provider: "mistral", + model: "codestral-2508", + }) + expect(resolveFimTarget("inception", "mercury-edit-2")).toEqual({ + provider: "inception", + model: "mercury-edit-2", + url: "https://api.inceptionlabs.ai/v1/fim/completions", + }) + }) + + test("preserves gateway model pass-through behavior", () => { + expect(resolveFimTarget()).toEqual({ + provider: "kilo", + model: "mistralai/codestral-2501", + url: "https://api.kilo.ai/api/fim/completions", + }) + expect(resolveFimTarget(undefined, "mistralai/codestral-2508")).toEqual({ + provider: "kilo", + model: "mistralai/codestral-2508", + url: "https://api.kilo.ai/api/fim/completions", + }) + expect(resolveFimTarget(undefined, "inception/mercury-edit")).toEqual({ + provider: "kilo", + model: "inception/mercury-edit", + url: "https://api.kilo.ai/api/fim/completions", + }) + expect(resolveFimTarget("kilo", "custom/fim-model")).toEqual({ + provider: "kilo", + model: "custom/fim-model", + url: "https://api.kilo.ai/api/fim/completions", + }) + }) +}) diff --git a/packages/kilo-gateway/test/mistral-fim-endpoint.test.ts b/packages/kilo-gateway/test/mistral-fim-endpoint.test.ts new file mode 100644 index 000000000000..d10c6177287a --- /dev/null +++ b/packages/kilo-gateway/test/mistral-fim-endpoint.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test } from "bun:test" +import { + CODESTRAL_FIM_URL, + MISTRAL_FIM_URL, + clearMistralFimEndpointCache, + getCachedMistralFimEndpoint, + requestMistralFim, +} from "../src/mistral-fim-endpoint" + +function response(status: number) { + return new Response(null, { status }) +} + +describe("Mistral FIM endpoint cache", () => { + test("remembers Codestral endpoint after successful fallback", async () => { + clearMistralFimEndpointCache() + const urls: string[] = [] + const first = await requestMistralFim(async (url) => { + urls.push(url) + return response(url === MISTRAL_FIM_URL ? 401 : 200) + }) + const second = await requestMistralFim(async (url) => { + urls.push(url) + return response(200) + }) + + expect(first.ok).toBe(true) + expect(second.ok).toBe(true) + expect(urls).toEqual([MISTRAL_FIM_URL, CODESTRAL_FIM_URL, CODESTRAL_FIM_URL]) + expect(getCachedMistralFimEndpoint()).toBe(CODESTRAL_FIM_URL) + }) + + test("does not remember fallback for invalid credentials", async () => { + clearMistralFimEndpointCache() + const urls: string[] = [] + const res = await requestMistralFim(async (url) => { + urls.push(url) + return response(401) + }) + + expect(res.status).toBe(401) + expect(urls).toEqual([MISTRAL_FIM_URL, CODESTRAL_FIM_URL]) + expect(getCachedMistralFimEndpoint()).toBeUndefined() + }) + + test("uses one process-local endpoint preference", async () => { + clearMistralFimEndpointCache() + const urls: string[] = [] + await requestMistralFim(async (url) => { + urls.push(url) + return response(url === MISTRAL_FIM_URL ? 403 : 200) + }) + await requestMistralFim(async (url) => { + urls.push(url) + return response(200) + }) + + expect(urls).toEqual([MISTRAL_FIM_URL, CODESTRAL_FIM_URL, CODESTRAL_FIM_URL]) + expect(getCachedMistralFimEndpoint()).toBe(CODESTRAL_FIM_URL) + }) + + test("clears stale preference and probes alternate endpoint", async () => { + clearMistralFimEndpointCache() + const urls: string[] = [] + await requestMistralFim(async (url) => response(url === MISTRAL_FIM_URL ? 401 : 200)) + const res = await requestMistralFim(async (url) => { + urls.push(url) + return response(url === CODESTRAL_FIM_URL ? 401 : 200) + }) + + expect(res.ok).toBe(true) + expect(urls).toEqual([CODESTRAL_FIM_URL, MISTRAL_FIM_URL]) + expect(getCachedMistralFimEndpoint()).toBe(MISTRAL_FIM_URL) + }) +}) diff --git a/packages/kilo-vscode/package.json b/packages/kilo-vscode/package.json index 196f9999df01..bcb8a931e321 100644 --- a/packages/kilo-vscode/package.json +++ b/packages/kilo-vscode/package.json @@ -824,14 +824,32 @@ "type": "string", "enum": [ "mistralai/codestral-2508", - "inception/mercury-edit-2" + "inception/mercury-edit-2", + "codestral-2508", + "mercury-edit-2" ], "enumDescriptions": [ - "Codestral by Mistral AI (default)", - "Mercury Edit 2 by Inception" + "Codestral via Kilo Gateway (default)", + "Mercury Edit 2 via Kilo Gateway", + "Codestral via your connected Mistral provider API key", + "Mercury Edit 2 via your connected Inception provider API key" ], "description": "Model to use for inline autocomplete suggestions" }, + "kilo-code.new.autocomplete.provider": { + "type": "string", + "enum": [ + "kilo", + "mistral", + "inception" + ], + "enumDescriptions": [ + "Use autocomplete models through Kilo Gateway", + "Use autocomplete models through your connected Mistral provider API key", + "Use autocomplete models through your connected Inception provider API key" + ], + "description": "Provider to use for inline autocomplete suggestions. If unset, Kilo Gateway is used." + }, "kilo-code.new.autocomplete.enableAutoTrigger": { "type": "boolean", "default": true, diff --git a/packages/kilo-vscode/src/services/autocomplete/AutocompleteServiceManager.ts b/packages/kilo-vscode/src/services/autocomplete/AutocompleteServiceManager.ts index a272f6773720..d3d0d67c85b5 100644 --- a/packages/kilo-vscode/src/services/autocomplete/AutocompleteServiceManager.ts +++ b/packages/kilo-vscode/src/services/autocomplete/AutocompleteServiceManager.ts @@ -23,11 +23,13 @@ export interface AutocompleteServiceSettings { function readSettings(): AutocompleteServiceSettings { const config = vscode.workspace.getConfiguration(CONFIG_SECTION) + const info = getAutocompleteModel(config.get("provider"), config.get("model")) return { enableAutoTrigger: config.get("enableAutoTrigger") ?? true, enableSmartInlineTaskKeybinding: config.get("enableSmartInlineTaskKeybinding") ?? true, enableChatAutocomplete: config.get("enableChatAutocomplete") ?? true, - model: getAutocompleteModel(config.get("model") ?? "").id, + provider: info.providerID, + model: info.modelID, snoozeUntil: config.get("snoozeUntil"), } } @@ -119,9 +121,7 @@ export class AutocompleteServiceManager { public async load() { this.settings = readSettings() - if (this.settings.model) { - this.inlineCompletionProvider.setModel(this.settings.model) - } + this.inlineCompletionProvider.setModel(getAutocompleteModel(this.settings.provider, this.settings.model).id) await this.updateGlobalContext() this.updateStatusBar() @@ -319,11 +319,13 @@ export class AutocompleteServiceManager { } private getCurrentModelName(): string { - return this.inlineCompletionProvider.getModelId() + const info = getAutocompleteModel(this.settings?.provider, this.settings?.model) + return info.label } private getCurrentProviderName(): string { - return getAutocompleteModel(this.inlineCompletionProvider.getModelId()).provider + const info = getAutocompleteModel(this.settings?.provider, this.settings?.model) + return info.provider } private hasNoUsableProvider(): boolean { diff --git a/packages/kilo-vscode/src/services/autocomplete/__tests__/settings.spec.ts b/packages/kilo-vscode/src/services/autocomplete/__tests__/settings.spec.ts index 5552d8c9fa6a..04574cad6e3d 100644 --- a/packages/kilo-vscode/src/services/autocomplete/__tests__/settings.spec.ts +++ b/packages/kilo-vscode/src/services/autocomplete/__tests__/settings.spec.ts @@ -24,16 +24,29 @@ describe("autocomplete settings", () => { update.mockClear() }) - it("includes the configured model in loaded settings", async () => { - state.set("model", "inception/mercury-edit-2") + it("includes the configured direct provider model in loaded settings", async () => { + state.set("provider", "inception") + state.set("model", "mercury-edit-2") const { buildAutocompleteSettingsMessage } = await import("../settings") - expect(buildAutocompleteSettingsMessage().settings.model).toBe("inception/mercury-edit-2") + expect(buildAutocompleteSettingsMessage().settings.provider).toBe("inception") + expect(buildAutocompleteSettingsMessage().settings.model).toBe("mercury-edit-2") + }) + + it("does not infer direct provider from a bare model name when provider is unset", async () => { + // Safety: a legacy `model` setting alone must never silently route to a + // direct BYOK provider. Direct providers require an explicit `provider`. + state.set("model", "mercury-edit-2") + const { buildAutocompleteSettingsMessage } = await import("../settings") + + expect(buildAutocompleteSettingsMessage().settings.provider).toBe("kilo") + expect(buildAutocompleteSettingsMessage().settings.model).toBe("mistralai/codestral-2508") }) it("defaults to codestral when no model is set", async () => { const { buildAutocompleteSettingsMessage } = await import("../settings") + expect(buildAutocompleteSettingsMessage().settings.provider).toBe("kilo") expect(buildAutocompleteSettingsMessage().settings.model).toBe("mistralai/codestral-2508") }) @@ -41,26 +54,39 @@ describe("autocomplete settings", () => { state.set("model", "some/removed-model") const { buildAutocompleteSettingsMessage } = await import("../settings") + expect(buildAutocompleteSettingsMessage().settings.provider).toBe("kilo") expect(buildAutocompleteSettingsMessage().settings.model).toBe("mistralai/codestral-2508") }) - it("maps legacy inception/mercury-edit to inception/mercury-edit-2", async () => { + it("maps legacy inception/mercury-edit to Kilo Gateway Mercury", async () => { state.set("model", "inception/mercury-edit") const { buildAutocompleteSettingsMessage } = await import("../settings") + expect(buildAutocompleteSettingsMessage().settings.provider).toBe("kilo") + expect(buildAutocompleteSettingsMessage().settings.model).toBe("inception/mercury-edit-2") + }) + + it("maps legacy inception/mercury-edit-2 to Kilo Gateway Mercury", async () => { + state.set("model", "inception/mercury-edit-2") + const { buildAutocompleteSettingsMessage } = await import("../settings") + + expect(buildAutocompleteSettingsMessage().settings.provider).toBe("kilo") expect(buildAutocompleteSettingsMessage().settings.model).toBe("inception/mercury-edit-2") }) it("validates supported model updates", async () => { const { validAutocompleteSetting } = await import("../settings") - expect(validAutocompleteSetting("model", "inception/mercury-edit-2")).toBe(true) + expect(validAutocompleteSetting("model", "mercury-edit-2")).toBe(true) + expect(validAutocompleteSetting("provider", "inception")).toBe(true) }) - it("rejects unsupported model updates", async () => { + it("rejects unsupported autocomplete updates", async () => { const { validAutocompleteSetting } = await import("../settings") expect(validAutocompleteSetting("model", "other/model")).toBe(false) + expect(validAutocompleteSetting("provider", undefined)).toBe(false) + expect(validAutocompleteSetting("provider", null)).toBe(false) }) it("rejects non-boolean toggle updates", async () => { diff --git a/packages/kilo-vscode/src/services/autocomplete/chat-autocomplete/ChatTextAreaAutocomplete.ts b/packages/kilo-vscode/src/services/autocomplete/chat-autocomplete/ChatTextAreaAutocomplete.ts index 3971e785a11a..9d2c0d008238 100644 --- a/packages/kilo-vscode/src/services/autocomplete/chat-autocomplete/ChatTextAreaAutocomplete.ts +++ b/packages/kilo-vscode/src/services/autocomplete/chat-autocomplete/ChatTextAreaAutocomplete.ts @@ -77,7 +77,7 @@ export class ChatTextAreaAutocomplete { async getCompletion(userText: string, visibleCodeContext?: VisibleCodeContext): Promise<{ suggestion: string }> { const cfg = vscode.workspace.getConfiguration("kilo-code.new.autocomplete") - const entry = getAutocompleteModel(cfg.get("model") ?? "") + const entry = getAutocompleteModel(cfg.get("provider"), cfg.get("model")) const startTime = Date.now() // Build context for telemetry diff --git a/packages/kilo-vscode/src/services/autocomplete/classic-auto-complete/AutocompleteInlineCompletionProvider.ts b/packages/kilo-vscode/src/services/autocomplete/classic-auto-complete/AutocompleteInlineCompletionProvider.ts index 78c16079f521..32ee332e6752 100644 --- a/packages/kilo-vscode/src/services/autocomplete/classic-auto-complete/AutocompleteInlineCompletionProvider.ts +++ b/packages/kilo-vscode/src/services/autocomplete/classic-auto-complete/AutocompleteInlineCompletionProvider.ts @@ -22,7 +22,7 @@ import { import { FimPromptBuilder } from "./FillInTheMiddle" import { hasValidCredentials } from "../fim" import type { KiloConnectionService } from "../../cli-backend" -import { getAutocompleteModel } from "../../../shared/autocomplete-models" +import { getAutocompleteModelById } from "../../../shared/autocomplete-models" import { ContextRetrievalService } from "../continuedev/core/autocomplete/context/ContextRetrievalService" import { VsCodeIde } from "../continuedev/core/vscode-test-harness/src/VSCodeIde" import { RecentlyVisitedRangesService } from "../continuedev/core/vscode-test-harness/src/autocomplete/RecentlyVisitedRangesService" @@ -345,7 +345,7 @@ export class AutocompleteInlineCompletionProvider implements vscode.InlineComple const telemetryContext: AutocompleteContext = { languageId: document.languageId, modelId: this.contextProvider.modelId, - provider: getAutocompleteModel(this.contextProvider.modelId).provider, + provider: getAutocompleteModelById(this.contextProvider.modelId).provider, } this.telemetry?.captureSuggestionRequested(telemetryContext) @@ -586,7 +586,7 @@ export class AutocompleteInlineCompletionProvider implements vscode.InlineComple const telemetryContext: AutocompleteContext = { languageId, modelId: this.contextProvider.modelId, - provider: getAutocompleteModel(this.contextProvider.modelId).provider, + provider: getAutocompleteModelById(this.contextProvider.modelId).provider, } // Defense-in-depth: credentials may become invalid between the provider gate and the actual diff --git a/packages/kilo-vscode/src/services/autocomplete/classic-auto-complete/FillInTheMiddle.ts b/packages/kilo-vscode/src/services/autocomplete/classic-auto-complete/FillInTheMiddle.ts index 6a36ae3ab32b..564423b2f184 100644 --- a/packages/kilo-vscode/src/services/autocomplete/classic-auto-complete/FillInTheMiddle.ts +++ b/packages/kilo-vscode/src/services/autocomplete/classic-auto-complete/FillInTheMiddle.ts @@ -65,36 +65,14 @@ export class FimPromptBuilder { signal?: AbortSignal, ): Promise { const { formattedPrefix, prunedSuffix, autocompleteInput } = prompt - let perflog = "" - const logtime = (() => { - let timestamp = performance.now() - return (msg: string) => { - const baseline = timestamp - timestamp = performance.now() - perflog += `${msg}: ${timestamp - baseline}\n` - } - })() - - logtime("snippets") - - console.log("[FIM] formattedPrefix:", formattedPrefix) - let response = "" const onChunk = (text: string) => { response += text } - logtime("prep fim") const usageInfo = await generateFim(connection, modelId, formattedPrefix, prunedSuffix, onChunk, signal) - logtime("fim network") - console.log("[FIM] response:", response) const fillInAtCursorSuggestion = processSuggestion(response) - if (fillInAtCursorSuggestion.text) { - console.info("Final FIM suggestion:", fillInAtCursorSuggestion) - } - logtime("processSuggestion") - console.log(perflog + `lengths: ${formattedPrefix.length + prunedSuffix.length}\n`) return { suggestion: fillInAtCursorSuggestion, cost: usageInfo.cost, diff --git a/packages/kilo-vscode/src/services/autocomplete/fim.ts b/packages/kilo-vscode/src/services/autocomplete/fim.ts index 8e5fcb429c6c..d45ecd65b9dd 100644 --- a/packages/kilo-vscode/src/services/autocomplete/fim.ts +++ b/packages/kilo-vscode/src/services/autocomplete/fim.ts @@ -1,6 +1,8 @@ import { ResponseMetaData } from "./types" import type { KiloConnectionService } from "../cli-backend" -import { getAutocompleteModel } from "../../shared/autocomplete-models" +import { getAutocompleteModelById } from "../../shared/autocomplete-models" + +const FIM_MAX_TOKENS = 256 /** * Generate a FIM (Fill-in-the-Middle) completion via the CLI backend. @@ -17,7 +19,7 @@ export async function generateFim( signal?: AbortSignal, ): Promise { const client = await connectionService.getClientAsync() - + const info = getAutocompleteModelById(modelId) let cost = 0 let inputTokens = 0 let outputTokens = 0 @@ -27,15 +29,16 @@ export async function generateFim( // ends the stream. Without this, errors never reach ErrorBackoff. let sseError: Error | undefined - const temp = getAutocompleteModel(modelId).temperature + console.info(`[FIM] request provider=${info.providerID} model=${info.requestModel} url=/kilo/fim`) const { stream } = await client.kilo.fim( { prefix, suffix, - model: modelId, - maxTokens: 256, - temperature: temp, + provider: info.providerID, + model: info.modelID, + maxTokens: FIM_MAX_TOKENS, + temperature: info.temperature, }, { signal, diff --git a/packages/kilo-vscode/src/services/autocomplete/settings.ts b/packages/kilo-vscode/src/services/autocomplete/settings.ts index 9ec341406dc4..1312de8f9b17 100644 --- a/packages/kilo-vscode/src/services/autocomplete/settings.ts +++ b/packages/kilo-vscode/src/services/autocomplete/settings.ts @@ -1,5 +1,9 @@ import * as vscode from "vscode" -import { AUTOCOMPLETE_MODELS, getAutocompleteModel } from "../../shared/autocomplete-models" +import { + getAutocompleteModel, + validAutocompleteModel, + validAutocompleteProvider, +} from "../../shared/autocomplete-models" type Message = { type: string @@ -18,13 +22,15 @@ export async function routeAutocompleteMessage(message: Message, post: Post): Pr export function buildAutocompleteSettingsMessage() { const config = vscode.workspace.getConfiguration("kilo-code.new.autocomplete") + const info = getAutocompleteModel(config.get("provider"), config.get("model")) return { type: "autocompleteSettingsLoaded" as const, settings: { enableAutoTrigger: config.get("enableAutoTrigger", true), enableSmartInlineTaskKeybinding: config.get("enableSmartInlineTaskKeybinding", false), enableChatAutocomplete: config.get("enableChatAutocomplete", false), - model: getAutocompleteModel(config.get("model") ?? "").id, + provider: info.providerID, + model: info.modelID, }, } } @@ -40,8 +46,11 @@ export function watchAutocompleteConfig(post: Post): vscode.Disposable { export function validAutocompleteSetting(key: string, value: unknown) { if (key === "model") { - if (typeof value !== "string") return false - return AUTOCOMPLETE_MODELS.some((m) => m.id === value) + return validAutocompleteModel(value) + } + + if (key === "provider") { + return validAutocompleteProvider(value) } if (key === "enableAutoTrigger") return typeof value === "boolean" diff --git a/packages/kilo-vscode/src/shared/autocomplete-models.ts b/packages/kilo-vscode/src/shared/autocomplete-models.ts index 6f46b56512bb..e67350a13136 100644 --- a/packages/kilo-vscode/src/shared/autocomplete-models.ts +++ b/packages/kilo-vscode/src/shared/autocomplete-models.ts @@ -1,52 +1,10 @@ -/** - * Single source of truth for autocomplete FIM model definitions. - * - * Shared between extension code (src/) and webview code (webview-ui/). - * When adding a new model, update ONLY this file and package.json's - * `kilo-code.new.autocomplete.model` enum. - */ - -export interface AutocompleteModelDef { - /** Full model ID sent to the gateway, e.g. "mistralai/codestral-2508" */ - readonly id: string - /** Human-readable label shown in the settings dropdown */ - readonly label: string - /** Provider display name for status bar / telemetry */ - readonly provider: string - /** FIM request temperature */ - readonly temperature: number -} - -const models: AutocompleteModelDef[] = [ - { - id: "mistralai/codestral-2508", - label: "Codestral (Mistral AI)", - provider: "Mistral AI", - temperature: 0.2, - }, - { - id: "inception/mercury-edit-2", - label: "Mercury Edit 2 (Inception)", - provider: "Inception", - temperature: 0, - }, -] - -export const AUTOCOMPLETE_MODELS: readonly AutocompleteModelDef[] = models - -export const DEFAULT_AUTOCOMPLETE_MODEL: AutocompleteModelDef = models[0]! - -// Map inception/mercury-edit to inception/mercury-edit-2 so users who -// already have the old id saved in settings.json keep working without a -// silent fallback to Codestral. Not exposed in the dropdown. -const aliases: Record = { - "inception/mercury-edit": "inception/mercury-edit-2", -} - -export function getAutocompleteModel(id: string): AutocompleteModelDef { - const resolved = aliases[id] ?? id - for (const m of models) { - if (m.id === resolved) return m - } - return DEFAULT_AUTOCOMPLETE_MODEL -} +export { + AUTOCOMPLETE_MODELS, + DEFAULT_AUTOCOMPLETE_MODEL, + getAutocompleteModel, + getAutocompleteModelById, + validAutocompleteModel, + validAutocompleteProvider, + type AutocompleteModelDef, + type AutocompleteProviderID, +} from "@kilocode/kilo-gateway/autocomplete" diff --git a/packages/kilo-vscode/tests/unit/autocomplete-model-selector.test.ts b/packages/kilo-vscode/tests/unit/autocomplete-model-selector.test.ts index 1996041e9176..d2bc87994043 100644 --- a/packages/kilo-vscode/tests/unit/autocomplete-model-selector.test.ts +++ b/packages/kilo-vscode/tests/unit/autocomplete-model-selector.test.ts @@ -1,19 +1,15 @@ import { describe, expect, it } from "vitest" -import { - AUTOCOMPLETE_PROVIDER_ID, - AUTOCOMPLETE_PROVIDER_NAME, - AUTOCOMPLETE_SELECTOR_MODELS, -} from "../../webview-ui/src/components/settings/autocomplete-model-selector" +import { AUTOCOMPLETE_SELECTOR_MODELS } from "../../webview-ui/src/components/settings/autocomplete-model-selector" import { AUTOCOMPLETE_MODELS } from "../../src/shared/autocomplete-models" describe("autocomplete model selector", () => { - it("shows only Kilo Gateway autocomplete models", () => { + it("shows autocomplete models grouped by their configured provider", () => { expect(AUTOCOMPLETE_SELECTOR_MODELS).toEqual( AUTOCOMPLETE_MODELS.map((m) => ({ - id: m.id, + id: m.modelID, name: m.label, - providerID: AUTOCOMPLETE_PROVIDER_ID, - providerName: AUTOCOMPLETE_PROVIDER_NAME, + providerID: m.providerID, + providerName: m.provider, })), ) }) diff --git a/packages/kilo-vscode/tests/unit/autocomplete-models-sync.test.ts b/packages/kilo-vscode/tests/unit/autocomplete-models-sync.test.ts index e7dc56578399..799daa11857f 100644 --- a/packages/kilo-vscode/tests/unit/autocomplete-models-sync.test.ts +++ b/packages/kilo-vscode/tests/unit/autocomplete-models-sync.test.ts @@ -7,8 +7,8 @@ describe("autocomplete model enum ↔ AUTOCOMPLETE_MODELS sync", () => { const pkg = JSON.parse(readFileSync(join(__dirname, "../../package.json"), "utf8")) const prop = pkg.contributes.configuration.properties["kilo-code.new.autocomplete.model"] - it("package.json enum matches AUTOCOMPLETE_MODELS ids", () => { - const ids = AUTOCOMPLETE_MODELS.map((m) => m.id) + it("package.json enum matches AUTOCOMPLETE_MODELS model IDs", () => { + const ids = AUTOCOMPLETE_MODELS.map((m) => m.modelID) expect(prop.enum).toEqual(ids) }) diff --git a/packages/kilo-vscode/webview-ui/src/components/settings/ModelsTab.tsx b/packages/kilo-vscode/webview-ui/src/components/settings/ModelsTab.tsx index 9a72e0883e90..26e631c8042f 100644 --- a/packages/kilo-vscode/webview-ui/src/components/settings/ModelsTab.tsx +++ b/packages/kilo-vscode/webview-ui/src/components/settings/ModelsTab.tsx @@ -9,7 +9,7 @@ import { DEFAULT_AUTOCOMPLETE_MODEL } from "../../../../src/shared/autocomplete- import { ModelSelectorBase } from "../shared/ModelSelector" import { ThinkingSelectorBase } from "../shared/ThinkingSelector" import SettingsRow from "./SettingsRow" -import { AUTOCOMPLETE_PROVIDER_ID, AUTOCOMPLETE_SELECTOR_MODELS } from "./autocomplete-model-selector" +import { AUTOCOMPLETE_SELECTOR_MODELS, getAutocompleteSelection } from "./autocomplete-model-selector" const ModelsTab: Component = () => { const { config, settings, updateConfig, updateSetting } = useConfig() @@ -17,7 +17,9 @@ const ModelsTab: Component = () => { const provider = useProvider() const session = useSession() - const autocompleteModel = () => String(settings()["autocomplete.model"] ?? DEFAULT_AUTOCOMPLETE_MODEL.id) + const autocompleteProvider = () => + String(settings()["autocomplete.provider"] ?? DEFAULT_AUTOCOMPLETE_MODEL.providerID) + const autocompleteModel = () => String(settings()["autocomplete.model"] ?? DEFAULT_AUTOCOMPLETE_MODEL.modelID) function handleModelSelect(configKey: "model" | "small_model") { return (providerID: string, modelID: string) => { @@ -71,7 +73,8 @@ const ModelsTab: Component = () => { } function handleAutocompleteModelSelect(providerID: string, modelID: string) { - if (providerID !== AUTOCOMPLETE_PROVIDER_ID || !modelID) return + if (!providerID || !modelID) return + updateSetting("autocomplete.provider", providerID) updateSetting("autocomplete.model", modelID) } @@ -129,7 +132,7 @@ const ModelsTab: Component = () => { last > ({ - id: m.id, + id: m.modelID, name: m.label, - providerID: AUTOCOMPLETE_PROVIDER_ID, - providerName: AUTOCOMPLETE_PROVIDER_NAME, + providerID: m.providerID, + providerName: m.provider, })) diff --git a/packages/kilo-vscode/webview-ui/src/context/config.tsx b/packages/kilo-vscode/webview-ui/src/context/config.tsx index 5587f042a852..36348c586e11 100644 --- a/packages/kilo-vscode/webview-ui/src/context/config.tsx +++ b/packages/kilo-vscode/webview-ui/src/context/config.tsx @@ -76,6 +76,7 @@ export const ConfigProvider: ParentComponent = (props) => { "autocomplete.enableAutoTrigger": message.settings.enableAutoTrigger, "autocomplete.enableSmartInlineTaskKeybinding": message.settings.enableSmartInlineTaskKeybinding, "autocomplete.enableChatAutocomplete": message.settings.enableChatAutocomplete, + "autocomplete.provider": message.settings.provider, "autocomplete.model": message.settings.model, }) return diff --git a/packages/kilo-vscode/webview-ui/src/types/messages/extension-messages.ts b/packages/kilo-vscode/webview-ui/src/types/messages/extension-messages.ts index 98628047a476..58748ae2af37 100644 --- a/packages/kilo-vscode/webview-ui/src/types/messages/extension-messages.ts +++ b/packages/kilo-vscode/webview-ui/src/types/messages/extension-messages.ts @@ -330,6 +330,7 @@ export interface AutocompleteSettingsLoadedMessage { enableAutoTrigger: boolean enableSmartInlineTaskKeybinding: boolean enableChatAutocomplete: boolean + provider: string model: string } } diff --git a/packages/opencode/src/kilocode/server/httpapi/groups/kilo-gateway.ts b/packages/opencode/src/kilocode/server/httpapi/groups/kilo-gateway.ts index d561d88325b8..df8a5747013c 100644 --- a/packages/opencode/src/kilocode/server/httpapi/groups/kilo-gateway.ts +++ b/packages/opencode/src/kilocode/server/httpapi/groups/kilo-gateway.ts @@ -138,6 +138,7 @@ export const OrganizationModes = Schema.Struct({ export const FimBody = Schema.Struct({ prefix: Schema.String, suffix: Schema.String, + provider: Schema.optional(Schema.String), model: Schema.optional(Schema.String), maxTokens: Schema.optional(Schema.Finite), temperature: Schema.optional(Schema.Finite), diff --git a/packages/opencode/src/kilocode/server/httpapi/handlers/kilo-gateway.ts b/packages/opencode/src/kilocode/server/httpapi/handlers/kilo-gateway.ts index 5c979617e1e2..27b99522c108 100644 --- a/packages/opencode/src/kilocode/server/httpapi/handlers/kilo-gateway.ts +++ b/packages/opencode/src/kilocode/server/httpapi/handlers/kilo-gateway.ts @@ -19,6 +19,7 @@ import { fetchOrganizationModes, fetchProfile, } from "@kilocode/kilo-gateway" +import { DIRECT_FIM_ENV, requestMistralFim, resolveFimTarget } from "@kilocode/kilo-gateway/fim" import { buildKiloHeaders } from "@kilocode/kilo-gateway" import { Effect } from "effect" import * as Stream from "effect/Stream" @@ -77,36 +78,50 @@ export const kiloGatewayHandlers = HttpApiBuilder.group(InstanceHttpApi, "kilo", }) const fim = Effect.fn("KiloGatewayHttpApi.fim")(function* (ctx: { payload: typeof FimBody.Type }) { - const info = yield* proxyAuth() - if (!info.auth) return yield* Effect.fail(new HttpApiError.Unauthorized({})) - if (!info.token) return yield* Effect.fail(new HttpApiError.Unauthorized({})) + const target = resolveFimTarget(ctx.payload.provider, ctx.payload.model) + const info = target.provider === "kilo" ? yield* proxyAuth() : undefined + const token = yield* Effect.gen(function* () { + if (target.provider === "kilo") return info?.token + const item = yield* auth.get(target.provider).pipe(Effect.mapError(() => new HttpApiError.Unauthorized({}))) + if (item?.type === "api") return item.key + return DIRECT_FIM_ENV[target.provider].map((key) => process.env[key]).find(Boolean) + }) + + if (target.provider === "kilo" && !info?.auth) return yield* Effect.fail(new HttpApiError.Unauthorized({})) + if (!token) return yield* Effect.fail(new HttpApiError.Unauthorized({})) const request = yield* HttpServerRequest.HttpServerRequest - const endpoint = new URL("fim/completions", `${KILO_API_BASE}/api/`) const signal = request.source instanceof Request ? AbortSignal.any([request.source.signal, AbortSignal.timeout(FIM_TIMEOUT_MS)]) : AbortSignal.timeout(FIM_TIMEOUT_MS) const response = yield* Effect.promise(async () => { try { - return await fetch(endpoint, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${info.token}`, - ...buildKiloHeaders(undefined, { kilocodeOrganizationId: info.organizationId }), - [HEADER_FEATURE]: "autocomplete", - }, - signal, - body: JSON.stringify({ - model: ctx.payload.model ?? "mistralai/codestral-2501", - prompt: ctx.payload.prefix, - suffix: ctx.payload.suffix, - max_tokens: ctx.payload.maxTokens ?? 256, - temperature: ctx.payload.temperature ?? 0.2, - stream: true, - }), - }) + const run = async (url: string): Promise => { + console.info(`[FIM] request provider=${target.provider} model=${target.model} url=${url}`) + return fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + ...(target.provider === "kilo" + ? buildKiloHeaders(undefined, { kilocodeOrganizationId: info?.organizationId }) + : {}), + ...(target.provider === "kilo" ? { [HEADER_FEATURE]: "autocomplete" } : {}), + }, + signal, + body: JSON.stringify({ + model: target.model, + prompt: ctx.payload.prefix, + suffix: ctx.payload.suffix, + max_tokens: ctx.payload.maxTokens ?? 256, + temperature: ctx.payload.temperature ?? 0.2, + stream: true, + }), + }) + } + if (target.provider === "mistral") return requestMistralFim(run) + return run(target.url) } catch (err) { if (err instanceof DOMException && err.name === "TimeoutError") return Response.json({ error: "FIM request timed out" }, { status: 504 }) diff --git a/packages/sdk/js/src/v2/gen/sdk.gen.ts b/packages/sdk/js/src/v2/gen/sdk.gen.ts index e8733856bf2c..950df5f0f4de 100644 --- a/packages/sdk/js/src/v2/gen/sdk.gen.ts +++ b/packages/sdk/js/src/v2/gen/sdk.gen.ts @@ -5761,6 +5761,7 @@ export class Kilo extends HeyApiClient { workspace?: string prefix?: string suffix?: string + provider?: string model?: string maxTokens?: number temperature?: number @@ -5776,6 +5777,7 @@ export class Kilo extends HeyApiClient { { in: "query", key: "workspace" }, { in: "body", key: "prefix" }, { in: "body", key: "suffix" }, + { in: "body", key: "provider" }, { in: "body", key: "model" }, { in: "body", key: "maxTokens" }, { in: "body", key: "temperature" }, diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index 066d3fd4c5e8..e351cb9aeb89 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -7739,6 +7739,7 @@ export type KiloFimData = { body?: { prefix: string suffix: string + provider?: string model?: string maxTokens?: number temperature?: number diff --git a/packages/sdk/openapi.json b/packages/sdk/openapi.json index 624758a6049d..224332dbfb53 100644 --- a/packages/sdk/openapi.json +++ b/packages/sdk/openapi.json @@ -10118,6 +10118,9 @@ "suffix": { "type": "string" }, + "provider": { + "type": "string" + }, "model": { "type": "string" },