Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions apps/api/src/routes/dev-plans.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1534,6 +1534,7 @@ const getStatus = createRoute({
projectId: z.string().nullable(),
apiKey: z.string().nullable(),
devPlanAllowAllModels: z.boolean(),
devPlanServiceTier: z.enum(["default", "flex"]),
retentionLevel: z.enum(["retain", "none"]),
defaultRoutingStrategy: z.enum([
"auto",
Expand Down Expand Up @@ -1588,6 +1589,7 @@ devPlans.openapi(getStatus, async (c) => {
projectId: null,
apiKey: null,
devPlanAllowAllModels: false,
devPlanServiceTier: "default" as const,
retentionLevel: "none" as const,
defaultRoutingStrategy: "auto" as const,
});
Expand Down Expand Up @@ -1645,6 +1647,7 @@ devPlans.openapi(getStatus, async (c) => {
projectId,
apiKey,
devPlanAllowAllModels: personalOrg.devPlanAllowAllModels,
devPlanServiceTier: personalOrg.devPlanServiceTier,
retentionLevel: personalOrg.retentionLevel,
defaultRoutingStrategy,
});
Expand All @@ -1660,6 +1663,10 @@ const updateSettings = createRoute({
"application/json": {
schema: z.object({
devPlanAllowAllModels: z.boolean().optional(),
// Default processing tier for DevPass routing. "flex" saves
// plan credits by using cheaper flex processing where the
// selected provider supports it.
devPlanServiceTier: z.enum(["default", "flex"]).optional(),
retentionLevel: z.enum(["retain", "none"]).optional(),
// Coding plans optimize for prompt caching, so only the
// default weighted routing or the price strategy are allowed.
Expand All @@ -1676,6 +1683,7 @@ const updateSettings = createRoute({
schema: z.object({
success: z.boolean(),
devPlanAllowAllModels: z.boolean(),
devPlanServiceTier: z.enum(["default", "flex"]),
retentionLevel: z.enum(["retain", "none"]),
defaultRoutingStrategy: z.enum([
"auto",
Expand All @@ -1693,8 +1701,12 @@ const updateSettings = createRoute({

devPlans.openapi(updateSettings, async (c) => {
const user = c.get("user");
const { devPlanAllowAllModels, retentionLevel, defaultRoutingStrategy } =
c.req.valid("json");
const {
devPlanAllowAllModels,
devPlanServiceTier,
retentionLevel,
defaultRoutingStrategy,
} = c.req.valid("json");

if (!user) {
throw new HTTPException(401, {
Expand Down Expand Up @@ -1730,12 +1742,16 @@ devPlans.openapi(updateSettings, async (c) => {

const updateData: {
devPlanAllowAllModels?: boolean;
devPlanServiceTier?: "default" | "flex";
retentionLevel?: "retain" | "none";
} = {};

if (devPlanAllowAllModels !== undefined) {
updateData.devPlanAllowAllModels = devPlanAllowAllModels;
}
if (devPlanServiceTier !== undefined) {
updateData.devPlanServiceTier = devPlanServiceTier;
}
if (retentionLevel !== undefined) {
updateData.retentionLevel = retentionLevel;
}
Expand All @@ -1757,6 +1773,15 @@ devPlans.openapi(updateSettings, async (c) => {
new: devPlanAllowAllModels,
};
}
if (
devPlanServiceTier !== undefined &&
devPlanServiceTier !== personalOrg.devPlanServiceTier
) {
changes.devPlanServiceTier = {
old: personalOrg.devPlanServiceTier,
new: devPlanServiceTier,
};
}
if (
retentionLevel !== undefined &&
retentionLevel !== personalOrg.retentionLevel
Expand Down Expand Up @@ -1816,6 +1841,7 @@ devPlans.openapi(updateSettings, async (c) => {
success: true,
devPlanAllowAllModels:
devPlanAllowAllModels ?? personalOrg.devPlanAllowAllModels,
devPlanServiceTier: devPlanServiceTier ?? personalOrg.devPlanServiceTier,
retentionLevel: retentionLevel ?? personalOrg.retentionLevel,
defaultRoutingStrategy: effectiveRoutingStrategy,
});
Expand Down
1 change: 1 addition & 0 deletions apps/api/src/routes/organization.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ const organizationSchema = z.object({
devPlanBillingCycleStart: z.date().nullable(),
devPlanExpiresAt: z.date().nullable(),
devPlanAllowAllModels: z.boolean(),
devPlanServiceTier: z.enum(["default", "flex"]),
devPlanBillingOverride: z.boolean(),
// Chat Plans fields
chatPlan: z.enum(["none", "starter", "plus", "pro"]),
Expand Down
1 change: 1 addition & 0 deletions apps/code/src/app/dashboard/(main)/settings/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ export default function SettingsPage() {

<DevPlanSettings
devPlanAllowAllModels={devPlanStatus.devPlanAllowAllModels ?? false}
devPlanServiceTier={devPlanStatus.devPlanServiceTier ?? "default"}
retentionLevel={devPlanStatus.retentionLevel ?? "none"}
defaultRoutingStrategy={devPlanStatus.defaultRoutingStrategy ?? "auto"}
/>
Expand Down
78 changes: 78 additions & 0 deletions apps/code/src/app/dashboard/components/DevPlanSettings.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,23 @@ const ROUTING_OPTIONS: Array<{
{ value: "latency", label: "Lowest latency", allowed: false },
];

type ServiceTier = "default" | "flex";

const SERVICE_TIER_OPTIONS: Array<{ value: ServiceTier; label: string }> = [
{ value: "default", label: "Standard (recommended)" },
{ value: "flex", label: "Flex" },
];

interface DevPlanSettingsProps {
devPlanAllowAllModels: boolean;
devPlanServiceTier: ServiceTier;
retentionLevel: "retain" | "none";
defaultRoutingStrategy: RoutingStrategy;
}

export default function DevPlanSettings({
devPlanAllowAllModels: initialAllowAllModels,
devPlanServiceTier: initialServiceTier,
retentionLevel: initialRetentionLevel,
defaultRoutingStrategy: initialRoutingStrategy,
}: DevPlanSettingsProps) {
Expand All @@ -57,6 +66,10 @@ export default function DevPlanSettings({
);
const [isUpdatingRouting, setIsUpdatingRouting] = useState(false);

const [serviceTier, setServiceTier] =
useState<ServiceTier>(initialServiceTier);
const [isUpdatingServiceTier, setIsUpdatingServiceTier] = useState(false);

const [advancedOpen, setAdvancedOpen] = useState(false);

const updateSettingsMutation = api.useMutation(
Expand Down Expand Up @@ -110,6 +123,31 @@ export default function DevPlanSettings({
}
};

const handleServiceTierChange = async (value: string) => {
const tier = value as ServiceTier;
if (tier !== "default" && tier !== "flex") {
return;
}
const previous = serviceTier;
setServiceTier(tier);
setIsUpdatingServiceTier(true);
try {
await updateSettingsMutation.mutateAsync({
body: { devPlanServiceTier: tier },
});
toast.success(
tier === "flex"
? "Requests default to flex processing"
: "Requests default to standard processing",
);
} catch {
setServiceTier(previous);
toast.error("Failed to update service tier");
} finally {
setIsUpdatingServiceTier(false);
}
};

const handleRetentionToggle = async (checked: boolean) => {
setIsUpdatingRetention(true);
try {
Expand Down Expand Up @@ -180,6 +218,46 @@ export default function DevPlanSettings({
</div>
</div>

<div className="rounded-xl border p-5 space-y-4">
<div className="flex items-center justify-between gap-4">
<div className="space-y-0.5">
<Label htmlFor="service-tier" className="text-sm font-medium">
Default service tier
</Label>
<p className="text-xs text-muted-foreground">
Flex processing costs less and saves your plan credits, but
responses may be slower during peak demand. Only applied for
models that support it — everything else stays on standard
processing.{" "}
<a
href="https://docs.llmgateway.io/features/service-tiers"
target="_blank"
rel="noreferrer"
className="underline underline-offset-2"
>
Learn more
</a>
</p>
</div>
<Select
value={serviceTier}
onValueChange={handleServiceTierChange}
disabled={isUpdatingServiceTier}
>
<SelectTrigger id="service-tier" size="sm" className="w-[180px]">
<SelectValue />
</SelectTrigger>
<SelectContent>
{SERVICE_TIER_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
</div>

<div className="rounded-xl border p-5 space-y-4">
<div className="flex items-center justify-between gap-4">
<div className="space-y-0.5">
Expand Down
129 changes: 129 additions & 0 deletions apps/gateway/src/api.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1034,6 +1034,135 @@ describe("api", () => {
expect(json.metadata?.used_service_tier).toBeUndefined();
});

test("/v1/chat/completions applies the dev-plan default flex service tier", async () => {
await db.insert(tables.apiKey).values({
id: "token-id-devplan-flex-default",
token: "real-token-devplan-flex-default",
projectId: "project-id",
description: "Test API Key",
createdBy: "user-id",
});

await db.insert(tables.providerKey).values({
id: "provider-key-id-devplan-flex-default",
token: "sk-test-key",
provider: "openai",
organizationId: "org-id",
baseUrl: mockServerUrl,
});

await harness.setDevPlan({ devPlan: "pro", serviceTier: "flex" });

// No service_tier on the request — the org-level DevPass setting should
// route it to flex processing on the flex-capable openai mapping.
const res = await app.request("/v1/chat/completions", {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: "Bearer real-token-devplan-flex-default",
},
body: JSON.stringify({
model: "gpt-5.5",
messages: [{ role: "user", content: "Hello!" }],
}),
});

expect(res.status).toBe(200);
const json = await res.json();
expect(json.service_tier).toBe("flex");
// The tier came from the org default, not the request, so only the
// used tier is surfaced.
expect(json.metadata?.requested_service_tier).toBeUndefined();
expect(json.metadata?.used_service_tier).toBe("flex");

const logs = await waitForLogs(1);
expect(logs.length).toBe(1);
expect(logs[0].requestedServiceTier).toBeNull();
expect(logs[0].usedServiceTier).toBe("flex");
});

test("/v1/chat/completions lets an explicit service_tier win over the dev-plan default", async () => {
await db.insert(tables.apiKey).values({
id: "token-id-devplan-flex-override",
token: "real-token-devplan-flex-override",
projectId: "project-id",
description: "Test API Key",
createdBy: "user-id",
});

await db.insert(tables.providerKey).values({
id: "provider-key-id-devplan-flex-override",
token: "sk-test-key",
provider: "openai",
organizationId: "org-id",
baseUrl: mockServerUrl,
});

await harness.setDevPlan({ devPlan: "pro", serviceTier: "flex" });

const res = await app.request("/v1/chat/completions", {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: "Bearer real-token-devplan-flex-override",
},
body: JSON.stringify({
model: "gpt-5.5",
service_tier: "priority",
messages: [{ role: "user", content: "Hello!" }],
}),
});

expect(res.status).toBe(200);
const json = await res.json();
expect(json.service_tier).toBe("priority");
expect(json.metadata?.requested_service_tier).toBe("priority");
expect(json.metadata?.used_service_tier).toBe("priority");
});

test("/v1/chat/completions skips the dev-plan flex default for models without flex support", async () => {
await db.insert(tables.apiKey).values({
id: "token-id-devplan-flex-unsupported",
token: "real-token-devplan-flex-unsupported",
projectId: "project-id",
description: "Test API Key",
createdBy: "user-id",
});

await db.insert(tables.providerKey).values({
id: "provider-key-id-devplan-flex-unsupported",
token: "sk-test-key",
provider: "openai",
organizationId: "org-id",
baseUrl: mockServerUrl,
});

await harness.setDevPlan({
devPlan: "pro",
allowAllModels: true,
serviceTier: "flex",
});

// gpt-4o has no flex-capable mapping — the default must fall back to
// standard processing instead of failing the request.
const res = await app.request("/v1/chat/completions", {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: "Bearer real-token-devplan-flex-unsupported",
},
body: JSON.stringify({
model: "gpt-4o",
messages: [{ role: "user", content: "Hello!" }],
}),
});

expect(res.status).toBe(200);
const json = await res.json();
expect(json.metadata?.requested_service_tier).toBeUndefined();
expect(json.metadata?.used_service_tier).toBeUndefined();
});

test("/v1/chat/completions streams service tier in the final usage chunk", async () => {
await db.insert(tables.apiKey).values({
id: "token-id-service-tier-stream",
Expand Down
Loading
Loading