Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
6520704
feat(gateway): dynamic routes via dynamic/<name>
steebchen Aug 5, 2026
0886704
fix(dynamic-routes): address PR review feedback
steebchen Aug 5, 2026
3754601
feat(ui): visual flow editor for dynamic routes
steebchen Aug 7, 2026
a3dcc20
fix(dynamic-routes): visual editor review feedback
steebchen Aug 7, 2026
3e9695f
Merge remote-tracking branch 'origin/main' into cloudflare-dynamic-ro…
steebchen Aug 7, 2026
4304c16
docs: dynamic routes changelog and feature page
steebchen Aug 7, 2026
17faf64
docs(learn): add dynamic routes knowledge base page
steebchen Aug 7, 2026
48f755c
chore(db): reset migrations to origin/main before merge
steebchen Aug 7, 2026
fc2f272
Merge remote-tracking branch 'origin/main' into cloudflare-dynamic-ro…
steebchen Aug 7, 2026
28917cb
chore(db): regenerate dynamic routes migration
steebchen Aug 7, 2026
32f2f9d
Optimised images with calibre/image-actions
steebchen Aug 7, 2026
62213a6
Optimised images with calibre/image-actions
steebchen Aug 7, 2026
da0122c
Optimised images with calibre/image-actions
steebchen Aug 7, 2026
6a71908
Optimised images with calibre/image-actions
steebchen Aug 7, 2026
9595a48
feat(ui): rich model/provider pickers in flow editor
steebchen Aug 7, 2026
4ad2238
Merge remote-tracking branch 'origin/cloudflare-dynamic-routing' into…
steebchen Aug 7, 2026
7e9349c
test(dynamic-routes): integration coverage + raw-body fix
steebchen Aug 7, 2026
97bbc83
chore(db): reset migrations to origin/main before merge
steebchen Aug 8, 2026
49f947d
Merge remote-tracking branch 'origin/main' into cloudflare-dynamic-ro…
steebchen Aug 8, 2026
cdc4941
chore(db): regenerate dynamic routes migration
steebchen Aug 8, 2026
8848ac0
Merge remote-tracking branch 'origin/main' into cloudflare-dynamic-ro…
steebchen Aug 8, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
524 changes: 524 additions & 0 deletions apps/api/src/routes/dynamic-routes.ts

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions apps/api/src/routes/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import { chats } from "./chats.js";
import { customModels } from "./custom-models.js";
import { devPlanCancellationFeedback } from "./dev-plan-cancellation-feedback.js";
import { devPlans } from "./dev-plans.js";
import { dynamicRoutes } from "./dynamic-routes.js";
import { guardrails } from "./guardrails.js";
import keysApi from "./keys-api.js";
import keysProvider from "./keys-provider.js";
Expand Down Expand Up @@ -88,6 +89,7 @@ routes.route("/model-ratings", modelRatings);
routes.route("/model-survey", modelSurvey);
routes.route("/guardrails", guardrails);
routes.route("/routing-config", routingConfig);
routes.route("/dynamic-routes", dynamicRoutes);
routes.route("/custom-models", customModels);
routes.route("/video", video);
routes.route("/sso", sso);
9 changes: 8 additions & 1 deletion apps/api/src/routes/keys-provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ import { isStealthProvider, providers } from "@llmgateway/models";
import {
CUSTOM_PROVIDER_NAME_MESSAGE,
CUSTOM_PROVIDER_NAME_REGEX,
RESERVED_CUSTOM_PROVIDER_NAME_MESSAGE,
RESERVED_CUSTOM_PROVIDER_NAMES,
} from "@llmgateway/shared";
import { getApiKeyFingerprint } from "@llmgateway/shared/api-key-hash";
import { maskToken } from "@llmgateway/shared/mask-token";
Expand Down Expand Up @@ -176,7 +178,12 @@ export function toPublicProviderKey(row: ProviderKeyRow) {
// model strings, so it must stay URL-safe and unique within the organization.
export const customProviderNameSchema = z
.string()
.regex(CUSTOM_PROVIDER_NAME_REGEX, CUSTOM_PROVIDER_NAME_MESSAGE);
.regex(CUSTOM_PROVIDER_NAME_REGEX, CUSTOM_PROVIDER_NAME_MESSAGE)
.refine(
(name) =>
!(RESERVED_CUSTOM_PROVIDER_NAMES as readonly string[]).includes(name),
RESERVED_CUSTOM_PROVIDER_NAME_MESSAGE,
);

export async function assertCustomProviderNameAvailable(
organizationId: string,
Expand Down
2 changes: 1 addition & 1 deletion apps/api/src/routes/routing-config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import type {

export const routingConfig = new OpenAPIHono<ServerTypes>();

async function checkProjectEnterpriseAccess(
export async function checkProjectEnterpriseAccess(
userId: string,
projectId: string,
): Promise<{
Expand Down
104 changes: 102 additions & 2 deletions apps/gateway/src/chat/chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ import {
shouldBillCancelledRequests,
zeroInferenceCosts,
} from "@/lib/costs.js";
import { getPublishedDynamicRoute } from "@/lib/dynamic-route-loader.js";
import {
assertOriginAllowed,
assertTestWalletModelAllowed,
Expand Down Expand Up @@ -204,6 +205,10 @@ import { countInputImages } from "./tools/count-input-images.js";
import { createLogEntry } from "./tools/create-log-entry.js";
import { estimateTokensFromContent } from "./tools/estimate-tokens-from-content.js";
import { estimateTokens } from "./tools/estimate-tokens.js";
import {
type DynamicRouteEvaluation,
evaluateDynamicRoute,
} from "./tools/evaluate-dynamic-route.js";
import {
extractAwsBedrockHttpError,
extractAwsBedrockStreamError,
Expand Down Expand Up @@ -3041,6 +3046,71 @@ chat.openapi(completions, async (c) => {
}
}

// Resolve a named dynamic route ("dynamic/<name>") to its target model and
// optional provider restriction. The route was tagged in parseModelInput and
// piggybacks the "auto" placeholder model until here, where project/org
// context needed by conditional and percentage nodes is available. The
// resolved target then flows through the auto-routing block below, which
// narrows candidates to the route's model/providers instead of the auto
// allowlist.
let dynamicRouteSelection:
| {
name: string;
version: number;
model: string;
providers?: string[];
path: string[];
}
| undefined;
if (parseResult.dynamicRouteName) {
const dynamicRouteName = parseResult.dynamicRouteName;
if (organization.plan !== "enterprise") {
throw new HTTPException(403, {
message:
"Dynamic routes are only available on the enterprise plan. Contact us at contact@llmgateway.io to upgrade.",
});
}
const publishedRoute = await getPublishedDynamicRoute(
project.id,
dynamicRouteName,
);
if (!publishedRoute) {
throw new HTTPException(404, {
message: `Dynamic route "${dynamicRouteName}" not found, disabled, or has no published version`,
});
}
let evaluation: DynamicRouteEvaluation;
try {
evaluation = evaluateDynamicRoute(publishedRoute.graph, {
getHeader: (name) => c.req.header(name),
body: validationResult.data as unknown as Record<string, unknown>,
metadata: {
orgId: project.organizationId,
projectId: project.id,
apiKeyId: apiKey.id,
plan: organization.plan,
},
splitKey: sessionId ?? requestId,
});
} catch (error) {
throw new HTTPException(400, {
message: `Dynamic route "${dynamicRouteName}" evaluation failed: ${toError(error).message}`,
});
}
if (evaluation.status === "end") {
throw new HTTPException(400, {
message: `Dynamic route "${dynamicRouteName}" ended without resolving to a model`,
});
}
dynamicRouteSelection = {
name: publishedRoute.name,
version: publishedRoute.version,
model: evaluation.model,
providers: evaluation.providers,
path: evaluation.path,
};
}

// Apply routing logic after apiKey and project are available
if (
(usedProvider === "llmgateway" && usedInternalModel === "auto") ||
Expand Down Expand Up @@ -3158,9 +3228,21 @@ chat.openapi(completions, async (c) => {
continue;
}

// A dynamic route already resolved the target model, so candidates
// narrow to exactly that model instead of the auto allowlist.
// free_models_only (including test-mode end-user wallets) still
// applies: a route resolving to a paid model must not select it.
if (dynamicRouteSelection) {
if (modelDef.id !== dynamicRouteSelection.model) {
continue;
}
if (effectiveFreeModelsOnly && !("free" in modelDef && modelDef.free)) {
continue;
}
}
// When free_models_only is true, only consider models marked as free
// Otherwise, only consider hardcoded allowed models
if (effectiveFreeModelsOnly) {
else if (effectiveFreeModelsOnly) {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if (!("free" in modelDef && modelDef.free)) {
continue;
Comment on lines 3323 to 3327
}
Expand Down Expand Up @@ -3216,7 +3298,9 @@ chat.openapi(completions, async (c) => {
(provider) =>
availableProviders.includes(provider.providerId) &&
(!candidateAllowedProviders ||
candidateAllowedProviders.includes(provider.providerId)),
candidateAllowedProviders.includes(provider.providerId)) &&
(!dynamicRouteSelection?.providers ||
dynamicRouteSelection.providers.includes(provider.providerId)),
);
const cachedFilteredProviders = isDevPlan
? availableModelProviders.filter(providerSupportsCachedInput)
Expand Down Expand Up @@ -3374,6 +3458,15 @@ chat.openapi(completions, async (c) => {
message: complianceBlockMessage(modelInfo.id),
});
}
// A dynamic route must never silently fall back to the hardcoded
// default model — fail with the route's resolved target instead.
if (dynamicRouteSelection) {
throw new HTTPException(400, {
message: effectiveFreeModelsOnly
? `Dynamic route "${dynamicRouteSelection.name}" resolved to model "${dynamicRouteSelection.model}" which is not available with free_models_only`
: `Dynamic route "${dynamicRouteSelection.name}" resolved to model "${dynamicRouteSelection.model}" but no matching provider is currently available for this request`,
});
}
if (effectiveFreeModelsOnly) {
// If free_models_only is true but no suitable model found, return error
throw new HTTPException(400, {
Expand All @@ -3392,6 +3485,13 @@ chat.openapi(completions, async (c) => {
usedExternalId = "claude-haiku-4-5";
usedProvider = "anthropic";
}
if (dynamicRouteSelection && routingMetadata) {
routingMetadata.dynamicRoute = {
name: dynamicRouteSelection.name,
version: dynamicRouteSelection.version,
path: dynamicRouteSelection.path,
};
}
// Update modelInfo to the selected model so retry/fallback logic can find
// alternative providers. Without this, modelInfo still points to the "auto"
// model definition which only has "llmgateway" as a provider, preventing retries.
Expand Down
Loading
Loading