diff --git a/apps/api/src/routes/organization.ts b/apps/api/src/routes/organization.ts index 9e862c12a7..080af0320c 100644 --- a/apps/api/src/routes/organization.ts +++ b/apps/api/src/routes/organization.ts @@ -62,6 +62,7 @@ const projectSchema = z.object({ organizationId: z.string(), cachingEnabled: z.boolean(), cacheDurationSeconds: z.number(), + providerCacheControlEnabled: z.boolean(), mode: z.enum(["api-keys", "credits", "hybrid"]), status: z.enum(["active", "inactive", "deleted"]).nullable(), }); diff --git a/apps/api/src/routes/projects.ts b/apps/api/src/routes/projects.ts index c3480f3c7e..5f627f60fc 100644 --- a/apps/api/src/routes/projects.ts +++ b/apps/api/src/routes/projects.ts @@ -20,6 +20,7 @@ const projectSchema = z.object({ organizationId: z.string(), cachingEnabled: z.boolean(), cacheDurationSeconds: z.number(), + providerCacheControlEnabled: z.boolean(), mode: z.enum(["api-keys", "credits", "hybrid"]), status: z.enum(["active", "inactive", "deleted"]).nullable(), }); @@ -29,6 +30,7 @@ const createProjectSchema = z.object({ organizationId: z.string().min(1), cachingEnabled: z.boolean().optional(), cacheDurationSeconds: z.number().min(10).max(31536000).optional(), + providerCacheControlEnabled: z.boolean().optional(), mode: z.enum(["api-keys", "credits", "hybrid"]).optional(), }); @@ -36,6 +38,7 @@ const updateProjectSchema = z.object({ name: z.string().min(1).max(255).optional(), cachingEnabled: z.boolean().optional(), cacheDurationSeconds: z.number().min(10).max(31536000).optional(), // Min 10 seconds, max 1 year + providerCacheControlEnabled: z.boolean().optional(), mode: z.enum(["api-keys", "credits", "hybrid"]).optional(), }); @@ -154,8 +157,13 @@ projects.openapi(updateProject, async (c) => { } const { id } = c.req.param(); - const { name, cachingEnabled, cacheDurationSeconds, mode } = - c.req.valid("json"); + const { + name, + cachingEnabled, + cacheDurationSeconds, + providerCacheControlEnabled, + mode, + } = c.req.valid("json"); const userOrgs = await db.query.userOrganization.findMany({ where: { @@ -201,6 +209,10 @@ projects.openapi(updateProject, async (c) => { updateData.cacheDurationSeconds = cacheDurationSeconds; } + if (providerCacheControlEnabled !== undefined) { + updateData.providerCacheControlEnabled = providerCacheControlEnabled; + } + if (mode !== undefined) { updateData.mode = mode; } @@ -234,6 +246,15 @@ projects.openapi(updateProject, async (c) => { new: cacheDurationSeconds, }; } + if ( + providerCacheControlEnabled !== undefined && + providerCacheControlEnabled !== project.providerCacheControlEnabled + ) { + changes.providerCacheControlEnabled = { + old: project.providerCacheControlEnabled, + new: providerCacheControlEnabled, + }; + } if (mode !== undefined && mode !== project.mode) { changes.mode = { old: project.mode, new: mode }; } @@ -305,6 +326,7 @@ export interface CreateProjectInput { name: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; mode?: "api-keys" | "credits" | "hybrid"; } @@ -318,6 +340,7 @@ export async function createProjectForOrg( name, cachingEnabled = false, cacheDurationSeconds = 60, + providerCacheControlEnabled = true, mode = "hybrid", } = input; @@ -372,6 +395,7 @@ export async function createProjectForOrg( organizationId, cachingEnabled, cacheDurationSeconds, + providerCacheControlEnabled, mode, }) .returning(); diff --git a/apps/api/src/routes/v1-master.ts b/apps/api/src/routes/v1-master.ts index 6fbb9c5dba..9a4e1a2c89 100644 --- a/apps/api/src/routes/v1-master.ts +++ b/apps/api/src/routes/v1-master.ts @@ -122,6 +122,7 @@ const projectSchema = z.object({ organizationId: z.string(), cachingEnabled: z.boolean(), cacheDurationSeconds: z.number(), + providerCacheControlEnabled: z.boolean(), mode: projectModeEnum, status: z.enum(["active", "inactive", "deleted"]).nullable(), }); @@ -130,6 +131,7 @@ const createProjectBody = z.object({ name: z.string().min(1).max(255), cachingEnabled: z.boolean().optional(), cacheDurationSeconds: z.number().min(10).max(31536000).optional(), + providerCacheControlEnabled: z.boolean().optional(), mode: projectModeEnum.optional(), }); @@ -313,6 +315,7 @@ const updateProjectBody = z name: z.string().min(1).max(255).optional(), cachingEnabled: z.boolean().optional(), cacheDurationSeconds: z.number().min(10).max(31536000).optional(), + providerCacheControlEnabled: z.boolean().optional(), mode: projectModeEnum.optional(), status: z.enum(["active", "inactive"]).optional(), }) diff --git a/apps/code/src/lib/api/v1.d.ts b/apps/code/src/lib/api/v1.d.ts index 83257afddc..4ceeafef4c 100644 --- a/apps/code/src/lib/api/v1.d.ts +++ b/apps/code/src/lib/api/v1.d.ts @@ -684,6 +684,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -708,6 +709,7 @@ export interface paths { name: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; }; @@ -729,6 +731,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -855,6 +858,7 @@ export interface paths { name?: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; /** @enum {string} */ @@ -878,6 +882,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -7146,6 +7151,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -7221,6 +7227,7 @@ export interface paths { name?: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; }; @@ -7243,6 +7250,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -7300,6 +7308,7 @@ export interface paths { organizationId: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; }; @@ -7321,6 +7330,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -8025,6 +8035,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ diff --git a/apps/docs/content/features/caching/provider-cache-control.mdx b/apps/docs/content/features/caching/provider-cache-control.mdx index 06bf7c7f25..13d7ed0f61 100644 --- a/apps/docs/content/features/caching/provider-cache-control.mdx +++ b/apps/docs/content/features/caching/provider-cache-control.mdx @@ -26,6 +26,10 @@ For most users, prompt caching just works — you do not need to change your req Providers including OpenAI, Anthropic (when prompts cross the provider's minimum size), Google, DeepSeek, xAI, and Alibaba inspect incoming requests for shared prefixes and cache them automatically. LLM Gateway forwards the provider's cache metadata back to you in the response, and bills the cached portion at the model's `cached_input` rate. +For **Anthropic** and **AWS Bedrock Claude**, prompt caching is strictly opt-in via `cache_control` / `cachePoint` markers on the request body. To get automatic cache benefits without rewriting your requests, LLM Gateway injects those markers for you on long system and user messages by default. If you send long prompts sporadically — with gaps wider than the 5-minute TTL — you may want to disable this entirely, since you would otherwise pay the cache-write premium (1.25× input for 5m, 2× for 1h) without ever benefiting from a cache read. + +To disable, open **Project Settings → Caching → Provider Cache Writes** and turn off "Allow provider cache writes". When disabled, the gateway strips **all** `cache_control` markers from outgoing requests for the project — both the ones it adds automatically and any markers your client sends. This covers callers that always emit markers regardless of the user's request cadence (e.g. Claude Code, Cursor, Cline). The change takes up to 5 minutes to take effect due to the project-settings cache. + To take advantage of automatic caching: - Put stable content (system prompt, instructions, tool definitions, long documents) at the **start** of your messages diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 82a38579ea..5d24fe70bf 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -3790,10 +3790,13 @@ chat.openapi(completions, async (c) => { // Check if caching is enabled for this project. Dev plan orgs never get // gateway-level response caching — the feature is offered only on regular // (non-devpass) organizations. - const { enabled: cachingEnabled, duration: cacheDuration } = - organization.devPlan !== "none" - ? { enabled: false, duration: 0 } - : await isCachingEnabled(project.id); + const { + enabled: projectCachingEnabled, + duration: cacheDuration, + providerCacheControlEnabled, + } = await isCachingEnabled(project.id); + const cachingEnabled = + organization.devPlan !== "none" ? false : projectCachingEnabled; let cacheKey: string | null = null; let streamingCacheKey: string | null = null; @@ -4413,6 +4416,7 @@ chat.openapi(completions, async (c) => { useResponsesApi, prompt_cache_key, prompt_cache_retention, + providerCacheControlEnabled, ); } catch (e) { // Surface typed pre-upstream input errors in the activity feed as a @@ -4628,6 +4632,7 @@ chat.openapi(completions, async (c) => { webSearchEnabled: !!webSearchTool, excludedEnvKeyIndices: failedEnvKeyIndicesByProvider.get(retryKey), excludedProviderKeyIds: failedTrackedKeyIdsByProvider.get(retryKey), + providerCacheControlEnabled, }, ); } diff --git a/apps/gateway/src/chat/tools/resolve-provider-context.ts b/apps/gateway/src/chat/tools/resolve-provider-context.ts index 4699b7ddc1..84830b9cee 100644 --- a/apps/gateway/src/chat/tools/resolve-provider-context.ts +++ b/apps/gateway/src/chat/tools/resolve-provider-context.ts @@ -121,6 +121,7 @@ export interface ProviderContextOptions { webSearchEnabled: boolean; excludedEnvKeyIndices?: ReadonlySet; excludedProviderKeyIds?: ReadonlySet; + providerCacheControlEnabled: boolean; } interface ProjectInfo { @@ -472,6 +473,7 @@ export async function resolveProviderContext( useResponsesApi, options.prompt_cache_key, options.prompt_cache_retention, + options.providerCacheControlEnabled, ); // Post-validation of max_tokens in request body diff --git a/apps/playground/src/lib/api/v1.d.ts b/apps/playground/src/lib/api/v1.d.ts index 83257afddc..4ceeafef4c 100644 --- a/apps/playground/src/lib/api/v1.d.ts +++ b/apps/playground/src/lib/api/v1.d.ts @@ -684,6 +684,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -708,6 +709,7 @@ export interface paths { name: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; }; @@ -729,6 +731,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -855,6 +858,7 @@ export interface paths { name?: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; /** @enum {string} */ @@ -878,6 +882,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -7146,6 +7151,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -7221,6 +7227,7 @@ export interface paths { name?: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; }; @@ -7243,6 +7250,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -7300,6 +7308,7 @@ export interface paths { organizationId: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; }; @@ -7321,6 +7330,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -8025,6 +8035,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ diff --git a/apps/ui/src/app/dashboard/[orgId]/[projectId]/settings/preferences/_components/caching-settings-rsc.tsx b/apps/ui/src/app/dashboard/[orgId]/[projectId]/settings/preferences/_components/caching-settings-rsc.tsx index 87ac36ab7d..ce323735a7 100644 --- a/apps/ui/src/app/dashboard/[orgId]/[projectId]/settings/preferences/_components/caching-settings-rsc.tsx +++ b/apps/ui/src/app/dashboard/[orgId]/[projectId]/settings/preferences/_components/caching-settings-rsc.tsx @@ -49,6 +49,7 @@ export const CachingSettingsRsc = async ({ preferences: { cachingEnabled: project.cachingEnabled, cacheDurationSeconds: project.cacheDurationSeconds, + providerCacheControlEnabled: project.providerCacheControlEnabled, }, }, }; diff --git a/apps/ui/src/components/settings/caching-settings.tsx b/apps/ui/src/components/settings/caching-settings.tsx index df5f7842eb..0ca9838c74 100644 --- a/apps/ui/src/components/settings/caching-settings.tsx +++ b/apps/ui/src/components/settings/caching-settings.tsx @@ -31,6 +31,7 @@ const cachingFormSchema = z.object({ 31536000, "Cache duration must not exceed 31,536,000 seconds (1 year)", ), + providerCacheControlEnabled: z.boolean(), }); type CachingFormData = z.infer; @@ -58,6 +59,8 @@ export function CachingSettings({ initialData.preferences.preferences.cachingEnabled ?? false, cacheDurationSeconds: initialData.preferences.preferences.cacheDurationSeconds ?? 60, + providerCacheControlEnabled: + initialData.preferences.preferences.providerCacheControlEnabled ?? true, }, }); @@ -81,6 +84,7 @@ export function CachingSettings({ body: { cachingEnabled: data.cachingEnabled, cacheDurationSeconds: data.cacheDurationSeconds, + providerCacheControlEnabled: data.providerCacheControlEnabled, }, }); @@ -159,6 +163,45 @@ export function CachingSettings({ )} /> + + +
+

Provider Cache Writes

+

+ Anthropic and AWS Bedrock (Claude) only +

+
+ + ( + + + + +
+ Allow provider cache writes + + When disabled, the gateway strips all{" "} + cache_control markers from outgoing requests — + both the ones it adds automatically for long prompts and any + markers your client sends (e.g. Claude Code, Cursor, Cline). + Cache writes are billed at 1.25× (5m) or 2× (1h) the input + price; reads are 0.1×. Disable this if you send long prompts + sporadically with gaps longer than the 5-minute cache TTL — + otherwise you pay the write premium without ever benefiting + from a cache read. Note: changing this setting may take up + to 5 minutes to take effect. + +
+
+ )} + /> +