From e3921bd7d82ec457b3a6ea293c53c861934a9985 Mon Sep 17 00:00:00 2001 From: ratchaw Date: Mon, 25 May 2026 18:22:53 +0100 Subject: [PATCH 1/9] feat(caching): add provider cache writes toggle Adds a project-level toggle that suppresses the gateway's automatic injection of cache_control / cachePoint markers into Anthropic and AWS Bedrock Claude requests. Caller-supplied markers still pass through; only the gateway's length-based heuristic auto-injection is gated. Lets users on sparse prompt patterns avoid paying the 1.25x/2x cache-write premium when their gap between requests exceeds the cache TTL. Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/api/src/routes/organization.ts | 1 + apps/api/src/routes/projects.ts | 28 +- apps/api/src/routes/v1-master.ts | 3 + apps/code/src/lib/api/v1.d.ts | 11 + .../caching/provider-cache-control.mdx | 4 + apps/gateway/src/chat/chat.ts | 8 +- apps/playground/src/lib/api/v1.d.ts | 11 + .../_components/caching-settings-rsc.tsx | 1 + .../components/settings/caching-settings.tsx | 40 + apps/ui/src/lib/api/v1.d.ts | 11 + apps/ui/src/types/settings.ts | 1 + ee/admin/src/lib/api/v1.d.ts | 11 + packages/actions/src/prepare-request-body.ts | 6 + .../src/transform-anthropic-messages.ts | 8 +- .../1779728096_needy_robbie_robertson.sql | 1 + .../migrations/meta/1779728096_snapshot.json | 18554 ++++++++++++++++ packages/db/migrations/meta/_journal.json | 7 + packages/db/src/cache-helpers.ts | 25 +- packages/db/src/schema.ts | 1 + 19 files changed, 18721 insertions(+), 11 deletions(-) create mode 100644 packages/db/migrations/1779728096_needy_robbie_robertson.sql create mode 100644 packages/db/migrations/meta/1779728096_snapshot.json diff --git a/apps/api/src/routes/organization.ts b/apps/api/src/routes/organization.ts index 9e862c12a7..080af0320c 100644 --- a/apps/api/src/routes/organization.ts +++ b/apps/api/src/routes/organization.ts @@ -62,6 +62,7 @@ const projectSchema = z.object({ organizationId: z.string(), cachingEnabled: z.boolean(), cacheDurationSeconds: z.number(), + providerCacheControlEnabled: z.boolean(), mode: z.enum(["api-keys", "credits", "hybrid"]), status: z.enum(["active", "inactive", "deleted"]).nullable(), }); diff --git a/apps/api/src/routes/projects.ts b/apps/api/src/routes/projects.ts index c3480f3c7e..5f627f60fc 100644 --- a/apps/api/src/routes/projects.ts +++ b/apps/api/src/routes/projects.ts @@ -20,6 +20,7 @@ const projectSchema = z.object({ organizationId: z.string(), cachingEnabled: z.boolean(), cacheDurationSeconds: z.number(), + providerCacheControlEnabled: z.boolean(), mode: z.enum(["api-keys", "credits", "hybrid"]), status: z.enum(["active", "inactive", "deleted"]).nullable(), }); @@ -29,6 +30,7 @@ const createProjectSchema = z.object({ organizationId: z.string().min(1), cachingEnabled: z.boolean().optional(), cacheDurationSeconds: z.number().min(10).max(31536000).optional(), + providerCacheControlEnabled: z.boolean().optional(), mode: z.enum(["api-keys", "credits", "hybrid"]).optional(), }); @@ -36,6 +38,7 @@ const updateProjectSchema = z.object({ name: z.string().min(1).max(255).optional(), cachingEnabled: z.boolean().optional(), cacheDurationSeconds: z.number().min(10).max(31536000).optional(), // Min 10 seconds, max 1 year + providerCacheControlEnabled: z.boolean().optional(), mode: z.enum(["api-keys", "credits", "hybrid"]).optional(), }); @@ -154,8 +157,13 @@ projects.openapi(updateProject, async (c) => { } const { id } = c.req.param(); - const { name, cachingEnabled, cacheDurationSeconds, mode } = - c.req.valid("json"); + const { + name, + cachingEnabled, + cacheDurationSeconds, + providerCacheControlEnabled, + mode, + } = c.req.valid("json"); const userOrgs = await db.query.userOrganization.findMany({ where: { @@ -201,6 +209,10 @@ projects.openapi(updateProject, async (c) => { updateData.cacheDurationSeconds = cacheDurationSeconds; } + if (providerCacheControlEnabled !== undefined) { + updateData.providerCacheControlEnabled = providerCacheControlEnabled; + } + if (mode !== undefined) { updateData.mode = mode; } @@ -234,6 +246,15 @@ projects.openapi(updateProject, async (c) => { new: cacheDurationSeconds, }; } + if ( + providerCacheControlEnabled !== undefined && + providerCacheControlEnabled !== project.providerCacheControlEnabled + ) { + changes.providerCacheControlEnabled = { + old: project.providerCacheControlEnabled, + new: providerCacheControlEnabled, + }; + } if (mode !== undefined && mode !== project.mode) { changes.mode = { old: project.mode, new: mode }; } @@ -305,6 +326,7 @@ export interface CreateProjectInput { name: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; mode?: "api-keys" | "credits" | "hybrid"; } @@ -318,6 +340,7 @@ export async function createProjectForOrg( name, cachingEnabled = false, cacheDurationSeconds = 60, + providerCacheControlEnabled = true, mode = "hybrid", } = input; @@ -372,6 +395,7 @@ export async function createProjectForOrg( organizationId, cachingEnabled, cacheDurationSeconds, + providerCacheControlEnabled, mode, }) .returning(); diff --git a/apps/api/src/routes/v1-master.ts b/apps/api/src/routes/v1-master.ts index 6fbb9c5dba..9a4e1a2c89 100644 --- a/apps/api/src/routes/v1-master.ts +++ b/apps/api/src/routes/v1-master.ts @@ -122,6 +122,7 @@ const projectSchema = z.object({ organizationId: z.string(), cachingEnabled: z.boolean(), cacheDurationSeconds: z.number(), + providerCacheControlEnabled: z.boolean(), mode: projectModeEnum, status: z.enum(["active", "inactive", "deleted"]).nullable(), }); @@ -130,6 +131,7 @@ const createProjectBody = z.object({ name: z.string().min(1).max(255), cachingEnabled: z.boolean().optional(), cacheDurationSeconds: z.number().min(10).max(31536000).optional(), + providerCacheControlEnabled: z.boolean().optional(), mode: projectModeEnum.optional(), }); @@ -313,6 +315,7 @@ const updateProjectBody = z name: z.string().min(1).max(255).optional(), cachingEnabled: z.boolean().optional(), cacheDurationSeconds: z.number().min(10).max(31536000).optional(), + providerCacheControlEnabled: z.boolean().optional(), mode: projectModeEnum.optional(), status: z.enum(["active", "inactive"]).optional(), }) diff --git a/apps/code/src/lib/api/v1.d.ts b/apps/code/src/lib/api/v1.d.ts index da3524aa3c..a4bdb38849 100644 --- a/apps/code/src/lib/api/v1.d.ts +++ b/apps/code/src/lib/api/v1.d.ts @@ -683,6 +683,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -707,6 +708,7 @@ export interface paths { name: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; }; @@ -728,6 +730,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -854,6 +857,7 @@ export interface paths { name?: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; /** @enum {string} */ @@ -877,6 +881,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -7135,6 +7140,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -7210,6 +7216,7 @@ export interface paths { name?: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; }; @@ -7232,6 +7239,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -7289,6 +7297,7 @@ export interface paths { organizationId: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; }; @@ -7310,6 +7319,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -8014,6 +8024,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ diff --git a/apps/docs/content/features/caching/provider-cache-control.mdx b/apps/docs/content/features/caching/provider-cache-control.mdx index 06bf7c7f25..0d91f2c362 100644 --- a/apps/docs/content/features/caching/provider-cache-control.mdx +++ b/apps/docs/content/features/caching/provider-cache-control.mdx @@ -26,6 +26,10 @@ For most users, prompt caching just works — you do not need to change your req Providers including OpenAI, Anthropic (when prompts cross the provider's minimum size), Google, DeepSeek, xAI, and Alibaba inspect incoming requests for shared prefixes and cache them automatically. LLM Gateway forwards the provider's cache metadata back to you in the response, and bills the cached portion at the model's `cached_input` rate. +For **Anthropic** and **AWS Bedrock Claude**, prompt caching is strictly opt-in via `cache_control` / `cachePoint` markers on the request body. To get automatic cache benefits without rewriting your requests, LLM Gateway injects those markers for you on long system and user messages by default. If you send long prompts sporadically — with gaps wider than the 5-minute TTL — you may want to disable this auto-injection, since you would otherwise pay the cache-write premium (1.25× input for 5m, 2× for 1h) without ever benefiting from a cache read. + +To disable, open **Project Settings → Caching → Provider Cache Writes** and turn off "Auto-mark long prompts as cacheable". Caller-supplied `cache_control` markers continue to pass through; only the gateway's automatic injection is suppressed. The change takes up to 5 minutes to take effect due to the project-settings cache. + To take advantage of automatic caching: - Put stable content (system prompt, instructions, tool definitions, long documents) at the **start** of your messages diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 0ba7b957f2..d8d1267653 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -3695,8 +3695,11 @@ chat.openapi(completions, async (c) => { } // Check if caching is enabled for this project - const { enabled: cachingEnabled, duration: cacheDuration } = - await isCachingEnabled(project.id); + const { + enabled: cachingEnabled, + duration: cacheDuration, + providerCacheControlEnabled, + } = await isCachingEnabled(project.id); let cacheKey: string | null = null; let streamingCacheKey: string | null = null; @@ -4319,6 +4322,7 @@ chat.openapi(completions, async (c) => { useResponsesApi, prompt_cache_key, prompt_cache_retention, + providerCacheControlEnabled, ); if (forceImageStreamUpstream) { diff --git a/apps/playground/src/lib/api/v1.d.ts b/apps/playground/src/lib/api/v1.d.ts index da3524aa3c..a4bdb38849 100644 --- a/apps/playground/src/lib/api/v1.d.ts +++ b/apps/playground/src/lib/api/v1.d.ts @@ -683,6 +683,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -707,6 +708,7 @@ export interface paths { name: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; }; @@ -728,6 +730,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -854,6 +857,7 @@ export interface paths { name?: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; /** @enum {string} */ @@ -877,6 +881,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -7135,6 +7140,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -7210,6 +7216,7 @@ export interface paths { name?: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; }; @@ -7232,6 +7239,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -7289,6 +7297,7 @@ export interface paths { organizationId: string; cachingEnabled?: boolean; cacheDurationSeconds?: number; + providerCacheControlEnabled?: boolean; /** @enum {string} */ mode?: "api-keys" | "credits" | "hybrid"; }; @@ -7310,6 +7319,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ @@ -8014,6 +8024,7 @@ export interface paths { organizationId: string; cachingEnabled: boolean; cacheDurationSeconds: number; + providerCacheControlEnabled: boolean; /** @enum {string} */ mode: "api-keys" | "credits" | "hybrid"; /** @enum {string|null} */ diff --git a/apps/ui/src/app/dashboard/[orgId]/[projectId]/settings/preferences/_components/caching-settings-rsc.tsx b/apps/ui/src/app/dashboard/[orgId]/[projectId]/settings/preferences/_components/caching-settings-rsc.tsx index 87ac36ab7d..ce323735a7 100644 --- a/apps/ui/src/app/dashboard/[orgId]/[projectId]/settings/preferences/_components/caching-settings-rsc.tsx +++ b/apps/ui/src/app/dashboard/[orgId]/[projectId]/settings/preferences/_components/caching-settings-rsc.tsx @@ -49,6 +49,7 @@ export const CachingSettingsRsc = async ({ preferences: { cachingEnabled: project.cachingEnabled, cacheDurationSeconds: project.cacheDurationSeconds, + providerCacheControlEnabled: project.providerCacheControlEnabled, }, }, }; diff --git a/apps/ui/src/components/settings/caching-settings.tsx b/apps/ui/src/components/settings/caching-settings.tsx index df5f7842eb..20b924c92e 100644 --- a/apps/ui/src/components/settings/caching-settings.tsx +++ b/apps/ui/src/components/settings/caching-settings.tsx @@ -31,6 +31,7 @@ const cachingFormSchema = z.object({ 31536000, "Cache duration must not exceed 31,536,000 seconds (1 year)", ), + providerCacheControlEnabled: z.boolean(), }); type CachingFormData = z.infer; @@ -58,6 +59,8 @@ export function CachingSettings({ initialData.preferences.preferences.cachingEnabled ?? false, cacheDurationSeconds: initialData.preferences.preferences.cacheDurationSeconds ?? 60, + providerCacheControlEnabled: + initialData.preferences.preferences.providerCacheControlEnabled ?? true, }, }); @@ -81,6 +84,7 @@ export function CachingSettings({ body: { cachingEnabled: data.cachingEnabled, cacheDurationSeconds: data.cacheDurationSeconds, + providerCacheControlEnabled: data.providerCacheControlEnabled, }, }); @@ -159,6 +163,42 @@ export function CachingSettings({ )} /> + + +
+

Provider Cache Writes

+

+ Anthropic and AWS Bedrock (Claude) only +

+
+ + ( + + + + +
+ Auto-mark long prompts as cacheable + + When enabled, the gateway automatically marks long system + and user prompts so the provider can cache them across + requests. Cache writes are billed at 1.25× (5m) or 2× (1h) + the input price; reads are 0.1×. Disable this if you send + long prompts sporadically — you'll pay the write + premium without benefiting from cache reads. Note: changing + this setting may take up to 5 minutes to take effect. + +
+ + )} + /> +