feat(ai): add native Vercel AI Gateway language models

packages/ai/README.md

@@ -1283,7 +1283,7 @@ const gateway = CloudflareAIGateway.configure({
 }).model("workers-ai/@cf/meta/llama-3.1-8b-instruct")
 ```
 
-Included LLM providers: OpenAI, Anthropic, Google (Gemini), Google Vertex, Amazon Bedrock, Azure OpenAI, Baseten, Cerebras, Cohere, Cloudflare AI Gateway, Cloudflare Workers AI, DeepInfra, DeepSeek, Fireworks, Groq, Mistral, OpenRouter, TogetherAI, and xAI. Z.ai currently exposes image generation. Generic Chat Completions, Responses, and Anthropic Messages-compatible entrypoints support custom endpoints.
+Included LLM providers: OpenAI, Anthropic, Google (Gemini), Google Vertex, Amazon Bedrock, Azure OpenAI, Baseten, Cerebras, Cohere, Cloudflare AI Gateway, Cloudflare Workers AI, DeepInfra, DeepSeek, Fireworks, Groq, Mistral, OpenRouter, TogetherAI, Vercel AI Gateway, and xAI. Z.ai currently exposes image generation. Generic Chat Completions, Responses, and Anthropic Messages-compatible entrypoints support custom endpoints.
 
 Each named provider owns its module, endpoint, authentication, and route setup. Providers with the same wire format compose the shared protocol directly:
 

packages/ai/src/cache-policy.ts

@@ -48,21 +48,22 @@ const RESPECTS_INLINE_HINTS = new Set([
   "meta-messages",
   "minimax-messages",
   "moonshot-messages",
+  "vercel-ai-gateway-messages",
   "zai-coding-messages",
   "bedrock-converse",
   "openrouter",
   "digitalocean",
 ])
 
-// OpenRouter upstreams other than Anthropic and Alibaba Qwen cache without breakpoints. Gemini uses only the last
-// breakpoint, so a conversation-tail breakpoint writes a new cache every step and costs more than none. Qwen ignores
-// breakpoints on tool definitions and caches tools with the system prompt.
+// OpenRouter and Vercel AI Gateway upstreams other than Anthropic and Alibaba Qwen cache without breakpoints.
+// Gemini uses only the last breakpoint, so a conversation-tail breakpoint writes a new cache every step and costs
+// more than none. Qwen ignores breakpoints on tool definitions and caches tools with the system prompt.
 const QWEN: CachePolicyObject = { system: true, messages: { tail: 1 } }
-const openRouterPolicy = (modelID: string): CachePolicyObject => {
+const gatewayPolicy = (modelID: string): CachePolicyObject => {
   // `~anthropic/claude-sonnet-latest` style IDs are OpenRouter aliases for the latest model in a family.
   const id = modelID.replace(/^~/, "")
   if (id.startsWith("anthropic/")) return AUTO
-  if (id.startsWith("qwen/")) return QWEN
+  if (id.startsWith("qwen/") || id.startsWith("alibaba/qwen")) return QWEN
   return NONE
 }
 
@@ -170,11 +171,13 @@ const countHints = (request: LLMRequest) =>
   )
 
 export const applyCachePolicy = (request: LLMRequest): LLMRequest => {
-  if (!RESPECTS_INLINE_HINTS.has(request.model.route.id)) return request
+  const route = request.model.route.id
+  if (!RESPECTS_INLINE_HINTS.has(route)) return request
   const policy =
-    request.model.route.id === "openrouter" && (request.cache === undefined || request.cache === "auto")
-      ? openRouterPolicy(request.model.id)
-      : request.model.route.id === "alibaba-chat" && (request.cache === undefined || request.cache === "auto")
+    (route === "openrouter" || route === "vercel-ai-gateway-messages") &&
+    (request.cache === undefined || request.cache === "auto")
+      ? gatewayPolicy(request.model.id)
+      : route === "alibaba-chat" && (request.cache === undefined || request.cache === "auto")
         ? request.model.id.toLowerCase().startsWith("qwen")
           ? QWEN
           : NONE

packages/ai/src/protocols/utils/gateway-protocol.ts

@@ -0,0 +1,90 @@
+import { Effect, Option, Schema } from "effect"
+import { Protocol } from "../../route/protocol.js"
+import { LLMEvent, mergeJsonRecords, type AIError, type LLMRequest } from "../../schema/index.js"
+import { JsonObject, lenient } from "../shared.js"
+
+interface ParserState<Inner> {
+  readonly inner: Inner
+  readonly gateway?: Record<string, unknown>
+}
+
+const GatewayHolder = Schema.Struct({
+  provider_metadata: lenient(
+    Schema.Struct({
+      gateway: lenient(JsonObject),
+    }),
+  ),
+})
+
+const GatewayEvent = Schema.Struct({
+  ...GatewayHolder.fields,
+  response: lenient(GatewayHolder),
+  choices: lenient(
+    Schema.Array(
+      Schema.Struct({
+        delta: lenient(GatewayHolder),
+      }),
+    ),
+  ),
+})
+const decodeGatewayEvent = Schema.decodeUnknownOption(GatewayEvent)
+
+function attachGatewayMetadata(
+  events: ReadonlyArray<LLMEvent>,
+  gateway: Record<string, unknown> | undefined,
+): ReadonlyArray<LLMEvent> {
+  if (!gateway || !events.some(LLMEvent.is.finish)) return events
+  return events.map((event) =>
+    LLMEvent.is.finish(event) ? { ...event, providerMetadata: { ...event.providerMetadata, gateway } } : event,
+  )
+}
+
+export function gatewayProtocol<Body, Event, State>(
+  protocol: Protocol<Body, string, Event, State>,
+  input: {
+    readonly id: string
+    readonly prepare: (
+      request: LLMRequest,
+    ) => Effect.Effect<{ readonly request: LLMRequest; readonly body: Record<string, unknown> }, AIError>
+  },
+) {
+  const initial = (request: LLMRequest): ParserState<State> => ({
+    inner: protocol.stream.initial(request),
+  })
+  const onHalt = protocol.stream.onHalt
+
+  return Protocol.make({
+    id: input.id,
+    body: {
+      schema: JsonObject,
+      from: Effect.fnUntraced(function* (request: LLMRequest) {
+        const prepared = yield* input.prepare(request)
+        const body = yield* protocol.body.from(prepared.request)
+        return { ...body, ...prepared.body }
+      }),
+    },
+    supportsEffortUpdates: protocol.supportsEffortUpdates,
+    sanitizer: protocol.sanitizer,
+    stream: {
+      event: protocol.stream.event,
+      initial,
+      step: Effect.fnUntraced(function* (state: ParserState<State>, event: Event) {
+        const decoded = Option.getOrUndefined(decodeGatewayEvent(event))
+        const gateway = decoded
+          ? mergeJsonRecords(
+              state.gateway,
+              decoded.provider_metadata?.gateway,
+              decoded.response?.provider_metadata?.gateway,
+              ...(decoded.choices ?? []).map((choice) => choice.delta?.provider_metadata?.gateway),
+            )
+          : state.gateway
+        const [inner, events] = yield* protocol.stream.step(state.inner, event)
+        return [{ inner, gateway }, attachGatewayMetadata(events, gateway)] as const
+      }),
+      terminal: protocol.stream.terminal,
+      onHalt: onHalt
+        ? (state) => onHalt(state.inner).pipe(Effect.map((events) => attachGatewayMetadata(events, state.gateway)))
+        : undefined,
+    },
+  })
+}