Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apps/gateway/src/chat/chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8865,6 +8865,7 @@ chat.openapi(completions, async (c) => {
{
const served = resolveServedServiceTier({
trafficType: data?.usageMetadata?.trafficType,
serviceTierBody: data?.usageMetadata?.serviceTier,
});
if (served) {
servedServiceTier = served;
Expand Down Expand Up @@ -12028,6 +12029,7 @@ chat.openapi(completions, async (c) => {
{
const served = resolveServedServiceTier({
trafficType: json?.usageMetadata?.trafficType,
serviceTierBody: json?.usageMetadata?.serviceTier,
});
if (served) {
servedServiceTier = served;
Expand Down
27 changes: 26 additions & 1 deletion packages/actions/src/apply-google-service-tier.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,35 @@ describe("resolveServedServiceTier", () => {
).toBeNull();
});

it("maps the AI Studio usageMetadata.serviceTier body field to a tier id", () => {
// Streaming AI Studio responses omit the header and carry the served tier
// in the body instead.
expect(resolveServedServiceTier({ serviceTierBody: "flex" })).toBe("flex");
expect(resolveServedServiceTier({ serviceTierBody: "priority" })).toBe(
"priority",
);
expect(
resolveServedServiceTier({ serviceTierBody: "standard" }),
).toBeNull();
});

it("prefers the header but falls back to the body field", () => {
expect(
resolveServedServiceTier({
serviceTierHeader: null,
serviceTierBody: "flex",
}),
).toBe("flex");
});

it("returns null when no signals are present", () => {
expect(resolveServedServiceTier({})).toBeNull();
expect(
resolveServedServiceTier({ trafficType: null, serviceTierHeader: null }),
resolveServedServiceTier({
trafficType: null,
serviceTierHeader: null,
serviceTierBody: null,
}),
).toBeNull();
});
});
Expand Down
15 changes: 11 additions & 4 deletions packages/actions/src/apply-google-service-tier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,14 +98,19 @@ export function providerKeyBaseUrlSupportsServiceTier(
* - Vertex AI reports the served tier in `usageMetadata.trafficType`
* (`ON_DEMAND_PRIORITY` / `ON_DEMAND_FLEX` / `ON_DEMAND`).
* - The Gemini Developer API (AI Studio / glacier) reports it in the
* `x-gemini-service-tier` response header (`priority` / `flex` / `standard`).
* `x-gemini-service-tier` response header (`priority` / `flex` / `standard`)
* on unary responses, but streaming responses omit that header and instead
* carry it in the body as `usageMetadata.serviceTier` (`flex` / `priority` /
* `standard`). Both are checked so streaming requests aren't misread as
* standard.
*
* Billing keys off this value rather than the requested tier so a downgraded
* request is charged at the rate it actually ran at.
*/
export function resolveServedServiceTier(signals: {
trafficType?: string | null;
serviceTierHeader?: string | null;
serviceTierBody?: string | null;
}): "flex" | "priority" | null {
const trafficType = signals.trafficType?.toUpperCase();
if (trafficType === "ON_DEMAND_PRIORITY") {
Expand All @@ -114,11 +119,13 @@ export function resolveServedServiceTier(signals: {
if (trafficType === "ON_DEMAND_FLEX") {
return "flex";
}
const header = signals.serviceTierHeader?.toLowerCase();
if (header === "priority") {
const tier = (
signals.serviceTierHeader ?? signals.serviceTierBody
)?.toLowerCase();
if (tier === "priority") {
return "priority";
}
if (header === "flex") {
if (tier === "flex") {
return "flex";
}
return null;
Expand Down
Loading