diff --git a/changelog.d/features/10590-google-ai-studio-tts.md b/changelog.d/features/10590-google-ai-studio-tts.md new file mode 100644 index 00000000000..9fcf9319196 --- /dev/null +++ b/changelog.d/features/10590-google-ai-studio-tts.md @@ -0,0 +1 @@ +- Added Google AI Studio Gemini batch text-to-speech support through `POST /v1/audio/speech`. diff --git a/open-sse/config/audioRegistry.ts b/open-sse/config/audioRegistry.ts index aaa727fc469..9cd89cb7699 100644 --- a/open-sse/config/audioRegistry.ts +++ b/open-sse/config/audioRegistry.ts @@ -287,6 +287,19 @@ export const AUDIO_TRANSLATION_PROVIDERS: Record = { }; export const AUDIO_SPEECH_PROVIDERS: Record = { + google: { + id: "google", + credentialProviderId: "gemini", + baseUrl: "https://generativelanguage.googleapis.com/v1beta/models", + authType: "apikey", + authHeader: "x-goog-api-key", + format: "gemini-tts", + models: [ + { id: "gemini-3.1-flash-tts-preview", name: "Gemini 3.1 Flash TTS" }, + { id: "gemini-2.5-flash-preview-tts", name: "Gemini 2.5 Flash TTS" }, + { id: "gemini-2.5-pro-preview-tts", name: "Gemini 2.5 Pro TTS" }, + ], + }, vertex: { id: "vertex", baseUrl: "https://us-central1-aiplatform.googleapis.com/v1", diff --git a/open-sse/executors/geminiTts.ts b/open-sse/executors/geminiTts.ts new file mode 100644 index 00000000000..de235a58eea --- /dev/null +++ b/open-sse/executors/geminiTts.ts @@ -0,0 +1,81 @@ +import { Buffer } from "node:buffer"; +import { extractInlineAudio, parsePcmSampleRate, pcmToWav } from "./vertexMedia.ts"; +import { CORS_HEADERS } from "../utils/cors.ts"; +import { upstreamErrorResponse } from "../utils/audioResponse.ts"; +import { errorResponse } from "../utils/error.ts"; + +type GeminiTtsCredentials = { + apiKey?: string | null; + accessToken?: string | null; +}; + +export class GeminiTtsUpstreamError extends Error { + constructor( + public readonly response: Response, + public readonly body: string + ) { + super(`Gemini TTS upstream error (${response.status})`); + } +} + +export async function geminiGenerateSpeech( + credentials: GeminiTtsCredentials, + options: { model: string; text: string; voice: string } +): Promise { + const headers: Record = { "Content-Type": "application/json" }; + if (credentials.apiKey) { + headers["x-goog-api-key"] = credentials.apiKey; + } else if (credentials.accessToken) { + headers.Authorization = `Bearer ${credentials.accessToken}`; + } + + const response = await fetch( + `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(options.model)}:generateContent`, + { + method: "POST", + headers, + body: JSON.stringify({ + contents: [{ parts: [{ text: options.text }] }], + generationConfig: { + responseModalities: ["AUDIO"], + speechConfig: { + voiceConfig: { + prebuiltVoiceConfig: { voiceName: options.voice }, + }, + }, + }, + }), + } + ); + if (!response.ok) { + throw new GeminiTtsUpstreamError(response, await response.text()); + } + + const inline = extractInlineAudio(await response.json()); + if (!inline) throw new Error("Gemini TTS response did not contain audio data"); + return pcmToWav(Buffer.from(inline.base64, "base64"), parsePcmSampleRate(inline.mimeType)); +} + +export async function handleGeminiTtsSpeech( + credentials: GeminiTtsCredentials, + options: { model: string; text: string; voice?: unknown } +): Promise { + try { + const wav = await geminiGenerateSpeech(credentials, { + model: options.model, + text: options.text, + voice: + typeof options.voice === "string" && options.voice.trim() ? options.voice.trim() : "Kore", + }); + return new Response(new Uint8Array(wav), { + status: 200, + headers: { ...CORS_HEADERS, "Content-Type": "audio/wav" }, + }); + } catch (error) { + if (error instanceof GeminiTtsUpstreamError) { + return upstreamErrorResponse(error.response, error.body); + } + const message = error instanceof Error ? error.message : String(error); + return errorResponse(500, `Speech request failed: ${message}`); + } +} diff --git a/open-sse/executors/vertexMedia.ts b/open-sse/executors/vertexMedia.ts index c21becedcd5..390b087a320 100644 --- a/open-sse/executors/vertexMedia.ts +++ b/open-sse/executors/vertexMedia.ts @@ -156,13 +156,13 @@ export function pcmToWav( return Buffer.concat([header, pcm]); } -function parseSampleRate(mimeType: string | undefined): number { +export function parsePcmSampleRate(mimeType: string | undefined): number { if (!mimeType) return 24000; const match = /rate=(\d+)/i.exec(mimeType); return match ? parseInt(match[1], 10) : 24000; } -function extractInlineAudio( +export function extractInlineAudio( data: unknown ): { base64: string; mimeType: string } | null { const parts = (data as { candidates?: Array<{ content?: { parts?: unknown[] } }> })?.candidates?.[0] @@ -215,7 +215,7 @@ export async function vertexGenerateSpeech( const inline = extractInlineAudio(data); if (!inline) throw new Error("Vertex TTS returned no audio content"); const pcm = Buffer.from(inline.base64, "base64"); - return { audio: pcmToWav(pcm, parseSampleRate(inline.mimeType)), contentType: "audio/wav" }; + return { audio: pcmToWav(pcm, parsePcmSampleRate(inline.mimeType)), contentType: "audio/wav" }; } /** Gemini transcription (audio → text). `audioBase64` is the raw file bytes, base64-encoded. */ diff --git a/open-sse/handlers/audioSpeech.ts b/open-sse/handlers/audioSpeech.ts index 9dc499a1e47..efcf369cf99 100644 --- a/open-sse/handlers/audioSpeech.ts +++ b/open-sse/handlers/audioSpeech.ts @@ -21,6 +21,7 @@ import { getSpeechProvider, parseSpeechModel } from "../config/audioRegistry.ts" import { buildAuthHeaders } from "../config/registryUtils.ts"; import { kieExecutor } from "../executors/kie.ts"; import { vertexGenerateSpeech } from "../executors/vertexMedia.ts"; +import { handleGeminiTtsSpeech } from "../executors/geminiTts.ts"; import { handleAwsPollySpeech } from "../executors/awsPollyTts.ts"; import { handleEdgeTtsSpeech } from "../executors/edgeTts.ts"; import { GttsUpstreamError, normalizeGttsLang, synthesizeGtts } from "../executors/gtts.ts"; @@ -889,6 +890,13 @@ export async function handleAudioSpeech({ headers: { ...CORS_HEADERS, "Content-Type": contentType }, }); } + if (providerConfig.format === "gemini-tts") { + return handleGeminiTtsSpeech(credentials, { + model: modelId, + text: body.input, + voice: body.voice, + }); + } if (providerConfig.format === "hyperbolic") { return handleHyperbolicSpeech(providerConfig, body, token); diff --git a/tests/unit/gemini-tts.test.ts b/tests/unit/gemini-tts.test.ts new file mode 100644 index 00000000000..15ae9eb05cf --- /dev/null +++ b/tests/unit/gemini-tts.test.ts @@ -0,0 +1,165 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { Buffer } from "node:buffer"; + +const { AUDIO_SPEECH_PROVIDERS, parseSpeechModel } = + await import("../../open-sse/config/audioRegistry.ts"); +const { geminiGenerateSpeech } = await import("../../open-sse/executors/geminiTts.ts"); +const { handleAudioSpeech } = await import("../../open-sse/handlers/audioSpeech.ts"); + +test("Google Gemini TTS models parse publicly and remap to Gemini credentials", () => { + assert.deepEqual(parseSpeechModel("google/gemini-2.5-flash-preview-tts"), { + provider: "google", + model: "gemini-2.5-flash-preview-tts", + }); + assert.equal(AUDIO_SPEECH_PROVIDERS.google.credentialProviderId, "gemini"); + assert.deepEqual( + AUDIO_SPEECH_PROVIDERS.google.models.map(({ id }) => id), + ["gemini-3.1-flash-tts-preview", "gemini-2.5-flash-preview-tts", "gemini-2.5-pro-preview-tts"] + ); +}); + +test("geminiGenerateSpeech sends the exact AI Studio generateContent contract and wraps PCM", async () => { + const originalFetch = globalThis.fetch; + const pcm = Buffer.from([1, 2, 3, 4]); + let captured: { url: string; init: RequestInit } | undefined; + globalThis.fetch = async (input, init = {}) => { + captured = { url: String(input), init }; + return Response.json({ + candidates: [ + { + content: { + parts: [ + { + inlineData: { + data: pcm.toString("base64"), + mimeType: "audio/L16;codec=pcm;rate=16000", + }, + }, + ], + }, + }, + ], + }); + }; + try { + const wav = await geminiGenerateSpeech( + { apiKey: "gemini-key" }, + { model: "gemini-2.5-flash-preview-tts", text: "Hello", voice: "Kore" } + ); + assert.equal( + captured?.url, + "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash-preview-tts:generateContent" + ); + assert.equal( + (captured?.init.headers as Record)["Content-Type"], + "application/json" + ); + assert.equal( + (captured?.init.headers as Record)["x-goog-api-key"], + "gemini-key" + ); + assert.equal((captured?.init.headers as Record).Authorization, undefined); + assert.deepEqual(JSON.parse(String(captured?.init.body)), { + contents: [{ parts: [{ text: "Hello" }] }], + generationConfig: { + responseModalities: ["AUDIO"], + speechConfig: { + voiceConfig: { prebuiltVoiceConfig: { voiceName: "Kore" } }, + }, + }, + }); + assert.equal(wav.subarray(0, 4).toString("ascii"), "RIFF"); + assert.equal(wav.readUInt32LE(24), 16000); + assert.deepEqual(wav.subarray(44), pcm); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("handleAudioSpeech returns WAV and defaults the AI Studio voice to Kore", async () => { + const originalFetch = globalThis.fetch; + let payload: { + generationConfig: { + speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: string } } }; + }; + }; + globalThis.fetch = async (_input, init = {}) => { + payload = JSON.parse(String(init.body)); + return Response.json({ + candidates: [ + { + content: { + parts: [ + { + inlineData: { + data: Buffer.from([5, 6]).toString("base64"), + mimeType: "audio/L16;rate=24000", + }, + }, + ], + }, + }, + ], + }); + }; + try { + const response = await handleAudioSpeech({ + body: { + model: "google/gemini-2.5-pro-preview-tts", + input: "Speak", + }, + credentials: { apiKey: "gemini-key" }, + }); + assert.equal(response.status, 200); + assert.equal(response.headers.get("content-type"), "audio/wav"); + assert.equal( + payload.generationConfig.speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName, + "Kore" + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("handleAudioSpeech rejects an AI Studio response without audio", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => Response.json({ candidates: [{ content: { parts: [] } }] }); + try { + const response = await handleAudioSpeech({ + body: { + model: "google/gemini-2.5-flash-preview-tts", + input: "Silent", + }, + credentials: { apiKey: "gemini-key" }, + }); + const payload = (await response.json()) as { error: { message: string } }; + assert.equal(response.status, 500); + assert.equal( + payload.error.message, + "Speech request failed: Gemini TTS response did not contain audio data" + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("handleAudioSpeech preserves AI Studio upstream errors", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => + Response.json({ error: { message: "quota exhausted" } }, { status: 429 }); + try { + const response = await handleAudioSpeech({ + body: { + model: "google/gemini-2.5-flash-preview-tts", + input: "Limited", + }, + credentials: { apiKey: "gemini-key" }, + }); + const payload = (await response.json()) as { error: { message: string } }; + assert.equal(response.status, 429); + assert.equal(payload.error.message, "quota exhausted"); + } finally { + globalThis.fetch = originalFetch; + } +});