diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 51cb3b8f799..e6161aae8f3 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -12,6 +12,7 @@ "_rebaseline_2026_06_15_3881_nous_validator": "Issue #3881 own growth: validation.ts 4394->4407 (+13 = treat any non-auth 4xx (400/404/422) from the Nous Research probe as a valid key — the request shape/model was wrong, not the credentials — mirroring the longcat/nvidia validators, plus the corrected Hermes-4-70B probe model). Cohesive validator branch; not extractable.", "_rebaseline_2026_06_15_3877_byteplus": "Issue #3877 own growth: providerRegistry.ts 4708->4730 (+22 = a byteplus (BytePlus ModelArk) registry entry — OpenAI-compatible, Ark base ap-southeast-1, Bearer, 4 seed models — modeled on the volcengine entry; byteplus was in APIKEY_PROVIDERS but unregistered here so validation returned {unsupported:true}). Cohesive provider registration; not extractable.", "_rebaseline_2026_06_15_3910_nested_combo_ctx": "PR #3910 net +1: providerRegistry.ts 4730->4731 (test-registry contextLength metadata for the nested combo-ref LCD regression test). opencode-plugin-only behavioral change; no core routing/virtualFactory touched.", + "_rebaseline_2026_06_15_3929_vertex_media": "PR #3929 own growth: audioSpeech.ts 952->965 (+13) and videoGeneration.ts 1026->1078 (+52) = vertex/* media branches (Gemini TTS, Veo predictLongRunning poll) wired into the speech/video handlers; new logic lives in open-sse/executors/vertexMedia.ts (341, under cap). Cohesive media-provider feature.", "cap": 800, "frozen": { "open-sse/config/providerRegistry.ts": 4731, @@ -26,13 +27,13 @@ "open-sse/executors/grok-web.ts": 1871, "open-sse/executors/muse-spark-web.ts": 1284, "open-sse/executors/perplexity-web.ts": 868, - "open-sse/handlers/audioSpeech.ts": 952, + "open-sse/handlers/audioSpeech.ts": 965, "open-sse/handlers/chatCore.ts": 5815, "open-sse/handlers/imageGeneration.ts": 3777, "open-sse/handlers/responseSanitizer.ts": 1103, "open-sse/handlers/search.ts": 1442, "open-sse/handlers/sseParser.ts": 812, - "open-sse/handlers/videoGeneration.ts": 1026, + "open-sse/handlers/videoGeneration.ts": 1078, "open-sse/mcp-server/schemas/tools.ts": 1437, "open-sse/mcp-server/server.ts": 1457, "open-sse/mcp-server/tools/advancedTools.ts": 1118, diff --git a/open-sse/config/audioRegistry.ts b/open-sse/config/audioRegistry.ts index 338b6fb4850..dfeac3a953d 100644 --- a/open-sse/config/audioRegistry.ts +++ b/open-sse/config/audioRegistry.ts @@ -23,6 +23,19 @@ export interface AudioProvider { } export const AUDIO_TRANSCRIPTION_PROVIDERS: Record = { + vertex: { + id: "vertex", + baseUrl: "https://us-central1-aiplatform.googleapis.com/v1", + authType: "apikey", + authHeader: "bearer", + format: "vertex-gemini", + models: [ + { id: "gemini-2.5-flash", name: "Gemini 2.5 Flash (Vertex Transcribe)" }, + { id: "gemini-2.5-pro", name: "Gemini 2.5 Pro (Vertex Transcribe)" }, + { id: "gemini-2.0-flash", name: "Gemini 2.0 Flash (Vertex Transcribe)" }, + ], + }, + openai: { id: "openai", baseUrl: "https://api.openai.com/v1/audio/transcriptions", @@ -148,6 +161,18 @@ export const AUDIO_TRANSCRIPTION_PROVIDERS: Record = { }; export const AUDIO_SPEECH_PROVIDERS: Record = { + vertex: { + id: "vertex", + baseUrl: "https://us-central1-aiplatform.googleapis.com/v1", + authType: "apikey", + authHeader: "bearer", + format: "vertex-gemini-tts", + models: [ + { id: "gemini-2.5-flash-preview-tts", name: "Gemini 2.5 Flash TTS (Vertex)" }, + { id: "gemini-2.5-pro-preview-tts", name: "Gemini 2.5 Pro TTS (Vertex)" }, + ], + }, + openai: { id: "openai", baseUrl: "https://api.openai.com/v1/audio/speech", diff --git a/open-sse/config/musicRegistry.ts b/open-sse/config/musicRegistry.ts index 7ca68d58910..7eda3e74254 100644 --- a/open-sse/config/musicRegistry.ts +++ b/open-sse/config/musicRegistry.ts @@ -24,6 +24,15 @@ interface MusicProvider { } export const MUSIC_PROVIDERS: Record = { + vertex: { + id: "vertex", + baseUrl: "https://us-central1-aiplatform.googleapis.com/v1", + authType: "apikey", + authHeader: "bearer", + format: "vertex-lyria", + models: [{ id: "lyria-002", name: "Lyria 2 (Vertex)" }], + }, + kie: { id: "kie", baseUrl: "https://api.kie.ai", diff --git a/open-sse/config/videoRegistry.ts b/open-sse/config/videoRegistry.ts index 6e65e5adfb7..12d0c8cbc3a 100644 --- a/open-sse/config/videoRegistry.ts +++ b/open-sse/config/videoRegistry.ts @@ -26,6 +26,19 @@ interface VideoProvider { } export const VIDEO_PROVIDERS: Record = { + vertex: { + id: "vertex", + baseUrl: "https://us-central1-aiplatform.googleapis.com/v1", + authType: "apikey", + authHeader: "bearer", + format: "vertex-veo", + models: [ + { id: "veo-3.0-generate-001", name: "Veo 3.0 (Vertex)" }, + { id: "veo-3.0-fast-generate-001", name: "Veo 3.0 Fast (Vertex)" }, + { id: "veo-2.0-generate-001", name: "Veo 2.0 (Vertex)" }, + ], + }, + kie: { id: "kie", baseUrl: "https://api.kie.ai", diff --git a/open-sse/executors/vertexMedia.ts b/open-sse/executors/vertexMedia.ts new file mode 100644 index 00000000000..278230c1e1f --- /dev/null +++ b/open-sse/executors/vertexMedia.ts @@ -0,0 +1,341 @@ +/** + * Vertex AI media generation client. + * + * Google's Vertex AI serves speech (Gemini TTS), transcription (Gemini), music + * (Lyria) and video (Veo) — but through the same `aiplatform.googleapis.com` + * surface that the chat executor authenticates against, NOT through the + * third-party media registries (kie/suno/deepgram/…). This module reuses the + * Vertex chat executor's auth (Service Account JSON → OAuth bearer, or Express + * API key) and implements the verified per-model contracts: + * + * - Speech: `{model}:generateContent` + responseModalities:["AUDIO"] → PCM L16 → WAV + * - Transcription: `{model}:generateContent` with inline audio + text prompt → text + * - Music (Lyria): `{model}:predict` → predictions[0].bytesBase64Encoded (WAV) + * - Video (Veo): `{model}:predictLongRunning` → poll `{model}:fetchPredictOperation` + * → response.videos[0].bytesBase64Encoded (MP4) + */ + +import { Buffer } from "node:buffer"; +import { sleep } from "../utils/sleep.ts"; +import { + parseSAFromApiKey, + getAccessToken, + looksLikeServiceAccountJson, + isExpressApiKey, +} from "./vertex.ts"; + +export interface VertexMediaCredentials { + apiKey?: string | null; + accessToken?: string | null; + providerSpecificData?: Record | null; +} + +interface ResolvedVertexAuth { + project: string; + region: string; + bearerToken: string | null; + expressKey: string | null; +} + +const DEFAULT_REGION = "us-central1"; + +function resolveRegion(credentials: VertexMediaCredentials | null | undefined): string { + const psd = credentials?.providerSpecificData; + if (psd && typeof psd === "object") { + const region = (psd as Record).region; + if (typeof region === "string" && region.trim().length > 0) return region.trim(); + } + return DEFAULT_REGION; +} + +async function resolveVertexAuth( + credentials: VertexMediaCredentials | null | undefined +): Promise { + const apiKey = typeof credentials?.apiKey === "string" ? credentials.apiKey.trim() : ""; + const region = resolveRegion(credentials); + let bearerToken = + typeof credentials?.accessToken === "string" && credentials.accessToken.trim().length > 0 + ? credentials.accessToken.trim() + : null; + let project = ""; + let expressKey: string | null = null; + + if (looksLikeServiceAccountJson(apiKey)) { + const sa = parseSAFromApiKey(apiKey); + project = typeof sa.project_id === "string" ? sa.project_id : ""; + if (!bearerToken) bearerToken = await getAccessToken(sa); + } else if (isExpressApiKey(apiKey)) { + expressKey = apiKey; + } + + return { project, region, bearerToken, expressKey }; +} + +/** + * Build the request URL + headers for a Vertex publisher-model action. + * SA path → project-scoped regional endpoint + Bearer auth. + * Express path (best-effort) → project-less global publisher endpoint + ?key=. + */ +function buildModelRequest( + auth: ResolvedVertexAuth, + model: string, + action: string +): { url: string; headers: Record } { + const headers: Record = { "Content-Type": "application/json" }; + + if (auth.bearerToken && auth.project) { + headers["Authorization"] = `Bearer ${auth.bearerToken}`; + return { + url: `https://${auth.region}-aiplatform.googleapis.com/v1/projects/${auth.project}/locations/${auth.region}/publishers/google/models/${model}:${action}`, + headers, + }; + } + + if (auth.expressKey) { + return { + url: `https://aiplatform.googleapis.com/v1/publishers/google/models/${model}:${action}?key=${encodeURIComponent( + auth.expressKey + )}`, + headers, + }; + } + + throw new Error( + "Vertex AI requires a Service Account JSON (with project_id) or a Vertex AI Express API key" + ); +} + +interface VertexHttpError extends Error { + status?: number; +} + +async function vertexError(res: Response): Promise { + let detail = ""; + try { + detail = await res.text(); + } catch { + /* ignore */ + } + let message = `Vertex AI error (${res.status})`; + if (detail) { + try { + const parsed = JSON.parse(detail); + message = parsed?.error?.message || message; + } catch { + message = detail.slice(0, 300); + } + } + const err = new Error(message) as VertexHttpError; + err.status = res.status; + return err; +} + +/** Wrap raw little-endian 16-bit PCM mono samples in a minimal WAV container. */ +export function pcmToWav(pcm: Buffer, sampleRate = 24000, channels = 1, bitsPerSample = 16): Buffer { + const blockAlign = (channels * bitsPerSample) / 8; + const byteRate = sampleRate * blockAlign; + const header = Buffer.alloc(44); + header.write("RIFF", 0); + header.writeUInt32LE(36 + pcm.length, 4); + header.write("WAVE", 8); + header.write("fmt ", 12); + header.writeUInt32LE(16, 16); + header.writeUInt16LE(1, 20); // PCM + header.writeUInt16LE(channels, 22); + header.writeUInt32LE(sampleRate, 24); + header.writeUInt32LE(byteRate, 28); + header.writeUInt16LE(blockAlign, 32); + header.writeUInt16LE(bitsPerSample, 34); + header.write("data", 36); + header.writeUInt32LE(pcm.length, 40); + return Buffer.concat([header, pcm]); +} + +function parseSampleRate(mimeType: string | undefined): number { + if (!mimeType) return 24000; + const match = /rate=(\d+)/i.exec(mimeType); + return match ? parseInt(match[1], 10) : 24000; +} + +function extractInlineAudio( + data: unknown +): { base64: string; mimeType: string } | null { + const parts = (data as { candidates?: Array<{ content?: { parts?: unknown[] } }> })?.candidates?.[0] + ?.content?.parts; + if (!Array.isArray(parts)) return null; + for (const part of parts) { + const inline = (part as { inlineData?: { data?: unknown; mimeType?: unknown } })?.inlineData; + if (inline && typeof inline.data === "string" && inline.data.length > 0) { + return { + base64: inline.data, + mimeType: typeof inline.mimeType === "string" ? inline.mimeType : "audio/L16;rate=24000", + }; + } + } + return null; +} + +function extractText(data: unknown): string { + const parts = (data as { candidates?: Array<{ content?: { parts?: unknown[] } }> })?.candidates?.[0] + ?.content?.parts; + if (!Array.isArray(parts)) return ""; + return parts + .map((part) => (part as { text?: unknown })?.text) + .filter((text): text is string => typeof text === "string") + .join("") + .trim(); +} + +/** Gemini TTS → WAV audio buffer. */ +export async function vertexGenerateSpeech( + credentials: VertexMediaCredentials, + options: { model: string; input: string; voice?: string } +): Promise<{ audio: Buffer; contentType: string }> { + const auth = await resolveVertexAuth(credentials); + const { url, headers } = buildModelRequest(auth, options.model, "generateContent"); + const payload = { + contents: [{ role: "user", parts: [{ text: options.input }] }], + generationConfig: { + responseModalities: ["AUDIO"], + speechConfig: { + voiceConfig: { + prebuiltVoiceConfig: { voiceName: options.voice && options.voice.trim() ? options.voice.trim() : "Kore" }, + }, + }, + }, + }; + const res = await fetch(url, { method: "POST", headers, body: JSON.stringify(payload) }); + if (!res.ok) throw await vertexError(res); + const data = await res.json(); + const inline = extractInlineAudio(data); + if (!inline) throw new Error("Vertex TTS returned no audio content"); + const pcm = Buffer.from(inline.base64, "base64"); + return { audio: pcmToWav(pcm, parseSampleRate(inline.mimeType)), contentType: "audio/wav" }; +} + +/** Gemini transcription (audio → text). `audioBase64` is the raw file bytes, base64-encoded. */ +export async function vertexTranscribe( + credentials: VertexMediaCredentials, + options: { model: string; audioBase64: string; mimeType?: string; prompt?: string; language?: string } +): Promise { + const auth = await resolveVertexAuth(credentials); + const { url, headers } = buildModelRequest(auth, options.model, "generateContent"); + const instruction = + options.prompt && options.prompt.trim().length > 0 + ? options.prompt.trim() + : `Transcribe this audio verbatim. Output only the spoken words${ + options.language ? ` (language: ${options.language})` : "" + }, with no commentary.`; + const payload = { + contents: [ + { + role: "user", + parts: [ + { text: instruction }, + { inlineData: { mimeType: options.mimeType || "audio/wav", data: options.audioBase64 } }, + ], + }, + ], + }; + const res = await fetch(url, { method: "POST", headers, body: JSON.stringify(payload) }); + if (!res.ok) throw await vertexError(res); + return extractText(await res.json()); +} + +/** Lyria music generation → { base64 WAV, format }. */ +export async function vertexGenerateMusic( + credentials: VertexMediaCredentials, + options: { model?: string; prompt: string; negativePrompt?: string; sampleCount?: number; seed?: number } +): Promise<{ base64: string; format: string }> { + const auth = await resolveVertexAuth(credentials); + const model = options.model && options.model.trim() ? options.model.trim() : "lyria-002"; + const { url, headers } = buildModelRequest(auth, model, "predict"); + const instance: Record = { prompt: options.prompt }; + if (options.negativePrompt) instance.negative_prompt = options.negativePrompt; + if (typeof options.seed === "number") instance.seed = options.seed; + const parameters: Record = {}; + if (typeof options.sampleCount === "number") parameters.sample_count = options.sampleCount; + const res = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify({ instances: [instance], parameters }), + }); + if (!res.ok) throw await vertexError(res); + const data = await res.json(); + const base64 = (data as { predictions?: Array<{ bytesBase64Encoded?: unknown }> })?.predictions?.[0] + ?.bytesBase64Encoded; + if (typeof base64 !== "string" || base64.length === 0) { + throw new Error("Vertex Lyria returned no audio"); + } + return { base64, format: "wav" }; +} + +/** Veo video generation (async long-running) → { base64 MP4 or gcsUri, format }. */ +export async function vertexGenerateVideo( + credentials: VertexMediaCredentials, + options: { + model: string; + prompt: string; + aspectRatio?: string; + durationSeconds?: number; + sampleCount?: number; + negativePrompt?: string; + image?: { bytesBase64Encoded: string; mimeType: string }; + pollIntervalMs?: number; + maxWaitMs?: number; + } +): Promise<{ base64?: string; url?: string; format: string }> { + const auth = await resolveVertexAuth(credentials); + const submit = buildModelRequest(auth, options.model, "predictLongRunning"); + + const instance: Record = { prompt: options.prompt }; + if (options.image) instance.image = options.image; + const parameters: Record = { + sampleCount: typeof options.sampleCount === "number" ? options.sampleCount : 1, + }; + if (options.aspectRatio) parameters.aspectRatio = options.aspectRatio; + if (typeof options.durationSeconds === "number") parameters.durationSeconds = options.durationSeconds; + if (options.negativePrompt) parameters.negativePrompt = options.negativePrompt; + + const submitRes = await fetch(submit.url, { + method: "POST", + headers: submit.headers, + body: JSON.stringify({ instances: [instance], parameters }), + }); + if (!submitRes.ok) throw await vertexError(submitRes); + const op = await submitRes.json(); + const operationName = (op as { name?: unknown })?.name; + if (typeof operationName !== "string" || operationName.length === 0) { + throw new Error("Vertex Veo did not return an operation name"); + } + + const poll = buildModelRequest(auth, options.model, "fetchPredictOperation"); + const intervalMs = options.pollIntervalMs && options.pollIntervalMs > 0 ? options.pollIntervalMs : 10000; + const maxWaitMs = options.maxWaitMs && options.maxWaitMs > 0 ? options.maxWaitMs : 5 * 60 * 1000; + const deadline = Date.now() + maxWaitMs; + + while (Date.now() < deadline) { + await sleep(intervalMs); + const pollRes = await fetch(poll.url, { + method: "POST", + headers: poll.headers, + body: JSON.stringify({ operationName }), + }); + if (!pollRes.ok) throw await vertexError(pollRes); + const pollData = await pollRes.json(); + if ((pollData as { done?: unknown })?.done) { + const opError = (pollData as { error?: { message?: unknown } })?.error; + if (opError) throw new Error(String(opError.message || "Veo operation failed")); + const videos = (pollData as { response?: { videos?: unknown } })?.response?.videos; + const video = Array.isArray(videos) ? (videos[0] as Record) : null; + if (video && typeof video.bytesBase64Encoded === "string") { + return { base64: video.bytesBase64Encoded, format: "mp4" }; + } + if (video && typeof video.gcsUri === "string") { + return { url: video.gcsUri, format: "mp4" }; + } + throw new Error("Veo operation completed but returned no video"); + } + } + throw new Error("Vertex Veo video generation timed out"); +} diff --git a/open-sse/handlers/audioSpeech.ts b/open-sse/handlers/audioSpeech.ts index ffa2542aa8b..1034349016d 100644 --- a/open-sse/handlers/audioSpeech.ts +++ b/open-sse/handlers/audioSpeech.ts @@ -21,6 +21,7 @@ import { stripTrailingSlashes } from "../utils/urlSanitize.ts"; import { getSpeechProvider, parseSpeechModel } from "../config/audioRegistry.ts"; import { buildAuthHeaders } from "../config/registryUtils.ts"; import { kieExecutor } from "../executors/kie.ts"; +import { vertexGenerateSpeech } from "../executors/vertexMedia.ts"; import { errorResponse } from "../utils/error.ts"; import { getKieCallbackUrl, @@ -872,6 +873,18 @@ export async function handleAudioSpeech({ try { // Route to provider-specific handler + if (providerConfig.format === "vertex-gemini-tts") { + const { audio, contentType } = await vertexGenerateSpeech(credentials, { + model: modelId, + input: body.input, + voice: body.voice, + }); + return new Response(audio, { + status: 200, + headers: { ...CORS_HEADERS, "Content-Type": contentType }, + }); + } + if (providerConfig.format === "hyperbolic") { return handleHyperbolicSpeech(providerConfig, body, token); } diff --git a/open-sse/handlers/audioTranscription.ts b/open-sse/handlers/audioTranscription.ts index b33ce44c605..af443427c80 100644 --- a/open-sse/handlers/audioTranscription.ts +++ b/open-sse/handlers/audioTranscription.ts @@ -21,6 +21,7 @@ import { } from "../config/audioRegistry.ts"; import { buildAuthHeaders } from "../config/registryUtils.ts"; import { kieExecutor } from "../executors/kie.ts"; +import { vertexTranscribe } from "../executors/vertexMedia.ts"; import { errorResponse } from "../utils/error.ts"; type TranscriptionCredentials = { @@ -462,6 +463,32 @@ export async function handleAudioTranscription({ } // Route to provider-specific handler + if (providerConfig.format === "vertex-gemini") { + try { + const buffer = Buffer.from(await file.arrayBuffer()); + const uploadedType = + typeof (file as { type?: unknown }).type === "string" && (file as { type?: string }).type + ? (file as { type: string }).type + : "audio/wav"; + const languageValue = formData.get("language"); + const promptValue = formData.get("prompt"); + const text = await vertexTranscribe(credentials ?? {}, { + model: modelId as string, + audioBase64: buffer.toString("base64"), + mimeType: uploadedType, + prompt: typeof promptValue === "string" ? promptValue : undefined, + language: typeof languageValue === "string" ? languageValue : undefined, + }); + return Response.json({ text }, { headers: { ...CORS_HEADERS } }); + } catch (err) { + const error = err as { message?: string; status?: number }; + return errorResponse( + typeof error?.status === "number" ? error.status : 500, + `Vertex transcription failed: ${error?.message || "unknown error"}` + ); + } + } + if (providerConfig.format === "deepgram") { return handleDeepgramTranscription(providerConfig, file, modelId, token, formData); } diff --git a/open-sse/handlers/musicGeneration.ts b/open-sse/handlers/musicGeneration.ts index 9a32cc70019..95b9ebd4091 100644 --- a/open-sse/handlers/musicGeneration.ts +++ b/open-sse/handlers/musicGeneration.ts @@ -16,6 +16,7 @@ import { getMusicProvider, parseMusicModel } from "../config/musicRegistry.ts"; import { kieExecutor } from "../executors/kie.ts"; +import { vertexGenerateMusic } from "../executors/vertexMedia.ts"; import { submitComfyWorkflow, pollComfyResult, @@ -94,6 +95,29 @@ export async function handleMusicGeneration({ body, credentials, log }) { }; } + if (providerConfig.format === "vertex-lyria") { + try { + const { base64, format } = await vertexGenerateMusic(credentials, { + model, + prompt: String(body.prompt ?? ""), + negativePrompt: typeof body.negative_prompt === "string" ? body.negative_prompt : undefined, + sampleCount: typeof body.sample_count === "number" ? body.sample_count : undefined, + seed: typeof body.seed === "number" ? body.seed : undefined, + }); + return { + success: true, + data: { created: Math.floor(Date.now() / 1000), data: [{ b64_json: base64, format }] }, + }; + } catch (err: any) { + log?.error?.("MUSIC", `Vertex Lyria generation failed: ${err?.message}`); + return { + success: false, + status: typeof err?.status === "number" ? err.status : 502, + error: sanitizeErrorMessage(err?.message || "Vertex Lyria generation failed"), + }; + } + } + if (providerConfig.format === "comfyui") { return handleComfyUIMusicGeneration({ model, provider, providerConfig, body, log }); } diff --git a/open-sse/handlers/videoGeneration.ts b/open-sse/handlers/videoGeneration.ts index 65890fdd04f..1581133b7fa 100644 --- a/open-sse/handlers/videoGeneration.ts +++ b/open-sse/handlers/videoGeneration.ts @@ -17,6 +17,7 @@ import { getVideoProvider, parseVideoModel } from "../config/videoRegistry.ts"; import { kieExecutor } from "../executors/kie.ts"; +import { vertexGenerateVideo } from "../executors/vertexMedia.ts"; import { getExecutor } from "../executors/index.ts"; import { isJsonObject, parseKieResultJson } from "../utils/kieTask.ts"; import { @@ -56,6 +57,10 @@ export async function handleVideoGeneration({ body, credentials, log }) { }; } + if (providerConfig.format === "vertex-veo") { + return handleVertexVeoGeneration({ model, body, credentials, log }); + } + if (providerConfig.format === "comfyui") { return handleComfyUIVideoGeneration({ model, provider, providerConfig, body, log }); } @@ -98,6 +103,53 @@ export async function handleVideoGeneration({ body, credentials, log }) { }; } +/** + * Veo video generation via Vertex AI (predictLongRunning → poll → MP4). + * Uses the Vertex chat credentials (Service Account JSON or Express key). + */ +async function handleVertexVeoGeneration({ model, body, credentials, log }) { + try { + const aspectRatio = + typeof body.aspect_ratio === "string" + ? body.aspect_ratio + : typeof body.aspectRatio === "string" + ? body.aspectRatio + : typeof body.size === "string" + ? body.size + : undefined; + const durationSeconds = + typeof body.duration === "number" + ? body.duration + : typeof body.durationSeconds === "number" + ? body.durationSeconds + : undefined; + + const result = await vertexGenerateVideo(credentials, { + model, + prompt: String(body.prompt ?? ""), + aspectRatio, + durationSeconds, + negativePrompt: typeof body.negative_prompt === "string" ? body.negative_prompt : undefined, + }); + + const item = result.base64 + ? { b64_json: result.base64, format: result.format } + : { url: result.url, format: result.format }; + + return { + success: true, + data: { created: Math.floor(Date.now() / 1000), data: [item] }, + }; + } catch (err: any) { + log?.error?.("VIDEO", `Vertex Veo generation failed: ${err?.message}`); + return { + success: false, + status: typeof err?.status === "number" ? err.status : 502, + error: sanitizeErrorMessage(err?.message || "Vertex Veo generation failed"), + }; + } +} + /** * Handle ComfyUI video generation * Submits an AnimateDiff or SVD workflow, polls for completion, fetches output video diff --git a/tests/unit/vertex-media.test.ts b/tests/unit/vertex-media.test.ts new file mode 100644 index 00000000000..b6b8e4196ed --- /dev/null +++ b/tests/unit/vertex-media.test.ts @@ -0,0 +1,171 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { Buffer } from "node:buffer"; + +import { + pcmToWav, + vertexGenerateSpeech, + vertexTranscribe, + vertexGenerateMusic, + vertexGenerateVideo, +} from "../../open-sse/executors/vertexMedia.ts"; + +// Service Account credential with a pre-set accessToken so resolveVertexAuth never +// performs a real OAuth token exchange (getAccessToken is skipped when accessToken is present). +function saCredentials(region = "us-central1") { + return { + apiKey: JSON.stringify({ project_id: "proj-test", client_email: "svc@x.iam", private_key: "x" }), + accessToken: "test-bearer-token", + providerSpecificData: { region }, + }; +} + +function expressCredentials() { + return { apiKey: "express-key-abc", accessToken: null, providerSpecificData: {} }; +} + +interface FetchCall { + url: string; + init: any; +} + +function installFetch(responders: Array<(call: FetchCall) => unknown>) { + const calls: FetchCall[] = []; + let i = 0; + (globalThis as any).fetch = async (url: string, init: any) => { + const call = { url: String(url), init }; + calls.push(call); + const payload = responders[Math.min(i, responders.length - 1)](call); + i += 1; + return { + ok: true, + status: 200, + json: async () => payload, + text: async () => JSON.stringify(payload), + }; + }; + return calls; +} + +test("pcmToWav writes a valid RIFF/WAVE header with correct sizes", () => { + const pcm = Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]); + const wav = pcmToWav(pcm, 24000); + assert.equal(wav.subarray(0, 4).toString("ascii"), "RIFF"); + assert.equal(wav.subarray(8, 12).toString("ascii"), "WAVE"); + assert.equal(wav.readUInt32LE(4), 36 + pcm.length); // RIFF chunk size + assert.equal(wav.readUInt32LE(24), 24000); // sample rate + assert.equal(wav.readUInt32LE(40), pcm.length); // data chunk size + assert.equal(wav.length, 44 + pcm.length); +}); + +test("vertexGenerateSpeech posts generateContent with AUDIO modality and returns WAV", async () => { + const pcmB64 = Buffer.from([10, 20, 30, 40]).toString("base64"); + const calls = installFetch([ + () => ({ + candidates: [ + { content: { parts: [{ inlineData: { data: pcmB64, mimeType: "audio/L16;codec=pcm;rate=24000" } }] } }, + ], + }), + ]); + + const { audio, contentType } = await vertexGenerateSpeech(saCredentials("europe-west4"), { + model: "gemini-2.5-flash-preview-tts", + input: "Hello world", + voice: "Puck", + }); + + assert.equal(contentType, "audio/wav"); + assert.equal(audio.subarray(0, 4).toString("ascii"), "RIFF"); + assert.equal( + calls[0].url, + "https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-test/locations/europe-west4/publishers/google/models/gemini-2.5-flash-preview-tts:generateContent" + ); + const body = JSON.parse(calls[0].init.body); + assert.deepEqual(body.generationConfig.responseModalities, ["AUDIO"]); + assert.equal(body.generationConfig.speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName, "Puck"); + assert.equal(calls[0].init.headers.Authorization, "Bearer test-bearer-token"); +}); + +test("vertexGenerateSpeech defaults the voice to Kore", async () => { + const pcmB64 = Buffer.from([1, 2]).toString("base64"); + const calls = installFetch([ + () => ({ candidates: [{ content: { parts: [{ inlineData: { data: pcmB64, mimeType: "audio/L16;rate=16000" } }] } }] }), + ]); + await vertexGenerateSpeech(saCredentials(), { model: "gemini-2.5-flash-preview-tts", input: "hi" }); + const body = JSON.parse(calls[0].init.body); + assert.equal(body.generationConfig.speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName, "Kore"); +}); + +test("vertexTranscribe posts audio inlineData and returns the joined text", async () => { + const calls = installFetch([ + () => ({ candidates: [{ content: { parts: [{ text: "the quick brown fox" }] } }] }), + ]); + const text = await vertexTranscribe(saCredentials(), { + model: "gemini-2.5-flash", + audioBase64: "QUJD", + mimeType: "audio/mpeg", + prompt: "Transcribe please", + }); + assert.equal(text, "the quick brown fox"); + const body = JSON.parse(calls[0].init.body); + const parts = body.contents[0].parts; + assert.equal(parts[0].text, "Transcribe please"); + assert.equal(parts[1].inlineData.mimeType, "audio/mpeg"); + assert.equal(parts[1].inlineData.data, "QUJD"); + assert.ok(calls[0].url.endsWith("/gemini-2.5-flash:generateContent")); +}); + +test("vertexGenerateMusic posts predict to lyria and returns base64 WAV", async () => { + const calls = installFetch([() => ({ predictions: [{ bytesBase64Encoded: "TY9MUA==" }] })]); + const { base64, format } = await vertexGenerateMusic(saCredentials(), { + model: "lyria-002", + prompt: "relaxing sax", + }); + assert.equal(base64, "TY9MUA=="); + assert.equal(format, "wav"); + assert.ok(calls[0].url.endsWith("/lyria-002:predict")); + const body = JSON.parse(calls[0].init.body); + assert.equal(body.instances[0].prompt, "relaxing sax"); +}); + +test("vertexGenerateVideo submits predictLongRunning then polls fetchPredictOperation", async () => { + const calls = installFetch([ + () => ({ name: "projects/proj-test/.../operations/op-1" }), // submit + () => ({ name: "projects/proj-test/.../operations/op-1" }), // poll #1 (not done) + () => ({ + done: true, + response: { videos: [{ bytesBase64Encoded: "TVA0VklERU8=" }] }, + }), // poll #2 (done) + ]); + + const result = await vertexGenerateVideo(saCredentials(), { + model: "veo-3.0-fast-generate-001", + prompt: "a cat playing piano", + aspectRatio: "16:9", + durationSeconds: 4, + pollIntervalMs: 1, + maxWaitMs: 5000, + }); + + assert.equal(result.base64, "TVA0VklERU8="); + assert.equal(result.format, "mp4"); + assert.ok(calls[0].url.endsWith("/veo-3.0-fast-generate-001:predictLongRunning")); + assert.ok(calls[1].url.endsWith("/veo-3.0-fast-generate-001:fetchPredictOperation")); + const submitBody = JSON.parse(calls[0].init.body); + assert.equal(submitBody.parameters.aspectRatio, "16:9"); + assert.equal(submitBody.parameters.durationSeconds, 4); + assert.equal(submitBody.instances[0].prompt, "a cat playing piano"); +}); + +test("Express API key uses the project-less publisher endpoint with ?key=", async () => { + const pcmB64 = Buffer.from([1]).toString("base64"); + const calls = installFetch([ + () => ({ candidates: [{ content: { parts: [{ inlineData: { data: pcmB64, mimeType: "audio/L16;rate=24000" } }] } }] }), + ]); + await vertexGenerateSpeech(expressCredentials(), { model: "gemini-2.5-flash-preview-tts", input: "hi" }); + assert.equal( + calls[0].url, + "https://aiplatform.googleapis.com/v1/publishers/google/models/gemini-2.5-flash-preview-tts:generateContent?key=express-key-abc" + ); + assert.equal(calls[0].init.headers.Authorization, undefined); +});