Skip to content

chore: enhance Inworld TTS support #2123

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
diegosouzapw merged 1 commit into from
May 10, 2026
Merged

chore: enhance Inworld TTS support #2123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions open-sse/config/audioRegistry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export interface AudioProvider {
authType: string;
authHeader: string;
format?: string;
supportedFormats?: string[];
async?: boolean;
models: AudioModel[];
}
Expand Down Expand Up @@ -124,8 +125,8 @@ export const AUDIO_SPEECH_PROVIDERS: Record<string, AudioProvider> = {
authType: "apikey",
authHeader: "bearer",
models: [
{ id: "tts-1", name: "TTS 1" },
{ id: "tts-1-hd", name: "TTS 1 HD" },
{ id: "tts-1", name: "TTS 1" },
{ id: "gpt-4o-mini-tts", name: "GPT-4o Mini TTS" },
],
},
Expand Down Expand Up @@ -226,8 +227,9 @@ export const AUDIO_SPEECH_PROVIDERS: Record<string, AudioProvider> = {
authType: "apikey",
authHeader: "basic",
format: "inworld",
supportedFormats: ["mp3", "wav", "opus", "pcm"],
models: [
{ id: "inworld-tts-1.5-max", name: "Inworld TTS 1.5 Max" },
{ id: "inworld-tts-2", name: "Inworld TTS 2" },
{ id: "inworld-tts-1.5-mini", name: "Inworld TTS 1.5 Mini" },
],
},
Expand All @@ -242,8 +244,8 @@ export const AUDIO_SPEECH_PROVIDERS: Record<string, AudioProvider> = {
authHeader: "x-api-key",
format: "cartesia",
models: [
{ id: "sonic-2", name: "Sonic 2" },
{ id: "sonic-3", name: "Sonic 3" },
{ id: "sonic-2", name: "Sonic 2" },
],
},

Expand Down Expand Up @@ -297,6 +299,7 @@ export const AUDIO_SPEECH_PROVIDERS: Record<string, AudioProvider> = {
authType: "apikey",
authHeader: "bearer",
format: "xiaomi-mimo-tts",
supportedFormats: ["mp3", "wav"],
models: [
{ id: "mimo-v2.5-tts", name: "MiMo V2.5 TTS" },
{ id: "mimo-v2.5-tts-voicedesign", name: "MiMo V2.5 Voice Design" },
Expand Down
4 changes: 1 addition & 3 deletions open-sse/config/runway.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,9 @@ export const RUNWAYML_SUPPORTED_VIDEO_MODELS = [
{ id: "gen4_turbo", name: "Gen-4 Turbo" },
{ id: "veo3.1", name: "Veo 3.1" },
{ id: "veo3.1_fast", name: "Veo 3.1 Fast" },
{ id: "veo3", name: "Veo 3" },
{ id: "gen3a_turbo", name: "Gen-3 Alpha Turbo" },
];

export const RUNWAYML_IMAGE_REQUIRED_MODELS = new Set(["gen4_turbo", "gen3a_turbo"]);
export const RUNWAYML_IMAGE_REQUIRED_MODELS = new Set(["gen4_turbo"]);

export function normalizeRunwayBaseUrl(baseUrl?: string | null) {
const normalized = String(baseUrl || "")
Expand Down
21 changes: 19 additions & 2 deletions open-sse/handlers/audioSpeech.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,21 @@ async function handleHuggingFaceTtsSpeech(providerConfig, body, modelId, token)
* POST { text, voiceId, modelId, audioConfig } → JSON { audioContent: "<base64>" }
* Docs: https://docs.inworld.ai/api-reference/ttsAPI/texttospeech/synthesize-speech
*/
const INWORLD_AUDIO_FORMATS = {
mp3: { audioEncoding: "MP3", mimeType: "audio/mpeg" },
wav: { audioEncoding: "WAV", mimeType: "audio/wav" },
opus: { audioEncoding: "OPUS", mimeType: "audio/opus" },
pcm: { audioEncoding: "PCM", mimeType: "audio/pcm" },
};

async function handleInworldSpeech(providerConfig, body, modelId, token) {
const requestedFormat =
typeof body.response_format === "string" ? body.response_format.toLowerCase() : "mp3";
const audioFormat = INWORLD_AUDIO_FORMATS[requestedFormat];
if (!audioFormat) {
return errorResponse(400, "Inworld TTS supports response_format mp3, wav, opus, or pcm only");
}

const res = await fetch(providerConfig.baseUrl, {
method: "POST",
headers: {
Expand All @@ -440,7 +454,7 @@ async function handleInworldSpeech(providerConfig, body, modelId, token) {
voiceId: body.voice || undefined,
modelId,
audioConfig: {
audioEncoding: body.response_format === "wav" ? "LINEAR16" : "MP3",
audioEncoding: audioFormat.audioEncoding,
},
}),
});
Expand All @@ -452,7 +466,10 @@ async function handleInworldSpeech(providerConfig, body, modelId, token) {
const data = await res.json();
// Decode base64 audioContent to binary
const audioBuffer = Uint8Array.from(atob(data.audioContent ?? ""), (c) => c.charCodeAt(0));
const mimeType = body.response_format === "wav" ? "audio/wav" : "audio/mpeg";
const mimeType =
typeof data.contentType === "string" && data.contentType
? data.contentType
: audioFormat.mimeType;

return new Response(audioBuffer, {
status: 200,
Expand Down
148 changes: 143 additions & 5 deletions src/app/(dashboard)/dashboard/cache/media/MediaPageClient.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ type MediaModelConfig = { id: string; name: string };
type MediaProviderConfig = {
id: string;
authType: string;
supportedFormats?: string[];
models: MediaModelConfig[];
};
type ProviderModelGroup = {
Expand Down Expand Up @@ -172,8 +173,147 @@ const VOICE_PRESETS: Record<string, { id: string; label: string }[]> = {
{ id: "aura-orion-en", label: "Orion (EN)" },
],
inworld: [
{ id: "Eva", label: "Eva (EN)" },
{ id: "Abby", label: "Abby (EN)" },
{ id: "Alex", label: "Alex (EN)" },
{ id: "Amina", label: "Amina (EN)" },
{ id: "Anjali", label: "Anjali (EN)" },
{ id: "Arjun", label: "Arjun (EN)" },
{ id: "Ashley", label: "Ashley (EN)" },
{ id: "Avery", label: "Avery (EN)" },
{ id: "Bianca", label: "Bianca (EN)" },
{ id: "Blake", label: "Blake (EN)" },
{ id: "Brandon", label: "Brandon (EN)" },
{ id: "Brian", label: "Brian (EN)" },
{ id: "Callum", label: "Callum (EN)" },
{ id: "Carter", label: "Carter (EN)" },
{ id: "Cedric", label: "Cedric (EN)" },
{ id: "Celeste", label: "Celeste (EN)" },
{ id: "Chloe", label: "Chloe (EN)" },
{ id: "Claire", label: "Claire (EN)" },
{ id: "Clive", label: "Clive (EN)" },
{ id: "Conrad", label: "Conrad (EN)" },
{ id: "Craig", label: "Craig (EN)" },
{ id: "Damon", label: "Damon (EN)" },
{ id: "Darlene", label: "Darlene (EN)" },
{ id: "Deborah", label: "Deborah (EN)" },
{ id: "Dennis", label: "Dennis (EN)" },
{ id: "Derek", label: "Derek (EN)" },
{ id: "Dominus", label: "Dominus (EN)" },
{ id: "Duncan", label: "Duncan (EN)" },
{ id: "Edward", label: "Edward (EN)" },
{ id: "Eleanor", label: "Eleanor (EN)" },
{ id: "Elliot", label: "Elliot (EN)" },
{ id: "Ethan", label: "Ethan (EN)" },
{ id: "Evan", label: "Evan (EN)" },
{ id: "Evelyn", label: "Evelyn (EN)" },
{ id: "Felix", label: "Felix (EN)" },
{ id: "Gareth", label: "Gareth (EN)" },
{ id: "Graham", label: "Graham (EN)" },
{ id: "Hades", label: "Hades (EN)" },
{ id: "Hamish", label: "Hamish (EN)" },
{ id: "Hana", label: "Hana (EN)" },
{ id: "Hank", label: "Hank (EN)" },
{ id: "James", label: "James (EN)" },
{ id: "Jason", label: "Jason (EN)" },
{ id: "Jessica", label: "Jessica (EN)" },
{ id: "Jonah", label: "Jonah (EN)" },
{ id: "Kelsey", label: "Kelsey (EN)" },
{ id: "Lauren", label: "Lauren (EN)" },
{ id: "Levi", label: "Levi (EN)" },
{ id: "Liam", label: "Liam (EN)" },
{ id: "Loretta", label: "Loretta (EN)" },
{ id: "Lucian", label: "Lucian (EN)" },
{ id: "Luna", label: "Luna (EN)" },
{ id: "Malcolm", label: "Malcolm (EN)" },
{ id: "Marcus", label: "Marcus (EN)" },
{ id: "Mark", label: "Mark (EN)" },
{ id: "Marlene", label: "Marlene (EN)" },
{ id: "Mia", label: "Mia (EN)" },
{ id: "Miranda", label: "Miranda (EN)" },
{ id: "Mortimer", label: "Mortimer (EN)" },
{ id: "Nadia", label: "Nadia (EN)" },
{ id: "Naomi", label: "Naomi (EN)" },
{ id: "Nate", label: "Nate (EN)" },
{ id: "Oliver", label: "Oliver (EN)" },
{ id: "Olivia", label: "Olivia (EN)" },
{ id: "Pippa", label: "Pippa (EN)" },
{ id: "Pixie", label: "Pixie (EN)" },
{ id: "Reed", label: "Reed (EN)" },
{ id: "Riley", label: "Riley (EN)" },
{ id: "Ronald", label: "Ronald (EN)" },
{ id: "Rupert", label: "Rupert (EN)" },
{ id: "Saanvi", label: "Saanvi (EN)" },
{ id: "Sarah", label: "Sarah (EN)" },
{ id: "Sebastian", label: "Sebastian (EN)" },
{ id: "Selene", label: "Selene (EN)" },
{ id: "Serena", label: "Serena (EN)" },
{ id: "Simon", label: "Simon (EN)" },
{ id: "Snik", label: "Snik (EN)" },
{ id: "Sophie", label: "Sophie (EN)" },
{ id: "Tessa", label: "Tessa (EN)" },
{ id: "Theodore", label: "Theodore (EN)" },
{ id: "Timothy", label: "Timothy (EN)" },
{ id: "Trevor", label: "Trevor (EN)" },
{ id: "Tristan", label: "Tristan (EN)" },
{ id: "Tyler", label: "Tyler (EN)" },
{ id: "Veronica", label: "Veronica (EN)" },
{ id: "Victor", label: "Victor (EN)" },
{ id: "Victoria", label: "Victoria (EN)" },
{ id: "Vinny", label: "Vinny (EN)" },
{ id: "Wendy", label: "Wendy (EN)" },
{ id: "Aanya", label: "Aanya (HI)" },
{ id: "Aarav", label: "Aarav (HI)" },
{ id: "Manoj", label: "Manoj (HI)" },
{ id: "Riya", label: "Riya (HI)" },
{ id: "Alain", label: "Alain (FR)" },
{ id: "Étienne", label: "Étienne (FR)" },
{ id: "Hélène", label: "Hélène (FR)" },
{ id: "Mathieu", label: "Mathieu (FR)" },
{ id: "Asuka", label: "Asuka (JP)" },
{ id: "Haruto", label: "Haruto (JP)" },
{ id: "Hina", label: "Hina (JP)" },
{ id: "Satoshi", label: "Satoshi (JP)" },
{ id: "Beatriz", label: "Beatriz (PT)" },
{ id: "Heitor", label: "Heitor (PT)" },
{ id: "Maitê", label: "Maitê (PT)" },
{ id: "Mariana", label: "Mariana (PT)" },
{ id: "Murilo", label: "Murilo (PT)" },
{ id: "Camila", label: "Camila (ES)" },
{ id: "Diego", label: "Diego (ES)" },
{ id: "Lupita", label: "Lupita (ES)" },
{ id: "Mateo", label: "Mateo (ES)" },
{ id: "Mauricio", label: "Mauricio (ES)" },
{ id: "Miguel", label: "Miguel (ES)" },
{ id: "Rafael", label: "Rafael (ES)" },
{ id: "Sofia", label: "Sofia (ES)" },
{ id: "Dmitry", label: "Dmitry (RU)" },
{ id: "Elena", label: "Elena (RU)" },
{ id: "Nikolai", label: "Nikolai (RU)" },
{ id: "Svetlana", label: "Svetlana (RU)" },
{ id: "Erik", label: "Erik (NL)" },
{ id: "Katrien", label: "Katrien (NL)" },
{ id: "Lennart", label: "Lennart (NL)" },
{ id: "Lore", label: "Lore (NL)" },
{ id: "Gianni", label: "Gianni (IT)" },
{ id: "Orietta", label: "Orietta (IT)" },
{ id: "Hyunwoo", label: "Hyunwoo (KO)" },
{ id: "Minji", label: "Minji (KO)" },
{ id: "Seojun", label: "Seojun (KO)" },
{ id: "Yoona", label: "Yoona (KO)" },
{ id: "Jing", label: "Jing (ZH)" },
{ id: "Mei", label: "Mei (ZH)" },
{ id: "Ming", label: "Ming (ZH)" },
{ id: "Xiaoyin", label: "Xiaoyin (ZH)" },
{ id: "Xinyi", label: "Xinyi (ZH)" },
{ id: "Yichen", label: "Yichen (ZH)" },
{ id: "Johanna", label: "Johanna (DE)" },
{ id: "Josef", label: "Josef (DE)" },
{ id: "Nour", label: "Nour (AR)" },
{ id: "Omar", label: "Omar (AR)" },
{ id: "Oren", label: "Oren (HE)" },
{ id: "Yael", label: "Yael (HE)" },
{ id: "Szymon", label: "Szymon (PL)" },
{ id: "Wojciech", label: "Wojciech (PL)" },
],
"xiaomi-mimo": [
{ id: "冰糖", label: "冰糖 (Chinese Female)" },
Expand All @@ -188,12 +328,10 @@ const VOICE_PRESETS: Record<string, { id: string; label: string }[]> = {
};

const SPEECH_FORMATS = ["mp3", "wav", "opus", "flac", "pcm"];
const SPEECH_FORMATS_BY_PROVIDER: Record<string, string[]> = {
"xiaomi-mimo": ["mp3", "wav"],
};

function getSpeechFormats(providerId: string): string[] {
return SPEECH_FORMATS_BY_PROVIDER[providerId] || SPEECH_FORMATS;
const providerFormats = AUDIO_SPEECH_PROVIDERS[providerId]?.supportedFormats;
return providerFormats?.length ? providerFormats : SPEECH_FORMATS;
}

function getVoiceList(providerId: string) {
Expand Down
61 changes: 58 additions & 3 deletions tests/unit/audio-speech-handler.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import test from "node:test";
import assert from "node:assert/strict";

const { handleAudioSpeech } = await import("../../open-sse/handlers/audioSpeech.ts");
const { AUDIO_SPEECH_PROVIDERS } = await import("../../open-sse/config/audioRegistry.ts");

test("handleAudioSpeech requires model", async () => {
const response = await handleAudioSpeech({
Expand Down Expand Up @@ -486,7 +487,7 @@ test("handleAudioSpeech maps Inworld requests to basic auth and wav output", asy
try {
const response = await handleAudioSpeech({
body: {
model: "inworld/inworld-tts-1.5-max",
model: "inworld/inworld-tts-2",
input: "inworld text",
voice: "voice-9",
response_format: "wav",
Expand All @@ -498,8 +499,8 @@ test("handleAudioSpeech maps Inworld requests to basic auth and wav output", asy
assert.deepEqual(captured.body, {
text: "inworld text",
voiceId: "voice-9",
modelId: "inworld-tts-1.5-max",
audioConfig: { audioEncoding: "LINEAR16" },
modelId: "inworld-tts-2",
audioConfig: { audioEncoding: "WAV" },
});
assert.equal(response.headers.get("content-type"), "audio/wav");
assert.deepEqual(Array.from(new Uint8Array(await response.arrayBuffer())), [1, 2, 3, 4]);
Expand All @@ -508,6 +509,60 @@ test("handleAudioSpeech maps Inworld requests to basic auth and wav output", asy
}
});

test("Inworld speech registry exposes supported formats without flac", () => {
assert.deepEqual(AUDIO_SPEECH_PROVIDERS.inworld.supportedFormats, ["mp3", "wav", "opus", "pcm"]);
});

test("handleAudioSpeech maps Inworld opus output and rejects flac", async () => {
const originalFetch = globalThis.fetch;
let captured;
let callCount = 0;

globalThis.fetch = async (_url, options = {}) => {
callCount += 1;
captured = JSON.parse(String(options.body || "{}"));

return new Response(JSON.stringify({ audioContent: "BQY=", contentType: "audio/opus" }), {
status: 200,
headers: { "content-type": "application/json" },
});
};

try {
const opusResponse = await handleAudioSpeech({
body: {
model: "inworld/inworld-tts-2",
input: "inworld text",
response_format: "opus",
},
credentials: { apiKey: "encoded-basic-token" },
});

assert.deepEqual(captured.audioConfig, { audioEncoding: "OPUS" });
assert.equal(opusResponse.headers.get("content-type"), "audio/opus");
assert.deepEqual(Array.from(new Uint8Array(await opusResponse.arrayBuffer())), [5, 6]);

const flacResponse = await handleAudioSpeech({
body: {
model: "inworld/inworld-tts-2",
input: "inworld text",
response_format: "flac",
},
credentials: { apiKey: "encoded-basic-token" },
});
const payload = (await flacResponse.json()) as any;

assert.equal(flacResponse.status, 400);
assert.equal(
payload.error.message,
"Inworld TTS supports response_format mp3, wav, opus, or pcm only"
);
assert.equal(callCount, 1);
} finally {
globalThis.fetch = originalFetch;
}
});

test("handleAudioSpeech supports local Coqui providers without credentials", async () => {
const originalFetch = globalThis.fetch;
let captured;
Expand Down