feat: add voice dictation beta

apps/desktop/src/settings/DesktopClientSettings.test.ts

@@ -33,6 +33,9 @@ const clientSettings: ClientSettings = {
   sidebarV2Enabled: false,
   sidebarV2ConfiguredByUser: false,
   timestampFormat: "24-hour",
+  voiceTranscriptionEnabled: true,
+  voiceTranscriptionProvider: "openai",
+  voiceTranscriptionApiKey: "",
   wordWrap: true,
 };
 

apps/server/src/http.ts

@@ -40,8 +40,18 @@ import {
 } from "./auth/http.ts";
 import * as ServerEnvironment from "./environment/ServerEnvironment.ts";
 import { browserApiCorsAllowedHeaders, browserApiCorsAllowedMethods } from "./httpCors.ts";
+import {
+  forwardVoiceTranscription,
+  MAX_TRANSCRIPTION_AUDIO_BYTES,
+  readTranscriptionAudio,
+  resolveTranscriptionProvider,
+  TranscriptionApiKeyMissingError,
+  TranscriptionEmptyAudioError,
+  TranscriptionProviderUnsupportedError,
+} from "./transcription.ts";
 
 const OTLP_TRACES_PROXY_PATH = "/api/observability/v1/traces";
+const TRANSCRIPTION_PATH = "/api/transcription";
 const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]);
 const DESKTOP_RENDERER_ORIGINS = ["t3code://app", "t3code-dev://app"];
 const GZIP_MIN_BYTES = 1024;
@@ -247,6 +257,60 @@ export const otlpTracesProxyRouteLayer = HttpRouter.add(
   ),
 );
 
+export const transcriptionRouteLayer = HttpRouter.add(
+  "POST",
+  TRANSCRIPTION_PATH,
+  Effect.gen(function* () {
+    yield* authenticateRawRouteWithScope(AuthOrchestrationOperateScope);
+    const request = yield* HttpServerRequest.HttpServerRequest;
+    const declaredLength = Number(request.headers["content-length"] ?? "0");
+    if (Number.isFinite(declaredLength) && declaredLength > MAX_TRANSCRIPTION_AUDIO_BYTES) {
+      return HttpServerResponse.jsonUnsafe(
+        { error: "The recording exceeds the 25 MB limit." },
+        { status: 413 },
+      );
+    }
+
+    const provider = resolveTranscriptionProvider(
+      request.headers["x-t3-transcription-provider"] ?? "",
+    );
+    if (!provider) {
+      return yield* new TranscriptionProviderUnsupportedError();
+    }
+
+    const audio = yield* readTranscriptionAudio(request.stream);
+    const text = yield* forwardVoiceTranscription({
+      audio,
+      audioMimeType: request.headers["content-type"] ?? "audio/webm",
+      provider,
+      apiKey: request.headers["x-t3-transcription-api-key"] ?? "",
+    });
+    return HttpServerResponse.jsonUnsafe({ text });
+  }).pipe(
+    Effect.catchTags({
+      EnvironmentAuthInvalidError: HttpServerRespondable.toResponse,
+      EnvironmentInternalError: HttpServerRespondable.toResponse,
+      EnvironmentScopeRequiredError: HttpServerRespondable.toResponse,
+      TranscriptionAudioTooLargeError: (error) =>
+        Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 413 })),
+      TranscriptionApiKeyMissingError: (error) =>
+        Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })),
+      TranscriptionBodyReadError: (error) =>
+        Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })),
+      TranscriptionEmptyAudioError: (error) =>
+        Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })),
+      TranscriptionProviderError: (error) =>
+        Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })),
+      TranscriptionProviderUnsupportedError: (error) =>
+        Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 400 })),
+      TranscriptionRequestError: (error) =>
+        Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })),
+      TranscriptionResponseError: (error) =>
+        Effect.succeed(HttpServerResponse.jsonUnsafe({ error: error.message }, { status: 502 })),
+    }),
+  ),
+);
+
 export const assetRouteLayer = HttpRouter.add(
   "GET",
   `${ASSET_ROUTE_PREFIX}/*`,

apps/server/src/httpCors.ts

@@ -5,6 +5,8 @@ export const browserApiCorsAllowedHeaders = [
   "traceparent",
   "content-type",
   "dpop",
+  "x-t3-transcription-api-key",
+  "x-t3-transcription-provider",
 ] as const;
 
 export const browserApiCorsHeaders = {

apps/server/src/server.test.ts

@@ -1300,6 +1300,8 @@ const assertBrowserApiCorsPreflightHeaders = (
     "content-type",
     "dpop",
     "traceparent",
+    "x-t3-transcription-api-key",
+    "x-t3-transcription-provider",
   ]);
 };
 const crossOriginClientOrigin = "http://remote-client.test:3773";
@@ -4209,6 +4211,8 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
         "content-type",
         "dpop",
         "traceparent",
+        "x-t3-transcription-api-key",
+        "x-t3-transcription-provider",
       ]);
     }).pipe(Effect.provide(NodeHttpServer.layerTest)),
   );

apps/server/src/server.ts

@@ -13,6 +13,7 @@ import * as ServerConfig from "./config.ts";
 import * as HttpResponseCompression from "./httpCompression/HttpResponseCompression.ts";
 import {
   otlpTracesProxyRouteLayer,
+  transcriptionRouteLayer,
   assetRouteLayer,
   serverEnvironmentHttpApiLayer,
   staticAndDevRouteLayer,
@@ -417,6 +418,7 @@ export const makeRoutesLayer = Layer.mergeAll(
       Layer.provide(environmentAuthenticatedAuthLayer),
     ),
     otlpTracesProxyRouteLayer,
+    transcriptionRouteLayer,
     assetRouteLayer,
     staticAndDevRouteLayer,
     websocketRpcRouteLayer,

apps/server/src/transcription.test.ts

@@ -0,0 +1,50 @@
+import { expect, it } from "@effect/vitest";
+import * as Effect from "effect/Effect";
+import * as Stream from "effect/Stream";
+import { describe } from "vite-plus/test";
+
+import {
+  MAX_TRANSCRIPTION_AUDIO_BYTES,
+  readTranscriptionAudio,
+  resolveTranscriptionProvider,
+  transcriptionProviderConfig,
+} from "./transcription.ts";
+
+describe("transcription providers", () => {
+  it("uses a fixed OpenAI endpoint and model", () => {
+    expect(resolveTranscriptionProvider("openai")).toBe("openai");
+    expect(transcriptionProviderConfig("openai")).toEqual({
+      endpoint: "https://api.openai.com/v1/audio/transcriptions",
+      model: "gpt-4o-mini-transcribe",
+    });
+  });
+
+  it("uses a fixed Groq endpoint and rejects custom providers", () => {
+    expect(resolveTranscriptionProvider("groq")).toBe("groq");
+    expect(transcriptionProviderConfig("groq")).toEqual({
+      endpoint: "https://api.groq.com/openai/v1/audio/transcriptions",
+      model: "whisper-large-v3-turbo",
+    });
+    expect(resolveTranscriptionProvider("http://127.0.0.1:8080/v1")).toBeNull();
+  });
+});
+
+describe("readTranscriptionAudio", () => {
+  it.effect("combines streamed audio chunks", () =>
+    Effect.gen(function* () {
+      const audio = yield* readTranscriptionAudio(
+        Stream.make(new Uint8Array([1, 2]), new Uint8Array([3, 4])),
+      );
+      expect(Array.from(audio)).toEqual([1, 2, 3, 4]);
+    }),
+  );
+
+  it.effect("stops when streamed audio exceeds the limit", () =>
+    Effect.gen(function* () {
+      const error = yield* readTranscriptionAudio(
+        Stream.make(new Uint8Array(MAX_TRANSCRIPTION_AUDIO_BYTES), new Uint8Array([1])),
+      ).pipe(Effect.flip);
+      expect(error._tag).toBe("TranscriptionAudioTooLargeError");
+    }),
+  );
+});

apps/server/src/transcription.ts

@@ -0,0 +1,203 @@
+import type { VoiceTranscriptionProvider } from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+import * as Stream from "effect/Stream";
+import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
+
+export const MAX_TRANSCRIPTION_AUDIO_BYTES = 25 * 1024 * 1024;
+
+const PROVIDERS = {
+  openai: {
+    endpoint: "https://api.openai.com/v1/audio/transcriptions",
+    model: "gpt-4o-mini-transcribe",
+  },
+  groq: {
+    endpoint: "https://api.groq.com/openai/v1/audio/transcriptions",
+    model: "whisper-large-v3-turbo",
+  },
+} as const satisfies Record<VoiceTranscriptionProvider, { endpoint: string; model: string }>;
+
+export interface VoiceTranscriptionInput {
+  readonly audio: Uint8Array;
+  readonly audioMimeType: string;
+  readonly provider: VoiceTranscriptionProvider;
+  readonly apiKey: string;
+}
+
+export class TranscriptionAudioTooLargeError extends Schema.TaggedErrorClass<TranscriptionAudioTooLargeError>()(
+  "TranscriptionAudioTooLargeError",
+  { receivedBytes: Schema.Number },
+) {
+  override get message(): string {
+    return "The recording exceeds the 25 MB limit.";
+  }
+}
+
+export class TranscriptionBodyReadError extends Schema.TaggedErrorClass<TranscriptionBodyReadError>()(
+  "TranscriptionBodyReadError",
+  { cause: Schema.Defect() },
+) {
+  override get message(): string {
+    return "Could not read the recording.";
+  }
+}
+
+export class TranscriptionEmptyAudioError extends Schema.TaggedErrorClass<TranscriptionEmptyAudioError>()(
+  "TranscriptionEmptyAudioError",
+  {},
+) {
+  override get message(): string {
+    return "The recording was empty.";
+  }
+}
+
+export class TranscriptionProviderUnsupportedError extends Schema.TaggedErrorClass<TranscriptionProviderUnsupportedError>()(
+  "TranscriptionProviderUnsupportedError",
+  {},
+) {
+  override get message(): string {
+    return "Select OpenAI or Groq for transcription.";
+  }
+}
+
+export class TranscriptionApiKeyMissingError extends Schema.TaggedErrorClass<TranscriptionApiKeyMissingError>()(
+  "TranscriptionApiKeyMissingError",
+  {},
+) {
+  override get message(): string {
+    return "Add an API key for the selected transcription provider.";
+  }
+}
+
+export class TranscriptionRequestError extends Schema.TaggedErrorClass<TranscriptionRequestError>()(
+  "TranscriptionRequestError",
+  {
+    provider: Schema.String,
+    cause: Schema.Defect(),
+  },
+) {
+  override get message(): string {
+    return "Could not reach the transcription provider.";
+  }
+}
+
+export class TranscriptionProviderError extends Schema.TaggedErrorClass<TranscriptionProviderError>()(
+  "TranscriptionProviderError",
+  {
+    provider: Schema.String,
+    providerStatus: Schema.Int,
+  },
+) {
+  override get message(): string {
+    return "The transcription provider rejected the request. Check the provider and API key.";
+  }
+}
+
+export class TranscriptionResponseError extends Schema.TaggedErrorClass<TranscriptionResponseError>()(
+  "TranscriptionResponseError",
+  {
+    provider: Schema.String,
+    cause: Schema.Defect(),
+  },
+) {
+  override get message(): string {
+    return "The transcription provider returned an invalid response.";
+  }
+}
+
+const TranscriptionResponse = Schema.Struct({ text: Schema.String });
+
+export function resolveTranscriptionProvider(provider: string): VoiceTranscriptionProvider | null {
+  return provider === "openai" || provider === "groq" ? provider : null;
+}
+
+export function transcriptionProviderConfig(provider: VoiceTranscriptionProvider) {
+  return PROVIDERS[provider];
+}
+
+export const readTranscriptionAudio = <E, R>(stream: Stream.Stream<Uint8Array, E, R>) =>
+  stream.pipe(
+    Stream.mapError((cause) => new TranscriptionBodyReadError({ cause })),
+    Stream.runFoldEffect(
+      () => ({ chunks: [] as Uint8Array[], size: 0 }),
+      (accumulator, chunk) => {
+        const size = accumulator.size + chunk.byteLength;
+        if (size > MAX_TRANSCRIPTION_AUDIO_BYTES) {
+          return Effect.fail(new TranscriptionAudioTooLargeError({ receivedBytes: size }));
+        }
+        accumulator.chunks.push(chunk);
+        return Effect.succeed({ chunks: accumulator.chunks, size });
+      },
+    ),
+    Effect.map(({ chunks, size }) => {
+      const audio = new Uint8Array(size);
+      let offset = 0;
+      for (const chunk of chunks) {
+        audio.set(chunk, offset);
+        offset += chunk.byteLength;
+      }
+      return audio;
+    }),
+  );
+
+function audioFileExtension(mimeType: string): string {
+  if (mimeType.includes("ogg")) return "ogg";
+  if (mimeType.includes("mp4") || mimeType.includes("m4a")) return "m4a";
+  if (mimeType.includes("wav")) return "wav";
+  return "webm";
+}
+
+export const forwardVoiceTranscription = Effect.fn("voiceTranscription.forward")(function* (
+  input: VoiceTranscriptionInput,
+) {
+  if (input.audio.byteLength === 0) {
+    return yield* new TranscriptionEmptyAudioError();
+  }
+  if (input.audio.byteLength > MAX_TRANSCRIPTION_AUDIO_BYTES) {
+    return yield* new TranscriptionAudioTooLargeError({
+      receivedBytes: input.audio.byteLength,
+    });
+  }
+  const apiKey = input.apiKey.trim();
+  if (!apiKey) {
+    return yield* new TranscriptionApiKeyMissingError();
+  }
+
+  const providerConfig = transcriptionProviderConfig(input.provider);
+  const mimeType = input.audioMimeType.split(";", 1)[0]?.trim() || "audio/webm";
+  const form = new FormData();
+  form.set("model", providerConfig.model);
+  form.set(
+    "file",
+    new Blob([input.audio], { type: mimeType }),
+    `recording.${audioFileExtension(mimeType)}`,
+  );
+
+  const httpClient = yield* HttpClient.HttpClient;
+  const payload = yield* HttpClientRequest.post(providerConfig.endpoint).pipe(
+    HttpClientRequest.bearerToken(apiKey),
+    HttpClientRequest.bodyFormData(form),
+    httpClient.execute,
+    Effect.mapError((cause) => new TranscriptionRequestError({ provider: input.provider, cause })),
+    Effect.flatMap((response) =>
+      Effect.gen(function* () {
+        if (response.status < 200 || response.status >= 300) {
+          return yield* new TranscriptionProviderError({
+            provider: input.provider,
+            providerStatus: response.status,
+          });
+        }
+        return yield* HttpClientResponse.schemaBodyJson(TranscriptionResponse)(response).pipe(
+          Effect.mapError(
+            (cause) => new TranscriptionResponseError({ provider: input.provider, cause }),
+          ),
+        );
+      }),
+    ),
+    Effect.timeout("2 minutes"),
+    Effect.catchTag("TimeoutError", (cause) =>
+      Effect.fail(new TranscriptionRequestError({ provider: input.provider, cause })),
+    ),
+  );
+  return payload.text.trim();
+});