Feat/add bytedance doubao plugin

livekit-agents/pyproject.toml

@@ -85,6 +85,7 @@ mistralai = ["livekit-plugins-mistralai>=1.2.14"]
 neuphonic = ["livekit-plugins-neuphonic>=1.2.14"]
 nltk = ["livekit-plugins-nltk>=1.2.14"]
 openai = ["livekit-plugins-openai>=1.2.14"]
+doubao = ["livekit-plugins-doubao>=1.2.14"]
 playai = ["livekit-plugins-playai>=1.2.14"]
 resemble = ["livekit-plugins-resemble>=1.2.14"]
 rime = ["livekit-plugins-rime>=1.2.14"]

livekit-plugins/livekit-plugins-doubao/README.md

@@ -0,0 +1,15 @@
+# Doubao plugin for LiveKit Agents
+
+Support for ByteDance Doubao realtime dialogue, LLM, TTS, and STT APIs.
+
+See [https://docs.livekit.io/agents/integrations/llm/doubao/](https://docs.livekit.io/agents/integrations/llm/doubao/) for more information.
+
+## Installation
+
+```bash
+pip install livekit-plugins-doubao
+```
+
+## Pre-requisites
+
+You'll need Doubao credentials to call the APIs. Set them as environment variables such as `DOUBAO_APP_ID`, `DOUBAO_ACCESS_KEY`, and `DOUBAO_ACCESS_TOKEN`.

livekit-plugins/livekit-plugins-doubao/livekit/plugins/doubao/__init__.py

@@ -0,0 +1,61 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Doubao plugin for LiveKit Agents
+
+Support for ByteDance Doubao realtime dialogue, LLM, TTS, and STT APIs.
+
+See https://docs.livekit.io/agents/integrations/llm/doubao/ for more information.
+"""
+
+from .embeddings import EmbeddingData, create_embeddings
+from .llm import LLM, LLMStream
+from .models import STTModels, TTSModels, TTSVoices
+from .stt import STT
+from .tts import TTS
+from .version import __version__
+
+__all__ = [
+    "STT",
+    "TTS",
+    "LLM",
+    "LLMStream",
+    "STTModels",
+    "TTSModels",
+    "TTSVoices",
+    "create_embeddings",
+    "EmbeddingData",
+    "__version__",
+]
+
+from livekit.agents import Plugin
+
+from .log import logger
+
+
+class DoubaoPlugin(Plugin):
+    def __init__(self) -> None:
+        super().__init__(__name__, __version__, __package__, logger)
+
+
+Plugin.register_plugin(DoubaoPlugin())
+
+# Cleanup docs of unexported modules
+_module = dir()
+NOT_IN_ALL = [m for m in _module if m not in __all__]
+
+__pdoc__ = {}
+
+for n in NOT_IN_ALL:
+    __pdoc__[n] = False

livekit-plugins/livekit-plugins-doubao/livekit/plugins/doubao/embeddings.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import base64
+import os
+import struct
+from dataclasses import dataclass
+
+import aiohttp
+
+from livekit.agents import utils
+
+from . import models
+
+
+@dataclass
+class EmbeddingData:
+    index: int
+    embedding: list[float]
+
+
+async def create_embeddings(
+    *,
+    input: list[str],
+    model: models.EmbeddingModels = "text-embedding-3-small",
+    dimensions: int | None = None,
+    api_key: str | None = None,
+    http_session: aiohttp.ClientSession | None = None,
+) -> list[EmbeddingData]:
+    http_session = http_session or utils.http_context.http_session()
+
+    api_key = api_key or os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        raise ValueError("OPENAI_API_KEY must be set")
+
+    async with http_session.post(
+        "https://api.openai.com/v1/embeddings",
+        headers={"Authorization": f"Bearer {api_key}"},
+        json={
+            "model": model,
+            "input": input,
+            "encoding_format": "base64",
+            "dimensions": dimensions,
+        },
+    ) as resp:
+        json = await resp.json()
+        data = json["data"]
+        list_data = []
+        for d in data:
+            bytes = base64.b64decode(d["embedding"])
+            num_floats = len(bytes) // 4
+            floats = list(struct.unpack("f" * num_floats, bytes))
+            list_data.append(EmbeddingData(index=d["index"], embedding=floats))
+
+        return list_data