Skip to content

Commit

Permalink
feat: realtime config
Browse files Browse the repository at this point in the history
  • Loading branch information
Dogtiti committed Nov 7, 2024
1 parent 283caba commit e44ebe3
Show file tree
Hide file tree
Showing 7 changed files with 320 additions and 58 deletions.
12 changes: 7 additions & 5 deletions app/components/chat.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -793,11 +793,13 @@ export function ChatActions(props: {
)}
</>
<div className={styles["chat-input-actions-end"]}>
<ChatAction
onClick={() => props.setShowChatSidePanel(true)}
text={"Realtime Chat"}
icon={<HeadphoneIcon />}
/>
{config.realtimeConfig.enable && (
<ChatAction
onClick={() => props.setShowChatSidePanel(true)}
text={"Realtime Chat"}
icon={<HeadphoneIcon />}
/>
)}
</div>
</div>
);
Expand Down
98 changes: 46 additions & 52 deletions app/components/realtime-chat/realtime-chat.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { useDebouncedCallback } from "use-debounce";
import VoiceIcon from "@/app/icons/voice.svg";
import VoiceOffIcon from "@/app/icons/voice-off.svg";
import PowerIcon from "@/app/icons/power.svg";
Expand All @@ -8,12 +7,7 @@ import clsx from "clsx";

import { useState, useRef, useEffect } from "react";

import {
useAccessStore,
useChatStore,
ChatMessage,
createMessage,
} from "@/app/store";
import { useChatStore, createMessage, useAppConfig } from "@/app/store";

import { IconButton } from "@/app/components/button";

Expand All @@ -23,7 +17,6 @@ import {
RTInputAudioItem,
RTResponse,
TurnDetection,
Voice,
} from "rt-client";
import { AudioHandler } from "@/app/lib/audio";
import { uploadImage } from "@/app/utils/chat";
Expand All @@ -39,41 +32,40 @@ export function RealtimeChat({
onStartVoice,
onPausedVoice,
}: RealtimeChatProps) {
const currentItemId = useRef<string>("");
const currentBotMessage = useRef<ChatMessage | null>();
const currentUserMessage = useRef<ChatMessage | null>();
const accessStore = useAccessStore.getState();
const chatStore = useChatStore();
const session = chatStore.currentSession();

const config = useAppConfig();
const [status, setStatus] = useState("");
const [isRecording, setIsRecording] = useState(false);
const [isConnected, setIsConnected] = useState(false);
const [isConnecting, setIsConnecting] = useState(false);
const [modality, setModality] = useState("audio");
const [isAzure, setIsAzure] = useState(false);
const [endpoint, setEndpoint] = useState("");
const [deployment, setDeployment] = useState("");
const [useVAD, setUseVAD] = useState(true);
const [voice, setVoice] = useState<Voice>("alloy");
const [temperature, setTemperature] = useState(0.9);

const clientRef = useRef<RTClient | null>(null);
const audioHandlerRef = useRef<AudioHandler | null>(null);
const initRef = useRef(false);

const apiKey = accessStore.openaiApiKey;
const temperature = config.realtimeConfig.temperature;
const apiKey = config.realtimeConfig.apiKey;
const model = config.realtimeConfig.model;
const azure = config.realtimeConfig.provider === "Azure";
const azureEndpoint = config.realtimeConfig.azure.endpoint;
const azureDeployment = config.realtimeConfig.azure.deployment;
const voice = config.realtimeConfig.voice;

const handleConnect = async () => {
if (isConnecting) return;
if (!isConnected) {
try {
setIsConnecting(true);
clientRef.current = isAzure
? new RTClient(new URL(endpoint), { key: apiKey }, { deployment })
: new RTClient(
clientRef.current = azure
? new RTClient(
new URL(azureEndpoint),
{ key: apiKey },
{ model: "gpt-4o-realtime-preview-2024-10-01" },
);
{ deployment: azureDeployment },
)
: new RTClient({ key: apiKey }, { model });
const modalities: Modality[] =
modality === "audio" ? ["text", "audio"] : ["text"];
const turnDetection: TurnDetection = useVAD
Expand Down Expand Up @@ -191,7 +183,6 @@ export function RealtimeChat({
const blob = audioHandlerRef.current?.savePlayFile();
uploadImage(blob!).then((audio_url) => {
botMessage.audio_url = audio_url;
// botMessage.date = new Date().toLocaleString();
// update text and audio_url
chatStore.updateTargetSession(session, (session) => {
session.messages = session.messages.concat();
Expand Down Expand Up @@ -258,31 +249,32 @@ export function RealtimeChat({
}
};

useEffect(
useDebouncedCallback(() => {
const initAudioHandler = async () => {
const handler = new AudioHandler();
await handler.initialize();
audioHandlerRef.current = handler;
await handleConnect();
await toggleRecording();
};
useEffect(() => {
// 防止重复初始化
if (initRef.current) return;
initRef.current = true;

initAudioHandler().catch((error) => {
setStatus(error);
console.error(error);
});
const initAudioHandler = async () => {
const handler = new AudioHandler();
await handler.initialize();
audioHandlerRef.current = handler;
await handleConnect();
await toggleRecording();
};

return () => {
if (isRecording) {
toggleRecording();
}
audioHandlerRef.current?.close().catch(console.error);
disconnect();
};
}),
[],
);
initAudioHandler().catch((error) => {
setStatus(error);
console.error(error);
});

return () => {
if (isRecording) {
toggleRecording();
}
audioHandlerRef.current?.close().catch(console.error);
disconnect();
};
}, []);

// update session params
useEffect(() => {
Expand All @@ -304,26 +296,28 @@ export function RealtimeChat({
<div className={styles["realtime-chat"]}>
<div
className={clsx(styles["circle-mic"], {
[styles["pulse"]]: true,
[styles["pulse"]]: isRecording,
})}
>
<div className={styles["icon-center"]}></div>
</div>
<div className={styles["bottom-icons"]}>
<div>
<IconButton
icon={isRecording ? <VoiceOffIcon /> : <VoiceIcon />}
icon={isRecording ? <VoiceIcon /> : <VoiceOffIcon />}
onClick={toggleRecording}
disabled={!isConnected}
type={isRecording ? "danger" : isConnected ? "primary" : null}
shadow
bordered
/>
</div>
<div className={styles["icon-center"]}>{status}</div>
<div>
<IconButton
icon={<PowerIcon />}
onClick={handleClose}
type={isConnecting || isConnected ? "danger" : "primary"}
shadow
bordered
/>
</div>
</div>
Expand Down
173 changes: 173 additions & 0 deletions app/components/realtime-chat/realtime-config.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
import { RealtimeConfig } from "@/app/store";

import Locale from "@/app/locales";
import { ListItem, Select, PasswordInput } from "@/app/components/ui-lib";

import { InputRange } from "@/app/components/input-range";
import { Voice } from "rt-client";
import { ServiceProvider } from "@/app/constant";

const providers = [ServiceProvider.OpenAI, ServiceProvider.Azure];

const models = ["gpt-4o-realtime-preview-2024-10-01"];

const voice = ["alloy", "shimmer", "echo"];

export function RealtimeConfigList(props: {
realtimeConfig: RealtimeConfig;
updateConfig: (updater: (config: RealtimeConfig) => void) => void;
}) {
const azureConfigComponent = props.realtimeConfig.provider ===
ServiceProvider.Azure && (
<>
<ListItem
title={Locale.Settings.Realtime.Azure.Endpoint.Title}
subTitle={Locale.Settings.Realtime.Azure.Endpoint.SubTitle}
>
<input
value={props.realtimeConfig?.azure?.endpoint}
type="text"
placeholder={Locale.Settings.Realtime.Azure.Endpoint.Title}
onChange={(e) => {
props.updateConfig(
(config) => (config.azure.endpoint = e.currentTarget.value),
);
}}
/>
</ListItem>
<ListItem
title={Locale.Settings.Realtime.Azure.Deployment.Title}
subTitle={Locale.Settings.Realtime.Azure.Deployment.SubTitle}
>
<input
value={props.realtimeConfig?.azure?.deployment}
type="text"
placeholder={Locale.Settings.Realtime.Azure.Deployment.Title}
onChange={(e) => {
props.updateConfig(
(config) => (config.azure.deployment = e.currentTarget.value),
);
}}
/>
</ListItem>
</>
);

return (
<>
<ListItem
title={Locale.Settings.Realtime.Enable.Title}
subTitle={Locale.Settings.Realtime.Enable.SubTitle}
>
<input
type="checkbox"
checked={props.realtimeConfig.enable}
onChange={(e) =>
props.updateConfig(
(config) => (config.enable = e.currentTarget.checked),
)
}
></input>
</ListItem>

{props.realtimeConfig.enable && (
<>
<ListItem
title={Locale.Settings.Realtime.Provider.Title}
subTitle={Locale.Settings.Realtime.Provider.SubTitle}
>
<Select
aria-label={Locale.Settings.Realtime.Provider.Title}
value={props.realtimeConfig.provider}
onChange={(e) => {
props.updateConfig(
(config) =>
(config.provider = e.target.value as ServiceProvider),
);
}}
>
{providers.map((v, i) => (
<option value={v} key={i}>
{v}
</option>
))}
</Select>
</ListItem>
<ListItem
title={Locale.Settings.Realtime.Model.Title}
subTitle={Locale.Settings.Realtime.Model.SubTitle}
>
<Select
aria-label={Locale.Settings.Realtime.Model.Title}
value={props.realtimeConfig.model}
onChange={(e) => {
props.updateConfig((config) => (config.model = e.target.value));
}}
>
{models.map((v, i) => (
<option value={v} key={i}>
{v}
</option>
))}
</Select>
</ListItem>
<ListItem
title={Locale.Settings.Realtime.ApiKey.Title}
subTitle={Locale.Settings.Realtime.ApiKey.SubTitle}
>
<PasswordInput
aria={Locale.Settings.ShowPassword}
aria-label={Locale.Settings.Realtime.ApiKey.Title}
value={props.realtimeConfig.apiKey}
type="text"
placeholder={Locale.Settings.Realtime.ApiKey.Placeholder}
onChange={(e) => {
props.updateConfig(
(config) => (config.apiKey = e.currentTarget.value),
);
}}
/>
</ListItem>
{azureConfigComponent}
<ListItem
title={Locale.Settings.TTS.Voice.Title}
subTitle={Locale.Settings.TTS.Voice.SubTitle}
>
<Select
value={props.realtimeConfig.voice}
onChange={(e) => {
props.updateConfig(
(config) => (config.voice = e.currentTarget.value as Voice),
);
}}
>
{voice.map((v, i) => (
<option value={v} key={i}>
{v}
</option>
))}
</Select>
</ListItem>
<ListItem
title={Locale.Settings.Realtime.Temperature.Title}
subTitle={Locale.Settings.Realtime.Temperature.SubTitle}
>
<InputRange
aria={Locale.Settings.Temperature.Title}
value={props.realtimeConfig?.temperature?.toFixed(1)}
min="0.6"
max="1"
step="0.1"
onChange={(e) => {
props.updateConfig(
(config) =>
(config.temperature = e.currentTarget.valueAsNumber),
);
}}
></InputRange>
</ListItem>
</>
)}
</>
);
}
Loading

0 comments on commit e44ebe3

Please sign in to comment.