feat: support customizing prompts & greeting (#343)

* feat: support customizing prompts & greeting

* remove unnecessary package

* fix react build issue

* fix: support mobile & ignore empty values

demo/package.json

@@ -13,7 +13,7 @@
     "@ant-design/icons": "^5.3.7",
     "@reduxjs/toolkit": "^2.2.3",
     "agora-rtc-sdk-ng": "^4.21.0",
-    "antd": "^5.15.3",
+    "antd": "^5.21.4",
     "axios": "^1.7.7",
     "next": "14.2.4",
     "protobufjs": "^7.2.5",

demo/src/app/api/agents/start/graph.tsx

@@ -61,7 +61,13 @@ export const voiceNameMap: LanguageMap = {
 
 // Get the graph properties based on the graph name, language, and voice type
 // This is the place where you can customize the properties for different graphs to override default property.json
-export const getGraphProperties = (graphName: string, language: string, voiceType: string) => {
+export const getGraphProperties = (
+    graphName: string,
+    language: string,
+    voiceType: string,
+    prompt: string | undefined,
+    greeting: string | undefined
+) => {
     let localizationOptions = {
         "greeting": "Hey, I\'m TEN Agent, I can speak, see, and reason from a knowledge base, ask me anything!",
         "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]",
@@ -91,7 +97,9 @@ export const getGraphProperties = (graphName: string, language: string, voiceTyp
             },
             "openai_chatgpt": {
                 "model": "gpt-4o",
-                ...localizationOptions
+                ...localizationOptions,
+                "prompt": prompt,
+                "greeting": greeting,
             },
             "azure_tts": {
                 "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType]
@@ -103,7 +111,9 @@ export const getGraphProperties = (graphName: string, language: string, voiceTyp
                 "model": "gpt-4o-realtime-preview",
                 "voice": voiceNameMap[language]["openai"][voiceType],
                 "language": language,
-                ...localizationOptions
+                ...localizationOptions,
+                "system_message": prompt,
+                "greeting": greeting,
             }
         }
     } else if (graphName == "va.openai.azure") {
@@ -113,7 +123,9 @@ export const getGraphProperties = (graphName: string, language: string, voiceTyp
             },
             "openai_chatgpt": {
                 "model": "gpt-4o-mini",
-                ...localizationOptions
+                ...localizationOptions,
+                "prompt": prompt,
+                "greeting": greeting,
             },
             "azure_tts": {
                 "azure_synthesis_voice_name": voiceNameMap[language]["azure"][voiceType]

demo/src/app/api/agents/start/route.tsx

@@ -24,6 +24,8 @@ export async function POST(request: NextRequest) {
       graph_name,
       language,
       voice_type,
+      prompt,
+      greeting,
     } = body;
 
     console.log(`Starting agent for request ID: ${JSON.stringify({
@@ -32,7 +34,7 @@ export async function POST(request: NextRequest) {
       user_uid,
       graph_name,
       // Get the graph properties based on the graph name, language, and voice type
-      properties: getGraphProperties(graph_name, language, voice_type),
+      properties: getGraphProperties(graph_name, language, voice_type, prompt, greeting),
     })}`);
 
     console.log(`AGENT_SERVER_URL: ${AGENT_SERVER_URL}/start`);
@@ -44,7 +46,7 @@ export async function POST(request: NextRequest) {
       user_uid,
       graph_name,
       // Get the graph properties based on the graph name, language, and voice type
-      properties: getGraphProperties(graph_name, language, voice_type),
+      properties: getGraphProperties(graph_name, language, voice_type, prompt, greeting),
     });
 
     const responseData = response.data;

demo/src/app/home/page.tsx

@@ -26,9 +26,9 @@ export default function Home() {
 
   return (
     mobile === null ? <></> :
-    <AuthInitializer>
-      {mobile ? <MobileEntry></MobileEntry> : <PCEntry></PCEntry>}
-    </AuthInitializer >
+      <AuthInitializer>
+        {mobile ? <MobileEntry></MobileEntry> : <PCEntry></PCEntry>}
+      </AuthInitializer >
   );
 }
 

demo/src/common/constant.ts

@@ -1,11 +1,18 @@
 import { IOptions, ColorItem, LanguageOptionItem, VoiceOptionItem, GraphOptionItem } from "@/types"
 export const GITHUB_URL = "https://github.com/TEN-framework/TEN-Agent"
 export const OPTIONS_KEY = "__options__"
+export const AGENT_SETTINGS_KEY = "__agent_settings__"
 export const DEFAULT_OPTIONS: IOptions = {
   channel: "",
   userName: "",
   userId: 0
 }
+
+export const DEFAULT_AGENT_SETTINGS = {
+  greeting: "",
+  prompt: ""
+}
+
 export const DESCRIPTION = "The World's First Multimodal AI Agent with the OpenAI Realtime API (Beta)"
 export const LANGUAGE_OPTIONS: LanguageOptionItem[] = [
   {

demo/src/common/request.ts

@@ -8,6 +8,8 @@ interface StartRequestConfig {
   graphName: string,
   language: Language,
   voiceType: "male" | "female"
+  prompt?: string,
+  greeting?: string,
 }
 
 interface GenAgoraDataConfig {
@@ -32,14 +34,16 @@ export const apiGenAgoraData = async (config: GenAgoraDataConfig) => {
 export const apiStartService = async (config: StartRequestConfig): Promise<any> => {
   // look at app/api/agents/start/route.tsx for the server-side implementation
   const url = `/api/agents/start`
-  const { channel, userId, graphName, language, voiceType } = config
+  const { channel, userId, graphName, language, voiceType, greeting, prompt } = config
   const data = {
     request_id: genUUID(),
     channel_name: channel,
     user_uid: userId,
     graph_name: graphName,
     language,
-    voice_type: voiceType
+    voice_type: voiceType,
+    greeting: greeting ? greeting : undefined,
+    prompt: prompt ? prompt : undefined
   }
   let resp: any = await axios.post(url, data)
   resp = (resp.data) || {}

demo/src/common/storage.ts

@@ -1,14 +1,19 @@
-import { IOptions } from "@/types"
-import { OPTIONS_KEY, DEFAULT_OPTIONS } from "./constant"
+import { IAgentSettings, IOptions } from "@/types"
+import { OPTIONS_KEY, DEFAULT_OPTIONS, AGENT_SETTINGS_KEY, DEFAULT_AGENT_SETTINGS } from "./constant"
 
-export const getOptionsFromLocal = () => {
+export const getOptionsFromLocal = (): {options:IOptions, settings: IAgentSettings} => {
+  let data = {options: DEFAULT_OPTIONS, settings: DEFAULT_AGENT_SETTINGS}
   if (typeof window !== "undefined") {
-    const data = localStorage.getItem(OPTIONS_KEY)
-    if (data) {
-      return JSON.parse(data)
+    const options = localStorage.getItem(OPTIONS_KEY)
+    if (options) {
+      data.options = JSON.parse(options)
+    }
+    const settings = localStorage.getItem(AGENT_SETTINGS_KEY)
+    if (settings) {
+      data.settings = JSON.parse(settings)
     }
   }
-  return DEFAULT_OPTIONS
+  return data
 }
 
 
@@ -18,4 +23,8 @@ export const setOptionsToLocal = (options: IOptions) => {
   }
 }
 
-
+export const setAgentSettingsToLocal = (settings: IAgentSettings) => {
+  if (typeof window !== "undefined") {
+    localStorage.setItem(AGENT_SETTINGS_KEY, JSON.stringify(settings))
+  }
+}

demo/src/components/authInitializer/index.tsx

@@ -2,7 +2,7 @@
 
 import { ReactNode, useEffect } from "react"
 import { useAppDispatch, getOptionsFromLocal } from "@/common"
-import { setOptions, reset } from "@/store/reducers/global"
+import { setOptions, reset, setAgentSettings } from "@/store/reducers/global"
 
 interface AuthInitializerProps {
   children: ReactNode;
@@ -14,10 +14,11 @@ const AuthInitializer = (props: AuthInitializerProps) => {
 
   useEffect(() => {
     if (typeof window !== "undefined") {
-      const options = getOptionsFromLocal()
-      if (options) {
+      const data = getOptionsFromLocal()
+      if (data) {
         dispatch(reset())
-        dispatch(setOptions(options))
+        dispatch(setOptions(data.options))
+        dispatch(setAgentSettings(data.settings))
       }
     }
   }, [dispatch])

demo/src/components/loginCard/index.tsx

@@ -1,11 +1,11 @@
 "use client"
 
-import type React from 'react'; 
+import type React from 'react';
 import { useRouter } from 'next/navigation'
 import { message } from "antd"
 import { useState, useEffect } from "react"
 import { GITHUB_URL, getRandomUserId, useAppDispatch, getRandomChannel } from "@/common"
-import { setOptions } from "@/store/reducers/global"
+import { setAgentSettings, setOptions } from "@/store/reducers/global"
 import styles from "./index.module.scss"
 
 import { GithubIcon } from "../icons"
@@ -60,8 +60,8 @@ const LoginCard = () => {
 
   return <div className={styles.card}>
     <section className={styles.top}>
-      <span 
-        className={styles.github} 
+      <span
+        className={styles.github}
         onClick={onClickGithub}
         onKeyDown={(e) => {
           if (e.key === 'Enter' || e.key === ' ') {
@@ -84,8 +84,8 @@ const LoginCard = () => {
         <input placeholder="User Name" value={userName} onChange={onUserNameChange} />
       </div>
       <div className={styles.section}>
-        <div 
-          className={`${styles.btn} ${!isLoadingSuccess && styles.btnLoadingBg}`} 
+        <div
+          className={`${styles.btn} ${!isLoadingSuccess && styles.btnLoadingBg}`}
           onClick={onClickJoin}
           onKeyDown={(e) => {
             if (e.key === 'Enter' || e.key === ' ') {

demo/src/components/settings/index.tsx

@@ -0,0 +1,90 @@
+import React, { useState } from 'react';
+import { Modal, Form, Input, Button, FloatButton, ConfigProvider, theme } from 'antd';
+import { SettingOutlined } from '@ant-design/icons';
+import { useAppDispatch, useAppSelector } from '@/common';
+import { setAgentSettings } from '@/store/reducers/global';
+
+interface FormValues {
+    greeting: string;
+    prompt: string;
+}
+
+const FormModal: React.FC = () => {
+    const [isModalVisible, setIsModalVisible] = useState(false);
+    const [form] = Form.useForm<FormValues>();
+    const dispatch = useAppDispatch();
+    const agentSettings = useAppSelector(state => state.global.agentSettings);
+
+    const showModal = () => {
+        form.setFieldsValue(agentSettings);
+        setIsModalVisible(true);
+    };
+
+    const handleOk = async () => {
+        try {
+            const values = await form.validateFields();
+            console.log('Form Values:', values);
+            // Handle the form submission logic here
+            dispatch(setAgentSettings(values));
+            setIsModalVisible(false);
+            form.resetFields();
+        } catch (errorInfo) {
+            console.log('Validate Failed:', errorInfo);
+        }
+    };
+
+    const handleCancel = () => {
+        setIsModalVisible(false);
+    };
+
+    return (
+        <>
+            <ConfigProvider
+                theme={
+                    {
+                        algorithm: theme.darkAlgorithm,
+                        components: {
+                            Modal: {
+                                contentBg: "#272A2F",
+                                headerBg: "#272A2F",
+                                footerBg: "#272A2F",
+                            }
+                        }
+                    }
+                }>
+                <FloatButton type="primary" icon={<SettingOutlined></SettingOutlined>} onClick={showModal}>
+                </FloatButton>
+                <Modal
+                    title="Settings"
+                    visible={isModalVisible}
+                    onOk={handleOk}
+                    onCancel={handleCancel}
+                    okText="OK"
+                    cancelText="Cancel"
+                >
+                    <Form
+                        form={form}
+                        layout="vertical"
+                        name="form_in_modal"
+                    >
+                        <Form.Item
+                            name="greeting"
+                            label="Greeting"
+                        >
+                            <Input.TextArea rows={2} placeholder="Enter the greeting, leave it blank if you want to use default one." />
+                        </Form.Item>
+
+                        <Form.Item
+                            name="prompt"
+                            label="Prompt"
+                        >
+                            <Input.TextArea rows={4} placeholder="Enter the prompt, leave it blank if you want to use default one." />
+                        </Form.Item>
+                    </Form>
+                </Modal>
+            </ConfigProvider>
+        </>
+    );
+};
+
+export default FormModal;

demo/src/platform/mobile/description/index.tsx

@@ -18,6 +18,7 @@ const Description = () => {
   const language = useAppSelector(state => state.global.language)
   const voiceType = useAppSelector(state => state.global.voiceType)
   const graphName = useAppSelector(state => state.global.graphName)
+  const agentSettings = useAppSelector(state => state.global.agentSettings)
   const [loading, setLoading] = useState(false)
 
   useEffect(() => {
@@ -50,7 +51,9 @@ const Description = () => {
         userId,
         graphName,
         language,
-        voiceType
+        voiceType,
+        greeting: agentSettings.greeting,
+        prompt: agentSettings.prompt
       })
       const { code, msg } = res || {}
       if (code != 0) {

demo/src/platform/mobile/entry/index.tsx

@@ -4,6 +4,7 @@ import Rtc from "../rtc"
 import Header from "../header"
 import Menu, { IMenuData } from "../menu"
 import styles from "./index.module.scss"
+import FormModal from "@/components/settings"
 
 
 const MenuData: IMenuData[] = [{
@@ -23,6 +24,7 @@ const MobileEntry = () => {
     <div className={styles.content}>
       <Menu data={MenuData}></Menu>
     </div>
+    <FormModal></FormModal>
   </div>
 }
 

demo/src/platform/pc/description/index.module.scss

@@ -45,6 +45,11 @@
     caret-color: transparent;
     box-sizing: border-box;
 
+    &.btnSetting {
+      background: #181A1D;
+      border: 1px solid #272A2F;
+    }
+
     .btnText {
       width: 100px;
       text-align: center;
@@ -70,4 +75,4 @@
     border: 1px solid var(--Error-400-T, #E95C7B);
   }
 
-}
+}