diff --git a/agents/examples/demo/property.json b/agents/examples/demo/property.json index 2945bcfd..98c4f950 100644 --- a/agents/examples/demo/property.json +++ b/agents/examples/demo/property.json @@ -518,6 +518,7 @@ "language": "en-US", "server_vad": true, "dump": true, + "greeting": "TEN Agent connected. How can I help you today?", "max_history": 10 } }, @@ -534,7 +535,6 @@ "max_tokens": 512, "prompt": "", "proxy_url": "${env:OPENAI_PROXY_URL}", - "greeting": "TEN Agent connected. How can I help you today?", "checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]", "max_memory_length": 10, "enable_tools": true @@ -596,6 +596,26 @@ } ] } + ], + "cmd": [ + { + "name": "on_user_joined", + "dest": [ + { + "extension_group": "llm", + "extension": "openai_v2v_python" + } + ] + }, + { + "name": "on_user_left", + "dest": [ + { + "extension_group": "llm", + "extension": "openai_v2v_python" + } + ] + } ] }, { @@ -688,102 +708,19 @@ ] }, { - "name": "tool_call_get_current_weather", - "dest": [ - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" - } - ] - } - } - ] - }, - { - "name": "tool_call_get_past_weather", - "dest": [ - { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" - } - ] - } - } - ] - }, - { - "name": "tool_call_get_future_weather", + "name": "tool_call", "dest": [ { "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" - } - ] - } - } - ] - }, - { - "name": "tool_call_bing_search", - "dest": [ + "extension": "vision_analyze_tool_python" + }, { "extension_group": "tools", - "extension": "bingsearch_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" - } - ] - } - } - ] - }, - { - "name": "tool_call_get_vision_chat_completion", - "dest": [ + "extension": "weatherapi_tool_python" + }, { "extension_group": "tools", - "extension": "vision_analyze_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" - } - ] - } + "extension": "bingsearch_tool_python" } ] } @@ -848,6 +785,7 @@ "language": "en-US", "server_vad": true, "dump": true, + "greeting": "TEN Agent connected. How can I help you today?", "max_history": 10 } }, @@ -975,82 +913,33 @@ ] }, { - "name": "tool_call_get_current_weather", + "name": "tool_call", "dest": [ { "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" - } - ] - } - } - ] - }, - { - "name": "tool_call_get_past_weather", - "dest": [ + "extension": "weatherapi_tool_python" + }, { "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" - } - ] - } + "extension": "bingsearch_tool_python" } ] }, { - "name": "tool_call_get_future_weather", + "name": "on_user_joined", "dest": [ { - "extension_group": "tools", - "extension": "weatherapi_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" - } - ] - } + "extension_group": "llm", + "extension": "openai_v2v_python" } ] }, { - "name": "tool_call_bing_search", + "name": "on_user_left", "dest": [ { - "extension_group": "tools", - "extension": "bingsearch_tool_python", - "msg_conversion": { - "type": "per_property", - "keep_original": true, - "rules": [ - { - "path": "_ten.name", - "conversion_mode": "fixed_value", - "value": "tool_call" - } - ] - } + "extension_group": "llm", + "extension": "openai_v2v_python" } ] } diff --git a/agents/ten_packages/extension/openai_v2v_python/extension.py b/agents/ten_packages/extension/openai_v2v_python/extension.py index 61912b23..305ac179 100644 --- a/agents/ten_packages/extension/openai_v2v_python/extension.py +++ b/agents/ten_packages/extension/openai_v2v_python/extension.py @@ -126,6 +126,9 @@ async def on_start(self, ten_env: AsyncTenEnv) -> None: self.memory.on(EVENT_MEMORY_EXPIRED, self._on_memory_expired) self.memory.on(EVENT_MEMORY_APPENDED, self._on_memory_appended) + + self.ctx = self.config.build_ctx() + self.ctx["greeting"] = self.config.greeting self.conn = RealtimeApiConnection( ten_env=ten_env, @@ -179,7 +182,7 @@ async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None: elif cmd_name == CMD_IN_ON_USER_JOINED: self.users_count += 1 # Send greeting when first user joined - if self.connected and self.users_count == 1: + if self.users_count == 1: await self._greeting() elif cmd_name == CMD_IN_ON_USER_LEFT: self.users_count -= 1 @@ -217,7 +220,6 @@ def get_time_ms() -> int: # self.ten_env.log_info(f"Received message: {message.type}") match message: case SessionCreated(): - self.connected = True self.ten_env.log_info(f"Session is created: {message.session}") self.session_id = message.session.id self.session = message.session @@ -402,9 +404,6 @@ async def _on_audio(self, buff: bytearray): await self.conn.send_audio_data(self.buff) self.buff = b'' - self.ctx = self.config.build_ctx() - self.ctx["greeting"] = self.config.greeting - async def _update_session(self) -> None: tools = [] @@ -610,8 +609,11 @@ def _convert_to_content_parts(self, content: Iterable[LLMChatCompletionContentPa return content_parts async def _greeting(self) -> None: - if self.config.greeting: + if self.connected and self.users_count == 1: text = self._greeting_text() + if self.config.greeting: + text = "Say '" + self.config.greeting + "' to me." + self.ten_env.log_info(f"send greeting {text}") await self.conn.send_request(ItemCreate(item=UserMessageItemParam(content=[{"type": ContentType.InputText, "text": text}]))) await self.conn.send_request(ResponseCreate())