Skip to content

Commit

Permalink
Merge pull request #457 from TEN-framework/fix/v2v_greeting
Browse files Browse the repository at this point in the history 
fix: fix v2v greeting bug
  • Loading branch information
@TomasBack2Future
TomasBack2Future authored Dec 4, 2024
2 parents 4230fdb + a1e3b80 commit 232bb68
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 155 deletions.
187 changes: 38 additions & 149 deletions agents/examples/demo/property.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -518,6 +518,7 @@
"language": "en-US",
"server_vad": true,
"dump": true,
"greeting": "TEN Agent connected. How can I help you today?",
"max_history": 10
}
},
Expand All @@ -534,7 +535,6 @@
"max_tokens": 512,
"prompt": "",
"proxy_url": "${env:OPENAI_PROXY_URL}",
"greeting": "TEN Agent connected. How can I help you today?",
"checking_vision_text_items": "[\"Let me take a look...\",\"Let me check your camera...\",\"Please wait for a second...\"]",
"max_memory_length": 10,
"enable_tools": true
Expand Down Expand Up @@ -596,6 +596,26 @@
}
]
}
],
"cmd": [
{
"name": "on_user_joined",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
},
{
"name": "on_user_left",
"dest": [
{
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
]
},
{
Expand Down Expand Up @@ -688,102 +708,19 @@
]
},
{
"name": "tool_call_get_current_weather",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_get_past_weather",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_get_future_weather",
"name": "tool_call",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_bing_search",
"dest": [
"extension": "vision_analyze_tool_python"
},
{
"extension_group": "tools",
"extension": "bingsearch_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_get_vision_chat_completion",
"dest": [
"extension": "weatherapi_tool_python"
},
{
"extension_group": "tools",
"extension": "vision_analyze_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
"extension": "bingsearch_tool_python"
}
]
}
Expand Down Expand Up @@ -848,6 +785,7 @@
"language": "en-US",
"server_vad": true,
"dump": true,
"greeting": "TEN Agent connected. How can I help you today?",
"max_history": 10
}
},
Expand Down Expand Up @@ -975,82 +913,33 @@
]
},
{
"name": "tool_call_get_current_weather",
"name": "tool_call",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
}
]
},
{
"name": "tool_call_get_past_weather",
"dest": [
"extension": "weatherapi_tool_python"
},
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
"extension": "bingsearch_tool_python"
}
]
},
{
"name": "tool_call_get_future_weather",
"name": "on_user_joined",
"dest": [
{
"extension_group": "tools",
"extension": "weatherapi_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
},
{
"name": "tool_call_bing_search",
"name": "on_user_left",
"dest": [
{
"extension_group": "tools",
"extension": "bingsearch_tool_python",
"msg_conversion": {
"type": "per_property",
"keep_original": true,
"rules": [
{
"path": "_ten.name",
"conversion_mode": "fixed_value",
"value": "tool_call"
}
]
}
"extension_group": "llm",
"extension": "openai_v2v_python"
}
]
}
Expand Down
14 changes: 8 additions & 6 deletions agents/ten_packages/extension/openai_v2v_python/extension.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,9 @@ async def on_start(self, ten_env: AsyncTenEnv) -> None:

self.memory.on(EVENT_MEMORY_EXPIRED, self._on_memory_expired)
self.memory.on(EVENT_MEMORY_APPENDED, self._on_memory_appended)

self.ctx = self.config.build_ctx()
self.ctx["greeting"] = self.config.greeting

self.conn = RealtimeApiConnection(
ten_env=ten_env,
Expand Down Expand Up @@ -179,7 +182,7 @@ async def on_cmd(self, ten_env: AsyncTenEnv, cmd: Cmd) -> None:
elif cmd_name == CMD_IN_ON_USER_JOINED:
self.users_count += 1
# Send greeting when first user joined
if self.connected and self.users_count == 1:
if self.users_count == 1:
await self._greeting()
elif cmd_name == CMD_IN_ON_USER_LEFT:
self.users_count -= 1
Expand Down Expand Up @@ -217,7 +220,6 @@ def get_time_ms() -> int:
# self.ten_env.log_info(f"Received message: {message.type}")
match message:
case SessionCreated():
self.connected = True
self.ten_env.log_info(f"Session is created: {message.session}")
self.session_id = message.session.id
self.session = message.session
Expand Down Expand Up @@ -402,9 +404,6 @@ async def _on_audio(self, buff: bytearray):
await self.conn.send_audio_data(self.buff)
self.buff = b''

self.ctx = self.config.build_ctx()
self.ctx["greeting"] = self.config.greeting

async def _update_session(self) -> None:
tools = []

Expand Down Expand Up @@ -610,8 +609,11 @@ def _convert_to_content_parts(self, content: Iterable[LLMChatCompletionContentPa
return content_parts

async def _greeting(self) -> None:
if self.config.greeting:
if self.connected and self.users_count == 1:
text = self._greeting_text()
if self.config.greeting:
text = "Say '" + self.config.greeting + "' to me."
self.ten_env.log_info(f"send greeting {text}")
await self.conn.send_request(ItemCreate(item=UserMessageItemParam(content=[{"type": ContentType.InputText, "text": text}])))
await self.conn.send_request(ResponseCreate())

Expand Down

0 comments on commit 232bb68

Please sign in to comment.