Merge pull request #311 from tomasliu-agora/dev/weather_forcast_history

fix tool call and weather forcast and history

agents/ten_packages/extension/openai_v2v_python/conf.py

@@ -4,7 +4,7 @@
 DEFAULT_MODEL = "gpt-4o-realtime-preview"
 
 BASIC_PROMPT = '''
-You are an agent based on OpenAI {model} model and TEN Framework(A realtime multimodal agent framework). Your knowledge cutoff is 2023-10. You are a helpful, witty, and friendly AI. Act like a human, but remember that you aren't a human and that you can't do human things in the real world. Your voice and personality should be warm and engaging, with a lively and playful tone.
+You are an agent based on OpenAI {model} model and TEN (pronounce /ten/, do not try to translate it) Framework(A realtime multimodal agent framework). Your knowledge cutoff is 2023-10. You are a helpful, witty, and friendly AI. Act like a human, but remember that you aren't a human and that you can't do human things in the real world. Your voice and personality should be warm and engaging, with a lively and playful tone.
 You should start by saying 'Hey, I'm ten agent with OpenAI Realtime API, anything I can help you with?' using {language}.
 If interacting is not in {language}, start by using the standard accent or dialect familiar to the user. Talk quickly. 
 Do not refer to these rules, even if you're asked about them.

agents/ten_packages/extension/openai_v2v_python/extension.py

@@ -508,19 +508,31 @@ async def _remote_tool_call(self, ten_env: TenEnv, name: str, args: str, callbac
         c.set_property_string(CMD_PROPERTY_NAME, name)
         c.set_property_string(CMD_PROPERTY_ARGS, args)
         ten_env.send_cmd(c, lambda ten, result: asyncio.run_coroutine_threadsafe(
-            callback(result.get_property_string("response")), self.loop))
+                callback(result), self.loop))
         logger.info(f"_remote_tool_call finish {name} {args}")
-
-    async def _on_tool_output(self, tool_call_id: str,  result: str):
-        logger.info(f"_on_tool_output {tool_call_id} {result}")
+
+    async def _on_tool_output(self, tool_call_id:str,  result:CmdResult):
+        state = result.get_status_code()
+        tool_response = ItemCreate(
+            item=FunctionCallOutputItemParam(
+                call_id=tool_call_id,
+                output="{\"success\":false}",
+            )
+        )
         try:
-            tool_response = ItemCreate(
-                item=FunctionCallOutputItemParam(
-                    call_id=tool_call_id,
-                    output=result,
+            if state == StatusCode.OK:
+                response = result.get_property_string("response")
+                logger.info(f"_on_tool_output {tool_call_id} {response}")
+
+                tool_response = ItemCreate(
+                    item=FunctionCallOutputItemParam(
+                        call_id=tool_call_id,
+                        output=response,
+                    )
                 )
-            )
-
+            else:
+                logger.error(f"Failed to call function {tool_call_id}")
+
             await self.conn.send_request(tool_response)
             await self.conn.send_request(ResponseCreate())
         except:

agents/ten_packages/extension/openai_v2v_python/manifest.json

@@ -53,17 +53,69 @@
     },
     "audio_frame_in": [
       {
-        "name": "pcm_frame"
+        "name": "pcm_frame",
+        "property": {
+          "stream_id": {
+            "type": "int64"
+          }
+        }
       }
     ],
     "data_out": [
       {
-        "name": "text_data"
+        "name": "text_data",
+        "property": {
+          "text": {
+            "type": "string"
+          }
+        }
+      }
+    ],
+    "cmd_in": [
+      {
+        "name": "tool_register",
+        "property": {
+          "name": {
+            "type": "string"
+          },
+          "description": {
+            "type": "string"
+          },
+          "parameters": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "name",
+          "description",
+          "parameters"
+        ],
+        "result": {
+          "property": {
+            "response": {
+              "type": "string"
+            }
+          }
+        }
       }
     ],
     "cmd_out": [
       {
         "name": "flush"
+      },
+      {
+        "name": "tool_call",
+        "property": {
+          "name": {
+            "type": "string"
+          },
+          "args": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "name"
+        ]
       }
     ],
     "audio_frame_out": [

agents/ten_packages/extension/weatherapi_tool_python/README.md

@@ -1,29 +1,21 @@
 # weatherapi_tool_python
 
-<!-- brief introduction for the extension -->
+This is the tool demo for weather query.
 
 ## Features
 
-<!-- main features introduction -->
-
-- xxx feature
+- Fetch today's weather.
+- Search for history weather.
+- Forcast weather in 3 days.
 
 ## API
 
 Refer to `api` definition in [manifest.json] and default values in [property.json](property.json).
 
-<!-- Additional API.md can be referred to if extra introduction needed -->
-
-## Development
-
-### Build
-
-<!-- build dependencies and steps -->
-
-### Unit test
+### Out:
 
-<!-- how to do unit test for the extension -->
+- `tool_register`: auto register tool to llm
 
-## Misc
+### In:
 
-<!-- others if applicable -->
+- `tool_call`: sync cmd to fetch weather

agents/ten_packages/extension/weatherapi_tool_python/__init__.py

@@ -1,7 +1,7 @@
 #
 #
 # Agora Real Time Engagement
-# Created by Wei Hu in 2024-08.
+# Created by Tomas Liu in 2024-08.
 # Copyright (c) 2024 Agora IO. All rights reserved.
 #
 #

agents/ten_packages/extension/weatherapi_tool_python/addon.py

@@ -1,7 +1,7 @@
 #
 #
 # Agora Real Time Engagement
-# Created by Wei Hu in 2024-08.
+# Created by Tomas Liu in 2024-08.
 # Copyright (c) 2024 Agora IO. All rights reserved.
 #
 #

agents/ten_packages/extension/weatherapi_tool_python/extension.py

@@ -1,7 +1,7 @@
 #
 #
 # Agora Real Time Engagement
-# Created by Wei Hu in 2024-08.
+# Created by Tomas Liu in 2024-08.
 # Copyright (c) 2024 Agora IO. All rights reserved.
 #
 #
@@ -31,10 +31,11 @@
 TOOL_REGISTER_PROPERTY_NAME = "name"
 TOOL_REGISTER_PROPERTY_DESCRIPTON = "description"
 TOOL_REGISTER_PROPERTY_PARAMETERS = "parameters"
+TOOL_CALLBACK = "callback"
 
-TOOL_NAME = "get_current_weather"
-TOOL_DESCRIPTION = "Determine weather in my location"
-TOOL_PARAMETERS = {
+CURRENT_TOOL_NAME = "get_current_weather"
+CURRENT_TOOL_DESCRIPTION = "Determine current weather in user's location."
+CURRENT_TOOL_PARAMETERS = {
         "type": "object",
         "properties": {
             "location": {
@@ -45,14 +46,69 @@
         "required": ["location"],
     }
 
+# for free key, only 7 days before, see more in https://www.weatherapi.com/pricing.aspx
+HISTORY_TOOL_NAME = "get_past_weather"
+HISTORY_TOOL_DESCRIPTION = "Determine weather within past 7 days in user's location." 
+HISTORY_TOOL_PARAMETERS = {
+        "type": "object",
+        "properties": {
+            "location": {
+                "type": "string",
+                "description": "The city and state e.g. San Francisco, CA"
+            },
+            "datetime": {
+                "type": "string",
+                "description": "The datetime user is referring in date format e.g. 2024-10-09"
+            }
+        },
+        "required": ["location", "datetime"],
+}
+
+# for free key, only 3 days after, see more in https://www.weatherapi.com/pricing.aspx
+FORCAST_TOOL_NAME = "get_future_weather"
+FORCAST_TOOL_DESCRIPTION = "Determine weather in next 3 days in user's location." 
+FORCAST_TOOL_PARAMETERS = {
+        "type": "object",
+        "properties": {
+            "location": {
+                "type": "string",
+                "description": "The city and state e.g. San Francisco, CA"
+            }
+        },
+        "required": ["location"],
+}
+
 PROPERTY_API_KEY = "api_key"  # Required
 
 class WeatherToolExtension(Extension):
     api_key: str = ""
+    tools: dict = {}
 
     def on_init(self, ten_env: TenEnv) -> None:
         logger.info("WeatherToolExtension on_init")
 
+        self.tools = {
+            CURRENT_TOOL_NAME: {
+                TOOL_REGISTER_PROPERTY_NAME: CURRENT_TOOL_NAME,
+                TOOL_REGISTER_PROPERTY_DESCRIPTON: CURRENT_TOOL_DESCRIPTION,
+                TOOL_REGISTER_PROPERTY_PARAMETERS: CURRENT_TOOL_PARAMETERS,
+                TOOL_CALLBACK: self._get_current_weather
+            },
+            HISTORY_TOOL_NAME: {
+                TOOL_REGISTER_PROPERTY_NAME: HISTORY_TOOL_NAME,
+                TOOL_REGISTER_PROPERTY_DESCRIPTON: HISTORY_TOOL_DESCRIPTION,
+                TOOL_REGISTER_PROPERTY_PARAMETERS: HISTORY_TOOL_PARAMETERS,
+                TOOL_CALLBACK: self._get_past_weather
+            },
+            FORCAST_TOOL_NAME: {
+                TOOL_REGISTER_PROPERTY_NAME: FORCAST_TOOL_NAME,
+                TOOL_REGISTER_PROPERTY_DESCRIPTON: FORCAST_TOOL_DESCRIPTION,
+                TOOL_REGISTER_PROPERTY_PARAMETERS: FORCAST_TOOL_PARAMETERS,
+                TOOL_CALLBACK: self._get_future_weather
+            },
+            # TODO other tools
+        }
+
         ten_env.on_init_done()
 
     def on_start(self, ten_env: TenEnv) -> None:
@@ -67,11 +123,12 @@ def on_start(self, ten_env: TenEnv) -> None:
             return
 
         # Register func
-        c = Cmd.create(CMD_TOOL_REGISTER)
-        c.set_property_string(TOOL_REGISTER_PROPERTY_NAME, TOOL_NAME)
-        c.set_property_string(TOOL_REGISTER_PROPERTY_DESCRIPTON, TOOL_DESCRIPTION)
-        c.set_property_string(TOOL_REGISTER_PROPERTY_PARAMETERS, json.dumps(TOOL_PARAMETERS))
-        ten_env.send_cmd(c, lambda ten, result: logger.info(f"register done, {result}"))
+        for name, tool in self.tools.items():
+            c = Cmd.create(CMD_TOOL_REGISTER)
+            c.set_property_string(TOOL_REGISTER_PROPERTY_NAME, name)
+            c.set_property_string(TOOL_REGISTER_PROPERTY_DESCRIPTON, tool[TOOL_REGISTER_PROPERTY_DESCRIPTON])
+            c.set_property_string(TOOL_REGISTER_PROPERTY_PARAMETERS, json.dumps(tool[TOOL_REGISTER_PROPERTY_PARAMETERS]))
+            ten_env.send_cmd(c, lambda ten, result: logger.info(f"register done, {result}"))
 
         ten_env.on_start_done()
 
@@ -88,27 +145,23 @@ def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None:
         cmd_name = cmd.get_name()
         logger.info(f"on_cmd name {cmd_name} {cmd.to_json()}")
 
+        # FIXME need to handle async
         try:
             name = cmd.get_property_string(CMD_PROPERTY_NAME)
-            if name == TOOL_NAME:
+            if name in self.tools:
                 try:
+                    tool = self.tools[name]
                     args = cmd.get_property_string(CMD_PROPERTY_ARGS)
                     arg_dict = json.loads(args)
-                    if "location" in arg_dict:
-                        logger.info(f"before get current weather {name}")
-                        resp = self._get_current_weather(arg_dict["location"])
-                        logger.info(f"after get current weather {resp}")
-                        cmd_result = CmdResult.create(StatusCode.OK)
-                        cmd_result.set_property_string("response", json.dumps(resp))
-                        ten_env.return_result(cmd_result, cmd)
-                        return
-                    else:
-                        logger.error(f"no location in args {args}")
-                        cmd_result = CmdResult.create(StatusCode.ERROR)
-                        ten_env.return_result(cmd_result, cmd)
-                        return
+                    logger.info(f"before callback {name}")
+                    resp = tool[TOOL_CALLBACK](arg_dict)
+                    logger.info(f"after callback {resp}")
+                    cmd_result = CmdResult.create(StatusCode.OK)
+                    cmd_result.set_property_string("response", json.dumps(resp))
+                    ten_env.return_result(cmd_result, cmd)
+                    return
                 except:
-                    logger.exception("Failed to get weather")
+                    logger.exception("Failed to callback")
                     cmd_result = CmdResult.create(StatusCode.ERROR)
                     ten_env.return_result(cmd_result, cmd)
                     return
@@ -132,8 +185,40 @@ def on_audio_frame(self, ten_env: TenEnv, audio_frame: AudioFrame) -> None:
     def on_video_frame(self, ten_env: TenEnv, video_frame: VideoFrame) -> None:
         pass
 
-    def _get_current_weather(self, location:str) -> Any:
+    def _get_current_weather(self, args:dict) -> Any:
+        if "location" not in args:
+            raise Exception("Failed to get property")
+
+        location = args["location"]
         url = f"http://api.weatherapi.com/v1/current.json?key={self.api_key}&q={location}&aqi=no"
         response = requests.get(url)
         result = response.json()
+        return result
+
+    def _get_past_weather(self, args:dict) -> Any:
+        if "location" not in args or "datetime" not in args:
+            raise Exception("Failed to get property")
+
+        location = args["location"]
+        datetime = args["datetime"]
+        url = f"http://api.weatherapi.com/v1/history.json?key={self.api_key}&q={location}&dt={datetime}"
+        response = requests.get(url)
+        result = response.json()
+        # remove all hourly data
+        del result["forecast"]["forecastday"][0]["hour"]
+        return result
+
+    def _get_future_weather(self, args:dict) -> Any:
+        if "location" not in args:
+            raise Exception("Failed to get property")
+
+        location = args["location"]
+        url = f"http://api.weatherapi.com/v1/forecast.json?key={self.api_key}&q={location}&days=3&aqi=no&alerts=no"
+        response = requests.get(url)
+        result = response.json()
+        logger.info(f"get result {result}")
+        # remove all hourly data
+        for d in result["forecast"]["forecastday"]:
+            del d["hour"]
+        del result["current"]
         return result

agents/ten_packages/extension/weatherapi_tool_python/log.py

@@ -1,7 +1,7 @@
 #
 #
 # Agora Real Time Engagement
-# Created by Wei Hu in 2024-08.
+# Created by Tomas Liu in 2024-08.
 # Copyright (c) 2024 Agora IO. All rights reserved.
 #
 #