From ac52373fdd9ece260687b7d1f6a44bf4c244ed98 Mon Sep 17 00:00:00 2001 From: Byron Xu Date: Fri, 12 Jan 2024 07:59:03 -0800 Subject: [PATCH 1/7] add SQL agent and Spider environment --- test/spider_env.py | 141 +++++++++++++++++++++++++++++++++++++++++++++ test/sqlagent.py | 52 +++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 test/spider_env.py create mode 100644 test/sqlagent.py diff --git a/test/spider_env.py b/test/spider_env.py new file mode 100644 index 000000000000..40400f618e76 --- /dev/null +++ b/test/spider_env.py @@ -0,0 +1,141 @@ +def download_spider(cache_dir: str) -> None: + import os + import zipfile + + # Not guaranteed to work indefinitely. + cmd = f'mkdir -p {cache_dir}; cd {cache_dir}; curl --insecure -LOJ "https://drive.google.com/uc?export=download&id=1iRDVHLr4mX2wQKSgA9J8Pire73Jahh0m&confirm="' + os.system(cmd) + + with zipfile.ZipFile(os.path.join(cache_dir, "spider.zip")) as zf: + zf.extractall(cache_dir) + assert os.path.isfile(os.path.join(cache_dir, "spider/train_spider.json")) + + +class SpiderEnv: + def __init__(self, cache_dir: str = "~/.cache/spider", random_seed=666): + import json + import os + import numpy as np + + # TODO: logger. + self._rng = np.random.default_rng(random_seed) + + cache_dir = os.path.expanduser(cache_dir) + self._db_dir = os.path.join(cache_dir, "spider/database") + + # Download and unzip the dataset if non-existent. + data_file = os.path.join(cache_dir, "spider/train_spider.json") + if not os.path.isfile(data_file): + print(f"Downloading Spider dataset to {cache_dir}") + download_spider(cache_dir) + else: + print(f"Loading cached Spider dataset from {cache_dir}") + + # TODO: Use other train & dev files. + with open(data_file) as f: + self._dataset = json.load(f) + + # Try to load every unique schema + unique_db_ids = set(data["db_id"] for data in self._dataset) + error_db_ids = [] + for db_id in unique_db_ids: + schema = self._get_schema(db_id) + if schema is None: + error_db_ids.append(db_id) + + # Remove data with schema errors + self._dataset = [ + data for data in self._dataset if data["db_id"] not in error_db_ids + ] + + def _get_schema(self, db_id: str) -> str: + import glob + + schema_files = glob.glob(f"{self._db_dir}/{db_id}/*.sql") + if len(schema_files) == 0: + print(f"Schema file not found for {self._db_dir}/{db_id}") + return None + if len(schema_files) > 1: + print(f"Multiple schema files found for {self._db_dir}/{db_id}") + return None + + try: + with open(schema_files[0]) as f: + # Extract all the "CREATE TABLE (...);" statements + schema = "" + in_create_table = False + for line in f: + line = line.strip() + if "CREATE TABLE " in line.upper(): + in_create_table = True + if in_create_table: + schema += line + "\n" + if ");" in line: + in_create_table = False + schema = schema.replace("`", '"') + except Exception as e: + print(e) + return None + + return schema + + def _run_query(self, db_id: str, query: str): + import os + import sqlite3 + + con = sqlite3.connect(os.path.join(self._db_dir, f"{db_id}/{db_id}.sqlite")) + cur = con.cursor() + result, error = None, None + + try: + result = cur.execute(query).fetchall() + except Exception as e: + error = str(e) + + return result, error + + def reset(self, k: int = None): + if k is None: + # Get a random question. + self._k = self._rng.integers(len(self._dataset)) # TODO: Replacement? + else: + self._k = k + + data = self._dataset[self._k] + db_id = data["db_id"] + + observation = { + "observation": db_id, + "instruction": data["question"], + "feedback": None, + } + self._info = { + "schema": self._get_schema(db_id), + "gold_query": data["query"], + "gold_result": self._run_query(db_id, data["query"])[0], + } + + return observation, self._info + + def step(self, query: str): + data = self._dataset[self._k] + db_id = data["db_id"] + + result, error = self._run_query(db_id, query) + + if error is not None: + reward = 0.0 + else: + # TODO: Add another reward for query exact_set_match comparing with query_toks_no_value. + reward = 1.0 if result == self._info["gold_result"] else 0.0 + + observation = { + "observation": db_id, + "instruction": data["question"], + "feedback": {"result": result, "error": error}, + } + + # TODO: Handle these. + terminated, truncated = False, False + + return observation, reward, terminated, truncated, self._info diff --git a/test/sqlagent.py b/test/sqlagent.py new file mode 100644 index 000000000000..d06a96b26e96 --- /dev/null +++ b/test/sqlagent.py @@ -0,0 +1,52 @@ +from autogen import AssistantAgent, UserProxyAgent, ConversableAgent, config_list_from_json +from spider_env import SpiderEnv +from typing import Annotated, Dict +import json + +# Load LLM inference endpoints from an env variable or a file +# See https://microsoft.github.io/autogen/docs/FAQ#set-your-api-endpoints +# and OAI_CONFIG_LIST_sample +config_list = config_list_from_json(env_or_file="OAI_CONFIG_LIST") + +def check_termination(msg: Dict): + if 'tool_responses' not in msg: + return False + json_str = msg['tool_responses'][0]['content'] + obj = json.loads(json_str) + return "error" not in obj or obj["error"] is None and obj["reward"] == 1 + +sql_writer = ConversableAgent("sql_writer", + llm_config={"config_list": config_list}, + system_message="You are good at writing SQL queries. Always respond with a function call to execute_sql().", + is_termination_msg=check_termination) +user_proxy = UserProxyAgent("user_proxy", human_input_mode="NEVER", max_consecutive_auto_reply=5) +spider = SpiderEnv() + +@sql_writer.register_for_llm(description="Function for executing SQL query and returning a response") +@user_proxy.register_for_execution() +def execute_sql(reflection: Annotated[str, "Think about what to do"], sql: Annotated[str, "SQL query"]) -> Annotated[Dict[str, str], "Dictionary with keys 'result' and 'error'"]: + observation, reward, terminated, truncated, info = spider.step(sql) + error = observation["feedback"]["error"] + if not error and reward == 0: + error = "The SQL query returned an incorrect result" + if error: + return { + "error": error, + "wrong_result": observation["feedback"]["result"], + "correct_result": info["gold_result"], + } + else: + return { + "result": observation["feedback"]["result"], + } + +for i in range(100): + observation, info = spider.reset() + question = observation["instruction"] + schema = info["schema"] + message = f"""Below is the schema for a SQL database: + {schema} + Generate a SQL query to answer the following question: + {question} + """ + user_proxy.initiate_chat(sql_writer, message=message) From 00674d73cf4e65fea91b118946304f72eb7dfc6e Mon Sep 17 00:00:00 2001 From: Wangda Zhang Date: Fri, 12 Jan 2024 09:51:20 -0800 Subject: [PATCH 2/7] Make a notebook. Clean up environment. --- notebook/agentchat_sql.ipynb | 100 +++++++++++++++++++++++++ test/spider_env.py | 141 ----------------------------------- test/sqlagent.py | 52 ------------- 3 files changed, 100 insertions(+), 193 deletions(-) create mode 100644 notebook/agentchat_sql.ipynb delete mode 100644 test/spider_env.py delete mode 100644 test/sqlagent.py diff --git a/notebook/agentchat_sql.ipynb b/notebook/agentchat_sql.ipynb new file mode 100644 index 000000000000..ab9106b2c90c --- /dev/null +++ b/notebook/agentchat_sql.ipynb @@ -0,0 +1,100 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SQL Agent" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from autogen import AssistantAgent, UserProxyAgent, ConversableAgent, config_list_from_json\n", + "from spider_env import SpiderEnv\n", + "from typing import Annotated, Dict\n", + "import json\n", + "\n", + "# Load LLM inference endpoints from an env variable or a file\n", + "# See https://microsoft.github.io/autogen/docs/FAQ#set-your-api-endpoints\n", + "# and OAI_CONFIG_LIST_sample\n", + "config_list = config_list_from_json(env_or_file=\"OAI_CONFIG_LIST\")\n", + "\n", + "\n", + "def check_termination(msg: Dict):\n", + " if \"tool_responses\" not in msg:\n", + " return False\n", + " json_str = msg[\"tool_responses\"][0][\"content\"]\n", + " obj = json.loads(json_str)\n", + " return \"error\" not in obj or obj[\"error\"] is None and obj[\"reward\"] == 1\n", + "\n", + "\n", + "sql_writer = ConversableAgent(\n", + " \"sql_writer\",\n", + " llm_config={\"config_list\": config_list},\n", + " system_message=\"You are good at writing SQL queries. Always respond with a function call to execute_sql().\",\n", + " is_termination_msg=check_termination,\n", + ")\n", + "user_proxy = UserProxyAgent(\"user_proxy\", human_input_mode=\"NEVER\", max_consecutive_auto_reply=5)\n", + "spider = SpiderEnv()\n", + "\n", + "\n", + "@sql_writer.register_for_llm(description=\"Function for executing SQL query and returning a response\")\n", + "@user_proxy.register_for_execution()\n", + "def execute_sql(\n", + " reflection: Annotated[str, \"Think about what to do\"], sql: Annotated[str, \"SQL query\"]\n", + ") -> Annotated[Dict[str, str], \"Dictionary with keys 'result' and 'error'\"]:\n", + " observation, reward, terminated, truncated, info = spider.step(sql)\n", + " error = observation[\"feedback\"][\"error\"]\n", + " if not error and reward == 0:\n", + " error = \"The SQL query returned an incorrect result\"\n", + " if error:\n", + " return {\n", + " \"error\": error,\n", + " \"wrong_result\": observation[\"feedback\"][\"result\"],\n", + " \"correct_result\": info[\"gold_result\"],\n", + " }\n", + " else:\n", + " return {\n", + " \"result\": observation[\"feedback\"][\"result\"],\n", + " }\n", + "\n", + "\n", + "for i in range(100):\n", + " observation, info = spider.reset()\n", + " question = observation[\"instruction\"]\n", + " schema = info[\"schema\"]\n", + " message = f\"\"\"Below is the schema for a SQL database:\n", + " {schema}\n", + " Generate a SQL query to answer the following question:\n", + " {question}\n", + " \"\"\"\n", + " user_proxy.initiate_chat(sql_writer, message=message)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/test/spider_env.py b/test/spider_env.py deleted file mode 100644 index 40400f618e76..000000000000 --- a/test/spider_env.py +++ /dev/null @@ -1,141 +0,0 @@ -def download_spider(cache_dir: str) -> None: - import os - import zipfile - - # Not guaranteed to work indefinitely. - cmd = f'mkdir -p {cache_dir}; cd {cache_dir}; curl --insecure -LOJ "https://drive.google.com/uc?export=download&id=1iRDVHLr4mX2wQKSgA9J8Pire73Jahh0m&confirm="' - os.system(cmd) - - with zipfile.ZipFile(os.path.join(cache_dir, "spider.zip")) as zf: - zf.extractall(cache_dir) - assert os.path.isfile(os.path.join(cache_dir, "spider/train_spider.json")) - - -class SpiderEnv: - def __init__(self, cache_dir: str = "~/.cache/spider", random_seed=666): - import json - import os - import numpy as np - - # TODO: logger. - self._rng = np.random.default_rng(random_seed) - - cache_dir = os.path.expanduser(cache_dir) - self._db_dir = os.path.join(cache_dir, "spider/database") - - # Download and unzip the dataset if non-existent. - data_file = os.path.join(cache_dir, "spider/train_spider.json") - if not os.path.isfile(data_file): - print(f"Downloading Spider dataset to {cache_dir}") - download_spider(cache_dir) - else: - print(f"Loading cached Spider dataset from {cache_dir}") - - # TODO: Use other train & dev files. - with open(data_file) as f: - self._dataset = json.load(f) - - # Try to load every unique schema - unique_db_ids = set(data["db_id"] for data in self._dataset) - error_db_ids = [] - for db_id in unique_db_ids: - schema = self._get_schema(db_id) - if schema is None: - error_db_ids.append(db_id) - - # Remove data with schema errors - self._dataset = [ - data for data in self._dataset if data["db_id"] not in error_db_ids - ] - - def _get_schema(self, db_id: str) -> str: - import glob - - schema_files = glob.glob(f"{self._db_dir}/{db_id}/*.sql") - if len(schema_files) == 0: - print(f"Schema file not found for {self._db_dir}/{db_id}") - return None - if len(schema_files) > 1: - print(f"Multiple schema files found for {self._db_dir}/{db_id}") - return None - - try: - with open(schema_files[0]) as f: - # Extract all the "CREATE TABLE (...);" statements - schema = "" - in_create_table = False - for line in f: - line = line.strip() - if "CREATE TABLE " in line.upper(): - in_create_table = True - if in_create_table: - schema += line + "\n" - if ");" in line: - in_create_table = False - schema = schema.replace("`", '"') - except Exception as e: - print(e) - return None - - return schema - - def _run_query(self, db_id: str, query: str): - import os - import sqlite3 - - con = sqlite3.connect(os.path.join(self._db_dir, f"{db_id}/{db_id}.sqlite")) - cur = con.cursor() - result, error = None, None - - try: - result = cur.execute(query).fetchall() - except Exception as e: - error = str(e) - - return result, error - - def reset(self, k: int = None): - if k is None: - # Get a random question. - self._k = self._rng.integers(len(self._dataset)) # TODO: Replacement? - else: - self._k = k - - data = self._dataset[self._k] - db_id = data["db_id"] - - observation = { - "observation": db_id, - "instruction": data["question"], - "feedback": None, - } - self._info = { - "schema": self._get_schema(db_id), - "gold_query": data["query"], - "gold_result": self._run_query(db_id, data["query"])[0], - } - - return observation, self._info - - def step(self, query: str): - data = self._dataset[self._k] - db_id = data["db_id"] - - result, error = self._run_query(db_id, query) - - if error is not None: - reward = 0.0 - else: - # TODO: Add another reward for query exact_set_match comparing with query_toks_no_value. - reward = 1.0 if result == self._info["gold_result"] else 0.0 - - observation = { - "observation": db_id, - "instruction": data["question"], - "feedback": {"result": result, "error": error}, - } - - # TODO: Handle these. - terminated, truncated = False, False - - return observation, reward, terminated, truncated, self._info diff --git a/test/sqlagent.py b/test/sqlagent.py deleted file mode 100644 index d06a96b26e96..000000000000 --- a/test/sqlagent.py +++ /dev/null @@ -1,52 +0,0 @@ -from autogen import AssistantAgent, UserProxyAgent, ConversableAgent, config_list_from_json -from spider_env import SpiderEnv -from typing import Annotated, Dict -import json - -# Load LLM inference endpoints from an env variable or a file -# See https://microsoft.github.io/autogen/docs/FAQ#set-your-api-endpoints -# and OAI_CONFIG_LIST_sample -config_list = config_list_from_json(env_or_file="OAI_CONFIG_LIST") - -def check_termination(msg: Dict): - if 'tool_responses' not in msg: - return False - json_str = msg['tool_responses'][0]['content'] - obj = json.loads(json_str) - return "error" not in obj or obj["error"] is None and obj["reward"] == 1 - -sql_writer = ConversableAgent("sql_writer", - llm_config={"config_list": config_list}, - system_message="You are good at writing SQL queries. Always respond with a function call to execute_sql().", - is_termination_msg=check_termination) -user_proxy = UserProxyAgent("user_proxy", human_input_mode="NEVER", max_consecutive_auto_reply=5) -spider = SpiderEnv() - -@sql_writer.register_for_llm(description="Function for executing SQL query and returning a response") -@user_proxy.register_for_execution() -def execute_sql(reflection: Annotated[str, "Think about what to do"], sql: Annotated[str, "SQL query"]) -> Annotated[Dict[str, str], "Dictionary with keys 'result' and 'error'"]: - observation, reward, terminated, truncated, info = spider.step(sql) - error = observation["feedback"]["error"] - if not error and reward == 0: - error = "The SQL query returned an incorrect result" - if error: - return { - "error": error, - "wrong_result": observation["feedback"]["result"], - "correct_result": info["gold_result"], - } - else: - return { - "result": observation["feedback"]["result"], - } - -for i in range(100): - observation, info = spider.reset() - question = observation["instruction"] - schema = info["schema"] - message = f"""Below is the schema for a SQL database: - {schema} - Generate a SQL query to answer the following question: - {question} - """ - user_proxy.initiate_chat(sql_writer, message=message) From 251d7f321ead554d8120243e9ed681037718ec16 Mon Sep 17 00:00:00 2001 From: Byron Xu Date: Fri, 12 Jan 2024 07:59:03 -0800 Subject: [PATCH 3/7] add SQL agent and Spider environment --- test/spider_env.py | 141 +++++++++++++++++++++++++++++++++++++++++++++ test/sqlagent.py | 52 +++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 test/spider_env.py create mode 100644 test/sqlagent.py diff --git a/test/spider_env.py b/test/spider_env.py new file mode 100644 index 000000000000..40400f618e76 --- /dev/null +++ b/test/spider_env.py @@ -0,0 +1,141 @@ +def download_spider(cache_dir: str) -> None: + import os + import zipfile + + # Not guaranteed to work indefinitely. + cmd = f'mkdir -p {cache_dir}; cd {cache_dir}; curl --insecure -LOJ "https://drive.google.com/uc?export=download&id=1iRDVHLr4mX2wQKSgA9J8Pire73Jahh0m&confirm="' + os.system(cmd) + + with zipfile.ZipFile(os.path.join(cache_dir, "spider.zip")) as zf: + zf.extractall(cache_dir) + assert os.path.isfile(os.path.join(cache_dir, "spider/train_spider.json")) + + +class SpiderEnv: + def __init__(self, cache_dir: str = "~/.cache/spider", random_seed=666): + import json + import os + import numpy as np + + # TODO: logger. + self._rng = np.random.default_rng(random_seed) + + cache_dir = os.path.expanduser(cache_dir) + self._db_dir = os.path.join(cache_dir, "spider/database") + + # Download and unzip the dataset if non-existent. + data_file = os.path.join(cache_dir, "spider/train_spider.json") + if not os.path.isfile(data_file): + print(f"Downloading Spider dataset to {cache_dir}") + download_spider(cache_dir) + else: + print(f"Loading cached Spider dataset from {cache_dir}") + + # TODO: Use other train & dev files. + with open(data_file) as f: + self._dataset = json.load(f) + + # Try to load every unique schema + unique_db_ids = set(data["db_id"] for data in self._dataset) + error_db_ids = [] + for db_id in unique_db_ids: + schema = self._get_schema(db_id) + if schema is None: + error_db_ids.append(db_id) + + # Remove data with schema errors + self._dataset = [ + data for data in self._dataset if data["db_id"] not in error_db_ids + ] + + def _get_schema(self, db_id: str) -> str: + import glob + + schema_files = glob.glob(f"{self._db_dir}/{db_id}/*.sql") + if len(schema_files) == 0: + print(f"Schema file not found for {self._db_dir}/{db_id}") + return None + if len(schema_files) > 1: + print(f"Multiple schema files found for {self._db_dir}/{db_id}") + return None + + try: + with open(schema_files[0]) as f: + # Extract all the "CREATE TABLE (...);" statements + schema = "" + in_create_table = False + for line in f: + line = line.strip() + if "CREATE TABLE " in line.upper(): + in_create_table = True + if in_create_table: + schema += line + "\n" + if ");" in line: + in_create_table = False + schema = schema.replace("`", '"') + except Exception as e: + print(e) + return None + + return schema + + def _run_query(self, db_id: str, query: str): + import os + import sqlite3 + + con = sqlite3.connect(os.path.join(self._db_dir, f"{db_id}/{db_id}.sqlite")) + cur = con.cursor() + result, error = None, None + + try: + result = cur.execute(query).fetchall() + except Exception as e: + error = str(e) + + return result, error + + def reset(self, k: int = None): + if k is None: + # Get a random question. + self._k = self._rng.integers(len(self._dataset)) # TODO: Replacement? + else: + self._k = k + + data = self._dataset[self._k] + db_id = data["db_id"] + + observation = { + "observation": db_id, + "instruction": data["question"], + "feedback": None, + } + self._info = { + "schema": self._get_schema(db_id), + "gold_query": data["query"], + "gold_result": self._run_query(db_id, data["query"])[0], + } + + return observation, self._info + + def step(self, query: str): + data = self._dataset[self._k] + db_id = data["db_id"] + + result, error = self._run_query(db_id, query) + + if error is not None: + reward = 0.0 + else: + # TODO: Add another reward for query exact_set_match comparing with query_toks_no_value. + reward = 1.0 if result == self._info["gold_result"] else 0.0 + + observation = { + "observation": db_id, + "instruction": data["question"], + "feedback": {"result": result, "error": error}, + } + + # TODO: Handle these. + terminated, truncated = False, False + + return observation, reward, terminated, truncated, self._info diff --git a/test/sqlagent.py b/test/sqlagent.py new file mode 100644 index 000000000000..d06a96b26e96 --- /dev/null +++ b/test/sqlagent.py @@ -0,0 +1,52 @@ +from autogen import AssistantAgent, UserProxyAgent, ConversableAgent, config_list_from_json +from spider_env import SpiderEnv +from typing import Annotated, Dict +import json + +# Load LLM inference endpoints from an env variable or a file +# See https://microsoft.github.io/autogen/docs/FAQ#set-your-api-endpoints +# and OAI_CONFIG_LIST_sample +config_list = config_list_from_json(env_or_file="OAI_CONFIG_LIST") + +def check_termination(msg: Dict): + if 'tool_responses' not in msg: + return False + json_str = msg['tool_responses'][0]['content'] + obj = json.loads(json_str) + return "error" not in obj or obj["error"] is None and obj["reward"] == 1 + +sql_writer = ConversableAgent("sql_writer", + llm_config={"config_list": config_list}, + system_message="You are good at writing SQL queries. Always respond with a function call to execute_sql().", + is_termination_msg=check_termination) +user_proxy = UserProxyAgent("user_proxy", human_input_mode="NEVER", max_consecutive_auto_reply=5) +spider = SpiderEnv() + +@sql_writer.register_for_llm(description="Function for executing SQL query and returning a response") +@user_proxy.register_for_execution() +def execute_sql(reflection: Annotated[str, "Think about what to do"], sql: Annotated[str, "SQL query"]) -> Annotated[Dict[str, str], "Dictionary with keys 'result' and 'error'"]: + observation, reward, terminated, truncated, info = spider.step(sql) + error = observation["feedback"]["error"] + if not error and reward == 0: + error = "The SQL query returned an incorrect result" + if error: + return { + "error": error, + "wrong_result": observation["feedback"]["result"], + "correct_result": info["gold_result"], + } + else: + return { + "result": observation["feedback"]["result"], + } + +for i in range(100): + observation, info = spider.reset() + question = observation["instruction"] + schema = info["schema"] + message = f"""Below is the schema for a SQL database: + {schema} + Generate a SQL query to answer the following question: + {question} + """ + user_proxy.initiate_chat(sql_writer, message=message) From 5666531a4ccb43fcd603e2e6628fed8cba2a7905 Mon Sep 17 00:00:00 2001 From: Wangda Zhang Date: Fri, 12 Jan 2024 09:51:20 -0800 Subject: [PATCH 4/7] Make a notebook. Clean up environment. --- notebook/agentchat_sql.ipynb | 317 +++++++++++++++++++++++++++++++++++ test/spider_env.py | 141 ---------------- test/sqlagent.py | 52 ------ 3 files changed, 317 insertions(+), 193 deletions(-) create mode 100644 notebook/agentchat_sql.ipynb delete mode 100644 test/spider_env.py delete mode 100644 test/sqlagent.py diff --git a/notebook/agentchat_sql.ipynb b/notebook/agentchat_sql.ipynb new file mode 100644 index 000000000000..58592b8ec94c --- /dev/null +++ b/notebook/agentchat_sql.ipynb @@ -0,0 +1,317 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SQL Agent" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook demonstrates a basic SQL agent that translates natural language questions into SQL queries." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Environment\n", + "\n", + "For this demo, we use a SQLite database environment based on a standard text-to-sql benchmark called [Spider](https://yale-lily.github.io/spider). The environment provides a gym-like interface and can be used as follows." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loading cached Spider dataset from /home/wangdazhang/.cache/spider\n", + "Schema file not found for /home/wangdazhang/.cache/spider/spider/database/flight_4\n", + "Schema file not found for /home/wangdazhang/.cache/spider/spider/database/small_bank_1\n", + "Schema file not found for /home/wangdazhang/.cache/spider/spider/database/icfp_1\n", + "Schema file not found for /home/wangdazhang/.cache/spider/spider/database/twitter_1\n", + "Schema file not found for /home/wangdazhang/.cache/spider/spider/database/epinions_1\n", + "Schema file not found for /home/wangdazhang/.cache/spider/spider/database/chinook_1\n", + "Schema file not found for /home/wangdazhang/.cache/spider/spider/database/company_1\n" + ] + } + ], + "source": [ + "# %pip install spider-env\n", + "\n", + "from spider_env import SpiderEnv\n", + "\n", + "gym = SpiderEnv()\n", + "\n", + "# Randomly select a question from Spider\n", + "observation, info = gym.reset()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Find the famous titles of artists that do not have any volume.\n" + ] + } + ], + "source": [ + "# The natural language question\n", + "question = observation[\"instruction\"]\n", + "print(question)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CREATE TABLE \"artist\" (\n", + "\"Artist_ID\" int,\n", + "\"Artist\" text,\n", + "\"Age\" int,\n", + "\"Famous_Title\" text,\n", + "\"Famous_Release_date\" text,\n", + "PRIMARY KEY (\"Artist_ID\")\n", + ");\n", + "CREATE TABLE \"volume\" (\n", + "\"Volume_ID\" int,\n", + "\"Volume_Issue\" text,\n", + "\"Issue_Date\" text,\n", + "\"Weeks_on_Top\" real,\n", + "\"Song\" text,\n", + "\"Artist_ID\" int,\n", + "PRIMARY KEY (\"Volume_ID\"),\n", + "FOREIGN KEY (\"Artist_ID\") REFERENCES \"artist\"(\"Artist_ID\")\n", + ");\n", + "CREATE TABLE \"music_festival\" (\n", + "\"ID\" int,\n", + "\"Music_Festival\" text,\n", + "\"Date_of_ceremony\" text,\n", + "\"Category\" text,\n", + "\"Volume\" int,\n", + "\"Result\" text,\n", + "PRIMARY KEY (\"ID\"),\n", + "FOREIGN KEY (\"Volume\") REFERENCES \"volume\"(\"Volume_ID\")\n", + ");\n", + "\n" + ] + } + ], + "source": [ + "# The schema of the corresponding database\n", + "schema = info[\"schema\"]\n", + "print(schema)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Agent Implementation\n", + "\n", + "Using AutoGen, a SQL agent can be implemented with a ConversableAgent. The gym environment executes the generated SQL query and the agent can take execution results as feedback to improve its generation in multiple rounds of conversations." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "from autogen import UserProxyAgent, ConversableAgent, config_list_from_json\n", + "from typing import Annotated, Dict\n", + "import json\n", + "import os\n", + "\n", + "os.environ[\"AUTOGEN_USE_DOCKER\"] = \"False\"\n", + "config_list = config_list_from_json(env_or_file=\"OAI_CONFIG_LIST\")\n", + "\n", + "\n", + "def check_termination(msg: Dict):\n", + " if \"tool_responses\" not in msg:\n", + " return False\n", + " json_str = msg[\"tool_responses\"][0][\"content\"]\n", + " obj = json.loads(json_str)\n", + " return \"error\" not in obj or obj[\"error\"] is None and obj[\"reward\"] == 1\n", + "\n", + "\n", + "sql_writer = ConversableAgent(\n", + " \"sql_writer\",\n", + " llm_config={\"config_list\": config_list},\n", + " system_message=\"You are good at writing SQL queries. Always respond with a function call to execute_sql().\",\n", + " is_termination_msg=check_termination,\n", + ")\n", + "user_proxy = UserProxyAgent(\"user_proxy\", human_input_mode=\"NEVER\", max_consecutive_auto_reply=5)\n", + "\n", + "\n", + "@sql_writer.register_for_llm(description=\"Function for executing SQL query and returning a response\")\n", + "@user_proxy.register_for_execution()\n", + "def execute_sql(\n", + " reflection: Annotated[str, \"Think about what to do\"], sql: Annotated[str, \"SQL query\"]\n", + ") -> Annotated[Dict[str, str], \"Dictionary with keys 'result' and 'error'\"]:\n", + " observation, reward, _, _, info = gym.step(sql)\n", + " error = observation[\"feedback\"][\"error\"]\n", + " if not error and reward == 0:\n", + " error = \"The SQL query returned an incorrect result\"\n", + " if error:\n", + " return {\n", + " \"error\": error,\n", + " \"wrong_result\": observation[\"feedback\"][\"result\"],\n", + " \"correct_result\": info[\"gold_result\"],\n", + " }\n", + " else:\n", + " return {\n", + " \"result\": observation[\"feedback\"][\"result\"],\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The agent can then take as input the schema and the text question, and generate the SQL query." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to sql_writer):\n", + "\n", + "Below is the schema for a SQL database:\n", + "CREATE TABLE \"artist\" (\n", + "\"Artist_ID\" int,\n", + "\"Artist\" text,\n", + "\"Age\" int,\n", + "\"Famous_Title\" text,\n", + "\"Famous_Release_date\" text,\n", + "PRIMARY KEY (\"Artist_ID\")\n", + ");\n", + "CREATE TABLE \"volume\" (\n", + "\"Volume_ID\" int,\n", + "\"Volume_Issue\" text,\n", + "\"Issue_Date\" text,\n", + "\"Weeks_on_Top\" real,\n", + "\"Song\" text,\n", + "\"Artist_ID\" int,\n", + "PRIMARY KEY (\"Volume_ID\"),\n", + "FOREIGN KEY (\"Artist_ID\") REFERENCES \"artist\"(\"Artist_ID\")\n", + ");\n", + "CREATE TABLE \"music_festival\" (\n", + "\"ID\" int,\n", + "\"Music_Festival\" text,\n", + "\"Date_of_ceremony\" text,\n", + "\"Category\" text,\n", + "\"Volume\" int,\n", + "\"Result\" text,\n", + "PRIMARY KEY (\"ID\"),\n", + "FOREIGN KEY (\"Volume\") REFERENCES \"volume\"(\"Volume_ID\")\n", + ");\n", + "\n", + "Generate a SQL query to answer the following question:\n", + "Find the famous titles of artists that do not have any volume.\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33msql_writer\u001b[0m (to user_proxy):\n", + "\n", + "\u001b[32m***** Suggested tool Call (call_eAu0OEzS8l3QvN3jQSn4w0hJ): execute_sql *****\u001b[0m\n", + "Arguments: \n", + "{\"reflection\":\"Generating SQL to find famous titles of artists without any volume\",\"sql\":\"SELECT a.Artist, a.Famous_Title FROM artist a WHERE NOT EXISTS (SELECT 1 FROM volume v WHERE v.Artist_ID = a.Artist_ID)\"}\n", + "\u001b[32m****************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION execute_sql...\u001b[0m\n", + "\u001b[33muser_proxy\u001b[0m (to sql_writer):\n", + "\n", + "\u001b[33muser_proxy\u001b[0m (to sql_writer):\n", + "\n", + "\u001b[32m***** Response from calling tool \"call_eAu0OEzS8l3QvN3jQSn4w0hJ\" *****\u001b[0m\n", + "{\"error\": \"The SQL query returned an incorrect result\", \"wrong_result\": [[\"Ophiolatry\", \"Antievangelistical Process (re-release)\"], [\"Triumfall\", \"Antithesis of All Flesh\"]], \"correct_result\": [[\"Antievangelistical Process (re-release)\"], [\"Antithesis of All Flesh\"]]}\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33msql_writer\u001b[0m (to user_proxy):\n", + "\n", + "\u001b[32m***** Suggested tool Call (call_5LXoKqdZ17kPCOHJbbpSz2yk): execute_sql *****\u001b[0m\n", + "Arguments: \n", + "{\"reflection\":\"Adjusting SQL to only select famous titles and exclude artist names for artists without any volume.\",\"sql\":\"SELECT a.Famous_Title FROM artist a WHERE NOT EXISTS (SELECT 1 FROM volume v WHERE v.Artist_ID = a.Artist_ID)\"}\n", + "\u001b[32m****************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION execute_sql...\u001b[0m\n", + "\u001b[33muser_proxy\u001b[0m (to sql_writer):\n", + "\n", + "\u001b[33muser_proxy\u001b[0m (to sql_writer):\n", + "\n", + "\u001b[32m***** Response from calling tool \"call_5LXoKqdZ17kPCOHJbbpSz2yk\" *****\u001b[0m\n", + "{\"result\": [[\"Antievangelistical Process (re-release)\"], [\"Antithesis of All Flesh\"]]}\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> NO HUMAN INPUT RECEIVED.\u001b[0m\n" + ] + } + ], + "source": [ + "message = f\"\"\"Below is the schema for a SQL database:\n", + "{schema}\n", + "Generate a SQL query to answer the following question:\n", + "{question}\n", + "\"\"\"\n", + "\n", + "user_proxy.initiate_chat(sql_writer, message=message)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/test/spider_env.py b/test/spider_env.py deleted file mode 100644 index 40400f618e76..000000000000 --- a/test/spider_env.py +++ /dev/null @@ -1,141 +0,0 @@ -def download_spider(cache_dir: str) -> None: - import os - import zipfile - - # Not guaranteed to work indefinitely. - cmd = f'mkdir -p {cache_dir}; cd {cache_dir}; curl --insecure -LOJ "https://drive.google.com/uc?export=download&id=1iRDVHLr4mX2wQKSgA9J8Pire73Jahh0m&confirm="' - os.system(cmd) - - with zipfile.ZipFile(os.path.join(cache_dir, "spider.zip")) as zf: - zf.extractall(cache_dir) - assert os.path.isfile(os.path.join(cache_dir, "spider/train_spider.json")) - - -class SpiderEnv: - def __init__(self, cache_dir: str = "~/.cache/spider", random_seed=666): - import json - import os - import numpy as np - - # TODO: logger. - self._rng = np.random.default_rng(random_seed) - - cache_dir = os.path.expanduser(cache_dir) - self._db_dir = os.path.join(cache_dir, "spider/database") - - # Download and unzip the dataset if non-existent. - data_file = os.path.join(cache_dir, "spider/train_spider.json") - if not os.path.isfile(data_file): - print(f"Downloading Spider dataset to {cache_dir}") - download_spider(cache_dir) - else: - print(f"Loading cached Spider dataset from {cache_dir}") - - # TODO: Use other train & dev files. - with open(data_file) as f: - self._dataset = json.load(f) - - # Try to load every unique schema - unique_db_ids = set(data["db_id"] for data in self._dataset) - error_db_ids = [] - for db_id in unique_db_ids: - schema = self._get_schema(db_id) - if schema is None: - error_db_ids.append(db_id) - - # Remove data with schema errors - self._dataset = [ - data for data in self._dataset if data["db_id"] not in error_db_ids - ] - - def _get_schema(self, db_id: str) -> str: - import glob - - schema_files = glob.glob(f"{self._db_dir}/{db_id}/*.sql") - if len(schema_files) == 0: - print(f"Schema file not found for {self._db_dir}/{db_id}") - return None - if len(schema_files) > 1: - print(f"Multiple schema files found for {self._db_dir}/{db_id}") - return None - - try: - with open(schema_files[0]) as f: - # Extract all the "CREATE TABLE (...);" statements - schema = "" - in_create_table = False - for line in f: - line = line.strip() - if "CREATE TABLE " in line.upper(): - in_create_table = True - if in_create_table: - schema += line + "\n" - if ");" in line: - in_create_table = False - schema = schema.replace("`", '"') - except Exception as e: - print(e) - return None - - return schema - - def _run_query(self, db_id: str, query: str): - import os - import sqlite3 - - con = sqlite3.connect(os.path.join(self._db_dir, f"{db_id}/{db_id}.sqlite")) - cur = con.cursor() - result, error = None, None - - try: - result = cur.execute(query).fetchall() - except Exception as e: - error = str(e) - - return result, error - - def reset(self, k: int = None): - if k is None: - # Get a random question. - self._k = self._rng.integers(len(self._dataset)) # TODO: Replacement? - else: - self._k = k - - data = self._dataset[self._k] - db_id = data["db_id"] - - observation = { - "observation": db_id, - "instruction": data["question"], - "feedback": None, - } - self._info = { - "schema": self._get_schema(db_id), - "gold_query": data["query"], - "gold_result": self._run_query(db_id, data["query"])[0], - } - - return observation, self._info - - def step(self, query: str): - data = self._dataset[self._k] - db_id = data["db_id"] - - result, error = self._run_query(db_id, query) - - if error is not None: - reward = 0.0 - else: - # TODO: Add another reward for query exact_set_match comparing with query_toks_no_value. - reward = 1.0 if result == self._info["gold_result"] else 0.0 - - observation = { - "observation": db_id, - "instruction": data["question"], - "feedback": {"result": result, "error": error}, - } - - # TODO: Handle these. - terminated, truncated = False, False - - return observation, reward, terminated, truncated, self._info diff --git a/test/sqlagent.py b/test/sqlagent.py deleted file mode 100644 index d06a96b26e96..000000000000 --- a/test/sqlagent.py +++ /dev/null @@ -1,52 +0,0 @@ -from autogen import AssistantAgent, UserProxyAgent, ConversableAgent, config_list_from_json -from spider_env import SpiderEnv -from typing import Annotated, Dict -import json - -# Load LLM inference endpoints from an env variable or a file -# See https://microsoft.github.io/autogen/docs/FAQ#set-your-api-endpoints -# and OAI_CONFIG_LIST_sample -config_list = config_list_from_json(env_or_file="OAI_CONFIG_LIST") - -def check_termination(msg: Dict): - if 'tool_responses' not in msg: - return False - json_str = msg['tool_responses'][0]['content'] - obj = json.loads(json_str) - return "error" not in obj or obj["error"] is None and obj["reward"] == 1 - -sql_writer = ConversableAgent("sql_writer", - llm_config={"config_list": config_list}, - system_message="You are good at writing SQL queries. Always respond with a function call to execute_sql().", - is_termination_msg=check_termination) -user_proxy = UserProxyAgent("user_proxy", human_input_mode="NEVER", max_consecutive_auto_reply=5) -spider = SpiderEnv() - -@sql_writer.register_for_llm(description="Function for executing SQL query and returning a response") -@user_proxy.register_for_execution() -def execute_sql(reflection: Annotated[str, "Think about what to do"], sql: Annotated[str, "SQL query"]) -> Annotated[Dict[str, str], "Dictionary with keys 'result' and 'error'"]: - observation, reward, terminated, truncated, info = spider.step(sql) - error = observation["feedback"]["error"] - if not error and reward == 0: - error = "The SQL query returned an incorrect result" - if error: - return { - "error": error, - "wrong_result": observation["feedback"]["result"], - "correct_result": info["gold_result"], - } - else: - return { - "result": observation["feedback"]["result"], - } - -for i in range(100): - observation, info = spider.reset() - question = observation["instruction"] - schema = info["schema"] - message = f"""Below is the schema for a SQL database: - {schema} - Generate a SQL query to answer the following question: - {question} - """ - user_proxy.initiate_chat(sql_writer, message=message) From 9a5a9f30d873a3064c8c0b89297faa87f60c00c5 Mon Sep 17 00:00:00 2001 From: Byron Xu Date: Fri, 26 Jan 2024 16:35:19 -0500 Subject: [PATCH 5/7] clarify that sql agent is for spider environment --- notebook/{agentchat_sql.ipynb => agentchat_sql_spider.ipynb} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename notebook/{agentchat_sql.ipynb => agentchat_sql_spider.ipynb} (99%) diff --git a/notebook/agentchat_sql.ipynb b/notebook/agentchat_sql_spider.ipynb similarity index 99% rename from notebook/agentchat_sql.ipynb rename to notebook/agentchat_sql_spider.ipynb index b056873516e1..0c850b7e0823 100644 --- a/notebook/agentchat_sql.ipynb +++ b/notebook/agentchat_sql_spider.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# SQL Agent" + "# SQL Agent for Spider text-to-SQL benchmark" ] }, { From 9a8bd052493db328021fed2dc67ae8443de7eac7 Mon Sep 17 00:00:00 2001 From: Byron Xu Date: Fri, 26 Jan 2024 16:42:01 -0500 Subject: [PATCH 6/7] add link to notebook in docs --- website/docs/Examples.md | 1 + 1 file changed, 1 insertion(+) diff --git a/website/docs/Examples.md b/website/docs/Examples.md index 11e5a4edcf44..6aacff17f02d 100644 --- a/website/docs/Examples.md +++ b/website/docs/Examples.md @@ -13,6 +13,7 @@ Links to notebook examples: - Automated Task Solving with Code Generation, Execution & Debugging - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb) - Automated Code Generation and Question Answering with Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat.ipynb) - Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_qdrant_RetrieveChat.ipynb) + - Natural Language Text to SQL Query using the [Spider](https://yale-lily.github.io/spider) Text-to-SQL Benchmark - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_sql_spider.ipynb) 1. **Multi-Agent Collaboration (>3 Agents)** From b886a59723ce0c136676a08da33a2e8da21631d3 Mon Sep 17 00:00:00 2001 From: Wangda Zhang Date: Sat, 27 Jan 2024 12:42:56 -0500 Subject: [PATCH 7/7] Update doc. --- website/docs/Examples.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/website/docs/Examples.md b/website/docs/Examples.md index 6aacff17f02d..7593363d96e7 100644 --- a/website/docs/Examples.md +++ b/website/docs/Examples.md @@ -13,7 +13,6 @@ Links to notebook examples: - Automated Task Solving with Code Generation, Execution & Debugging - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb) - Automated Code Generation and Question Answering with Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat.ipynb) - Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_qdrant_RetrieveChat.ipynb) - - Natural Language Text to SQL Query using the [Spider](https://yale-lily.github.io/spider) Text-to-SQL Benchmark - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_sql_spider.ipynb) 1. **Multi-Agent Collaboration (>3 Agents)** @@ -41,18 +40,24 @@ Links to notebook examples: - Agent Chat with Whisper - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_video_transcript_translate_with_whisper.ipynb) - Constrained Responses via Guidance - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_guidance.ipynb) - Browse the Web with Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_surfer.ipynb) + - **SQL**: Natural Language Text to SQL Query using the [Spider](https://yale-lily.github.io/spider) Text-to-SQL Benchmark - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_sql_spider.ipynb) + 1. **Human Involvement** + - Simple example in ChatGPT style [View example](https://github.com/microsoft/autogen/blob/main/samples/simple_chat.py) - Auto Code Generation, Execution, Debugging and **Human Feedback** - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_human_feedback.ipynb) - Automated Task Solving with GPT-4 + **Multiple Human Users** - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_two_users.ipynb) - Agent Chat with **Async Human Inputs** - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/Async_human_input.ipynb) + 1. **Agent Teaching and Learning** + - Teach Agents New Skills & Reuse via Automated Chat - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teaching.ipynb) - Teach Agents New Facts, User Preferences and Skills Beyond Coding - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teachability.ipynb) - Teach OpenAI Assistants Through GPTAssistantAgent - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teachable_oai_assistants.ipynb) - Agent Optimizer: Train Agents in an Agentic Way - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_agentoptimizer.ipynb) 1. **Multi-Agent Chat with OpenAI Assistants in the loop** + - Hello-World Chat with OpenAi Assistant in AutoGen - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_twoagents_basic.ipynb) - Chat with OpenAI Assistant using Function Call - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_function_call.ipynb) - Chat with OpenAI Assistant with Code Interpreter - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_code_interpreter.ipynb) @@ -60,14 +65,21 @@ Links to notebook examples: - OpenAI Assistant in a Group Chat - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_groupchat.ipynb) 1. **Multimodal Agent** + - Multimodal Agent Chat with DALLE and GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_dalle_and_gpt4v.ipynb) - Multimodal Agent Chat with Llava - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb) - Multimodal Agent Chat with GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_gpt-4v.ipynb) + 1. **Long Context Handling** + - Conversations with Chat History Compression Enabled - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_compression.ipynb) + 1. **Evaluation and Assessment** + - AgentEval: A Multi-Agent System for Assess Utility of LLM-powered Applications - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agenteval_cq_math.ipynb) + 1. **Automatic Agent Building** + - Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_basic.ipynb) - Automatically Build Multi-agent System from Agent Library - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_agent_library.ipynb)