response filter (#1039)

* response filter * rewrite implement based on the filter * multi responses * abs path * code handling * option to not use docker * context * eval_only -> raise_error * notebook * utils * utils * separate tests * test * test * test * test * test * test * test * test * **config in test() * test * test * filename
microsoft · May 21, 2023 · e463146 · e463146
1 parent 7de4eb3
commit e463146
Show file tree

Hide file tree

Showing 21 changed files with 2,253 additions and 1,820 deletions.
diff --git a/.github/workflows/openai.yml b/.github/workflows/openai.yml
@@ -7,18 +7,18 @@ on:
   pull_request:
     branches: ['main']
     paths:
-      - 'flaml/integrations/oai/**'
-      - 'test/openai/**'
-      - 'notebook/integrate_openai.ipynb'
-      - 'notebook/integrate_chatgpt_math.ipynb'
+      - 'flaml/autogen/**'
+      - 'test/autogen/**'
+      - 'notebook/autogen_openai_completion.ipynb'
+      - 'notebook/autogen_chatgpt_gpt4.ipynb'
       - '.github/workflows/openai.yml'
 
 jobs:
   test:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.9]
+        python-version: ["3.9", "3.10", "3.11"]
     runs-on: ${{ matrix.os }}
     environment: openai
     steps:
@@ -33,14 +33,27 @@ jobs:
           python -m pip install --upgrade pip wheel
           pip install -e .[autogen,blendsearch]
           python -c "import flaml"
+          pip install coverage pytest datasets
       - name: Coverage
+        if: matrix.python-version == '3.9'
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
         run: |
-          pip install coverage pytest datasets nbconvert nbformat ipykernel
-          coverage run -a -m pytest test/openai
+          coverage run -a -m pytest test/autogen
           coverage xml
-          cat "$(pwd)/test/openai/executed_openai_notebook_output.txt"
+      - name: Coverage and check notebook outputs
+        if: matrix.python-version != '3.9'
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
+        run: |
+          pip install nbconvert nbformat ipykernel
+          coverage run -a -m pytest test/autogen/oai/test_notebook.py
+          coverage xml
+          cat "$(pwd)/test/autogen/oai/executed_openai_notebook_output.txt"
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

diff --git a/flaml/autogen/agent/agent.py b/flaml/autogen/agent/agent.py
@@ -37,7 +37,7 @@ def _send(self, message, recipient):
 
     def _receive(self, message, sender):
         """Receive a message from another agent."""
-        print("****", self.name, "received message from", sender.name, "****")
+        print("\n****", self.name, "received message from", sender.name, "****\n")
         print(message)
         self._conversations[sender.name].append({"content": message, "role": "user"})
 

diff --git a/flaml/autogen/agent/human_proxy_agent.py → flaml/autogen/agent/user_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py → flaml/autogen/agent/user_proxy_agent.py
@@ -3,8 +3,8 @@
 from collections import defaultdict
 
 
-class HumanProxyAgent(Agent):
-    """(Experimental) A proxy agent for human, that can execute code and provide feedback to the other agents."""
+class UserProxyAgent(Agent):
+    """(Experimental) A proxy agent for the user, that can execute code and provide feedback to the other agents."""
 
     MAX_CONSECUTIVE_AUTO_REPLY = 100  # maximum number of consecutive auto replies (subject to future change)
 
@@ -16,6 +16,7 @@ def __init__(
         human_input_mode="ALWAYS",
         max_consecutive_auto_reply=None,
         is_termination_msg=None,
+        use_docker=True,
         **config,
     ):
         """
@@ -51,22 +52,26 @@ def __init__(
             max_consecutive_auto_reply if max_consecutive_auto_reply is not None else self.MAX_CONSECUTIVE_AUTO_REPLY
         )
         self._consecutive_auto_reply_counter = defaultdict(int)
+        self._use_docker = use_docker
 
     def _execute_code(self, code, lang):
         """Execute the code and return the result."""
-        if lang == "bash":
-            assert code.startswith("python "), code
+        if lang in ["bash", "shell"]:
+            if not code.startswith("python "):
+                return 1, f"please do not suggest bash or shell commands like {code}"
             file_name = code[len("python ") :]
-            exitcode, logs = execute_code(filename=file_name, work_dir=self._work_dir)
+            exitcode, logs = execute_code(filename=file_name, work_dir=self._work_dir, use_docker=self._use_docker)
+            logs = logs.decode("utf-8")
         elif lang == "python":
             if code.startswith("# filename: "):
                 filename = code[11 : code.find("\n")].strip()
             else:
                 filename = None
-            exitcode, logs = execute_code(code, work_dir=self._work_dir, filename=filename)
+            exitcode, logs = execute_code(code, work_dir=self._work_dir, filename=filename, use_docker=self._use_docker)
+            logs = logs.decode("utf-8")
         else:
             # TODO: could this happen?
-            exitcode, logs = 1, "unknown language"
+            exitcode, logs = 1, f"unknown language {lang}"
             # raise NotImplementedError
         return exitcode, logs
 
@@ -80,7 +85,7 @@ def auto_reply(self, message, sender, default_reply=""):
             # try to execute the code
             exitcode, logs = self._execute_code(code, lang)
             exitcode2str = "execution succeeded" if exitcode == 0 else "execution failed"
-            self._send(f"exitcode: {exitcode} ({exitcode2str})\nCode output: {logs.decode('utf-8')}", sender)
+            self._send(f"exitcode: {exitcode} ({exitcode2str})\nCode output: {logs}", sender)
 
     def receive(self, message, sender):
         """Receive a message from the sender agent.

diff --git a/flaml/autogen/code_utils.py b/flaml/autogen/code_utils.py
@@ -37,8 +37,7 @@ def generate_code(pattern: str = CODE_BLOCK_PATTERN, **config) -> Tuple[str, flo
         float: The cost of the generation.
     """
     response = oai.Completion.create(**config)
-    cost = oai.Completion.cost(response)
-    return extract_code(oai.Completion.extract_text(response)[0], pattern), cost
+    return extract_code(oai.Completion.extract_text(response)[0], pattern), response["cost"]
 
 
 _IMPROVE_FUNCTION_CONFIG = {
@@ -59,8 +58,7 @@ def improve_function(file_name, func_name, objective, **config):
     response = oai.Completion.create(
         {"func_name": func_name, "objective": objective, "file_string": file_string}, **params
     )
-    cost = oai.Completion.cost(response)
-    return oai.Completion.extract_text(response)[0], cost
+    return oai.Completion.extract_text(response)[0], response["cost"]
 
 
 _IMPROVE_CODE_CONFIG = {
@@ -97,8 +95,7 @@ def improve_code(files, objective, suggest_only=True, **config):
     params = {**_IMPROVE_CODE_CONFIG, **config}
     followup = "" if suggest_only else " followed by the improved code"
     response = oai.Completion.create({"objective": objective, "code": code, "followup": followup}, **params)
-    cost = oai.Completion.cost(response)
-    return oai.Completion.extract_text(response)[0], cost
+    return oai.Completion.extract_text(response)[0], response["cost"]
 
 
 def timeout_handler(signum, frame):
@@ -281,9 +278,8 @@ def generate_assertions(definition: str, **config) -> Tuple[str, float]:
         {"definition": definition},
         **params,
     )
-    cost = oai.Completion.cost(response)
     assertions = oai.Completion.extract_text(response)[0]
-    return assertions, cost
+    return assertions, response["cost"]
 
 
 def _remove_check(response):
@@ -387,6 +383,23 @@ def eval_function_completions(
 ]
 
 
+class PassAssertionFilter:
+    def __init__(self, assertions):
+        self._assertions = assertions
+        self.cost = 0
+        self.metrics = self.responses = None
+
+    def pass_assertions(self, context, response, **_):
+        """Check if the response passes the assertions."""
+        responses = oai.Completion.extract_text(response)
+        metrics = eval_function_completions(responses, context["definition"], assertions=self._assertions)
+        self._assertions = metrics["assertions"]
+        self.cost += metrics["gen_cost"]
+        self.metrics = metrics
+        self.responses = responses
+        return metrics["succeed_assertions"]
+
+
 def implement(
     definition: str,
     configs: Optional[List[Dict]] = None,
@@ -408,12 +421,19 @@ def implement(
     configs = configs or _IMPLEMENT_CONFIGS
     if len(configs) > 1 and callable(assertions):
         assertions, cost = assertions(definition)
-    for i, config in enumerate(configs):
-        response = oai.Completion.create({"definition": definition}, **config)
-        cost += oai.Completion.cost(response)
-        responses = oai.Completion.extract_text(response)
-        metrics = eval_function_completions(responses, definition, assertions=assertions)
-        assertions = metrics["assertions"]
-        cost += metrics["gen_cost"]
-        if metrics["succeed_assertions"] or i == len(configs) - 1:
-            return responses[metrics["index_selected"]], cost, i
+    assertion_filter = PassAssertionFilter(assertions)
+    response = oai.Completion.create(
+        {"definition": definition}, config_list=configs, filter_func=assertion_filter.pass_assertions
+    )
+    cost += assertion_filter.cost + response["cost"]
+    return assertion_filter.responses[assertion_filter.metrics["index_selected"]], cost, response["config_id"]
+
+    # for i, config in enumerate(configs):
+    #     response = oai.Completion.create({"definition": definition}, **config)
+    #     cost += oai.Completion.cost(response)
+    #     responses = oai.Completion.extract_text(response)
+    #     metrics = eval_function_completions(responses, definition, assertions=assertions)
+    #     assertions = metrics["assertions"]
+    #     cost += metrics["gen_cost"]
+    #     if metrics["succeed_assertions"] or i == len(configs) - 1:
+    #         return responses[metrics["index_selected"]], cost, i
diff --git a/flaml/autogen/math_utils.py b/flaml/autogen/math_utils.py
@@ -20,9 +20,8 @@ def solve_problem(problem: str, **config) -> str:
     """
     params = {**_MATH_CONFIG, **config}
     response = oai.Completion.create({"problem": problem}, **params)
-    cost = oai.Completion.cost(response)
     results = eval_math_responses(oai.Completion.extract_text(response))
-    return results.get("voted_answer"), cost
+    return results.get("voted_answer"), response["cost"]
 
 
 def remove_boxed(string: str) -> Optional[str]:

diff --git a/flaml/autogen/oai/__init__.py b/flaml/autogen/oai/__init__.py
@@ -1,3 +1,4 @@
 from flaml.autogen.oai.completion import Completion, ChatCompletion
+from flaml.autogen.oai.openai_utils import get_config_list, config_list_gpt4_gpt35, config_list_openai_aoai
 
-__all__ = ["Completion", "ChatCompletion"]
+__all__ = ["Completion", "ChatCompletion", "get_config_list", "config_list_gpt4_gpt35", "config_list_openai_aoai"]