Sets the umask before executing the task in Docker. (#593)

* Sets the umask before executing the task in Docker.

* Added version backward compatibility for disabling cache and setting timeouts.

Author: afourney

samples/tools/testbed/README.md

@@ -2,7 +2,7 @@
 
 The Autogen Testbed environment is a tool for repeatedly running a set of pre-defined Autogen scenarios in a setting with tightly-controlled initial conditions. With each run, Autogen will start from a blank slate, working out what code needs to be written, and what libraries or dependencies to install. The results of each run are logged, and can be ingested by analysis or metrics scripts (see the HumanEval example later in this README). By default, all runs are conducted in freshly-initialized docker containers, providing the recommended level of consistency and safety.
 
-This Testbed sample has been tested in, and is known to work with, Autogen versions 0.1.14 and 0.2.0b1
+This Testbed sample has been tested in, and is known to work with, Autogen versions 0.1.14 and 0.2.0b5
 
 ## Setup
 

samples/tools/testbed/includes/testbed_utils.py

@@ -1,10 +1,40 @@
-from importlib.metadata import version as lib_version
+from pkg_resources import packaging
 from datetime import datetime
 import os
 import autogen
 import json
 
 
+def default_llm_config(config_list, timeout=180):
+    """Return a default config list with a given timeout, and with caching disabled.
+    The formatting depends on the version of Autogen installed.
+
+    Args:
+        config_list (list): the OAI config list to include in the final llm_config
+        timeout (int): the timeout for calls to the LLM
+
+    Returns:
+        None
+    """
+    llm_config = {
+        "config_list": config_list,
+    }
+
+    # Add options depending on the version
+    version = packaging.version.parse(autogen.__version__)
+    if version < packaging.version.parse("0.2.0b1"):
+        llm_config["request_timeout"] = timeout
+        llm_config["use_cache"] = False
+    elif version < packaging.version.parse("0.2.0b4"):
+        llm_config["timeout"] = timeout
+        llm_config["cache"] = None
+    else:
+        llm_config["timeout"] = timeout
+        llm_config["cache_seed"] = None
+
+    return llm_config
+
+
 def init():
     """Helper function to initialize logging in a testbed scenario.
     Specifically, write timestamp and version information, then
@@ -20,7 +50,7 @@ def init():
     # Print some information about the run
     with open("timestamp.txt", "wt") as f:
         f.write("Timestamp: " + datetime.now().isoformat() + "\n")
-        f.write("pyautogen version: " + lib_version("pyautogen") + "\n")
+        f.write("pyautogen version: " + str(autogen.__version__) + "\n")
 
 
 def finalize(agents):

samples/tools/testbed/run_scenarios.py

@@ -189,6 +189,7 @@ def run_scenario_in_docker(work_dir, timeout=600):
     with open(os.path.join(work_dir, "run.sh"), "wt") as f:
         f.write(
             """#
+umask 000
 . ./ENV
 pip install pyautogen
 python scenario.py

samples/tools/testbed/scenarios/default_two_agents.py

@@ -1,25 +1,22 @@
-from autogen import AssistantAgent, UserProxyAgent, config_list_from_json
 import os
 import json
+import autogen
 import testbed_utils
 
 testbed_utils.init()
 ##############################
 
-config_list = config_list_from_json(
+config_list = autogen.config_list_from_json(
     "OAI_CONFIG_LIST",
     filter_dict={"model": ["__MODEL__"]},
 )
 
-assistant = AssistantAgent(
+assistant = autogen.AssistantAgent(
     "assistant",
     is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
-    llm_config={
-        # "request_timeout": 180,  # Remove for autogen version >= 0.2, and OpenAI version >= 1.0
-        "config_list": config_list,
-    },
+    llm_config=testbed_utils.default_llm_config(config_list, timeout=180),
 )
-user_proxy = UserProxyAgent(
+user_proxy = autogen.UserProxyAgent(
     "user_proxy",
     human_input_mode="NEVER",
     is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,

samples/tools/testbed/scenarios/human_eval_two_agents.py

@@ -1,8 +1,8 @@
-from autogen import AssistantAgent, UserProxyAgent, config_list_from_json
 import os
 import json
 import base64
 import testbed_utils
+import autogen
 
 # NOTE:
 # This scenario runs Human Eval in a slightly unconventional way:
@@ -36,20 +36,17 @@ def run_tests(candidate):
 
 
 # Ok, now get autogen to solve it.
-config_list = config_list_from_json(
+config_list = autogen.config_list_from_json(
     "OAI_CONFIG_LIST",
     filter_dict={"model": ["__MODEL__"]},
 )
 
-assistant = AssistantAgent(
+assistant = autogen.AssistantAgent(
     "assistant",
     is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,
-    llm_config={
-        # "request_timeout": 180,  # Remove for autogen version >= 0.2, and OpenAI version >= 1.0
-        "config_list": config_list,
-    },
+    llm_config=testbed_utils.default_llm_config(config_list, timeout=180),
 )
-user_proxy = UserProxyAgent(
+user_proxy = autogen.UserProxyAgent(
     "user_proxy",
     human_input_mode="NEVER",
     is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,