Skip tests that depend on OpenAI via --skip-openai

.github/workflows/build.yml

@@ -41,17 +41,15 @@ jobs:
           pip install -e .
           python -c "import autogen"
           pip install -e. pytest mock
-          pip uninstall -y openai
       - name: Test with pytest
         if: matrix.python-version != '3.10'
         run: |
-          pytest test
+          pytest test --skip-openai
       - name: Coverage
         if: matrix.python-version == '3.10'
         run: |
           pip install -e .[test]
-          pip uninstall -y openai
-          coverage run -a -m pytest test --ignore=test/agentchat/contrib
+          coverage run -a -m pytest test --ignore=test/agentchat/contrib --skip-openai
           coverage xml
       - name: Upload coverage to Codecov
         if: matrix.python-version == '3.10'

.gitignore

@@ -171,3 +171,7 @@ test/my_tmp/*
 
 # Storage for the AgentEval output
 test/test_files/agenteval-in-out/out/
+
+# Files created by tests
+*tmp_code_*
+test/agentchat/test_agent_scripts/*

test/agentchat/contrib/test_agent_builder.py

@@ -5,6 +5,7 @@
 from packaging.requirements import Requirement
 from autogen.agentchat.contrib.agent_builder import AgentBuilder
 from autogen import UserProxyAgent
+from conftest import skip_openai
 
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST  # noqa: E402
@@ -20,15 +21,15 @@
     from openai.types.completion import Completion
     from openai.types.completion_usage import CompletionUsage
     import diskcache
-
-    OPENAI_INSTALLED = True
 except ImportError:
-    OPENAI_INSTALLED = False
+    skip = True
+else:
+    skip = False or skip_openai
 
 
 @pytest.mark.skipif(
-    not OPENAI_INSTALLED,
-    reason="do not run when dependency is not installed",
+    skip,
+    reason="openai not installed OR requested to skip",
 )
 def test_build():
     builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4")
@@ -57,8 +58,8 @@ def test_build():
 
 
 @pytest.mark.skipif(
-    not OPENAI_INSTALLED,
-    reason="do not run when dependency is not installed",
+    skip,
+    reason="openai not installed OR requested to skip",
 )
 def test_save():
     builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4")
@@ -93,8 +94,8 @@ def test_save():
 
 
 @pytest.mark.skipif(
-    not OPENAI_INSTALLED,
-    reason="do not run when dependency is not installed",
+    skip,
+    reason="openai not installed OR requested to skip",
 )
 def test_load():
     builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4")
@@ -128,8 +129,8 @@ def test_load():
 
 
 @pytest.mark.skipif(
-    not OPENAI_INSTALLED,
-    reason="do not run when dependency is not installed",
+    skip,
+    reason="openai not installed OR requested to skip",
 )
 def test_clear_agent():
     builder = AgentBuilder(config_path=oai_config_path, builder_model="gpt-4", agent_model="gpt-4")

test/agentchat/contrib/test_compressible_agent.py

@@ -2,6 +2,7 @@
 import sys
 import autogen
 import os
+from conftest import skip_openai
 from autogen.agentchat.contrib.compressible_agent import CompressibleAgent
 
 here = os.path.abspath(os.path.dirname(__file__))
@@ -19,15 +20,15 @@
 
 try:
     import openai
-
-    OPENAI_INSTALLED = True
 except ImportError:
-    OPENAI_INSTALLED = False
+    skip = True
+else:
+    skip = False or skip_openai
 
 
 @pytest.mark.skipif(
-    sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED,
-    reason="do not run on MacOS or windows or dependency is not installed",
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
 )
 def test_mode_compress():
     conversations = {}
@@ -65,8 +66,8 @@ def test_mode_compress():
 
 
 @pytest.mark.skipif(
-    sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED,
-    reason="do not run on MacOS or windows or dependency is not installed",
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
 )
 def test_mode_customized():
     try:
@@ -135,8 +136,8 @@ def constrain_num_messages(messages):
 
 
 @pytest.mark.skipif(
-    sys.platform in ["darwin", "win32"] or not OPENAI_INSTALLED,
-    reason="do not run on MacOS or windows or dependency is not installed",
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
 )
 def test_compress_message():
     assistant = CompressibleAgent(
@@ -169,6 +170,10 @@ def test_compress_message():
     assert is_success, "Compression failed."
 
 
+@pytest.mark.skipif(
+    skip,
+    reason="do not run if dependency is not installed OR requested to skip",
+)
 def test_mode_terminate():
     assistant = CompressibleAgent(
         name="assistant",

test/agentchat/contrib/test_gpt_assistant.py

@@ -3,6 +3,7 @@
 import sys
 import autogen
 from autogen import OpenAIWrapper
+from conftest import skip_openai
 
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST  # noqa: E402
@@ -11,10 +12,10 @@
     import openai
     from autogen.agentchat.contrib.gpt_assistant_agent import GPTAssistantAgent
     from autogen.oai.openai_utils import retrieve_assistants_by_name
-
-    skip_test = False
 except ImportError:
-    skip_test = True
+    skip = True
+else:
+    skip = False or skip_openai
 
 config_list = autogen.config_list_from_json(
     OAI_CONFIG_LIST, file_location=KEY_LOC, filter_dict={"api_type": ["openai"]}
@@ -26,8 +27,8 @@ def ask_ossinsight(question):
 
 
 @pytest.mark.skipif(
-    sys.platform in ["darwin", "win32"] or skip_test,
-    reason="do not run on MacOS or windows or dependency is not installed",
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
 )
 def test_gpt_assistant_chat():
     ossinsight_api_schema = {
@@ -73,8 +74,8 @@ def test_gpt_assistant_chat():
 
 
 @pytest.mark.skipif(
-    sys.platform in ["darwin", "win32"] or skip_test,
-    reason="do not run on MacOS or windows or dependency is not installed",
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
 )
 def test_get_assistant_instructions():
     """
@@ -97,8 +98,8 @@ def test_get_assistant_instructions():
 
 
 @pytest.mark.skipif(
-    sys.platform in ["darwin", "win32"] or skip_test,
-    reason="do not run on MacOS or windows or dependency is not installed",
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
 )
 def test_gpt_assistant_instructions_overwrite():
     """
@@ -142,8 +143,8 @@ def test_gpt_assistant_instructions_overwrite():
 
 
 @pytest.mark.skipif(
-    sys.platform in ["darwin", "win32"] or skip_test,
-    reason="do not run on MacOS or windows or dependency is not installed",
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
 )
 def test_gpt_assistant_existing_no_instructions():
     """
@@ -178,8 +179,8 @@ def test_gpt_assistant_existing_no_instructions():
 
 
 @pytest.mark.skipif(
-    sys.platform in ["darwin", "win32"] or skip_test,
-    reason="do not run on MacOS or windows or dependency is not installed",
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
 )
 def test_get_assistant_files():
     """
@@ -212,8 +213,8 @@ def test_get_assistant_files():
 
 
 @pytest.mark.skipif(
-    sys.platform in ["darwin", "win32"] or skip_test,
-    reason="do not run on MacOS or windows or dependency is not installed",
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
 )
 def test_assistant_retrieval():
     """
@@ -283,8 +284,8 @@ def test_assistant_retrieval():
 
 
 @pytest.mark.skipif(
-    sys.platform in ["darwin", "win32"] or skip_test,
-    reason="do not run on MacOS or windows or dependency is not installed",
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
 )
 def test_assistant_mismatch_retrieval():
     """Test function to check if the GPTAssistantAgent can filter out the mismatch assistant"""

test/agentchat/contrib/test_teachable_agent.py

@@ -2,6 +2,7 @@
 import os
 import sys
 from autogen import ConversableAgent, config_list_from_json
+from conftest import skip_openai
 
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 from test_assistant_agent import OAI_CONFIG_LIST, KEY_LOC  # noqa: E402
@@ -12,7 +13,7 @@
 except ImportError:
     skip = True
 else:
-    skip = False
+    skip = False or skip_openai
 
 try:
     from termcolor import colored

test/agentchat/test_assistant_agent.py

@@ -2,23 +2,26 @@
 import sys
 import pytest
 import autogen
+from conftest import skip_openai
 from autogen.agentchat import AssistantAgent, UserProxyAgent
 
+try:
+    from openai import OpenAI
+except ImportError:
+    skip = True
+else:
+    skip = False or skip_openai
+
 KEY_LOC = "notebook"
 OAI_CONFIG_LIST = "OAI_CONFIG_LIST"
 here = os.path.abspath(os.path.dirname(__file__))
 
 
 @pytest.mark.skipif(
-    sys.platform in ["darwin", "win32"],
-    reason="do not run on MacOS or windows",
+    sys.platform in ["darwin", "win32"] or skip,
+    reason="do not run on MacOS or windows OR openai not installed OR requested to skip",
 )
 def test_ai_user_proxy_agent():
-    try:
-        import openai
-    except ImportError:
-        return
-
     conversations = {}
     # autogen.ChatCompletion.start_logging(conversations)
 
@@ -57,11 +60,8 @@ def test_ai_user_proxy_agent():
     print(conversations)
 
 
+@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")
 def test_gpt35(human_input_mode="NEVER", max_consecutive_auto_reply=5):
-    try:
-        import openai
-    except ImportError:
-        return
     config_list = autogen.config_list_from_json(
         OAI_CONFIG_LIST,
         file_location=KEY_LOC,
@@ -115,12 +115,8 @@ def test_gpt35(human_input_mode="NEVER", max_consecutive_auto_reply=5):
     assert not isinstance(user.use_docker, bool)  # None or str
 
 
+@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")
 def test_create_execute_script(human_input_mode="NEVER", max_consecutive_auto_reply=10):
-    try:
-        import openai
-    except ImportError:
-        return
-
     config_list = autogen.config_list_from_json(OAI_CONFIG_LIST, file_location=KEY_LOC)
     conversations = {}
     # autogen.ChatCompletion.start_logging(conversations)
@@ -160,12 +156,8 @@ def test_create_execute_script(human_input_mode="NEVER", max_consecutive_auto_re
     # autogen.ChatCompletion.stop_logging()
 
 
+@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")
 def test_tsp(human_input_mode="NEVER", max_consecutive_auto_reply=10):
-    try:
-        import openai
-    except ImportError:
-        return
-
     config_list = autogen.config_list_from_json(
         OAI_CONFIG_LIST,
         file_location=KEY_LOC,

test/agentchat/test_async.py

@@ -1,8 +1,16 @@
 import pytest
 import asyncio
 import autogen
+from conftest import skip_openai
 from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST
 
+try:
+    from openai import OpenAI
+except ImportError:
+    skip = True
+else:
+    skip = False or skip_openai
+
 
 def get_market_news(ind, ind_upper):
     data = {
@@ -45,13 +53,9 @@ def get_market_news(ind, ind_upper):
     return feeds_summary
 
 
+@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")
 @pytest.mark.asyncio
 async def test_async_groupchat():
-    try:
-        import openai
-    except ImportError:
-        return
-
     config_list = autogen.config_list_from_json(OAI_CONFIG_LIST, KEY_LOC)
 
     llm_config = {
@@ -91,12 +95,9 @@ async def test_async_groupchat():
     assert len(user_proxy.chat_messages) > 0
 
 
+@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")
 @pytest.mark.asyncio
 async def test_stream():
-    try:
-        import openai
-    except ImportError:
-        return
     config_list = autogen.config_list_from_json(OAI_CONFIG_LIST, KEY_LOC)
     data = asyncio.Future()