Streamline Testing with pytest-cov and pytest Defaults (#2490)

* done

* update docs

* try fix

* update workflows

* undo minor fix

* resolve comments

* adds back pytest-asyncio

* minor fix

* add branch coverage

* restore pip install e.

* test with coverage

* fix mypy

* fix coverage + docker + windows combo

* fix bash command

* formatter

* formatter

* one last fix

* I lied, last fix

* fix

* fix retrieve chat test

* fix windows paths

* change cache seed

* down grade openai version

* fix openai mypy

* better error type

* fix image gen cache test

* fix

* experimenting

* fix lmm

* skip cosmos test

* remove cosmos db

* unused imports

* handle more cosmosdb skips

* fix flaky test

Author: WaelKarkoub

.github/workflows/build.yml

@@ -46,7 +46,7 @@ jobs:
           python -m pip install --upgrade pip wheel
           pip install -e .[cosmosdb]
           python -c "import autogen"
-          pip install pytest mock
+          pip install pytest-cov>=5 mock
       - name: Install optional dependencies for code executors
         # code executors and udfs auto skip without deps, so only run for python 3.11
         if: matrix.python-version == '3.11'
@@ -71,12 +71,11 @@ jobs:
         if: matrix.python-version == '3.10'
         run: |
           pip install -e .[test,redis,websockets]
-          coverage run -a -m pytest test --ignore=test/agentchat/contrib --skip-openai --durations=10 --durations-min=1.0
-          coverage xml
+          pytest test --ignore=test/agentchat/contrib --skip-openai --durations=10 --durations-min=1.0
       - name: Test with Cosmos DB
         run: |
           pip install -e .[test,cosmosdb]
-          coverage run -a -m pytest test/cache/test_cosmos_db_cache.py --skip-openai --durations=10 --durations-min=1.0
+          pytest test/cache/test_cosmos_db_cache.py --skip-openai --durations=10 --durations-min=1.0
       - name: Upload coverage to Codecov
         if: matrix.python-version == '3.10'
         uses: codecov/codecov-action@v3

.github/workflows/contrib-openai.yml

@@ -5,14 +5,15 @@ name: OpenAI4ContribTests
 
 on:
   pull_request:
-    branches: ['main']
+    branches: ["main"]
     paths:
-      - 'autogen/**'
-      - 'test/agentchat/contrib/**'
-      - '.github/workflows/contrib-openai.yml'
-      - 'setup.py'
-permissions: {}
-    # actions: read
+      - "autogen/**"
+      - "test/agentchat/contrib/**"
+      - ".github/workflows/contrib-openai.yml"
+      - "setup.py"
+permissions:
+  {}
+  # actions: read
   # checks: read
   # contents: read
   # deployments: read
@@ -55,7 +56,7 @@ jobs:
           python -m pip install --upgrade pip wheel
           pip install -e .
           python -c "import autogen"
-          pip install coverage pytest-asyncio
+          pip install pytest-cov>=5 pytest-asyncio
       - name: Install packages for test when needed
         run: |
           pip install docker
@@ -67,8 +68,7 @@ jobs:
           AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
           OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
         run: |
-          coverage run -a -m pytest -k test_retrievechat test/agentchat/contrib/retrievechat
-          coverage xml
+          pytest test/agentchat/contrib/retrievechat/ test/agentchat/contrib/retrievechat
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
@@ -98,7 +98,7 @@ jobs:
           python -m pip install --upgrade pip wheel
           pip install -e .
           python -c "import autogen"
-          pip install coverage pytest-asyncio
+          pip install pytest-cov>=5 pytest-asyncio
       - name: Install packages for test when needed
         run: |
           pip install docker
@@ -109,8 +109,7 @@ jobs:
           AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
           OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
         run: |
-          coverage run -a -m pytest test/agentchat/contrib/test_compressible_agent.py
-          coverage xml
+          pytest test/agentchat/contrib/test_compressible_agent.py
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
@@ -139,7 +138,7 @@ jobs:
           python -m pip install --upgrade pip wheel
           pip install -e .
           python -c "import autogen"
-          pip install coverage pytest-asyncio
+          pip install pytest-cov>=5 pytest-asyncio
       - name: Install packages for test when needed
         run: |
           pip install docker
@@ -150,8 +149,7 @@ jobs:
           AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
           OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
         run: |
-          coverage run -a -m pytest test/agentchat/contrib/test_gpt_assistant.py
-          coverage xml
+          pytest test/agentchat/contrib/test_gpt_assistant.py
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
@@ -180,16 +178,15 @@ jobs:
           python -m pip install --upgrade pip wheel
           pip install -e .[teachable]
           python -c "import autogen"
-          pip install coverage pytest
+          pip install pytest-cov>=5
       - name: Coverage
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
           AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
           OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
         run: |
-          coverage run -a -m pytest test/agentchat/contrib/capabilities/test_teachable_agent.py
-          coverage xml
+          pytest test/agentchat/contrib/capabilities/test_teachable_agent.py
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
@@ -198,8 +195,8 @@ jobs:
   AgentBuilder:
     strategy:
       matrix:
-        os: [ ubuntu-latest ]
-        python-version: [ "3.11" ]
+        os: [ubuntu-latest]
+        python-version: ["3.11"]
     runs-on: ${{ matrix.os }}
     environment: openai1
     steps:
@@ -218,7 +215,7 @@ jobs:
           python -m pip install --upgrade pip wheel
           pip install -e .
           python -c "import autogen"
-          pip install coverage pytest-asyncio
+          pip install pytest-cov>=5 pytest-asyncio
       - name: Install packages for test when needed
         run: |
           pip install -e .[autobuild]
@@ -229,8 +226,7 @@ jobs:
           AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
           OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
         run: |
-          coverage run -a -m pytest test/agentchat/contrib/test_agent_builder.py
-          coverage xml
+          pytest test/agentchat/contrib/test_agent_builder.py
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
@@ -259,7 +255,7 @@ jobs:
           python -m pip install --upgrade pip wheel
           pip install -e .[websurfer]
           python -c "import autogen"
-          pip install coverage pytest
+          pip install pytest-cov>=5
       - name: Coverage
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -268,93 +264,90 @@ jobs:
           OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
           BING_API_KEY: ${{ secrets.BING_API_KEY }}
         run: |
-          coverage run -a -m pytest test/agentchat/contrib/test_web_surfer.py
-          coverage xml
+          pytest test/agentchat/contrib/test_web_surfer.py
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
           file: ./coverage.xml
           flags: unittests
   ContextHandling:
-      strategy:
-        matrix:
-          os: [ubuntu-latest]
-          python-version: ["3.11"]
-      runs-on: ${{ matrix.os }}
-      environment: openai1
-      steps:
-        # checkout to pr branch
-        - name: Checkout
-          uses: actions/checkout@v4
-          with:
-            ref: ${{ github.event.pull_request.head.sha }}
-        - name: Set up Python ${{ matrix.python-version }}
-          uses: actions/setup-python@v5
-          with:
-            python-version: ${{ matrix.python-version }}
-        - name: Install packages and dependencies
-          run: |
-            docker --version
-            python -m pip install --upgrade pip wheel
-            pip install -e .
-            python -c "import autogen"
-            pip install coverage pytest
-        - name: Coverage
-          env:
-            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-            AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
-            AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
-            OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
-            BING_API_KEY: ${{ secrets.BING_API_KEY }}
-          run: |
-            coverage run -a -m pytest test/agentchat/contrib/capabilities/test_context_handling.py
-            coverage xml
-        - name: Upload coverage to Codecov
-          uses: codecov/codecov-action@v3
-          with:
-            file: ./coverage.xml
-            flags: unittests
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.11"]
+    runs-on: ${{ matrix.os }}
+    environment: openai1
+    steps:
+      # checkout to pr branch
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install packages and dependencies
+        run: |
+          docker --version
+          python -m pip install --upgrade pip wheel
+          pip install -e .
+          python -c "import autogen"
+          pip install pytest-cov>=5
+      - name: Coverage
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
+          OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
+          BING_API_KEY: ${{ secrets.BING_API_KEY }}
+        run: |
+          pytest test/agentchat/contrib/capabilities/test_context_handling.py
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          flags: unittests
   ImageGen:
-      strategy:
-        matrix:
-          os: [ubuntu-latest]
-          python-version: ["3.12"]
-      runs-on: ${{ matrix.os }}
-      environment: openai1
-      steps:
-        # checkout to pr branch
-        - name: Checkout
-          uses: actions/checkout@v4
-          with:
-            ref: ${{ github.event.pull_request.head.sha }}
-        - name: Set up Python ${{ matrix.python-version }}
-          uses: actions/setup-python@v5
-          with:
-            python-version: ${{ matrix.python-version }}
-        - name: Install packages and dependencies
-          run: |
-            docker --version
-            python -m pip install --upgrade pip wheel
-            pip install -e .[lmm]
-            python -c "import autogen"
-            pip install coverage pytest
-        - name: Coverage
-          env:
-            OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          run: |
-            coverage run -a -m pytest test/agentchat/contrib/capabilities/test_image_generation_capability.py
-            coverage xml
-        - name: Upload coverage to Codecov
-          uses: codecov/codecov-action@v3
-          with:
-            file: ./coverage.xml
-            flags: unittests
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.12"]
+    runs-on: ${{ matrix.os }}
+    environment: openai1
+    steps:
+      # checkout to pr branch
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install packages and dependencies
+        run: |
+          docker --version
+          python -m pip install --upgrade pip wheel
+          pip install -e .[lmm]
+          python -c "import autogen"
+          pip install pytest-cov>=5
+      - name: Coverage
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          pytest test/agentchat/contrib/capabilities/test_image_generation_capability.py
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          flags: unittests
 
   AgentOptimizer:
     strategy:
       matrix:
-        os: [ ubuntu-latest ]
-        python-version: [ "3.11" ]
+        os: [ubuntu-latest]
+        python-version: ["3.11"]
     runs-on: ${{ matrix.os }}
     environment: openai1
     steps:
@@ -373,16 +366,15 @@ jobs:
           python -m pip install --upgrade pip wheel
           pip install -e .
           python -c "import autogen"
-          pip install coverage pytest
+          pip install pytest-cov>=5
       - name: Coverage
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
           AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
           OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
         run: |
-          coverage run -a -m pytest test/agentchat/contrib/test_agent_optimizer.py
-          coverage xml
+          pytest test/agentchat/contrib/test_agent_optimizer.py
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with: