Add coverage report handling (#189)

* add coverage report handling

* test the foobar file

* fix coverage run

* forgot checkout

* specify shell

* debug

* debug

* try again

* do not fail on combine

* add docs

.github/actions/report-coverage/action.yml

@@ -0,0 +1,41 @@
+name: "Report Coverage"
+description: "Combine Coverage and Report"
+inputs:
+  fail_under:
+    description: "The coverage amount to fail under"
+    default: 80
+
+runs:
+  using: "composite"
+  steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          # Use latest Python, so it understands all syntax.
+          python-version: "3.11"
+
+      - run: python -Im pip install --upgrade coverage[toml]
+        shell: bash
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: coverage-data
+
+      - name: Combine coverage & fail if it's <${{inputs.fail_under}}.
+        shell: bash
+        run: |
+          python -Im coverage combine || true
+          python -Im coverage html --skip-covered --skip-empty
+
+          # Report and write to summary.
+          python -Im coverage report | sed 's/^/    /' >> $GITHUB_STEP_SUMMARY
+
+          # Report again and fail if under threshold
+          python -Im coverage report --fail-under=${{inputs.fail_under}}
+
+      - name: Upload HTML report if check failed.
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-report
+          path: htmlcov
+        if: ${{ failure() }}

.github/actions/upload-coverage/action.yml

@@ -0,0 +1,16 @@
+name: "Upload Coverage"
+description: "Upload the coverage report(s) for this job"
+inputs:
+  file_pattern:
+    description: "The coverage file pattern"
+    default: ".coverage"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Upload coverage data
+      uses: actions/upload-artifact@v3
+      with:
+        name: coverage-data
+        path: ${{inputs.file_pattern}}
+        if-no-files-found: ignore

.github/workflows/tests.yml

@@ -278,6 +278,26 @@ jobs:
       - uses: ./.github/actions/base-setup
       - uses: ./.github/actions/test-sdist
 
+  coverage_test:
+    name: Test coverage
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/base-setup
+      - run: |
+          pip install coverage pytest
+          python -m coverage run -m pytest foobar.py
+          ls -a
+      - uses: ./.github/actions/upload-coverage
+
+  coverage_report:
+    name: Combine & check coverage
+    needs: coverage_test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/report-coverage
+
   tests_check: # This job does nothing and is only used for the branch protection
     if: always()
     needs:
@@ -292,6 +312,7 @@ jobs:
       - update_snapshots-manual-server
       - update_snapshots
       - test_sdist
+      - coverage_report
     runs-on: ubuntu-latest
     steps:
       - name: Decide whether the needed jobs succeeded or failed

README.md

@@ -358,6 +358,46 @@ jobs:
           association: ${{ github.event.comment.author_association }}
 ```
 
+## Upload Coverage and Report Coverage
+
+These actions are meant to be used together, to combine and enforce coverage.
+A coverage snapshot will be included in the workflow summary.  If coverage
+is below threshold, the `report-coverage` action will fail and upload the
+html report.
+
+```yaml
+name: Tests
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+      - run: |
+          pip install -e ".[test]"
+          python -m coverage run -m pytest
+      - uses: jupyterlab/maintainer-tools/.github/actions/upload-coverage@v1
+  coverage_report:
+    name: Combine & check coverage
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: jupyterlab/maintainer-tools/.github/actions/report-coverage@v1
+        with:
+          fail_under: 90
+```
+
 ## Update snapshots
 
 You can use _update snapshots_ action to commit on a branch