test(pytest): refactor autotests (#1458)

* use shared pytest fixtures
* update developer docs with pytest info
* remove unneeded files (readme.txt, old nosetest script)
* rename t-numbered test scripts according to scope & purpose
* update pytest.ini to match both test_[something].py and [something]_tests.py
* add --keep <dir> pytest CLI argument (saves test outputs from automatic cleanup)
* use pytest markers to condition tests on executable availability & optional deps
* use temporary directories in example scripts, tutorials & notebooks (with optional --keep <dir> arg for scripts)
* generate coupled model tests dynamically from example models with pytest_generate_tests
* use file lock to coordinate mf6 example model sharing (https://pypi.org/project/pytest-xdist/#making-session-scoped-fixtures-execute-only-once)
* fold example scripts, tutorials & notebooks into CI test step (and include them on Windows CI)
* add jupyter and jupytext as test dependencies (since example scripts, tutorials and notebooks included in test suite)
* don't fail network-bound tests if resource is unavailable
* sequester pymake comparison usage in regression test module
* use pytest-benchmark for performance tests

.github/workflows/ci-mf6.yml

@@ -94,7 +94,7 @@ jobs:
       run: |
         pytest -v --durations=0 get_exes.py
 
-    - name: Run pytest
+    - name: Run tests
       working-directory: ./modflow6/autotest
       run: |
         pytest -v -n auto -k "test_gw" --durations=0 --cov=flopy --cov-report=xml

.github/workflows/ci-windows.yml

@@ -100,10 +100,10 @@ jobs:
         pip install xmipy
         pip install .
 
-    - name: Prepare for the autotests
-      working-directory: ./autotest
+    - name: Install modflow executables
       run: |
-        pytest -v ci_prepare.py
+        md C:\Users\runneradmin\.local\bin
+        get-modflow "C:\Users\runneradmin\.local\bin"
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -115,10 +115,12 @@ jobs:
       run: |
         echo "C:\Users\runneradmin\.local\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
 
-    - name: Run pytest on autotest scripts
+    - name: Run tests
       working-directory: ./autotest
       run: |
-        pytest -v -n auto --durations=0 --cov=flopy --cov-report=xml
+        pytest -v -n auto --benchmark-disable --cov=flopy --cov-report=xml --durations=0
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Print coverage report before upload
       working-directory: ./autotest

.github/workflows/ci.yml

@@ -116,15 +116,6 @@ jobs:
         python-version: [3.9, 3.8, 3.7]
         run-type: [std]
         test-pth: [""]
-        include:
-          - os: ubuntu-latest
-            python-version: 3.9
-            run-type: nb
-            test-path: run_notebooks.py
-          - os: ubuntu-latest
-            python-version: 3.9
-            run-type: script
-            test-path: run_scripts.py
     defaults:
       run:
         shell: bash
@@ -159,10 +150,12 @@ jobs:
       run: |
         pip install .[test,optional]
 
-    - name: Prepare for the autotests
+    - name: Install modflow executables
       working-directory: ./autotest
       run: |
-        pytest -v ci_prepare.py
+        mkdir -p $HOME/.local/bin
+        get-modflow $HOME/.local/bin
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -176,17 +169,13 @@ jobs:
         [ ! -d ./failedTests ] && mkdir ./failedTests
         echo "artifact folder for diagnosing failures." > ./failedTests/readme.txt
 
-    - name: Run pytest on autotest scripts
+    - name: Run tests
       if: matrix.run-type == 'std'
       working-directory: ./autotest
       run: |
-        pytest -v -n auto --durations=0 --cov=flopy --cov-report=xml
-
-    - name: Run pytest on scripts and notebooks
-      if: matrix.run-type != 'std'
-      working-directory: ./autotest
-      run: |
-        pytest -v -n auto --durations=0 --cov=flopy --cov-report=xml ${{ matrix.test-path }}
+        pytest -v -n auto --benchmark-disable --cov=flopy --cov-report=xml --durations=0
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Upload failedTests folder as an artifact
       uses: actions/upload-artifact@v2

.gitignore

@@ -33,9 +33,10 @@ MANIFEST
 pip-log.txt
 pip-delete-this-directory.txt
 
-# Unit test / coverage reports
+# test coverage / benchmarking reports
 htmlcov/
 .tox/
+**/.benchmarks
 .coverage
 .cache
 nosetests.xml