Add a workflow and Makefile target to test old GMT versions every Tuesday

.github/workflows/ci_tests_backward.yaml

@@ -0,0 +1,105 @@
+# This workflow installs PyGMT and runs tests
+
+name: GMT Backward Tests
+
+on:
+  # push:
+  #   branches: [ main ]
+  pull_request:
+    # types: [ready_for_review]
+    paths-ignore:
+      - 'doc/**'
+      - 'examples/**'
+      - '*.md'
+      - 'README.rst'
+      - 'LICENSE.txt'
+      - '.gitignore'
+  # Schedule tests on Monday/Wednesday/Friday
+  schedule:
+    - cron: '0 0 * * 1,3,5'
+
+jobs:
+  test:
+    name: ${{ matrix.os }} - GMT ${{ matrix.gmt_version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.10']
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+        gmt_version: ['6.3']
+    timeout-minutes: 30
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      # Cancel previous runs that are not completed
+      - name: Cancel Previous Runs
+        uses: styfle/cancel-workflow-action@0.10.0
+        with:
+          access_token: ${{ github.token }}
+
+      # Checkout current git repository
+      - name: Checkout
+        uses: actions/checkout@v3.0.2
+        with:
+          # fetch all history so that setuptools-scm works
+          fetch-depth: 0
+
+      # Install Mambaforge with conda-forge dependencies
+      - name: Setup Mambaforge
+        uses: conda-incubator/setup-miniconda@v2.1.1
+        with:
+          activate-environment: pygmt
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge,nodefaults
+          channel-priority: strict
+          miniforge-version: latest
+          miniforge-variant: Mambaforge
+          mamba-version: "*"
+          use-mamba: true
+
+      # Install GMT and other required dependencies from conda-forge
+      - name: Install dependencies
+        run: |
+          mamba install gmt=${{ matrix.gmt_version }} numpy \
+                        pandas xarray netCDF4 packaging geopandas \
+                        build dvc make pytest>=6.0 \
+                        pytest-cov pytest-doctestplus pytest-mpl sphinx-gallery
+
+      # Show installed pkg information for postmortem diagnostic
+      - name: List installed packages
+        run: mamba list
+
+      # Download cached remote files (artifacts) from GitHub
+      - name: Download remote data from GitHub
+        uses: dawidd6/action-download-artifact@v2.22.0
+        with:
+          workflow: cache_data.yaml
+          workflow_conclusion: success
+          name: gmt-cache
+          path: .gmt
+
+      # Move downloaded files to ~/.gmt directory and list them
+      - name: Move and list downloaded remote files
+        run: |
+          mkdir -p ~/.gmt
+          mv .gmt/* ~/.gmt
+          # Change modification times of the two files, so GMT won't refresh it
+          touch ~/.gmt/server/gmt_data_server.txt ~/.gmt/server/gmt_hash_server.txt
+          ls -lhR ~/.gmt
+
+      # Pull baseline image data from dvc remote (DAGsHub)
+      - name: Pull baseline image data from dvc remote
+        run: |
+          dvc pull --verbose
+          ls -lhR pygmt/tests/baseline/
+
+      # Install the package that we want to test
+      - name: Install the package
+        run: make install
+
+      # Run the tests but skip images
+      - name: Run tests
+        run: make test_no_images PYTEST_EXTRA="-r P"

Makefile

@@ -14,15 +14,16 @@ LINT_FILES=$(PROJECT) doc/conf.py
 help:
 	@echo "Commands:"
 	@echo ""
-	@echo "  install   install in editable mode"
-	@echo "  package   build source and wheel distributions"
-	@echo "  test      run the test suite (including some doctests) and report coverage"
-	@echo "  fulltest  run the test suite (including all doctests) and report coverage"
-	@echo "  format    run black, blackdoc, docformatter and isort to automatically format the code"
-	@echo "  check     run code style and quality checks (black, blackdoc, docformatter, flakeheaven and isort)"
-	@echo "  lint      run pylint for a deeper (and slower) quality check"
-	@echo "  clean     clean up build and generated files"
-	@echo "  distclean clean up build and generated files, including project metadata files"
+	@echo "  install        install in editable mode"
+	@echo "  package        build source and wheel distributions"
+	@echo "  test           run the test suite (including some doctests) and report coverage"
+	@echo "  fulltest       run the test suite (including all doctests) and report coverage"
+	@echo "  test_no_images run the test suite (including all doctests) but skip image comparisons"
+	@echo "  format         run black, blackdoc, docformatter and isort to automatically format the code"
+	@echo "  check          run code style and quality checks (black, blackdoc, docformatter, flakeheaven and isort)"
+	@echo "  lint           run pylint for a deeper (and slower) quality check"
+	@echo "  clean          clean up build and generated files"
+	@echo "  distclean      clean up build and generated files, including project metadata files"
 	@echo ""
 
 install:
@@ -53,6 +54,20 @@ fulltest:
 	cp -r $(TESTDIR)/htmlcov .
 	rm -r $(TESTDIR)
 
+test_no_images:
+	# Run a tmp folder to make sure the tests are run on the installed version
+	mkdir -p $(TESTDIR)
+	@echo ""
+	@cd $(TESTDIR); python -c "import $(PROJECT); $(PROJECT).show_versions()"
+	@echo ""
+	# run pytest without the --mpl option to disable image comparisons
+	# use -o to override the addopts in pyproject.toml file
+	cd $(TESTDIR); \
+		PYGMT_USE_EXTERNAL_DISPLAY="false" \
+		pytest -o addopts="--verbose --durations=0 --durations-min=0.2 --doctest-modules" \
+		$(PYTEST_COV_ARGS) $(PROJECT)
+	rm -r $(TESTDIR)
+
 format:
 	isort .
 	docformatter --in-place $(DOCFORMATTER_OPTIONS) $(DOCFORMATTER_FILES)