Merge pull request #2096 from devitocodes/refactor_reqs

reqs: Move pyrevolve to optionals and introduce testing-only reqs

.github/workflows/examples-mpi.yml

@@ -43,7 +43,7 @@ jobs:
     - name: Install dependencies
       run: |
         pip install --upgrade pip
-        pip install -e .[extras,mpi]
+        pip install -e .[extras,mpi,tests]
 
     - name: Test mpi notebooks
       run : |

.github/workflows/examples.yml

@@ -42,8 +42,8 @@ jobs:
 
     - name: Install dependencies
       run: |
-        pip install -e .
-        pip install matplotlib
+        pip install -e .[tests]
+        pip install matplotlib pyrevolve
 
     - name: Tests in examples
       run: py.test --cov --cov-config=.coveragerc --cov-report=xml examples/

.github/workflows/pytest-aws.yaml

@@ -66,7 +66,7 @@ jobs:
       if: "!contains(matrix.name, 'docker')"
       run: |
         pip3 install --upgrade pip
-        pip3 install -U -e .
+        pip3 install -U -e .[tests]
         pip3 install pytest-xdist
   
     - name: Test with pytest

.github/workflows/pytest-core-mpi.yml

@@ -36,7 +36,7 @@ jobs:
       run: |
         sudo apt-get update && sudo apt install mpich -y
         pip3 install --upgrade pip
-        pip3 install -e .[extras,mpi]
+        pip3 install -e .[extras,mpi,tests]
 
     - name: Test with pytest
       run: |

.github/workflows/pytest-core-nompi.yml

@@ -153,7 +153,7 @@ jobs:
       if: "!contains(matrix.name, 'docker')"
       run: |
         pip install --upgrade pip
-        pip install -e .
+        pip install -e .[tests]
         pip install sympy==${{matrix.sympy}}
 
     - name: Test with pytest

.github/workflows/tutorials.yml

@@ -85,7 +85,7 @@ jobs:
       if: matrix.name != 'tutos-docker-gcc-py37'
       run: |
         python -m pip install --upgrade pip
-        pip install -e .
+        pip install -e .[tests]
         pip install matplotlib blosc
 
     - name: Seismic notebooks

devito/__init__.py

@@ -23,7 +23,13 @@
 from devito.data.allocators import *  # noqa
 from devito.logger import error, warning, info, set_log_level  # noqa
 from devito.mpi import MPI  # noqa
-from devito.checkpointing import DevitoCheckpoint, CheckpointOperator  # noqa
+try:
+    from devito.checkpointing import DevitoCheckpoint, CheckpointOperator  # noqa
+    from pyrevolve import Revolver
+except ImportError:
+    from devito.checkpointing import NoopCheckpoint as DevitoCheckpoint  # noqa
+    from devito.checkpointing import NoopCheckpointOperator as CheckpointOperator  # noqa
+    from devito.checkpointing import NoopRevolver as Revolver  # noqa
 
 # Imports required to initialize Devito
 from devito.arch import compiler_registry, platform_registry

devito/checkpointing/__init__.py

@@ -1 +1,24 @@
-from .checkpoint import *  # noqa
+try:
+    import pyrevolve as pyrevolve  # noqa
+    from .checkpoint import *  # noqa
+except ImportError:
+    pass
+
+
+class Noop(object):
+    """ Dummy replacement in case pyrevolve isn't available. """
+
+    def __init__(self, *args, **kwargs):
+        raise ImportError("Missing required `pyrevolve`; cannot use checkpointing")
+
+
+class NoopCheckpointOperator(Noop):
+    pass
+
+
+class NoopCheckpoint(Noop):
+    pass
+
+
+class NoopRevolver(Noop):
+    pass

docker/Dockerfile.devito

@@ -18,7 +18,7 @@ RUN python3 -m venv /venv && \
     /venv/bin/pip install --no-cache-dir --upgrade pip && \
     /venv/bin/pip install --no-cache-dir jupyter && \
     /venv/bin/pip install --no-cache-dir wheel && \
-    /venv/bin/pip install --no-cache-dir -e /app/devito[extras,mpi] && \
+    /venv/bin/pip install --no-cache-dir -e /app/devito[extras,mpi,tests] && \
     rm -rf ~/.cache/pip
 
 # Safety cleanup

docs/source/download.rst

@@ -45,7 +45,7 @@ To install the `latest Devito release`_ along with any additional dependencies,
 
    pip install devito
    # ...or to install additional dependencies:
-   # pip install devito[extras,mpi,nvidia]
+   # pip install devito[extras,mpi,nvidia,tests]
 
 .. _latest Devito release: https://pypi.org/project/devito/#history
 
@@ -55,11 +55,12 @@ To install the latest Devito development version, without the tutorials, follow:
 
    pip install git+https://github.com/devitocodes/devito.git
    # ...or to install additional dependencies:
-   # pip install git+https://github.com/devitocodes/devito.git#egg=project[extras,mpi,nvidia]
+   # pip install git+https://github.com/devitocodes/devito.git#egg=project[extras,mpi,nvidia,tests]
 
 Additional dependencies:
 
 - extras : optional dependencies for Jupyter notebooks, plotting, benchmarking
+- tests : optional dependencies required for testing infrastructure
 - mpi : optional dependencies for MPI (mpi4py)
 - nvidia : optional dependencies for targetting GPU deployment
 
@@ -98,7 +99,7 @@ and finally, install Devito along with any extra dependencies:
 
    pip install devito
    # ... or to install additional dependencies
-   # pip install devito[extras,mpi,nvidia]
+   # pip install devito[extras,mpi,nvidia,tests]
 
 
 For developers
@@ -117,7 +118,7 @@ and then install the requirements in your virtual environment (venv or conda):
    # Install requirements
    pip install -e .
    # ...or to install additional dependencies
-   # pip install -e .[extras,mpi,nvidia]
+   # pip install -e .[extras,mpi,nvidia,tests]
 
 
 Facing issues?

examples/seismic/acoustic/wavesolver.py

@@ -1,9 +1,8 @@
-from devito import Function, TimeFunction, DevitoCheckpoint, CheckpointOperator
+from devito import Function, TimeFunction, DevitoCheckpoint, CheckpointOperator, Revolver
 from devito.tools import memoized_meth
 from examples.seismic.acoustic.operators import (
     ForwardOperator, AdjointOperator, GradientOperator, BornOperator
 )
-from pyrevolve import Revolver
 
 
 class AcousticWaveSolver(object):

examples/seismic/tti/wavesolver.py

@@ -1,10 +1,10 @@
 # coding: utf-8
-from devito import Function, TimeFunction, warning, DevitoCheckpoint, CheckpointOperator
+from devito import (Function, TimeFunction, warning,
+                    DevitoCheckpoint, CheckpointOperator, Revolver)
 from devito.tools import memoized_meth
 from examples.seismic.tti.operators import ForwardOperator, AdjointOperator
 from examples.seismic.tti.operators import JacobianOperator, JacobianAdjOperator
 from examples.seismic.tti.operators import particle_velocity_fields
-from pyrevolve import Revolver
 
 
 class AnisotropicWaveSolver(object):

examples/seismic/viscoacoustic/wavesolver.py

@@ -1,11 +1,10 @@
 from devito import (VectorTimeFunction, TimeFunction, Function, NODE,
-                    DevitoCheckpoint, CheckpointOperator)
+                    DevitoCheckpoint, CheckpointOperator, Revolver)
 from devito.tools import memoized_meth
 from examples.seismic import PointSource
 from examples.seismic.viscoacoustic.operators import (
     ForwardOperator, AdjointOperator, GradientOperator, BornOperator
 )
-from pyrevolve import Revolver
 
 
 class ViscoacousticWaveSolver(object):

requirements-optional.txt

@@ -1,2 +1,3 @@
 matplotlib
 pandas
+pyrevolve

requirements-testing.txt

@@ -0,0 +1,4 @@
+pytest>=7.2,<8.0
+pytest-runner
+pytest-cov
+codecov

requirements.txt

@@ -12,8 +12,4 @@ codepy>=2019.1
 click<9.0
 multidict
 anytree>=2.4.3,<=2.8
-pyrevolve>=2.1.3
-distributed<2023.5
-pytest>=7.2,<8.0
-pytest-runner
-pytest-cov
+distributed<2023.4

setup.py

@@ -9,6 +9,9 @@
 with open('requirements-optional.txt') as f:
     optionals = f.read().splitlines()
 
+with open('requirements-testing.txt') as f:
+    testing = f.read().splitlines()
+
 with open('requirements-mpi.txt') as f:
     mpis = f.read().splitlines()
 
@@ -24,7 +27,8 @@
         reqs += [ir]
 
 extras_require = {}
-for mreqs, mode in zip([optionals, mpis, nvidias], ['extras', 'mpi', 'nvidia']):
+for mreqs, mode in (zip([optionals, mpis, nvidias, testing],
+                        ['extras', 'mpi', 'nvidia', 'tests'])):
     opt_reqs = []
     for ir in mreqs:
         # For conditionals like pytest=2.1; python == 3.6

tests/conftest.py

@@ -4,7 +4,8 @@
 import pytest
 import sys
 
-from devito import Eq, configuration  # noqa
+from devito import Eq, configuration, Revolver  # noqa
+from devito.checkpointing import NoopRevolver
 from devito.finite_differences.differentiable import EvalDerivative
 from devito.arch import Cpu64, Device, sniff_mpi_distro, Arm
 from devito.arch.compiler import compiler_registry, IntelCompiler, NvidiaCompiler
@@ -23,19 +24,17 @@ def skipif(items, whole_module=False):
     # Sanity check
     accepted = set()
     accepted.update({'device', 'device-C', 'device-openmp', 'device-openacc',
-                     'device-aomp', 'cpu64-icc', 'cpu64-nvc', 'cpu64-arm'})
+                     'device-aomp', 'cpu64-icc', 'cpu64-nvc', 'cpu64-arm', 'chkpnt'})
     accepted.update({'nompi', 'nodevice'})
     unknown = sorted(set(items) - accepted)
     if unknown:
         raise ValueError("Illegal skipif argument(s) `%s`" % unknown)
     skipit = False
     for i in items:
         # Skip if no MPI
-        if i == 'nompi':
-            if MPI is None:
-                skipit = "mpi4py/MPI not installed"
-                break
-            continue
+        if i == 'nompi' and MPI is None:
+            skipit = "mpi4py/MPI not installed"
+            break
         # Skip if won't run on GPUs
         if i == 'device' and isinstance(configuration['platform'], Device):
             skipit = "device `%s` unsupported" % configuration['platform'].name
@@ -73,6 +72,10 @@ def skipif(items, whole_module=False):
         if i == 'cpu64-arm' and isinstance(configuration['platform'], Arm):
             skipit = "Arm doesn't support x86-specific instructions"
             break
+        # Skip if pyrevolve not installed
+        if i == 'chkpnt' and Revolver is NoopRevolver:
+            skipit = "pyrevolve not installed"
+            break
 
     if skipit is False:
         return pytest.mark.skipif(False, reason='')

tests/test_checkpointing.py

@@ -1,11 +1,11 @@
 from functools import reduce
 
 import pytest
-from pyrevolve import Revolver
 import numpy as np
 
+from conftest import skipif
 from devito import (Grid, TimeFunction, Operator, Function, Eq, switchconfig, Constant,
-                    DevitoCheckpoint, CheckpointOperator)
+                    Revolver, CheckpointOperator, DevitoCheckpoint)
 from examples.seismic.acoustic.acoustic_example import acoustic_setup
 
 
@@ -107,6 +107,7 @@ def test_segmented_averaging():
     assert (f_ref.data_with_halo[1, -1] == 1.).all()
 
 
+@skipif('chkpnt')
 @switchconfig(log_level='WARNING')
 @pytest.mark.parametrize('space_order', [4])
 @pytest.mark.parametrize('kernel', ['OT2'])
@@ -160,6 +161,7 @@ def test_acoustic_save_and_nosave(shape=(50, 50), spacing=(15.0, 15.0), tn=500.,
     assert(np.allclose(rec.data, rec_bk))
 
 
+@skipif('chkpnt')
 def test_index_alignment():
     """ A much simpler test meant to ensure that the forward and reverse indices are
     correctly aligned (i.e. u * v , where u is the forward field and v the reverse field

tests/test_gradient.py

@@ -9,12 +9,12 @@
 from examples.seismic.acoustic.operators import iso_stencil
 from examples.seismic import Receiver, demo_model, setup_geometry
 from examples.seismic.tti import tti_setup
-from examples.seismic.viscoacoustic import viscoacoustic_setup
+from examples.seismic.viscoacoustic import viscoacoustic_setup as vsc_setup
 
 
 class TestGradient(object):
 
-    @skipif('cpu64-icc')
+    @skipif(['chkpnt', 'cpu64-icc'])
     @pytest.mark.parametrize('dtype', [np.float32, np.float64])
     @pytest.mark.parametrize('opt', [('advanced', {'openmp': True}),
                                      ('noop', {'openmp': True})])
@@ -148,14 +148,14 @@ def test_gradient_equivalence(self, shape, kernel, space_order, preset, nbl, dty
 
     @skipif('cpu64-icc')
     @pytest.mark.parametrize('kernel, shape, ckp, setup_func, time_order', [
-        ('OT2', (50, 60), True, iso_setup, 2),
+        pytest.param('OT2', (50, 60), True, iso_setup, 2, marks=skipif('chkpnt')),
         ('OT2', (50, 60), False, iso_setup, 2),
-        ('centered', (50, 60), True, tti_setup, 2),
+        pytest.param('centered', (50, 60), True, tti_setup, 2, marks=skipif('chkpnt')),
         ('centered', (50, 60), False, tti_setup, 2),
-        ('sls', (50, 60), True, viscoacoustic_setup, 2),
-        ('sls', (50, 60), False, viscoacoustic_setup, 2),
-        ('sls', (50, 60), True, viscoacoustic_setup, 1),
-        ('sls', (50, 60), False, viscoacoustic_setup, 1),
+        pytest.param('sls', (50, 60), True, vsc_setup, 2, marks=skipif('chkpnt')),
+        ('sls', (50, 60), False, vsc_setup, 2),
+        pytest.param('sls', (50, 60), True, vsc_setup, 1, marks=skipif('chkpnt')),
+        ('sls', (50, 60), False, vsc_setup, 1),
     ])
     @pytest.mark.parametrize('space_order', [4])
     @pytest.mark.parametrize('dtype', [np.float32, np.float64])
@@ -241,8 +241,8 @@ def initializer(data):
     @skipif('cpu64-icc')
     @pytest.mark.parametrize('kernel, shape, spacing, setup_func, time_order', [
         ('OT2', (70, 80), (15., 15.), iso_setup, 2),
-        ('sls', (70, 80), (20., 20.), viscoacoustic_setup, 2),
-        ('sls', (70, 80), (20., 20.), viscoacoustic_setup, 1),
+        ('sls', (70, 80), (20., 20.), vsc_setup, 2),
+        ('sls', (70, 80), (20., 20.), vsc_setup, 1),
         ('centered', (70, 80), (15., 15.), tti_setup, 2),
     ])
     @pytest.mark.parametrize('space_order', [4])