diff --git a/.github/workflows/examples-mpi.yml b/.github/workflows/examples-mpi.yml index 48c76c4240..6c46297580 100644 --- a/.github/workflows/examples-mpi.yml +++ b/.github/workflows/examples-mpi.yml @@ -43,7 +43,7 @@ jobs: - name: Install dependencies run: | pip install --upgrade pip - pip install -e .[extras,mpi] + pip install -e .[extras,mpi,tests] - name: Test mpi notebooks run : | diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 28ecfcf378..f260a69b8d 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -42,8 +42,8 @@ jobs: - name: Install dependencies run: | - pip install -e . - pip install matplotlib + pip install -e .[tests] + pip install matplotlib pyrevolve - name: Tests in examples run: py.test --cov --cov-config=.coveragerc --cov-report=xml examples/ diff --git a/.github/workflows/pytest-aws.yaml b/.github/workflows/pytest-aws.yaml index 15992b6837..f57b83fef9 100644 --- a/.github/workflows/pytest-aws.yaml +++ b/.github/workflows/pytest-aws.yaml @@ -66,7 +66,7 @@ jobs: if: "!contains(matrix.name, 'docker')" run: | pip3 install --upgrade pip - pip3 install -U -e . + pip3 install -U -e .[tests] pip3 install pytest-xdist - name: Test with pytest diff --git a/.github/workflows/pytest-core-mpi.yml b/.github/workflows/pytest-core-mpi.yml index 1307baa85e..4774fe517d 100644 --- a/.github/workflows/pytest-core-mpi.yml +++ b/.github/workflows/pytest-core-mpi.yml @@ -36,7 +36,7 @@ jobs: run: | sudo apt-get update && sudo apt install mpich -y pip3 install --upgrade pip - pip3 install -e .[extras,mpi] + pip3 install -e .[extras,mpi,tests] - name: Test with pytest run: | diff --git a/.github/workflows/pytest-core-nompi.yml b/.github/workflows/pytest-core-nompi.yml index c5e431ffc0..5f5d3b0f37 100644 --- a/.github/workflows/pytest-core-nompi.yml +++ b/.github/workflows/pytest-core-nompi.yml @@ -153,7 +153,7 @@ jobs: if: "!contains(matrix.name, 'docker')" run: | pip install --upgrade pip - pip install -e . + pip install -e .[tests] pip install sympy==${{matrix.sympy}} - name: Test with pytest diff --git a/.github/workflows/tutorials.yml b/.github/workflows/tutorials.yml index 526a0899cf..32c4e1b65b 100644 --- a/.github/workflows/tutorials.yml +++ b/.github/workflows/tutorials.yml @@ -85,7 +85,7 @@ jobs: if: matrix.name != 'tutos-docker-gcc-py37' run: | python -m pip install --upgrade pip - pip install -e . + pip install -e .[tests] pip install matplotlib blosc - name: Seismic notebooks diff --git a/devito/__init__.py b/devito/__init__.py index ae4d819eaf..cd61626b28 100644 --- a/devito/__init__.py +++ b/devito/__init__.py @@ -23,7 +23,13 @@ from devito.data.allocators import * # noqa from devito.logger import error, warning, info, set_log_level # noqa from devito.mpi import MPI # noqa -from devito.checkpointing import DevitoCheckpoint, CheckpointOperator # noqa +try: + from devito.checkpointing import DevitoCheckpoint, CheckpointOperator # noqa + from pyrevolve import Revolver +except ImportError: + from devito.checkpointing import NoopCheckpoint as DevitoCheckpoint # noqa + from devito.checkpointing import NoopCheckpointOperator as CheckpointOperator # noqa + from devito.checkpointing import NoopRevolver as Revolver # noqa # Imports required to initialize Devito from devito.arch import compiler_registry, platform_registry diff --git a/devito/checkpointing/__init__.py b/devito/checkpointing/__init__.py index 5102c7a2a8..22bca408b7 100644 --- a/devito/checkpointing/__init__.py +++ b/devito/checkpointing/__init__.py @@ -1 +1,24 @@ -from .checkpoint import * # noqa +try: + import pyrevolve as pyrevolve # noqa + from .checkpoint import * # noqa +except ImportError: + pass + + +class Noop(object): + """ Dummy replacement in case pyrevolve isn't available. """ + + def __init__(self, *args, **kwargs): + raise ImportError("Missing required `pyrevolve`; cannot use checkpointing") + + +class NoopCheckpointOperator(Noop): + pass + + +class NoopCheckpoint(Noop): + pass + + +class NoopRevolver(Noop): + pass diff --git a/docker/Dockerfile.devito b/docker/Dockerfile.devito index 0be27a1383..81d907400c 100644 --- a/docker/Dockerfile.devito +++ b/docker/Dockerfile.devito @@ -18,7 +18,7 @@ RUN python3 -m venv /venv && \ /venv/bin/pip install --no-cache-dir --upgrade pip && \ /venv/bin/pip install --no-cache-dir jupyter && \ /venv/bin/pip install --no-cache-dir wheel && \ - /venv/bin/pip install --no-cache-dir -e /app/devito[extras,mpi] && \ + /venv/bin/pip install --no-cache-dir -e /app/devito[extras,mpi,tests] && \ rm -rf ~/.cache/pip # Safety cleanup diff --git a/docs/source/download.rst b/docs/source/download.rst index 71549de86d..2ee073f69b 100644 --- a/docs/source/download.rst +++ b/docs/source/download.rst @@ -45,7 +45,7 @@ To install the `latest Devito release`_ along with any additional dependencies, pip install devito # ...or to install additional dependencies: - # pip install devito[extras,mpi,nvidia] + # pip install devito[extras,mpi,nvidia,tests] .. _latest Devito release: https://pypi.org/project/devito/#history @@ -55,11 +55,12 @@ To install the latest Devito development version, without the tutorials, follow: pip install git+https://github.com/devitocodes/devito.git # ...or to install additional dependencies: - # pip install git+https://github.com/devitocodes/devito.git#egg=project[extras,mpi,nvidia] + # pip install git+https://github.com/devitocodes/devito.git#egg=project[extras,mpi,nvidia,tests] Additional dependencies: - extras : optional dependencies for Jupyter notebooks, plotting, benchmarking +- tests : optional dependencies required for testing infrastructure - mpi : optional dependencies for MPI (mpi4py) - nvidia : optional dependencies for targetting GPU deployment @@ -98,7 +99,7 @@ and finally, install Devito along with any extra dependencies: pip install devito # ... or to install additional dependencies - # pip install devito[extras,mpi,nvidia] + # pip install devito[extras,mpi,nvidia,tests] For developers @@ -117,7 +118,7 @@ and then install the requirements in your virtual environment (venv or conda): # Install requirements pip install -e . # ...or to install additional dependencies - # pip install -e .[extras,mpi,nvidia] + # pip install -e .[extras,mpi,nvidia,tests] Facing issues? diff --git a/examples/seismic/acoustic/wavesolver.py b/examples/seismic/acoustic/wavesolver.py index 20a2241bbc..3636a86616 100644 --- a/examples/seismic/acoustic/wavesolver.py +++ b/examples/seismic/acoustic/wavesolver.py @@ -1,9 +1,8 @@ -from devito import Function, TimeFunction, DevitoCheckpoint, CheckpointOperator +from devito import Function, TimeFunction, DevitoCheckpoint, CheckpointOperator, Revolver from devito.tools import memoized_meth from examples.seismic.acoustic.operators import ( ForwardOperator, AdjointOperator, GradientOperator, BornOperator ) -from pyrevolve import Revolver class AcousticWaveSolver(object): diff --git a/examples/seismic/tti/wavesolver.py b/examples/seismic/tti/wavesolver.py index c9043d7c10..5aef427401 100644 --- a/examples/seismic/tti/wavesolver.py +++ b/examples/seismic/tti/wavesolver.py @@ -1,10 +1,10 @@ # coding: utf-8 -from devito import Function, TimeFunction, warning, DevitoCheckpoint, CheckpointOperator +from devito import (Function, TimeFunction, warning, + DevitoCheckpoint, CheckpointOperator, Revolver) from devito.tools import memoized_meth from examples.seismic.tti.operators import ForwardOperator, AdjointOperator from examples.seismic.tti.operators import JacobianOperator, JacobianAdjOperator from examples.seismic.tti.operators import particle_velocity_fields -from pyrevolve import Revolver class AnisotropicWaveSolver(object): diff --git a/examples/seismic/viscoacoustic/wavesolver.py b/examples/seismic/viscoacoustic/wavesolver.py index c40efd8a7b..05125807b8 100755 --- a/examples/seismic/viscoacoustic/wavesolver.py +++ b/examples/seismic/viscoacoustic/wavesolver.py @@ -1,11 +1,10 @@ from devito import (VectorTimeFunction, TimeFunction, Function, NODE, - DevitoCheckpoint, CheckpointOperator) + DevitoCheckpoint, CheckpointOperator, Revolver) from devito.tools import memoized_meth from examples.seismic import PointSource from examples.seismic.viscoacoustic.operators import ( ForwardOperator, AdjointOperator, GradientOperator, BornOperator ) -from pyrevolve import Revolver class ViscoacousticWaveSolver(object): diff --git a/requirements-optional.txt b/requirements-optional.txt index babdd14a51..d13e1ca946 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -1,2 +1,3 @@ matplotlib pandas +pyrevolve \ No newline at end of file diff --git a/requirements-testing.txt b/requirements-testing.txt new file mode 100644 index 0000000000..8fce562f37 --- /dev/null +++ b/requirements-testing.txt @@ -0,0 +1,4 @@ +pytest>=7.2,<8.0 +pytest-runner +pytest-cov +codecov \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 56b39e0b33..2a359c0568 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,8 +12,4 @@ codepy>=2019.1 click<9.0 multidict anytree>=2.4.3,<=2.8 -pyrevolve>=2.1.3 -distributed<2023.5 -pytest>=7.2,<8.0 -pytest-runner -pytest-cov +distributed<2023.4 diff --git a/setup.py b/setup.py index 799bec8407..3d78741600 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,9 @@ with open('requirements-optional.txt') as f: optionals = f.read().splitlines() +with open('requirements-testing.txt') as f: + testing = f.read().splitlines() + with open('requirements-mpi.txt') as f: mpis = f.read().splitlines() @@ -24,7 +27,8 @@ reqs += [ir] extras_require = {} -for mreqs, mode in zip([optionals, mpis, nvidias], ['extras', 'mpi', 'nvidia']): +for mreqs, mode in (zip([optionals, mpis, nvidias, testing], + ['extras', 'mpi', 'nvidia', 'tests'])): opt_reqs = [] for ir in mreqs: # For conditionals like pytest=2.1; python == 3.6 diff --git a/tests/conftest.py b/tests/conftest.py index 51d8e63138..55afa7cede 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,8 @@ import pytest import sys -from devito import Eq, configuration # noqa +from devito import Eq, configuration, Revolver # noqa +from devito.checkpointing import NoopRevolver from devito.finite_differences.differentiable import EvalDerivative from devito.arch import Cpu64, Device, sniff_mpi_distro, Arm from devito.arch.compiler import compiler_registry, IntelCompiler, NvidiaCompiler @@ -23,7 +24,7 @@ def skipif(items, whole_module=False): # Sanity check accepted = set() accepted.update({'device', 'device-C', 'device-openmp', 'device-openacc', - 'device-aomp', 'cpu64-icc', 'cpu64-nvc', 'cpu64-arm'}) + 'device-aomp', 'cpu64-icc', 'cpu64-nvc', 'cpu64-arm', 'chkpnt'}) accepted.update({'nompi', 'nodevice'}) unknown = sorted(set(items) - accepted) if unknown: @@ -31,11 +32,9 @@ def skipif(items, whole_module=False): skipit = False for i in items: # Skip if no MPI - if i == 'nompi': - if MPI is None: - skipit = "mpi4py/MPI not installed" - break - continue + if i == 'nompi' and MPI is None: + skipit = "mpi4py/MPI not installed" + break # Skip if won't run on GPUs if i == 'device' and isinstance(configuration['platform'], Device): skipit = "device `%s` unsupported" % configuration['platform'].name @@ -73,6 +72,10 @@ def skipif(items, whole_module=False): if i == 'cpu64-arm' and isinstance(configuration['platform'], Arm): skipit = "Arm doesn't support x86-specific instructions" break + # Skip if pyrevolve not installed + if i == 'chkpnt' and Revolver is NoopRevolver: + skipit = "pyrevolve not installed" + break if skipit is False: return pytest.mark.skipif(False, reason='') diff --git a/tests/test_checkpointing.py b/tests/test_checkpointing.py index 3c5616191a..75cca861cc 100644 --- a/tests/test_checkpointing.py +++ b/tests/test_checkpointing.py @@ -1,11 +1,11 @@ from functools import reduce import pytest -from pyrevolve import Revolver import numpy as np +from conftest import skipif from devito import (Grid, TimeFunction, Operator, Function, Eq, switchconfig, Constant, - DevitoCheckpoint, CheckpointOperator) + Revolver, CheckpointOperator, DevitoCheckpoint) from examples.seismic.acoustic.acoustic_example import acoustic_setup @@ -107,6 +107,7 @@ def test_segmented_averaging(): assert (f_ref.data_with_halo[1, -1] == 1.).all() +@skipif('chkpnt') @switchconfig(log_level='WARNING') @pytest.mark.parametrize('space_order', [4]) @pytest.mark.parametrize('kernel', ['OT2']) @@ -160,6 +161,7 @@ def test_acoustic_save_and_nosave(shape=(50, 50), spacing=(15.0, 15.0), tn=500., assert(np.allclose(rec.data, rec_bk)) +@skipif('chkpnt') def test_index_alignment(): """ A much simpler test meant to ensure that the forward and reverse indices are correctly aligned (i.e. u * v , where u is the forward field and v the reverse field diff --git a/tests/test_gradient.py b/tests/test_gradient.py index b401421085..9c91138c84 100644 --- a/tests/test_gradient.py +++ b/tests/test_gradient.py @@ -9,12 +9,12 @@ from examples.seismic.acoustic.operators import iso_stencil from examples.seismic import Receiver, demo_model, setup_geometry from examples.seismic.tti import tti_setup -from examples.seismic.viscoacoustic import viscoacoustic_setup +from examples.seismic.viscoacoustic import viscoacoustic_setup as vsc_setup class TestGradient(object): - @skipif('cpu64-icc') + @skipif(['chkpnt', 'cpu64-icc']) @pytest.mark.parametrize('dtype', [np.float32, np.float64]) @pytest.mark.parametrize('opt', [('advanced', {'openmp': True}), ('noop', {'openmp': True})]) @@ -148,14 +148,14 @@ def test_gradient_equivalence(self, shape, kernel, space_order, preset, nbl, dty @skipif('cpu64-icc') @pytest.mark.parametrize('kernel, shape, ckp, setup_func, time_order', [ - ('OT2', (50, 60), True, iso_setup, 2), + pytest.param('OT2', (50, 60), True, iso_setup, 2, marks=skipif('chkpnt')), ('OT2', (50, 60), False, iso_setup, 2), - ('centered', (50, 60), True, tti_setup, 2), + pytest.param('centered', (50, 60), True, tti_setup, 2, marks=skipif('chkpnt')), ('centered', (50, 60), False, tti_setup, 2), - ('sls', (50, 60), True, viscoacoustic_setup, 2), - ('sls', (50, 60), False, viscoacoustic_setup, 2), - ('sls', (50, 60), True, viscoacoustic_setup, 1), - ('sls', (50, 60), False, viscoacoustic_setup, 1), + pytest.param('sls', (50, 60), True, vsc_setup, 2, marks=skipif('chkpnt')), + ('sls', (50, 60), False, vsc_setup, 2), + pytest.param('sls', (50, 60), True, vsc_setup, 1, marks=skipif('chkpnt')), + ('sls', (50, 60), False, vsc_setup, 1), ]) @pytest.mark.parametrize('space_order', [4]) @pytest.mark.parametrize('dtype', [np.float32, np.float64]) @@ -241,8 +241,8 @@ def initializer(data): @skipif('cpu64-icc') @pytest.mark.parametrize('kernel, shape, spacing, setup_func, time_order', [ ('OT2', (70, 80), (15., 15.), iso_setup, 2), - ('sls', (70, 80), (20., 20.), viscoacoustic_setup, 2), - ('sls', (70, 80), (20., 20.), viscoacoustic_setup, 1), + ('sls', (70, 80), (20., 20.), vsc_setup, 2), + ('sls', (70, 80), (20., 20.), vsc_setup, 1), ('centered', (70, 80), (15., 15.), tti_setup, 2), ]) @pytest.mark.parametrize('space_order', [4])