ci: reinstate amd mpi tests

devitocodes · Feb 1, 2023 · 874713c · 874713c
1 parent 7d25d52
commit 874713c
Show file tree

Hide file tree

Showing 6 changed files with 17 additions and 23 deletions.
diff --git a/.github/workflows/pytest-gpu.yml b/.github/workflows/pytest-gpu.yml
@@ -46,31 +46,34 @@ jobs:
           pytest-gpu-acc-nvidia,
           pytest-gpu-omp-amd
         ]
+        test_examples: ["examples/seismic/tti/tti_example.py examples/seismic/acoustic/acoustic_example.py examples/seismic/viscoacoustic/viscoacoustic_example.py examples/seismic/viscoelastic/viscoelastic_example.py examples/seismic/elastic/elastic_example.py"]
+
         include:
         - name: pytest-gpu-omp-nvidia
           test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openmp.py"
           base: "devitocodes/bases:nvidia-clang"
           tags: ["self-hosted", "nvidiagpu"]
           test_drive_cmd: "nvidia-smi"
           flags: '--gpus all --rm --name testrun-clang-nvidia'
+          mpi_flags: ''
 
         - name: pytest-gpu-acc-nvidia
           test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openacc.py"
           base: "devitocodes/bases:nvidia-nvc"
           tags: ["self-hosted", "nvidiagpu"]
           test_drive_cmd: "nvidia-smi"
           flags: '--gpus all --rm --name testrun-nvc'
+          mpi_flags: ''
 
         - name: pytest-gpu-omp-amd
-          # We only test the OpenMP generation due to issue with the offloading discussed in
-          # https://github.com/ROCm-Developer-Tools/aomp/issues/397
-          test_files: "tests/test_gpu_openmp.py"
+          test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openmp.py"
           tags: ["self-hosted", "amdgpu"]
           base: "devitocodes/bases:amd"
           test_drive_cmd: "rocm-smi"
           # Attach the AMD GPU devices `/dev` and add user to video and render (109 on wampa) group
           # Options from https://rocmdocs.amd.com/en/latest/ROCm_Virtualization_Containers/ROCm-Virtualization-&-Containers.html
           flags: "--network=host --device=/dev/kfd --device=/dev/dri --ipc=host --group-add video --group-add 109 --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --rm --name testrun-amd"
+          mpi_flags: "--mca btl '^openib' -x UCX_TLS=sm,self,rocm_copy,rocm_ipc  --mca pml ucx"
 
     steps:
     - name: Checkout devito
@@ -93,17 +96,8 @@ jobs:
 
     - name: Test examples
       run: |
-        docker run ${{ matrix.flags }} ${{ matrix.name }} pytest examples/seismic/acoustic/acoustic_example.py
-        docker run ${{ matrix.flags }} ${{ matrix.name }} pytest examples/seismic/elastic/elastic_example.py
-        docker run ${{ matrix.flags }} ${{ matrix.name }} pytest examples/seismic/tti/tti_example.py
-        docker run ${{ matrix.flags }} ${{ matrix.name }} pytest examples/seismic/viscoacoustic/viscoacoustic_example.py
-        docker run ${{ matrix.flags }} ${{ matrix.name }} pytest examples/seismic/viscoelastic/viscoelastic_example.py
+        docker run ${{ matrix.flags }} ${{ matrix.name }} pytest ${{ matrix.test_examples }}
 
     - name: Test examples with MPI
-      if: matrix.name != 'pytest-gpu-omp-amd'
       run: |
-        docker run ${{ matrix.flags }} --env DEVITO_MPI=1 ${{ matrix.name }} mpirun -n 2 pytest examples/seismic/acoustic/acoustic_example.py
-        docker run ${{ matrix.flags }} --env DEVITO_MPI=1 ${{ matrix.name }} mpirun -n 2 pytest examples/seismic/elastic/elastic_example.py
-        docker run ${{ matrix.flags }} --env DEVITO_MPI=1 ${{ matrix.name }} mpirun -n 2 pytest examples/seismic/tti/tti_example.py
-        docker run ${{ matrix.flags }} --env DEVITO_MPI=1 ${{ matrix.name }} mpirun -n 2 pytest examples/seismic/viscoacoustic/viscoacoustic_example.py
-        docker run ${{ matrix.flags }} --env DEVITO_MPI=1 ${{ matrix.name }} mpirun -n 2 pytest examples/seismic/viscoelastic/viscoelastic_example.py
+        docker run ${{ matrix.flags }} --env DEVITO_MPI=1 ${{ matrix.name }} mpirun ${{ matrix.mpi_flags }} -n 2 pytest ${{ matrix.test_examples }}
diff --git a/docker/Dockerfile.amd b/docker/Dockerfile.amd
@@ -16,7 +16,10 @@ ENV ROCM_HOME /opt/rocm
 ENV HIP_HOME $ROCM_HOME/hip
 
 # Some utils needed
-RUN apt-get update && apt-get install -y wget git autoconf dh-autoreconf flex python3-venv python3-dev vim libnuma1 tmux
+RUN apt-get update && \
+    apt-get install -y wget git autoconf dh-autoreconf flex \
+                       python3-venv python3-dev \
+                       vim libnuma-dev tmux numactl
 
 # Install tmpi
 RUN curl https://raw.githubusercontent.com/Azrael3000/tmpi/master/tmpi -o /usr/local/bin/tmpi

diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu
@@ -11,7 +11,7 @@ FROM ${pyversion} as base
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install for basic base not containing it
-RUN apt-get update && apt-get install -y vim wget git libnuma1 tmux
+RUN apt-get update && apt-get install -y vim wget git libnuma-dev tmux numactl
 
 # Install tmpi
 RUN curl https://raw.githubusercontent.com/Azrael3000/tmpi/master/tmpi -o /usr/local/bin/tmpi
@@ -24,7 +24,7 @@ RUN apt-get install -y libgl1-mesa-glx
 ##############################################################
 FROM base as gcc
 
-RUN apt-get install -y mpich libmpich-dev 
+RUN apt-get install -y mpich libmpich-dev
 # Env vars defaults
 ENV DEVITO_ARCH="gcc"
 ENV DEVITO_LANGUAGE="openmp"

diff --git a/docker/Dockerfile.nvidia b/docker/Dockerfile.nvidia
@@ -12,7 +12,7 @@ FROM ${pyversion}-slim-bullseye as sdk-base
 
 ENV DEBIAN_FRONTEND noninteractive
 
-RUN apt-get update -y && apt-get install -y -q gpg apt-utils curl wget vim libnuma1 tmux
+RUN apt-get update -y && apt-get install -y -q gpg apt-utils curl wget vim libnuma-dev tmux numactl
 
 # nodesource: nvdashboard requires nodejs>=10
 RUN curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | gpg --yes --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg

diff --git a/examples/seismic/tti/tti_example.py b/examples/seismic/tti/tti_example.py
@@ -71,12 +71,12 @@ def run(shape=(50, 50, 50), spacing=(20.0, 20.0, 20.0), tn=250.0,
     return summary.gflopss, summary.oi, summary.timings, [rec, u, v]
 
 
-@pytest.mark.parametrize('shape', [(51, 51), (16, 16, 16)])
+@pytest.mark.parametrize('shape', [(51, 51), (32, 16, 16)])
 @pytest.mark.parametrize('kernel', ['centered', 'staggered'])
 def test_tti_stability(shape, kernel):
     spacing = tuple([20]*len(shape))
     _, _, _, [rec, _, _] = run(shape=shape, spacing=spacing, kernel=kernel,
-                               tn=16000.0, nbl=0)
+                               tn=16000.0, nbl=2)
     assert np.isfinite(norm(rec))
 
 

diff --git a/tests/test_gpu_openmp.py b/tests/test_gpu_openmp.py
@@ -23,7 +23,6 @@ def test_init_omp_env(self):
         assert str(op.body.init[0].body[0]) ==\
             'if (deviceid != -1)\n{\n  omp_set_default_device(deviceid);\n}'
 
-    @skipif('device-aomp')
     @pytest.mark.parallel(mode=1)
     def test_init_omp_env_w_mpi(self):
         grid = Grid(shape=(3, 3, 3))
@@ -345,7 +344,6 @@ def test_iso_acoustic(self, opt):
 
 class TestMPI(object):
 
-    @skipif('device-aomp')
     @pytest.mark.parallel(mode=[2, 4])
     def test_mpi_nocomms(self):
         grid = Grid(shape=(3, 3, 3))
@@ -362,7 +360,6 @@ def test_mpi_nocomms(self):
 
         assert np.all(np.array(u.data[0, :, :, :]) == time_steps)
 
-    @skipif('device-aomp')
     @pytest.mark.parallel(mode=[2, 4])
     def test_iso_ac(self):
         TestOperator().iso_acoustic(opt='advanced')