merge

radiasoft · Aug 12, 2024 · b8de621 · b8de621
2 parents 4cb1c4b + 502e5ea
commit b8de621
Show file tree

Hide file tree

Showing 527 changed files with 12,946 additions and 8,311 deletions.
diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml
@@ -18,6 +18,7 @@ jobs:
     LAPACKPP_HOME: '/usr/local'
     OMP_NUM_THREADS: 1
     WARPX_CI_CCACHE: 'TRUE'
+    WARPX_CI_CLEAN_TESTS: 'TRUE'
     WARPX_CI_NUM_MAKE_JOBS: 2
     WARPX_CI_OPENPMD: 'TRUE'
     WARPX_CI_TMP: '/tmp/ci'
@@ -109,6 +110,17 @@ jobs:
           -DCMAKE_CXX_STANDARD=17                       \
           -Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON
       fi
+      if [[ "${WARPX_CI_REGULAR_CARTESIAN_3D:-FALSE}" == "TRUE" ]]; then
+        cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/[email protected] \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)          \
+          -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF    \
+          -DHeffte_ENABLE_FFTW=ON -DHeffte_ENABLE_TESTING=OFF    \
+          -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=OFF      \
+          -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF     \
+          -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \
+          -DHeffte_ENABLE_MAGMA=OFF                              \
+          -DCMAKE_VERBOSE_MAKEFILE=ON
+      fi
       rm -rf ${CEI_TMP}
       df -h
     displayName: 'Install dependencies'

diff --git a/.github/workflows/clang_sanitizers.yml b/.github/workflows/clang_sanitizers.yml
@@ -0,0 +1,172 @@
+name: 🧴 clang sanitizers
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.head_ref }}-clangsanitizers
+  cancel-in-progress: true
+
+jobs:
+  build_UB_sanitizer:
+    name: Clang UB sanitizer
+    runs-on: ubuntu-22.04
+    if: github.event.pull_request.draft == false
+    env:
+      CC: clang
+      CXX: clang++
+      # On CI for this test, Ninja is slower than the default:
+      #CMAKE_GENERATOR: Ninja
+    steps:
+    - uses: actions/checkout@v4
+    - name: install dependencies
+      run: |
+        .github/workflows/dependencies/clang15.sh
+    - name: CCache Cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
+    - name: build WarpX
+      run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=100M
+        ccache -z
+
+        export CXX=$(which clang++-15)
+        export CC=$(which clang-15)
+        export CXXFLAGS="-fsanitize=undefined,address,pointer-compare -fno-sanitize-recover=all"
+
+        cmake -S . -B build                 \
+          -GNinja                           \
+          -DCMAKE_VERBOSE_MAKEFILE=ON       \
+          -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+          -DWarpX_DIMS="RZ;1;2;3"           \
+          -DWarpX_FFT=ON                    \
+          -DWarpX_QED=ON                    \
+          -DWarpX_QED_TABLE_GEN=ON          \
+          -DWarpX_OPENPMD=ON                \
+          -DWarpX_PRECISION=SINGLE          \
+          -DWarpX_PARTICLE_PRECISION=SINGLE
+        cmake --build build -j 4
+
+        ccache -s
+        du -hs ~/.cache/ccache
+
+    - name: run with UB sanitizer
+      run: |
+
+        export OMP_NUM_THREADS=2
+
+        #MPI implementations often leak memory
+        export "ASAN_OPTIONS=detect_leaks=0"
+
+        mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz
+        mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d
+        mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d
+        mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d
+
+  build_thread_sanitizer:
+    name: Clang thread sanitizer
+    runs-on: ubuntu-22.04
+    container: ubuntu:23.10
+    if: github.event.pull_request.draft == false
+    env:
+      CC: clang
+      CXX: clang++
+      # On CI for this test, Ninja is slower than the default:
+      #CMAKE_GENERATOR: Ninja
+    steps:
+    - uses: actions/checkout@v4
+    - name: install dependencies
+      run: |
+        .github/workflows/dependencies/clang17.sh
+    - name: CCache Cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
+    - name: build WarpX
+      run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=100M
+        ccache -z
+
+        export CXX=$(which clang++-17)
+        export CC=$(which clang-17)
+        export CXXFLAGS="-fsanitize=thread"
+
+        cmake -S . -B build                 \
+          -GNinja                           \
+          -DCMAKE_VERBOSE_MAKEFILE=ON       \
+          -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+          -DWarpX_DIMS="RZ;1;2;3"           \
+          -DWarpX_FFT=ON                    \
+          -DWarpX_QED=ON                    \
+          -DWarpX_QED_TABLE_GEN=ON          \
+          -DWarpX_OPENPMD=ON                \
+          -DWarpX_EB=OFF                    \
+          -DWarpX_PRECISION=DOUBLE          \
+          -DWarpX_PARTICLE_PRECISION=DOUBLE
+        cmake --build build -j 4
+
+        cmake -S . -B build_EB              \
+          -GNinja                           \
+          -DCMAKE_VERBOSE_MAKEFILE=ON       \
+          -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+          -DWarpX_DIMS="2"                  \
+          -DWarpX_FFT=ON                    \
+          -DWarpX_QED=ON                    \
+          -DWarpX_QED_TABLE_GEN=ON          \
+          -DWarpX_OPENPMD=ON                \
+          -DWarpX_EB=ON                     \
+          -DWarpX_PRECISION=DOUBLE          \
+          -DWarpX_PARTICLE_PRECISION=DOUBLE
+        cmake --build build_EB -j 4
+
+        ccache -s
+        du -hs ~/.cache/ccache
+
+    - name: run with thread sanitizer
+      run: |
+        export PMIX_MCA_gds=hash
+        export TSAN_OPTIONS='ignore_noninstrumented_modules=1'
+        export ARCHER_OPTIONS="verbose=1"
+
+        # We need these two lines because these tests run inside a docker container
+        export OMPI_ALLOW_RUN_AS_ROOT=1
+        export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
+        export OMP_NUM_THREADS=2
+
+        mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz warpx.serialize_initial_conditions = 0
+        mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d warpx.serialize_initial_conditions = 0
+        mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d warpx.serialize_initial_conditions = 0
+        mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d warpx.serialize_initial_conditions = 0
+
+        git clone https://github.com/ECP-WarpX/warpx-data ../warpx-data
+        cd Examples/Tests/embedded_circle
+
+        ulimit -c unlimited
+
+        mpirun -n 2 ../../../build_EB/bin/warpx.2d inputs_2d warpx.serialize_initial_conditions = 0
+
+  save_pr_number:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Save PR number
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+        run: |
+          echo $PR_NUMBER > pr_number.txt
+      - uses: actions/upload-artifact@v4
+        with:
+          name: pr_number
+          path: pr_number.txt
+          retention-days: 1
diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml
@@ -13,6 +13,7 @@ jobs:
         dim: [1, 2, RZ, 3]
     name: clang-tidy-${{ matrix.dim }}D
     runs-on: ubuntu-22.04
+    timeout-minutes: 120
     if: github.event.pull_request.draft == false
     steps:
     - uses: actions/checkout@v4
@@ -43,7 +44,7 @@ jobs:
           -DWarpX_DIMS="${{ matrix.dim }}"  \
           -DWarpX_MPI=ON                    \
           -DWarpX_COMPUTE=OMP               \
-          -DWarpX_PSATD=ON                  \
+          -DWarpX_FFT=ON                    \
           -DWarpX_QED=ON                    \
           -DWarpX_QED_TABLE_GEN=ON          \
           -DWarpX_OPENPMD=ON                \

diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -43,6 +43,11 @@ jobs:
 
         export CEI_SUDO="sudo"
         export CEI_TMP="/tmp/cei"
+
+        export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
+        export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
+        which nvcc || echo "nvcc not in PATH!"
+
         cmake-easyinstall --prefix=/usr/local \
           git+https://github.com/openPMD/[email protected] \
           -DopenPMD_USE_PYTHON=OFF    \
@@ -51,6 +56,16 @@ jobs:
           -DBUILD_CLI_TOOLS=OFF       \
           -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
           -DCMAKE_VERBOSE_MAKEFILE=ON
+        cmake-easyinstall --prefix=/usr/local                    \
+          git+https://github.com/icl-utk-edu/[email protected]   \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)          \
+          -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF    \
+          -DHeffte_ENABLE_FFTW=OFF -DHeffte_ENABLE_TESTING=OFF   \
+          -DHeffte_ENABLE_CUDA=ON -DHeffte_ENABLE_ROCM=OFF       \
+          -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF     \
+          -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \
+          -DHeffte_ENABLE_MAGMA=OFF                              \
+          -DCMAKE_VERBOSE_MAKEFILE=ON
     - name: build WarpX
       run: |
         export CCACHE_COMPRESS=1
@@ -70,7 +85,8 @@ jobs:
           -DWarpX_OPENPMD=ON           \
           -DWarpX_openpmd_internal=OFF \
           -DWarpX_PRECISION=SINGLE     \
-          -DWarpX_PSATD=ON             \
+          -DWarpX_FFT=ON               \
+          -DWarpX_HEFFTE=ON            \
           -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
           -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
         cmake --build build_sp -j 4
@@ -115,8 +131,8 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach 2a3955a5f5aac1aef6e6e72687f182331e049c39 && cd -
-        make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4
+        cd ../amrex && git checkout --detach 24.08 && cd -
+        make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4
 
         ccache -s
         du -hs ~/.cache/ccache
@@ -168,7 +184,7 @@ jobs:
           -DWarpX_PYTHON=OFF           \
           -DAMReX_CUDA_ARCH=8.0        \
           -DWarpX_OPENPMD=ON           \
-          -DWarpX_PSATD=ON             \
+          -DWarpX_FFT=ON               \
           -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
           -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
         cmake --build build -j 4

diff --git a/.github/workflows/dependencies/clang17.sh b/.github/workflows/dependencies/clang17.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024 The WarpX Community
+#
+# License: BSD-3-Clause-LBNL
+# Authors: Luca Fedeli
+
+set -eu -o pipefail
+
+# This dependency file is currently used within a docker container,
+# which does not come with sudo.
+apt-get -qqq update
+apt-get -y install sudo
+
+# `man apt.conf`:
+#   Number of retries to perform. If this is non-zero APT will retry
+#   failed files the given number of times.
+echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
+
+# This dependency file is currently used within a docker container,
+# which does not come (among others) with wget, xz-utils, curl, git,
+# ccache, and pkg-config pre-installed.
+sudo apt-get -qqq update
+sudo apt-get install -y \
+    cmake               \
+    clang-17            \
+    clang-tidy-17       \
+    libblas-dev         \
+    libc++-17-dev       \
+    libboost-math-dev   \
+    libfftw3-dev        \
+    libfftw3-mpi-dev    \
+    libhdf5-openmpi-dev \
+    liblapack-dev       \
+    libopenmpi-dev      \
+    libomp-17-dev       \
+    ninja-build         \
+    wget                \
+    xz-utils            \
+    curl                \
+    git                 \
+    ccache              \
+    pkg-config
+
+# Use clang 17
+export CXX=$(which clang++-17)
+export CC=$(which clang-17)
+
+# cmake-easyinstall
+#
+sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall
+sudo chmod a+x /usr/local/bin/cmake-easyinstall
+export CEI_SUDO="sudo"
+export CEI_TMP="/tmp/cei"
+
+# BLAS++ & LAPACK++
+cmake-easyinstall \
+  --prefix=/usr/local                           \
+  git+https://github.com/icl-utk-edu/blaspp.git \
+  -Duse_openmp=OFF                              \
+  -Dbuild_tests=OFF                             \
+  -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
+  -DCMAKE_VERBOSE_MAKEFILE=ON
+
+cmake-easyinstall \
+  --prefix=/usr/local                             \
+  git+https://github.com/icl-utk-edu/lapackpp.git \
+  -Duse_cmake_find_lapack=ON                      \
+  -Dbuild_tests=OFF                               \
+  -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)   \
+  -DCMAKE_VERBOSE_MAKEFILE=ON