Skip to content

Commit b25467e

Browse files
committed
Merge remote-tracking branch 'upstream/development' into HSdevmerge_240807
2 parents 4005b09 + ffb16f3 commit b25467e

File tree

330 files changed

+8011
-4571
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

330 files changed

+8011
-4571
lines changed

.github/workflows/clang_tidy.yml

+2-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ jobs:
1313
dim: [1, 2, RZ, 3]
1414
name: clang-tidy-${{ matrix.dim }}D
1515
runs-on: ubuntu-22.04
16+
timeout-minutes: 120
1617
if: github.event.pull_request.draft == false
1718
steps:
1819
- uses: actions/checkout@v4
@@ -43,7 +44,7 @@ jobs:
4344
-DWarpX_DIMS="${{ matrix.dim }}" \
4445
-DWarpX_MPI=ON \
4546
-DWarpX_COMPUTE=OMP \
46-
-DWarpX_PSATD=ON \
47+
-DWarpX_FFT=ON \
4748
-DWarpX_QED=ON \
4849
-DWarpX_QED_TABLE_GEN=ON \
4950
-DWarpX_OPENPMD=ON \

.github/workflows/cuda.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ jobs:
7070
-DWarpX_OPENPMD=ON \
7171
-DWarpX_openpmd_internal=OFF \
7272
-DWarpX_PRECISION=SINGLE \
73-
-DWarpX_PSATD=ON \
73+
-DWarpX_FFT=ON \
7474
-DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
7575
-DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
7676
cmake --build build_sp -j 4
@@ -115,8 +115,8 @@ jobs:
115115
which nvcc || echo "nvcc not in PATH!"
116116
117117
git clone https://github.com/AMReX-Codes/amrex.git ../amrex
118-
cd ../amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd -
119-
make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4
118+
cd ../amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd -
119+
make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4
120120
121121
ccache -s
122122
du -hs ~/.cache/ccache
@@ -168,7 +168,7 @@ jobs:
168168
-DWarpX_PYTHON=OFF \
169169
-DAMReX_CUDA_ARCH=8.0 \
170170
-DWarpX_OPENPMD=ON \
171-
-DWarpX_PSATD=ON \
171+
-DWarpX_FFT=ON \
172172
-DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
173173
-DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
174174
cmake --build build -j 4

.github/workflows/hip.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ jobs:
5555
-DWarpX_MPI=ON \
5656
-DWarpX_OPENPMD=ON \
5757
-DWarpX_PRECISION=SINGLE \
58-
-DWarpX_PSATD=ON
58+
-DWarpX_FFT=ON
5959
cmake --build build_sp -j 4
6060
6161
export WARPX_MPI=OFF
@@ -115,7 +115,7 @@ jobs:
115115
-DWarpX_MPI=ON \
116116
-DWarpX_OPENPMD=ON \
117117
-DWarpX_PRECISION=DOUBLE \
118-
-DWarpX_PSATD=ON
118+
-DWarpX_FFT=ON
119119
cmake --build build_2d -j 4
120120
121121
export WARPX_MPI=OFF

.github/workflows/macos.yml

+3-2
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ jobs:
4444
- name: CCache Cache
4545
uses: actions/cache@v4
4646
with:
47-
path: /Users/runner/Library/Caches/ccache
47+
path: ~/Library/Caches/ccache
4848
key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
4949
restore-keys: |
5050
ccache-${{ github.workflow }}-${{ github.job }}-git-
@@ -53,7 +53,7 @@ jobs:
5353
export CCACHE_COMPRESS=1
5454
export CCACHE_COMPRESSLEVEL=10
5555
export CCACHE_MAXSIZE=100M
56-
export CCACHE_DEPEND=1
56+
export CCACHE_SLOPPINESS=time_macros
5757
ccache -z
5858
5959
source py-venv/bin/activate
@@ -76,6 +76,7 @@ jobs:
7676
cmake --build build_sp -j 3
7777
cmake --build build_sp --target pip_install
7878
79+
du -hs ~/Library/Caches/ccache
7980
ccache -s
8081
8182
- name: run pywarpx

.github/workflows/ubuntu.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ jobs:
7777
-DCMAKE_VERBOSE_MAKEFILE=ON \
7878
-DWarpX_DIMS="1;2" \
7979
-DWarpX_EB=OFF \
80-
-DWarpX_PSATD=ON \
80+
-DWarpX_FFT=ON \
8181
-DWarpX_QED_TABLE_GEN=ON \
8282
-DWarpX_QED_TOOLS=ON
8383
@@ -127,7 +127,7 @@ jobs:
127127
-DCMAKE_VERBOSE_MAKEFILE=ON \
128128
-DWarpX_DIMS="RZ;3" \
129129
-DWarpX_EB=OFF \
130-
-DWarpX_PSATD=ON \
130+
-DWarpX_FFT=ON \
131131
-DWarpX_PRECISION=SINGLE \
132132
-DWarpX_PARTICLE_PRECISION=SINGLE \
133133
-DWarpX_QED_TABLE_GEN=ON
@@ -211,7 +211,7 @@ jobs:
211211
cmake -S . -B build \
212212
-DCMAKE_VERBOSE_MAKEFILE=ON \
213213
-DWarpX_APP=OFF \
214-
-DWarpX_PSATD=ON \
214+
-DWarpX_FFT=ON \
215215
-DWarpX_PYTHON=ON \
216216
-DWarpX_QED_TABLE_GEN=ON
217217
cmake --build build -j 4 --target pip_install
@@ -260,7 +260,7 @@ jobs:
260260
-GNinja \
261261
-DCMAKE_VERBOSE_MAKEFILE=ON \
262262
-DWarpX_DIMS="RZ;1;2;3" \
263-
-DWarpX_PSATD=ON \
263+
-DWarpX_FFT=ON \
264264
-DWarpX_QED=ON \
265265
-DWarpX_QED_TABLE_GEN=ON \
266266
-DWarpX_OPENPMD=ON \

.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ Python/pywarpx/libwarpx*.so
44
d/
55
f/
66
o/
7-
build/
7+
build*/
88
tmp_build_dir/
99
test_dir
1010
test_dir/

CMakeLists.txt

+78-12
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Preamble ####################################################################
22
#
33
cmake_minimum_required(VERSION 3.20.0)
4-
project(WarpX VERSION 24.05)
4+
project(WarpX VERSION 24.07)
55

66
include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake)
77

@@ -16,6 +16,12 @@ endif()
1616

1717
# CMake policies ##############################################################
1818
#
19+
# Setting a cmake_policy to OLD is deprecated by definition and will raise a
20+
# verbose warning
21+
if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR)
22+
set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "" FORCE)
23+
endif()
24+
1925
# AMReX 21.06+ supports CUDA_ARCHITECTURES with CMake 3.20+
2026
# CMake 3.18+: CMAKE_CUDA_ARCHITECTURES
2127
# https://cmake.org/cmake/help/latest/policy/CMP0104.html
@@ -72,7 +78,8 @@ option(WarpX_EB "Embedded boundary support" OFF)
7278
option(WarpX_LIB "Build WarpX as a library" OFF)
7379
option(WarpX_MPI "Multi-node support (message-passing)" ON)
7480
option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON)
75-
option(WarpX_PSATD "spectral solver support" OFF)
81+
option(WarpX_FFT "FFT-based solvers" OFF)
82+
option(WarpX_HEFFTE "Multi-node FFT-based solvers" OFF)
7683
option(WarpX_PYTHON "Python bindings" OFF)
7784
option(WarpX_SENSEI "SENSEI in situ diagnostics" OFF)
7885
option(WarpX_QED "QED support (requires PICSAR)" ON)
@@ -130,15 +137,28 @@ mark_as_advanced(WarpX_MPI_THREAD_MULTIPLE)
130137

131138
option(WarpX_amrex_internal "Download & build AMReX" ON)
132139

140+
if(WarpX_HEFFTE AND NOT WarpX_MPI)
141+
message(FATAL_ERROR "WarpX_HEFFTE (${WarpX_HEFFTE}) can only be used if WarpX_MPI is ON.")
142+
endif()
143+
133144
# change the default build type to Release (or RelWithDebInfo) instead of Debug
134145
set_default_build_type("Release")
135146

136147
# Option to enable interprocedural optimization
137148
# (also know as "link-time optimization" or "whole program optimization")
138-
option(WarpX_IPO "Compile WarpX with interprocedural optimization (will take more time)" OFF)
149+
set(_WarpX_IPO_DEFAULT OFF)
150+
set(_WarpX_PYTHON_IPO_DEFAULT ON)
151+
if(DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION)
152+
set(_WarpX_IPO_DEFAULT ${CMAKE_INTERPROCEDURAL_OPTIMIZATION})
153+
set(_WarpX_PYTHON_IPO_DEFAULT ${CMAKE_INTERPROCEDURAL_OPTIMIZATION})
154+
endif()
155+
option(WarpX_IPO
156+
"Compile WarpX with interprocedural optimization (will take more time)"
157+
${_WarpX_IPO_DEFAULT}
158+
)
139159
option(WarpX_PYTHON_IPO
140160
"Compile Python bindings with interprocedural optimization (IPO) / link-time optimization (LTO)"
141-
ON
161+
${_WarpX_PYTHON_IPO_DEFAULT}
142162
)
143163

144164
set(pyWarpX_VERSION_INFO "" CACHE STRING
@@ -149,16 +169,28 @@ if(WarpX_APP OR WarpX_PYTHON)
149169
set(WarpX_LIB ON CACHE STRING "Build WarpX as a library" FORCE)
150170
endif()
151171

172+
# deprecated options: transition phase
173+
174+
if(DEFINED WarpX_PSATD)
175+
message(WARNING "CMake option WarpX_PSATD is deprecated. Use WarpX_FFT instead.\n"
176+
"Overwriting WarpX_FFT with '${WarpX_PSATD}' because WarpX_PSATD was set.")
177+
set(WarpX_FFT ${WarpX_PSATD} CACHE STRING "FFT-based solvers" FORCE)
178+
endif()
179+
152180
# note: we could skip this if we solely build WarpX_APP, but if we build a
153181
# shared WarpX library or a third party, like ImpactX, uses ablastr in a
154182
# shared library (e.g., for Python bindings), then we need relocatable code.
155183
option(ABLASTR_POSITION_INDEPENDENT_CODE
156184
"Build ABLASTR with position independent code" ON)
157185
mark_as_advanced(ABLASTR_POSITION_INDEPENDENT_CODE)
158186

159-
option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_PSATD})
160-
if(WarpX_PSATD)
161-
set(ABLASTR_FFT ON CACHE STRING "compile AnyFFT wrappers" FORCE)
187+
option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_FFT})
188+
if(WarpX_FFT)
189+
set(ABLASTR_FFT ON CACHE STRING "FFT-based solvers" FORCE)
190+
endif()
191+
option(ABLASTR_HEFFTE "compile AnyFFT wrappers" ${WarpX_HEFFTE})
192+
if(WarpX_HEFFTE)
193+
set(ABLASTR_HEFFTE ON CACHE STRING "Multi-Node FFT-based solvers" FORCE)
162194
endif()
163195

164196
# this defined the variable BUILD_TESTING which is ON by default
@@ -192,7 +224,7 @@ include(${WarpX_SOURCE_DIR}/cmake/dependencies/openPMD.cmake)
192224

193225
# PSATD
194226
include(${WarpX_SOURCE_DIR}/cmake/dependencies/FFT.cmake)
195-
if(WarpX_PSATD)
227+
if(WarpX_FFT)
196228
# BLASPP and LAPACKPP
197229
if(RZ IN_LIST WarpX_DIMS)
198230
find_package(blaspp CONFIG REQUIRED)
@@ -201,6 +233,23 @@ if(WarpX_PSATD)
201233
endif()
202234
endif()
203235

236+
# multi-node FFT
237+
if(WarpX_HEFFTE)
238+
if(WarpX_COMPUTE STREQUAL CUDA)
239+
set(_heFFTe_COMPS CUDA)
240+
elseif(WarpX_COMPUTE STREQUAL HIP)
241+
set(_heFFTe_COMPS ROCM)
242+
elseif(WarpX_COMPUTE STREQUAL SYCL)
243+
set(_heFFTe_COMPS ONEAPI)
244+
else() # NOACC, OMP
245+
set(_heFFTe_COMPS FFTW) # or MKL
246+
endif()
247+
# note: we could also enforce GPUAWARE for CUDA and HIP, which can still be
248+
# disabled at runtime
249+
250+
find_package(Heffte REQUIRED COMPONENTS ${_heFFTe_COMPS})
251+
endif()
252+
204253
# Python
205254
if(WarpX_PYTHON)
206255
find_package(Python COMPONENTS Interpreter Development.Module REQUIRED)
@@ -415,7 +464,7 @@ endif()
415464

416465
# Interprocedural optimization (IPO) / Link-Time Optimization (LTO)
417466
if(WarpX_IPO)
418-
enable_IPO("${_ALL_TARGETS}")
467+
warpx_enable_IPO("${_ALL_TARGETS}")
419468
endif()
420469

421470
# link dependencies
@@ -444,10 +493,20 @@ foreach(D IN LISTS WarpX_DIMS)
444493
endif()
445494
endif()
446495

496+
if(ABLASTR_HEFFTE)
497+
target_link_libraries(ablastr_${SD} PUBLIC Heffte::Heffte)
498+
endif()
499+
447500
if(WarpX_PYTHON)
448501
target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::module pybind11::windows_extras)
449502
if(WarpX_PYTHON_IPO)
450-
target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::lto)
503+
if(DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION)
504+
warpx_enable_IPO(pyWarpX_${SD})
505+
else()
506+
# conditionally defined target in pybind11
507+
# https://github.com/pybind/pybind11/blob/v2.12.0/tools/pybind11Common.cmake#L397-L403
508+
target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::lto)
509+
endif()
451510
endif()
452511
endif()
453512

@@ -520,14 +579,21 @@ foreach(D IN LISTS WarpX_DIMS)
520579
endif()
521580
endif()
522581

523-
if(WarpX_PSATD)
524-
target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_PSATD)
582+
if(WarpX_FFT)
583+
target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_FFT)
525584
endif()
526585
if(ABLASTR_FFT)
527586
# We need to enable FFT support in ABLASTR for PSATD solver
528587
target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_FFT)
529588
endif()
530589

590+
if(WarpX_HEFFTE)
591+
target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_HEFFTE)
592+
endif()
593+
if(ABLASTR_HEFFTE)
594+
target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_HEFFTE)
595+
endif()
596+
531597
if(WarpX_PYTHON AND pyWarpX_VERSION_INFO)
532598
# for module __version__
533599
target_compile_definitions(pyWarpX_${SD} PRIVATE

Docs/Doxyfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -2301,7 +2301,7 @@ PREDEFINED = AMREX_Linux=1 \
23012301
WARPX_DIM_XZ=1 \
23022302
WARPX_USE_GPU=1 \
23032303
WARPX_USE_OPENPMD=1 \
2304-
WARPX_USE_PSATD=1 \
2304+
WARPX_USE_FFT=1 \
23052305
WARPX_QED=1 \
23062306
WARPX_QED_TABLE_GEN=1
23072307

Docs/requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ openpmd-viewer # for checksumAPI
1313

1414
# PICMI API docs
1515
# note: keep in sync with version in ../requirements.txt
16-
picmistandard==0.28.0
16+
picmistandard==0.29.0
1717
# for development against an unreleased PICMI version, use:
1818
# picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python
1919

Docs/source/acknowledge_us.rst

+4-4
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,14 @@ Please add the following sentence to your publications, it helps contributors ke
2323

2424
**Plain text:**
2525

26-
This research used the open-source particle-in-cell code WarpX https://github.com/ECP-WarpX/WarpX, primarily funded by the US DOE Exascale Computing Project. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors.
26+
This research used the open-source particle-in-cell code WarpX https://github.com/ECP-WarpX/WarpX. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors.
2727

2828
**LaTeX:**
2929

3030
.. code-block:: latex
3131

3232
\usepackage{hyperref}
33-
This research used the open-source particle-in-cell code WarpX \url{https://github.com/ECP-WarpX/WarpX}, primarily funded by the US DOE Exascale Computing Project.
33+
This research used the open-source particle-in-cell code WarpX \url{https://github.com/ECP-WarpX/WarpX}.
3434
Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies.
3535
We acknowledge all WarpX contributors.
3636

@@ -55,8 +55,8 @@ If your project uses a specific algorithm or component, please consider citing t
5555

5656
- Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A.
5757
**Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**.
58-
Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024.
59-
`preprint <http://arxiv.org/abs/2402.17248>__`
58+
Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024.
59+
`DOI:10.1145/3659914.3659937 <https://doi.org/10.1145/3659914.3659937>`__
6060

6161
- Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A.
6262
**Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**.

Docs/source/acknowledgements.rst

+7-2
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,18 @@
33
Funding and Acknowledgements
44
============================
55

6-
WarpX is supported by the Exascale Computing Project (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy organizations (Office of Science and the National Nuclear Security Administration) responsible for the planning and preparation of a capable exascale ecosystem, including software, applications, hardware, advanced system engineering, and early testbed platforms, in support of the nation's exascale computing imperative.
6+
WarpX is hosted by the High Performance Computing Foundation (HPSF).
7+
If your organization wants to help steer the evolution of the HPC software ecosystem, visit `hpsf.io <https://hpsf.io>`__ and consider joining!
78

89
WarpX is supported by the CAMPA collaboration, a project of the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research and Office of High Energy Physics, Scientific Discovery through Advanced Computing (SciDAC) program.
910

10-
ABLASTR seed development is supported by the Laboratory Directed Research and Development Program of Lawrence Berkeley National Laboratory under U.S. Department of Energy Contract No. DE-AC02-05CH11231.
11+
WarpX is supported by the KISMET project, a project of the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research and Office of Fusion Energy Science, Scientific Discovery through Advanced Computing (SciDAC) program.
1112

1213
CEA-LIDYL actively contributes to the co-development of WarpX.
1314
As part of this initiative, WarpX also receives funding from the French National Research Agency (ANR - Plasm-On-Chip), the Horizon H2020 program and CEA.
1415

16+
WarpX was supported by the Exascale Computing Project (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy organizations (Office of Science and the National Nuclear Security Administration) responsible for the planning and preparation of a capable exascale ecosystem, including software, applications, hardware, advanced system engineering, and early testbed platforms, in support of the nation's exascale computing imperative.
17+
18+
ABLASTR seed development was supported by the Laboratory Directed Research and Development Program of Lawrence Berkeley National Laboratory under U.S. Department of Energy Contract No. DE-AC02-05CH11231.
19+
1520
We acknowledge all the contributors and users of the WarpX community who participate to the code quality with valuable code improvement and important feedback.

Docs/source/conf.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs):
103103
# built documents.
104104
#
105105
# The short X.Y version.
106-
version = u'24.05'
106+
version = u'24.07'
107107
# The full version, including alpha/beta/rc tags.
108-
release = u'24.05'
108+
release = u'24.07'
109109

110110
# The language for content autogenerated by Sphinx. Refer to documentation
111111
# for a list of supported languages.

0 commit comments

Comments
 (0)