Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify overwrite_amrex_parser_defaults WarpXAMReXInit.cpp #4965

Open
wants to merge 6 commits into
base: development
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 64 additions & 37 deletions Source/Initialization/WarpXAMReXInit.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,66 +17,93 @@
#include <string>

namespace {
/** Overwrite defaults in AMReX Inputs
*
* This overwrites defaults in amrex::ParmParse for inputs.
*/
void
overwrite_amrex_parser_defaults ()
{
amrex::ParmParse pp_amrex("amrex");

#ifdef AMREX_USE_GPU
constexpr auto amrex_use_gpu = true;
#else
constexpr auto amrex_use_gpu = false;
#endif

void override_default_abort_on_out_of_gpu_memory ()
{
// https://amrex-codes.github.io/amrex/docs_html/GPU.html#inputs-parameters
bool abort_on_out_of_gpu_memory = true; // AMReX' default: false
auto pp_amrex = amrex::ParmParse{"amrex"};
bool abort_on_out_of_gpu_memory = true; // AMReX's default: false
pp_amrex.queryAdd("abort_on_out_of_gpu_memory", abort_on_out_of_gpu_memory);
}

bool the_arena_is_managed = false; // AMReX' default: true
void override_default_the_arena_is_managed ()
{
auto pp_amrex = amrex::ParmParse{"amrex"};
bool the_arena_is_managed = false; // AMReX's default: true
pp_amrex.queryAdd("the_arena_is_managed", the_arena_is_managed);
}

void override_default_omp_threads ()
{
// https://amrex-codes.github.io/amrex/docs_html/InputsComputeBackends.html
std::string omp_threads = "nosmt"; // AMReX' default: system
auto pp_amrex = amrex::ParmParse{"amrex"};
std::string omp_threads = "nosmt"; // AMReX's default: system
pp_amrex.queryAdd("omp_threads", omp_threads);
}

void set_device_synchronization ()
{
//See https://github.com/AMReX-Codes/amrex/pull/3763
auto warpx_do_device_synchronize = amrex_use_gpu;

auto pp_warpx = amrex::ParmParse{"warpx"};
pp_warpx.query("do_device_synchronize", warpx_do_device_synchronize);
bool do_device_synchronize = warpx_do_device_synchronize;

auto pp_tiny_profiler = amrex::ParmParse{"tiny_profiler"};
if (pp_tiny_profiler.queryAdd("device_synchronize_around_region", do_device_synchronize) )
{
WARPX_ALWAYS_ASSERT_WITH_MESSAGE(
do_device_synchronize == warpx_do_device_synchronize,
"tiny_profiler.device_synchronize_around_region overrides warpx.do_device_synchronize.");
}

}

void apply_workaround_for_warpx_numprocs ()
{
// Work-around:
// If warpx.numprocs is used for the domain decomposition, we will not use blocking factor
// to generate grids. Nonetheless, AMReX has asserts in place that validate that the
// number of cells is a multiple of blocking factor. We set the blocking factor to 1 so those
// AMReX asserts will always pass.
const amrex::ParmParse pp_warpx("warpx");
const auto pp_warpx = amrex::ParmParse{"warpx"};
if (pp_warpx.contains("numprocs"))
{
amrex::ParmParse pp_amr("amr");
pp_amr.add("blocking_factor", 1);
}
}

//See https://github.com/AMReX-Codes/amrex/pull/3763
#ifdef AMREX_USE_GPU
bool warpx_do_device_synchronize = true;
#else
bool warpx_do_device_synchronize = false;
#endif
pp_warpx.query("do_device_synchronize", warpx_do_device_synchronize);
bool do_device_synchronize = warpx_do_device_synchronize;
amrex::ParmParse pp_tiny_profiler("tiny_profiler");
if (pp_tiny_profiler.queryAdd("device_synchronize_around_region", do_device_synchronize) )
{
WARPX_ALWAYS_ASSERT_WITH_MESSAGE(
do_device_synchronize == warpx_do_device_synchronize,
"tiny_profiler.device_synchronize_around_region overrides warpx.do_device_synchronize.");
}

void override_default_tiling_option_for_particles ()
{
// Here we override the default tiling option for particles, which is always
// "false" in AMReX, to "false" if compiling for GPU execution and "true"
// if compiling for CPU.
{
amrex::ParmParse pp_particles("particles");
#ifdef AMREX_USE_GPU
bool do_tiling = false; // By default, tiling is off on GPU
#else
bool do_tiling = true;
#endif
pp_particles.queryAdd("do_tiling", do_tiling);
}
auto pp_particles = amrex::ParmParse{"particles"};
auto do_tiling = !amrex_use_gpu; // By default, tiling is off on GPU
pp_particles.queryAdd("do_tiling", do_tiling);
}

/** Overwrite defaults in AMReX Inputs
*
* This overwrites defaults in amrex::ParmParse for inputs.
*/
void
overwrite_amrex_parser_defaults ()
{
override_default_abort_on_out_of_gpu_memory();
override_default_the_arena_is_managed();
override_default_omp_threads();
apply_workaround_for_warpx_numprocs();
set_device_synchronization();
override_default_tiling_option_for_particles();
}
}

Expand Down
Loading