Add hook for UserWorkBeforeLoop

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Current develop
 
 ### Added (new features/APIs/variables/...)
+- [[PR 971]](https://github.com/parthenon-hpc-lab/parthenon/pull/971) Add UserWorkBeforeLoop
 - [[PR 907]](https://github.com/parthenon-hpc-lab/parthenon/pull/907) PEP1: Allow subclassing StateDescriptor
 - [[PR 932]](https://github.com/parthenon-hpc-lab/parthenon/pull/932) Add GetOrAddFlag to metadata
 - [[PR 931]](https://github.com/parthenon-hpc-lab/parthenon/pull/931) Allow SparsePacks with subsets of blocks

doc/sphinx/src/interface/state.rst

@@ -112,6 +112,13 @@ several useful features and functions.
   deletgates to the ``std::function`` member ``PostStepDiagnosticsMesh``
   if set (defaults to ``nullptr`` an therefore a no-op) to print
   diagnostics after the time-integration advance
+- ``void UserWorkBeforeLoopMesh(Mesh *, ParameterInput *pin, SimTime
+  &tm)`` performs a per-package, mesh-wide calculation after the mesh
+  has been generated, and problem generators called, but before any
+  time evolution. This work is done both on first initialization and
+  on restart. If you would like to avoid doing the work upon restart,
+  you can check for the const ``is_restart`` member field of the ``Mesh``
+  object.
 
 The reasoning for providing ``FillDerived*`` and ``EstimateTimestep*``
 function pointers appropriate for usage with both ``MeshData`` and

example/advection/advection_package.cpp

@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020-2021. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -13,12 +13,14 @@
 
 #include <algorithm>
 #include <cmath>
+#include <iostream>
 #include <limits>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include <coordinates/coordinates.hpp>
+#include <globals.hpp>
 #include <parthenon/package.hpp>
 
 #include "advection_package.hpp"
@@ -215,10 +217,19 @@ std::shared_ptr<StateDescriptor> Initialize(ParameterInput *pin) {
   }
   pkg->CheckRefinementBlock = CheckRefinement;
   pkg->EstimateTimestepBlock = EstimateTimestepBlock;
+  pkg->UserWorkBeforeLoopMesh = AdvectionGreetings;
 
   return pkg;
 }
 
+void AdvectionGreetings(Mesh *pmesh, ParameterInput *pin, parthenon::SimTime &tm) {
+  if (parthenon::Globals::my_rank == 0) {
+    std::cout << "Hello from the advection package in the advection example!\n"
+              << "This run is a restart: " << pmesh->is_restart << "\n"
+              << std::endl;
+  }
+}
+
 AmrTag CheckRefinement(MeshBlockData<Real> *rc) {
   // refine on advected, for example.  could also be a derived quantity
   auto pmb = rc->GetBlockPointer();

example/advection/advection_package.hpp

@@ -1,5 +1,5 @@
 //========================================================================================
-// (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved.
+// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001 for Los
 // Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC
@@ -21,6 +21,7 @@ namespace advection_package {
 using namespace parthenon::package::prelude;
 
 std::shared_ptr<StateDescriptor> Initialize(ParameterInput *pin);
+void AdvectionGreetings(Mesh *pmes, ParameterInput *pin, parthenon::SimTime &tm);
 AmrTag CheckRefinement(MeshBlockData<Real> *rc);
 void PreFill(MeshBlockData<Real> *rc);
 void SquareIt(MeshBlockData<Real> *rc);

src/application_input.hpp

@@ -48,6 +48,7 @@ struct ApplicationInput {
       PostStepDiagnosticsInLoop = nullptr;
 
   std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkAfterLoop = nullptr;
+  std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkBeforeLoop = nullptr;
   BValFunc boundary_conditions[BOUNDARY_NFACES] = {nullptr};
   SBValFunc swarm_boundary_conditions[BOUNDARY_NFACES] = {nullptr};
 

src/driver/driver.cpp

@@ -75,6 +75,16 @@ DriverStatus EvolutionDriver::Execute() {
   // Defaults must be set across all ranks
   DumpInputParameters();
 
+  // Before loop do work
+  // App input version
+  if (app_input->UserWorkBeforeLoop != nullptr) {
+    app_input->UserWorkBeforeLoop(pmesh, pinput, tm);
+  }
+  // packages version
+  for (auto &[name, pkg] : pmesh->packages.AllPackages()) {
+    pkg->UserWorkBeforeLoop(pmesh, pinput, tm);
+  }
+
   Kokkos::Profiling::pushRegion("Driver_Main");
   while (tm.KeepGoing()) {
     if (Globals::my_rank == 0) OutputCycleDiagnostics();

src/interface/state_descriptor.hpp

@@ -406,6 +406,10 @@ class StateDescriptor {
     if (InitNewlyAllocatedVarsBlock != nullptr) return InitNewlyAllocatedVarsBlock(rc);
   }
 
+  void UserWorkBeforeLoop(Mesh *pmesh, ParameterInput *pin, SimTime &tm) const {
+    if (UserWorkBeforeLoopMesh != nullptr) return UserWorkBeforeLoopMesh(pmesh, pin, tm);
+  }
+
   std::vector<std::shared_ptr<AMRCriteria>> amr_criteria;
 
   std::function<void(MeshBlockData<Real> *rc)> PreCommFillDerivedBlock = nullptr;
@@ -416,6 +420,8 @@ class StateDescriptor {
   std::function<void(MeshData<Real> *rc)> PostFillDerivedMesh = nullptr;
   std::function<void(MeshBlockData<Real> *rc)> FillDerivedBlock = nullptr;
   std::function<void(MeshData<Real> *rc)> FillDerivedMesh = nullptr;
+  std::function<void(Mesh *, ParameterInput *, SimTime &)> UserWorkBeforeLoopMesh =
+      nullptr;
 
   std::function<void(SimTime const &simtime, MeshData<Real> *rc)> PreStepDiagnosticsMesh =
       nullptr;

src/mesh/mesh.cpp

@@ -63,7 +63,7 @@ namespace parthenon {
 Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, Packages_t &packages,
            int mesh_test)
     : // public members:
-      modified(true),
+      modified(true), is_restart(false),
       // aggregate initialization of RegionSize struct:
       mesh_size({pin->GetReal("parthenon/mesh", "x1min"),
                  pin->GetReal("parthenon/mesh", "x2min"),
@@ -484,7 +484,7 @@ Mesh::Mesh(ParameterInput *pin, ApplicationInput *app_in, RestartReader &rr,
     : // public members:
       // aggregate initialization of RegionSize struct:
       // (will be overwritten by memcpy from restart file, in this case)
-      modified(true),
+      modified(true), is_restart(true),
       // aggregate initialization of RegionSize struct:
       mesh_size({pin->GetReal("parthenon/mesh", "x1min"),
                  pin->GetReal("parthenon/mesh", "x2min"),

src/mesh/mesh.hpp

@@ -94,6 +94,7 @@ class Mesh {
 
   // data
   bool modified;
+  const bool is_restart;
   RegionSize mesh_size;
   BoundaryFlag mesh_bcs[BOUNDARY_NFACES];
   const int ndim; // number of dimensions