Add checks for parthenon/mesh/multigrid = true when using multigrid block lists

CHANGELOG.md

@@ -32,6 +32,7 @@
 - [[PR 1172]](https://github.com/parthenon-hpc-lab/parthenon/pull/1172) Make parthenon manager robust against external MPI init and finalize calls
 
 ### Fixed (not changing behavior/API/variables/...)
+- [[PR 1241]](https://github.com/parthenon-hpc-lab/parthenon/pull/1241) Emit an error message when using uninitialized multigrid blocks
 - [[PR 1248]](https://github.com/parthenon-hpc-lab/parthenon/pull/1248) Fix edge case regarding AMR de-refinement logic
 - [[PR 1240]](https://github.com/parthenon-hpc-lab/parthenon/pull/1240) Fix I/O for CellMemAligned variables
 - [[PR 1229]](https://github.com/parthenon-hpc-lab/parthenon/pull/1229) Ensure builds function on 32 bit architectures

doc/sphinx/src/mesh/mesh.rst

@@ -62,7 +62,7 @@ Multi-grid Grids Stored in ``Mesh``
 If the parameter ``parthenon/mesh/multigrid`` is set to ``true``, the ``Mesh``
 constructor and AMR routines populate both 
 ``std::vector<LogicalLocMap_t> Mesh::gmg_grid_locs`` and 
-``std::vector<BlockList_t> gmg_block_lists``, where each entry into the vectors 
+``std::vector<BlockList_t> gmg_block_lists_``, where each entry into the vectors 
 describes one level of the of the geometric multi-grid (GMG) mesh. For refined 
 meshes, each GMG level only includes blocks that are at a given logical level 
 (starting from the finest logical level on the grid and including both internal 

src/mesh/mesh-gmg.cpp

@@ -117,18 +117,18 @@ void Mesh::BuildGMGBlockLists(ParameterInput *pin, ApplicationInput *app_in) {
   const int gmg_min_level = -gmg_level_offset;
   gmg_min_logical_level_ = gmg_min_level;
   for (int level = gmg_min_level; level <= current_level; ++level) {
-    gmg_block_lists[level] = BlockList_t();
+    gmg_block_lists_[level] = BlockList_t();
   }
 
   // Fill/create gmg block lists based on this ranks block list
   for (auto &pmb : block_list) {
     const int level = pmb->loc.level();
     // Add the leaf block to its level
-    gmg_block_lists[level].push_back(pmb);
+    gmg_block_lists_[level].push_back(pmb);
 
     // Add the leaf block to the next finer level if required
     if (level < current_level) {
-      gmg_block_lists[level + 1].push_back(pmb);
+      gmg_block_lists_[level + 1].push_back(pmb);
     }
 
     // Create internal blocks that share a Morton number with this block
@@ -139,15 +139,15 @@ void Mesh::BuildGMGBlockLists(ParameterInput *pin, ApplicationInput *app_in) {
       RegionSize block_size = GetDefaultBlockSize();
       BoundaryFlag block_bcs[6];
       SetBlockSizeAndBoundaries(loc, block_size, block_bcs);
-      gmg_block_lists[loc.level()].push_back(
+      gmg_block_lists_[loc.level()].push_back(
           MeshBlock::Make(forest.GetGid(loc), -1, loc, block_size, block_bcs, this, pin,
                           app_in, packages, resolved_packages, gflag));
       loc = loc.GetParent();
     }
   }
 
   // Sort the gmg block lists by gid
-  for (auto &[level, bl] : gmg_block_lists) {
+  for (auto &[level, bl] : gmg_block_lists_) {
     std::sort(bl.begin(), bl.end(), [](auto &a, auto &b) { return a->gid < b->gid; });
     BuildBlockPartitions(GridIdentifier::two_level_composite(level));
   }
@@ -157,7 +157,7 @@ void Mesh::SetGMGNeighbors() {
   if (!multigrid) return;
   const int gmg_min_level = GetGMGMinLevel();
   // Sort the gmg block lists by gid and find neighbors
-  for (auto &[level, bl] : gmg_block_lists) {
+  for (auto &[level, bl] : gmg_block_lists_) {
     for (auto &pmb : bl) {
       // Coarser neighbor
       pmb->gmg_coarser_neighbors.clear();

src/mesh/mesh.cpp

@@ -390,7 +390,7 @@ Mesh::~Mesh() {
 
 void Mesh::BuildBlockPartitions(GridIdentifier grid) {
   auto partition_blocklists = partition::ToSizeN(
-      grid.type == GridType::leaf ? block_list : gmg_block_lists[grid.logical_level],
+      grid.type == GridType::leaf ? block_list : gmg_block_lists_[grid.logical_level],
       DefaultPackSize());
   std::vector<std::shared_ptr<BlockListPartition>> out;
   int id = 0;

src/mesh/mesh.hpp

@@ -138,7 +138,13 @@ class Mesh {
 
   DataCollection<MeshData<Real>> mesh_data;
 
-  std::map<int, BlockList_t> gmg_block_lists;
+  const BlockList_t &GetGMGBlockList(int level) const {
+    PARTHENON_REQUIRE(multigrid, "Asking for multigrid blocks on a Mesh that was created "
+                                 "without parthenon/mesh/multigrid = true set.");
+    PARTHENON_REQUIRE(gmg_block_lists_.count(level),
+                      "Asking for a multigrid level that doesn't exist.");
+    return gmg_block_lists_.at(level);
+  }
   int GetGMGMaxLevel() const { return current_level; }
   int GetGMGMinLevel() const { return gmg_min_logical_level_; }
 
@@ -160,6 +166,12 @@ class Mesh {
 
   const std::vector<std::shared_ptr<BlockListPartition>> &
   GetDefaultBlockPartitions(GridIdentifier grid = GridIdentifier::leaf()) const {
+    if (grid.type == GridType::two_level_composite)
+      PARTHENON_REQUIRE(multigrid, "Asking for a partition of a multigrid grid when "
+                                   "parthenon/mesh/multigrid = false.")
+    PARTHENON_REQUIRE(
+        block_partitions_.count(grid),
+        "There isn't a block partition available for this grid for some reason.");
     return block_partitions_.at(grid);
   }
 
@@ -288,6 +300,9 @@ class Mesh {
 
   std::vector<LogicalLocation> loclist;
 
+  // Block lists for internal nodes in the tree corresponding to multigrid levels
+  std::map<int, BlockList_t> gmg_block_lists_;
+
   // flags are false if using non-uniform or user meshgen function
   bool use_uniform_meshgen_fn_[4];