Merge pull request #902 from parthenon-hpc-lab/jmm/sparse-is-nan

Add sparse seed nans flag to output

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Current develop
 
 ### Added (new features/APIs/variables/...)
+- [[PR 902]](https://github.com/parthenon-hpc-lab/parthenon/pull/902) Add ability to output NaNs for de-allocated sparse fields
 - [[PR 887]](https://github.com/parthenon-hpc-lab/parthenon/pull/887) Add ability to dump more types of params and read them from restarts
 - [[PR 884]](https://github.com/parthenon-hpc-lab/parthenon/pull/884) Add constant derivative BC and expose GenericBC
 - [[PR 892]](https://github.com/parthenon-hpc-lab/parthenon/pull/892) Cost-based load balancing and memory diagnostics

doc/sphinx/src/outputs.rst

@@ -59,6 +59,13 @@ look like
    file_number_width = 6 # default: 5
    use_final_label = true # default: true
 
+   # Sparse variables may not be allocated on every block. By default
+   # parthenon outputs de-allocated variables as 0 in the output
+   # file. However, it is often convenient to output them as NaN
+   # instead, marking deallocated and allocated but zero as
+   # separate. This flag turns this functionality on.
+   sparse_seed_nans = false # default false
+
 This will produce an hdf5 (``.phdf``) output file every 1 units of
 simulation time containing the density, velocity, and energy of each
 cell. The files will be identified by a 6-digit ID, and the output file

src/outputs/outputs.cpp

@@ -152,8 +152,11 @@ Outputs::Outputs(Mesh *pm, ParameterInput *pin, SimTime *tm) {
       if (is_hdf5_output) {
         op.single_precision_output =
             pin->GetOrAddBoolean(op.block_name, "single_precision_output", false);
+        op.sparse_seed_nans =
+            pin->GetOrAddBoolean(op.block_name, "sparse_seed_nans", false);
       } else {
         op.single_precision_output = false;
+        op.sparse_seed_nans = false;
 
         if (pin->DoesParameterExist(op.block_name, "single_precision_output")) {
           std::stringstream warn;

src/outputs/outputs.hpp

@@ -59,12 +59,14 @@ struct OutputParameters {
   int file_number;
   bool include_ghost_zones, cartesian_vector;
   bool single_precision_output;
+  bool sparse_seed_nans;
   int hdf5_compression_level;
   // TODO(felker): some of the parameters in this class are not initialized in constructor
   OutputParameters()
       : block_number(0), next_time(0.0), dt(-1.0), file_number(0),
         include_ghost_zones(false), cartesian_vector(false),
-        single_precision_output(false), hdf5_compression_level(5) {}
+        single_precision_output(false), sparse_seed_nans(false),
+        hdf5_compression_level(5) {}
 };
 
 //----------------------------------------------------------------------------------------

src/outputs/parthenon_hdf5.cpp

@@ -538,7 +538,9 @@ void PHDF5Output::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, SimTime *tm
             varSize =
                 vinfo.nx6 * vinfo.nx5 * vinfo.nx4 * vinfo.nx3 * vinfo.nx2 * vinfo.nx1;
           }
-          memset(tmpData.data() + index, 0, varSize * sizeof(OutT));
+          auto fill_val =
+              output_params.sparse_seed_nans ? std::numeric_limits<OutT>::quiet_NaN() : 0;
+          std::fill(tmpData.data() + index, tmpData.data() + index + varSize, fill_val);
           index += varSize;
         } else {
           std::stringstream msg;