From 5ca50df5127202852ad65b0a6f83c2dc355618c4 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Fri, 19 Nov 2021 12:43:23 +0000
Subject: [PATCH 01/63] Add WIP CatalystAdaptor class

---
 Source/Insitu/CatalystAdaptor.cpp | 41 ++++++++++++++++++
 Source/Insitu/CatalystAdaptor.hpp | 69 +++++++++++++++++++++++++++++++
 2 files changed, 110 insertions(+)
 create mode 100644 Source/Insitu/CatalystAdaptor.cpp
 create mode 100644 Source/Insitu/CatalystAdaptor.hpp

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
new file mode 100644
index 000000000..3cb0193e1
--- /dev/null
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -0,0 +1,41 @@
+/* GRChombo
+ * Copyright 2012 The GRChombo collaboration.
+ * Please refer to LICENSE in GRChombo's root directory.
+ */
+
+#include "CatalystAdaptor.hpp"
+
+#ifdef USE_CATALYST
+
+CatalystAdaptor::CatalystAdaptor() {}
+
+CatalystAdaptor::CatalystAdaptor(const GRAMR *a_gr_amr_ptr,
+                                 std::string a_python_script_path)
+{
+    initialise(a_gr_amr_ptr, a_python_script_path);
+}
+
+CatalystAdaptor::~CatalystAdaptor() {}
+
+void CatalystAdaptor::initialise(const GRAMR *a_gr_amr_ptr,
+                                 std::string a_python_script_path)
+{
+    if (!a_gr_amr_ptr)
+    {
+        std::cerr << "CatalystAdaptor::initialise: failed to initalise due to"
+                     " invalid GRAMR pointer"
+                  << std::endl;
+        m_initialised = false;
+        return;
+    }
+    m_gr_amr_ptr = a_gr_amr_ptr;
+}
+void CatalystAdaptor::finalise();
+
+void CatalystAdaptor::update_vtk_grid();
+
+void CatalystAdaptor::add_var();
+
+void CatalystAdaptor::coprocess(double time);
+
+#endif /* USE_CATALYST */
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
new file mode 100644
index 000000000..5c081f7ff
--- /dev/null
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -0,0 +1,69 @@
+/* GRChombo
+ * Copyright 2012 The GRChombo collaboration.
+ * Please refer to LICENSE in GRChombo's root directory.
+ */
+
+#ifndef CATALYSTADAPTOR_HPP_
+#define CATALYSTADAPTOR_HPP_
+
+#ifdef USE_CATALYST
+
+// Standard library includes
+#include <string>
+
+// GRChombo includes
+#include "GRAMR.hpp"
+#include "UserVariables.hpp"
+
+// ParaView/VTK/Catalyst includes
+#include <vtkAMRBox.h>
+#include <vtkCPDataDescription.h>
+#include <vtkCPInputDataDescription.h>
+#include <vtkCPProcessor.h>
+#include <vtkCPPythonScriptPipeline.h>
+#include <vtkCellData.h>
+#include <vtkDoubleArray.h>
+#include <vtkNew.h>
+#include <vtkOverlappingAMR.h>
+#include <vtkUniformGrid.h>
+
+// Class that interfaces with ParaView Catalyst for insitu visualisation
+class CatalystAdaptor
+{
+  public:
+    // empty constructor (doesn't call initialise)
+    CatalystAdaptor();
+
+    // full constructor (calls initialise)
+    CatalystAdaptor(const GRAMR *a_gr_amr_ptr,
+                    std::string a_python_script_path);
+
+    // destructor
+    ~CatalystAdaptor();
+
+    // Initialisation/Finalisation
+    void initialise(const GRAMR *m_gr_amr_ptr,
+                    std::string a_python_script_path);
+    void finalise();
+
+    // update the AMR grid (no grid data)
+    void update_vtk_grid();
+
+    // send variables to catalyst
+    void add_var();
+
+    // do Catalyst processing
+    void coprocess(double time);
+
+  private:
+    bool m_initialised = false;
+    bool m_finalised = false;
+    const GRAMR *m_gr_amr_ptr = nullptr;
+    std::string m_python_script_path;
+
+    vtkCPProcessor *m_proc_ptr = nullptr;
+    vtkOverlappingAMR *m_vtk_grid_ptr = nullptr;
+};
+
+#endif /* USE_CATALYST */
+#endif /* CATALYSTADAPTOR_HPP_ */

From 8bf656095b7c6eefc87b790013b881896439c905 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 23 Nov 2021 17:52:51 +0000
Subject: [PATCH 02/63] Add Make.insitu

This file allows GRChombo examples to be built with Catalyst. Also add
some more work on the CatalystAdaptor code.
---
 Examples/BinaryBH/GNUmakefile     |  4 ++
 Source/GRChomboCore/GRAMR.hpp     |  4 ++
 Source/Insitu/CatalystAdaptor.cpp | 61 ++++++++++++++++++++++++++++---
 Source/Insitu/CatalystAdaptor.hpp | 10 ++++-
 Source/Insitu/Make.insitu         | 29 +++++++++++++++
 5 files changed, 102 insertions(+), 6 deletions(-)
 create mode 100644 Source/Insitu/Make.insitu

diff --git a/Examples/BinaryBH/GNUmakefile b/Examples/BinaryBH/GNUmakefile
index fe043e5f7..8d2f2a9f8 100644
--- a/Examples/BinaryBH/GNUmakefile
+++ b/Examples/BinaryBH/GNUmakefile
@@ -41,4 +41,8 @@ ifeq ($(USE_TWOPUNCTURES),TRUE)
   src_dirs += $(TWOPUNCTURES_SOURCE)
 endif
 
+# ParaView Catalyst insitu visualisation
+# Set PARAVIEW_DIR environment variable to use
+include $(GRCHOMBO_SOURCE)/Insitu/Make.insitu
+
 include $(CHOMBO_HOME)/mk/Make.test
diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp
index 06f51614e..a675ea523 100644
--- a/Source/GRChomboCore/GRAMR.hpp
+++ b/Source/GRChomboCore/GRAMR.hpp
@@ -18,6 +18,10 @@
 #include <ratio>
 #include <vector>
 
+#ifdef USE_CATALYST
+#include "CatalystAdaptor.hpp"
+#endif
+
 // Chombo namespace
 #include "UsingNamespace.H"
 
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 3cb0193e1..ba35c83b6 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -15,11 +15,21 @@ CatalystAdaptor::CatalystAdaptor(const GRAMR *a_gr_amr_ptr,
     initialise(a_gr_amr_ptr, a_python_script_path);
 }
 
-CatalystAdaptor::~CatalystAdaptor() {}
+CatalystAdaptor::~CatalystAdaptor()
+{
+    if (m_initialised)
+    {
+        finalise();
+    }
+}
 
 void CatalystAdaptor::initialise(const GRAMR *a_gr_amr_ptr,
                                  std::string a_python_script_path)
 {
+    // don't initalise twice
+    if (m_initialised)
+        return;
+
     if (!a_gr_amr_ptr)
     {
         std::cerr << "CatalystAdaptor::initialise: failed to initalise due to"
@@ -29,13 +39,54 @@ void CatalystAdaptor::initialise(const GRAMR *a_gr_amr_ptr,
         return;
     }
     m_gr_amr_ptr = a_gr_amr_ptr;
+
+    // Initialise VTK CP Processor
+    if (!m_proc_ptr)
+    {
+        m_proc_ptr = vtkCPProcessor::New();
+        m_proc_ptr->Initialize();
+    }
+    else
+    {
+        m_proc_ptr->RemoveAllPipelines();
+    }
+
+    // Create Python script pipeline and add it to the VTK CP Processor
+    if (auto pipeline = vtkCPPythonScriptPipeline::CreateAndInitializePipeline(
+            a_python_script_path.c_str()))
+    {
+        m_proc_ptr->AddPipeline(pipeline);
+    }
+    else
+    {
+        std::string pipeline_fail_warning =
+            "Failed to set up pipeline for script:";
+        pipeline_fail_warning += a_python_script_path;
+        MayDay::Warning(pipeline_fail_warning.c_str());
+    }
+
+    m_initialised = true;
+}
+
+void CatalystAdaptor::finalise()
+{
+    if (m_proc_ptr)
+    {
+        m_proc_ptr->Delete();
+        m_proc_ptr = nullptr;
+    }
+    if (m_vtk_grid_ptr)
+    {
+        m_vtk_grid_ptr->Delete();
+        m_vtk_grid_ptr = nullptr;
+    }
+    m_initialised = false;
 }
-void CatalystAdaptor::finalise();
 
-void CatalystAdaptor::update_vtk_grid();
+void CatalystAdaptor::update_vtk_grid() {}
 
-void CatalystAdaptor::add_var();
+void CatalystAdaptor::add_var() {}
 
-void CatalystAdaptor::coprocess(double time);
+void CatalystAdaptor::coprocess(double time) {}
 
 #endif /* USE_CATALYST */
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index 5c081f7ff..fd5be40d5 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -11,6 +11,9 @@
 // Standard library includes
 #include <string>
 
+// Chombo includes
+#include "MayDay.H"
+
 // GRChombo includes
 #include "GRAMR.hpp"
 #include "UserVariables.hpp"
@@ -27,6 +30,12 @@
 #include <vtkOverlappingAMR.h>
 #include <vtkUniformGrid.h>
 
+// Chombo namespace
+#include "UsingNamespace.H"
+
+// Forward declaration of GRAMR class
+class GRAMR;
+
 // Class that interfaces with ParaView Catalyst for insitu visualisation
 class CatalystAdaptor
 {
@@ -57,7 +66,6 @@ class CatalystAdaptor
 
   private:
     bool m_initialised = false;
-    bool m_finalised = false;
     const GRAMR *m_gr_amr_ptr = nullptr;
     std::string m_python_script_path;
 
diff --git a/Source/Insitu/Make.insitu b/Source/Insitu/Make.insitu
new file mode 100644
index 000000000..0faa9e2d8
--- /dev/null
+++ b/Source/Insitu/Make.insitu
@@ -0,0 +1,29 @@
+# In order to build with ParaView Catalyst insitu visualisation, include this 
+# makefile before mk/Make.test is included in your example and after src_dirs
+# are set
+
+# Decide whether we want to build with ParaView Catalyst by checking whether 
+# PARAVIEW_DIR environment variable is set
+ifdef PARAVIEW_DIR
+ifneq (,$(wildcard $(PARAVIEW_DIR)/bin/paraview-config))
+  $(info ================== Using ParaView Catalyst ==================)
+  USE_CATALYST := TRUE
+else
+  $(error No paraview-config in \$\(PARAVIEW_DIR\)/bin/. Has ParaView been built with Catalyst?)
+endif
+endif
+
+ifeq ($(USE_CATALYST),TRUE)
+  src_dirs += $(GRCHOMBO_SOURCE)/Insitu
+  # This won't work if these make variables are overwritten in Make.defs.local
+  cxxcppflags += -DUSE_CATALYST
+  # These components are correct in v5.9 but may differ in other versions
+  # The paraview-config command may also change
+  PARAVIEW_COMPONENTS := PythonCatalyst
+  VTK_COMPONENTS := CommonDataModel PythonUsed
+  PARAVIEW_CONFIG_EXEC := $(PARAVIEW_DIR)/bin/paraview-config
+  # The Chombo mkdep script cannot cope with the space in "-isystem /include/path"
+  # so we have to remove it
+  cxxcppflags += $(subst -isystem ,-isystem,$(shell $(PARAVIEW_CONFIG_EXEC) --cppflags -c "$(PARAVIEW_COMPONENTS)" -v "$(VTK_COMPONENTS)"))
+  XTRALDFLAGS += $(shell $(PARAVIEW_CONFIG_EXEC) --ldflags -c "$(PARAVIEW_COMPONENTS)" -v "$(VTK_COMPONENTS)")
+endif

From 2bbabedd0475b74073d1d0c24e5244a318e7fc1c Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Thu, 25 Nov 2021 14:18:46 +0000
Subject: [PATCH 03/63] Remove code that strips spaces in isystem flag

This can be handled by the updated mkdep script in GRChombo/Chombo#32.
---
 Source/Insitu/Make.insitu | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Source/Insitu/Make.insitu b/Source/Insitu/Make.insitu
index 0faa9e2d8..e82c0d272 100644
--- a/Source/Insitu/Make.insitu
+++ b/Source/Insitu/Make.insitu
@@ -22,8 +22,6 @@ ifeq ($(USE_CATALYST),TRUE)
   PARAVIEW_COMPONENTS := PythonCatalyst
   VTK_COMPONENTS := CommonDataModel PythonUsed
   PARAVIEW_CONFIG_EXEC := $(PARAVIEW_DIR)/bin/paraview-config
-  # The Chombo mkdep script cannot cope with the space in "-isystem /include/path"
-  # so we have to remove it
-  cxxcppflags += $(subst -isystem ,-isystem,$(shell $(PARAVIEW_CONFIG_EXEC) --cppflags -c "$(PARAVIEW_COMPONENTS)" -v "$(VTK_COMPONENTS)"))
+  cxxcppflags += $(shell $(PARAVIEW_CONFIG_EXEC) --cppflags -c "$(PARAVIEW_COMPONENTS)" -v "$(VTK_COMPONENTS)")
   XTRALDFLAGS += $(shell $(PARAVIEW_CONFIG_EXEC) --ldflags -c "$(PARAVIEW_COMPONENTS)" -v "$(VTK_COMPONENTS)")
 endif

From 276c3d99c335ec60234136ebfb207d0ace96ee39 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 30 Nov 2021 11:56:44 +0000
Subject: [PATCH 04/63] Add script to cache output from paraview-config

paraview-config is extremely slow and we only want to call it once and
use the cached output otherwise.
---
 Source/Insitu/Make.insitu |  8 ++++----
 Source/Insitu/cache.sh    | 38 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 4 deletions(-)
 create mode 100755 Source/Insitu/cache.sh

diff --git a/Source/Insitu/Make.insitu b/Source/Insitu/Make.insitu
index e82c0d272..c8f32f299 100644
--- a/Source/Insitu/Make.insitu
+++ b/Source/Insitu/Make.insitu
@@ -19,9 +19,9 @@ ifeq ($(USE_CATALYST),TRUE)
   cxxcppflags += -DUSE_CATALYST
   # These components are correct in v5.9 but may differ in other versions
   # The paraview-config command may also change
-  PARAVIEW_COMPONENTS := PythonCatalyst
-  VTK_COMPONENTS := CommonDataModel PythonUsed
+  PARAVIEW_COMPONENT_FLAGS := -c PythonCatalyst
+  VTK_COMPONENT_FLAGS := -v CommonDataModel -v PythonUsed
   PARAVIEW_CONFIG_EXEC := $(PARAVIEW_DIR)/bin/paraview-config
-  cxxcppflags += $(shell $(PARAVIEW_CONFIG_EXEC) --cppflags -c "$(PARAVIEW_COMPONENTS)" -v "$(VTK_COMPONENTS)")
-  XTRALDFLAGS += $(shell $(PARAVIEW_CONFIG_EXEC) --ldflags -c "$(PARAVIEW_COMPONENTS)" -v "$(VTK_COMPONENTS)")
+  cxxcppflags += $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --cppflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
+  XTRALDFLAGS += $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --ldflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
 endif
diff --git a/Source/Insitu/cache.sh b/Source/Insitu/cache.sh
new file mode 100755
index 000000000..78cb87a84
--- /dev/null
+++ b/Source/Insitu/cache.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# This script caches the ouput of shell commands when first run and then uses
+# the cache if the arguments haven't changed.
+# Modified version of https://stackoverflow.com/a/36092050 to avoid race
+# condition.
+
+# hash all arguments
+KEY="$@"
+
+# hash last modified dates of any files
+for arg in "$@"
+do
+  if [ -f "$arg" ]
+  then
+    KEY+=`date -r "$arg" +\ %s`
+  fi
+done
+
+# use the hash as a name for temporary file
+FILE="/tmp/command_cache.`echo -n "$KEY" | md5sum | cut -c -10`"
+FILE_LOCK="${FILE}.lock"
+
+# check file isn't locked
+while [ -f $FILE_LOCK ]
+do
+  sleep 1
+done
+
+# use cached file or execute the command and cache it
+if [ -f $FILE ]
+then
+  cat $FILE
+else
+  # lock file
+  touch $FILE_LOCK
+  $@ | tee $FILE
+  rm -f $FILE_LOCK
+fi

From 1ab1957da4f304cd67198d0881ce79205d213c8c Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 30 Nov 2021 17:01:43 +0000
Subject: [PATCH 05/63] Add CatalystAdaptor to GRAMR

Also add compile_commands.json database.
---
 Source/GRChomboCore/GRAMR.cpp | 17 +++++++++++++++++
 Source/GRChomboCore/GRAMR.hpp | 13 +++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp
index 090c5ee0c..49f6a9de7 100644
--- a/Source/GRChomboCore/GRAMR.cpp
+++ b/Source/GRChomboCore/GRAMR.cpp
@@ -8,6 +8,14 @@
 
 GRAMR::GRAMR() : m_interpolator(nullptr) {}
 
+void GRAMR::conclude()
+{
+    AMR::conclude();
+#ifdef USE_CATALYST
+    m_insitu->finalise();
+#endif
+}
+
 // Called after AMR object set up
 void GRAMR::set_interpolator(AMRInterpolator<Lagrange<4>> *a_interpolator)
 {
@@ -51,3 +59,12 @@ void GRAMR::fill_multilevel_ghosts(const VariableType a_var_type,
         level.fillAllGhosts(a_var_type, a_comps);
     }
 }
+
+#ifdef USE_CATALYST
+void GRAMR::setup_catalyst(const std::string &a_python_script_path)
+{
+    pout() << "GRAMR::setup_catalyst" << std::endl;
+    m_insitu = new CatalystAdaptor;
+    m_insitu->initialise(this, a_python_script_path);
+}
+#endif
\ No newline at end of file
diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp
index a675ea523..9962844e0 100644
--- a/Source/GRChomboCore/GRAMR.hpp
+++ b/Source/GRChomboCore/GRAMR.hpp
@@ -38,6 +38,9 @@ class GRAMRLevel;
 // Forward declaration for AMRInterpolator
 template <typename InterpAlgo> class AMRInterpolator;
 
+// Forward declaration for CatalystAdaptor
+class CatalystAdaptor;
+
 class GRAMR : public AMR
 {
   private:
@@ -50,6 +53,9 @@ class GRAMR : public AMR
 
     GRAMR();
 
+    // just calls AMR::conclude unless USE_CATALYST == TRUE
+    void conclude();
+
     // defined here due to auto return type
     auto get_walltime()
     {
@@ -75,6 +81,13 @@ class GRAMR : public AMR
         const Interval &a_comps = Interval(0, std::numeric_limits<int>::max()),
         const int a_min_level = 0,
         const int a_max_level = std::numeric_limits<int>::max()) const;
+
+#ifdef USE_CATALYST
+    void setup_catalyst(const std::string &a_python_script_path);
+#endif
+
+  protected:
+    CatalystAdaptor *m_insitu;
 };
 
 #endif /* GRAMR_HPP_ */

From 2955d39d4c0ed60d2515d55306e1beb6b08bca1f Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 1 Dec 2021 11:59:25 +0000
Subject: [PATCH 06/63] Add Catalyst parameters

---
 Source/GRChomboCore/ChomboParameters.hpp | 44 ++++++++++++++++++++++++
 Source/GRChomboCore/GRAMR.cpp            | 18 +++++++---
 Source/GRChomboCore/GRAMR.hpp            |  6 +++-
 Source/GRChomboCore/SetupFunctions.hpp   |  4 +++
 4 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index d3ba7cb26..1fff91755 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -21,6 +21,10 @@
 #include <algorithm>
 #include <string>
 
+#ifdef USE_CATALYST
+#include <vtkCPPythonPipeline.h>
+#endif
+
 // Chombo namespace
 #include "UsingNamespace.H"
 
@@ -126,6 +130,15 @@ class ChomboParameters
 
         pp.load("print_progress_only_to_rank_0", print_progress_only_to_rank_0,
                 false);
+
+#ifdef USE_CATALYST
+        pp.load("activate_catalyst", activate_catalyst, false);
+        pp.load("catalyst_script_path", catalyst_script_path);
+        pp.load("catalyst_coprocess_level", catalyst_coprocess_level, 0);
+        UserVariables::load_vars_to_vector(pp, "catalyst_vars",
+                                           "num_catalyst_vars", catalyst_vars,
+                                           num_catalyst_vars);
+#endif
     }
 
     void read_filesystem_params(GRParmParse &pp)
@@ -464,6 +477,28 @@ class ChomboParameters
                                 "parity type undefined");
             }
         }
+
+#ifdef USE_CATALYST
+        if (activate_catalyst)
+        {
+            bool catalyst_script_exists =
+                (access(catalyst_script_path.c_str(), R_OK) == 0);
+            check_parameter("catalyst_script_path", catalyst_script_path,
+                            catalyst_script_exists, "file does not exist");
+            bool catalyst_script_valid =
+                (vtkCPPythonPipeline::DetectScriptVersion(
+                     catalyst_script_path.c_str()) != 0);
+            check_parameter("catalyst_script_path", catalyst_script_path,
+                            catalyst_script_valid,
+                            "not a valid ParaView Catalyst script");
+
+            check_parameter("catalyst_coprocess_level",
+                            catalyst_coprocess_level,
+                            catalyst_coprocess_level >= 0 &&
+                                catalyst_coprocess_level <= max_level,
+                            "not a valid level");
+        }
+#endif
     }
 
     // General parameters
@@ -524,6 +559,15 @@ class ChomboParameters
     bool just_check_params = false;
     bool print_progress_only_to_rank_0;
 
+#ifdef USE_CATALYST
+    bool activate_catalyst;
+    std::string catalyst_script_path;
+    int catalyst_coprocess_level;
+    // variables to pass to Catalyst
+    int num_catalyst_vars;
+    std::vector<std::pair<int, VariableType>> catalyst_vars;
+#endif
+
   protected:
     // the low and high corners of the domain taking into account reflective BCs
     // only used in parameter checks hence protected
diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp
index 49f6a9de7..14dd8600e 100644
--- a/Source/GRChomboCore/GRAMR.cpp
+++ b/Source/GRChomboCore/GRAMR.cpp
@@ -12,7 +12,10 @@ void GRAMR::conclude()
 {
     AMR::conclude();
 #ifdef USE_CATALYST
-    m_insitu->finalise();
+    if (m_activate_catalyst)
+    {
+        m_insitu->finalise();
+    }
 #endif
 }
 
@@ -61,10 +64,15 @@ void GRAMR::fill_multilevel_ghosts(const VariableType a_var_type,
 }
 
 #ifdef USE_CATALYST
-void GRAMR::setup_catalyst(const std::string &a_python_script_path)
+void GRAMR::setup_catalyst(bool a_activate_catalyst,
+                           const std::string &a_python_script_path)
 {
-    pout() << "GRAMR::setup_catalyst" << std::endl;
-    m_insitu = new CatalystAdaptor;
-    m_insitu->initialise(this, a_python_script_path);
+    m_activate_catalyst = a_activate_catalyst;
+    if (m_activate_catalyst)
+    {
+        pout() << "GRAMR::setup_catalyst" << std::endl;
+        m_insitu = new CatalystAdaptor;
+        m_insitu->initialise(this, a_python_script_path);
+    }
 }
 #endif
\ No newline at end of file
diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp
index 9962844e0..8c61cddbd 100644
--- a/Source/GRChomboCore/GRAMR.hpp
+++ b/Source/GRChomboCore/GRAMR.hpp
@@ -83,11 +83,15 @@ class GRAMR : public AMR
         const int a_max_level = std::numeric_limits<int>::max()) const;
 
 #ifdef USE_CATALYST
-    void setup_catalyst(const std::string &a_python_script_path);
+    void setup_catalyst(bool a_activate_catalyst,
+                        const std::string &a_python_script_path);
 #endif
 
   protected:
+#ifdef USE_CATALYST
+    bool m_activate_catalyst = false;
     CatalystAdaptor *m_insitu;
+#endif
 };
 
 #endif /* GRAMR_HPP_ */
diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 7e77141f1..867d02bc8 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -181,6 +181,10 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
         MayDay::Error("GRChombo restart only defined with hdf5");
 #endif
     }
+#ifdef USE_CATALYST
+    gr_amr.setup_catalyst(chombo_params.activate_catalyst,
+                          chombo_params.catalyst_script_path);
+#endif
 }
 
 #endif /* SETUP_FUNCTIONS_HPP_ */

From cd576ce3d375d05a4cde171f045ec5a8b2a4fd05 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 1 Dec 2021 16:15:16 +0000
Subject: [PATCH 07/63] Add CatalystAdaptor::build_vtk_grid function

This currently passes ghosted boxes to Catalyst.
---
 Source/Insitu/CatalystAdaptor.cpp | 73 ++++++++++++++++++++++++++++++-
 Source/Insitu/CatalystAdaptor.hpp |  2 +-
 2 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index ba35c83b6..ae90e5d50 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -4,6 +4,7 @@
  */
 
 #include "CatalystAdaptor.hpp"
+#include "GRAMRLevel.hpp"
 
 #ifdef USE_CATALYST
 
@@ -83,7 +84,77 @@ void CatalystAdaptor::finalise()
     m_initialised = false;
 }
 
-void CatalystAdaptor::update_vtk_grid() {}
+void CatalystAdaptor::build_vtk_grid()
+{
+    if (m_vtk_grid_ptr != nullptr)
+        m_vtk_grid_ptr->Delete();
+
+    m_vtk_grid_ptr = vtkOverlappingAMR::New();
+
+    const auto gramrlevels = m_gr_amr_ptr->get_gramrlevels();
+
+    // need the number of levels and the number of boxes on each level to
+    // initialise the vtkOverlappingAMR object
+    int num_levels = gramrlevels.size();
+    int *boxes_per_level = new int[num_levels];
+    for (int ilevel = 0; ilevel < num_levels; ++ilevel)
+    {
+        const GRAMRLevel *level = gramrlevels[ilevel];
+        const DisjointBoxLayout &level_box_layout =
+            level->getLevelData().disjointBoxLayout();
+        boxes_per_level[ilevel] = level_box_layout.boxArray().size();
+    }
+    m_vtk_grid_ptr->Initialize(num_levels, boxes_per_level);
+
+    // The origin is always at (0, 0, 0) in Chombo
+    double origin_global[3] = {0., 0., 0.};
+    m_vtk_grid_ptr->SetOrigin(origin_global);
+
+    // now add all the boxes
+    for (int ilevel = 0; ilevel < num_levels; ++ilevel)
+    {
+        const GRAMRLevel *level = gramrlevels[ilevel];
+        const double dx = level->get_dx();
+        const double dx_arr[3] = {dx, dx, dx};
+        m_vtk_grid_ptr->SetSpacing(ilevel, dx_arr);
+        m_vtk_grid_ptr->SetRefinementRatio(ilevel, level->refRatio());
+        const GRLevelData &level_data = level->getLevelData();
+        const DisjointBoxLayout &level_box_layout =
+            level_data.disjointBoxLayout();
+        LayoutIterator lit = level_box_layout.layoutIterator();
+        int ibox;
+        for (ibox = 0, lit.begin(); lit.ok(); ++lit, ++ibox)
+        {
+            // first get the box without ghosts
+            Box box = level_box_layout[lit];
+
+            // now grow the box to make the ghosted box
+            Box ghosted_box = box;
+            ghosted_box.grow(level_data.ghostVect());
+            const IntVect &small_end = ghosted_box.smallEnd();
+            const IntVect &big_end = ghosted_box.bigEnd();
+
+            vtkAMRBox vtk_amr_box(small_end.dataPtr(), big_end.dataPtr());
+            m_vtk_grid_ptr->SetAMRBox(ilevel, ibox, vtk_amr_box);
+
+            bool local_box = (procID() == level_box_layout.procID(lit()));
+            // only need to do the following for local boxes
+            if (local_box)
+            {
+                vtkNew<vtkUniformGrid> vtk_uniform_grid_ptr;
+                vtk_uniform_grid_ptr->SetOrigin(origin_global);
+                vtk_uniform_grid_ptr->SetSpacing(dx_arr);
+                vtk_uniform_grid_ptr->SetExtent(small_end[0], big_end[0] + 1,
+                                                small_end[1], big_end[1] + 1,
+                                                small_end[2], big_end[2] + 1);
+                m_vtk_grid_ptr->SetDataSet(ilevel, ibox, vtk_uniform_grid_ptr);
+            }
+        }
+    }
+
+    // not sure if this is necessary but it was on the OverlappingAMR example
+    m_vtk_grid_ptr->GenerateParentChildInformation();
+}
 
 void CatalystAdaptor::add_var() {}
 
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index fd5be40d5..965a1393c 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -56,7 +56,7 @@ class CatalystAdaptor
     void finalise();
 
     // update the AMR grid (no grid data)
-    void update_vtk_grid();
+    void build_vtk_grid();
 
     // send variables to catalyst
     void add_var();

From 87079463462fc31dbeb4e4f74597dfd207a9c59c Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 1 Dec 2021 18:13:58 +0000
Subject: [PATCH 08/63] Add CatalystAdaptor::add_vars

In theory only the variables requested in the script should be sent to
Catalyst.
---
 Source/Insitu/CatalystAdaptor.cpp | 92 ++++++++++++++++++++++++++++++-
 Source/Insitu/CatalystAdaptor.hpp | 14 +++--
 2 files changed, 97 insertions(+), 9 deletions(-)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index ae90e5d50..ad6823600 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -10,7 +10,7 @@
 
 CatalystAdaptor::CatalystAdaptor() {}
 
-CatalystAdaptor::CatalystAdaptor(const GRAMR *a_gr_amr_ptr,
+CatalystAdaptor::CatalystAdaptor(GRAMR *a_gr_amr_ptr,
                                  std::string a_python_script_path)
 {
     initialise(a_gr_amr_ptr, a_python_script_path);
@@ -24,7 +24,7 @@ CatalystAdaptor::~CatalystAdaptor()
     }
 }
 
-void CatalystAdaptor::initialise(const GRAMR *a_gr_amr_ptr,
+void CatalystAdaptor::initialise(GRAMR *a_gr_amr_ptr,
                                  std::string a_python_script_path)
 {
     // don't initalise twice
@@ -156,8 +156,94 @@ void CatalystAdaptor::build_vtk_grid()
     m_vtk_grid_ptr->GenerateParentChildInformation();
 }
 
-void CatalystAdaptor::add_var() {}
+void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
+{
+    std::array<bool, NUM_VARS> requested_evolution_vars;
+    std::array<bool, NUM_DIAGNOSTIC_VARS> requested_diagnostic_vars;
+
+    for (int ivar = 0; ivar < NUM_VARS; ++ivar)
+    {
+        requested_evolution_vars[ivar] = a_input_data_desc->IsFieldNeeded(
+            UserVariables::variable_names[ivar].c_str(), vtkDataObject::CELL);
+    }
+    for (int ivar = 0; ivar < NUM_DIAGNOSTIC_VARS; ++ivar)
+    {
+        requested_diagnostic_vars[ivar] = a_input_data_desc->IsFieldNeeded(
+            DiagnosticVariables::variable_names[ivar].c_str(),
+            vtkDataObject::CELL);
+    }
+
+    vtkAMRInformation *amr_info = m_vtk_grid_ptr->GetAMRInfo();
+    auto gramrlevels = m_gr_amr_ptr->get_gramrlevels();
+
+    for (int ilevel = 0; ilevel < gramrlevels.size(); ++ilevel)
+    {
+        GRAMRLevel *level = gramrlevels[ilevel];
+        // Unfortunately it doesn't seem possible to pass const pointers
+        // to Catalyst
+        GRLevelData &evolution_level_data = const_cast<GRLevelData &>(
+            level->getLevelData(VariableType::evolution));
+        GRLevelData &diagnostic_level_data = const_cast<GRLevelData &>(
+            level->getLevelData(VariableType::diagnostic));
+
+        const DisjointBoxLayout &level_box_layout =
+            evolution_level_data.disjointBoxLayout();
+        LayoutIterator lit = level_box_layout.layoutIterator();
+        int ibox;
+        for (ibox = 0, lit.begin(); lit.ok(); ++lit, ++ibox)
+        {
+            // only add data that we have locally
+            bool local_box = (procID() == level_box_layout.procID(lit()));
+            if (local_box)
+            {
+                vtkUniformGrid *vtk_uniform_grid_ptr =
+                    m_vtk_grid_ptr->GetDataSet(ilevel, ibox);
+                // hopefully this promotion works
+                DataIndex dind(lit());
+                FArrayBox &evolution_fab = evolution_level_data[dind];
+                FArrayBox &diagnostic_fab = diagnostic_level_data[dind];
+
+                for (int ivar = 0; ivar < NUM_VARS; ++ivar)
+                {
+                    if (requested_evolution_vars[ivar])
+                    {
+                        vtkDoubleArray *vtk_double_arr = fab_to_vtk_array(
+                            evolution_fab, ivar,
+                            UserVariables::variable_names[ivar]);
+                        vtk_uniform_grid_ptr->GetCellData()->AddArray(
+                            vtk_double_arr);
+                    }
+                }
+                for (int ivar = 0; ivar < NUM_DIAGNOSTIC_VARS; ++ivar)
+                {
+                    if (requested_diagnostic_vars[ivar])
+                    {
+                        vtkDoubleArray *vtk_double_arr = fab_to_vtk_array(
+                            diagnostic_fab, ivar,
+                            DiagnosticVariables::variable_names[ivar]);
+                        vtk_uniform_grid_ptr->GetCellData()->AddArray(
+                            vtk_double_arr);
+                    }
+                }
+            }
+        }
+    }
+}
 
 void CatalystAdaptor::coprocess(double time) {}
 
+vtkDoubleArray *CatalystAdaptor::fab_to_vtk_array(FArrayBox &a_fab, int a_var,
+                                                  const std::string &a_name)
+{
+    vtkDoubleArray *out = vtkDoubleArray::New();
+    vtkIdType num_cells = a_fab.size().product();
+    out->SetNumberOfTuples(num_cells);
+    out->SetName(a_name.c_str());
+    // this prevents Catalyst from deallocating the Chombo
+    // data pointers
+    int save_data = 1;
+    out->SetArray(a_fab.dataPtr(a_var), num_cells, save_data);
+    return out;
+}
+
 #endif /* USE_CATALYST */
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index 965a1393c..aa012b0d8 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -20,6 +20,7 @@
 
 // ParaView/VTK/Catalyst includes
 #include <vtkAMRBox.h>
+#include <vtkAMRInformation.h>
 #include <vtkCPDataDescription.h>
 #include <vtkCPInputDataDescription.h>
 #include <vtkCPProcessor.h>
@@ -44,29 +45,30 @@ class CatalystAdaptor
     CatalystAdaptor();
 
     // full constructor (calls initialise)
-    CatalystAdaptor(const GRAMR *a_gr_amr_ptr,
-                    std::string a_python_script_path);
+    CatalystAdaptor(GRAMR *a_gr_amr_ptr, std::string a_python_script_path);
 
     // destructor
     ~CatalystAdaptor();
 
     // Initialisation/Finalisation
-    void initialise(const GRAMR *m_gr_amr_ptr,
-                    std::string a_python_script_path);
+    void initialise(GRAMR *m_gr_amr_ptr, std::string a_python_script_path);
     void finalise();
 
     // update the AMR grid (no grid data)
     void build_vtk_grid();
 
     // send variables to catalyst
-    void add_var();
+    void add_vars(vtkCPInputDataDescription *a_input_data_desc);
 
     // do Catalyst processing
     void coprocess(double time);
 
   private:
+    vtkDoubleArray *fab_to_vtk_array(FArrayBox &a_fab, int a_var,
+                                     const std::string &a_name);
+
     bool m_initialised = false;
-    const GRAMR *m_gr_amr_ptr = nullptr;
+    GRAMR *m_gr_amr_ptr = nullptr;
     std::string m_python_script_path;
 
     vtkCPProcessor *m_proc_ptr = nullptr;

From ae6cddcf184bd1642c0861e37ef86f59c074e58d Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 7 Dec 2021 14:51:36 +0000
Subject: [PATCH 09/63] Implement CatalystAdaptor::coprocess and call it

---
 Source/GRChomboCore/GRAMR.cpp      |  2 +-
 Source/GRChomboCore/GRAMR.hpp      |  1 -
 Source/GRChomboCore/GRAMRLevel.cpp | 21 +++++++++++++
 Source/GRChomboCore/GRAMRLevel.hpp |  4 +++
 Source/Insitu/CatalystAdaptor.cpp  | 50 +++++++++++++++++++++++++++---
 Source/Insitu/CatalystAdaptor.hpp  | 15 +++++----
 6 files changed, 81 insertions(+), 12 deletions(-)

diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp
index 14dd8600e..c6a8c22ca 100644
--- a/Source/GRChomboCore/GRAMR.cpp
+++ b/Source/GRChomboCore/GRAMR.cpp
@@ -72,7 +72,7 @@ void GRAMR::setup_catalyst(bool a_activate_catalyst,
     {
         pout() << "GRAMR::setup_catalyst" << std::endl;
         m_insitu = new CatalystAdaptor;
-        m_insitu->initialise(this, a_python_script_path);
+        m_insitu->initialise(this, a_python_script_path, m_verbosity);
     }
 }
 #endif
\ No newline at end of file
diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp
index 8c61cddbd..880b5a0ee 100644
--- a/Source/GRChomboCore/GRAMR.hpp
+++ b/Source/GRChomboCore/GRAMR.hpp
@@ -87,7 +87,6 @@ class GRAMR : public AMR
                         const std::string &a_python_script_path);
 #endif
 
-  protected:
 #ifdef USE_CATALYST
     bool m_activate_catalyst = false;
     CatalystAdaptor *m_insitu;
diff --git a/Source/GRChomboCore/GRAMRLevel.cpp b/Source/GRChomboCore/GRAMRLevel.cpp
index 6239f5be9..fe5b72210 100644
--- a/Source/GRChomboCore/GRAMRLevel.cpp
+++ b/Source/GRChomboCore/GRAMRLevel.cpp
@@ -178,6 +178,27 @@ void GRAMRLevel::postTimeStep()
 
     specificPostTimeStep();
 
+#ifdef USE_CATALYST
+    if (m_p.activate_catalyst && m_level == m_p.catalyst_coprocess_level)
+    {
+        if (m_verbosity)
+        {
+            pout() << "GRAMRLevel::postTimestep: Calling Catalyst CoProcess"
+                   << std::endl;
+        }
+
+        unsigned int level_timestep = std::round(m_time / m_dt);
+        m_gr_amr.m_insitu->coprocess(m_time, level_timestep);
+
+        if (m_verbosity)
+        {
+            pout() << "GRAMRLevel::postTimestep: Calling Catalyst CoProcess "
+                      "finished"
+                   << std::endl;
+        }
+    }
+#endif
+
     // enforce solution BCs - this is required after the averaging
     // and postentially after specificPostTimeStep actions
     fillBdyGhosts(m_state_new);
diff --git a/Source/GRChomboCore/GRAMRLevel.hpp b/Source/GRChomboCore/GRAMRLevel.hpp
index e3bad9bbb..2e7a9534a 100644
--- a/Source/GRChomboCore/GRAMRLevel.hpp
+++ b/Source/GRChomboCore/GRAMRLevel.hpp
@@ -188,6 +188,10 @@ class GRAMRLevel : public AMRLevel, public InterpSource
         const VariableType var_type = VariableType::evolution,
         const Interval &a_comps = Interval(0, std::numeric_limits<int>::max()));
 
+#ifdef USE_CATALYST
+    virtual void preCatalystCoProcess() {}
+#endif
+
   protected:
     /// Fill all evolution ghosts cells (i.e. those in m_state_new)
     virtual void
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index ad6823600..ca9d94480 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -11,9 +11,10 @@
 CatalystAdaptor::CatalystAdaptor() {}
 
 CatalystAdaptor::CatalystAdaptor(GRAMR *a_gr_amr_ptr,
-                                 std::string a_python_script_path)
+                                 std::string a_python_script_path,
+                                 int a_verbosity)
 {
-    initialise(a_gr_amr_ptr, a_python_script_path);
+    initialise(a_gr_amr_ptr, a_python_script_path, a_verbosity);
 }
 
 CatalystAdaptor::~CatalystAdaptor()
@@ -25,7 +26,8 @@ CatalystAdaptor::~CatalystAdaptor()
 }
 
 void CatalystAdaptor::initialise(GRAMR *a_gr_amr_ptr,
-                                 std::string a_python_script_path)
+                                 std::string a_python_script_path,
+                                 int a_verbosity)
 {
     // don't initalise twice
     if (m_initialised)
@@ -66,6 +68,8 @@ void CatalystAdaptor::initialise(GRAMR *a_gr_amr_ptr,
         MayDay::Warning(pipeline_fail_warning.c_str());
     }
 
+    m_verbosity = a_verbosity;
+
     m_initialised = true;
 }
 
@@ -86,6 +90,10 @@ void CatalystAdaptor::finalise()
 
 void CatalystAdaptor::build_vtk_grid()
 {
+    if (m_verbosity)
+    {
+        pout() << "CatalystAdaptor::build_vtk_grid" << std::endl;
+    }
     if (m_vtk_grid_ptr != nullptr)
         m_vtk_grid_ptr->Delete();
 
@@ -158,20 +166,36 @@ void CatalystAdaptor::build_vtk_grid()
 
 void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
 {
+    if (m_verbosity)
+    {
+        pout() << "CatalystAdaptor::add_vars" << std::endl;
+    }
+
     std::array<bool, NUM_VARS> requested_evolution_vars;
     std::array<bool, NUM_DIAGNOSTIC_VARS> requested_diagnostic_vars;
 
+    if (m_verbosity > 1)
+    {
+        pout() << "CatalystAdaptor Requested variables:\n";
+    }
+
     for (int ivar = 0; ivar < NUM_VARS; ++ivar)
     {
         requested_evolution_vars[ivar] = a_input_data_desc->IsFieldNeeded(
             UserVariables::variable_names[ivar].c_str(), vtkDataObject::CELL);
+        if (m_verbosity > 1 && requested_evolution_vars[ivar])
+            pout() << UserVariables::variable_names[ivar] << " ";
     }
     for (int ivar = 0; ivar < NUM_DIAGNOSTIC_VARS; ++ivar)
     {
         requested_diagnostic_vars[ivar] = a_input_data_desc->IsFieldNeeded(
             DiagnosticVariables::variable_names[ivar].c_str(),
             vtkDataObject::CELL);
+        if (m_verbosity > 1 && requested_diagnostic_vars[ivar])
+            pout() << DiagnosticVariables::variable_names[ivar] << " ";
     }
+    if (m_verbosity > 1)
+        pout() << std::endl;
 
     vtkAMRInformation *amr_info = m_vtk_grid_ptr->GetAMRInfo();
     auto gramrlevels = m_gr_amr_ptr->get_gramrlevels();
@@ -230,7 +254,25 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
     }
 }
 
-void CatalystAdaptor::coprocess(double time) {}
+void CatalystAdaptor::coprocess(double a_time, unsigned int a_timestep)
+{
+    pout() << "CatalystAdaptor::coprocess at time " << a_time << " and step "
+           << a_timestep << std::endl;
+
+    vtkNew<vtkCPDataDescription> data_description;
+    data_description->AddInput("input");
+    data_description->SetTimeData(a_time, a_timestep);
+
+    if (m_proc_ptr->RequestDataDescription(data_description) != 0)
+    {
+        build_vtk_grid();
+        auto input_data_description =
+            data_description->GetInputDescriptionByName("input");
+        add_vars(input_data_description);
+        input_data_description->SetGrid(m_vtk_grid_ptr);
+        m_proc_ptr->CoProcess(data_description);
+    }
+}
 
 vtkDoubleArray *CatalystAdaptor::fab_to_vtk_array(FArrayBox &a_fab, int a_var,
                                                   const std::string &a_name)
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index aa012b0d8..217aeea41 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -45,28 +45,31 @@ class CatalystAdaptor
     CatalystAdaptor();
 
     // full constructor (calls initialise)
-    CatalystAdaptor(GRAMR *a_gr_amr_ptr, std::string a_python_script_path);
+    CatalystAdaptor(GRAMR *a_gr_amr_ptr, std::string a_python_script_path,
+                    int a_verbosity);
 
     // destructor
     ~CatalystAdaptor();
 
     // Initialisation/Finalisation
-    void initialise(GRAMR *m_gr_amr_ptr, std::string a_python_script_path);
+    void initialise(GRAMR *m_gr_amr_ptr, std::string a_python_script_path,
+                    int a_verbosity);
     void finalise();
 
+    // do Catalyst processing
+    void coprocess(double a_time, unsigned int a_timestep);
+
+  private:
     // update the AMR grid (no grid data)
     void build_vtk_grid();
 
     // send variables to catalyst
     void add_vars(vtkCPInputDataDescription *a_input_data_desc);
 
-    // do Catalyst processing
-    void coprocess(double time);
-
-  private:
     vtkDoubleArray *fab_to_vtk_array(FArrayBox &a_fab, int a_var,
                                      const std::string &a_name);
 
+    int m_verbosity;
     bool m_initialised = false;
     GRAMR *m_gr_amr_ptr = nullptr;
     std::string m_python_script_path;

From cf8069c595bdad8aa0f19cf79f205a1ba19ccb49 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 15 Dec 2021 17:31:24 +0000
Subject: [PATCH 10/63] Modify construction of vtkOverlappingAMR

* Change constructor for each vtkAMRBox.
* Generate ghost array for each vtkUniformGrid.
* Audit grid at the end of construction.
---
 Source/Insitu/CatalystAdaptor.cpp | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index ca9d94480..7080b689e 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -135,14 +135,23 @@ void CatalystAdaptor::build_vtk_grid()
         {
             // first get the box without ghosts
             Box box = level_box_layout[lit];
+            const IntVect &small_end = box.smallEnd();
+            const IntVect &big_end = box.bigEnd();
 
             // now grow the box to make the ghosted box
             Box ghosted_box = box;
             ghosted_box.grow(level_data.ghostVect());
-            const IntVect &small_end = ghosted_box.smallEnd();
-            const IntVect &big_end = ghosted_box.bigEnd();
+            const IntVect &small_ghosted_end = ghosted_box.smallEnd();
+            const IntVect &big_ghosted_end = ghosted_box.bigEnd();
 
-            vtkAMRBox vtk_amr_box(small_end.dataPtr(), big_end.dataPtr());
+            // vtkAMRBox vtk_amr_box(small_ghosted_end.dataPtr(),
+            //                       big_ghosted_end.dataPtr());
+            double origin[3] = {dx_arr[0] * small_ghosted_end[0],
+                                dx_arr[1] * small_ghosted_end[1],
+                                dx_arr[2] * small_ghosted_end[2]};
+
+            vtkAMRBox vtk_amr_box(origin, ghosted_box.size().dataPtr(), dx_arr,
+                                  origin_global);
             m_vtk_grid_ptr->SetAMRBox(ilevel, ibox, vtk_amr_box);
 
             bool local_box = (procID() == level_box_layout.procID(lit()));
@@ -152,16 +161,24 @@ void CatalystAdaptor::build_vtk_grid()
                 vtkNew<vtkUniformGrid> vtk_uniform_grid_ptr;
                 vtk_uniform_grid_ptr->SetOrigin(origin_global);
                 vtk_uniform_grid_ptr->SetSpacing(dx_arr);
-                vtk_uniform_grid_ptr->SetExtent(small_end[0], big_end[0] + 1,
-                                                small_end[1], big_end[1] + 1,
-                                                small_end[2], big_end[2] + 1);
+                vtk_uniform_grid_ptr->SetExtent(
+                    small_ghosted_end[0], big_ghosted_end[0] + 1,
+                    small_ghosted_end[1], big_ghosted_end[1] + 1,
+                    small_ghosted_end[2], big_ghosted_end[2] + 1);
+                // add the ghost cell information
+                int no_ghost[6] = {small_end[0], big_end[0] + 1,
+                                   small_end[1], big_end[1] + 1,
+                                   small_end[2], big_end[2] + 1};
+                bool cell_data = true;
+                vtk_uniform_grid_ptr->GenerateGhostArray(no_ghost, cell_data);
                 m_vtk_grid_ptr->SetDataSet(ilevel, ibox, vtk_uniform_grid_ptr);
             }
         }
     }
 
+    m_vtk_grid_ptr->Audit();
     // not sure if this is necessary but it was on the OverlappingAMR example
-    m_vtk_grid_ptr->GenerateParentChildInformation();
+    // m_vtk_grid_ptr->GenerateParentChildInformation();
 }
 
 void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)

From 8c8257210721d22ca67216ac25bcafa2251d3f96 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 15 Dec 2021 17:38:04 +0000
Subject: [PATCH 11/63] Add example Catalyst script and params file

---
 .../BinaryBH/catalyst_scripts/slice_chi.py    | 149 ++++++++++++++
 Examples/BinaryBH/params_catalyst.txt         | 191 ++++++++++++++++++
 2 files changed, 340 insertions(+)
 create mode 100644 Examples/BinaryBH/catalyst_scripts/slice_chi.py
 create mode 100644 Examples/BinaryBH/params_catalyst.txt

diff --git a/Examples/BinaryBH/catalyst_scripts/slice_chi.py b/Examples/BinaryBH/catalyst_scripts/slice_chi.py
new file mode 100644
index 000000000..49c369954
--- /dev/null
+++ b/Examples/BinaryBH/catalyst_scripts/slice_chi.py
@@ -0,0 +1,149 @@
+# script-version: 2.0
+# Catalyst state generated using paraview version 5.9.1
+
+#### import the simple module from the paraview
+from paraview.simple import *
+#### disable automatic camera reset on 'Show'
+paraview.simple._DisableFirstRenderCameraReset()
+
+# ----------------------------------------------------------------
+# setup views used in the visualization
+# ----------------------------------------------------------------
+
+# get the material library
+materialLibrary1 = GetMaterialLibrary()
+
+# Create a new 'Render View'
+renderView1 = CreateView('RenderView')
+renderView1.ViewSize = [1131, 939]
+renderView1.AxesGrid = 'GridAxes3DActor'
+renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
+renderView1.StereoType = 'Crystal Eyes'
+renderView1.CameraPosition = [-24.234943738007217, 12.79030062375982, 77.14896782254773]
+renderView1.CameraFocalPoint = [31.99999999999993, 31.99999999999998, 31.99999999999996]
+renderView1.CameraViewUp = [0.6572599982077688, -0.31659445000046216, 0.6839424310457926]
+renderView1.CameraFocalDisk = 1.0
+renderView1.CameraParallelScale = 60.6217782649107
+renderView1.BackEnd = 'OSPRay raycaster'
+renderView1.OSPRayMaterialLibrary = materialLibrary1
+
+SetActiveView(None)
+
+# ----------------------------------------------------------------
+# setup view layouts
+# ----------------------------------------------------------------
+
+# create new layout object 'Layout #1'
+layout1 = CreateLayout(name='Layout #1')
+layout1.AssignView(0, renderView1)
+layout1.SetSize(1131, 939)
+
+# ----------------------------------------------------------------
+# restore active view
+SetActiveView(renderView1)
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup the data processing pipelines
+# ----------------------------------------------------------------
+
+# create a new 'VisItChomboReader'
+binaryBH_chi_ghosts3dhdf5 = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_chi_ghosts.3d.hdf5'])
+binaryBH_chi_ghosts3dhdf5.MeshStatus = ['Mesh']
+binaryBH_chi_ghosts3dhdf5.CellArrayStatus = ['chi']
+
+# create a new 'Slice AMR data'
+sliceAMRdata1 = SliceAMRdata(registrationName='SliceAMRdata1', Input=binaryBH_chi_ghosts3dhdf5)
+sliceAMRdata1.Level = 2
+sliceAMRdata1.OffSet = 35.0
+sliceAMRdata1.Normal = 'Z-Normal'
+
+# ----------------------------------------------------------------
+# setup the visualization in view 'renderView1'
+# ----------------------------------------------------------------
+
+# show data from sliceAMRdata1
+sliceAMRdata1Display = Show(sliceAMRdata1, renderView1, 'AMRRepresentation')
+
+# get color transfer function/color map for 'chi'
+chiLUT = GetColorTransferFunction('chi')
+chiLUT.RGBPoints = [4.8189884420197785e-05, 0.231373, 0.298039, 0.752941, 0.4837166353392167, 0.865003, 0.865003, 0.865003, 0.9673850807940133, 0.705882, 0.0156863, 0.14902]
+chiLUT.ScalarRangeInitialized = 1.0
+
+# get opacity transfer function/opacity map for 'chi'
+chiPWF = GetOpacityTransferFunction('chi')
+chiPWF.Points = [4.8189884420197785e-05, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
+chiPWF.ScalarRangeInitialized = 1
+
+# trace defaults for the display properties.
+sliceAMRdata1Display.Representation = 'Surface'
+sliceAMRdata1Display.ColorArrayName = ['CELLS', 'chi']
+sliceAMRdata1Display.LookupTable = chiLUT
+sliceAMRdata1Display.SelectTCoordArray = 'None'
+sliceAMRdata1Display.SelectNormalArray = 'None'
+sliceAMRdata1Display.SelectTangentArray = 'None'
+sliceAMRdata1Display.OSPRayScaleFunction = 'PiecewiseFunction'
+sliceAMRdata1Display.SelectOrientationVectors = 'None'
+sliceAMRdata1Display.ScaleFactor = 7.0
+sliceAMRdata1Display.SelectScaleArray = 'None'
+sliceAMRdata1Display.GlyphType = 'Arrow'
+sliceAMRdata1Display.GlyphTableIndexArray = 'None'
+sliceAMRdata1Display.GaussianRadius = 0.35000000000000003
+sliceAMRdata1Display.SetScaleArray = [None, '']
+sliceAMRdata1Display.ScaleTransferFunction = 'PiecewiseFunction'
+sliceAMRdata1Display.OpacityArray = [None, '']
+sliceAMRdata1Display.OpacityTransferFunction = 'PiecewiseFunction'
+sliceAMRdata1Display.DataAxesGrid = 'GridAxesRepresentation'
+sliceAMRdata1Display.PolarAxes = 'PolarAxesRepresentation'
+sliceAMRdata1Display.ScalarOpacityUnitDistance = 1.7175096404638197
+sliceAMRdata1Display.ScalarOpacityFunction = chiPWF
+
+# setup the color legend parameters for each legend in this view
+
+# get color legend/bar for chiLUT in view renderView1
+chiLUTColorBar = GetScalarBar(chiLUT, renderView1)
+chiLUTColorBar.Title = 'chi'
+chiLUTColorBar.ComponentTitle = ''
+
+# set color bar visibility
+chiLUTColorBar.Visibility = 1
+
+# show color legend
+sliceAMRdata1Display.SetScalarBarVisibility(renderView1, True)
+
+# ----------------------------------------------------------------
+# setup color maps and opacity mapes used in the visualization
+# note: the Get..() functions create a new object, if needed
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup extractors
+# ----------------------------------------------------------------
+
+# create extractor
+pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
+# trace defaults for the extractor.
+# init the 'PNG' selected for 'Writer'
+pNG1.Writer.FileName = 'RenderView1_%.6ts%cm.png'
+pNG1.Writer.ImageResolution = [1131, 939]
+pNG1.Writer.Format = 'PNG'
+
+# ----------------------------------------------------------------
+# restore active source
+SetActiveSource(sliceAMRdata1)
+# ----------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# Catalyst options
+from paraview import catalyst
+options = catalyst.Options()
+options.GlobalTrigger = 'TimeStep'
+options.EnableCatalystLive = 1
+options.CatalystLiveTrigger = 'TimeStep'
+
+# ------------------------------------------------------------------------------
+if __name__ == '__main__':
+    from paraview.simple import SaveExtractsUsingCatalystOptions
+    # Code for non in-situ environments; if executing in post-processing
+    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
+    SaveExtractsUsingCatalystOptions(options)
diff --git a/Examples/BinaryBH/params_catalyst.txt b/Examples/BinaryBH/params_catalyst.txt
new file mode 100644
index 000000000..170b33b93
--- /dev/null
+++ b/Examples/BinaryBH/params_catalyst.txt
@@ -0,0 +1,191 @@
+#################################################
+# Filesystem parameters
+
+verbosity = 1
+
+# location / naming of output files
+# output_path = "" # Main path for all files. Must exist!
+chk_prefix = BinaryBH_
+plot_prefix = BinaryBHp_
+# restart_file = BinaryBHChk_000000.3d.hdf5
+
+# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval
+checkpoint_interval = -1
+# set to 0 to turn off plot files (except at t=0 and t=stop_time)
+# set to -1 to never ever print plotfiles
+plot_interval = 1
+num_plot_vars = 1
+plot_vars = chi
+
+# subpaths - specific directories for hdf5, pout, extraction data
+# (these are created at runtime)
+hdf5_subpath = "hdf5"
+pout_subpath = "pout"
+data_subpath = "data"
+
+# change the name of output files
+# pout_prefix = "pout"
+print_progress_only_to_rank_0 = 1
+
+# ignore_checkpoint_name_mismatch = 0
+write_plot_ghosts = 1
+
+#################################################
+# Initial Data parameters
+
+# provide 'offset' or 'center'
+
+massA = 0.48847892320123
+massB = 0.48847892320123
+
+offsetA = 0.0 6.10679 0.0
+offsetB = 0.0 -6.10679 0.0
+# centerA = 256 232 256
+# centerB = 256 250 256
+
+momentumA = -0.0841746 -0.000510846 0.0
+momentumB =  0.0841746  0.000510846 0.0
+
+#################################################
+# Grid parameters
+
+# 'N' is the number of subdivisions in each direction of a cubic box
+# 'L' is the length of the longest side of the box, dx_coarsest = L/N
+# NB - If you use reflective BC and want to specify the subdivisions and side
+# of the box were there are no symmetries, specify 'N_full' and 'L_full' instead
+# NB - if you have a non-cubic grid, you can specify 'N1' or 'N1_full',
+# 'N2' or 'N2_full' and 'N3' or 'N3_full' ( then dx_coarsest = L/N(max) )
+# NB - the N values need to be multiples of the block_factor
+N_full = 64
+L_full = 64
+
+# Maximum number of times you can regrid above coarsest level
+max_level = 3 # There are (max_level+1) grids, so min is zero
+
+# Frequency of regridding at each level and thresholds on the tagging
+# Need one for each level except the top one, ie max_level items
+# Generally you do not need to regrid frequently on every level
+# Level Regridding: 0   1   2   3   4   5   6   7   8
+regrid_interval =   0   0   0   64  64  64  64  64  64
+regrid_threshold = 0.02
+
+# Max and min box sizes
+max_box_size = 16
+min_box_size = 16
+
+# tag_buffer_size = 3
+# grid_buffer_size = 8
+# fill_ratio = 0.7
+# num_ghosts = 3
+# center = 256.0 256.0 256.0 # defaults to center of the grid
+
+#################################################
+# Boundary Conditions parameters
+
+# Periodic directions - 0 = false, 1 = true
+isPeriodic = 0 0 0
+# if not periodic, then specify the boundary type
+# 0 = static, 1 = sommerfeld, 2 = reflective
+# (see BoundaryConditions.hpp for details)
+hi_boundary = 1 1 1
+lo_boundary = 1 1 1
+
+# if reflective boundaries selected, must set
+# parity of all vars (in order given by UserVariables.hpp)
+# 0 = even
+# 1,2,3 = odd x, y, z
+# 4,5,6 = odd xy, yz, xz
+# 7     = odd xyz
+vars_parity            = 0 0 4 6 0 5 0    #chi and hij
+                         0 0 4 6 0 5 0    #K and Aij
+                         0 1 2 3          #Theta and Gamma
+                         0 1 2 3 1 2 3    #lapse shift and B
+vars_parity_diagnostic = 0 1 2 3          #Ham and Mom
+                         0 7              #Weyl
+
+# if sommerfeld boundaries selected, must select
+# asymptotic values
+num_nonzero_asymptotic_vars = 5
+nonzero_asymptotic_vars = chi h11 h22 h33 lapse
+nonzero_asymptotic_values = 1.0 1.0 1.0 1.0 1.0
+
+# if you are using extrapolating BC:
+# extrapolation_order = 1
+# num_extrapolating_vars = -1
+# extrapolating_vars =
+
+#################################################
+# Evolution parameters
+
+# dt will be dx*dt_multiplier on each grid level
+dt_multiplier = 0.25
+stop_time = 5.0
+# max_steps = 100
+
+# Spatial derivative order (only affects CCZ4 RHS)
+max_spatial_derivative_order = 4 # can be 4 or 6
+
+nan_check = 1
+
+# Lapse evolution
+lapse_advec_coeff = 1.0
+lapse_coeff = 2.0
+lapse_power = 1.0
+
+# Shift evolution
+shift_advec_coeff = 0.0 # Usually no advection for beta
+shift_Gamma_coeff = 0.75
+eta = 1.0 # eta of gamma driver, should be of order ~1/M_ADM of spacetime
+
+# CCZ4 parameters
+formulation = 0 # 1 for BSSN, 0 for CCZ4
+kappa1 = 0.1
+kappa2 = 0.
+kappa3 = 1.
+covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse
+
+# coefficient for KO numerical dissipation
+sigma = 1.0
+
+track_punctures = 1
+puncture_tracking_level = 3
+
+# calculate_constraint_norms = 0
+
+# min_chi = 1.e-4
+# min_lapse = 1.e-4
+
+#################################################
+# Extraction parameters
+
+# extraction_center = 256 256 256 # defaults to center
+activate_extraction = 0
+num_extraction_radii = 0
+extraction_radii = 1.0 1.0
+extraction_levels = 2 1
+num_points_phi = 24
+num_points_theta = 37
+num_modes = 8
+modes = 2 0 # l m for spherical harmonics
+        2 1
+        2 2
+        4 0
+        4 1
+        4 2
+        4 3
+        4 4
+
+# integral_file_prefix = "Weyl4_mode_"
+
+# write_extraction = 0
+# extraction_subpath = "data/extraction" # directory for 'write_extraction = 1'
+# extraction_file_prefix = "Weyl4_extraction_"
+
+#################################################
+# ParaView Catalyst parameters
+
+activate_catalyst = true
+catalyst_script_path = catalyst_scripts/slice_chi.py
+catalyst_coprocess_level = 2
+num_catalyst_vars = 1
+caralyst_vars = chi

From 06ea334c842c59b63412a08223a9ebe9f1f3de67 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Mon, 20 Dec 2021 16:33:49 +0000
Subject: [PATCH 12/63] Add Catalyst volume rendering script

---
 .../catalyst_scripts/volume_raytrace_chi.py   | 133 ++++++++++++++++++
 1 file changed, 133 insertions(+)
 create mode 100644 Examples/BinaryBH/catalyst_scripts/volume_raytrace_chi.py

diff --git a/Examples/BinaryBH/catalyst_scripts/volume_raytrace_chi.py b/Examples/BinaryBH/catalyst_scripts/volume_raytrace_chi.py
new file mode 100644
index 000000000..7ab651950
--- /dev/null
+++ b/Examples/BinaryBH/catalyst_scripts/volume_raytrace_chi.py
@@ -0,0 +1,133 @@
+# script-version: 2.0
+# Catalyst state generated using paraview version 5.9.1
+
+#### import the simple module from the paraview
+from paraview.simple import *
+#### disable automatic camera reset on 'Show'
+paraview.simple._DisableFirstRenderCameraReset()
+
+# ----------------------------------------------------------------
+# setup views used in the visualization
+# ----------------------------------------------------------------
+
+# get the material library
+materialLibrary1 = GetMaterialLibrary()
+
+# Create a new 'Render View'
+renderView1 = CreateView('RenderView')
+renderView1.ViewSize = [885, 778]
+renderView1.AxesGrid = 'GridAxes3DActor'
+renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
+renderView1.StereoType = 'Crystal Eyes'
+renderView1.CameraPosition = [20.770091346798225, 54.44320054580248, 46.07511197528731]
+renderView1.CameraFocalPoint = [32.000000000000014, 31.999999999999986, 31.999999999999993]
+renderView1.CameraViewUp = [0.37830852443531104, -0.3485213720084932, 0.8575637081831853]
+renderView1.CameraFocalDisk = 1.0
+renderView1.CameraParallelScale = 60.6217782649107
+renderView1.EnableRayTracing = 1
+renderView1.BackEnd = 'OSPRay raycaster'
+renderView1.OSPRayMaterialLibrary = materialLibrary1
+
+SetActiveView(None)
+
+# ----------------------------------------------------------------
+# setup view layouts
+# ----------------------------------------------------------------
+
+# create new layout object 'Layout #1'
+layout1 = CreateLayout(name='Layout #1')
+layout1.AssignView(0, renderView1)
+layout1.SetSize(885, 778)
+
+# ----------------------------------------------------------------
+# restore active view
+SetActiveView(renderView1)
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup the data processing pipelines
+# ----------------------------------------------------------------
+
+# create a new 'VisItChomboReader'
+binaryBHp_00000 = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000002.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000003.3d.hdf5'])
+binaryBHp_00000.MeshStatus = ['Mesh']
+binaryBHp_00000.CellArrayStatus = ['chi']
+
+# ----------------------------------------------------------------
+# setup the visualization in view 'renderView1'
+# ----------------------------------------------------------------
+
+# show data from binaryBHp_00000
+binaryBHp_00000Display = Show(binaryBHp_00000, renderView1, 'AMRRepresentation')
+
+# get color transfer function/color map for 'chi'
+chiLUT = GetColorTransferFunction('chi')
+chiLUT.RGBPoints = [0.006705302783959541, 0.0862745098039216, 0.00392156862745098, 0.298039215686275, 0.03584677977638898, 0.113725, 0.0235294, 0.45098, 0.060048963685229544, 0.105882, 0.0509804, 0.509804, 0.07684234902507095, 0.0392157, 0.0392157, 0.560784, 0.09314182943734466, 0.0313725, 0.0980392, 0.6, 0.10894734875818932, 0.0431373, 0.164706, 0.639216, 0.13166776172045955, 0.054902, 0.243137, 0.678431, 0.16179708557943404, 0.054902, 0.317647, 0.709804, 0.19884125838596878, 0.0509804, 0.396078, 0.741176, 0.22285825283622032, 0.0392157, 0.466667, 0.768627, 0.24687524728647184, 0.0313725, 0.537255, 0.788235, 0.27194187206163456, 0.0313725, 0.615686, 0.811765, 0.2976258269170425, 0.0235294, 0.709804, 0.831373, 0.3233098379363089, 0.0509804, 0.8, 0.85098, 0.3445484799520089, 0.0705882, 0.854902, 0.870588, 0.3643054071849909, 0.262745, 0.901961, 0.862745, 0.3815926974524059, 0.423529, 0.941176, 0.87451, 0.40826451832681654, 0.572549, 0.964706, 0.835294, 0.4260457696856576, 0.658824, 0.980392, 0.843137, 0.43888774711336165, 0.764706, 0.980392, 0.866667, 0.4527175624781391, 0.827451, 0.980392, 0.886275, 0.4798833163620467, 0.913725, 0.988235, 0.937255, 0.4882800090319688, 1.0, 1.0, 0.972549019607843, 0.49667670170189104, 0.988235, 0.980392, 0.870588, 0.5090247742020214, 0.992156862745098, 0.972549019607843, 0.803921568627451, 0.518409304809017, 0.992157, 0.964706, 0.713725, 0.5342148241298618, 0.988235, 0.956863, 0.643137, 0.559404874057699, 0.980392, 0.917647, 0.509804, 0.5806435722372544, 0.968627, 0.87451, 0.407843, 0.6023761753443805, 0.94902, 0.823529, 0.321569, 0.6181816946652244, 0.929412, 0.776471, 0.278431, 0.6413960687189206, 0.909804, 0.717647, 0.235294, 0.6621408058070486, 0.890196, 0.658824, 0.196078, 0.679181147390997, 0.878431, 0.619608, 0.168627, 0.7031981418412483, 0.870588, 0.54902, 0.156863, 0.7272151362914998, 0.85098, 0.47451, 0.145098, 0.751232130741751, 0.831373, 0.411765, 0.133333, 0.7752491251920026, 0.811765, 0.345098, 0.113725, 0.7992661196422539, 0.788235, 0.266667, 0.0941176, 0.8232831140925052, 0.741176, 0.184314, 0.0745098, 0.8473001085427566, 0.690196, 0.12549, 0.0627451, 0.8713171029930079, 0.619608, 0.0627451, 0.0431373, 0.8937905905179446, 0.54902, 0.027451, 0.0705882, 0.9135475010772842, 0.470588, 0.0156863, 0.0901961, 0.935774024030506, 0.4, 0.00392157, 0.101961, 0.9673850807940133, 0.188235294117647, 0.0, 0.0705882352941176]
+chiLUT.ColorSpace = 'Lab'
+chiLUT.NanColor = [1.0, 0.0, 0.0]
+chiLUT.ScalarRangeInitialized = 1.0
+
+# get opacity transfer function/opacity map for 'chi'
+chiPWF = GetOpacityTransferFunction('chi')
+chiPWF.Points = [0.006705302783959541, 1.0, 0.5, 0.0, 0.055023517459630966, 0.6094674468040466, 0.5, 0.0, 0.126079723238945, 0.4378698170185089, 0.5, 0.0, 0.22840063273906708, 0.23076923191547394, 0.5, 0.0, 0.37619754672050476, 0.08875739574432373, 0.5, 0.0, 0.5808393955230713, 0.0, 0.5, 0.0, 0.9673850536346436, 0.0, 0.5, 0.0]
+chiPWF.ScalarRangeInitialized = 1
+
+# trace defaults for the display properties.
+binaryBHp_00000Display.Representation = 'Volume'
+binaryBHp_00000Display.ColorArrayName = ['CELLS', 'chi']
+binaryBHp_00000Display.LookupTable = chiLUT
+binaryBHp_00000Display.SelectTCoordArray = 'None'
+binaryBHp_00000Display.SelectNormalArray = 'None'
+binaryBHp_00000Display.SelectTangentArray = 'None'
+binaryBHp_00000Display.OSPRayScaleFunction = 'PiecewiseFunction'
+binaryBHp_00000Display.SelectOrientationVectors = 'None'
+binaryBHp_00000Display.ScaleFactor = 7.0
+binaryBHp_00000Display.SelectScaleArray = 'None'
+binaryBHp_00000Display.GlyphType = 'Arrow'
+binaryBHp_00000Display.GlyphTableIndexArray = 'None'
+binaryBHp_00000Display.GaussianRadius = 0.35000000000000003
+binaryBHp_00000Display.SetScaleArray = [None, '']
+binaryBHp_00000Display.ScaleTransferFunction = 'PiecewiseFunction'
+binaryBHp_00000Display.OpacityArray = [None, '']
+binaryBHp_00000Display.OpacityTransferFunction = 'PiecewiseFunction'
+binaryBHp_00000Display.DataAxesGrid = 'GridAxesRepresentation'
+binaryBHp_00000Display.PolarAxes = 'PolarAxesRepresentation'
+binaryBHp_00000Display.ScalarOpacityUnitDistance = 0.8192401766441247
+binaryBHp_00000Display.ScalarOpacityFunction = chiPWF
+
+# ----------------------------------------------------------------
+# setup color maps and opacity mapes used in the visualization
+# note: the Get..() functions create a new object, if needed
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup extractors
+# ----------------------------------------------------------------
+
+# create extractor
+pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
+# trace defaults for the extractor.
+# init the 'PNG' selected for 'Writer'
+pNG1.Writer.FileName = 'RenderView1_%.6ts%cm.png'
+pNG1.Writer.ImageResolution = [885, 778]
+pNG1.Writer.Format = 'PNG'
+
+# ----------------------------------------------------------------
+# restore active source
+SetActiveSource(pNG1)
+# ----------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# Catalyst options
+from paraview import catalyst
+options = catalyst.Options()
+options.GlobalTrigger = 'TimeStep'
+options.EnableCatalystLive = 1
+options.CatalystLiveTrigger = 'TimeStep'
+
+# ------------------------------------------------------------------------------
+if __name__ == '__main__':
+    from paraview.simple import SaveExtractsUsingCatalystOptions
+    # Code for non in-situ environments; if executing in post-processing
+    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
+    SaveExtractsUsingCatalystOptions(options)

From a95360e19ee7713d0a693d766caf7d12dd5bcacc Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 21 Dec 2021 12:12:34 +0000
Subject: [PATCH 13/63] Add another Catalyst volume rendering script

This one includes resampling to an image and no raytracing.
---
 .../catalyst_scripts/volume_noraytrace_chi.py | 153 ++++++++++++++++++
 1 file changed, 153 insertions(+)
 create mode 100644 Examples/BinaryBH/catalyst_scripts/volume_noraytrace_chi.py

diff --git a/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_chi.py b/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_chi.py
new file mode 100644
index 000000000..c3ecb7ed3
--- /dev/null
+++ b/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_chi.py
@@ -0,0 +1,153 @@
+# script-version: 2.0
+# Catalyst state generated using paraview version 5.9.1
+
+#### import the simple module from the paraview
+from paraview.simple import *
+#### disable automatic camera reset on 'Show'
+paraview.simple._DisableFirstRenderCameraReset()
+
+# ----------------------------------------------------------------
+# setup views used in the visualization
+# ----------------------------------------------------------------
+
+# get the material library
+materialLibrary1 = GetMaterialLibrary()
+
+# Create a new 'Render View'
+renderView1 = CreateView('RenderView')
+renderView1.ViewSize = [885, 774]
+renderView1.AxesGrid = 'GridAxes3DActor'
+renderView1.CenterOfRotation = [8.0, 8.0, 4.0]
+renderView1.StereoType = 'Crystal Eyes'
+renderView1.CameraPosition = [8.038575486273551, 7.488142173440099, 14.569596590973248]
+renderView1.CameraFocalPoint = [8.008182364514674, 7.891428269059167, 6.241949497853552]
+renderView1.CameraViewUp = [-0.754397824803147, -0.6557763921833819, -0.029004230505694627]
+renderView1.CameraFocalDisk = 1.0
+renderView1.CameraParallelScale = 14.517231140957975
+renderView1.BackEnd = 'OSPRay raycaster'
+renderView1.OSPRayMaterialLibrary = materialLibrary1
+
+SetActiveView(None)
+
+# ----------------------------------------------------------------
+# setup view layouts
+# ----------------------------------------------------------------
+
+# create new layout object 'Layout #1'
+layout1 = CreateLayout(name='Layout #1')
+layout1.AssignView(0, renderView1)
+layout1.SetSize(885, 774)
+
+# ----------------------------------------------------------------
+# restore active view
+SetActiveView(renderView1)
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup the data processing pipelines
+# ----------------------------------------------------------------
+
+# create a new 'VisItChomboReader'
+input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000003.3d.hdf5'])
+input.MeshStatus = ['Mesh']
+input.CellArrayStatus = ['chi']
+
+# create a new 'Extract Level'
+extractLevel1 = ExtractLevel(registrationName='ExtractLevel1', Input=input)
+extractLevel1.Levels = [2]
+
+# create a new 'Resample To Image'
+resampleToImage1 = ResampleToImage(registrationName='ResampleToImage1', Input=extractLevel1)
+resampleToImage1.SamplingBounds = [3.75, 12.25, 5.75, 10.25, -0.25, 2.25]
+
+# ----------------------------------------------------------------
+# setup the visualization in view 'renderView1'
+# ----------------------------------------------------------------
+
+# show data from resampleToImage1
+resampleToImage1Display = Show(resampleToImage1, renderView1, 'UniformGridRepresentation')
+
+# get color transfer function/color map for 'chi'
+chiLUT = GetColorTransferFunction('chi')
+chiLUT.RGBPoints = [0.0, 0.0862745098039216, 0.00392156862745098, 0.298039215686275, 0.029344877158935277, 0.113725, 0.0235294, 0.45098, 0.0537159862131328, 0.105882, 0.0509804, 0.509804, 0.0706265851511459, 0.0392157, 0.0392157, 0.560784, 0.0870398318296482, 0.0313725, 0.0980392, 0.6, 0.10295566969276873, 0.0431373, 0.164706, 0.639216, 0.12583466541255697, 0.054902, 0.243137, 0.678431, 0.15617428436025385, 0.054902, 0.317647, 0.709804, 0.19347701615880114, 0.0509804, 0.396078, 0.741176, 0.2176616431786517, 0.0392157, 0.466667, 0.768627, 0.24184627019850216, 0.0313725, 0.537255, 0.788235, 0.267087853698452, 0.0313725, 0.615686, 0.811765, 0.2929510760870558, 0.0235294, 0.709804, 0.831373, 0.3188143550315277, 0.0509804, 0.8, 0.85098, 0.3402012374169067, 0.0705882, 0.854902, 0.870588, 0.36009606302373837, 0.262745, 0.901961, 0.862745, 0.3775040142212683, 0.423529, 0.941176, 0.87451, 0.4043619976847706, 0.572549, 0.964706, 0.835294, 0.42226735769767926, 0.658824, 0.980392, 0.843137, 0.43519896889198123, 0.764706, 0.980392, 0.866667, 0.4491253128832477, 0.827451, 0.980392, 0.886275, 0.47648067688419493, 0.913725, 0.988235, 0.937255, 0.48493597635320285, 1.0, 1.0, 0.972549019607843, 0.49339127582221104, 0.988235, 0.980392, 0.870588, 0.5058255347569962, 0.992156862745098, 0.972549019607843, 0.803921568627451, 0.5152755670229687, 0.992157, 0.964706, 0.713725, 0.5311914048860893, 0.988235, 0.956863, 0.643137, 0.5565572750151794, 0.980392, 0.917647, 0.509804, 0.5779442139564234, 0.968627, 0.87451, 0.407843, 0.5998285051571812, 0.94902, 0.823529, 0.321569, 0.615744343020301, 0.929412, 0.776471, 0.278431, 0.639120747555468, 0.909804, 0.717647, 0.235294, 0.6600102776813318, 0.890196, 0.658824, 0.196078, 0.6771695565558091, 0.878431, 0.619608, 0.168627, 0.7013541835756594, 0.870588, 0.54902, 0.156863, 0.7255388105955098, 0.85098, 0.47451, 0.145098, 0.7497234376153601, 0.831373, 0.411765, 0.133333, 0.7739080646352107, 0.811765, 0.345098, 0.113725, 0.798092691655061, 0.788235, 0.266667, 0.0941176, 0.8222773186749113, 0.741176, 0.184314, 0.0745098, 0.8464619456947616, 0.690196, 0.12549, 0.0627451, 0.870646572714612, 0.619608, 0.0627451, 0.0431373, 0.893276919519733, 0.54902, 0.027451, 0.0705882, 0.9131717283365444, 0.470588, 0.0156863, 0.0901961, 0.9355533868194685, 0.4, 0.00392157, 0.101961, 0.9673850807940133, 0.188235294117647, 0.0, 0.0705882352941176]
+chiLUT.ColorSpace = 'Lab'
+chiLUT.NanColor = [1.0, 0.0, 0.0]
+chiLUT.ScalarRangeInitialized = 1.0
+
+# get opacity transfer function/opacity map for 'chi'
+chiPWF = GetOpacityTransferFunction('chi')
+chiPWF.Points = [0.0, 1.0, 0.5, 0.0, 0.04865546501470074, 0.6094674468040466, 0.5, 0.0, 0.12020762722887196, 0.4378698170185089, 0.5, 0.0, 0.22324271380797622, 0.23076923191547394, 0.5, 0.0, 0.37207121721551956, 0.08875739574432373, 0.5, 0.0, 0.578141420384064, 0.0, 0.5, 0.0, 0.9673850807940133, 0.0, 0.5, 0.0]
+chiPWF.ScalarRangeInitialized = 1
+
+# trace defaults for the display properties.
+resampleToImage1Display.Representation = 'Volume'
+resampleToImage1Display.ColorArrayName = ['POINTS', 'chi']
+resampleToImage1Display.LookupTable = chiLUT
+resampleToImage1Display.SelectTCoordArray = 'None'
+resampleToImage1Display.SelectNormalArray = 'None'
+resampleToImage1Display.SelectTangentArray = 'None'
+resampleToImage1Display.OSPRayScaleArray = 'chi'
+resampleToImage1Display.OSPRayScaleFunction = 'PiecewiseFunction'
+resampleToImage1Display.SelectOrientationVectors = 'None'
+resampleToImage1Display.ScaleFactor = 0.84999915
+resampleToImage1Display.SelectScaleArray = 'None'
+resampleToImage1Display.GlyphType = 'Arrow'
+resampleToImage1Display.GlyphTableIndexArray = 'None'
+resampleToImage1Display.GaussianRadius = 0.042499957500000005
+resampleToImage1Display.SetScaleArray = ['POINTS', 'chi']
+resampleToImage1Display.ScaleTransferFunction = 'PiecewiseFunction'
+resampleToImage1Display.OpacityArray = ['POINTS', 'chi']
+resampleToImage1Display.OpacityTransferFunction = 'PiecewiseFunction'
+resampleToImage1Display.DataAxesGrid = 'GridAxesRepresentation'
+resampleToImage1Display.PolarAxes = 'PolarAxesRepresentation'
+resampleToImage1Display.ScalarOpacityUnitDistance = 0.10037670222093371
+resampleToImage1Display.ScalarOpacityFunction = chiPWF
+resampleToImage1Display.OpacityArrayName = ['POINTS', 'chi']
+resampleToImage1Display.SliceFunction = 'Plane'
+resampleToImage1Display.Slice = 49
+
+# init the 'PiecewiseFunction' selected for 'ScaleTransferFunction'
+resampleToImage1Display.ScaleTransferFunction.Points = [0.007669444988778632, 0.0, 0.5, 0.0, 0.6724393877650426, 1.0, 0.5, 0.0]
+
+# init the 'PiecewiseFunction' selected for 'OpacityTransferFunction'
+resampleToImage1Display.OpacityTransferFunction.Points = [0.007669444988778632, 0.0, 0.5, 0.0, 0.6724393877650426, 1.0, 0.5, 0.0]
+
+# init the 'Plane' selected for 'SliceFunction'
+resampleToImage1Display.SliceFunction.Origin = [8.0, 8.0, 1.0]
+
+# ----------------------------------------------------------------
+# setup color maps and opacity mapes used in the visualization
+# note: the Get..() functions create a new object, if needed
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup extractors
+# ----------------------------------------------------------------
+
+# create extractor
+pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
+# trace defaults for the extractor.
+# init the 'PNG' selected for 'Writer'
+pNG1.Writer.FileName = 'VolumeChi_%.6ts%cm.png'
+pNG1.Writer.ImageResolution = [885, 774]
+pNG1.Writer.Format = 'PNG'
+
+# ----------------------------------------------------------------
+# restore active source
+SetActiveSource(pNG1)
+# ----------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# Catalyst options
+from paraview import catalyst
+options = catalyst.Options()
+options.GlobalTrigger = 'TimeStep'
+options.EnableCatalystLive = 1
+options.CatalystLiveTrigger = 'TimeStep'
+
+# ------------------------------------------------------------------------------
+if __name__ == '__main__':
+    from paraview.simple import SaveExtractsUsingCatalystOptions
+    # Code for non in-situ environments; if executing in post-processing
+    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
+    SaveExtractsUsingCatalystOptions(options)

From 43dd07b35e583c225835b1518d5437f8362c51b9 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 22 Dec 2021 17:41:17 +0000
Subject: [PATCH 14/63] Attempt to fix building of VTK grid

Note that now the vtkAMRBox indices are (0,0,0) where the lower
corner of the ghosted domain on the coarsest level is e.g. (-3,-3,-3)
in Chombo indices.
In theory this might make it annoying to calculate coordinates, but
these should be calculated from the associated vtkUniformGrid objects
anyway which have their origin at the correct place (i.e. Chombo's
(0,0,0)).
This is necessary as VTK expects the "origin" of the vtkAMRBoxes to be
the lower bound.
---
 Source/Insitu/CatalystAdaptor.cpp | 79 ++++++++++++++++++++++++++++---
 1 file changed, 72 insertions(+), 7 deletions(-)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 7080b689e..8fc3645dc 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -116,14 +116,20 @@ void CatalystAdaptor::build_vtk_grid()
 
     // The origin is always at (0, 0, 0) in Chombo
     double origin_global[3] = {0., 0., 0.};
-    m_vtk_grid_ptr->SetOrigin(origin_global);
+    const IntVect &coarsest_ghost_vect =
+        gramrlevels[0]->getLevelData().ghostVect();
+    const double coarsest_dx = gramrlevels[0]->get_dx();
+    RealVect ghosted_origin_global_vect = -coarsest_dx * coarsest_ghost_vect;
+    m_vtk_grid_ptr->SetOrigin(ghosted_origin_global_vect.dataPtr());
 
     // now add all the boxes
     for (int ilevel = 0; ilevel < num_levels; ++ilevel)
     {
+        // pout() << "========================================\n";
+        // pout() << "Level: " << ilevel << std::endl;
         const GRAMRLevel *level = gramrlevels[ilevel];
         const double dx = level->get_dx();
-        const double dx_arr[3] = {dx, dx, dx};
+        double dx_arr[3] = {dx, dx, dx};
         m_vtk_grid_ptr->SetSpacing(ilevel, dx_arr);
         m_vtk_grid_ptr->SetRefinementRatio(ilevel, level->refRatio());
         const GRLevelData &level_data = level->getLevelData();
@@ -149,9 +155,12 @@ void CatalystAdaptor::build_vtk_grid()
             double origin[3] = {dx_arr[0] * small_ghosted_end[0],
                                 dx_arr[1] * small_ghosted_end[1],
                                 dx_arr[2] * small_ghosted_end[2]};
+            // RealVect level_origin = -dx * level_data.ghostVect();
 
             vtkAMRBox vtk_amr_box(origin, ghosted_box.size().dataPtr(), dx_arr,
-                                  origin_global);
+                                  ghosted_origin_global_vect.dataPtr());
+            // vtk_amr_box.Print(pout());
+            // pout() << "\n";
             m_vtk_grid_ptr->SetAMRBox(ilevel, ibox, vtk_amr_box);
 
             bool local_box = (procID() == level_box_layout.procID(lit()));
@@ -159,6 +168,7 @@ void CatalystAdaptor::build_vtk_grid()
             if (local_box)
             {
                 vtkNew<vtkUniformGrid> vtk_uniform_grid_ptr;
+                /*
                 vtk_uniform_grid_ptr->SetOrigin(origin_global);
                 vtk_uniform_grid_ptr->SetSpacing(dx_arr);
                 vtk_uniform_grid_ptr->SetExtent(
@@ -171,9 +181,19 @@ void CatalystAdaptor::build_vtk_grid()
                                    small_end[2], big_end[2] + 1};
                 bool cell_data = true;
                 vtk_uniform_grid_ptr->GenerateGhostArray(no_ghost, cell_data);
+                */
+
+                vtk_uniform_grid_ptr->Initialize(
+                    &vtk_amr_box, origin_global, dx_arr,
+                    level_data.ghostVect().dataPtr());
                 m_vtk_grid_ptr->SetDataSet(ilevel, ibox, vtk_uniform_grid_ptr);
             }
+            else
+            {
+                m_vtk_grid_ptr->SetDataSet(ilevel, ibox, nullptr);
+            }
         }
+        pout() << std::endl;
     }
 
     m_vtk_grid_ptr->Audit();
@@ -191,7 +211,7 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
     std::array<bool, NUM_VARS> requested_evolution_vars;
     std::array<bool, NUM_DIAGNOSTIC_VARS> requested_diagnostic_vars;
 
-    if (m_verbosity > 1)
+    if (m_verbosity)
     {
         pout() << "CatalystAdaptor Requested variables:\n";
     }
@@ -200,7 +220,7 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
     {
         requested_evolution_vars[ivar] = a_input_data_desc->IsFieldNeeded(
             UserVariables::variable_names[ivar].c_str(), vtkDataObject::CELL);
-        if (m_verbosity > 1 && requested_evolution_vars[ivar])
+        if (m_verbosity && requested_evolution_vars[ivar])
             pout() << UserVariables::variable_names[ivar] << " ";
     }
     for (int ivar = 0; ivar < NUM_DIAGNOSTIC_VARS; ++ivar)
@@ -208,10 +228,10 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
         requested_diagnostic_vars[ivar] = a_input_data_desc->IsFieldNeeded(
             DiagnosticVariables::variable_names[ivar].c_str(),
             vtkDataObject::CELL);
-        if (m_verbosity > 1 && requested_diagnostic_vars[ivar])
+        if (m_verbosity && requested_diagnostic_vars[ivar])
             pout() << DiagnosticVariables::variable_names[ivar] << " ";
     }
-    if (m_verbosity > 1)
+    if (m_verbosity)
         pout() << std::endl;
 
     vtkAMRInformation *amr_info = m_vtk_grid_ptr->GetAMRInfo();
@@ -239,11 +259,54 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
             {
                 vtkUniformGrid *vtk_uniform_grid_ptr =
                     m_vtk_grid_ptr->GetDataSet(ilevel, ibox);
+
                 // hopefully this promotion works
                 DataIndex dind(lit());
                 FArrayBox &evolution_fab = evolution_level_data[dind];
                 FArrayBox &diagnostic_fab = diagnostic_level_data[dind];
 
+#if DEBUG
+                vtkAMRBox vtk_box = m_vtk_grid_ptr->GetAMRBox(ilevel, ibox);
+                // shift to account for different indexing
+                IntVect vtk_origin_offset =
+                    -ipow(2, ilevel) * evolution_level_data.ghostVect();
+                vtk_box.Shift(vtk_origin_offset.dataPtr());
+                const int *vtk_box_lo = vtk_box.GetLoCorner();
+                const int *vtk_box_hi = vtk_box.GetHiCorner();
+
+                const Box &evolution_box = evolution_fab.box();
+                const IntVect &evolution_box_lo = evolution_box.smallEnd();
+                const IntVect &evolution_box_hi = evolution_box.bigEnd();
+
+                const Box &diagnostic_box = diagnostic_fab.box();
+                const IntVect &diagnostic_box_lo = diagnostic_box.smallEnd();
+                const IntVect &diagnostic_box_hi = diagnostic_box.bigEnd();
+                // add 1 to hi corner of VTK box as this is what is meant by hi
+                // corner in VTK
+                bool all_same_boxes =
+                    (vtk_box_lo[0] == evolution_box_lo[0]) &&
+                    (vtk_box_lo[1] == evolution_box_lo[1]) &&
+                    (vtk_box_lo[2] == evolution_box_lo[2]) &&
+                    (vtk_box_hi[0] + 1 == evolution_box_hi[0]) &&
+                    (vtk_box_hi[1] + 1 == evolution_box_hi[1]) &&
+                    (vtk_box_hi[2] + 1 == evolution_box_hi[2]) &&
+                    (vtk_box_lo[0] == diagnostic_box_lo[0]) &&
+                    (vtk_box_lo[1] == diagnostic_box_lo[1]) &&
+                    (vtk_box_lo[2] == diagnostic_box_lo[2]) &&
+                    (vtk_box_hi[0] + 1 == diagnostic_box_hi[0]) &&
+                    (vtk_box_hi[1] + 1 == diagnostic_box_hi[1]) &&
+                    (vtk_box_hi[2] + 1 == diagnostic_box_hi[2]);
+                if (!all_same_boxes)
+                {
+                    pout() << "Boxes do not agree: \n";
+                    pout() << "vtk_box: ";
+                    vtk_box.Print(pout());
+                    pout() << "\n";
+                    pout() << "evolution_box: " << evolution_box << "\n";
+                    pout() << "diagnostic_box: " << diagnostic_box << std::endl;
+                }
+#endif
+
                 for (int ivar = 0; ivar < NUM_VARS; ++ivar)
                 {
                     if (requested_evolution_vars[ivar])
@@ -286,6 +349,8 @@ void CatalystAdaptor::coprocess(double a_time, unsigned int a_timestep)
         auto input_data_description =
             data_description->GetInputDescriptionByName("input");
         add_vars(input_data_description);
+        // vtkNew<vtkOverlappingAMR> stripped_vtk_grid;
+        // vtkAMRUtilities::StripGhostLayers(m_vtk_grid_ptr, stripped_vtk_grid);
         input_data_description->SetGrid(m_vtk_grid_ptr);
         m_proc_ptr->CoProcess(data_description);
     }

From 15740031ea2aa175cb5eb69c79ebe1cbaad29c7b Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Thu, 23 Dec 2021 17:44:19 +0000
Subject: [PATCH 15/63] Refactor calling of Catalyst CoProcess

Also change back to the old construction of vtkUniformGrids. I think the
one that used the vtkAMRBoxes assumed node centering leading to the
wrong dimensions.
---
 Examples/BinaryBH/Main_BinaryBH.cpp |  8 ++++-
 Source/GRChomboCore/GRAMRLevel.cpp  | 47 ++++++++++++++++++-----------
 Source/GRChomboCore/GRAMRLevel.hpp  |  3 ++
 Source/Insitu/CatalystAdaptor.cpp   | 37 ++++++++++++-----------
 4 files changed, 59 insertions(+), 36 deletions(-)

diff --git a/Examples/BinaryBH/Main_BinaryBH.cpp b/Examples/BinaryBH/Main_BinaryBH.cpp
index 120dec3a8..0d9a6fbdd 100644
--- a/Examples/BinaryBH/Main_BinaryBH.cpp
+++ b/Examples/BinaryBH/Main_BinaryBH.cpp
@@ -81,11 +81,17 @@ int runGRChombo(int argc, char *argv[])
 
     std::chrono::time_point<Clock> start_time = Clock::now();
 
-    // Add a scheduler to call specificPostTimeStep on every AMRLevel at t=0
+    // Add a scheduler to call specificPostTimeStep and catalystCoProcess
+    // on every AMRLevel at t=0
     auto task = [](GRAMRLevel *level)
     {
         if (level->time() == 0.)
+        {
             level->specificPostTimeStep();
+#ifdef USE_CATALYST
+            level->catalystCoProcess();
+#endif
+        }
     };
     // call 'now' really now
     MultiLevelTaskPtr<> call_task(task);
diff --git a/Source/GRChomboCore/GRAMRLevel.cpp b/Source/GRChomboCore/GRAMRLevel.cpp
index fe5b72210..f88212927 100644
--- a/Source/GRChomboCore/GRAMRLevel.cpp
+++ b/Source/GRChomboCore/GRAMRLevel.cpp
@@ -179,24 +179,7 @@ void GRAMRLevel::postTimeStep()
     specificPostTimeStep();
 
 #ifdef USE_CATALYST
-    if (m_p.activate_catalyst && m_level == m_p.catalyst_coprocess_level)
-    {
-        if (m_verbosity)
-        {
-            pout() << "GRAMRLevel::postTimestep: Calling Catalyst CoProcess"
-                   << std::endl;
-        }
-
-        unsigned int level_timestep = std::round(m_time / m_dt);
-        m_gr_amr.m_insitu->coprocess(m_time, level_timestep);
-
-        if (m_verbosity)
-        {
-            pout() << "GRAMRLevel::postTimestep: Calling Catalyst CoProcess "
-                      "finished"
-                   << std::endl;
-        }
-    }
+    catalystCoProcess();
 #endif
 
     // enforce solution BCs - this is required after the averaging
@@ -207,6 +190,34 @@ void GRAMRLevel::postTimeStep()
         pout() << "GRAMRLevel::postTimeStep " << m_level << " finished" << endl;
 }
 
+#ifdef USE_CATALYST
+void GRAMRLevel::catalystCoProcess()
+{
+    if (m_p.activate_catalyst)
+    {
+        if (at_level_timestep_multiple(m_p.catalyst_coprocess_level))
+        {
+            preCatalystCoProcess();
+        }
+        if (m_level == m_p.catalyst_coprocess_level)
+        {
+            if (m_verbosity)
+            {
+                pout() << "GRAMRLevel::catalystCoProcess" << std::endl;
+            }
+
+            unsigned int level_timestep = std::round(m_time / m_dt);
+            m_gr_amr.m_insitu->coprocess(m_time, level_timestep);
+
+            if (m_verbosity)
+            {
+                pout() << "GRAMRLevel::catalystCoProcess finished" << std::endl;
+            }
+        }
+    }
+}
+#endif
+
 // for examples that don't implement a computeTaggingCriterion with diagnostic
 // variables
 void GRAMRLevel::computeTaggingCriterion(
diff --git a/Source/GRChomboCore/GRAMRLevel.hpp b/Source/GRChomboCore/GRAMRLevel.hpp
index 2e7a9534a..1485f25cb 100644
--- a/Source/GRChomboCore/GRAMRLevel.hpp
+++ b/Source/GRChomboCore/GRAMRLevel.hpp
@@ -189,6 +189,9 @@ class GRAMRLevel : public AMRLevel, public InterpSource
         const Interval &a_comps = Interval(0, std::numeric_limits<int>::max()));
 
 #ifdef USE_CATALYST
+    /// Calls Catalyst CoProcess
+    void catalystCoProcess();
+
     virtual void preCatalystCoProcess() {}
 #endif
 
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 8fc3645dc..149a8d91c 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -168,7 +168,7 @@ void CatalystAdaptor::build_vtk_grid()
             if (local_box)
             {
                 vtkNew<vtkUniformGrid> vtk_uniform_grid_ptr;
-                /*
+
                 vtk_uniform_grid_ptr->SetOrigin(origin_global);
                 vtk_uniform_grid_ptr->SetSpacing(dx_arr);
                 vtk_uniform_grid_ptr->SetExtent(
@@ -176,29 +176,32 @@ void CatalystAdaptor::build_vtk_grid()
                     small_ghosted_end[1], big_ghosted_end[1] + 1,
                     small_ghosted_end[2], big_ghosted_end[2] + 1);
                 // add the ghost cell information
-                int no_ghost[6] = {small_end[0], big_end[0] + 1,
-                                   small_end[1], big_end[1] + 1,
-                                   small_end[2], big_end[2] + 1};
-                bool cell_data = true;
-                vtk_uniform_grid_ptr->GenerateGhostArray(no_ghost, cell_data);
-                */
-
-                vtk_uniform_grid_ptr->Initialize(
-                    &vtk_amr_box, origin_global, dx_arr,
-                    level_data.ghostVect().dataPtr());
+                // int no_ghost[6] = {small_end[0], big_end[0] + 1,
+                //    small_end[1], big_end[1] + 1,
+                //    small_end[2], big_end[2] + 1};
+                // bool cell_data = true;
+                // vtk_uniform_grid_ptr->GenerateGhostArray(no_ghost,
+                // cell_data);
+
+                // vtk_uniform_grid_ptr->Initialize(
+                // &vtk_amr_box, origin_global, dx_arr,
+                // level_data.ghostVect().dataPtr());
+
                 m_vtk_grid_ptr->SetDataSet(ilevel, ibox, vtk_uniform_grid_ptr);
             }
-            else
-            {
-                m_vtk_grid_ptr->SetDataSet(ilevel, ibox, nullptr);
-            }
         }
-        pout() << std::endl;
     }
 
     m_vtk_grid_ptr->Audit();
+    const double *vtk_grid_bounds = m_vtk_grid_ptr->GetAMRInfo()->GetBounds();
+    pout() << "VTK Grid Bounds:"
+           << "(" << vtk_grid_bounds[0] << "," << vtk_grid_bounds[2] << ","
+           << vtk_grid_bounds[4] << "), (" << vtk_grid_bounds[1] << ","
+           << vtk_grid_bounds[3] << "," << vtk_grid_bounds[5] << ")"
+           << std::endl;
+
     // not sure if this is necessary but it was on the OverlappingAMR example
-    // m_vtk_grid_ptr->GenerateParentChildInformation();
+    m_vtk_grid_ptr->GenerateParentChildInformation();
 }
 
 void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)

From f88ce83e1c8b4852c62d1c036aa1f58d58cdba58 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Thu, 23 Dec 2021 17:47:31 +0000
Subject: [PATCH 16/63] Modify Catalyst Scripts

These have been generated from more representative data of the current
parameter file.
---
 .../BinaryBH/catalyst_scripts/slice_chi.py    | 41 +++++++++++--------
 .../catalyst_scripts/volume_noraytrace_chi.py | 40 +++++++++---------
 Examples/BinaryBH/params_catalyst.txt         |  4 +-
 3 files changed, 45 insertions(+), 40 deletions(-)

diff --git a/Examples/BinaryBH/catalyst_scripts/slice_chi.py b/Examples/BinaryBH/catalyst_scripts/slice_chi.py
index 49c369954..dd0a4ab22 100644
--- a/Examples/BinaryBH/catalyst_scripts/slice_chi.py
+++ b/Examples/BinaryBH/catalyst_scripts/slice_chi.py
@@ -15,18 +15,22 @@
 
 # Create a new 'Render View'
 renderView1 = CreateView('RenderView')
-renderView1.ViewSize = [1131, 939]
+renderView1.ViewSize = [885, 774]
 renderView1.AxesGrid = 'GridAxes3DActor'
+renderView1.CenterAxesVisibility = 1
 renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
 renderView1.StereoType = 'Crystal Eyes'
-renderView1.CameraPosition = [-24.234943738007217, 12.79030062375982, 77.14896782254773]
-renderView1.CameraFocalPoint = [31.99999999999993, 31.99999999999998, 31.99999999999996]
-renderView1.CameraViewUp = [0.6572599982077688, -0.31659445000046216, 0.6839424310457926]
+renderView1.CameraPosition = [32.0, 32.0, 223.24355652982143]
+renderView1.CameraFocalPoint = [32.0, 32.0, 32.0]
 renderView1.CameraFocalDisk = 1.0
-renderView1.CameraParallelScale = 60.6217782649107
+renderView1.CameraParallelScale = 49.49747468305833
+renderView1.CameraParallelProjection = 1
 renderView1.BackEnd = 'OSPRay raycaster'
 renderView1.OSPRayMaterialLibrary = materialLibrary1
 
+# init the 'GridAxes3DActor' selected for 'AxesGrid'
+renderView1.AxesGrid.Visibility = 1
+
 SetActiveView(None)
 
 # ----------------------------------------------------------------
@@ -36,7 +40,7 @@
 # create new layout object 'Layout #1'
 layout1 = CreateLayout(name='Layout #1')
 layout1.AssignView(0, renderView1)
-layout1.SetSize(1131, 939)
+layout1.SetSize(885, 774)
 
 # ----------------------------------------------------------------
 # restore active view
@@ -48,13 +52,13 @@
 # ----------------------------------------------------------------
 
 # create a new 'VisItChomboReader'
-binaryBH_chi_ghosts3dhdf5 = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_chi_ghosts.3d.hdf5'])
-binaryBH_chi_ghosts3dhdf5.MeshStatus = ['Mesh']
-binaryBH_chi_ghosts3dhdf5.CellArrayStatus = ['chi']
+input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000003.3d.hdf5'])
+input.MeshStatus = ['Mesh']
+input.CellArrayStatus = ['chi']
 
 # create a new 'Slice AMR data'
-sliceAMRdata1 = SliceAMRdata(registrationName='SliceAMRdata1', Input=binaryBH_chi_ghosts3dhdf5)
-sliceAMRdata1.Level = 2
+sliceAMRdata1 = SliceAMRdata(registrationName='SliceAMRdata1', Input=input)
+sliceAMRdata1.Level = 3
 sliceAMRdata1.OffSet = 35.0
 sliceAMRdata1.Normal = 'Z-Normal'
 
@@ -67,12 +71,14 @@
 
 # get color transfer function/color map for 'chi'
 chiLUT = GetColorTransferFunction('chi')
-chiLUT.RGBPoints = [4.8189884420197785e-05, 0.231373, 0.298039, 0.752941, 0.4837166353392167, 0.865003, 0.865003, 0.865003, 0.9673850807940133, 0.705882, 0.0156863, 0.14902]
+chiLUT.AutomaticRescaleRangeMode = 'Never'
+chiLUT.RGBPoints = [0.0, 0.267004, 0.004874, 0.329415, 0.003921999999999982, 0.26851, 0.009605, 0.335427, 0.007843000000000046, 0.269944, 0.014625, 0.341379, 0.011765000000000027, 0.271305, 0.019942, 0.347269, 0.01568599999999998, 0.272594, 0.025563, 0.353093, 0.019607999999999962, 0.273809, 0.031497, 0.358853, 0.02352900000000003, 0.274952, 0.037752, 0.364543, 0.027451000000000014, 0.276022, 0.044167, 0.370164, 0.03137299999999999, 0.277018, 0.050344, 0.375715, 0.035294000000000054, 0.277941, 0.056324, 0.381191, 0.039216000000000036, 0.278791, 0.062145, 0.386592, 0.04313699999999999, 0.279566, 0.067836, 0.391917, 0.04705899999999996, 0.280267, 0.073417, 0.397163, 0.05098000000000004, 0.280894, 0.078907, 0.402329, 0.05490200000000001, 0.281446, 0.08432, 0.407414, 0.05882399999999999, 0.281924, 0.089666, 0.412415, 0.06274499999999995, 0.282327, 0.094955, 0.417331, 0.06666700000000005, 0.282656, 0.100196, 0.42216, 0.07058799999999998, 0.28291, 0.105393, 0.426902, 0.07450999999999998, 0.283091, 0.110553, 0.431554, 0.07843100000000004, 0.283197, 0.11568, 0.436115, 0.0823529999999999, 0.283229, 0.120777, 0.440584, 0.08627499999999999, 0.283187, 0.125848, 0.44496, 0.09019599999999996, 0.283072, 0.130895, 0.449241, 0.09411799999999991, 0.282884, 0.13592, 0.453427, 0.098039, 0.282623, 0.140926, 0.457517, 0.10196099999999998, 0.28229, 0.145912, 0.46151, 0.10588199999999992, 0.281887, 0.150881, 0.465405, 0.10980400000000001, 0.281412, 0.155834, 0.469201, 0.11372499999999998, 0.280868, 0.160771, 0.472899, 0.11764699999999993, 0.280255, 0.165693, 0.476498, 0.12156899999999993, 0.279574, 0.170599, 0.479997, 0.12549, 0.278826, 0.17549, 0.483397, 0.12941199999999997, 0.278012, 0.180367, 0.486697, 0.13333300000000006, 0.277134, 0.185228, 0.489898, 0.13725500000000004, 0.276194, 0.190074, 0.493001, 0.141176, 0.275191, 0.194905, 0.496005, 0.14509799999999998, 0.274128, 0.199721, 0.498911, 0.14902000000000007, 0.273006, 0.20452, 0.501721, 0.15294100000000002, 0.271828, 0.209303, 0.504434, 0.1568630000000001, 0.270595, 0.214069, 0.507052, 0.1607840000000001, 0.269308, 0.218818, 0.509577, 0.16470600000000007, 0.267968, 0.223549, 0.512008, 0.168627, 0.26658, 0.228262, 0.514349, 0.17254900000000012, 0.265145, 0.232956, 0.516599, 0.1764710000000001, 0.263663, 0.237631, 0.518762, 0.18039200000000005, 0.262138, 0.242286, 0.520837, 0.1843140000000001, 0.260571, 0.246922, 0.522828, 0.18823500000000007, 0.258965, 0.251537, 0.524736, 0.19215700000000005, 0.257322, 0.25613, 0.526563, 0.1960780000000001, 0.255645, 0.260703, 0.528312, 0.2000000000000001, 0.253935, 0.265254, 0.529983, 0.20392200000000008, 0.252194, 0.269783, 0.531579, 0.20784300000000003, 0.250425, 0.27429, 0.533103, 0.211765, 0.248629, 0.278775, 0.534556, 0.21568599999999996, 0.246811, 0.283237, 0.535941, 0.21960800000000005, 0.244972, 0.287675, 0.53726, 0.22352900000000014, 0.243113, 0.292092, 0.538516, 0.22745099999999996, 0.241237, 0.296485, 0.539709, 0.23137300000000008, 0.239346, 0.300855, 0.540844, 0.23529400000000006, 0.237441, 0.305202, 0.541921, 0.23921600000000004, 0.235526, 0.309527, 0.542944, 0.24313700000000008, 0.233603, 0.313828, 0.543914, 0.24705900000000008, 0.231674, 0.318106, 0.544834, 0.25098000000000004, 0.229739, 0.322361, 0.545706, 0.254902, 0.227802, 0.326594, 0.546532, 0.258824, 0.225863, 0.330805, 0.547314, 0.2627450000000001, 0.223925, 0.334994, 0.548053, 0.26666700000000004, 0.221989, 0.339161, 0.548752, 0.27058800000000016, 0.220057, 0.343307, 0.549413, 0.27451, 0.21813, 0.347432, 0.550038, 0.27843100000000004, 0.21621, 0.351535, 0.550627, 0.282353, 0.214298, 0.355619, 0.551184, 0.2862750000000001, 0.212395, 0.359683, 0.55171, 0.29019600000000007, 0.210503, 0.363727, 0.552206, 0.29411800000000016, 0.208623, 0.367752, 0.552675, 0.298039, 0.206756, 0.371758, 0.553117, 0.3019610000000001, 0.204903, 0.375746, 0.553533, 0.30588200000000004, 0.203063, 0.379716, 0.553925, 0.30980400000000013, 0.201239, 0.38367, 0.554294, 0.3137249999999999, 0.19943, 0.387607, 0.554642, 0.31764700000000007, 0.197636, 0.391528, 0.554969, 0.32156900000000005, 0.19586, 0.395433, 0.555276, 0.32549, 0.1941, 0.399323, 0.555565, 0.329412, 0.192357, 0.403199, 0.555836, 0.33333300000000005, 0.190631, 0.407061, 0.556089, 0.3372550000000002, 0.188923, 0.41091, 0.556326, 0.341176, 0.187231, 0.414746, 0.556547, 0.3450980000000001, 0.185556, 0.41857, 0.556753, 0.34902000000000005, 0.183898, 0.422383, 0.556944, 0.352941, 0.182256, 0.426184, 0.55712, 0.356863, 0.180629, 0.429975, 0.557282, 0.36078400000000005, 0.179019, 0.433756, 0.55743, 0.36470600000000003, 0.177423, 0.437527, 0.557565, 0.368627, 0.175841, 0.44129, 0.557685, 0.3725490000000001, 0.174274, 0.445044, 0.557792, 0.37647100000000006, 0.172719, 0.448791, 0.557885, 0.380392, 0.171176, 0.45253, 0.557965, 0.384314, 0.169646, 0.456262, 0.55803, 0.3882350000000001, 0.168126, 0.459988, 0.558082, 0.39215700000000003, 0.166617, 0.463708, 0.558119, 0.39607800000000015, 0.165117, 0.467423, 0.558141, 0.4000000000000001, 0.163625, 0.471133, 0.558148, 0.4039220000000001, 0.162142, 0.474838, 0.55814, 0.407843, 0.160665, 0.47854, 0.558115, 0.41176500000000016, 0.159194, 0.482237, 0.558073, 0.41568600000000006, 0.157729, 0.485932, 0.558013, 0.4196080000000001, 0.15627, 0.489624, 0.557936, 0.423529, 0.154815, 0.493313, 0.55784, 0.4274510000000001, 0.153364, 0.497, 0.557724, 0.43137300000000006, 0.151918, 0.500685, 0.557587, 0.435294, 0.150476, 0.504369, 0.55743, 0.4392160000000001, 0.149039, 0.508051, 0.55725, 0.44313700000000006, 0.147607, 0.511733, 0.557049, 0.44705900000000004, 0.14618, 0.515413, 0.556823, 0.45098, 0.144759, 0.519093, 0.556572, 0.4549019999999999, 0.143343, 0.522773, 0.556295, 0.45882400000000007, 0.141935, 0.526453, 0.555991, 0.4627450000000002, 0.140536, 0.530132, 0.555659, 0.466667, 0.139147, 0.533812, 0.555298, 0.4705880000000001, 0.13777, 0.537492, 0.554906, 0.47451000000000004, 0.136408, 0.541173, 0.554483, 0.4784310000000001, 0.135066, 0.544853, 0.554029, 0.4823530000000001, 0.133743, 0.548535, 0.553541, 0.4862750000000001, 0.132444, 0.552216, 0.553018, 0.490196, 0.131172, 0.555899, 0.552459, 0.4941180000000001, 0.129933, 0.559582, 0.551864, 0.49803900000000006, 0.128729, 0.563265, 0.551229, 0.501961, 0.127568, 0.566949, 0.550556, 0.505882, 0.126453, 0.570633, 0.549841, 0.509804, 0.125394, 0.574318, 0.549086, 0.5137250000000001, 0.124395, 0.578002, 0.548287, 0.5176470000000002, 0.123463, 0.581687, 0.547445, 0.5215690000000001, 0.122606, 0.585371, 0.546557, 0.5254900000000002, 0.121831, 0.589055, 0.545623, 0.5294120000000004, 0.121148, 0.592739, 0.544641, 0.5333330000000002, 0.120565, 0.596422, 0.543611, 0.5372550000000001, 0.120092, 0.600104, 0.54253, 0.5411760000000003, 0.119738, 0.603785, 0.5414, 0.5450980000000003, 0.119512, 0.607464, 0.540218, 0.5490200000000003, 0.119423, 0.611141, 0.538982, 0.552941, 0.119483, 0.614817, 0.537692, 0.5568630000000002, 0.119699, 0.61849, 0.536347, 0.5607840000000002, 0.120081, 0.622161, 0.534946, 0.5647060000000002, 0.120638, 0.625828, 0.533488, 0.5686270000000001, 0.12138, 0.629492, 0.531973, 0.5725490000000001, 0.122312, 0.633153, 0.530398, 0.5764710000000003, 0.123444, 0.636809, 0.528763, 0.5803920000000002, 0.12478, 0.640461, 0.527068, 0.5843140000000001, 0.126326, 0.644107, 0.525311, 0.5882350000000002, 0.128087, 0.647749, 0.523491, 0.5921570000000003, 0.130067, 0.651384, 0.521608, 0.5960780000000001, 0.132268, 0.655014, 0.519661, 0.6, 0.134692, 0.658636, 0.517649, 0.6039220000000002, 0.137339, 0.662252, 0.515571, 0.6078430000000002, 0.14021, 0.665859, 0.513427, 0.6117650000000003, 0.143303, 0.669459, 0.511215, 0.6156860000000001, 0.146616, 0.67305, 0.508936, 0.6196080000000003, 0.150148, 0.676631, 0.506589, 0.6235290000000002, 0.153894, 0.680203, 0.504172, 0.6274510000000002, 0.157851, 0.683765, 0.501686, 0.6313730000000002, 0.162016, 0.687316, 0.499129, 0.6352940000000001, 0.166383, 0.690856, 0.496502, 0.6392160000000003, 0.170948, 0.694384, 0.493803, 0.6431370000000001, 0.175707, 0.6979, 0.491033, 0.6470590000000003, 0.180653, 0.701402, 0.488189, 0.6509800000000002, 0.185783, 0.704891, 0.485273, 0.6549020000000002, 0.19109, 0.708366, 0.482284, 0.6588240000000003, 0.196571, 0.711827, 0.479221, 0.6627450000000001, 0.202219, 0.715272, 0.476084, 0.6666670000000003, 0.20803, 0.718701, 0.472873, 0.6705880000000001, 0.214, 0.722114, 0.469588, 0.6745100000000003, 0.220124, 0.725509, 0.466226, 0.6784310000000002, 0.226397, 0.728888, 0.462789, 0.6823530000000004, 0.232815, 0.732247, 0.459277, 0.6862750000000001, 0.239374, 0.735588, 0.455688, 0.6901960000000003, 0.24607, 0.73891, 0.452024, 0.6941180000000002, 0.252899, 0.742211, 0.448284, 0.6980390000000002, 0.259857, 0.745492, 0.444467, 0.7019610000000002, 0.266941, 0.748751, 0.440573, 0.7058820000000001, 0.274149, 0.751988, 0.436601, 0.7098040000000001, 0.281477, 0.755203, 0.432552, 0.7137250000000003, 0.288921, 0.758394, 0.428426, 0.7176470000000004, 0.296479, 0.761561, 0.424223, 0.7215690000000002, 0.304148, 0.764704, 0.419943, 0.7254900000000003, 0.311925, 0.767822, 0.415586, 0.7294120000000002, 0.319809, 0.770914, 0.411152, 0.7333330000000003, 0.327796, 0.77398, 0.40664, 0.7372550000000001, 0.335885, 0.777018, 0.402049, 0.7411760000000003, 0.344074, 0.780029, 0.397381, 0.7450980000000003, 0.35236, 0.783011, 0.392636, 0.7490200000000001, 0.360741, 0.785964, 0.387814, 0.7529410000000001, 0.369214, 0.788888, 0.382914, 0.7568630000000001, 0.377779, 0.791781, 0.377939, 0.7607840000000001, 0.386433, 0.794644, 0.372886, 0.7647060000000001, 0.395174, 0.797475, 0.367757, 0.768627, 0.404001, 0.800275, 0.362552, 0.7725490000000002, 0.412913, 0.803041, 0.357269, 0.7764710000000001, 0.421908, 0.805774, 0.35191, 0.7803920000000001, 0.430983, 0.808473, 0.346476, 0.7843140000000001, 0.440137, 0.811138, 0.340967, 0.7882350000000002, 0.449368, 0.813768, 0.335384, 0.7921570000000001, 0.458674, 0.816363, 0.329727, 0.7960780000000001, 0.468053, 0.818921, 0.323998, 0.8000000000000002, 0.477504, 0.821444, 0.318195, 0.803922, 0.487026, 0.823929, 0.312321, 0.8078430000000002, 0.496615, 0.826376, 0.306377, 0.8117650000000002, 0.506271, 0.828786, 0.300362, 0.8156860000000001, 0.515992, 0.831158, 0.294279, 0.819608, 0.525776, 0.833491, 0.288127, 0.8235290000000002, 0.535621, 0.835785, 0.281908, 0.8274510000000003, 0.545524, 0.838039, 0.275626, 0.8313730000000001, 0.555484, 0.840254, 0.269281, 0.835294, 0.565498, 0.84243, 0.262877, 0.8392160000000001, 0.575563, 0.844566, 0.256415, 0.8431370000000002, 0.585678, 0.846661, 0.249897, 0.8470590000000001, 0.595839, 0.848717, 0.243329, 0.8509800000000001, 0.606045, 0.850733, 0.236712, 0.8549020000000003, 0.616293, 0.852709, 0.230052, 0.8588240000000003, 0.626579, 0.854645, 0.223353, 0.8627450000000001, 0.636902, 0.856542, 0.21662, 0.866667, 0.647257, 0.8584, 0.209861, 0.8705880000000001, 0.657642, 0.860219, 0.203082, 0.8745100000000002, 0.668054, 0.861999, 0.196293, 0.8784310000000002, 0.678489, 0.863742, 0.189503, 0.8823530000000003, 0.688944, 0.865448, 0.182725, 0.886275, 0.699415, 0.867117, 0.175971, 0.8901960000000002, 0.709898, 0.868751, 0.169257, 0.8941180000000002, 0.720391, 0.87035, 0.162603, 0.8980390000000001, 0.730889, 0.871916, 0.156029, 0.901961, 0.741388, 0.873449, 0.149561, 0.9058820000000002, 0.751884, 0.874951, 0.143228, 0.9098040000000002, 0.762373, 0.876424, 0.137064, 0.9137250000000001, 0.772852, 0.877868, 0.131109, 0.9176470000000002, 0.783315, 0.879285, 0.125405, 0.9215690000000001, 0.79376, 0.880678, 0.120005, 0.9254900000000004, 0.804182, 0.882046, 0.114965, 0.9294120000000001, 0.814576, 0.883393, 0.110347, 0.9333330000000003, 0.82494, 0.88472, 0.106217, 0.9372550000000002, 0.83527, 0.886029, 0.102646, 0.9411760000000003, 0.845561, 0.887322, 0.099702, 0.945098, 0.85581, 0.888601, 0.097452, 0.94902, 0.866013, 0.889868, 0.095953, 0.9529410000000003, 0.876168, 0.891125, 0.09525, 0.9568630000000002, 0.886271, 0.892374, 0.095374, 0.960784, 0.89632, 0.893616, 0.096335, 0.964706, 0.906311, 0.894855, 0.098125, 0.9686270000000003, 0.916242, 0.896091, 0.100717, 0.9725490000000003, 0.926106, 0.89733, 0.104071, 0.9764710000000002, 0.935904, 0.89857, 0.108131, 0.980392, 0.945636, 0.899815, 0.112838, 0.984314, 0.9553, 0.901065, 0.118128, 0.9882350000000002, 0.964894, 0.902323, 0.123941, 0.9921570000000003, 0.974417, 0.90359, 0.130215, 0.996078, 0.983868, 0.904867, 0.136897, 1.0, 0.993248, 0.906157, 0.143936]
+chiLUT.NanColor = [1.0, 0.0, 0.0]
 chiLUT.ScalarRangeInitialized = 1.0
 
 # get opacity transfer function/opacity map for 'chi'
 chiPWF = GetOpacityTransferFunction('chi')
-chiPWF.Points = [4.8189884420197785e-05, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
+chiPWF.Points = [0.0, 1.0, 0.5, 0.0, 1.0, 1.0, 0.5, 0.0]
 chiPWF.ScalarRangeInitialized = 1
 
 # trace defaults for the display properties.
@@ -95,7 +101,7 @@
 sliceAMRdata1Display.OpacityTransferFunction = 'PiecewiseFunction'
 sliceAMRdata1Display.DataAxesGrid = 'GridAxesRepresentation'
 sliceAMRdata1Display.PolarAxes = 'PolarAxesRepresentation'
-sliceAMRdata1Display.ScalarOpacityUnitDistance = 1.7175096404638197
+sliceAMRdata1Display.ScalarOpacityUnitDistance = 2.3023915422888135
 sliceAMRdata1Display.ScalarOpacityFunction = chiPWF
 
 # setup the color legend parameters for each legend in this view
@@ -124,13 +130,14 @@
 pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
 # trace defaults for the extractor.
 # init the 'PNG' selected for 'Writer'
-pNG1.Writer.FileName = 'RenderView1_%.6ts%cm.png'
-pNG1.Writer.ImageResolution = [1131, 939]
+pNG1.Writer.FileName = 'SliceChi_%.6ts%cm.png'
+pNG1.Writer.ImageResolution = [885, 774]
 pNG1.Writer.Format = 'PNG'
+pNG1.Writer.ResetDisplay = 1
 
 # ----------------------------------------------------------------
 # restore active source
-SetActiveSource(sliceAMRdata1)
+SetActiveSource(pNG1)
 # ----------------------------------------------------------------
 
 # ------------------------------------------------------------------------------
diff --git a/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_chi.py b/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_chi.py
index c3ecb7ed3..69ab93893 100644
--- a/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_chi.py
+++ b/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_chi.py
@@ -17,13 +17,14 @@
 renderView1 = CreateView('RenderView')
 renderView1.ViewSize = [885, 774]
 renderView1.AxesGrid = 'GridAxes3DActor'
-renderView1.CenterOfRotation = [8.0, 8.0, 4.0]
+renderView1.OrientationAxesVisibility = 0
+renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
 renderView1.StereoType = 'Crystal Eyes'
-renderView1.CameraPosition = [8.038575486273551, 7.488142173440099, 14.569596590973248]
-renderView1.CameraFocalPoint = [8.008182364514674, 7.891428269059167, 6.241949497853552]
-renderView1.CameraViewUp = [-0.754397824803147, -0.6557763921833819, -0.029004230505694627]
+renderView1.CameraPosition = [60.8256392162813, 9.229565961645996, 15.146416734320407]
+renderView1.CameraFocalPoint = [43.86533773456258, 22.62713273414431, 25.06265314762171]
+renderView1.CameraViewUp = [0.7001156857637189, 0.5438174917264287, 0.4627100196018815]
 renderView1.CameraFocalDisk = 1.0
-renderView1.CameraParallelScale = 14.517231140957975
+renderView1.CameraParallelScale = 60.6217782649107
 renderView1.BackEnd = 'OSPRay raycaster'
 renderView1.OSPRayMaterialLibrary = materialLibrary1
 
@@ -52,13 +53,9 @@
 input.MeshStatus = ['Mesh']
 input.CellArrayStatus = ['chi']
 
-# create a new 'Extract Level'
-extractLevel1 = ExtractLevel(registrationName='ExtractLevel1', Input=input)
-extractLevel1.Levels = [2]
-
 # create a new 'Resample To Image'
-resampleToImage1 = ResampleToImage(registrationName='ResampleToImage1', Input=extractLevel1)
-resampleToImage1.SamplingBounds = [3.75, 12.25, 5.75, 10.25, -0.25, 2.25]
+resampleToImage1 = ResampleToImage(registrationName='ResampleToImage1', Input=input)
+resampleToImage1.SamplingBounds = [-3.0, 67.0, -3.0, 67.0, -3.0, 67.0]
 
 # ----------------------------------------------------------------
 # setup the visualization in view 'renderView1'
@@ -69,14 +66,15 @@
 
 # get color transfer function/color map for 'chi'
 chiLUT = GetColorTransferFunction('chi')
-chiLUT.RGBPoints = [0.0, 0.0862745098039216, 0.00392156862745098, 0.298039215686275, 0.029344877158935277, 0.113725, 0.0235294, 0.45098, 0.0537159862131328, 0.105882, 0.0509804, 0.509804, 0.0706265851511459, 0.0392157, 0.0392157, 0.560784, 0.0870398318296482, 0.0313725, 0.0980392, 0.6, 0.10295566969276873, 0.0431373, 0.164706, 0.639216, 0.12583466541255697, 0.054902, 0.243137, 0.678431, 0.15617428436025385, 0.054902, 0.317647, 0.709804, 0.19347701615880114, 0.0509804, 0.396078, 0.741176, 0.2176616431786517, 0.0392157, 0.466667, 0.768627, 0.24184627019850216, 0.0313725, 0.537255, 0.788235, 0.267087853698452, 0.0313725, 0.615686, 0.811765, 0.2929510760870558, 0.0235294, 0.709804, 0.831373, 0.3188143550315277, 0.0509804, 0.8, 0.85098, 0.3402012374169067, 0.0705882, 0.854902, 0.870588, 0.36009606302373837, 0.262745, 0.901961, 0.862745, 0.3775040142212683, 0.423529, 0.941176, 0.87451, 0.4043619976847706, 0.572549, 0.964706, 0.835294, 0.42226735769767926, 0.658824, 0.980392, 0.843137, 0.43519896889198123, 0.764706, 0.980392, 0.866667, 0.4491253128832477, 0.827451, 0.980392, 0.886275, 0.47648067688419493, 0.913725, 0.988235, 0.937255, 0.48493597635320285, 1.0, 1.0, 0.972549019607843, 0.49339127582221104, 0.988235, 0.980392, 0.870588, 0.5058255347569962, 0.992156862745098, 0.972549019607843, 0.803921568627451, 0.5152755670229687, 0.992157, 0.964706, 0.713725, 0.5311914048860893, 0.988235, 0.956863, 0.643137, 0.5565572750151794, 0.980392, 0.917647, 0.509804, 0.5779442139564234, 0.968627, 0.87451, 0.407843, 0.5998285051571812, 0.94902, 0.823529, 0.321569, 0.615744343020301, 0.929412, 0.776471, 0.278431, 0.639120747555468, 0.909804, 0.717647, 0.235294, 0.6600102776813318, 0.890196, 0.658824, 0.196078, 0.6771695565558091, 0.878431, 0.619608, 0.168627, 0.7013541835756594, 0.870588, 0.54902, 0.156863, 0.7255388105955098, 0.85098, 0.47451, 0.145098, 0.7497234376153601, 0.831373, 0.411765, 0.133333, 0.7739080646352107, 0.811765, 0.345098, 0.113725, 0.798092691655061, 0.788235, 0.266667, 0.0941176, 0.8222773186749113, 0.741176, 0.184314, 0.0745098, 0.8464619456947616, 0.690196, 0.12549, 0.0627451, 0.870646572714612, 0.619608, 0.0627451, 0.0431373, 0.893276919519733, 0.54902, 0.027451, 0.0705882, 0.9131717283365444, 0.470588, 0.0156863, 0.0901961, 0.9355533868194685, 0.4, 0.00392157, 0.101961, 0.9673850807940133, 0.188235294117647, 0.0, 0.0705882352941176]
-chiLUT.ColorSpace = 'Lab'
+chiLUT.RGBPoints = [9.673850807940142e-05, 0.267004, 0.004874, 0.329415, 0.003890443386524834, 0.26851, 0.009605, 0.335427, 0.00768318097662798, 0.269944, 0.014625, 0.341379, 0.011476885855073412, 0.271305, 0.019942, 0.347269, 0.015269623445176558, 0.272594, 0.025563, 0.353093, 0.01906332832362199, 0.273809, 0.031497, 0.358853, 0.02285606591372514, 0.274952, 0.037752, 0.364543, 0.026649770792170572, 0.276022, 0.044167, 0.370164, 0.030443475670616, 0.277018, 0.050344, 0.375715, 0.03423621326071915, 0.277941, 0.056324, 0.381191, 0.038029918139164585, 0.278791, 0.062145, 0.386592, 0.04182265572926773, 0.279566, 0.067836, 0.391917, 0.04561636060771316, 0.280267, 0.073417, 0.397163, 0.049409098197816304, 0.280894, 0.078907, 0.402329, 0.05320280307626174, 0.281446, 0.08432, 0.407414, 0.056996507954707176, 0.281924, 0.089666, 0.412415, 0.060789245544810314, 0.282327, 0.094955, 0.417331, 0.06458295042325576, 0.282656, 0.100196, 0.42216, 0.0683756880133589, 0.28291, 0.105393, 0.426902, 0.07216939289180434, 0.283091, 0.110553, 0.431554, 0.07596213048190747, 0.283197, 0.11568, 0.436115, 0.07975583536035291, 0.283229, 0.120777, 0.440584, 0.08354954023879835, 0.283187, 0.125848, 0.44496, 0.08734227782890149, 0.283072, 0.130895, 0.449241, 0.09113598270734692, 0.282884, 0.13592, 0.453427, 0.09492872029745007, 0.282623, 0.140926, 0.457517, 0.0987224251758955, 0.28229, 0.145912, 0.46151, 0.10251516276599866, 0.281887, 0.150881, 0.465405, 0.10630886764444408, 0.281412, 0.155834, 0.469201, 0.11010160523454723, 0.280868, 0.160771, 0.472899, 0.11389531011299266, 0.280255, 0.165693, 0.476498, 0.11768901499143809, 0.279574, 0.170599, 0.479997, 0.12148175258154123, 0.278826, 0.17549, 0.483397, 0.12527545745998667, 0.278012, 0.180367, 0.486697, 0.12906819505008982, 0.277134, 0.185228, 0.489898, 0.13286189992853525, 0.276194, 0.190074, 0.493001, 0.1366546375186384, 0.275191, 0.194905, 0.496005, 0.14044834239708384, 0.274128, 0.199721, 0.498911, 0.14424204727552928, 0.273006, 0.20452, 0.501721, 0.1480347848656324, 0.271828, 0.209303, 0.504434, 0.15182848974407784, 0.270595, 0.214069, 0.507052, 0.155621227334181, 0.269308, 0.218818, 0.509577, 0.15941493221262643, 0.267968, 0.223549, 0.512008, 0.16320766980272958, 0.26658, 0.228262, 0.514349, 0.167001374681175, 0.265145, 0.232956, 0.516599, 0.17079507955962042, 0.263663, 0.237631, 0.518762, 0.17458781714972357, 0.262138, 0.242286, 0.520837, 0.178381522028169, 0.260571, 0.246922, 0.522828, 0.18217425961827216, 0.258965, 0.251537, 0.524736, 0.1859679644967176, 0.257322, 0.25613, 0.526563, 0.18976070208682075, 0.255645, 0.260703, 0.528312, 0.19355440696526618, 0.253935, 0.265254, 0.529983, 0.1973481118437116, 0.252194, 0.269783, 0.531579, 0.20114084943381474, 0.250425, 0.27429, 0.533103, 0.20493455431226018, 0.248629, 0.278775, 0.534556, 0.20872729190236333, 0.246811, 0.283237, 0.535941, 0.21252099678080877, 0.244972, 0.287675, 0.53726, 0.21631373437091192, 0.243113, 0.292092, 0.538516, 0.22010743924935733, 0.241237, 0.296485, 0.539709, 0.22390114412780276, 0.239346, 0.300855, 0.540844, 0.22769388171790592, 0.237441, 0.305202, 0.541921, 0.23148758659635135, 0.235526, 0.309527, 0.542944, 0.2352803241864545, 0.233603, 0.313828, 0.543914, 0.23907402906489994, 0.231674, 0.318106, 0.544834, 0.24286676665500306, 0.229739, 0.322361, 0.545706, 0.24666047153344853, 0.227802, 0.326594, 0.546532, 0.25045417641189394, 0.225863, 0.330805, 0.547314, 0.2542469140019971, 0.223925, 0.334994, 0.548053, 0.2580406188804425, 0.221989, 0.339161, 0.548752, 0.2618333564705457, 0.220057, 0.343307, 0.549413, 0.26562706134899106, 0.21813, 0.347432, 0.550038, 0.26941979893909423, 0.21621, 0.351535, 0.550627, 0.2732135038175397, 0.214298, 0.355619, 0.551184, 0.2770072086959851, 0.212395, 0.359683, 0.55171, 0.2807999462860883, 0.210503, 0.363727, 0.552206, 0.28459365116453367, 0.208623, 0.367752, 0.552675, 0.28838638875463685, 0.206756, 0.371758, 0.553117, 0.2921800936330822, 0.204903, 0.375746, 0.553533, 0.2959728312231854, 0.203063, 0.379716, 0.553925, 0.2997665361016309, 0.201239, 0.38367, 0.554294, 0.30355927369173397, 0.19943, 0.387607, 0.554642, 0.30735297857017946, 0.197636, 0.391528, 0.554969, 0.31114668344862484, 0.19586, 0.395433, 0.555276, 0.314939421038728, 0.1941, 0.399323, 0.555565, 0.31873312591717345, 0.192357, 0.403199, 0.555836, 0.3225258635072766, 0.190631, 0.407061, 0.556089, 0.32631956838572207, 0.188923, 0.41091, 0.556326, 0.33011230597582514, 0.187231, 0.414746, 0.556547, 0.33390601085427063, 0.185556, 0.41857, 0.556753, 0.337699715732716, 0.183898, 0.422383, 0.556944, 0.3414924533228192, 0.182256, 0.426184, 0.55712, 0.3452861582012646, 0.180629, 0.429975, 0.557282, 0.34907889579136775, 0.179019, 0.433756, 0.55743, 0.3528726006698132, 0.177423, 0.437527, 0.557565, 0.3566653382599163, 0.175841, 0.44129, 0.557685, 0.3604590431383618, 0.174274, 0.445044, 0.557792, 0.3642527480168072, 0.172719, 0.448791, 0.557885, 0.36804548560691036, 0.171176, 0.45253, 0.557965, 0.3718391904853558, 0.169646, 0.456262, 0.55803, 0.3756319280754589, 0.168126, 0.459988, 0.558082, 0.37942563295390436, 0.166617, 0.463708, 0.558119, 0.3832183705440075, 0.165117, 0.467423, 0.558141, 0.387012075422453, 0.163625, 0.471133, 0.558148, 0.39080578030089835, 0.162142, 0.474838, 0.55814, 0.39459851789100153, 0.160665, 0.47854, 0.558115, 0.39839222276944697, 0.159194, 0.482237, 0.558073, 0.4021849603595501, 0.157729, 0.485932, 0.558013, 0.40597866523799553, 0.15627, 0.489624, 0.557936, 0.40977140282809865, 0.154815, 0.493313, 0.55784, 0.41356510770654414, 0.153364, 0.497, 0.557724, 0.4173588125849896, 0.151918, 0.500685, 0.557587, 0.4211515501750927, 0.150476, 0.504369, 0.55743, 0.42494525505353814, 0.149039, 0.508051, 0.55725, 0.42873799264364126, 0.147607, 0.511733, 0.557049, 0.4325316975220867, 0.14618, 0.515413, 0.556823, 0.4363244351121898, 0.144759, 0.519093, 0.556572, 0.44011813999063526, 0.143343, 0.522773, 0.556295, 0.44391184486908075, 0.141935, 0.526453, 0.555991, 0.4477045824591839, 0.140536, 0.530132, 0.555659, 0.4514982873376293, 0.139147, 0.533812, 0.555298, 0.45529102492773244, 0.13777, 0.537492, 0.554906, 0.45908472980617787, 0.136408, 0.541173, 0.554483, 0.462877467396281, 0.135066, 0.544853, 0.554029, 0.46667117227472643, 0.133743, 0.548535, 0.553541, 0.4704648771531719, 0.132444, 0.552216, 0.553018, 0.47425761474327505, 0.131172, 0.555899, 0.552459, 0.4780513196217205, 0.129933, 0.559582, 0.551864, 0.4818440572118236, 0.128729, 0.563265, 0.551229, 0.48563776209026904, 0.127568, 0.566949, 0.550556, 0.4894304996803722, 0.126453, 0.570633, 0.549841, 0.49322420455881766, 0.125394, 0.574318, 0.549086, 0.4970169421489208, 0.124395, 0.578002, 0.548287, 0.5008106470273662, 0.123463, 0.581687, 0.547445, 0.5046043519058115, 0.122606, 0.585371, 0.546557, 0.5083970894959148, 0.121831, 0.589055, 0.545623, 0.5121907943743602, 0.121148, 0.592739, 0.544641, 0.5159835319644633, 0.120565, 0.596422, 0.543611, 0.5197772368429089, 0.120092, 0.600104, 0.54253, 0.5235699744330119, 0.119738, 0.603785, 0.5414, 0.5273636793114573, 0.119512, 0.607464, 0.540218, 0.5311573841899028, 0.119423, 0.611141, 0.538982, 0.534950121780006, 0.119483, 0.614817, 0.537692, 0.5387438266584513, 0.119699, 0.61849, 0.536347, 0.5425365642485545, 0.120081, 0.622161, 0.534946, 0.546330269127, 0.120638, 0.625828, 0.533488, 0.5501230067171031, 0.12138, 0.629492, 0.531973, 0.5539167115955486, 0.122312, 0.633153, 0.530398, 0.5577104164739939, 0.123444, 0.636809, 0.528763, 0.5615031540640971, 0.12478, 0.640461, 0.527068, 0.5652968589425426, 0.126326, 0.644107, 0.525311, 0.5690895965326457, 0.128087, 0.647749, 0.523491, 0.5728833014110912, 0.130067, 0.651384, 0.521608, 0.5766760390011942, 0.132268, 0.655014, 0.519661, 0.5804697438796397, 0.134692, 0.658636, 0.517649, 0.5842634487580851, 0.137339, 0.662252, 0.515571, 0.5880561863481883, 0.14021, 0.665859, 0.513427, 0.5918498912266338, 0.143303, 0.669459, 0.511215, 0.5956426288167368, 0.146616, 0.67305, 0.508936, 0.5994363336951823, 0.150148, 0.676631, 0.506589, 0.6032290712852855, 0.153894, 0.680203, 0.504172, 0.6070227761637309, 0.157851, 0.683765, 0.501686, 0.6108164810421762, 0.162016, 0.687316, 0.499129, 0.6146092186322795, 0.166383, 0.690856, 0.496502, 0.6184029235107249, 0.170948, 0.694384, 0.493803, 0.622195661100828, 0.175707, 0.6979, 0.491033, 0.6259893659792736, 0.180653, 0.701402, 0.488189, 0.6297821035693766, 0.185783, 0.704891, 0.485273, 0.633575808447822, 0.19109, 0.708366, 0.482284, 0.6373695133262675, 0.196571, 0.711827, 0.479221, 0.6411622509163707, 0.202219, 0.715272, 0.476084, 0.6449559557948161, 0.20803, 0.718701, 0.472873, 0.6487486933849191, 0.214, 0.722114, 0.469588, 0.6525423982633647, 0.220124, 0.725509, 0.466226, 0.6563351358534678, 0.226397, 0.728888, 0.462789, 0.6601288407319132, 0.232815, 0.732247, 0.459277, 0.6639225456103586, 0.239374, 0.735588, 0.455688, 0.6677152832004618, 0.24607, 0.73891, 0.452024, 0.6715089880789072, 0.252899, 0.742211, 0.448284, 0.6753017256690104, 0.259857, 0.745492, 0.444467, 0.6790954305474558, 0.266941, 0.748751, 0.440573, 0.6828881681375589, 0.274149, 0.751988, 0.436601, 0.6866818730160044, 0.281477, 0.755203, 0.432552, 0.6904746106061076, 0.288921, 0.758394, 0.428426, 0.694268315484553, 0.296479, 0.761561, 0.424223, 0.6980620203629985, 0.304148, 0.764704, 0.419943, 0.7018547579531015, 0.311925, 0.767822, 0.415586, 0.7056484628315469, 0.319809, 0.770914, 0.411152, 0.7094412004216502, 0.327796, 0.77398, 0.40664, 0.7132349053000956, 0.335885, 0.777018, 0.402049, 0.7170276428901987, 0.344074, 0.780029, 0.397381, 0.7208213477686441, 0.35236, 0.783011, 0.392636, 0.7246150526470896, 0.360741, 0.785964, 0.387814, 0.7284077902371927, 0.369214, 0.788888, 0.382914, 0.7322014951156381, 0.377779, 0.791781, 0.377939, 0.7359942327057413, 0.386433, 0.794644, 0.372886, 0.7397879375841867, 0.395174, 0.797475, 0.367757, 0.7435806751742898, 0.404001, 0.800275, 0.362552, 0.7473743800527354, 0.412913, 0.803041, 0.357269, 0.7511680849311808, 0.421908, 0.805774, 0.35191, 0.7549608225212838, 0.430983, 0.808473, 0.346476, 0.7587545273997293, 0.440137, 0.811138, 0.340967, 0.7625472649898325, 0.449368, 0.813768, 0.335384, 0.7663409698682779, 0.458674, 0.816363, 0.329727, 0.7701337074583811, 0.468053, 0.818921, 0.323998, 0.7739274123368265, 0.477504, 0.821444, 0.318195, 0.7777211172152719, 0.487026, 0.823929, 0.312321, 0.781513854805375, 0.496615, 0.826376, 0.306377, 0.7853075596838205, 0.506271, 0.828786, 0.300362, 0.7891002972739236, 0.515992, 0.831158, 0.294279, 0.792894002152369, 0.525776, 0.833491, 0.288127, 0.7966867397424722, 0.535621, 0.835785, 0.281908, 0.8004804446209177, 0.545524, 0.838039, 0.275626, 0.8042741494993632, 0.555484, 0.840254, 0.269281, 0.8080668870894662, 0.565498, 0.84243, 0.262877, 0.8118605919679116, 0.575563, 0.844566, 0.256415, 0.8156533295580148, 0.585678, 0.846661, 0.249897, 0.8194470344364603, 0.595839, 0.848717, 0.243329, 0.8232397720265634, 0.606045, 0.850733, 0.236712, 0.8270334769050088, 0.616293, 0.852709, 0.230052, 0.8308271817834543, 0.626579, 0.854645, 0.223353, 0.8346199193735574, 0.636902, 0.856542, 0.21662, 0.8384136242520028, 0.647257, 0.8584, 0.209861, 0.8422063618421061, 0.657642, 0.860219, 0.203082, 0.8460000667205514, 0.668054, 0.861999, 0.196293, 0.8497928043106545, 0.678489, 0.863742, 0.189503, 0.8535865091891001, 0.688944, 0.865448, 0.182725, 0.8573802140675455, 0.699415, 0.867117, 0.175971, 0.8611729516576485, 0.709898, 0.868751, 0.169257, 0.864966656536094, 0.720391, 0.87035, 0.162603, 0.8687593941261972, 0.730889, 0.871916, 0.156029, 0.8725530990046426, 0.741388, 0.873449, 0.149561, 0.8763458365947457, 0.751884, 0.874951, 0.143228, 0.8801395414731911, 0.762373, 0.876424, 0.137064, 0.8839322790632943, 0.772852, 0.877868, 0.131109, 0.8877259839417397, 0.783315, 0.879285, 0.125405, 0.8915196888201852, 0.79376, 0.880678, 0.120005, 0.8953124264102884, 0.804182, 0.882046, 0.114965, 0.8991061312887337, 0.814576, 0.883393, 0.110347, 0.9028988688788369, 0.82494, 0.88472, 0.106217, 0.9066925737572823, 0.83527, 0.886029, 0.102646, 0.9104853113473855, 0.845561, 0.887322, 0.099702, 0.914279016225831, 0.85581, 0.888601, 0.097452, 0.9180727211042763, 0.866013, 0.889868, 0.095953, 0.9218654586943795, 0.876168, 0.891125, 0.09525, 0.925659163572825, 0.886271, 0.892374, 0.095374, 0.9294519011629281, 0.89632, 0.893616, 0.096335, 0.9332456060413734, 0.906311, 0.894855, 0.098125, 0.9370383436314766, 0.916242, 0.896091, 0.100717, 0.9408320485099221, 0.926106, 0.89733, 0.104071, 0.9446257533883675, 0.935904, 0.89857, 0.108131, 0.9484184909784708, 0.945636, 0.899815, 0.112838, 0.9522121958569161, 0.9553, 0.901065, 0.118128, 0.9560049334470192, 0.964894, 0.902323, 0.123941, 0.9597986383254646, 0.974417, 0.90359, 0.130215, 0.9635913759155679, 0.983868, 0.904867, 0.136897, 0.9673850807940133, 0.993248, 0.906157, 0.143936]
+chiLUT.ShowDataHistogram = 1
+chiLUT.DataHistogramNumberOfBins = 44
 chiLUT.NanColor = [1.0, 0.0, 0.0]
 chiLUT.ScalarRangeInitialized = 1.0
 
 # get opacity transfer function/opacity map for 'chi'
 chiPWF = GetOpacityTransferFunction('chi')
-chiPWF.Points = [0.0, 1.0, 0.5, 0.0, 0.04865546501470074, 0.6094674468040466, 0.5, 0.0, 0.12020762722887196, 0.4378698170185089, 0.5, 0.0, 0.22324271380797622, 0.23076923191547394, 0.5, 0.0, 0.37207121721551956, 0.08875739574432373, 0.5, 0.0, 0.578141420384064, 0.0, 0.5, 0.0, 0.9673850807940133, 0.0, 0.5, 0.0]
+chiPWF.Points = [9.673850807940134e-05, 1.0, 0.5, 0.0, 0.07736533880233765, 0.7692307829856873, 0.5, 0.0, 0.1517721265554428, 0.6153846383094788, 0.5, 0.0, 0.24334973096847534, 0.43195265531539917, 0.5, 0.0, 0.38071611523628235, 0.21893490850925446, 0.5, 0.0, 0.5781803447501049, 0.0, 0.5, 0.0, 0.9673850807940133, 0.0, 0.5, 0.0]
 chiPWF.ScalarRangeInitialized = 1
 
 # trace defaults for the display properties.
@@ -89,31 +87,31 @@
 resampleToImage1Display.OSPRayScaleArray = 'chi'
 resampleToImage1Display.OSPRayScaleFunction = 'PiecewiseFunction'
 resampleToImage1Display.SelectOrientationVectors = 'None'
-resampleToImage1Display.ScaleFactor = 0.84999915
+resampleToImage1Display.ScaleFactor = 6.999993000000001
 resampleToImage1Display.SelectScaleArray = 'None'
 resampleToImage1Display.GlyphType = 'Arrow'
 resampleToImage1Display.GlyphTableIndexArray = 'None'
-resampleToImage1Display.GaussianRadius = 0.042499957500000005
+resampleToImage1Display.GaussianRadius = 0.34999965000000005
 resampleToImage1Display.SetScaleArray = ['POINTS', 'chi']
 resampleToImage1Display.ScaleTransferFunction = 'PiecewiseFunction'
 resampleToImage1Display.OpacityArray = ['POINTS', 'chi']
 resampleToImage1Display.OpacityTransferFunction = 'PiecewiseFunction'
 resampleToImage1Display.DataAxesGrid = 'GridAxesRepresentation'
 resampleToImage1Display.PolarAxes = 'PolarAxesRepresentation'
-resampleToImage1Display.ScalarOpacityUnitDistance = 0.10037670222093371
+resampleToImage1Display.ScalarOpacityUnitDistance = 1.224681164507726
 resampleToImage1Display.ScalarOpacityFunction = chiPWF
 resampleToImage1Display.OpacityArrayName = ['POINTS', 'chi']
 resampleToImage1Display.SliceFunction = 'Plane'
 resampleToImage1Display.Slice = 49
 
 # init the 'PiecewiseFunction' selected for 'ScaleTransferFunction'
-resampleToImage1Display.ScaleTransferFunction.Points = [0.007669444988778632, 0.0, 0.5, 0.0, 0.6724393877650426, 1.0, 0.5, 0.0]
+resampleToImage1Display.ScaleTransferFunction.Points = [0.16307053589122933, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
 
 # init the 'PiecewiseFunction' selected for 'OpacityTransferFunction'
-resampleToImage1Display.OpacityTransferFunction.Points = [0.007669444988778632, 0.0, 0.5, 0.0, 0.6724393877650426, 1.0, 0.5, 0.0]
+resampleToImage1Display.OpacityTransferFunction.Points = [0.16307053589122933, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
 
 # init the 'Plane' selected for 'SliceFunction'
-resampleToImage1Display.SliceFunction.Origin = [8.0, 8.0, 1.0]
+resampleToImage1Display.SliceFunction.Origin = [32.0, 32.0, 32.0]
 
 # ----------------------------------------------------------------
 # setup color maps and opacity mapes used in the visualization
@@ -134,7 +132,7 @@
 
 # ----------------------------------------------------------------
 # restore active source
-SetActiveSource(pNG1)
+SetActiveSource(resampleToImage1)
 # ----------------------------------------------------------------
 
 # ------------------------------------------------------------------------------
diff --git a/Examples/BinaryBH/params_catalyst.txt b/Examples/BinaryBH/params_catalyst.txt
index 170b33b93..08397f427 100644
--- a/Examples/BinaryBH/params_catalyst.txt
+++ b/Examples/BinaryBH/params_catalyst.txt
@@ -10,7 +10,7 @@ plot_prefix = BinaryBHp_
 # restart_file = BinaryBHChk_000000.3d.hdf5
 
 # HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval
-checkpoint_interval = -1
+checkpoint_interval = 100
 # set to 0 to turn off plot files (except at t=0 and t=stop_time)
 # set to -1 to never ever print plotfiles
 plot_interval = 1
@@ -186,6 +186,6 @@ modes = 2 0 # l m for spherical harmonics
 
 activate_catalyst = true
 catalyst_script_path = catalyst_scripts/slice_chi.py
-catalyst_coprocess_level = 2
+catalyst_coprocess_level = 0
 num_catalyst_vars = 1
 caralyst_vars = chi

From fee88cdc462af362308705e8ab217cb7a81a2dff Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 4 Jan 2022 14:18:21 +0000
Subject: [PATCH 17/63] Update ParaView Catalyst scripts

Add some recent new ones and remove some old ones.
---
 .../BinaryBH/catalyst_scripts/amr_blocks.py   | 117 ++++++++++++++
 Examples/BinaryBH/catalyst_scripts/outline.py | 119 ++++++++++++++
 .../catalyst_scripts/slice_amr_blocks.py      | 120 ++++++++++++++
 .../catalyst_scripts/volume_noraytrace_chi.py | 151 -----------------
 .../volume_noraytrace_resample_chi.py         | 152 +++++++++++++++++
 .../catalyst_scripts/volume_raytrace_chi.py   | 133 ---------------
 .../volume_raytrace_resample_chi.py           | 153 ++++++++++++++++++
 Examples/BinaryBH/params_catalyst.txt         |   4 +-
 8 files changed, 663 insertions(+), 286 deletions(-)
 create mode 100644 Examples/BinaryBH/catalyst_scripts/amr_blocks.py
 create mode 100644 Examples/BinaryBH/catalyst_scripts/outline.py
 create mode 100644 Examples/BinaryBH/catalyst_scripts/slice_amr_blocks.py
 delete mode 100644 Examples/BinaryBH/catalyst_scripts/volume_noraytrace_chi.py
 create mode 100644 Examples/BinaryBH/catalyst_scripts/volume_noraytrace_resample_chi.py
 delete mode 100644 Examples/BinaryBH/catalyst_scripts/volume_raytrace_chi.py
 create mode 100644 Examples/BinaryBH/catalyst_scripts/volume_raytrace_resample_chi.py

diff --git a/Examples/BinaryBH/catalyst_scripts/amr_blocks.py b/Examples/BinaryBH/catalyst_scripts/amr_blocks.py
new file mode 100644
index 000000000..64af0e09d
--- /dev/null
+++ b/Examples/BinaryBH/catalyst_scripts/amr_blocks.py
@@ -0,0 +1,117 @@
+# script-version: 2.0
+# Catalyst state generated using paraview version 5.9.1
+
+#### import the simple module from the paraview
+from paraview.simple import *
+#### disable automatic camera reset on 'Show'
+paraview.simple._DisableFirstRenderCameraReset()
+
+# ----------------------------------------------------------------
+# setup views used in the visualization
+# ----------------------------------------------------------------
+
+# get the material library
+materialLibrary1 = GetMaterialLibrary()
+
+# Create a new 'Render View'
+renderView1 = CreateView('RenderView')
+renderView1.ViewSize = [885, 774]
+renderView1.AxesGrid = 'GridAxes3DActor'
+renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
+renderView1.StereoType = 'Crystal Eyes'
+renderView1.CameraPosition = [158.08172115703206, 212.13325123979374, 112.7252021311981]
+renderView1.CameraFocalPoint = [32.00000000000007, 31.99999999999995, 32.0000000000001]
+renderView1.CameraViewUp = [0.05928269610132273, -0.44249529710522584, 0.8948091830008877]
+renderView1.CameraFocalDisk = 1.0
+renderView1.CameraParallelScale = 60.6217782649107
+renderView1.BackEnd = 'OSPRay raycaster'
+renderView1.OSPRayMaterialLibrary = materialLibrary1
+
+# init the 'GridAxes3DActor' selected for 'AxesGrid'
+renderView1.AxesGrid.Visibility = 1
+
+SetActiveView(None)
+
+# ----------------------------------------------------------------
+# setup view layouts
+# ----------------------------------------------------------------
+
+# create new layout object 'Layout #1'
+layout1 = CreateLayout(name='Layout #1')
+layout1.AssignView(0, renderView1)
+layout1.SetSize(885, 774)
+
+# ----------------------------------------------------------------
+# restore active view
+SetActiveView(renderView1)
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup the data processing pipelines
+# ----------------------------------------------------------------
+
+# create a new 'VisItChomboReader'
+input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000003.3d.hdf5'])
+input.MeshStatus = ['Mesh']
+input.CellArrayStatus = []
+
+# ----------------------------------------------------------------
+# setup the visualization in view 'renderView1'
+# ----------------------------------------------------------------
+
+# show data from input
+inputDisplay = Show(input, renderView1, 'AMRRepresentation')
+
+# trace defaults for the display properties.
+inputDisplay.Representation = 'AMR Blocks'
+inputDisplay.ColorArrayName = ['POINTS', '']
+inputDisplay.SelectTCoordArray = 'None'
+inputDisplay.SelectNormalArray = 'None'
+inputDisplay.SelectTangentArray = 'None'
+inputDisplay.OSPRayScaleFunction = 'PiecewiseFunction'
+inputDisplay.SelectOrientationVectors = 'None'
+inputDisplay.ScaleFactor = 7.0
+inputDisplay.SelectScaleArray = 'None'
+inputDisplay.GlyphType = 'Arrow'
+inputDisplay.GlyphTableIndexArray = 'None'
+inputDisplay.GaussianRadius = 0.35000000000000003
+inputDisplay.SetScaleArray = [None, '']
+inputDisplay.ScaleTransferFunction = 'PiecewiseFunction'
+inputDisplay.OpacityArray = [None, '']
+inputDisplay.OpacityTransferFunction = 'PiecewiseFunction'
+inputDisplay.DataAxesGrid = 'GridAxesRepresentation'
+inputDisplay.PolarAxes = 'PolarAxesRepresentation'
+inputDisplay.ScalarOpacityUnitDistance = 0.8192401766441247
+
+# ----------------------------------------------------------------
+# setup extractors
+# ----------------------------------------------------------------
+
+# create extractor
+pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
+# trace defaults for the extractor.
+# init the 'PNG' selected for 'Writer'
+pNG1.Writer.FileName = 'AMRBlocks_%.6ts%cm.png'
+pNG1.Writer.ImageResolution = [885, 774]
+pNG1.Writer.Format = 'PNG'
+pNG1.Writer.ResetDisplay = 1
+
+# ----------------------------------------------------------------
+# restore active source
+SetActiveSource(pNG1)
+# ----------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# Catalyst options
+from paraview import catalyst
+options = catalyst.Options()
+options.GlobalTrigger = 'TimeStep'
+options.EnableCatalystLive = 1
+options.CatalystLiveTrigger = 'TimeStep'
+
+# ------------------------------------------------------------------------------
+if __name__ == '__main__':
+    from paraview.simple import SaveExtractsUsingCatalystOptions
+    # Code for non in-situ environments; if executing in post-processing
+    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
+    SaveExtractsUsingCatalystOptions(options)
diff --git a/Examples/BinaryBH/catalyst_scripts/outline.py b/Examples/BinaryBH/catalyst_scripts/outline.py
new file mode 100644
index 000000000..53832a9c0
--- /dev/null
+++ b/Examples/BinaryBH/catalyst_scripts/outline.py
@@ -0,0 +1,119 @@
+# script-version: 2.0
+# Catalyst state generated using paraview version 5.9.1
+
+#### import the simple module from the paraview
+from paraview.simple import *
+#### disable automatic camera reset on 'Show'
+paraview.simple._DisableFirstRenderCameraReset()
+
+# ----------------------------------------------------------------
+# setup views used in the visualization
+# ----------------------------------------------------------------
+
+# get the material library
+materialLibrary1 = GetMaterialLibrary()
+
+# Create a new 'Render View'
+renderView1 = CreateView('RenderView')
+renderView1.ViewSize = [885, 774]
+renderView1.AxesGrid = 'GridAxes3DActor'
+renderView1.CenterAxesVisibility = 1
+renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
+renderView1.StereoType = 'Crystal Eyes'
+renderView1.CameraPosition = [145.67639666481145, -111.0984129619794, 178.49801336649438]
+renderView1.CameraFocalPoint = [32.000000000000014, 31.999999999999982, 32.00000000000003]
+renderView1.CameraViewUp = [-0.10746516487670288, 0.6682514214076767, 0.7361326484572212]
+renderView1.CameraFocalDisk = 1.0
+renderView1.CameraParallelScale = 60.6217782649107
+renderView1.CameraParallelProjection = 1
+renderView1.BackEnd = 'OSPRay raycaster'
+renderView1.OSPRayMaterialLibrary = materialLibrary1
+
+# init the 'GridAxes3DActor' selected for 'AxesGrid'
+renderView1.AxesGrid.Visibility = 1
+
+SetActiveView(None)
+
+# ----------------------------------------------------------------
+# setup view layouts
+# ----------------------------------------------------------------
+
+# create new layout object 'Layout #1'
+layout1 = CreateLayout(name='Layout #1')
+layout1.AssignView(0, renderView1)
+layout1.SetSize(885, 774)
+
+# ----------------------------------------------------------------
+# restore active view
+SetActiveView(renderView1)
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup the data processing pipelines
+# ----------------------------------------------------------------
+
+# create a new 'VisItChomboReader'
+input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000003.3d.hdf5'])
+input.MeshStatus = ['Mesh']
+input.CellArrayStatus = []
+
+# ----------------------------------------------------------------
+# setup the visualization in view 'renderView1'
+# ----------------------------------------------------------------
+
+# show data from input
+inputDisplay = Show(input, renderView1, 'AMRRepresentation')
+
+# trace defaults for the display properties.
+inputDisplay.Representation = 'Outline'
+inputDisplay.ColorArrayName = [None, '']
+inputDisplay.SelectTCoordArray = 'None'
+inputDisplay.SelectNormalArray = 'None'
+inputDisplay.SelectTangentArray = 'None'
+inputDisplay.OSPRayScaleFunction = 'PiecewiseFunction'
+inputDisplay.SelectOrientationVectors = 'None'
+inputDisplay.ScaleFactor = 7.0
+inputDisplay.SelectScaleArray = 'None'
+inputDisplay.GlyphType = 'Arrow'
+inputDisplay.GlyphTableIndexArray = 'None'
+inputDisplay.GaussianRadius = 0.35000000000000003
+inputDisplay.SetScaleArray = [None, '']
+inputDisplay.ScaleTransferFunction = 'PiecewiseFunction'
+inputDisplay.OpacityArray = [None, '']
+inputDisplay.OpacityTransferFunction = 'PiecewiseFunction'
+inputDisplay.DataAxesGrid = 'GridAxesRepresentation'
+inputDisplay.PolarAxes = 'PolarAxesRepresentation'
+inputDisplay.ScalarOpacityUnitDistance = 0.8192401766441247
+
+# ----------------------------------------------------------------
+# setup extractors
+# ----------------------------------------------------------------
+
+# create extractor
+pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
+# trace defaults for the extractor.
+# init the 'PNG' selected for 'Writer'
+pNG1.Writer.FileName = 'RenderView1_%.6ts%cm.png'
+pNG1.Writer.ImageResolution = [885, 774]
+pNG1.Writer.Format = 'PNG'
+pNG1.Writer.ResetDisplay = 1
+
+# ----------------------------------------------------------------
+# restore active source
+SetActiveSource(input)
+# ----------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# Catalyst options
+from paraview import catalyst
+options = catalyst.Options()
+options.GlobalTrigger = 'TimeStep'
+options.EnableCatalystLive = 1
+options.CatalystLiveTrigger = 'TimeStep'
+
+# ------------------------------------------------------------------------------
+if __name__ == '__main__':
+    from paraview.simple import SaveExtractsUsingCatalystOptions
+    # Code for non in-situ environments; if executing in post-processing
+    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
+    SaveExtractsUsingCatalystOptions(options)
diff --git a/Examples/BinaryBH/catalyst_scripts/slice_amr_blocks.py b/Examples/BinaryBH/catalyst_scripts/slice_amr_blocks.py
new file mode 100644
index 000000000..133925d91
--- /dev/null
+++ b/Examples/BinaryBH/catalyst_scripts/slice_amr_blocks.py
@@ -0,0 +1,120 @@
+# script-version: 2.0
+# Catalyst state generated using paraview version 5.9.1
+
+#### import the simple module from the paraview
+from paraview.simple import *
+#### disable automatic camera reset on 'Show'
+paraview.simple._DisableFirstRenderCameraReset()
+
+# ----------------------------------------------------------------
+# setup views used in the visualization
+# ----------------------------------------------------------------
+
+# get the material library
+materialLibrary1 = GetMaterialLibrary()
+
+# Create a new 'Render View'
+renderView1 = CreateView('RenderView')
+renderView1.ViewSize = [885, 774]
+renderView1.AxesGrid = 'GridAxes3DActor'
+renderView1.CenterAxesVisibility = 1
+renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
+renderView1.StereoType = 'Crystal Eyes'
+renderView1.CameraPosition = [32.0, 32.0, 190.05252605770363]
+renderView1.CameraFocalPoint = [32.0, 32.0, 32.0]
+renderView1.CameraFocalDisk = 1.0
+renderView1.CameraParallelScale = 49.49747468305833
+renderView1.BackEnd = 'OSPRay raycaster'
+renderView1.OSPRayMaterialLibrary = materialLibrary1
+renderView1.Background = [0.0, 0.0, 0.0]
+
+SetActiveView(None)
+
+# ----------------------------------------------------------------
+# setup view layouts
+# ----------------------------------------------------------------
+
+# create new layout object 'Layout #1'
+layout1 = CreateLayout(name='Layout #1')
+layout1.AssignView(0, renderView1)
+layout1.SetSize(885, 774)
+
+# ----------------------------------------------------------------
+# restore active view
+SetActiveView(renderView1)
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup the data processing pipelines
+# ----------------------------------------------------------------
+
+# create a new 'VisItChomboReader'
+input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000003.3d.hdf5'])
+input.MeshStatus = ['Mesh']
+input.CellArrayStatus = []
+
+# create a new 'Slice AMR data'
+sliceAMRdata1 = SliceAMRdata(registrationName='SliceAMRdata1', Input=input)
+sliceAMRdata1.Level = 3
+sliceAMRdata1.OffSet = 35.0
+sliceAMRdata1.Normal = 'Z-Normal'
+
+# ----------------------------------------------------------------
+# setup the visualization in view 'renderView1'
+# ----------------------------------------------------------------
+
+# show data from sliceAMRdata1
+sliceAMRdata1Display = Show(sliceAMRdata1, renderView1, 'AMRRepresentation')
+
+# trace defaults for the display properties.
+sliceAMRdata1Display.Representation = 'Outline'
+sliceAMRdata1Display.ColorArrayName = ['POINTS', '']
+sliceAMRdata1Display.SelectTCoordArray = 'None'
+sliceAMRdata1Display.SelectNormalArray = 'None'
+sliceAMRdata1Display.SelectTangentArray = 'None'
+sliceAMRdata1Display.OSPRayScaleFunction = 'PiecewiseFunction'
+sliceAMRdata1Display.SelectOrientationVectors = 'None'
+sliceAMRdata1Display.ScaleFactor = 7.0
+sliceAMRdata1Display.SelectScaleArray = 'None'
+sliceAMRdata1Display.GlyphType = 'Arrow'
+sliceAMRdata1Display.GlyphTableIndexArray = 'None'
+sliceAMRdata1Display.GaussianRadius = 0.35000000000000003
+sliceAMRdata1Display.SetScaleArray = [None, '']
+sliceAMRdata1Display.ScaleTransferFunction = 'PiecewiseFunction'
+sliceAMRdata1Display.OpacityArray = [None, '']
+sliceAMRdata1Display.OpacityTransferFunction = 'PiecewiseFunction'
+sliceAMRdata1Display.DataAxesGrid = 'GridAxesRepresentation'
+sliceAMRdata1Display.PolarAxes = 'PolarAxesRepresentation'
+sliceAMRdata1Display.ScalarOpacityUnitDistance = 2.3023915422888135
+
+# ----------------------------------------------------------------
+# setup extractors
+# ----------------------------------------------------------------
+
+# create extractor
+pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
+# trace defaults for the extractor.
+# init the 'PNG' selected for 'Writer'
+pNG1.Writer.FileName = 'AMRBlocksSlice_%.6ts%cm.png'
+pNG1.Writer.ImageResolution = [885, 774]
+pNG1.Writer.Format = 'PNG'
+
+# ----------------------------------------------------------------
+# restore active source
+SetActiveSource(pNG1)
+# ----------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# Catalyst options
+from paraview import catalyst
+options = catalyst.Options()
+options.GlobalTrigger = 'TimeStep'
+options.EnableCatalystLive = 1
+options.CatalystLiveTrigger = 'TimeStep'
+
+# ------------------------------------------------------------------------------
+if __name__ == '__main__':
+    from paraview.simple import SaveExtractsUsingCatalystOptions
+    # Code for non in-situ environments; if executing in post-processing
+    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
+    SaveExtractsUsingCatalystOptions(options)
diff --git a/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_chi.py b/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_chi.py
deleted file mode 100644
index 69ab93893..000000000
--- a/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_chi.py
+++ /dev/null
@@ -1,151 +0,0 @@
-# script-version: 2.0
-# Catalyst state generated using paraview version 5.9.1
-
-#### import the simple module from the paraview
-from paraview.simple import *
-#### disable automatic camera reset on 'Show'
-paraview.simple._DisableFirstRenderCameraReset()
-
-# ----------------------------------------------------------------
-# setup views used in the visualization
-# ----------------------------------------------------------------
-
-# get the material library
-materialLibrary1 = GetMaterialLibrary()
-
-# Create a new 'Render View'
-renderView1 = CreateView('RenderView')
-renderView1.ViewSize = [885, 774]
-renderView1.AxesGrid = 'GridAxes3DActor'
-renderView1.OrientationAxesVisibility = 0
-renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
-renderView1.StereoType = 'Crystal Eyes'
-renderView1.CameraPosition = [60.8256392162813, 9.229565961645996, 15.146416734320407]
-renderView1.CameraFocalPoint = [43.86533773456258, 22.62713273414431, 25.06265314762171]
-renderView1.CameraViewUp = [0.7001156857637189, 0.5438174917264287, 0.4627100196018815]
-renderView1.CameraFocalDisk = 1.0
-renderView1.CameraParallelScale = 60.6217782649107
-renderView1.BackEnd = 'OSPRay raycaster'
-renderView1.OSPRayMaterialLibrary = materialLibrary1
-
-SetActiveView(None)
-
-# ----------------------------------------------------------------
-# setup view layouts
-# ----------------------------------------------------------------
-
-# create new layout object 'Layout #1'
-layout1 = CreateLayout(name='Layout #1')
-layout1.AssignView(0, renderView1)
-layout1.SetSize(885, 774)
-
-# ----------------------------------------------------------------
-# restore active view
-SetActiveView(renderView1)
-# ----------------------------------------------------------------
-
-# ----------------------------------------------------------------
-# setup the data processing pipelines
-# ----------------------------------------------------------------
-
-# create a new 'VisItChomboReader'
-input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000003.3d.hdf5'])
-input.MeshStatus = ['Mesh']
-input.CellArrayStatus = ['chi']
-
-# create a new 'Resample To Image'
-resampleToImage1 = ResampleToImage(registrationName='ResampleToImage1', Input=input)
-resampleToImage1.SamplingBounds = [-3.0, 67.0, -3.0, 67.0, -3.0, 67.0]
-
-# ----------------------------------------------------------------
-# setup the visualization in view 'renderView1'
-# ----------------------------------------------------------------
-
-# show data from resampleToImage1
-resampleToImage1Display = Show(resampleToImage1, renderView1, 'UniformGridRepresentation')
-
-# get color transfer function/color map for 'chi'
-chiLUT = GetColorTransferFunction('chi')
-chiLUT.RGBPoints = [9.673850807940142e-05, 0.267004, 0.004874, 0.329415, 0.003890443386524834, 0.26851, 0.009605, 0.335427, 0.00768318097662798, 0.269944, 0.014625, 0.341379, 0.011476885855073412, 0.271305, 0.019942, 0.347269, 0.015269623445176558, 0.272594, 0.025563, 0.353093, 0.01906332832362199, 0.273809, 0.031497, 0.358853, 0.02285606591372514, 0.274952, 0.037752, 0.364543, 0.026649770792170572, 0.276022, 0.044167, 0.370164, 0.030443475670616, 0.277018, 0.050344, 0.375715, 0.03423621326071915, 0.277941, 0.056324, 0.381191, 0.038029918139164585, 0.278791, 0.062145, 0.386592, 0.04182265572926773, 0.279566, 0.067836, 0.391917, 0.04561636060771316, 0.280267, 0.073417, 0.397163, 0.049409098197816304, 0.280894, 0.078907, 0.402329, 0.05320280307626174, 0.281446, 0.08432, 0.407414, 0.056996507954707176, 0.281924, 0.089666, 0.412415, 0.060789245544810314, 0.282327, 0.094955, 0.417331, 0.06458295042325576, 0.282656, 0.100196, 0.42216, 0.0683756880133589, 0.28291, 0.105393, 0.426902, 0.07216939289180434, 0.283091, 0.110553, 0.431554, 0.07596213048190747, 0.283197, 0.11568, 0.436115, 0.07975583536035291, 0.283229, 0.120777, 0.440584, 0.08354954023879835, 0.283187, 0.125848, 0.44496, 0.08734227782890149, 0.283072, 0.130895, 0.449241, 0.09113598270734692, 0.282884, 0.13592, 0.453427, 0.09492872029745007, 0.282623, 0.140926, 0.457517, 0.0987224251758955, 0.28229, 0.145912, 0.46151, 0.10251516276599866, 0.281887, 0.150881, 0.465405, 0.10630886764444408, 0.281412, 0.155834, 0.469201, 0.11010160523454723, 0.280868, 0.160771, 0.472899, 0.11389531011299266, 0.280255, 0.165693, 0.476498, 0.11768901499143809, 0.279574, 0.170599, 0.479997, 0.12148175258154123, 0.278826, 0.17549, 0.483397, 0.12527545745998667, 0.278012, 0.180367, 0.486697, 0.12906819505008982, 0.277134, 0.185228, 0.489898, 0.13286189992853525, 0.276194, 0.190074, 0.493001, 0.1366546375186384, 0.275191, 0.194905, 0.496005, 0.14044834239708384, 0.274128, 0.199721, 0.498911, 0.14424204727552928, 0.273006, 0.20452, 0.501721, 0.1480347848656324, 0.271828, 0.209303, 0.504434, 0.15182848974407784, 0.270595, 0.214069, 0.507052, 0.155621227334181, 0.269308, 0.218818, 0.509577, 0.15941493221262643, 0.267968, 0.223549, 0.512008, 0.16320766980272958, 0.26658, 0.228262, 0.514349, 0.167001374681175, 0.265145, 0.232956, 0.516599, 0.17079507955962042, 0.263663, 0.237631, 0.518762, 0.17458781714972357, 0.262138, 0.242286, 0.520837, 0.178381522028169, 0.260571, 0.246922, 0.522828, 0.18217425961827216, 0.258965, 0.251537, 0.524736, 0.1859679644967176, 0.257322, 0.25613, 0.526563, 0.18976070208682075, 0.255645, 0.260703, 0.528312, 0.19355440696526618, 0.253935, 0.265254, 0.529983, 0.1973481118437116, 0.252194, 0.269783, 0.531579, 0.20114084943381474, 0.250425, 0.27429, 0.533103, 0.20493455431226018, 0.248629, 0.278775, 0.534556, 0.20872729190236333, 0.246811, 0.283237, 0.535941, 0.21252099678080877, 0.244972, 0.287675, 0.53726, 0.21631373437091192, 0.243113, 0.292092, 0.538516, 0.22010743924935733, 0.241237, 0.296485, 0.539709, 0.22390114412780276, 0.239346, 0.300855, 0.540844, 0.22769388171790592, 0.237441, 0.305202, 0.541921, 0.23148758659635135, 0.235526, 0.309527, 0.542944, 0.2352803241864545, 0.233603, 0.313828, 0.543914, 0.23907402906489994, 0.231674, 0.318106, 0.544834, 0.24286676665500306, 0.229739, 0.322361, 0.545706, 0.24666047153344853, 0.227802, 0.326594, 0.546532, 0.25045417641189394, 0.225863, 0.330805, 0.547314, 0.2542469140019971, 0.223925, 0.334994, 0.548053, 0.2580406188804425, 0.221989, 0.339161, 0.548752, 0.2618333564705457, 0.220057, 0.343307, 0.549413, 0.26562706134899106, 0.21813, 0.347432, 0.550038, 0.26941979893909423, 0.21621, 0.351535, 0.550627, 0.2732135038175397, 0.214298, 0.355619, 0.551184, 0.2770072086959851, 0.212395, 0.359683, 0.55171, 0.2807999462860883, 0.210503, 0.363727, 0.552206, 0.28459365116453367, 0.208623, 0.367752, 0.552675, 0.28838638875463685, 0.206756, 0.371758, 0.553117, 0.2921800936330822, 0.204903, 0.375746, 0.553533, 0.2959728312231854, 0.203063, 0.379716, 0.553925, 0.2997665361016309, 0.201239, 0.38367, 0.554294, 0.30355927369173397, 0.19943, 0.387607, 0.554642, 0.30735297857017946, 0.197636, 0.391528, 0.554969, 0.31114668344862484, 0.19586, 0.395433, 0.555276, 0.314939421038728, 0.1941, 0.399323, 0.555565, 0.31873312591717345, 0.192357, 0.403199, 0.555836, 0.3225258635072766, 0.190631, 0.407061, 0.556089, 0.32631956838572207, 0.188923, 0.41091, 0.556326, 0.33011230597582514, 0.187231, 0.414746, 0.556547, 0.33390601085427063, 0.185556, 0.41857, 0.556753, 0.337699715732716, 0.183898, 0.422383, 0.556944, 0.3414924533228192, 0.182256, 0.426184, 0.55712, 0.3452861582012646, 0.180629, 0.429975, 0.557282, 0.34907889579136775, 0.179019, 0.433756, 0.55743, 0.3528726006698132, 0.177423, 0.437527, 0.557565, 0.3566653382599163, 0.175841, 0.44129, 0.557685, 0.3604590431383618, 0.174274, 0.445044, 0.557792, 0.3642527480168072, 0.172719, 0.448791, 0.557885, 0.36804548560691036, 0.171176, 0.45253, 0.557965, 0.3718391904853558, 0.169646, 0.456262, 0.55803, 0.3756319280754589, 0.168126, 0.459988, 0.558082, 0.37942563295390436, 0.166617, 0.463708, 0.558119, 0.3832183705440075, 0.165117, 0.467423, 0.558141, 0.387012075422453, 0.163625, 0.471133, 0.558148, 0.39080578030089835, 0.162142, 0.474838, 0.55814, 0.39459851789100153, 0.160665, 0.47854, 0.558115, 0.39839222276944697, 0.159194, 0.482237, 0.558073, 0.4021849603595501, 0.157729, 0.485932, 0.558013, 0.40597866523799553, 0.15627, 0.489624, 0.557936, 0.40977140282809865, 0.154815, 0.493313, 0.55784, 0.41356510770654414, 0.153364, 0.497, 0.557724, 0.4173588125849896, 0.151918, 0.500685, 0.557587, 0.4211515501750927, 0.150476, 0.504369, 0.55743, 0.42494525505353814, 0.149039, 0.508051, 0.55725, 0.42873799264364126, 0.147607, 0.511733, 0.557049, 0.4325316975220867, 0.14618, 0.515413, 0.556823, 0.4363244351121898, 0.144759, 0.519093, 0.556572, 0.44011813999063526, 0.143343, 0.522773, 0.556295, 0.44391184486908075, 0.141935, 0.526453, 0.555991, 0.4477045824591839, 0.140536, 0.530132, 0.555659, 0.4514982873376293, 0.139147, 0.533812, 0.555298, 0.45529102492773244, 0.13777, 0.537492, 0.554906, 0.45908472980617787, 0.136408, 0.541173, 0.554483, 0.462877467396281, 0.135066, 0.544853, 0.554029, 0.46667117227472643, 0.133743, 0.548535, 0.553541, 0.4704648771531719, 0.132444, 0.552216, 0.553018, 0.47425761474327505, 0.131172, 0.555899, 0.552459, 0.4780513196217205, 0.129933, 0.559582, 0.551864, 0.4818440572118236, 0.128729, 0.563265, 0.551229, 0.48563776209026904, 0.127568, 0.566949, 0.550556, 0.4894304996803722, 0.126453, 0.570633, 0.549841, 0.49322420455881766, 0.125394, 0.574318, 0.549086, 0.4970169421489208, 0.124395, 0.578002, 0.548287, 0.5008106470273662, 0.123463, 0.581687, 0.547445, 0.5046043519058115, 0.122606, 0.585371, 0.546557, 0.5083970894959148, 0.121831, 0.589055, 0.545623, 0.5121907943743602, 0.121148, 0.592739, 0.544641, 0.5159835319644633, 0.120565, 0.596422, 0.543611, 0.5197772368429089, 0.120092, 0.600104, 0.54253, 0.5235699744330119, 0.119738, 0.603785, 0.5414, 0.5273636793114573, 0.119512, 0.607464, 0.540218, 0.5311573841899028, 0.119423, 0.611141, 0.538982, 0.534950121780006, 0.119483, 0.614817, 0.537692, 0.5387438266584513, 0.119699, 0.61849, 0.536347, 0.5425365642485545, 0.120081, 0.622161, 0.534946, 0.546330269127, 0.120638, 0.625828, 0.533488, 0.5501230067171031, 0.12138, 0.629492, 0.531973, 0.5539167115955486, 0.122312, 0.633153, 0.530398, 0.5577104164739939, 0.123444, 0.636809, 0.528763, 0.5615031540640971, 0.12478, 0.640461, 0.527068, 0.5652968589425426, 0.126326, 0.644107, 0.525311, 0.5690895965326457, 0.128087, 0.647749, 0.523491, 0.5728833014110912, 0.130067, 0.651384, 0.521608, 0.5766760390011942, 0.132268, 0.655014, 0.519661, 0.5804697438796397, 0.134692, 0.658636, 0.517649, 0.5842634487580851, 0.137339, 0.662252, 0.515571, 0.5880561863481883, 0.14021, 0.665859, 0.513427, 0.5918498912266338, 0.143303, 0.669459, 0.511215, 0.5956426288167368, 0.146616, 0.67305, 0.508936, 0.5994363336951823, 0.150148, 0.676631, 0.506589, 0.6032290712852855, 0.153894, 0.680203, 0.504172, 0.6070227761637309, 0.157851, 0.683765, 0.501686, 0.6108164810421762, 0.162016, 0.687316, 0.499129, 0.6146092186322795, 0.166383, 0.690856, 0.496502, 0.6184029235107249, 0.170948, 0.694384, 0.493803, 0.622195661100828, 0.175707, 0.6979, 0.491033, 0.6259893659792736, 0.180653, 0.701402, 0.488189, 0.6297821035693766, 0.185783, 0.704891, 0.485273, 0.633575808447822, 0.19109, 0.708366, 0.482284, 0.6373695133262675, 0.196571, 0.711827, 0.479221, 0.6411622509163707, 0.202219, 0.715272, 0.476084, 0.6449559557948161, 0.20803, 0.718701, 0.472873, 0.6487486933849191, 0.214, 0.722114, 0.469588, 0.6525423982633647, 0.220124, 0.725509, 0.466226, 0.6563351358534678, 0.226397, 0.728888, 0.462789, 0.6601288407319132, 0.232815, 0.732247, 0.459277, 0.6639225456103586, 0.239374, 0.735588, 0.455688, 0.6677152832004618, 0.24607, 0.73891, 0.452024, 0.6715089880789072, 0.252899, 0.742211, 0.448284, 0.6753017256690104, 0.259857, 0.745492, 0.444467, 0.6790954305474558, 0.266941, 0.748751, 0.440573, 0.6828881681375589, 0.274149, 0.751988, 0.436601, 0.6866818730160044, 0.281477, 0.755203, 0.432552, 0.6904746106061076, 0.288921, 0.758394, 0.428426, 0.694268315484553, 0.296479, 0.761561, 0.424223, 0.6980620203629985, 0.304148, 0.764704, 0.419943, 0.7018547579531015, 0.311925, 0.767822, 0.415586, 0.7056484628315469, 0.319809, 0.770914, 0.411152, 0.7094412004216502, 0.327796, 0.77398, 0.40664, 0.7132349053000956, 0.335885, 0.777018, 0.402049, 0.7170276428901987, 0.344074, 0.780029, 0.397381, 0.7208213477686441, 0.35236, 0.783011, 0.392636, 0.7246150526470896, 0.360741, 0.785964, 0.387814, 0.7284077902371927, 0.369214, 0.788888, 0.382914, 0.7322014951156381, 0.377779, 0.791781, 0.377939, 0.7359942327057413, 0.386433, 0.794644, 0.372886, 0.7397879375841867, 0.395174, 0.797475, 0.367757, 0.7435806751742898, 0.404001, 0.800275, 0.362552, 0.7473743800527354, 0.412913, 0.803041, 0.357269, 0.7511680849311808, 0.421908, 0.805774, 0.35191, 0.7549608225212838, 0.430983, 0.808473, 0.346476, 0.7587545273997293, 0.440137, 0.811138, 0.340967, 0.7625472649898325, 0.449368, 0.813768, 0.335384, 0.7663409698682779, 0.458674, 0.816363, 0.329727, 0.7701337074583811, 0.468053, 0.818921, 0.323998, 0.7739274123368265, 0.477504, 0.821444, 0.318195, 0.7777211172152719, 0.487026, 0.823929, 0.312321, 0.781513854805375, 0.496615, 0.826376, 0.306377, 0.7853075596838205, 0.506271, 0.828786, 0.300362, 0.7891002972739236, 0.515992, 0.831158, 0.294279, 0.792894002152369, 0.525776, 0.833491, 0.288127, 0.7966867397424722, 0.535621, 0.835785, 0.281908, 0.8004804446209177, 0.545524, 0.838039, 0.275626, 0.8042741494993632, 0.555484, 0.840254, 0.269281, 0.8080668870894662, 0.565498, 0.84243, 0.262877, 0.8118605919679116, 0.575563, 0.844566, 0.256415, 0.8156533295580148, 0.585678, 0.846661, 0.249897, 0.8194470344364603, 0.595839, 0.848717, 0.243329, 0.8232397720265634, 0.606045, 0.850733, 0.236712, 0.8270334769050088, 0.616293, 0.852709, 0.230052, 0.8308271817834543, 0.626579, 0.854645, 0.223353, 0.8346199193735574, 0.636902, 0.856542, 0.21662, 0.8384136242520028, 0.647257, 0.8584, 0.209861, 0.8422063618421061, 0.657642, 0.860219, 0.203082, 0.8460000667205514, 0.668054, 0.861999, 0.196293, 0.8497928043106545, 0.678489, 0.863742, 0.189503, 0.8535865091891001, 0.688944, 0.865448, 0.182725, 0.8573802140675455, 0.699415, 0.867117, 0.175971, 0.8611729516576485, 0.709898, 0.868751, 0.169257, 0.864966656536094, 0.720391, 0.87035, 0.162603, 0.8687593941261972, 0.730889, 0.871916, 0.156029, 0.8725530990046426, 0.741388, 0.873449, 0.149561, 0.8763458365947457, 0.751884, 0.874951, 0.143228, 0.8801395414731911, 0.762373, 0.876424, 0.137064, 0.8839322790632943, 0.772852, 0.877868, 0.131109, 0.8877259839417397, 0.783315, 0.879285, 0.125405, 0.8915196888201852, 0.79376, 0.880678, 0.120005, 0.8953124264102884, 0.804182, 0.882046, 0.114965, 0.8991061312887337, 0.814576, 0.883393, 0.110347, 0.9028988688788369, 0.82494, 0.88472, 0.106217, 0.9066925737572823, 0.83527, 0.886029, 0.102646, 0.9104853113473855, 0.845561, 0.887322, 0.099702, 0.914279016225831, 0.85581, 0.888601, 0.097452, 0.9180727211042763, 0.866013, 0.889868, 0.095953, 0.9218654586943795, 0.876168, 0.891125, 0.09525, 0.925659163572825, 0.886271, 0.892374, 0.095374, 0.9294519011629281, 0.89632, 0.893616, 0.096335, 0.9332456060413734, 0.906311, 0.894855, 0.098125, 0.9370383436314766, 0.916242, 0.896091, 0.100717, 0.9408320485099221, 0.926106, 0.89733, 0.104071, 0.9446257533883675, 0.935904, 0.89857, 0.108131, 0.9484184909784708, 0.945636, 0.899815, 0.112838, 0.9522121958569161, 0.9553, 0.901065, 0.118128, 0.9560049334470192, 0.964894, 0.902323, 0.123941, 0.9597986383254646, 0.974417, 0.90359, 0.130215, 0.9635913759155679, 0.983868, 0.904867, 0.136897, 0.9673850807940133, 0.993248, 0.906157, 0.143936]
-chiLUT.ShowDataHistogram = 1
-chiLUT.DataHistogramNumberOfBins = 44
-chiLUT.NanColor = [1.0, 0.0, 0.0]
-chiLUT.ScalarRangeInitialized = 1.0
-
-# get opacity transfer function/opacity map for 'chi'
-chiPWF = GetOpacityTransferFunction('chi')
-chiPWF.Points = [9.673850807940134e-05, 1.0, 0.5, 0.0, 0.07736533880233765, 0.7692307829856873, 0.5, 0.0, 0.1517721265554428, 0.6153846383094788, 0.5, 0.0, 0.24334973096847534, 0.43195265531539917, 0.5, 0.0, 0.38071611523628235, 0.21893490850925446, 0.5, 0.0, 0.5781803447501049, 0.0, 0.5, 0.0, 0.9673850807940133, 0.0, 0.5, 0.0]
-chiPWF.ScalarRangeInitialized = 1
-
-# trace defaults for the display properties.
-resampleToImage1Display.Representation = 'Volume'
-resampleToImage1Display.ColorArrayName = ['POINTS', 'chi']
-resampleToImage1Display.LookupTable = chiLUT
-resampleToImage1Display.SelectTCoordArray = 'None'
-resampleToImage1Display.SelectNormalArray = 'None'
-resampleToImage1Display.SelectTangentArray = 'None'
-resampleToImage1Display.OSPRayScaleArray = 'chi'
-resampleToImage1Display.OSPRayScaleFunction = 'PiecewiseFunction'
-resampleToImage1Display.SelectOrientationVectors = 'None'
-resampleToImage1Display.ScaleFactor = 6.999993000000001
-resampleToImage1Display.SelectScaleArray = 'None'
-resampleToImage1Display.GlyphType = 'Arrow'
-resampleToImage1Display.GlyphTableIndexArray = 'None'
-resampleToImage1Display.GaussianRadius = 0.34999965000000005
-resampleToImage1Display.SetScaleArray = ['POINTS', 'chi']
-resampleToImage1Display.ScaleTransferFunction = 'PiecewiseFunction'
-resampleToImage1Display.OpacityArray = ['POINTS', 'chi']
-resampleToImage1Display.OpacityTransferFunction = 'PiecewiseFunction'
-resampleToImage1Display.DataAxesGrid = 'GridAxesRepresentation'
-resampleToImage1Display.PolarAxes = 'PolarAxesRepresentation'
-resampleToImage1Display.ScalarOpacityUnitDistance = 1.224681164507726
-resampleToImage1Display.ScalarOpacityFunction = chiPWF
-resampleToImage1Display.OpacityArrayName = ['POINTS', 'chi']
-resampleToImage1Display.SliceFunction = 'Plane'
-resampleToImage1Display.Slice = 49
-
-# init the 'PiecewiseFunction' selected for 'ScaleTransferFunction'
-resampleToImage1Display.ScaleTransferFunction.Points = [0.16307053589122933, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
-
-# init the 'PiecewiseFunction' selected for 'OpacityTransferFunction'
-resampleToImage1Display.OpacityTransferFunction.Points = [0.16307053589122933, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
-
-# init the 'Plane' selected for 'SliceFunction'
-resampleToImage1Display.SliceFunction.Origin = [32.0, 32.0, 32.0]
-
-# ----------------------------------------------------------------
-# setup color maps and opacity mapes used in the visualization
-# note: the Get..() functions create a new object, if needed
-# ----------------------------------------------------------------
-
-# ----------------------------------------------------------------
-# setup extractors
-# ----------------------------------------------------------------
-
-# create extractor
-pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
-# trace defaults for the extractor.
-# init the 'PNG' selected for 'Writer'
-pNG1.Writer.FileName = 'VolumeChi_%.6ts%cm.png'
-pNG1.Writer.ImageResolution = [885, 774]
-pNG1.Writer.Format = 'PNG'
-
-# ----------------------------------------------------------------
-# restore active source
-SetActiveSource(resampleToImage1)
-# ----------------------------------------------------------------
-
-# ------------------------------------------------------------------------------
-# Catalyst options
-from paraview import catalyst
-options = catalyst.Options()
-options.GlobalTrigger = 'TimeStep'
-options.EnableCatalystLive = 1
-options.CatalystLiveTrigger = 'TimeStep'
-
-# ------------------------------------------------------------------------------
-if __name__ == '__main__':
-    from paraview.simple import SaveExtractsUsingCatalystOptions
-    # Code for non in-situ environments; if executing in post-processing
-    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
-    SaveExtractsUsingCatalystOptions(options)
diff --git a/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_resample_chi.py b/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_resample_chi.py
new file mode 100644
index 000000000..643d4786c
--- /dev/null
+++ b/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_resample_chi.py
@@ -0,0 +1,152 @@
+# script-version: 2.0
+# Catalyst state generated using paraview version 5.9.1
+
+#### import the simple module from the paraview
+from paraview.simple import *
+#### disable automatic camera reset on 'Show'
+paraview.simple._DisableFirstRenderCameraReset()
+
+# ----------------------------------------------------------------
+# setup views used in the visualization
+# ----------------------------------------------------------------
+
+# get the material library
+materialLibrary1 = GetMaterialLibrary()
+
+# Create a new 'Render View'
+renderView1 = CreateView('RenderView')
+renderView1.ViewSize = [885, 774]
+renderView1.AxesGrid = 'GridAxes3DActor'
+renderView1.CenterAxesVisibility = 1
+renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
+renderView1.StereoType = 'Crystal Eyes'
+renderView1.CameraPosition = [20.770091346798225, 54.44320054580248, 46.07511197528731]
+renderView1.CameraFocalPoint = [32.000000000000014, 31.999999999999986, 31.999999999999993]
+renderView1.CameraViewUp = [0.37830852443531104, -0.3485213720084932, 0.8575637081831853]
+renderView1.CameraFocalDisk = 1.0
+renderView1.CameraParallelScale = 60.6217782649107
+renderView1.BackEnd = 'OSPRay raycaster'
+renderView1.OSPRayMaterialLibrary = materialLibrary1
+renderView1.Background = [0.0, 0.0, 0.0]
+
+SetActiveView(None)
+
+# ----------------------------------------------------------------
+# setup view layouts
+# ----------------------------------------------------------------
+
+# create new layout object 'Layout #1'
+layout1 = CreateLayout(name='Layout #1')
+layout1.AssignView(0, renderView1)
+layout1.SetSize(885, 774)
+
+# ----------------------------------------------------------------
+# restore active view
+SetActiveView(renderView1)
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup the data processing pipelines
+# ----------------------------------------------------------------
+
+# create a new 'VisItChomboReader'
+input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000002.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000003.3d.hdf5'])
+input.MeshStatus = ['Mesh']
+input.CellArrayStatus = ['chi']
+
+# create a new 'Resample To Image'
+resampleToImage1 = ResampleToImage(registrationName='ResampleToImage1', Input=input)
+resampleToImage1.SamplingBounds = [-3.0, 67.0, -3.0, 67.0, -3.0, 67.0]
+
+# ----------------------------------------------------------------
+# setup the visualization in view 'renderView1'
+# ----------------------------------------------------------------
+
+# show data from resampleToImage1
+resampleToImage1Display = Show(resampleToImage1, renderView1, 'UniformGridRepresentation')
+
+# get color transfer function/color map for 'chi'
+chiLUT = GetColorTransferFunction('chi')
+chiLUT.RGBPoints = [0.006705302783959541, 1.0, 0.988235, 0.968627, 0.025918898344160616, 1.0, 0.952941, 0.878431, 0.05473929168446223, 0.968627, 0.905882, 0.776471, 0.10277328058496492, 0.94902, 0.898039, 0.647059, 0.1508072694854676, 0.901961, 0.878431, 0.556863, 0.1988412583859703, 0.847059, 0.858824, 0.482353, 0.24687524728647298, 0.690196, 0.819608, 0.435294, 0.2949092361869757, 0.513725, 0.768627, 0.384314, 0.3429432250874783, 0.337255, 0.721569, 0.337255, 0.3909772139879811, 0.278431, 0.658824, 0.392157, 0.4390112028884838, 0.231373, 0.639216, 0.435294, 0.4870451917889864, 0.203922, 0.6, 0.486275, 0.5350791806894891, 0.172549, 0.568627, 0.537255, 0.5831131695899918, 0.141176, 0.517647, 0.54902, 0.6311471584904945, 0.133333, 0.458824, 0.541176, 0.6791811473909971, 0.12549, 0.396078, 0.529412, 0.7272151362914999, 0.117647, 0.321569, 0.521569, 0.7752491251920026, 0.121569, 0.258824, 0.509804, 0.8232831140925052, 0.133333, 0.227451, 0.501961, 0.8713171029930079, 0.145098, 0.192157, 0.490196, 0.9193510918935106, 0.188235, 0.164706, 0.470588, 0.9673850807940133, 0.258824, 0.196078, 0.439216]
+chiLUT.ColorSpace = 'RGB'
+chiLUT.NanColor = [0.25, 0.0, 0.0]
+chiLUT.NanOpacity = 0.0
+chiLUT.ScalarRangeInitialized = 1.0
+
+# get opacity transfer function/opacity map for 'chi'
+chiPWF = GetOpacityTransferFunction('chi')
+chiPWF.Points = [0.006705302783959541, 1.0, 0.5, 0.0, 0.055023517459630966, 0.9053254723548889, 0.5, 0.0, 0.1061839833855629, 0.7692307829856873, 0.5, 0.0, 0.21418939530849457, 0.47337278723716736, 0.5, 0.0, 0.37335529923439026, 0.23076923191547394, 0.5, 0.0, 0.5808394117544118, 0.0, 0.5, 0.0, 0.9673850807940133, 0.0, 0.5, 0.0]
+chiPWF.ScalarRangeInitialized = 1
+
+# trace defaults for the display properties.
+resampleToImage1Display.Representation = 'Volume'
+resampleToImage1Display.ColorArrayName = ['POINTS', 'chi']
+resampleToImage1Display.LookupTable = chiLUT
+resampleToImage1Display.SelectTCoordArray = 'None'
+resampleToImage1Display.SelectNormalArray = 'None'
+resampleToImage1Display.SelectTangentArray = 'None'
+resampleToImage1Display.OSPRayScaleArray = 'chi'
+resampleToImage1Display.OSPRayScaleFunction = 'PiecewiseFunction'
+resampleToImage1Display.SelectOrientationVectors = 'None'
+resampleToImage1Display.ScaleFactor = 6.999993000000001
+resampleToImage1Display.SelectScaleArray = 'None'
+resampleToImage1Display.GlyphType = 'Arrow'
+resampleToImage1Display.GlyphTableIndexArray = 'None'
+resampleToImage1Display.GaussianRadius = 0.34999965000000005
+resampleToImage1Display.SetScaleArray = ['POINTS', 'chi']
+resampleToImage1Display.ScaleTransferFunction = 'PiecewiseFunction'
+resampleToImage1Display.OpacityArray = ['POINTS', 'chi']
+resampleToImage1Display.OpacityTransferFunction = 'PiecewiseFunction'
+resampleToImage1Display.DataAxesGrid = 'GridAxesRepresentation'
+resampleToImage1Display.PolarAxes = 'PolarAxesRepresentation'
+resampleToImage1Display.ScalarOpacityUnitDistance = 1.224681164507726
+resampleToImage1Display.ScalarOpacityFunction = chiPWF
+resampleToImage1Display.OpacityArrayName = ['POINTS', 'chi']
+resampleToImage1Display.SliceFunction = 'Plane'
+resampleToImage1Display.Slice = 49
+
+# init the 'PiecewiseFunction' selected for 'ScaleTransferFunction'
+resampleToImage1Display.ScaleTransferFunction.Points = [0.16307053589122933, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
+
+# init the 'PiecewiseFunction' selected for 'OpacityTransferFunction'
+resampleToImage1Display.OpacityTransferFunction.Points = [0.16307053589122933, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
+
+# init the 'Plane' selected for 'SliceFunction'
+resampleToImage1Display.SliceFunction.Origin = [32.0, 32.0, 32.0]
+
+# ----------------------------------------------------------------
+# setup color maps and opacity mapes used in the visualization
+# note: the Get..() functions create a new object, if needed
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup extractors
+# ----------------------------------------------------------------
+
+# create extractor
+pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
+# trace defaults for the extractor.
+# init the 'PNG' selected for 'Writer'
+pNG1.Writer.FileName = 'VolumeChi_%.6ts%cm.png'
+pNG1.Writer.ImageResolution = [885, 778]
+pNG1.Writer.Format = 'PNG'
+
+# ----------------------------------------------------------------
+# restore active source
+SetActiveSource(pNG1)
+# ----------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# Catalyst options
+from paraview import catalyst
+options = catalyst.Options()
+options.GlobalTrigger = 'TimeStep'
+options.EnableCatalystLive = 1
+options.CatalystLiveTrigger = 'TimeStep'
+
+# ------------------------------------------------------------------------------
+if __name__ == '__main__':
+    from paraview.simple import SaveExtractsUsingCatalystOptions
+    # Code for non in-situ environments; if executing in post-processing
+    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
+    SaveExtractsUsingCatalystOptions(options)
diff --git a/Examples/BinaryBH/catalyst_scripts/volume_raytrace_chi.py b/Examples/BinaryBH/catalyst_scripts/volume_raytrace_chi.py
deleted file mode 100644
index 7ab651950..000000000
--- a/Examples/BinaryBH/catalyst_scripts/volume_raytrace_chi.py
+++ /dev/null
@@ -1,133 +0,0 @@
-# script-version: 2.0
-# Catalyst state generated using paraview version 5.9.1
-
-#### import the simple module from the paraview
-from paraview.simple import *
-#### disable automatic camera reset on 'Show'
-paraview.simple._DisableFirstRenderCameraReset()
-
-# ----------------------------------------------------------------
-# setup views used in the visualization
-# ----------------------------------------------------------------
-
-# get the material library
-materialLibrary1 = GetMaterialLibrary()
-
-# Create a new 'Render View'
-renderView1 = CreateView('RenderView')
-renderView1.ViewSize = [885, 778]
-renderView1.AxesGrid = 'GridAxes3DActor'
-renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
-renderView1.StereoType = 'Crystal Eyes'
-renderView1.CameraPosition = [20.770091346798225, 54.44320054580248, 46.07511197528731]
-renderView1.CameraFocalPoint = [32.000000000000014, 31.999999999999986, 31.999999999999993]
-renderView1.CameraViewUp = [0.37830852443531104, -0.3485213720084932, 0.8575637081831853]
-renderView1.CameraFocalDisk = 1.0
-renderView1.CameraParallelScale = 60.6217782649107
-renderView1.EnableRayTracing = 1
-renderView1.BackEnd = 'OSPRay raycaster'
-renderView1.OSPRayMaterialLibrary = materialLibrary1
-
-SetActiveView(None)
-
-# ----------------------------------------------------------------
-# setup view layouts
-# ----------------------------------------------------------------
-
-# create new layout object 'Layout #1'
-layout1 = CreateLayout(name='Layout #1')
-layout1.AssignView(0, renderView1)
-layout1.SetSize(885, 778)
-
-# ----------------------------------------------------------------
-# restore active view
-SetActiveView(renderView1)
-# ----------------------------------------------------------------
-
-# ----------------------------------------------------------------
-# setup the data processing pipelines
-# ----------------------------------------------------------------
-
-# create a new 'VisItChomboReader'
-binaryBHp_00000 = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000002.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000003.3d.hdf5'])
-binaryBHp_00000.MeshStatus = ['Mesh']
-binaryBHp_00000.CellArrayStatus = ['chi']
-
-# ----------------------------------------------------------------
-# setup the visualization in view 'renderView1'
-# ----------------------------------------------------------------
-
-# show data from binaryBHp_00000
-binaryBHp_00000Display = Show(binaryBHp_00000, renderView1, 'AMRRepresentation')
-
-# get color transfer function/color map for 'chi'
-chiLUT = GetColorTransferFunction('chi')
-chiLUT.RGBPoints = [0.006705302783959541, 0.0862745098039216, 0.00392156862745098, 0.298039215686275, 0.03584677977638898, 0.113725, 0.0235294, 0.45098, 0.060048963685229544, 0.105882, 0.0509804, 0.509804, 0.07684234902507095, 0.0392157, 0.0392157, 0.560784, 0.09314182943734466, 0.0313725, 0.0980392, 0.6, 0.10894734875818932, 0.0431373, 0.164706, 0.639216, 0.13166776172045955, 0.054902, 0.243137, 0.678431, 0.16179708557943404, 0.054902, 0.317647, 0.709804, 0.19884125838596878, 0.0509804, 0.396078, 0.741176, 0.22285825283622032, 0.0392157, 0.466667, 0.768627, 0.24687524728647184, 0.0313725, 0.537255, 0.788235, 0.27194187206163456, 0.0313725, 0.615686, 0.811765, 0.2976258269170425, 0.0235294, 0.709804, 0.831373, 0.3233098379363089, 0.0509804, 0.8, 0.85098, 0.3445484799520089, 0.0705882, 0.854902, 0.870588, 0.3643054071849909, 0.262745, 0.901961, 0.862745, 0.3815926974524059, 0.423529, 0.941176, 0.87451, 0.40826451832681654, 0.572549, 0.964706, 0.835294, 0.4260457696856576, 0.658824, 0.980392, 0.843137, 0.43888774711336165, 0.764706, 0.980392, 0.866667, 0.4527175624781391, 0.827451, 0.980392, 0.886275, 0.4798833163620467, 0.913725, 0.988235, 0.937255, 0.4882800090319688, 1.0, 1.0, 0.972549019607843, 0.49667670170189104, 0.988235, 0.980392, 0.870588, 0.5090247742020214, 0.992156862745098, 0.972549019607843, 0.803921568627451, 0.518409304809017, 0.992157, 0.964706, 0.713725, 0.5342148241298618, 0.988235, 0.956863, 0.643137, 0.559404874057699, 0.980392, 0.917647, 0.509804, 0.5806435722372544, 0.968627, 0.87451, 0.407843, 0.6023761753443805, 0.94902, 0.823529, 0.321569, 0.6181816946652244, 0.929412, 0.776471, 0.278431, 0.6413960687189206, 0.909804, 0.717647, 0.235294, 0.6621408058070486, 0.890196, 0.658824, 0.196078, 0.679181147390997, 0.878431, 0.619608, 0.168627, 0.7031981418412483, 0.870588, 0.54902, 0.156863, 0.7272151362914998, 0.85098, 0.47451, 0.145098, 0.751232130741751, 0.831373, 0.411765, 0.133333, 0.7752491251920026, 0.811765, 0.345098, 0.113725, 0.7992661196422539, 0.788235, 0.266667, 0.0941176, 0.8232831140925052, 0.741176, 0.184314, 0.0745098, 0.8473001085427566, 0.690196, 0.12549, 0.0627451, 0.8713171029930079, 0.619608, 0.0627451, 0.0431373, 0.8937905905179446, 0.54902, 0.027451, 0.0705882, 0.9135475010772842, 0.470588, 0.0156863, 0.0901961, 0.935774024030506, 0.4, 0.00392157, 0.101961, 0.9673850807940133, 0.188235294117647, 0.0, 0.0705882352941176]
-chiLUT.ColorSpace = 'Lab'
-chiLUT.NanColor = [1.0, 0.0, 0.0]
-chiLUT.ScalarRangeInitialized = 1.0
-
-# get opacity transfer function/opacity map for 'chi'
-chiPWF = GetOpacityTransferFunction('chi')
-chiPWF.Points = [0.006705302783959541, 1.0, 0.5, 0.0, 0.055023517459630966, 0.6094674468040466, 0.5, 0.0, 0.126079723238945, 0.4378698170185089, 0.5, 0.0, 0.22840063273906708, 0.23076923191547394, 0.5, 0.0, 0.37619754672050476, 0.08875739574432373, 0.5, 0.0, 0.5808393955230713, 0.0, 0.5, 0.0, 0.9673850536346436, 0.0, 0.5, 0.0]
-chiPWF.ScalarRangeInitialized = 1
-
-# trace defaults for the display properties.
-binaryBHp_00000Display.Representation = 'Volume'
-binaryBHp_00000Display.ColorArrayName = ['CELLS', 'chi']
-binaryBHp_00000Display.LookupTable = chiLUT
-binaryBHp_00000Display.SelectTCoordArray = 'None'
-binaryBHp_00000Display.SelectNormalArray = 'None'
-binaryBHp_00000Display.SelectTangentArray = 'None'
-binaryBHp_00000Display.OSPRayScaleFunction = 'PiecewiseFunction'
-binaryBHp_00000Display.SelectOrientationVectors = 'None'
-binaryBHp_00000Display.ScaleFactor = 7.0
-binaryBHp_00000Display.SelectScaleArray = 'None'
-binaryBHp_00000Display.GlyphType = 'Arrow'
-binaryBHp_00000Display.GlyphTableIndexArray = 'None'
-binaryBHp_00000Display.GaussianRadius = 0.35000000000000003
-binaryBHp_00000Display.SetScaleArray = [None, '']
-binaryBHp_00000Display.ScaleTransferFunction = 'PiecewiseFunction'
-binaryBHp_00000Display.OpacityArray = [None, '']
-binaryBHp_00000Display.OpacityTransferFunction = 'PiecewiseFunction'
-binaryBHp_00000Display.DataAxesGrid = 'GridAxesRepresentation'
-binaryBHp_00000Display.PolarAxes = 'PolarAxesRepresentation'
-binaryBHp_00000Display.ScalarOpacityUnitDistance = 0.8192401766441247
-binaryBHp_00000Display.ScalarOpacityFunction = chiPWF
-
-# ----------------------------------------------------------------
-# setup color maps and opacity mapes used in the visualization
-# note: the Get..() functions create a new object, if needed
-# ----------------------------------------------------------------
-
-# ----------------------------------------------------------------
-# setup extractors
-# ----------------------------------------------------------------
-
-# create extractor
-pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
-# trace defaults for the extractor.
-# init the 'PNG' selected for 'Writer'
-pNG1.Writer.FileName = 'RenderView1_%.6ts%cm.png'
-pNG1.Writer.ImageResolution = [885, 778]
-pNG1.Writer.Format = 'PNG'
-
-# ----------------------------------------------------------------
-# restore active source
-SetActiveSource(pNG1)
-# ----------------------------------------------------------------
-
-# ------------------------------------------------------------------------------
-# Catalyst options
-from paraview import catalyst
-options = catalyst.Options()
-options.GlobalTrigger = 'TimeStep'
-options.EnableCatalystLive = 1
-options.CatalystLiveTrigger = 'TimeStep'
-
-# ------------------------------------------------------------------------------
-if __name__ == '__main__':
-    from paraview.simple import SaveExtractsUsingCatalystOptions
-    # Code for non in-situ environments; if executing in post-processing
-    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
-    SaveExtractsUsingCatalystOptions(options)
diff --git a/Examples/BinaryBH/catalyst_scripts/volume_raytrace_resample_chi.py b/Examples/BinaryBH/catalyst_scripts/volume_raytrace_resample_chi.py
new file mode 100644
index 000000000..5e08ff80b
--- /dev/null
+++ b/Examples/BinaryBH/catalyst_scripts/volume_raytrace_resample_chi.py
@@ -0,0 +1,153 @@
+# script-version: 2.0
+# Catalyst state generated using paraview version 5.9.1
+
+#### import the simple module from the paraview
+from paraview.simple import *
+#### disable automatic camera reset on 'Show'
+paraview.simple._DisableFirstRenderCameraReset()
+
+# ----------------------------------------------------------------
+# setup views used in the visualization
+# ----------------------------------------------------------------
+
+# get the material library
+materialLibrary1 = GetMaterialLibrary()
+
+# Create a new 'Render View'
+renderView1 = CreateView('RenderView')
+renderView1.ViewSize = [885, 774]
+renderView1.AxesGrid = 'GridAxes3DActor'
+renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
+renderView1.StereoType = 'Crystal Eyes'
+renderView1.CameraPosition = [20.770091346798225, 54.44320054580248, 46.07511197528731]
+renderView1.CameraFocalPoint = [32.000000000000014, 31.999999999999986, 31.999999999999993]
+renderView1.CameraViewUp = [0.37830852443531104, -0.3485213720084932, 0.8575637081831853]
+renderView1.CameraFocalDisk = 1.0
+renderView1.CameraParallelScale = 60.6217782649107
+renderView1.EnableRayTracing = 1
+renderView1.BackEnd = 'OSPRay raycaster'
+renderView1.SamplesPerPixel = 3
+renderView1.OSPRayMaterialLibrary = materialLibrary1
+renderView1.Background = [0.0, 0.0, 0.0]
+
+SetActiveView(None)
+
+# ----------------------------------------------------------------
+# setup view layouts
+# ----------------------------------------------------------------
+
+# create new layout object 'Layout #1'
+layout1 = CreateLayout(name='Layout #1')
+layout1.AssignView(0, renderView1)
+layout1.SetSize(885, 774)
+
+# ----------------------------------------------------------------
+# restore active view
+SetActiveView(renderView1)
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup the data processing pipelines
+# ----------------------------------------------------------------
+
+# create a new 'VisItChomboReader'
+input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000002.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000003.3d.hdf5'])
+input.MeshStatus = ['Mesh']
+input.CellArrayStatus = ['chi']
+
+# create a new 'Resample To Image'
+resampleToImage1 = ResampleToImage(registrationName='ResampleToImage1', Input=input)
+resampleToImage1.SamplingBounds = [-3.0, 67.0, -3.0, 67.0, -3.0, 67.0]
+
+# ----------------------------------------------------------------
+# setup the visualization in view 'renderView1'
+# ----------------------------------------------------------------
+
+# show data from resampleToImage1
+resampleToImage1Display = Show(resampleToImage1, renderView1, 'UniformGridRepresentation')
+
+# get color transfer function/color map for 'chi'
+chiLUT = GetColorTransferFunction('chi')
+chiLUT.RGBPoints = [0.006705302783959541, 1.0, 0.988235, 0.968627, 0.025918898344160616, 1.0, 0.952941, 0.878431, 0.05473929168446223, 0.968627, 0.905882, 0.776471, 0.10277328058496492, 0.94902, 0.898039, 0.647059, 0.1508072694854676, 0.901961, 0.878431, 0.556863, 0.1988412583859703, 0.847059, 0.858824, 0.482353, 0.24687524728647298, 0.690196, 0.819608, 0.435294, 0.2949092361869757, 0.513725, 0.768627, 0.384314, 0.3429432250874783, 0.337255, 0.721569, 0.337255, 0.3909772139879811, 0.278431, 0.658824, 0.392157, 0.4390112028884838, 0.231373, 0.639216, 0.435294, 0.4870451917889864, 0.203922, 0.6, 0.486275, 0.5350791806894891, 0.172549, 0.568627, 0.537255, 0.5831131695899918, 0.141176, 0.517647, 0.54902, 0.6311471584904945, 0.133333, 0.458824, 0.541176, 0.6791811473909971, 0.12549, 0.396078, 0.529412, 0.7272151362914999, 0.117647, 0.321569, 0.521569, 0.7752491251920026, 0.121569, 0.258824, 0.509804, 0.8232831140925052, 0.133333, 0.227451, 0.501961, 0.8713171029930079, 0.145098, 0.192157, 0.490196, 0.9193510918935106, 0.188235, 0.164706, 0.470588, 0.9673850807940133, 0.258824, 0.196078, 0.439216]
+chiLUT.ColorSpace = 'RGB'
+chiLUT.NanColor = [0.25, 0.0, 0.0]
+chiLUT.NanOpacity = 0.0
+chiLUT.ScalarRangeInitialized = 1.0
+
+# get opacity transfer function/opacity map for 'chi'
+chiPWF = GetOpacityTransferFunction('chi')
+chiPWF.Points = [0.006705302783959541, 1.0, 0.5, 0.0, 0.055023517459630966, 0.9053254723548889, 0.5, 0.0, 0.1061839833855629, 0.7692307829856873, 0.5, 0.0, 0.21418939530849457, 0.47337278723716736, 0.5, 0.0, 0.37335529923439026, 0.23076923191547394, 0.5, 0.0, 0.5808394117544118, 0.0, 0.5, 0.0, 0.9673850807940133, 0.0, 0.5, 0.0]
+chiPWF.ScalarRangeInitialized = 1
+
+# trace defaults for the display properties.
+resampleToImage1Display.Representation = 'Volume'
+resampleToImage1Display.ColorArrayName = ['POINTS', 'chi']
+resampleToImage1Display.LookupTable = chiLUT
+resampleToImage1Display.SelectTCoordArray = 'None'
+resampleToImage1Display.SelectNormalArray = 'None'
+resampleToImage1Display.SelectTangentArray = 'None'
+resampleToImage1Display.OSPRayScaleArray = 'chi'
+resampleToImage1Display.OSPRayScaleFunction = 'PiecewiseFunction'
+resampleToImage1Display.SelectOrientationVectors = 'None'
+resampleToImage1Display.ScaleFactor = 6.999993000000001
+resampleToImage1Display.SelectScaleArray = 'None'
+resampleToImage1Display.GlyphType = 'Arrow'
+resampleToImage1Display.GlyphTableIndexArray = 'None'
+resampleToImage1Display.GaussianRadius = 0.34999965000000005
+resampleToImage1Display.SetScaleArray = ['POINTS', 'chi']
+resampleToImage1Display.ScaleTransferFunction = 'PiecewiseFunction'
+resampleToImage1Display.OpacityArray = ['POINTS', 'chi']
+resampleToImage1Display.OpacityTransferFunction = 'PiecewiseFunction'
+resampleToImage1Display.DataAxesGrid = 'GridAxesRepresentation'
+resampleToImage1Display.PolarAxes = 'PolarAxesRepresentation'
+resampleToImage1Display.ScalarOpacityUnitDistance = 1.224681164507726
+resampleToImage1Display.ScalarOpacityFunction = chiPWF
+resampleToImage1Display.OpacityArrayName = ['POINTS', 'chi']
+resampleToImage1Display.SliceFunction = 'Plane'
+resampleToImage1Display.Slice = 49
+
+# init the 'PiecewiseFunction' selected for 'ScaleTransferFunction'
+resampleToImage1Display.ScaleTransferFunction.Points = [0.16307053589122933, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
+
+# init the 'PiecewiseFunction' selected for 'OpacityTransferFunction'
+resampleToImage1Display.OpacityTransferFunction.Points = [0.16307053589122933, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
+
+# init the 'Plane' selected for 'SliceFunction'
+resampleToImage1Display.SliceFunction.Origin = [32.0, 32.0, 32.0]
+
+# ----------------------------------------------------------------
+# setup color maps and opacity mapes used in the visualization
+# note: the Get..() functions create a new object, if needed
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup extractors
+# ----------------------------------------------------------------
+
+# create extractor
+pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
+# trace defaults for the extractor.
+# init the 'PNG' selected for 'Writer'
+pNG1.Writer.FileName = 'OSPRayVolumeChi_%.6ts%cm.png'
+pNG1.Writer.ImageResolution = [885, 778]
+pNG1.Writer.Format = 'PNG'
+
+# ----------------------------------------------------------------
+# restore active source
+SetActiveSource(resampleToImage1)
+# ----------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# Catalyst options
+from paraview import catalyst
+options = catalyst.Options()
+options.GlobalTrigger = 'TimeStep'
+options.EnableCatalystLive = 1
+options.CatalystLiveTrigger = 'TimeStep'
+
+# ------------------------------------------------------------------------------
+if __name__ == '__main__':
+    from paraview.simple import SaveExtractsUsingCatalystOptions
+    # Code for non in-situ environments; if executing in post-processing
+    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
+    SaveExtractsUsingCatalystOptions(options)
diff --git a/Examples/BinaryBH/params_catalyst.txt b/Examples/BinaryBH/params_catalyst.txt
index 08397f427..5d14a8486 100644
--- a/Examples/BinaryBH/params_catalyst.txt
+++ b/Examples/BinaryBH/params_catalyst.txt
@@ -185,7 +185,7 @@ modes = 2 0 # l m for spherical harmonics
 # ParaView Catalyst parameters
 
 activate_catalyst = true
-catalyst_script_path = catalyst_scripts/slice_chi.py
-catalyst_coprocess_level = 0
+catalyst_script_path = catalyst_scripts/volume_raytrace_resample_chi.py
+catalyst_coprocess_level = 1
 num_catalyst_vars = 1
 caralyst_vars = chi

From 5df0a4ee42ec15895f1f7837dd9957d3c39bc90c Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Fri, 21 Jan 2022 14:07:09 +0000
Subject: [PATCH 18/63] Create insitu makefile target

This includes "Insitu" in the filename. Note that the default example
executable will be built and linked with ParaView Catalyst if
PARAVIEW_DIR is set correctly but will not be named accordingly.
---
 Source/Insitu/Make.insitu | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/Source/Insitu/Make.insitu b/Source/Insitu/Make.insitu
index c8f32f299..8c910faa9 100644
--- a/Source/Insitu/Make.insitu
+++ b/Source/Insitu/Make.insitu
@@ -17,11 +17,17 @@ ifeq ($(USE_CATALYST),TRUE)
   src_dirs += $(GRCHOMBO_SOURCE)/Insitu
   # This won't work if these make variables are overwritten in Make.defs.local
   cxxcppflags += -DUSE_CATALYST
-  # These components are correct in v5.9 but may differ in other versions
-  # The paraview-config command may also change
+  # These components are correct in v5.9 and v5.10 but may differ in other 
+  # versions. The paraview-config command may also change
   PARAVIEW_COMPONENT_FLAGS := -c PythonCatalyst
   VTK_COMPONENT_FLAGS := -v CommonDataModel -v PythonUsed
   PARAVIEW_CONFIG_EXEC := $(PARAVIEW_DIR)/bin/paraview-config
   cxxcppflags += $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --cppflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
   XTRALDFLAGS += $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --ldflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
 endif
+
+insitu: all-test
+  ifndef PARAVIEW_DIR
+    $(error Must set PARAVIEW_DIR to build insitu executable)
+  endif
+	$(ECHO)mv $(_app_configs) $(ebase)_Insitu.$(config).ex
\ No newline at end of file

From d6669101a4051bdc2c485e572636200eea05b212 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 25 Jan 2022 17:17:17 +0000
Subject: [PATCH 19/63] Implement catalyst_vars parameter

This allows the user to restrict which variables are passed to Catalyst.
In theory this could be determined from the Catalyst script but the
functions to determine this don't seem to work well with newer Catalyst
scripts (i.e. those generated by v5.9 or later) and always request all
variables.
---
 Examples/BinaryBH/params_catalyst.txt    |  2 +-
 Source/GRChomboCore/ChomboParameters.hpp |  2 ++
 Source/GRChomboCore/GRAMR.cpp            |  7 ++---
 Source/GRChomboCore/GRAMR.hpp            |  6 +++--
 Source/GRChomboCore/SetupFunctions.hpp   |  4 ++-
 Source/Insitu/CatalystAdaptor.cpp        | 33 +++++++++++++++++++-----
 Source/Insitu/CatalystAdaptor.hpp        |  4 +++
 7 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/Examples/BinaryBH/params_catalyst.txt b/Examples/BinaryBH/params_catalyst.txt
index 5d14a8486..8572b1750 100644
--- a/Examples/BinaryBH/params_catalyst.txt
+++ b/Examples/BinaryBH/params_catalyst.txt
@@ -188,4 +188,4 @@ activate_catalyst = true
 catalyst_script_path = catalyst_scripts/volume_raytrace_resample_chi.py
 catalyst_coprocess_level = 1
 num_catalyst_vars = 1
-caralyst_vars = chi
+catalyst_vars = chi
diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index 1fff91755..021eaec49 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -564,6 +564,8 @@ class ChomboParameters
     std::string catalyst_script_path;
     int catalyst_coprocess_level;
     // variables to pass to Catalyst
+    // only restricts vars if num_catalyst_vars > 0, otherwise all requested
+    // are passed
     int num_catalyst_vars;
     std::vector<std::pair<int, VariableType>> catalyst_vars;
 #endif
diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp
index c6a8c22ca..d36bdbce5 100644
--- a/Source/GRChomboCore/GRAMR.cpp
+++ b/Source/GRChomboCore/GRAMR.cpp
@@ -64,15 +64,16 @@ void GRAMR::fill_multilevel_ghosts(const VariableType a_var_type,
 }
 
 #ifdef USE_CATALYST
-void GRAMR::setup_catalyst(bool a_activate_catalyst,
-                           const std::string &a_python_script_path)
+void GRAMR::setup_catalyst(
+    bool a_activate_catalyst, const std::string &a_python_script_path,
+    const std::vector<std::pair<int, VariableType>> &a_vars)
 {
     m_activate_catalyst = a_activate_catalyst;
     if (m_activate_catalyst)
     {
         pout() << "GRAMR::setup_catalyst" << std::endl;
         m_insitu = new CatalystAdaptor;
-        m_insitu->initialise(this, a_python_script_path, m_verbosity);
+        m_insitu->initialise(this, a_python_script_path, a_vars, m_verbosity);
     }
 }
 #endif
\ No newline at end of file
diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp
index 880b5a0ee..c655c7fd8 100644
--- a/Source/GRChomboCore/GRAMR.hpp
+++ b/Source/GRChomboCore/GRAMR.hpp
@@ -83,8 +83,10 @@ class GRAMR : public AMR
         const int a_max_level = std::numeric_limits<int>::max()) const;
 
 #ifdef USE_CATALYST
-    void setup_catalyst(bool a_activate_catalyst,
-                        const std::string &a_python_script_path);
+    void
+    setup_catalyst(bool a_activate_catalyst,
+                   const std::string &a_python_script_path,
+                   const std::vector<std::pair<int, VariableType>> &a_vars);
 #endif
 
 #ifdef USE_CATALYST
diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 867d02bc8..203a331d8 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -13,6 +13,7 @@
 #include "parstream.H" //Gives us pout()
 
 // Other includes
+#include <algorithm>
 #include <iostream>
 using std::cerr;
 using std::endl;
@@ -183,7 +184,8 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
     }
 #ifdef USE_CATALYST
     gr_amr.setup_catalyst(chombo_params.activate_catalyst,
-                          chombo_params.catalyst_script_path);
+                          chombo_params.catalyst_script_path,
+                          chombo_params.catalyst_vars);
 #endif
 }
 
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 149a8d91c..f103c9d9a 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -10,11 +10,11 @@
 
 CatalystAdaptor::CatalystAdaptor() {}
 
-CatalystAdaptor::CatalystAdaptor(GRAMR *a_gr_amr_ptr,
-                                 std::string a_python_script_path,
-                                 int a_verbosity)
+CatalystAdaptor::CatalystAdaptor(
+    GRAMR *a_gr_amr_ptr, std::string a_python_script_path,
+    const std::vector<std::pair<int, VariableType>> &a_vars, int a_verbosity)
 {
-    initialise(a_gr_amr_ptr, a_python_script_path, a_verbosity);
+    initialise(a_gr_amr_ptr, a_python_script_path, a_vars, a_verbosity);
 }
 
 CatalystAdaptor::~CatalystAdaptor()
@@ -25,9 +25,9 @@ CatalystAdaptor::~CatalystAdaptor()
     }
 }
 
-void CatalystAdaptor::initialise(GRAMR *a_gr_amr_ptr,
-                                 std::string a_python_script_path,
-                                 int a_verbosity)
+void CatalystAdaptor::initialise(
+    GRAMR *a_gr_amr_ptr, std::string a_python_script_path,
+    const std::vector<std::pair<int, VariableType>> &a_vars, int a_verbosity)
 {
     // don't initalise twice
     if (m_initialised)
@@ -42,6 +42,7 @@ void CatalystAdaptor::initialise(GRAMR *a_gr_amr_ptr,
         return;
     }
     m_gr_amr_ptr = a_gr_amr_ptr;
+    m_vars = a_vars;
 
     // Initialise VTK CP Processor
     if (!m_proc_ptr)
@@ -193,12 +194,14 @@ void CatalystAdaptor::build_vtk_grid()
     }
 
     m_vtk_grid_ptr->Audit();
+#if DEBUG
     const double *vtk_grid_bounds = m_vtk_grid_ptr->GetAMRInfo()->GetBounds();
     pout() << "VTK Grid Bounds:"
            << "(" << vtk_grid_bounds[0] << "," << vtk_grid_bounds[2] << ","
            << vtk_grid_bounds[4] << "), (" << vtk_grid_bounds[1] << ","
            << vtk_grid_bounds[3] << "," << vtk_grid_bounds[5] << ")"
            << std::endl;
+#endif
 
     // not sure if this is necessary but it was on the OverlappingAMR example
     m_vtk_grid_ptr->GenerateParentChildInformation();
@@ -223,6 +226,14 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
     {
         requested_evolution_vars[ivar] = a_input_data_desc->IsFieldNeeded(
             UserVariables::variable_names[ivar].c_str(), vtkDataObject::CELL);
+        if (m_vars.size() > 0)
+        {
+            bool pass_var =
+                !(std::find(m_vars.begin(), m_vars.end(),
+                            std::make_pair(ivar, VariableType::evolution)) ==
+                  m_vars.end());
+            requested_evolution_vars[ivar] &= pass_var;
+        }
         if (m_verbosity && requested_evolution_vars[ivar])
             pout() << UserVariables::variable_names[ivar] << " ";
     }
@@ -231,6 +242,14 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
         requested_diagnostic_vars[ivar] = a_input_data_desc->IsFieldNeeded(
             DiagnosticVariables::variable_names[ivar].c_str(),
             vtkDataObject::CELL);
+        if (m_vars.size() > 0)
+        {
+            bool pass_var =
+                !(std::find(m_vars.begin(), m_vars.end(),
+                            std::make_pair(ivar, VariableType::diagnostic)) ==
+                  m_vars.end());
+            requested_diagnostic_vars[ivar] &= pass_var;
+        }
         if (m_verbosity && requested_diagnostic_vars[ivar])
             pout() << DiagnosticVariables::variable_names[ivar] << " ";
     }
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index 217aeea41..f0f6055ae 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -46,6 +46,7 @@ class CatalystAdaptor
 
     // full constructor (calls initialise)
     CatalystAdaptor(GRAMR *a_gr_amr_ptr, std::string a_python_script_path,
+                    const std::vector<std::pair<int, VariableType>> &a_vars,
                     int a_verbosity);
 
     // destructor
@@ -53,6 +54,7 @@ class CatalystAdaptor
 
     // Initialisation/Finalisation
     void initialise(GRAMR *m_gr_amr_ptr, std::string a_python_script_path,
+                    const std::vector<std::pair<int, VariableType>> &a_vars,
                     int a_verbosity);
     void finalise();
 
@@ -72,6 +74,8 @@ class CatalystAdaptor
     int m_verbosity;
     bool m_initialised = false;
     GRAMR *m_gr_amr_ptr = nullptr;
+    // variables to pass to Catalyst
+    std::vector<std::pair<int, VariableType>> m_vars;
     std::string m_python_script_path;
 
     vtkCPProcessor *m_proc_ptr = nullptr;

From f828470acdbdf1e84ca2f9147cc3359f0178142a Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Thu, 3 Feb 2022 14:50:17 +0000
Subject: [PATCH 20/63] Add support for multiple Catalyst scripts

In order to do this, parameter names have been changed.
---
 Examples/BinaryBH/params_catalyst.txt    |  6 ++-
 Source/GRChomboCore/ChomboParameters.hpp | 50 ++++++++++++++++--------
 Source/GRChomboCore/GRAMR.cpp            |  4 +-
 Source/GRChomboCore/GRAMR.hpp            |  2 +-
 Source/GRChomboCore/SetupFunctions.hpp   |  2 +-
 Source/Insitu/CatalystAdaptor.cpp        | 30 ++++++++------
 Source/Insitu/CatalystAdaptor.hpp        |  7 ++--
 7 files changed, 64 insertions(+), 37 deletions(-)

diff --git a/Examples/BinaryBH/params_catalyst.txt b/Examples/BinaryBH/params_catalyst.txt
index 8572b1750..1e58c3255 100644
--- a/Examples/BinaryBH/params_catalyst.txt
+++ b/Examples/BinaryBH/params_catalyst.txt
@@ -185,7 +185,11 @@ modes = 2 0 # l m for spherical harmonics
 # ParaView Catalyst parameters
 
 activate_catalyst = true
-catalyst_script_path = catalyst_scripts/volume_raytrace_resample_chi.py
+# path to directory containing catalyst scripts
+catalyst_scripts_path = catalyst_scripts
+num_catalyst_scripts = 1
+# list of Catalyst scripts
+catalyst_scripts = volume_raytrace_resample_chi.py
 catalyst_coprocess_level = 1
 num_catalyst_vars = 1
 catalyst_vars = chi
diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index 021eaec49..26004e390 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -133,11 +133,23 @@ class ChomboParameters
 
 #ifdef USE_CATALYST
         pp.load("activate_catalyst", activate_catalyst, false);
-        pp.load("catalyst_script_path", catalyst_script_path);
-        pp.load("catalyst_coprocess_level", catalyst_coprocess_level, 0);
-        UserVariables::load_vars_to_vector(pp, "catalyst_vars",
-                                           "num_catalyst_vars", catalyst_vars,
-                                           num_catalyst_vars);
+        if (activate_catalyst)
+        {
+            pp.load("catalyst_scripts_path", catalyst_scripts_path,
+                    std::string());
+            pp.load("num_catalyst_scripts", num_catalyst_scripts, 1);
+            pp.load("catalyst_scripts", catalyst_scripts, num_catalyst_scripts);
+            // prepend the path to each script
+            if (!catalyst_scripts_path.empty() &&
+                catalyst_scripts_path.back() != '/')
+                catalyst_scripts_path += "/";
+            for (auto &script : catalyst_scripts)
+                script = catalyst_scripts_path + script;
+            pp.load("catalyst_coprocess_level", catalyst_coprocess_level, 0);
+            UserVariables::load_vars_to_vector(
+                pp, "catalyst_vars", "num_catalyst_vars", catalyst_vars,
+                num_catalyst_vars);
+        }
 #endif
     }
 
@@ -481,16 +493,19 @@ class ChomboParameters
 #ifdef USE_CATALYST
         if (activate_catalyst)
         {
-            bool catalyst_script_exists =
-                (access(catalyst_script_path.c_str(), R_OK) == 0);
-            check_parameter("catalyst_script_path", catalyst_script_path,
-                            catalyst_script_exists, "file does not exist");
-            bool catalyst_script_valid =
-                (vtkCPPythonPipeline::DetectScriptVersion(
-                     catalyst_script_path.c_str()) != 0);
-            check_parameter("catalyst_script_path", catalyst_script_path,
-                            catalyst_script_valid,
-                            "not a valid ParaView Catalyst script");
+            for (int iscript = 0; iscript < catalyst_scripts.size(); ++iscript)
+            {
+                const std::string &script = catalyst_scripts[iscript];
+                bool script_exists = (access(script.c_str(), R_OK) == 0);
+                const std::string script_parameter_name =
+                    "catalyst_scripts[" + std::to_string(iscript) + "]";
+                check_parameter(script_parameter_name, script, script_exists,
+                                "does not exist");
+                bool script_valid = (vtkCPPythonPipeline::DetectScriptVersion(
+                                         script.c_str()) != 0);
+                check_parameter(script_parameter_name, script, script_valid,
+                                "not a valid ParaView Catalyst script");
+            }
 
             check_parameter("catalyst_coprocess_level",
                             catalyst_coprocess_level,
@@ -561,7 +576,10 @@ class ChomboParameters
 
 #ifdef USE_CATALYST
     bool activate_catalyst;
-    std::string catalyst_script_path;
+    // ignores output_path
+    std::string catalyst_scripts_path;
+    int num_catalyst_scripts;
+    std::vector<std::string> catalyst_scripts;
     int catalyst_coprocess_level;
     // variables to pass to Catalyst
     // only restricts vars if num_catalyst_vars > 0, otherwise all requested
diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp
index d36bdbce5..1469cd1b5 100644
--- a/Source/GRChomboCore/GRAMR.cpp
+++ b/Source/GRChomboCore/GRAMR.cpp
@@ -65,7 +65,7 @@ void GRAMR::fill_multilevel_ghosts(const VariableType a_var_type,
 
 #ifdef USE_CATALYST
 void GRAMR::setup_catalyst(
-    bool a_activate_catalyst, const std::string &a_python_script_path,
+    bool a_activate_catalyst, const std::vector<std::string> &a_python_scripts,
     const std::vector<std::pair<int, VariableType>> &a_vars)
 {
     m_activate_catalyst = a_activate_catalyst;
@@ -73,7 +73,7 @@ void GRAMR::setup_catalyst(
     {
         pout() << "GRAMR::setup_catalyst" << std::endl;
         m_insitu = new CatalystAdaptor;
-        m_insitu->initialise(this, a_python_script_path, a_vars, m_verbosity);
+        m_insitu->initialise(this, a_python_scripts, a_vars, m_verbosity);
     }
 }
 #endif
\ No newline at end of file
diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp
index c655c7fd8..f5066fa5f 100644
--- a/Source/GRChomboCore/GRAMR.hpp
+++ b/Source/GRChomboCore/GRAMR.hpp
@@ -85,7 +85,7 @@ class GRAMR : public AMR
 #ifdef USE_CATALYST
     void
     setup_catalyst(bool a_activate_catalyst,
-                   const std::string &a_python_script_path,
+                   const std::vector<std::string> &a_python_scripts,
                    const std::vector<std::pair<int, VariableType>> &a_vars);
 #endif
 
diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 203a331d8..9697ca3d0 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -184,7 +184,7 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
     }
 #ifdef USE_CATALYST
     gr_amr.setup_catalyst(chombo_params.activate_catalyst,
-                          chombo_params.catalyst_script_path,
+                          chombo_params.catalyst_scripts,
                           chombo_params.catalyst_vars);
 #endif
 }
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index f103c9d9a..4693ebb7d 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -11,10 +11,10 @@
 CatalystAdaptor::CatalystAdaptor() {}
 
 CatalystAdaptor::CatalystAdaptor(
-    GRAMR *a_gr_amr_ptr, std::string a_python_script_path,
+    GRAMR *a_gr_amr_ptr, const std::vector<std::string> &a_python_scripts,
     const std::vector<std::pair<int, VariableType>> &a_vars, int a_verbosity)
 {
-    initialise(a_gr_amr_ptr, a_python_script_path, a_vars, a_verbosity);
+    initialise(a_gr_amr_ptr, a_python_scripts, a_vars, a_verbosity);
 }
 
 CatalystAdaptor::~CatalystAdaptor()
@@ -26,7 +26,7 @@ CatalystAdaptor::~CatalystAdaptor()
 }
 
 void CatalystAdaptor::initialise(
-    GRAMR *a_gr_amr_ptr, std::string a_python_script_path,
+    GRAMR *a_gr_amr_ptr, const std::vector<std::string> &a_python_scripts,
     const std::vector<std::pair<int, VariableType>> &a_vars, int a_verbosity)
 {
     // don't initalise twice
@@ -56,17 +56,21 @@ void CatalystAdaptor::initialise(
     }
 
     // Create Python script pipeline and add it to the VTK CP Processor
-    if (auto pipeline = vtkCPPythonScriptPipeline::CreateAndInitializePipeline(
-            a_python_script_path.c_str()))
+    for (const std::string &script : a_python_scripts)
     {
-        m_proc_ptr->AddPipeline(pipeline);
-    }
-    else
-    {
-        std::string pipeline_fail_warning =
-            "Failed to set up pipeline for script:";
-        pipeline_fail_warning += a_python_script_path;
-        MayDay::Warning(pipeline_fail_warning.c_str());
+        if (auto pipeline =
+                vtkCPPythonScriptPipeline::CreateAndInitializePipeline(
+                    script.c_str()))
+        {
+            m_proc_ptr->AddPipeline(pipeline);
+        }
+        else
+        {
+            std::string pipeline_fail_warning =
+                "Failed to set up pipeline for script: ";
+            pipeline_fail_warning += script;
+            MayDay::Warning(pipeline_fail_warning.c_str());
+        }
     }
 
     m_verbosity = a_verbosity;
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index f0f6055ae..2f52927d1 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -45,7 +45,8 @@ class CatalystAdaptor
     CatalystAdaptor();
 
     // full constructor (calls initialise)
-    CatalystAdaptor(GRAMR *a_gr_amr_ptr, std::string a_python_script_path,
+    CatalystAdaptor(GRAMR *a_gr_amr_ptr,
+                    const std::vector<std::string> &a_python_scripts,
                     const std::vector<std::pair<int, VariableType>> &a_vars,
                     int a_verbosity);
 
@@ -53,7 +54,8 @@ class CatalystAdaptor
     ~CatalystAdaptor();
 
     // Initialisation/Finalisation
-    void initialise(GRAMR *m_gr_amr_ptr, std::string a_python_script_path,
+    void initialise(GRAMR *m_gr_amr_ptr,
+                    const std::vector<std::string> &a_python_scripts,
                     const std::vector<std::pair<int, VariableType>> &a_vars,
                     int a_verbosity);
     void finalise();
@@ -76,7 +78,6 @@ class CatalystAdaptor
     GRAMR *m_gr_amr_ptr = nullptr;
     // variables to pass to Catalyst
     std::vector<std::pair<int, VariableType>> m_vars;
-    std::string m_python_script_path;
 
     vtkCPProcessor *m_proc_ptr = nullptr;
     vtkOverlappingAMR *m_vtk_grid_ptr = nullptr;

From f3d9c372782e3d7297dc92cd959026cfc9356bbc Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Fri, 4 Feb 2022 15:44:09 +0000
Subject: [PATCH 21/63] Make VTK/Catalyst log to a pout file

This reduces VTK log output to stderr and instead sends it to a
catalyst_pout (this name can be changed with the catalyst_pout_prefix
parameter) file (for each rank) under the usual pout_subpath like the
normal Chombo output (in the case MPI=TRUE). There is also a
catalyst_verbosity parameter which controls both the verbosity of the
VTK log output and the messages from CatalystAdaptor in the normal pout
file.

The -rdynamic flag will also be passed to the linker for the insitu
build target so that the VTK log stack trace (which is output in the
case of some termination signal) has more information for the [GR]Chombo
part of the stack.
---
 Examples/BinaryBH/params_catalyst.txt    |  2 ++
 Source/GRChomboCore/ChomboParameters.hpp |  7 +++++
 Source/GRChomboCore/GRAMR.cpp            |  4 +--
 Source/GRChomboCore/GRAMR.hpp            |  8 +++---
 Source/GRChomboCore/SetupFunctions.hpp   | 33 +++++++++++++++++++++---
 Source/Insitu/CatalystAdaptor.cpp        |  2 +-
 Source/Insitu/Make.insitu                |  2 +-
 7 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/Examples/BinaryBH/params_catalyst.txt b/Examples/BinaryBH/params_catalyst.txt
index 1e58c3255..f0823adac 100644
--- a/Examples/BinaryBH/params_catalyst.txt
+++ b/Examples/BinaryBH/params_catalyst.txt
@@ -185,6 +185,8 @@ modes = 2 0 # l m for spherical harmonics
 # ParaView Catalyst parameters
 
 activate_catalyst = true
+catalyst_verbosity = 2
+# catalyst_pout_prefix = catalyst_pout
 # path to directory containing catalyst scripts
 catalyst_scripts_path = catalyst_scripts
 num_catalyst_scripts = 1
diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index 26004e390..b50c991b3 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -135,6 +135,9 @@ class ChomboParameters
         pp.load("activate_catalyst", activate_catalyst, false);
         if (activate_catalyst)
         {
+            pp.load("catalyst_verbosity", catalyst_verbosity, verbosity);
+            pp.load("catalyst_pout_prefix", catalyst_pout_prefix,
+                    std::string("catalyst_pout"));
             pp.load("catalyst_scripts_path", catalyst_scripts_path,
                     std::string());
             pp.load("num_catalyst_scripts", num_catalyst_scripts, 1);
@@ -576,6 +579,10 @@ class ChomboParameters
 
 #ifdef USE_CATALYST
     bool activate_catalyst;
+    int catalyst_verbosity;
+    // prefix of filename for Catalyst log output (appended by .<rank>)
+    // file will be in pout_path
+    std::string catalyst_pout_prefix;
     // ignores output_path
     std::string catalyst_scripts_path;
     int num_catalyst_scripts;
diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp
index 1469cd1b5..a9cf8edf6 100644
--- a/Source/GRChomboCore/GRAMR.cpp
+++ b/Source/GRChomboCore/GRAMR.cpp
@@ -66,14 +66,14 @@ void GRAMR::fill_multilevel_ghosts(const VariableType a_var_type,
 #ifdef USE_CATALYST
 void GRAMR::setup_catalyst(
     bool a_activate_catalyst, const std::vector<std::string> &a_python_scripts,
-    const std::vector<std::pair<int, VariableType>> &a_vars)
+    const std::vector<std::pair<int, VariableType>> &a_vars, int a_verbosity)
 {
     m_activate_catalyst = a_activate_catalyst;
     if (m_activate_catalyst)
     {
         pout() << "GRAMR::setup_catalyst" << std::endl;
         m_insitu = new CatalystAdaptor;
-        m_insitu->initialise(this, a_python_scripts, a_vars, m_verbosity);
+        m_insitu->initialise(this, a_python_scripts, a_vars, a_verbosity);
     }
 }
 #endif
\ No newline at end of file
diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp
index f5066fa5f..ff7d87756 100644
--- a/Source/GRChomboCore/GRAMR.hpp
+++ b/Source/GRChomboCore/GRAMR.hpp
@@ -83,10 +83,10 @@ class GRAMR : public AMR
         const int a_max_level = std::numeric_limits<int>::max()) const;
 
 #ifdef USE_CATALYST
-    void
-    setup_catalyst(bool a_activate_catalyst,
-                   const std::vector<std::string> &a_python_scripts,
-                   const std::vector<std::pair<int, VariableType>> &a_vars);
+    void setup_catalyst(bool a_activate_catalyst,
+                        const std::vector<std::string> &a_python_scripts,
+                        const std::vector<std::pair<int, VariableType>> &a_vars,
+                        int a_verbosity);
 #endif
 
 #ifdef USE_CATALYST
diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 9697ca3d0..77de686d5 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -30,6 +30,10 @@ using std::endl;
 #include "DebuggingTools.hpp"
 #endif
 
+#ifdef USE_CATALYST
+#include "vtkLogger.h"
+#endif
+
 #ifdef _OPENMP
 #include <omp.h>
 #endif
@@ -183,9 +187,32 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
 #endif
     }
 #ifdef USE_CATALYST
-    gr_amr.setup_catalyst(chombo_params.activate_catalyst,
-                          chombo_params.catalyst_scripts,
-                          chombo_params.catalyst_vars);
+    // set vtkLogger internal messages to only appear at verbosity 2
+    vtkLogger::SetInternalVerbosityLevel(vtkLogger::VERBOSITY_3);
+#ifdef CH_MPI
+    vtkLogger::SetStderrVerbosity(vtkLogger::VERBOSITY_ERROR);
+    std::string catalyst_log_file = chombo_params.pout_path +
+                                    chombo_params.catalyst_pout_prefix +
+                                    std::string(".") + std::to_string(procID());
+    vtkLogger::FileMode vtk_logger_file_mode = vtkLogger::TRUNCATE;
+    // if we're writing to the normal pout file we don't want the vtkLogger to
+    // delete the file at this point
+    if (chombo_params.catalyst_pout_prefix == chombo_params.pout_prefix)
+    {
+        vtk_logger_file_mode = vtkLogger::APPEND;
+    }
+    vtkLogger::LogToFile(
+        catalyst_log_file.c_str(), vtk_logger_file_mode,
+        static_cast<vtkLogger::Verbosity>(chombo_params.catalyst_verbosity));
+    // Only write VTK stderr messages if there is an error
+#else
+    vtkLogger::SetStderrVerbosity(
+        static_cast<vtkLogger::Verbosity>(chombo_params.verbosity));
+#endif
+    vtkLogger::Init();
+    gr_amr.setup_catalyst(
+        chombo_params.activate_catalyst, chombo_params.catalyst_scripts,
+        chombo_params.catalyst_vars, chombo_params.catalyst_verbosity);
 #endif
 }
 
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 4693ebb7d..cc20584cf 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -223,7 +223,7 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
 
     if (m_verbosity)
     {
-        pout() << "CatalystAdaptor Requested variables:\n";
+        pout() << "CatalystAdaptor Requested variables: ";
     }
 
     for (int ivar = 0; ivar < NUM_VARS; ++ivar)
diff --git a/Source/Insitu/Make.insitu b/Source/Insitu/Make.insitu
index 8c910faa9..5a970a645 100644
--- a/Source/Insitu/Make.insitu
+++ b/Source/Insitu/Make.insitu
@@ -23,7 +23,7 @@ ifeq ($(USE_CATALYST),TRUE)
   VTK_COMPONENT_FLAGS := -v CommonDataModel -v PythonUsed
   PARAVIEW_CONFIG_EXEC := $(PARAVIEW_DIR)/bin/paraview-config
   cxxcppflags += $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --cppflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
-  XTRALDFLAGS += $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --ldflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
+  XTRALDFLAGS += -rdynamic $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --ldflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
 endif
 
 insitu: all-test

From 502c0255d453ae395a986379ca649ff9988dbd7d Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Mon, 7 Feb 2022 16:11:22 +0000
Subject: [PATCH 22/63] Use vtkLogger function to cast int to verbosity

---
 Source/GRChomboCore/SetupFunctions.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 77de686d5..46334c76a 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -203,7 +203,7 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
     }
     vtkLogger::LogToFile(
         catalyst_log_file.c_str(), vtk_logger_file_mode,
-        static_cast<vtkLogger::Verbosity>(chombo_params.catalyst_verbosity));
+        vtkLogger::ConvertToVerbosity(chombo_params.catalyst_verbosity));
     // Only write VTK stderr messages if there is an error
 #else
     vtkLogger::SetStderrVerbosity(

From 9e9ec91a0ca14ae42f16310988b8f90d1b3b221b Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Mon, 7 Feb 2022 16:12:41 +0000
Subject: [PATCH 23/63] Add Catalyst Insitu readme

This provides instructions on building and using ParaView Catalyst
in-situ visualization with a GRChombo example.
---
 Source/Insitu/README.md | 135 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 135 insertions(+)
 create mode 100644 Source/Insitu/README.md

diff --git a/Source/Insitu/README.md b/Source/Insitu/README.md
new file mode 100644
index 000000000..4a480f027
--- /dev/null
+++ b/Source/Insitu/README.md
@@ -0,0 +1,135 @@
+# Using ParaView Catalyst in-situ visualization with GRChombo
+
+## Why use in-situ visualization?
+
+Whilst the processing power of supercomputers has grown rapidly over recent
+years, the corresponding growth in IO capabilities and data storage has not kept
+pace. As a result, it is common to only write checkpoints/plot files relatively
+infrequently in order to save on disk space. This means that traditional 
+post-processed visualization can only be performed at relatively poor 
+resolutions in time. Alternatively, separate "visualization runs" are sometimes
+performed at significantly lower resolutions than that of the production
+simulations which takes more time and can also result in inaccurate output. One
+solution to this problem is performing the visualization _in-situ_ (i.e. during
+the simulation itself).
+
+In this guide, we describe how to use the in-situ visualization capabilities of
+ParaView Catalyst with a GRChombo example.
+
+![Catalyst Logo](https://www.paraview.org/Wiki/images/8/8a/CatalystLogo.png)
+
+## Prerequisites
+
+You will need a build of ParaView with Catalyst support. Unfortunately, the 
+pre-built binaries available on the ParaView website for various OSs and 
+architecture are not built with Catalyst support at the time of writing 
+(latest version: v5.10.0) so it is necessary to build ParaView from source.
+Given the large number of dependencies, it is probably easiest to use the 
+[superbuild](https://gitlab.kitware.com/paraview/paraview-superbuild/) 
+which downloads and builds all dependencies before building ParaView. When
+configuring the superbuild using CMake, you will want to make sure the following
+options are set
+```cmake
+ENABLE_osmesa = ON # enables rendering without a screen
+USE_SYSTEM_llvm = ON # a prerequisite for OSMesa above
+USE_SYSTEM_mpi = ON # you will want to use the same MPI as for [GR]Chombo
+PARAVIEW_BUILD_EDITION = CATALYST_RENDERING # This will build without IO
+# libraries and a user interface so will be much easier and quicker
+```
+Note that the above setting for `PARAVIEW_BUILD_EDITION` will not build
+the usual ParaView graphical user interface as this can take quite a bit longer
+and can be more complicated. For more details see 
+[this Kitware blog post](https://www.kitware.com/paraview-editions/). In order
+to [generate ParaView Catalyst scripts](#generating-paraview-catalyst-scripts), 
+you will probably want to use the GUI but for this part, you can use the
+pre-built ParaView binaries available on the [ParaView
+website](https://www.paraview.org/download/). Alternatively, you can set 
+```cmake
+PARAVIEW_BUILD_EDITION = CANONICAL
+```
+
+Since building ParaView can be a little daunting, unless you area already 
+familiar with building software with CMake, it may be advisable to ask
+your HPC system administrator to build a suitable version for you. If so, make
+sure you specify that you need a version with Catalyst enabled and with 
+off-screen rendering.
+
+This in-situ adaptor has been tested with the following versions of ParaView:
+
+| Tested ParaView versions |
+---
+| 5.9.1 |
+| 5.10.0 |
+
+It may work with other versions but these have not been tested.
+
+## Adding ParaView Catalyst to an existing GRChombo example
+
+To add ParaView Catalyst to your existing example, make sure your example is 
+up-to-date with this branch. Then simply add the line
+```make
+include $(GRCHOMBO_SOURCE)/Insitu/Make.insitu
+```
+in the `GNUmakefile` directly after the `src_dirs` variable has been set. See 
+the [BinaryBH's `GNUmakefile`](../../Examples/BinaryBH/GNUmakefile) for an
+example. Note that this script uses the `XTRALDFLAGS` and `cxxcppflags` Chombo
+make variables so do *not* set these in your `Make.defs.local` file.
+
+The in-situ visualization pipelines are called in 
+`GRAMRLevel::postTimeStep()` after `GRAMRLevel::specificPostTimeStep()`.
+If you wish to also process the pipeline before the first timestep, 
+you can use the [`MultiLevelTask` class](../utils/MultiLevelTask.hpp). See the 
+BinaryBH's [Main_BinaryBH.cpp](../../Examples/BinaryBH/Main_BinaryBH.cpp) for 
+an example. 
+
+If you are adding any code to your example that is specific to
+Catalyst it is a good idea to conditionally compile it so that you can continue
+to compile your example without Catalyst if you wish. For example
+```cpp
+#ifdef USE_CATALYST
+<code that depends on Catalyst>
+#endif
+```
+
+
+## Building a GRChombo example with ParaView Catalyst
+
+First, make sure you have updated [Chombo](https://github.com/GRChombo/Chombo) 
+to the latest commit on `main`.
+
+To build a GRChombo example with ParaView Catalyst (for example the [BinaryBH
+example](../../Examples/BinaryBH/)), next set the `PARAVIEW_DIR` to the path
+which contains the build of ParaView as described [above](#prerequisites), for
+example
+```bash
+export PARAVIEW_DIR=/path/to/paraview_build_dir
+```
+This directory should contain an executable at `bin/paraview-config` as this is
+the script that will be used to determine the compiler/linker flags needed to
+build with ParaView Catalyst. Since this script takes a long time to run, the
+outputs are cached using the [cache.sh](./cache.sh) script to 
+`/tmp/command_cache.<hash>`. If something messes up and this cached output is
+incorrect, it can be helpful to remove these files using e.g.
+```bash
+rm -f /tmp/command_cache.*
+```
+
+Finally `make` the `insitu` target e.g.
+```
+make insitu -j 4
+```
+An executable binary with a filename containing '`_Insitu`' will be created. 
+Note that if the `PARAVIEW_DIR` command is set, you can also use `make` the 
+usual `all` target to link with ParaView Catalyst but the executable filename
+will be the conventional one (i.e. without '`_Insitu`').
+
+If you wish to stop compiling/linking with ParaView Catalyst, unset the 
+`PARAVIEW_DIR` environment variable with
+```bash
+unset PARAVIEW_DIR
+```
+
+
+## Generating ParaView Catalyst scripts
+
+## Using the in-situ visualization capabilities
\ No newline at end of file

From ce6528d7f9203330f2dc739ee792d6a358120f43 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Mon, 7 Feb 2022 17:13:50 +0000
Subject: [PATCH 24/63] Add more instructions to Catalyst readme

---
 Source/Insitu/README.md | 67 +++++++++++++++++++++++++++++++++++++++--
 1 file changed, 65 insertions(+), 2 deletions(-)

diff --git a/Source/Insitu/README.md b/Source/Insitu/README.md
index 4a480f027..adcf767bb 100644
--- a/Source/Insitu/README.md
+++ b/Source/Insitu/README.md
@@ -57,7 +57,7 @@ off-screen rendering.
 This in-situ adaptor has been tested with the following versions of ParaView:
 
 | Tested ParaView versions |
----
+| --- |
 | 5.9.1 |
 | 5.10.0 |
 
@@ -132,4 +132,67 @@ unset PARAVIEW_DIR
 
 ## Generating ParaView Catalyst scripts
 
-## Using the in-situ visualization capabilities
\ No newline at end of file
+_Note that the procedure to generate Catalyst scripts varies between different
+versions of ParaView. In particular, there are significant differences before
+v5.9. The following instructions apply to v5.9.1 and v5.10.0._
+
+The easiest way to generate a suitable ParaView Catalyst Python script using the
+GUI is to load an HDF5 file that is similar to the configuration that will exist
+in your simulation. For example, you could use a plot/checkpoint file from the
+initial data of the simulation. Once you have loaded the file into ParaView,
+**make sure you rename the source to 'input'** (by right clicking on it and
+selecting 'Rename'). This is necessary so that Catalyst can replace the HDF5
+data in the script with that coming from the simulation. 
+
+Next apply filters and adjust the camera view as desired. Note that some filters
+do not currently work in situ (e.g. 'Slice AMR Data').
+
+In order to generate a valid Catalyst script, it is then necessary to add an 
+'Extractor' which will, for example, save an image. This can be done using the
+menu options, Extractors → Image → PNG.
+
+Finally, to generate the Catalyst script, use the menu options File → 
+Save Catalyst State. By default, the output directory for Extracts will be
+'dataset'.
+
+Unfortunately, if you have built the `CATALYST_RENDERING` ParaView edition
+as described [above](#prerequisites), since this does not contain any IO 
+libraries, it is necessary to manually edit the generated Catalyst script
+and replace the line
+```python
+input = VisItChomboReader(registrationName='input', FileName=<list of hdf5 files>)
+```
+with
+```python
+input = AMRGaussianPulseSource(registrationName='input')
+```
+in order to be able to use it.
+
+## Using the in-situ visualization capabilities
+
+There are several Catalyst specific parameters which control the in-situ
+visualization processing which are described below:
+ * `activate_catalyst = true/false`: This enables/disable Catalyst coprocessing
+ at runtime. Note that if set to `false`, the other parameters are not read in.
+ * `catalyst_verbosity = -2,...,10`: This controls the verbosity of information printed
+ to `catalyst_pout` files and that printed to the normal `pout` files from
+ the `CatalystAdaptor` class which interfaces with ParaView Catalyst.
+ * `catalyst_pout_prefix`: This is the filename prefix for the `catalyst_pout`
+ files. They are written to `pout_subpath` and appended by '`.<rank number>`' 
+ as for the normal `pout` files. Note that you can set this to the same
+ string as 'pout_prefix` to send this output to the same
+ * `catalyst_scripts_path`: This is the path that contains the Catalyst scripts
+ generated as described [above](#generating-paraview-catalyst-scripts).
+ * `num_catalyst_scripts`: Number of Catalyst scripts to process.
+ * `catalyst_scripts`: Filenames of Catalyst scripts
+ * `catalyst_coprocess_level`: This is the level for which the Catalyst
+ coprocess routine is called at the end of each timestep.
+ 
+Note that scripts generated with versions of ParaView v5.8 or earlier provide 
+information to the pipeline about the specific variables they require. Since 
+this feature is not currently provided by ParaView v5.9 or later, scripts
+generated with these versions request all variables by default. In order to
+only pass specific variables to Catalyst, it is possible to restrict the 
+variables passed using the `num_catalyst_vars` and `catalyst_vars` parameters.
+If these are not set, and a script generated by ParaView v5.9 or later is used,
+all variables will be passed to Catalyst.

From 4f6d9fba84514715ac937d2ddc7d1a1575b7e244 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 8 Feb 2022 17:59:58 +0000
Subject: [PATCH 25/63] Add minor changes to Insitu readme

---
 Source/Insitu/README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Source/Insitu/README.md b/Source/Insitu/README.md
index adcf767bb..c6f512a6f 100644
--- a/Source/Insitu/README.md
+++ b/Source/Insitu/README.md
@@ -90,6 +90,9 @@ to compile your example without Catalyst if you wish. For example
 <code that depends on Catalyst>
 #endif
 ```
+If you need to compute diagnostic quantities for visualization, you can override
+the virtual function `GRAMRLevel::preCatalystCoProcess()` which is called before
+Catalyst CoProcessing.
 
 
 ## Building a GRChombo example with ParaView Catalyst
@@ -119,7 +122,7 @@ Finally `make` the `insitu` target e.g.
 make insitu -j 4
 ```
 An executable binary with a filename containing '`_Insitu`' will be created. 
-Note that if the `PARAVIEW_DIR` command is set, you can also use `make` the 
+Note that if the `PARAVIEW_DIR` command is set, you can also `make` the 
 usual `all` target to link with ParaView Catalyst but the executable filename
 will be the conventional one (i.e. without '`_Insitu`').
 

From 76603c88ad971c28def068444fbc5235a11b6de2 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 9 Feb 2022 17:49:52 +0000
Subject: [PATCH 26/63] Add parameter to abort on catalyst error

---
 Examples/BinaryBH/params_catalyst.txt    |  1 +
 Source/GRChomboCore/ChomboParameters.hpp |  2 +
 Source/GRChomboCore/GRAMR.cpp            |  6 ++-
 Source/GRChomboCore/GRAMR.hpp            |  2 +-
 Source/GRChomboCore/SetupFunctions.hpp   |  3 +-
 Source/Insitu/CatalystAdaptor.cpp        | 51 +++++++++++++++++++-----
 Source/Insitu/CatalystAdaptor.hpp        |  5 ++-
 Source/Insitu/README.md                  |  2 +
 8 files changed, 57 insertions(+), 15 deletions(-)

diff --git a/Examples/BinaryBH/params_catalyst.txt b/Examples/BinaryBH/params_catalyst.txt
index f0823adac..fe55b7480 100644
--- a/Examples/BinaryBH/params_catalyst.txt
+++ b/Examples/BinaryBH/params_catalyst.txt
@@ -195,3 +195,4 @@ catalyst_scripts = volume_raytrace_resample_chi.py
 catalyst_coprocess_level = 1
 num_catalyst_vars = 1
 catalyst_vars = chi
+abort_on_catalyst_error = true
diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index b50c991b3..5da1ae349 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -152,6 +152,7 @@ class ChomboParameters
             UserVariables::load_vars_to_vector(
                 pp, "catalyst_vars", "num_catalyst_vars", catalyst_vars,
                 num_catalyst_vars);
+            pp.load("abort_on_catalyst_error", abort_on_catalyst_error, false);
         }
 #endif
     }
@@ -593,6 +594,7 @@ class ChomboParameters
     // are passed
     int num_catalyst_vars;
     std::vector<std::pair<int, VariableType>> catalyst_vars;
+    bool abort_on_catalyst_error;
 #endif
 
   protected:
diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp
index a9cf8edf6..3b863e46e 100644
--- a/Source/GRChomboCore/GRAMR.cpp
+++ b/Source/GRChomboCore/GRAMR.cpp
@@ -66,14 +66,16 @@ void GRAMR::fill_multilevel_ghosts(const VariableType a_var_type,
 #ifdef USE_CATALYST
 void GRAMR::setup_catalyst(
     bool a_activate_catalyst, const std::vector<std::string> &a_python_scripts,
-    const std::vector<std::pair<int, VariableType>> &a_vars, int a_verbosity)
+    const std::vector<std::pair<int, VariableType>> &a_vars,
+    bool a_abort_on_catalyst_error, int a_verbosity)
 {
     m_activate_catalyst = a_activate_catalyst;
     if (m_activate_catalyst)
     {
         pout() << "GRAMR::setup_catalyst" << std::endl;
         m_insitu = new CatalystAdaptor;
-        m_insitu->initialise(this, a_python_scripts, a_vars, a_verbosity);
+        m_insitu->initialise(this, a_python_scripts, a_vars,
+                             a_abort_on_catalyst_error, a_verbosity);
     }
 }
 #endif
\ No newline at end of file
diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp
index ff7d87756..e0dbee7c2 100644
--- a/Source/GRChomboCore/GRAMR.hpp
+++ b/Source/GRChomboCore/GRAMR.hpp
@@ -86,7 +86,7 @@ class GRAMR : public AMR
     void setup_catalyst(bool a_activate_catalyst,
                         const std::vector<std::string> &a_python_scripts,
                         const std::vector<std::pair<int, VariableType>> &a_vars,
-                        int a_verbosity);
+                        bool a_abort_on_catalyst_error, int a_verbosity);
 #endif
 
 #ifdef USE_CATALYST
diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 46334c76a..c113cb62a 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -212,7 +212,8 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
     vtkLogger::Init();
     gr_amr.setup_catalyst(
         chombo_params.activate_catalyst, chombo_params.catalyst_scripts,
-        chombo_params.catalyst_vars, chombo_params.catalyst_verbosity);
+        chombo_params.catalyst_vars, chombo_params.abort_on_catalyst_error,
+        chombo_params.catalyst_verbosity);
 #endif
 }
 
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index cc20584cf..5cd2841b9 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -12,9 +12,11 @@ CatalystAdaptor::CatalystAdaptor() {}
 
 CatalystAdaptor::CatalystAdaptor(
     GRAMR *a_gr_amr_ptr, const std::vector<std::string> &a_python_scripts,
-    const std::vector<std::pair<int, VariableType>> &a_vars, int a_verbosity)
+    const std::vector<std::pair<int, VariableType>> &a_vars,
+    bool a_abort_on_catalyst_error, int a_verbosity)
 {
-    initialise(a_gr_amr_ptr, a_python_scripts, a_vars, a_verbosity);
+    initialise(a_gr_amr_ptr, a_python_scripts, a_vars,
+               a_abort_on_catalyst_error, a_verbosity);
 }
 
 CatalystAdaptor::~CatalystAdaptor()
@@ -27,7 +29,8 @@ CatalystAdaptor::~CatalystAdaptor()
 
 void CatalystAdaptor::initialise(
     GRAMR *a_gr_amr_ptr, const std::vector<std::string> &a_python_scripts,
-    const std::vector<std::pair<int, VariableType>> &a_vars, int a_verbosity)
+    const std::vector<std::pair<int, VariableType>> &a_vars,
+    bool a_abort_on_catalyst_error, int a_verbosity)
 {
     // don't initalise twice
     if (m_initialised)
@@ -43,12 +46,22 @@ void CatalystAdaptor::initialise(
     }
     m_gr_amr_ptr = a_gr_amr_ptr;
     m_vars = a_vars;
+    m_abort_on_catalyst_error = a_abort_on_catalyst_error;
 
     // Initialise VTK CP Processor
     if (!m_proc_ptr)
     {
         m_proc_ptr = vtkCPProcessor::New();
-        m_proc_ptr->Initialize();
+        int m_proc_ptr_intialization_success = m_proc_ptr->Initialize();
+        if (!m_proc_ptr_intialization_success)
+        {
+            std::string error_msg = "Failed to initialize vtkCPProcessor in "
+                                    "CatalystAdaptor::initialise";
+            if (m_abort_on_catalyst_error)
+                MayDay::Error(error_msg.c_str());
+            else
+                MayDay::Warning(error_msg.c_str());
+        }
     }
     else
     {
@@ -62,14 +75,27 @@ void CatalystAdaptor::initialise(
                 vtkCPPythonScriptPipeline::CreateAndInitializePipeline(
                     script.c_str()))
         {
-            m_proc_ptr->AddPipeline(pipeline);
+            int add_pipeline_success = m_proc_ptr->AddPipeline(pipeline);
+            if (!add_pipeline_success)
+            {
+                std::string add_pipeline_fail_msg =
+                    "Failed to add pipeline for script: ";
+                add_pipeline_fail_msg += script;
+                if (m_abort_on_catalyst_error)
+                    MayDay::Error(add_pipeline_fail_msg.c_str());
+                else
+                    MayDay::Warning(add_pipeline_fail_msg.c_str());
+            }
         }
         else
         {
-            std::string pipeline_fail_warning =
+            std::string pipeline_fail_msg =
                 "Failed to set up pipeline for script: ";
-            pipeline_fail_warning += script;
-            MayDay::Warning(pipeline_fail_warning.c_str());
+            pipeline_fail_msg += script;
+            if (m_abort_on_catalyst_error)
+                MayDay::Error(pipeline_fail_msg.c_str());
+            else
+                MayDay::Warning(pipeline_fail_msg.c_str());
         }
     }
 
@@ -378,7 +404,14 @@ void CatalystAdaptor::coprocess(double a_time, unsigned int a_timestep)
         // vtkNew<vtkOverlappingAMR> stripped_vtk_grid;
         // vtkAMRUtilities::StripGhostLayers(m_vtk_grid_ptr, stripped_vtk_grid);
         input_data_description->SetGrid(m_vtk_grid_ptr);
-        m_proc_ptr->CoProcess(data_description);
+        int coprocess_success = m_proc_ptr->CoProcess(data_description);
+        if (!coprocess_success)
+        {
+            if (m_abort_on_catalyst_error)
+                MayDay::Error("Error in Catalyst CoProcess");
+            else
+                MayDay::Warning("Error in Catalyst CoProcess");
+        }
     }
 }
 
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index 2f52927d1..d285d1d06 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -48,7 +48,7 @@ class CatalystAdaptor
     CatalystAdaptor(GRAMR *a_gr_amr_ptr,
                     const std::vector<std::string> &a_python_scripts,
                     const std::vector<std::pair<int, VariableType>> &a_vars,
-                    int a_verbosity);
+                    bool a_abort_on_catalyst_error, int a_verbosity);
 
     // destructor
     ~CatalystAdaptor();
@@ -57,7 +57,7 @@ class CatalystAdaptor
     void initialise(GRAMR *m_gr_amr_ptr,
                     const std::vector<std::string> &a_python_scripts,
                     const std::vector<std::pair<int, VariableType>> &a_vars,
-                    int a_verbosity);
+                    bool a_abort_on_catalyst_error, int a_verbosity);
     void finalise();
 
     // do Catalyst processing
@@ -75,6 +75,7 @@ class CatalystAdaptor
 
     int m_verbosity;
     bool m_initialised = false;
+    bool m_abort_on_catalyst_error = false;
     GRAMR *m_gr_amr_ptr = nullptr;
     // variables to pass to Catalyst
     std::vector<std::pair<int, VariableType>> m_vars;
diff --git a/Source/Insitu/README.md b/Source/Insitu/README.md
index c6f512a6f..067c9c1fe 100644
--- a/Source/Insitu/README.md
+++ b/Source/Insitu/README.md
@@ -190,6 +190,8 @@ visualization processing which are described below:
  * `catalyst_scripts`: Filenames of Catalyst scripts
  * `catalyst_coprocess_level`: This is the level for which the Catalyst
  coprocess routine is called at the end of each timestep.
+ * `abort_on_catalyst_error = true/false`: If set to `true`, the code will abort
+ if there is an error in Catalyst. The default is `false`.
  
 Note that scripts generated with versions of ParaView v5.8 or earlier provide 
 information to the pipeline about the specific variables they require. Since 

From 2df36c90159f7692440ad2c72897f2d4f7dbbd98 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Fri, 11 Feb 2022 17:32:02 +0000
Subject: [PATCH 27/63] Fix Make.insitu when PARAVIEW_DIR is not set

---
 Source/Insitu/Make.insitu | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/Source/Insitu/Make.insitu b/Source/Insitu/Make.insitu
index 5a970a645..5e75045f0 100644
--- a/Source/Insitu/Make.insitu
+++ b/Source/Insitu/Make.insitu
@@ -26,8 +26,10 @@ ifeq ($(USE_CATALYST),TRUE)
   XTRALDFLAGS += -rdynamic $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --ldflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
 endif
 
+ifdef PARAVIEW_DIR
 insitu: all-test
-  ifndef PARAVIEW_DIR
-    $(error Must set PARAVIEW_DIR to build insitu executable)
-  endif
-	$(ECHO)mv $(_app_configs) $(ebase)_Insitu.$(config).ex
\ No newline at end of file
+	$(ECHO)mv $(_app_configs) $(ebase)_Insitu.$(config).ex
+else
+insitu:
+	@echo "Set PARAVIEW_DIR environment variable to build insitu target.";
+endif
\ No newline at end of file

From 63b4a7066ab74d6d5a0333e5172c6ee0b1e619d5 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Mon, 14 Feb 2022 17:25:42 +0000
Subject: [PATCH 28/63] Fix incorrect casting from IntVect to RealVect

Also refactor error/warning in CatalystAdaptor
---
 Source/Insitu/CatalystAdaptor.cpp | 56 ++++++++++++++-----------------
 Source/Insitu/CatalystAdaptor.hpp |  4 +++
 2 files changed, 29 insertions(+), 31 deletions(-)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 5cd2841b9..e94408d68 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -53,15 +53,9 @@ void CatalystAdaptor::initialise(
     {
         m_proc_ptr = vtkCPProcessor::New();
         int m_proc_ptr_intialization_success = m_proc_ptr->Initialize();
-        if (!m_proc_ptr_intialization_success)
-        {
-            std::string error_msg = "Failed to initialize vtkCPProcessor in "
-                                    "CatalystAdaptor::initialise";
-            if (m_abort_on_catalyst_error)
-                MayDay::Error(error_msg.c_str());
-            else
-                MayDay::Warning(error_msg.c_str());
-        }
+        std::string error_msg = "Failed to initialize vtkCPProcessor in "
+                                "CatalystAdaptor::initialise";
+        catalyst_error_or_warning(m_proc_ptr_intialization_success, error_msg);
     }
     else
     {
@@ -76,26 +70,18 @@ void CatalystAdaptor::initialise(
                     script.c_str()))
         {
             int add_pipeline_success = m_proc_ptr->AddPipeline(pipeline);
-            if (!add_pipeline_success)
-            {
-                std::string add_pipeline_fail_msg =
-                    "Failed to add pipeline for script: ";
-                add_pipeline_fail_msg += script;
-                if (m_abort_on_catalyst_error)
-                    MayDay::Error(add_pipeline_fail_msg.c_str());
-                else
-                    MayDay::Warning(add_pipeline_fail_msg.c_str());
-            }
+            std::string add_pipeline_fail_msg =
+                "Failed to add pipeline for script: ";
+            add_pipeline_fail_msg += script;
+            catalyst_error_or_warning(add_pipeline_success,
+                                      add_pipeline_fail_msg);
         }
         else
         {
             std::string pipeline_fail_msg =
                 "Failed to set up pipeline for script: ";
             pipeline_fail_msg += script;
-            if (m_abort_on_catalyst_error)
-                MayDay::Error(pipeline_fail_msg.c_str());
-            else
-                MayDay::Warning(pipeline_fail_msg.c_str());
+            catalyst_error_or_warning(false, pipeline_fail_msg);
         }
     }
 
@@ -150,7 +136,8 @@ void CatalystAdaptor::build_vtk_grid()
     const IntVect &coarsest_ghost_vect =
         gramrlevels[0]->getLevelData().ghostVect();
     const double coarsest_dx = gramrlevels[0]->get_dx();
-    RealVect ghosted_origin_global_vect = -coarsest_dx * coarsest_ghost_vect;
+    RealVect ghosted_origin_global_vect = coarsest_ghost_vect;
+    ghosted_origin_global_vect *= -coarsest_dx;
     m_vtk_grid_ptr->SetOrigin(ghosted_origin_global_vect.dataPtr());
 
     // now add all the boxes
@@ -405,13 +392,8 @@ void CatalystAdaptor::coprocess(double a_time, unsigned int a_timestep)
         // vtkAMRUtilities::StripGhostLayers(m_vtk_grid_ptr, stripped_vtk_grid);
         input_data_description->SetGrid(m_vtk_grid_ptr);
         int coprocess_success = m_proc_ptr->CoProcess(data_description);
-        if (!coprocess_success)
-        {
-            if (m_abort_on_catalyst_error)
-                MayDay::Error("Error in Catalyst CoProcess");
-            else
-                MayDay::Warning("Error in Catalyst CoProcess");
-        }
+        catalyst_error_or_warning(coprocess_success,
+                                  "Error in Catalyst CoProcess");
     }
 }
 
@@ -429,4 +411,16 @@ vtkDoubleArray *CatalystAdaptor::fab_to_vtk_array(FArrayBox &a_fab, int a_var,
     return out;
 }
 
+void CatalystAdaptor::catalyst_error_or_warning(bool a_success,
+                                                std::string a_msg)
+{
+    if (a_success)
+        return;
+
+    if (m_abort_on_catalyst_error)
+        MayDay::Error(a_msg.c_str());
+    else
+        MayDay::Warning(a_msg.c_str());
+}
+
 #endif /* USE_CATALYST */
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index d285d1d06..d42ef6a0a 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -73,6 +73,10 @@ class CatalystAdaptor
     vtkDoubleArray *fab_to_vtk_array(FArrayBox &a_fab, int a_var,
                                      const std::string &a_name);
 
+    // if a_success = false, either aborts or prints a warning depending on
+    // m_abort_on_catalyst_error
+    void catalyst_error_or_warning(bool a_success, std::string a_msg);
+
     int m_verbosity;
     bool m_initialised = false;
     bool m_abort_on_catalyst_error = false;

From 774340d696818422e3997f259244f0fe8607d966 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 16 Feb 2022 12:59:14 +0000
Subject: [PATCH 29/63] Add user to include extra VTK/ParaVIew components

These can be specified with the XTRA_VTK_COMPONENT_FLAGS and
XTRA_PARAVIEW_COMPONENT_FLAGS variables in the makefile which will be
passed to paraview-config.
---
 Source/Insitu/Make.insitu | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Source/Insitu/Make.insitu b/Source/Insitu/Make.insitu
index 5e75045f0..5e48cf9c9 100644
--- a/Source/Insitu/Make.insitu
+++ b/Source/Insitu/Make.insitu
@@ -19,8 +19,8 @@ ifeq ($(USE_CATALYST),TRUE)
   cxxcppflags += -DUSE_CATALYST
   # These components are correct in v5.9 and v5.10 but may differ in other 
   # versions. The paraview-config command may also change
-  PARAVIEW_COMPONENT_FLAGS := -c PythonCatalyst
-  VTK_COMPONENT_FLAGS := -v CommonDataModel -v PythonUsed
+  PARAVIEW_COMPONENT_FLAGS := -c PythonCatalyst ${XTRA_PARAVIEW_COMPONENT_FLAGS}
+  VTK_COMPONENT_FLAGS := -v CommonDataModel -v PythonUsed ${XTRA_VTK_COMPONENT_FLAGS}
   PARAVIEW_CONFIG_EXEC := $(PARAVIEW_DIR)/bin/paraview-config
   cxxcppflags += $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --cppflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
   XTRALDFLAGS += -rdynamic $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --ldflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))

From 951b69e67f05e13c9003b52e2fd224ceadcdfdce Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 16 Feb 2022 14:33:06 +0000
Subject: [PATCH 30/63] Add Catalyst Insitu test

This uses a script generated by ParaView 5.9.1 to extract an image of a
slice and compares this image to a known valid one.
---
 .gitignore                                    |   6 +
 .../CatalystInsituTest/CatalystInsituTest.cpp | 134 +++++++++++++++
 .../CatalystInsituTest.inputs                 |  43 +++++
 Tests/CatalystInsituTest/GNUmakefile          |  26 +++
 Tests/CatalystInsituTest/InsituTestLevel.hpp  |  36 ++++
 Tests/CatalystInsituTest/Polynomial.hpp       |   1 +
 .../SimulationParameters.hpp                  |  28 ++++
 Tests/CatalystInsituTest/UserVariables.hpp    |  29 ++++
 .../extracts/SliceA_expected.png              | Bin 0 -> 10451 bytes
 Tests/CatalystInsituTest/slice_A.py           | 157 ++++++++++++++++++
 10 files changed, 460 insertions(+)
 create mode 100644 Tests/CatalystInsituTest/CatalystInsituTest.cpp
 create mode 100644 Tests/CatalystInsituTest/CatalystInsituTest.inputs
 create mode 100644 Tests/CatalystInsituTest/GNUmakefile
 create mode 100644 Tests/CatalystInsituTest/InsituTestLevel.hpp
 create mode 120000 Tests/CatalystInsituTest/Polynomial.hpp
 create mode 100644 Tests/CatalystInsituTest/SimulationParameters.hpp
 create mode 100644 Tests/CatalystInsituTest/UserVariables.hpp
 create mode 100644 Tests/CatalystInsituTest/extracts/SliceA_expected.png
 create mode 100644 Tests/CatalystInsituTest/slice_A.py

diff --git a/.gitignore b/.gitignore
index 290526597..1b4095f2b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,9 @@ time.table.*
 ipo_out.optrpt
 
 Doxygen/html
+
+# ParaView Catalyst
+catalyst_pout.*
+*.png
+*.jpg
+datasets/*
diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.cpp b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
new file mode 100644
index 000000000..08250b455
--- /dev/null
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
@@ -0,0 +1,134 @@
+/* GRChombo
+ * Copyright 2012 The GRChombo collaboration.
+ * Please refer to LICENSE in GRChombo's root directory.
+ */
+
+#ifdef CH_LANG_CC
+/*
+ *      _______              __
+ *     / ___/ /  ___  __ _  / /  ___
+ *    / /__/ _ \/ _ \/  V \/ _ \/ _ \
+ *    \___/_//_/\___/_/_/_/_.__/\___/
+ *    Please refer to LICENSE, in Chombo's root directory.
+ */
+#endif
+
+// Chombo includes
+#include "parstream.H" //Gives us pout()
+
+// General includes:
+#include <algorithm>
+#include <cmath>
+
+#include "GRAMR.hpp"
+
+#include "GRParmParse.hpp"
+#include "MultiLevelTask.hpp"
+#include "SetupFunctions.hpp"
+#include "SimulationParameters.hpp"
+
+// Problem specific includes:
+#include "DefaultLevelFactory.hpp"
+#include "InsituTestLevel.hpp"
+#include "UserVariables.hpp"
+
+// VTK/Paraview includes
+#ifdef USE_CATALYST
+#include <vtkImageDifference.h>
+#include <vtkNew.h>
+#include <vtkPNGReader.h>
+#include <vtkPVVersion.h>
+#endif
+
+// Chombo namespace
+#include "UsingNamespace.H"
+
+#ifdef USE_CATALYST
+int runInsituTest(int argc, char *argv[])
+{
+    int status = 0;
+    // We use a Catalyst v2.0 script that only works with ParaView 5.9 or higher
+    if (PARAVIEW_VERSION_CHECK(PARAVIEW_VERSION_MAJOR, PARAVIEW_VERSION_MINOR,
+                               PARAVIEW_VERSION_PATCH) <
+        PARAVIEW_VERSION_CHECK(5, 9, 0))
+        return -2;
+
+    // Load the parameter file and construct the SimulationParameter class
+    // To add more parameters edit the SimulationParameters file.
+    std::string in_string = argv[argc - 1];
+    pout() << in_string << std::endl;
+    char const *in_file = argv[argc - 1];
+    GRParmParse pp(0, argv + argc, NULL, in_file);
+    SimulationParameters sim_params(pp);
+
+    GRAMR gr_amr;
+    DefaultLevelFactory<InsituTestLevel> insitu_test_level_factory(gr_amr,
+                                                                   sim_params);
+    setupAMRObject(gr_amr, insitu_test_level_factory);
+
+    auto task = [](GRAMRLevel *a_level) {
+        if (a_level->time() == 0.0)
+        {
+            a_level->catalystCoProcess();
+        }
+    };
+
+    MultiLevelTaskPtr<> call_task(task);
+    call_task.execute(gr_amr);
+    // gr_amr.run(sim_params.stop_time, sim_params.max_steps);
+    // gr_amr.conclude();
+
+    // read the newly generated PNG and the expected PNG
+    vtkNew<vtkPNGReader> generated_png_reader;
+    generated_png_reader->SetFileName(sim_params.generated_png_file.c_str());
+    vtkNew<vtkPNGReader> valid_png_reader;
+    valid_png_reader->SetFileName(sim_params.valid_png_file.c_str());
+
+    // Calculate the difference
+    vtkNew<vtkImageDifference> image_difference;
+    image_difference->SetInputConnection(generated_png_reader->GetOutputPort());
+    image_difference->SetImageConnection(valid_png_reader->GetOutputPort());
+    image_difference->Update();
+
+    double error = image_difference->GetError();
+
+    pout() << "error           = " << error << "\n";
+    pout() << "error_threshold = " << sim_params.error_threshold << std::endl;
+
+    status = (error < sim_params.error_threshold) ? 0 : 1;
+
+    return status;
+}
+#endif
+
+int main(int argc, char *argv[])
+{
+#ifdef USE_CATALYST
+    mainSetup(argc, argv);
+
+    int status = runInsituTest(argc, argv);
+#else
+    int status = -1;
+#endif
+
+    if (status == 0)
+        pout() << "Catalyst Insitu test passed." << std::endl;
+    else if (status == -1)
+    {
+        pout() << "Catalyst Insitu test skipped (USE_CATALYST undefined)."
+               << std::endl;
+    }
+    else if (status == -2)
+    {
+        pout() << "Catalyst Insitu test skipped (ParaView version < 5.9)."
+               << std::endl;
+    }
+    else
+        pout() << "Catalyst Insitu test failed with return code " << status
+               << std::endl;
+#ifdef USE_CATALYST
+    mainFinalize();
+#endif
+
+    return std::max(status, 0);
+}
diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.inputs b/Tests/CatalystInsituTest/CatalystInsituTest.inputs
new file mode 100644
index 000000000..5c3a41948
--- /dev/null
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.inputs
@@ -0,0 +1,43 @@
+verbosity = 0
+N_full = 32
+L_full = 16
+
+chk_prefix = TestChk_
+plot_prefix = TestPlt_
+stop_time = 0
+
+num_points = 30
+
+#num_ghosts = 3
+max_level = 1
+regrid_interval = 1 1
+
+#boundaries and periodicity of grid
+#Periodic directions - 0 = false, 1 = true
+isPeriodic = 0 0 0
+# if not periodic, then specify the boundary type
+# 0 = static, 1 = sommerfeld, 2 = reflective
+# (see BoundaryConditions.hpp for details)
+hi_boundary = 0 0 0
+lo_boundary = 0 0 0
+
+#################################################
+# ParaView Catalyst parameters
+
+activate_catalyst = true
+catalyst_verbosity = 0
+# catalyst_pout_prefix = catalyst_pout
+# path to directory containing catalyst scripts
+# catalyst_scripts_path = catalyst_scripts
+num_catalyst_scripts = 1
+# list of Catalyst scripts
+catalyst_scripts = slice_A.py
+catalyst_coprocess_level = 1
+# num_catalyst_vars = 1
+# catalyst_vars = A
+abort_on_catalyst_error = true
+
+# Test parameters
+error_threshold = 0.15 # arbitrary threshold but this is what VTK tests use
+valid_png_file = extracts/SliceA_expected.png
+generated_png_file = extracts/SliceA_000000.png
\ No newline at end of file
diff --git a/Tests/CatalystInsituTest/GNUmakefile b/Tests/CatalystInsituTest/GNUmakefile
new file mode 100644
index 000000000..23ef85e54
--- /dev/null
+++ b/Tests/CatalystInsituTest/GNUmakefile
@@ -0,0 +1,26 @@
+# -*- Mode: Makefile -*-
+
+### This makefile produces an executable for each name in the `ebase'
+###  variable using the libraries named in the `LibNames' variable.
+
+# Included makefiles need an absolute path to the Chombo installation
+# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash)
+
+GRCHOMBO_SOURCE = $(shell pwd)/../../Source
+
+ebase := CatalystInsituTest
+
+LibNames := AMRTimeDependent AMRTools BoxTools
+
+src_dirs := $(GRCHOMBO_SOURCE)/utils \
+            $(GRCHOMBO_SOURCE)/simd  \
+            $(GRCHOMBO_SOURCE)/BoxUtils  \
+            $(GRCHOMBO_SOURCE)/CCZ4  \
+            $(GRCHOMBO_SOURCE)/GRChomboCore  \
+            $(GRCHOMBO_SOURCE)/AMRInterpolator
+
+XTRA_VTK_COMPONENT_FLAGS = -v ImagingCore -v ImagingSources -v IOImage
+
+include $(GRCHOMBO_SOURCE)/Insitu/Make.insitu
+
+include $(CHOMBO_HOME)/mk/Make.test
diff --git a/Tests/CatalystInsituTest/InsituTestLevel.hpp b/Tests/CatalystInsituTest/InsituTestLevel.hpp
new file mode 100644
index 000000000..0a5669494
--- /dev/null
+++ b/Tests/CatalystInsituTest/InsituTestLevel.hpp
@@ -0,0 +1,36 @@
+/* GRChombo
+ * Copyright 2012 The GRChombo collaboration.
+ * Please refer to LICENSE in GRChombo's root directory.
+ */
+
+#ifndef INSITUTESTLEVEL_HPP_
+#define INSITUTESTLEVEL_HPP_
+
+#include "BoxLoops.hpp"
+#include "GRAMRLevel.hpp"
+#include "Polynomial.hpp"
+#include "SetValue.hpp"
+
+class InsituTestLevel : public GRAMRLevel
+{
+    friend class DefaultLevelFactory<InsituTestLevel>;
+    // Inherit the contructors from GRAMRLevel
+    using GRAMRLevel::GRAMRLevel;
+
+    // initialize data
+    virtual void initialData()
+    {
+        BoxLoops::loop(Polynomial(m_p.center, m_dx), m_state_new, m_state_new,
+                       FILL_GHOST_CELLS);
+    }
+
+    virtual void specificEvalRHS(GRLevelData &a_soln, GRLevelData &a_rhs,
+                                 const double a_time)
+    {
+    }
+
+    virtual void computeTaggingCriterion(FArrayBox &tagging_criterion,
+                                         const FArrayBox &current_state){};
+};
+
+#endif /* INSITUTESTLEVEL_HPP_ */
diff --git a/Tests/CatalystInsituTest/Polynomial.hpp b/Tests/CatalystInsituTest/Polynomial.hpp
new file mode 120000
index 000000000..c2e85b310
--- /dev/null
+++ b/Tests/CatalystInsituTest/Polynomial.hpp
@@ -0,0 +1 @@
+../AMRInterpolatorTest/Polynomial.hpp
\ No newline at end of file
diff --git a/Tests/CatalystInsituTest/SimulationParameters.hpp b/Tests/CatalystInsituTest/SimulationParameters.hpp
new file mode 100644
index 000000000..4d9ef293f
--- /dev/null
+++ b/Tests/CatalystInsituTest/SimulationParameters.hpp
@@ -0,0 +1,28 @@
+/* GRChombo
+ * Copyright 2012 The GRChombo collaboration.
+ * Please refer to LICENSE in GRChombo's root directory.
+ */
+
+#ifndef SIMULATIONPARAMETERS_HPP_
+#define SIMULATIONPARAMETERS_HPP_
+
+// General includes
+#include "ChomboParameters.hpp"
+#include "GRParmParse.hpp"
+
+class SimulationParameters : public ChomboParameters
+{
+  public:
+    SimulationParameters(GRParmParse &pp) : ChomboParameters(pp)
+    {
+        pp.load("error_threshold", error_threshold, 0.15);
+        pp.load("valid_png_file", valid_png_file);
+        pp.load("generated_png_file", generated_png_file);
+    }
+
+    double error_threshold;
+    std::string valid_png_file;
+    std::string generated_png_file;
+};
+
+#endif /* SIMULATIONPARAMETERS_HPP_ */
diff --git a/Tests/CatalystInsituTest/UserVariables.hpp b/Tests/CatalystInsituTest/UserVariables.hpp
new file mode 100644
index 000000000..b1eea0df2
--- /dev/null
+++ b/Tests/CatalystInsituTest/UserVariables.hpp
@@ -0,0 +1,29 @@
+/* GRChombo
+ * Copyright 2012 The GRChombo collaboration.
+ * Please refer to LICENSE in GRChombo's root directory.
+ */
+
+#ifndef USERVARIABLES_HPP
+#define USERVARIABLES_HPP
+
+#include "EmptyDiagnosticVariables.hpp"
+#include <array>
+#include <string>
+
+// assign enum to each variable
+enum
+{
+    c_A,
+    c_B,
+
+    NUM_VARS
+};
+
+namespace UserVariables
+{
+static const std::array<std::string, NUM_VARS> variable_names = {"A", "B"};
+}
+
+#include "UserVariables.inc.hpp"
+
+#endif /* USERVARIABLES_HPP */
diff --git a/Tests/CatalystInsituTest/extracts/SliceA_expected.png b/Tests/CatalystInsituTest/extracts/SliceA_expected.png
new file mode 100644
index 0000000000000000000000000000000000000000..c80d8606e4a2a3ab4c4972511ecdfa3d94404414
GIT binary patch
literal 10451
zcmb_?d010d`u{~mZDncHs-QrP0%8fs8bBacHrW-l0s;{sAiE%|l7wmts0d|O0>lLp
z5g{5tmIzTnA;dr>EJnbHY>k91tl7x-#!jc5-}IU1H+J~r<Z^QF$vx-YKcDx?Q5QS;
zFO|Op06^aUh_xF4to;T6)_81O1I~QC^=nP=V{43!N6dHU&V>d0`^5xDMgT|sqW!)P
zM#bk|$~^%9-vIX3-yKizof=|9XnS5z{TOLuojr&+;ZvDku_488qs45y``VHcR>K#W
zX!{Z+6Zr7W6#Z{@1bxWz+n55|XrfbI<~D_xewFSKbSPR9gNjX}d)WtS;5Y1=x)7>o
zpwGJouWvYS1HQ=r<j=gEyzPCKNQhmYLI`u_dg;6mgu%gF+~lxee^pgSFTu&3<@D%g
zhoQ>YqF4S(<ouKTt7$hTR=Vn_ryPe)jSG-$(R}T3Yipm6ijtKI>6LaYpA!Px3W3!6
zicKE;1clz|tEg*OdunnKSNlq{HsYOj=<T#LugPAD*lR!drCBq(vJ46h@##pNl1|uK
zdiob!D2h^ZDLxLV)hS=wTPkN2*Bt&b+sU>y&>>~=TE*sagNM!a`_=P`kP!~{=;f&j
zt(${r18W^6h#K`uK!UI_FozAJjm}OPryS=Akz#+Ex?~ZKHApWHuoeyAsLE}%N3E?z
z6;Zp~bc3tK&C^S<b(!Mki1=7arCME-Fsh-cQY~VVvb-26!K^Aux?Dr)vpWd2K;|f3
zOk2T9kh=YDc6xG<2FGDGjMNXsi%o*?ol`pfH=)@hM$n5&X=k1LYJa)BK&EZ-t5L6J
zT6nI3-m$hv@CoOZ?2S5faN17syy29!IP4>3m*g;g>>b;hA2YL@sN%@@le$&zbx=g7
zt|UP8Fu!p*o<7OWi0pTR#9J+z5Xzn06X)nZNERk$uzUpXUPj!uUw-YSbGZqJ-Gf)`
zTW%;Q@<a8W7K)D0GRx2IN_HL7)=}^J@lx!5-@&41lsm!A=^Uk#DPfyWpTLxkwjDQA
zcuI`kR*)TZ_?g+)@QK-~8xGT?&3o>kc8o>BWxgM?bHF&+i~vddgQBz&HMTx*k2;TD
zPkOOczGa`JdO5o!1*qWdQAZH&L;;q0)%CgNTYRs$M-{JH%xLz+)D_^p`2)_$C-_;}
zYVNO10DOn1w?oMnvXjTXefSMJfMl<ZI?)RJ<&!vrgp=IRmM;v5EshqKClrdV{p$`|
zm5Ow_TC*&zYe;DN7dd!uBMbR|d%e#0F^L++6SFr-!b0JWk+ic719W!8ng<=3xuP3a
zU&^c;@BHA#b22PCwX1i0SH!`>l-ETiQL!|Qzh0^H!$hO=ygIl~(Uiv1`~UtzMH%5k
z4{R(4bXjV|3F7WQZh0RWs>?!7Jr2#fC>d+PSK3UA>nZ%<5qN0%#et!f@ufi6*AP7_
zqvR?sp7q-FU6s136K8Z*Pslfe=uMOHh3-on*R+!A(3jaReG55W-L$kq<kChSyQ#*!
zJl{Xha+T9>h%HH{ghk#!7!L-z*0JK*MO9FDfl3pO?EWF{?cj8u+A}!34&kDUNfO#U
zc3I^LC*klGxI}0bBJg=3@whf99C|tNz0ZS`x^332kH=CXeTz;%y%M5))cLOB;b(VF
zFiF~@*rSTG?S<=)Gk?8&^x@gwh0KWj1?regLV)wy`@sv}?enM!X}liV_}#h$r$1&2
z^_}N8z1K66zejkz;r8j*_k%kO8M?`BJ%+amZPk9fWRp{7d!2jaz7bSqeL{@7<k8OB
zxwRIDp56J5Y2nQr+WmASRN38m?N>!QhGz?PopUz5pEJ5%PLSE?vR&DbQ9z){&aM=`
zJkA8x5jV*fT7E&`7z6^UNal?Y<>XtZU*F=auj2H5^{0opUU4HKZzuk_4vRrERl(K{
zRDp6Co1%JyM<HGsQyh?58Dz#X0wYMM&Tott8dHvm7IY4d6BI+QY+z1#eRx*SSe>6t
z(ua0LcwQU)Rv4zgG!M>Z`&JL*?-O%^N;&ppE&LHY#+hz;@rN$_MQ7qdwgM;}yj(8>
zyip%TY@Nz2HXkve4wOA-`?6Ot31VR#ssGZPxYrfyNv?)}z*fmliz9L_6cuG~`5mYM
z9oGG!n3fIalz4Y_5iSj3EQ1t!zGUbU0qF_U%kf_qs1d!5CuAUy^MXD7^X}{4pBc47
zVv3?!TeYEH5e>XQ%@l30u5tQNlq*xe<B#WHTXrK*fQ~I@VXRH7fRT7e+bCqZU_H-1
z+0pH;c7tMhyJkb!m)qo*Pw_9BByY-5OXlD1kUhWomjj!4^Rn-sk8U`PxflJcXr1A!
zOQIy1`U8BK8*2Bx4`|BB<KJfEb%E^~2w>hSZxiA4>vPq4f0_uSm%2aF)-&%o6<w~^
zKfL;Fhj%LSt!Zfau~5%uk;BYO*>isRsD!#9xiXAkf0;t6Lx0@ORlnsk+VmZ<+Vr9R
zX94ki=a;6&pg$4SC1xZ_<IeB)h7m?nobXqBz`MFUS2#-->g14d<zDn>qe3_wA!$MC
zH>vbPmkG<DXh%|OTQASW!RAgsVRV*8;Vx!Dp=aKp3a+EgOo?hwi+rmE@UdXNe;#6$
z-*1R68SQE!xL=@Ai7&Yq5mW<Su8%%cJj7O%Ut75=<cRuoNZ$mBac<>(fo?~i&s?MO
z682d2e9}%yDC~K#J^pRcZMDa2Q1_<thsDz&FxjUkLvp>Sg7*~#xG7!r<=DY~QU^P-
zsvBNVv=cjGWlf*i2F%*+APydTn05dHnCZDR+&vBPg90&UilXhFL7!U#zqXo;Wn8kc
zwl;zQ7Y=;--tqcnd^>QcLK!K(d$<H9yJ`6pV4-v$G{2SWGRakPCk{V@zFPCw7teiv
zB>I<(%;fjGdlJY_sFE``J2pdvW?cU_{ZYtNw94by^RrlFNYdNR#41!(ON8l@A&%P!
z-MP5HE8nJT{jY((DRl#k*+q?CT5fkQwoFYj4rHQ~=o6K<6g4Clgp(EIQ%$xa&g{}t
z$t(Ax4_#hho5e$3qwg{)xG8<iez$n!21L;CP#fck9QG(tL4!@DMQ9<O1e*jA;z5DA
zUua=+r1BEn5EHvwjB(l&6pZJ|VuWUGIDdgvh`tA51vpD^ytlJMUA6-`4r&$ImaFol
zV1YRee-YJx4r>%vJsjL&S#E~HzHzT+KN+403<=(Gb_RM-a}vH`^<k^pLZz7keOmC;
zA2!zzHuN>A!moK~S2JaGt;7Peb52rOmLMlTu}$+sYG&#?N~&l<@1O!;a$gx$g<5S)
zY;Js~;xs*3PeO`et?{@8hB|Tg(7X)Le(WItn7RMgLp5u@++$oYBH~5NEO7kP7SnD_
z9_aT&rZ(g^C<w<YrVxm7qkOMvYCgF~U-zxSQ+pw&Wzj_2s=VcBYP94;#}A{5btIyn
zZ<B2jycscznBNM<^0`m>TYJ+~L{Y)8dsd_Hs>y7|6ZFN!d4&1d3m5C_#DfbFu(_X-
z5(E$P6CNf?76TWkVU0e0(B+qztkH+X{ZQnMYWhnIH9z41QSHzw#ct{6?sE@#Z3s!>
zcvp^=FeAdOtW6I+V7%+&#tnJg!qkgx-*Haopc%e;Y?v;%&UpU@WO)~AvMS+v3(v3G
zG*@JWRiXM`n>|7%ZX%8_rWju9lILUNCw4&~in1$k+=BeK#N8Js0pL*fC0XE?oB!3&
zw9(bZ=i}6L$i7;+cuINy?VdD`kn&pX+MY3Hb0=V~I}<^gevDlm=}*Hk;=<KcuYidY
zW~B`YMsP-CHR<hwj&Mr{N9^_CJ4+=9Q|dMtsg2tj;ArnpXL5d2ZXk@zt)``=B_(uP
zmZt<TRJ%R8X`(`DXiypP7(+6X^wHCKkA_Ra-vrt@w6HTub7GI!?GcxDliaMW1+#7D
zRqE5|K_m(E4@hF4NXz$uO~PLO{MSJ8R@qI!ErZV(CId#{S+dEOy&YU$)_@tmHo^zo
zQ209({3_n^fx2osXEcVAzw&$>y;Nz5el_imz}x_RV@f7x)EJizhPx#pxKq<6ZnnkI
z*iMbRfk=X}kr>F>Eg7QM=dT<Vc#LIYmB_S|`MucbJg)hbwA-aKw4L8ik+fNDo*^x3
zVVvzDXA^Fd6=SLF2R0@!R>3wPdGBZJ<kJC629!#-)AIXvyu~+Qlo4ro&UG<<z<)G6
zbZ{sz9<PRw4jDhE4`D!0fp@WOwG;#6+#jy07pHT-J;B^!iZ|aA?z&$^hcm*SmFlag
z;be!`hfP+-MJhpd*_JNHcB%%mh<MBem)s$6^@q-w<$(0Np(b?z(0=s4w&8zRy^HOI
zEuMhMtFqAmr%)MHTJHdN+pqsfE}EGy2e{MoRu>p)I}qaqpnWa#Xm85mYuJPw;X-_7
znyHG5=46pY=_A7@ZVLsLt43r}uoi26Zwh$m*)F!$hCjJ0PxGqn47=@WhEZ43{P1YO
zd=c7fM29ktsmre~@cG%FUa}4V_G)<mK=J1cnl0OY1w>mKa*NV~VNdroV7Aa0pc5?j
zuudOHiuMK5B8tdzQE#+hC@3e(*Wi=th~m3QN-(+(|861LdFObTrTwbMSPmFVvC>%T
zPeYW4Mc-o)k>g3x-q(`8k7udb%TYj;8@~U*X8QU-SoW4I3m*VD;`X_R{>kaO*fJut
z6oJ&=U!C>DQ);z7#-81@-No9JL>TWUQ%V?@NuQQx1$?847=3D>wKn)-OqytSCC3!!
z3VOG0S~k|Xe+@p@tq-c$Jwr6+5V7e2H>+t;c#9RtG5Cnrs4r$IV}P4!I-beetNDD>
z`v2BX{mH4t?pHPjmE8(d_BN^QdN<J{LXcqEIqFM`+=_SY<OEoAt6lH=u}p;vndI?o
zUxPd2Esk*-#!g*H`XmDWwNmf}H+8goq2(&e+nR>g%(5tR6hzq?nwN{ThGt7TsCSn+
z1;HWp8^?oY>gkrrW~iJN!>Ye;rN<4!K5PO~>vM}TQm(g}-k%dkz<9U(9QoXLGkU}v
z=b3jRGGDWH!L?5mpGhF%eIEO15OU;0qu23AkbP_LJqubsA#uIjyHBbEM%2X%{`Tw%
zBKxSGsvje5AM392NPT|&NkUAtQ>MF%l|XwP@QvNGHNdY<pD~zyUHh18`XzuEaf>#S
z7W$=0Z>nrmwK>bKmaK<mNj{sBc2mO`I;ZAC$JIV&yAafZtb3A7io-k%x~x0tV0O`j
zW>5(z*sOiri?r_9l&kw#sYmA(cZ-3J8y2N{L25ziV4>j1M==#+3tSY+0|dLHW6HP7
z0m<f{?FEnbv`;5|32Xq>IYVlRT54duxxuK(Sq#YrYuY_fs7j|U4ttLia5>*7lLzqb
zn+i*LZPb9sn4O*XlE}HWt=oCSqH0%?XTic9t2o9RVh&_%2c3Y&=->QuZXY=^3;?pX
zY6HogpD{deKQ=Ffu6?>2tnmH4HqCm(3AqDjXzTA1a)K8(hT0bOT3MttK2WdsZcA8n
zTpDkf3h&<PvL6L|9bx*sFsdlE5PANHrl~v&z8e7SzuX2K3i{lkul!!rkKrF>5?uaa
zM31=RqXT#P0%i~QqF^#z@AyxgmB&5pCDyn6Uaz<FG_1@eZyT;D?K%kn(VwyPPbc<(
zIajmu%9?XB;BUT=8g38D^BQ38zv!t*yKhpw6Y&{p`kC7OpB@VXMRTO%fOW1EysM0b
zMZf+?&IfdkCH5g8`_yB9NV}&;vPsz5B}lT+qrnvc(dL3|_uHpsfEdE(4aSwS3EH?v
zqsAO<N1tovpmus$JTHX66l>E%uUu@qaIfuXLG8MleL?5vy&XcPYfWDkx~?8gJx;d0
zo(OBbYl4Fhzi@o~EwG~g2|D^ud;9<J1k&w$7`w-9U-0VRteBDI4I(XRF(%Zo)!XAa
ziLlVVr21XM-xGC0zi?^M^A*>)Mf*><xSJ4%Wkz$X%x)GnLLkR@^l`P2nSkrcb)mae
zU${3v;=4Wek$I|nPRmr~b5&NV?o6*_J2#|vUI5!CAB=@b(VZHOJ+<xIyXm%|Ct`*g
zoA;;}J0UvL^3R!hGHLr7m|D;fF#rArD`Gm>&y3)}WJpP#{_<xR0$fwaU-0v|S`dbj
zCbA0mi2JqZ!J}4O?N^!5#Y=M=$x~kfm>(QzoLL9pZwYqq;)@en4b2C2!SE?e1hXy|
z%}^glsqYln=k3O&tK%MIIfIc$oQ-kg+t9t<=M-YgvNe6Jd$<OT9$<m}y@TL{v*yme
zUAU5Caja})oXKH*$xvGqScIh9E%c3Z&^+&Y|3Xelde5epd)lRHAPcHN%eOM09bs8g
z+kB-9SFby9k<|qPb}0NiA{zfL8SSFE=L{9;Xx^@SFbI#qFx<LBby!Aw?o`QIYB?I8
zVk&+Us}EM(m8=;TU*6O%NR~_fZnUHME-*Z`r?xH3D&b=o<zvt{55tT-cV?FpXQq<O
z$hxq$z5^LD@4qU@X8s*B>LX{FwAG-E)Zv_6@#Pl1d+u=aYulTCPh$p`W*`vZ0$C_6
zRWOX*v7>vu`odVTwYG6AxA{;)Z_MMG!R%hH{bFlj$#GvnZx>#Zzoj;vQ}^x{-opIM
zucEc>B-1io557@b3K0osp$eSiScNW~4FL>lu_;-4{AYKNfpck~)_is<3}$7HkuIHC
zQ_V`yI{OKc-l=PC;c=%HqYi@APoB>b#jf>dWSo=5a@rBqvb2%CIBx$MMF&Q+8<@!4
zEU0b2yuc)J+7ioo2RY+8dVy?rRJ+C=e9=oVun`v2_fW$yj0$d{k$9+Jp%6^WSa;2c
z1;r%t1TDJQ(NYNKaN0|BcKKj-t$q5Pj=HAR-}T6!e5oZSXGj*^===hDhSdj3WK0TE
zyfMz@MZHRyrz@!A(sy6H1?AC3?y*kp&yUfMH?yhPJD)l4HkNt9@+Lx@vOq&zmqHx$
z6W^#cJ7~gLqtSV2H~wf+v1KWQeqt%FFp^B~<QOf(L$4fsSE{+^;^uQP@*_I{g(xjk
zrN85FSmCcsA5Tpd4wjyO2Udknb>*^q&g`njb-B9Jzn`*Nk}*6k-lNUBGsc;yt3{G&
zAnXZdLm-7DIV2d>^%op>rn>ITb~jUkoa8vZ)WCUHRV#jjFJjWf&Vu)LTv9cOlT;1j
z2(Fsdp#-~iZGGn=qP!d8WC3bBSPbp^+=2wGV^DDOi8qfy(#P417!Ot=JyNhW+Y<6I
zS3C`Ys1LnoKJ!1jGVVxsS>%UL6`(+o*dyMUQUYnk36OAM`#4_A3`r1#5646wp%VPf
zOg~9Lz)*wW!V(%fEt8X?F<H*jr0NYJ%lcw{8*XIZX2tRCQZJXpJ29;is}uPo=$nh%
zqGRMQ@BAMKqVl!r<KcmDxpHT-c=x3frwCy`l#nSsb_|f?I>sEx=(~*?n(iLxQjGPy
z271>@agncav9|xx&wc8YKwLH&cpY;*2V@KE1-YGSl6JhvUvRy!WOG!WufZAn)Pl*d
zPuuYgQ0$quSwg*~dN?NeNuptMptO)E*VVa23~Vz5;&_33E`T+kcb8rZI$E2*X(HN9
zBW89>V$$kdT)a`PSBhD#mxGGK^|Z2lFS?tfPlp@bjXqW5d5ZgPDyDW@;(<VTP2}XL
zJS8LWF3KKpiL0|HPtzyfg`iv_L#eKaa=qfB>eL#)_%~oVpH_$wxK@DNi0DxVnDRQ$
z(?IB{3apH@PqJo{ilvfs4Oxa>JhH3{hWRB@-)3Zc>N05pv`<|BfOKR;?2BDgj;dy@
zGTyqGD}ab{?(HY_@5G|6Hn%;^jYo&3$+DZ0>9Zfl&wVXq89`53nLg%LuKldkbO~yX
zU=I#<(jWGa&BG*5dd^6+k{yrDEXArisDS5UrXWA?>O52m(2=NooZ3WzzvoLgA~_vI
zP&Y%)tBjj*LGVw%-;bIurFT}T{mOVL&|FyHx2yVKBULQ|ENeqdJ5A?%F@0F4a?XBF
zCgPS4nce&ftWB$<@FiE=1@}M~!X#n1%+y>_@v9q_;+G5dc?j2THoCz9yPU>1qi|O>
zaDz>+RJ+AL(Bj{$_JSrgy$Hf1bhr>av6JjU5W;SH8`kE>KVCZB+;Q8AH?-iVJl-4H
z8b2bGEI*Ynwj81Qn3DapvHBR3*$EV@^<;mK8e4UTL0PNE$CX&u#5#%~adYyPxv6WJ
zVB}@9pIEF+ASkv0o4#$!Z=!*Gy8Y+*<X3@R6B8icHGnEoTXLZa@248Ch?WF#zA0t-
zx>N-X8I${)WA`D^*INp3hS<Dabl&WVHnO+;{Ik78Cy<VmERB;^BN8q0Q2Xh}$F=gy
zwIPp4EtV;0mF3ts{eGvUL>fZEF8J---@-;_Qu{aLV2?KBqxNySi&8c-0bpZ<v=?Ae
z2NVW=DpauUpm~tOKK`hs@Igupd0H9s>#0vj0gFfsS}k*cbMympU~G$zUhl1Q^EG=x
zn4yh2d@qB=)WuHsI>#~6JgPwW3|s6%(Ck<p17-0^AUp;J#$#1PMuJEYCKi2Aj;_{I
zetDuZl2C{9Tb0U1K?8fRxnVyA6boJ<QZn*`1gH6P^J-MU$PTIsTXkh#AKA9+k#Ip%
zB5e8Ll2PuC4l=)yC|aHR^pAs>$6Jn|@Dt^dB?co8pEL^0Yb0qikqYVJ%d0PX5xYe5
zc;p@vi2SmS_x;;N^e$zfGx@f|K93(gfnW0e`oh{;ZxbNUJN||2wX_`$xz&95BA{8>
zI_w(oz5!%7&QWE5xf0c(B71EkRS)?6a*NEZEhD)HzuX4VSaV48rB*!1ZaJ+>c1d@i
zesgU$8;xCL#@!q@dv(k*T-HZ%>N|>3KFFpN96Z$T1&Vv6)Eb72{P;vkVde<(!+cz)
zf6pWBlU6O?GmBKD<Pq2k5RGtsa#iwZTf=g}7;;sl_`7U%Qe7{X*^2K(!)jmAk1BEv
z#n;|Rk!!{AC|FeF4bYwNszXc8r&^vl%Q{-iRp?^jcgC|eeC5A%=*11+^CWv>-om2}
zHHj$H4At*#%o<I~Tb+i?J6~Yoq?oG3-lgAMQnmkwUU*$-R+m(a3iJY>`?)Rn_E~W+
zuEI}Jeq;!(;<SpxNfO_?D)NWMLqPxhD1fM7M?778HGm=@?sSj??Q0@_7{OI=t-o!Z
zYlfMFct+OYF54$xgUit?DtavXdLK#_$6LHU?o*v|d%j;+Rxh6m%VfiVl1dTqO7Vow
zjY~G^oXn`MuI>Ni`P?6#t!H)o>%<LNK7Y+QKi=8{h~o{XL29u28w3*2al!!Osr$D(
zJ%grfR$tz^mcRF@x>nT67A1@hew3``VO=vAIW{x=PpZ-R8T1YcI!cs#`l3CvS!6Qy
z>7IY(CqWg-cMYexDkWdbQbY5jI#BKpL0W&T<6mde+9N+&u0FS&1GZ};1iX=4dSLa$
ze9oY|1r$ma)<|r`)okx3q~i;a1c~o&shI4EBO=iCCCE&WkXKvo2c3@)y;OCpR5j4L
zwVu9u_v7$>TzXm*4t+1SLxY|R0<QEbC3SHrk7m%lPzs_Z1j;7{zQ@_vw63U|Kf4+T
z<`mcj5`XRZN!Hc#^A{+fCWTasmtM5h_B64bBB|6w3fMif0*A9$1NvR21|YCu-5Lnu
z?D(<B6`;iW5BfTO6OT#;X7~Qaam!v??@vFVoRV_jcV2v}wzA>SQ==i}HRjC(y<S7$
zJ1KRZ6kO;HWRilNjN}i2T>USG$U=?ne|kht-9cb=)$z)aR2fwt29|4k#jLdDt1q`z
z7ETWrPAkZ$x-*kQl+mAN2u@7Vn*T`Yb8=1>TKcd5Gt190uHvkR{AuRneH6TkfNlnU
zUm60Iv<CTXB3ep?ZidYD4<OkQ#_aHj{!0)DVz7DjAxjiE9|79|(lART>A{IH)gZeP
zRxP=A0c<%k-@y#EZ58QVTx;M2g0e3ca@ELe3!db7(-^zFV1ZmaJh9P}Kl5&()GOnJ
zU?mge0P&jh8kF(qVvH?B_{rdk2dN#+qLE24$Tl(pmp=x6l1hYNR?HZXN-85#>en8W
zLA?TXw`&VEeoU+1&(YYdR~ETahylAH+J0||SqC}}?lSAj?2m+e_ThF($6<`}zqOj|
z^B|sySD3j<;Aczh;ALZM9w3VP_ih+2nd&o}OX!{*Sr8rQ_=}bmaGej@n5u&|EeARz
z+@{h$H=F#?;`5jScF`CPO^s=uT#eiER~<eBmGQEFbls@AtG(s5F4YeIc#}C?W}LKH
z@u^yYnigO0>+5T0%Zpx=jovi5%pAK}FHTsGNP1USbbwDg!DN2*|E~N&&ZQve!_tx*
zriwyhDQ0KnQ=0ywWyZRt;l(kuNou?e^FSffph}0e_m7sD-*)j^n-66<XtoM${ov!9
zCjO!eDQqce|LW4rgtQ3?*`%!-(ZO(TXi%*pefW6aJG{d%kFga8_DRKs7<zVNQ4(Uy
zfJBw*<2WRYNRsr=O@c^QD<e?o{=~!4#=14TM5WetRN>?)eV9s>oqb^BaOh|bYeqGE
zIEie(gsME|sG|fMR(7NfJ?m@C&35V=)#&2~^t^p|Io2{Mebg=I;CXh~W7<gxWe%;c
z1i_jG)cxd}pTcOyahGH3NE9^qL8%xB7uR1O$fQq&i_e{6H<7SHaErnB{wH00;3DQJ
a61CIKl@Gi>-jcR20`@j8)>Vg3U-~~R){C0}

literal 0
HcmV?d00001

diff --git a/Tests/CatalystInsituTest/slice_A.py b/Tests/CatalystInsituTest/slice_A.py
new file mode 100644
index 000000000..8b8d16009
--- /dev/null
+++ b/Tests/CatalystInsituTest/slice_A.py
@@ -0,0 +1,157 @@
+# script-version: 2.0
+# Catalyst state generated using paraview version 5.9.1
+
+#### import the simple module from the paraview
+from paraview.simple import *
+#### disable automatic camera reset on 'Show'
+paraview.simple._DisableFirstRenderCameraReset()
+
+# ----------------------------------------------------------------
+# setup views used in the visualization
+# ----------------------------------------------------------------
+
+# get the material library
+materialLibrary1 = GetMaterialLibrary()
+
+# Create a new 'Render View'
+renderView1 = CreateView('RenderView')
+renderView1.ViewSize = [1092, 778]
+renderView1.AxesGrid = 'GridAxes3DActor'
+renderView1.OrientationAxesVisibility = 0
+renderView1.CenterOfRotation = [8.0, 8.0, 8.0]
+renderView1.StereoType = 'Crystal Eyes'
+renderView1.CameraPosition = [-31.513131514425908, 8.0, 8.0]
+renderView1.CameraFocalPoint = [8.0, 8.0, 8.0]
+renderView1.CameraViewUp = [0.0, 0.0, 1.0]
+renderView1.CameraFocalDisk = 1.0
+renderView1.CameraParallelScale = 12.374368670764582
+renderView1.BackEnd = 'OSPRay raycaster'
+renderView1.OSPRayMaterialLibrary = materialLibrary1
+renderView1.Background = [0.0, 0.0, 0.0]
+
+SetActiveView(None)
+
+# ----------------------------------------------------------------
+# setup view layouts
+# ----------------------------------------------------------------
+
+# create new layout object 'Layout #1'
+layout1 = CreateLayout(name='Layout #1')
+layout1.AssignView(0, renderView1)
+layout1.SetSize(1092, 778)
+
+# ----------------------------------------------------------------
+# restore active view
+SetActiveView(renderView1)
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup the data processing pipelines
+# ----------------------------------------------------------------
+
+# create a new 'AMRGaussianPulseSource'
+input = AMRGaussianPulseSource(registrationName='input')
+input.MeshStatus = ['Mesh']
+input.CellArrayStatus = ['A']
+
+# create a new 'Slice'
+slice1 = Slice(registrationName='Slice1', Input=input)
+slice1.SliceType = 'Plane'
+slice1.HyperTreeGridSlicer = 'Plane'
+slice1.SliceOffsetValues = [0.0]
+
+# init the 'Plane' selected for 'SliceType'
+slice1.SliceType.Origin = [8.0, 8.0, 8.0]
+
+# init the 'Plane' selected for 'HyperTreeGridSlicer'
+slice1.HyperTreeGridSlicer.Origin = [8.0, 8.0, 8.0]
+
+# ----------------------------------------------------------------
+# setup the visualization in view 'renderView1'
+# ----------------------------------------------------------------
+
+# show data from slice1
+slice1Display = Show(slice1, renderView1, 'GeometryRepresentation')
+
+# get color transfer function/color map for 'A'
+aLUT = GetColorTransferFunction('A')
+aLUT.RGBPoints = [42.015869140625, 0.267004, 0.004874, 0.329415, 64.0117780078125, 0.26851, 0.009605, 0.335427, 86.00207853515624, 0.269944, 0.014625, 0.341379, 107.99798740234375, 0.271305, 0.019942, 0.347269, 129.9882879296875, 0.272594, 0.025563, 0.353093, 151.984196796875, 0.273809, 0.031497, 0.358853, 173.97449732421876, 0.274952, 0.037752, 0.364543, 195.97040619140625, 0.276022, 0.044167, 0.370164, 217.96631505859375, 0.277018, 0.050344, 0.375715, 239.9566155859375, 0.277941, 0.056324, 0.381191, 261.952524453125, 0.278791, 0.062145, 0.386592, 283.94282498046874, 0.279566, 0.067836, 0.391917, 305.93873384765624, 0.280267, 0.073417, 0.397163, 327.92903437499996, 0.280894, 0.078907, 0.402329, 349.9249432421875, 0.281446, 0.08432, 0.407414, 371.920852109375, 0.281924, 0.089666, 0.412415, 393.9111526367187, 0.282327, 0.094955, 0.417331, 415.9070615039063, 0.282656, 0.100196, 0.42216, 437.89736203125, 0.28291, 0.105393, 0.426902, 459.89327089843755, 0.283091, 0.110553, 0.431554, 481.88357142578127, 0.283197, 0.11568, 0.436115, 503.8794802929687, 0.283229, 0.120777, 0.440584, 525.8753891601563, 0.283187, 0.125848, 0.44496, 547.8656896875, 0.283072, 0.130895, 0.449241, 569.8615985546875, 0.282884, 0.13592, 0.453427, 591.8518990820313, 0.282623, 0.140926, 0.457517, 613.8478079492187, 0.28229, 0.145912, 0.46151, 635.8381084765625, 0.281887, 0.150881, 0.465405, 657.83401734375, 0.281412, 0.155834, 0.469201, 679.8243178710937, 0.280868, 0.160771, 0.472899, 701.8202267382812, 0.280255, 0.165693, 0.476498, 723.8161356054687, 0.279574, 0.170599, 0.479997, 745.8064361328125, 0.278826, 0.17549, 0.483397, 767.802345, 0.278012, 0.180367, 0.486697, 789.7926455273438, 0.277134, 0.185228, 0.489898, 811.7885543945312, 0.276194, 0.190074, 0.493001, 833.778854921875, 0.275191, 0.194905, 0.496005, 855.7747637890625, 0.274128, 0.199721, 0.498911, 877.7706726562501, 0.273006, 0.20452, 0.501721, 899.7609731835937, 0.271828, 0.209303, 0.504434, 921.7568820507813, 0.270595, 0.214069, 0.507052, 943.747182578125, 0.269308, 0.218818, 0.509577, 965.7430914453124, 0.267968, 0.223549, 0.512008, 987.7333919726562, 0.26658, 0.228262, 0.514349, 1009.7293008398437, 0.265145, 0.232956, 0.516599, 1031.7252097070311, 0.263663, 0.237631, 0.518762, 1053.715510234375, 0.262138, 0.242286, 0.520837, 1075.7114191015626, 0.260571, 0.246922, 0.522828, 1097.7017196289064, 0.258965, 0.251537, 0.524736, 1119.6976284960938, 0.257322, 0.25613, 0.526563, 1141.6879290234376, 0.255645, 0.260703, 0.528312, 1163.683837890625, 0.253935, 0.265254, 0.529983, 1185.6797467578124, 0.252194, 0.269783, 0.531579, 1207.6700472851562, 0.250425, 0.27429, 0.533103, 1229.6659561523438, 0.248629, 0.278775, 0.534556, 1251.6562566796874, 0.246811, 0.283237, 0.535941, 1273.652165546875, 0.244972, 0.287675, 0.53726, 1295.6424660742189, 0.243113, 0.292092, 0.538516, 1317.6383749414063, 0.241237, 0.296485, 0.539709, 1339.6342838085936, 0.239346, 0.300855, 0.540844, 1361.6245843359375, 0.237441, 0.305202, 0.541921, 1383.620493203125, 0.235526, 0.309527, 0.542944, 1405.6107937304687, 0.233603, 0.313828, 0.543914, 1427.6067025976563, 0.231674, 0.318106, 0.544834, 1449.597003125, 0.229739, 0.322361, 0.545706, 1471.5929119921875, 0.227802, 0.326594, 0.546532, 1493.588820859375, 0.225863, 0.330805, 0.547314, 1515.5791213867187, 0.223925, 0.334994, 0.548053, 1537.575030253906, 0.221989, 0.339161, 0.548752, 1559.56533078125, 0.220057, 0.343307, 0.549413, 1581.5612396484373, 0.21813, 0.347432, 0.550038, 1603.5515401757812, 0.21621, 0.351535, 0.550627, 1625.5474490429688, 0.214298, 0.355619, 0.551184, 1647.5433579101561, 0.212395, 0.359683, 0.55171, 1669.5336584375, 0.210503, 0.363727, 0.552206, 1691.5295673046874, 0.208623, 0.367752, 0.552675, 1713.5198678320312, 0.206756, 0.371758, 0.553117, 1735.5157766992186, 0.204903, 0.375746, 0.553533, 1757.5060772265624, 0.203063, 0.379716, 0.553925, 1779.5019860937502, 0.201239, 0.38367, 0.554294, 1801.4922866210936, 0.19943, 0.387607, 0.554642, 1823.4881954882812, 0.197636, 0.391528, 0.554969, 1845.4841043554686, 0.19586, 0.395433, 0.555276, 1867.4744048828125, 0.1941, 0.399323, 0.555565, 1889.4703137499998, 0.192357, 0.403199, 0.555836, 1911.4606142773437, 0.190631, 0.407061, 0.556089, 1933.4565231445315, 0.188923, 0.41091, 0.556326, 1955.4468236718749, 0.187231, 0.414746, 0.556547, 1977.4427325390625, 0.185556, 0.41857, 0.556753, 1999.4386414062499, 0.183898, 0.422383, 0.556944, 2021.4289419335937, 0.182256, 0.426184, 0.55712, 2043.424850800781, 0.180629, 0.429975, 0.557282, 2065.415151328125, 0.179019, 0.433756, 0.55743, 2087.4110601953125, 0.177423, 0.437527, 0.557565, 2109.401360722656, 0.175841, 0.44129, 0.557685, 2131.3972695898437, 0.174274, 0.445044, 0.557792, 2153.3931784570314, 0.172719, 0.448791, 0.557885, 2175.383478984375, 0.171176, 0.45253, 0.557965, 2197.3793878515626, 0.169646, 0.456262, 0.55803, 2219.369688378906, 0.168126, 0.459988, 0.558082, 2241.365597246094, 0.166617, 0.463708, 0.558119, 2263.3558977734374, 0.165117, 0.467423, 0.558141, 2285.351806640625, 0.163625, 0.471133, 0.558148, 2307.3477155078126, 0.162142, 0.474838, 0.55814, 2329.338016035156, 0.160665, 0.47854, 0.558115, 2351.333924902344, 0.159194, 0.482237, 0.558073, 2373.3242254296874, 0.157729, 0.485932, 0.558013, 2395.320134296875, 0.15627, 0.489624, 0.557936, 2417.3104348242186, 0.154815, 0.493313, 0.55784, 2439.3063436914063, 0.153364, 0.497, 0.557724, 2461.302252558594, 0.151918, 0.500685, 0.557587, 2483.2925530859375, 0.150476, 0.504369, 0.55743, 2505.288461953125, 0.149039, 0.508051, 0.55725, 2527.2787624804687, 0.147607, 0.511733, 0.557049, 2549.2746713476563, 0.14618, 0.515413, 0.556823, 2571.264971875, 0.144759, 0.519093, 0.556572, 2593.2608807421875, 0.143343, 0.522773, 0.556295, 2615.256789609375, 0.141935, 0.526453, 0.555991, 2637.2470901367187, 0.140536, 0.530132, 0.555659, 2659.2429990039063, 0.139147, 0.533812, 0.555298, 2681.23329953125, 0.13777, 0.537492, 0.554906, 2703.2292083984375, 0.136408, 0.541173, 0.554483, 2725.219508925781, 0.135066, 0.544853, 0.554029, 2747.2154177929688, 0.133743, 0.548535, 0.553541, 2769.2113266601564, 0.132444, 0.552216, 0.553018, 2791.2016271875, 0.131172, 0.555899, 0.552459, 2813.1975360546876, 0.129933, 0.559582, 0.551864, 2835.187836582031, 0.128729, 0.563265, 0.551229, 2857.183745449219, 0.127568, 0.566949, 0.550556, 2879.174045976563, 0.126453, 0.570633, 0.549841, 2901.16995484375, 0.125394, 0.574318, 0.549086, 2923.1602553710936, 0.124395, 0.578002, 0.548287, 2945.1561642382812, 0.123463, 0.581687, 0.547445, 2967.1520731054684, 0.122606, 0.585371, 0.546557, 2989.1423736328125, 0.121831, 0.589055, 0.545623, 3011.1382825, 0.121148, 0.592739, 0.544641, 3033.1285830273437, 0.120565, 0.596422, 0.543611, 3055.1244918945313, 0.120092, 0.600104, 0.54253, 3077.114792421875, 0.119738, 0.603785, 0.5414, 3099.1107012890625, 0.119512, 0.607464, 0.540218, 3121.1066101562496, 0.119423, 0.611141, 0.538982, 3143.0969106835937, 0.119483, 0.614817, 0.537692, 3165.0928195507813, 0.119699, 0.61849, 0.536347, 3187.083120078125, 0.120081, 0.622161, 0.534946, 3209.0790289453125, 0.120638, 0.625828, 0.533488, 3231.069329472656, 0.12138, 0.629492, 0.531973, 3253.0652383398437, 0.122312, 0.633153, 0.530398, 3275.061147207031, 0.123444, 0.636809, 0.528763, 3297.051447734375, 0.12478, 0.640461, 0.527068, 3319.0473566015626, 0.126326, 0.644107, 0.525311, 3341.037657128906, 0.128087, 0.647749, 0.523491, 3363.033565996094, 0.130067, 0.651384, 0.521608, 3385.0238665234374, 0.132268, 0.655014, 0.519661, 3407.019775390625, 0.134692, 0.658636, 0.517649, 3429.015684257812, 0.137339, 0.662252, 0.515571, 3451.005984785156, 0.14021, 0.665859, 0.513427, 3473.001893652344, 0.143303, 0.669459, 0.511215, 3494.9921941796874, 0.146616, 0.67305, 0.508936, 3516.9881030468755, 0.150148, 0.676631, 0.506589, 3538.9784035742186, 0.153894, 0.680203, 0.504172, 3560.9743124414063, 0.157851, 0.683765, 0.501686, 3582.9702213085934, 0.162016, 0.687316, 0.499129, 3604.9605218359375, 0.166383, 0.690856, 0.496502, 3626.956430703125, 0.170948, 0.694384, 0.493803, 3648.9467312304687, 0.175707, 0.6979, 0.491033, 3670.9426400976567, 0.180653, 0.701402, 0.488189, 3692.932940625, 0.185783, 0.704891, 0.485273, 3714.9288494921875, 0.19109, 0.708366, 0.482284, 3736.9247583593747, 0.196571, 0.711827, 0.479221, 3758.9150588867187, 0.202219, 0.715272, 0.476084, 3780.9109677539063, 0.20803, 0.718701, 0.472873, 3802.90126828125, 0.214, 0.722114, 0.469588, 3824.897177148438, 0.220124, 0.725509, 0.466226, 3846.887477675781, 0.226397, 0.728888, 0.462789, 3868.8833865429688, 0.232815, 0.732247, 0.459277, 3890.879295410156, 0.239374, 0.735588, 0.455688, 3912.8695959375, 0.24607, 0.73891, 0.452024, 3934.8655048046876, 0.252899, 0.742211, 0.448284, 3956.855805332031, 0.259857, 0.745492, 0.444467, 3978.8517141992183, 0.266941, 0.748751, 0.440573, 4000.8420147265624, 0.274149, 0.751988, 0.436601, 4022.83792359375, 0.281477, 0.755203, 0.432552, 4044.828224121094, 0.288921, 0.758394, 0.428426, 4066.8241329882812, 0.296479, 0.761561, 0.424223, 4088.820041855469, 0.304148, 0.764704, 0.419943, 4110.810342382812, 0.311925, 0.767822, 0.415586, 4132.80625125, 0.319809, 0.770914, 0.411152, 4154.796551777344, 0.327796, 0.77398, 0.40664, 4176.792460644531, 0.335885, 0.777018, 0.402049, 4198.782761171875, 0.344074, 0.780029, 0.397381, 4220.7786700390625, 0.35236, 0.783011, 0.392636, 4242.77457890625, 0.360741, 0.785964, 0.387814, 4264.764879433594, 0.369214, 0.788888, 0.382914, 4286.760788300781, 0.377779, 0.791781, 0.377939, 4308.751088828125, 0.386433, 0.794644, 0.372886, 4330.7469976953125, 0.395174, 0.797475, 0.367757, 4352.737298222656, 0.404001, 0.800275, 0.362552, 4374.733207089844, 0.412913, 0.803041, 0.357269, 4396.729115957031, 0.421908, 0.805774, 0.35191, 4418.719416484375, 0.430983, 0.808473, 0.346476, 4440.715325351563, 0.440137, 0.811138, 0.340967, 4462.705625878906, 0.449368, 0.813768, 0.335384, 4484.701534746094, 0.458674, 0.816363, 0.329727, 4506.691835273437, 0.468053, 0.818921, 0.323998, 4528.687744140625, 0.477504, 0.821444, 0.318195, 4550.683653007813, 0.487026, 0.823929, 0.312321, 4572.673953535156, 0.496615, 0.826376, 0.306377, 4594.669862402344, 0.506271, 0.828786, 0.300362, 4616.660162929687, 0.515992, 0.831158, 0.294279, 4638.656071796875, 0.525776, 0.833491, 0.288127, 4660.646372324219, 0.535621, 0.835785, 0.281908, 4682.642281191406, 0.545524, 0.838039, 0.275626, 4704.638190058594, 0.555484, 0.840254, 0.269281, 4726.6284905859375, 0.565498, 0.84243, 0.262877, 4748.624399453125, 0.575563, 0.844566, 0.256415, 4770.614699980469, 0.585678, 0.846661, 0.249897, 4792.610608847656, 0.595839, 0.848717, 0.243329, 4814.600909375, 0.606045, 0.850733, 0.236712, 4836.5968182421875, 0.616293, 0.852709, 0.230052, 4858.592727109375, 0.626579, 0.854645, 0.223353, 4880.583027636719, 0.636902, 0.856542, 0.21662, 4902.578936503906, 0.647257, 0.8584, 0.209861, 4924.56923703125, 0.657642, 0.860219, 0.203082, 4946.565145898438, 0.668054, 0.861999, 0.196293, 4968.555446425781, 0.678489, 0.863742, 0.189503, 4990.551355292969, 0.688944, 0.865448, 0.182725, 5012.547264160156, 0.699415, 0.867117, 0.175971, 5034.5375646875, 0.709898, 0.868751, 0.169257, 5056.533473554688, 0.720391, 0.87035, 0.162603, 5078.523774082031, 0.730889, 0.871916, 0.156029, 5100.519682949219, 0.741388, 0.873449, 0.149561, 5122.509983476562, 0.751884, 0.874951, 0.143228, 5144.50589234375, 0.762373, 0.876424, 0.137064, 5166.496192871094, 0.772852, 0.877868, 0.131109, 5188.492101738281, 0.783315, 0.879285, 0.125405, 5210.488010605469, 0.79376, 0.880678, 0.120005, 5232.478311132812, 0.804182, 0.882046, 0.114965, 5254.47422, 0.814576, 0.883393, 0.110347, 5276.464520527344, 0.82494, 0.88472, 0.106217, 5298.460429394531, 0.83527, 0.886029, 0.102646, 5320.450729921875, 0.845561, 0.887322, 0.099702, 5342.4466387890625, 0.85581, 0.888601, 0.097452, 5364.44254765625, 0.866013, 0.889868, 0.095953, 5386.432848183594, 0.876168, 0.891125, 0.09525, 5408.428757050781, 0.886271, 0.892374, 0.095374, 5430.419057578125, 0.89632, 0.893616, 0.096335, 5452.4149664453125, 0.906311, 0.894855, 0.098125, 5474.405266972656, 0.916242, 0.896091, 0.100717, 5496.401175839844, 0.926106, 0.89733, 0.104071, 5518.397084707031, 0.935904, 0.89857, 0.108131, 5540.387385234375, 0.945636, 0.899815, 0.112838, 5562.383294101563, 0.9553, 0.901065, 0.118128, 5584.373594628906, 0.964894, 0.902323, 0.123941, 5606.369503496094, 0.974417, 0.90359, 0.130215, 5628.359804023437, 0.983868, 0.904867, 0.136897, 5650.355712890625, 0.993248, 0.906157, 0.143936]
+aLUT.NanColor = [1.0, 0.0, 0.0]
+aLUT.ScalarRangeInitialized = 1.0
+
+# trace defaults for the display properties.
+slice1Display.Representation = 'Surface'
+slice1Display.ColorArrayName = ['CELLS', 'A']
+slice1Display.LookupTable = aLUT
+slice1Display.SelectTCoordArray = 'None'
+slice1Display.SelectNormalArray = 'None'
+slice1Display.SelectTangentArray = 'None'
+slice1Display.OSPRayScaleFunction = 'PiecewiseFunction'
+slice1Display.SelectOrientationVectors = 'None'
+slice1Display.ScaleFactor = 1.75
+slice1Display.SelectScaleArray = 'None'
+slice1Display.GlyphType = 'Arrow'
+slice1Display.GlyphTableIndexArray = 'None'
+slice1Display.GaussianRadius = 0.08750000000000001
+slice1Display.SetScaleArray = [None, '']
+slice1Display.ScaleTransferFunction = 'PiecewiseFunction'
+slice1Display.OpacityArray = [None, '']
+slice1Display.OpacityTransferFunction = 'PiecewiseFunction'
+slice1Display.DataAxesGrid = 'GridAxesRepresentation'
+slice1Display.PolarAxes = 'PolarAxesRepresentation'
+
+# setup the color legend parameters for each legend in this view
+
+# get color legend/bar for aLUT in view renderView1
+#aLUTColorBar = GetScalarBar(aLUT, renderView1)
+#aLUTColorBar.Title = 'A'
+#aLUTColorBar.ComponentTitle = ''
+
+# set color bar visibility
+#aLUTColorBar.Visibility = 1
+
+# show color legend
+slice1Display.SetScalarBarVisibility(renderView1, False)
+
+# ----------------------------------------------------------------
+# setup color maps and opacity mapes used in the visualization
+# note: the Get..() functions create a new object, if needed
+# ----------------------------------------------------------------
+
+# get opacity transfer function/opacity map for 'A'
+aPWF = GetOpacityTransferFunction('A')
+aPWF.Points = [42.015869140625, 1.0, 0.5, 0.0, 5650.35546875, 1.0, 0.5, 0.0]
+aPWF.ScalarRangeInitialized = 1
+
+# ----------------------------------------------------------------
+# setup extractors
+# ----------------------------------------------------------------
+
+# create extractor
+pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
+# trace defaults for the extractor.
+# init the 'PNG' selected for 'Writer'
+pNG1.Writer.FileName = 'SliceA_%.6ts%cm.png'
+pNG1.Writer.ImageResolution = [800, 600]
+pNG1.Writer.OverrideColorPalette = 'BlackBackground'
+pNG1.Writer.Format = 'PNG'
+
+# ----------------------------------------------------------------
+# restore active source
+SetActiveSource(slice1)
+# ----------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# Catalyst options
+from paraview import catalyst
+options = catalyst.Options()
+options.ExtractsOutputDirectory = 'extracts'
+options.GlobalTrigger = 'TimeStep'
+options.CatalystLiveTrigger = 'TimeStep'
+
+# ------------------------------------------------------------------------------
+if __name__ == '__main__':
+    from paraview.simple import SaveExtractsUsingCatalystOptions
+    # Code for non in-situ environments; if executing in post-processing
+    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
+    SaveExtractsUsingCatalystOptions(options)

From d36b883a9861ca327c02248a170a70f97198b986 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 16 Feb 2022 15:41:49 +0000
Subject: [PATCH 31/63] Fix CatalystAdaptor bug with no diagnostic vars

This bug is not apparent in the case OPT=HIGH (so NDEBUG is defined) as
the Chombo asserts are skipped but appears otherwise.
---
 Source/Insitu/CatalystAdaptor.cpp | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index e94408d68..6df52ff8a 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -283,8 +283,11 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
         // to Catalyst
         GRLevelData &evolution_level_data = const_cast<GRLevelData &>(
             level->getLevelData(VariableType::evolution));
-        GRLevelData &diagnostic_level_data = const_cast<GRLevelData &>(
-            level->getLevelData(VariableType::diagnostic));
+        GRLevelData &diagnostic_level_data =
+            (NUM_DIAGNOSTIC_VARS > 0)
+                ? const_cast<GRLevelData &>(
+                      level->getLevelData(VariableType::diagnostic))
+                : evolution_level_data;
 
         const DisjointBoxLayout &level_box_layout =
             evolution_level_data.disjointBoxLayout();
@@ -302,7 +305,9 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
                 // hopefully this promotion works
                 DataIndex dind(lit());
                 FArrayBox &evolution_fab = evolution_level_data[dind];
-                FArrayBox &diagnostic_fab = diagnostic_level_data[dind];
+                FArrayBox &diagnostic_fab = (NUM_DIAGNOSTIC_VARS > 0)
+                                                ? diagnostic_level_data[dind]
+                                                : evolution_level_data[dind];
 
 #if DEBUG
                 vtkAMRBox vtk_box = m_vtk_grid_ptr->GetAMRBox(ilevel, ibox);

From 6b0344d33f1a0938182ed1a2b5a0a684389e0b57 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 16 Feb 2022 15:45:53 +0000
Subject: [PATCH 32/63] Fix various problems with CatalystInsituTest

* vtkPVVersion.h not present in all builds so replace with vtkPVConfig.h
* Need to add the ParaView::RemotingCore component for the above header
* Replace error from vtkImageDifference with thresholded error otherwise
the test is way too sensitive
---
 Tests/CatalystInsituTest/CatalystInsituTest.cpp    | 14 +++++++-------
 Tests/CatalystInsituTest/CatalystInsituTest.inputs |  6 +++---
 Tests/CatalystInsituTest/GNUmakefile               |  1 +
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.cpp b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
index 08250b455..5bdd13233 100644
--- a/Tests/CatalystInsituTest/CatalystInsituTest.cpp
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
@@ -37,7 +37,7 @@
 #include <vtkImageDifference.h>
 #include <vtkNew.h>
 #include <vtkPNGReader.h>
-#include <vtkPVVersion.h>
+#include <vtkPVConfig.h>
 #endif
 
 // Chombo namespace
@@ -48,9 +48,9 @@ int runInsituTest(int argc, char *argv[])
 {
     int status = 0;
     // We use a Catalyst v2.0 script that only works with ParaView 5.9 or higher
-    if (PARAVIEW_VERSION_CHECK(PARAVIEW_VERSION_MAJOR, PARAVIEW_VERSION_MINOR,
-                               PARAVIEW_VERSION_PATCH) <
-        PARAVIEW_VERSION_CHECK(5, 9, 0))
+    if (10000 * PARAVIEW_VERSION_MAJOR + 100 * PARAVIEW_VERSION_MINOR +
+            PARAVIEW_VERSION_PATCH <
+        50900)
         return -2;
 
     // Load the parameter file and construct the SimulationParameter class
@@ -75,8 +75,8 @@ int runInsituTest(int argc, char *argv[])
 
     MultiLevelTaskPtr<> call_task(task);
     call_task.execute(gr_amr);
-    // gr_amr.run(sim_params.stop_time, sim_params.max_steps);
-    // gr_amr.conclude();
+    gr_amr.run(sim_params.stop_time, sim_params.max_steps);
+    gr_amr.conclude();
 
     // read the newly generated PNG and the expected PNG
     vtkNew<vtkPNGReader> generated_png_reader;
@@ -90,7 +90,7 @@ int runInsituTest(int argc, char *argv[])
     image_difference->SetImageConnection(valid_png_reader->GetOutputPort());
     image_difference->Update();
 
-    double error = image_difference->GetError();
+    double error = image_difference->GetThresholdedError();
 
     pout() << "error           = " << error << "\n";
     pout() << "error_threshold = " << sim_params.error_threshold << std::endl;
diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.inputs b/Tests/CatalystInsituTest/CatalystInsituTest.inputs
index 5c3a41948..d483b197b 100644
--- a/Tests/CatalystInsituTest/CatalystInsituTest.inputs
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.inputs
@@ -9,8 +9,8 @@ stop_time = 0
 num_points = 30
 
 #num_ghosts = 3
-max_level = 1
-regrid_interval = 1 1
+max_level = 0
+regrid_interval = 1
 
 #boundaries and periodicity of grid
 #Periodic directions - 0 = false, 1 = true
@@ -32,7 +32,7 @@ catalyst_verbosity = 0
 num_catalyst_scripts = 1
 # list of Catalyst scripts
 catalyst_scripts = slice_A.py
-catalyst_coprocess_level = 1
+catalyst_coprocess_level = 0
 # num_catalyst_vars = 1
 # catalyst_vars = A
 abort_on_catalyst_error = true
diff --git a/Tests/CatalystInsituTest/GNUmakefile b/Tests/CatalystInsituTest/GNUmakefile
index 23ef85e54..8d10b1156 100644
--- a/Tests/CatalystInsituTest/GNUmakefile
+++ b/Tests/CatalystInsituTest/GNUmakefile
@@ -20,6 +20,7 @@ src_dirs := $(GRCHOMBO_SOURCE)/utils \
             $(GRCHOMBO_SOURCE)/AMRInterpolator
 
 XTRA_VTK_COMPONENT_FLAGS = -v ImagingCore -v ImagingSources -v IOImage
+XTRA_PARAVIEW_COMPONENT_FLAGS = -c RemotingCore
 
 include $(GRCHOMBO_SOURCE)/Insitu/Make.insitu
 

From 9bc6a0e89f2d263d57d4b0b7c627a27b961fc93c Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 16 Feb 2022 17:32:21 +0000
Subject: [PATCH 33/63] Make ParaView version checking more portable

Unfortunately it still won't work in some cases but I don't think
there's an easy way to do so.
---
 Tests/CatalystInsituTest/CatalystInsituTest.cpp | 11 +++++++++++
 Tests/CatalystInsituTest/GNUmakefile            |  1 -
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.cpp b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
index 5bdd13233..b4a3419ac 100644
--- a/Tests/CatalystInsituTest/CatalystInsituTest.cpp
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
@@ -37,7 +37,16 @@
 #include <vtkImageDifference.h>
 #include <vtkNew.h>
 #include <vtkPNGReader.h>
+// ParaView version checking doesn't seem to be very stable between versions
+#ifdef __has_include
+#if __has_include(<vtkPVVersion.h>)
+#include <vtkPVVersion.h>
+#define PARAVIEW_VERSION_KNOWN
+#elif __has_include(<vtkPVConfig.h>)
 #include <vtkPVConfig.h>
+#define PARAVIEW_VERSION_KNOWN
+#endif
+#endif
 #endif
 
 // Chombo namespace
@@ -47,11 +56,13 @@
 int runInsituTest(int argc, char *argv[])
 {
     int status = 0;
+#ifdef PARAVIEW_VERSION_KNOWN
     // We use a Catalyst v2.0 script that only works with ParaView 5.9 or higher
     if (10000 * PARAVIEW_VERSION_MAJOR + 100 * PARAVIEW_VERSION_MINOR +
             PARAVIEW_VERSION_PATCH <
         50900)
         return -2;
+#endif
 
     // Load the parameter file and construct the SimulationParameter class
     // To add more parameters edit the SimulationParameters file.
diff --git a/Tests/CatalystInsituTest/GNUmakefile b/Tests/CatalystInsituTest/GNUmakefile
index 8d10b1156..23ef85e54 100644
--- a/Tests/CatalystInsituTest/GNUmakefile
+++ b/Tests/CatalystInsituTest/GNUmakefile
@@ -20,7 +20,6 @@ src_dirs := $(GRCHOMBO_SOURCE)/utils \
             $(GRCHOMBO_SOURCE)/AMRInterpolator
 
 XTRA_VTK_COMPONENT_FLAGS = -v ImagingCore -v ImagingSources -v IOImage
-XTRA_PARAVIEW_COMPONENT_FLAGS = -c RemotingCore
 
 include $(GRCHOMBO_SOURCE)/Insitu/Make.insitu
 

From 59dd99a9290d985c0f1d68c12b1b15b6b0f63a49 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Fri, 18 Feb 2022 16:23:07 +0000
Subject: [PATCH 34/63] Make Catalyst aware of the output_path parameter

This means that any outputs for visualization pipelines will be created
under this path.
---
 Source/GRChomboCore/ChomboParameters.hpp        | 11 ++++++++++-
 Source/GRChomboCore/GRAMR.cpp                   |  3 ++-
 Source/GRChomboCore/GRAMR.hpp                   |  1 +
 Source/GRChomboCore/SetupFunctions.hpp          |  3 ++-
 Source/Insitu/CatalystAdaptor.cpp               | 17 ++++++++++++++---
 Source/Insitu/CatalystAdaptor.hpp               |  2 ++
 Source/Insitu/README.md                         | 13 ++++++++++---
 .../CatalystInsituTest.inputs                   |  2 ++
 .../CatalystInsituTest/SimulationParameters.hpp |  4 ++++
 9 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index 5da1ae349..e636c9b9b 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -146,8 +146,17 @@ class ChomboParameters
             if (!catalyst_scripts_path.empty() &&
                 catalyst_scripts_path.back() != '/')
                 catalyst_scripts_path += "/";
+            // make each catalyst_script a full path so that Catalyst doesn't
+            // look in its working directory (set to output_path)
+            char cwd[4096];
+            getcwd(cwd, 4096);
+            std::string current_working_directory = cwd;
+            current_working_directory += "/";
             for (auto &script : catalyst_scripts)
-                script = catalyst_scripts_path + script;
+            {
+                script =
+                    current_working_directory + catalyst_scripts_path + script;
+            }
             pp.load("catalyst_coprocess_level", catalyst_coprocess_level, 0);
             UserVariables::load_vars_to_vector(
                 pp, "catalyst_vars", "num_catalyst_vars", catalyst_vars,
diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp
index 3b863e46e..d5ee668e1 100644
--- a/Source/GRChomboCore/GRAMR.cpp
+++ b/Source/GRChomboCore/GRAMR.cpp
@@ -66,6 +66,7 @@ void GRAMR::fill_multilevel_ghosts(const VariableType a_var_type,
 #ifdef USE_CATALYST
 void GRAMR::setup_catalyst(
     bool a_activate_catalyst, const std::vector<std::string> &a_python_scripts,
+    const std::string &a_output_path,
     const std::vector<std::pair<int, VariableType>> &a_vars,
     bool a_abort_on_catalyst_error, int a_verbosity)
 {
@@ -74,7 +75,7 @@ void GRAMR::setup_catalyst(
     {
         pout() << "GRAMR::setup_catalyst" << std::endl;
         m_insitu = new CatalystAdaptor;
-        m_insitu->initialise(this, a_python_scripts, a_vars,
+        m_insitu->initialise(this, a_python_scripts, a_output_path, a_vars,
                              a_abort_on_catalyst_error, a_verbosity);
     }
 }
diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp
index e0dbee7c2..ac020951f 100644
--- a/Source/GRChomboCore/GRAMR.hpp
+++ b/Source/GRChomboCore/GRAMR.hpp
@@ -85,6 +85,7 @@ class GRAMR : public AMR
 #ifdef USE_CATALYST
     void setup_catalyst(bool a_activate_catalyst,
                         const std::vector<std::string> &a_python_scripts,
+                        const std::string &a_output_path,
                         const std::vector<std::pair<int, VariableType>> &a_vars,
                         bool a_abort_on_catalyst_error, int a_verbosity);
 #endif
diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index c113cb62a..1ac4b0505 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -212,7 +212,8 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
     vtkLogger::Init();
     gr_amr.setup_catalyst(
         chombo_params.activate_catalyst, chombo_params.catalyst_scripts,
-        chombo_params.catalyst_vars, chombo_params.abort_on_catalyst_error,
+        chombo_params.output_path, chombo_params.catalyst_vars,
+        chombo_params.abort_on_catalyst_error,
         chombo_params.catalyst_verbosity);
 #endif
 }
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 6df52ff8a..108c5f15d 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -12,10 +12,11 @@ CatalystAdaptor::CatalystAdaptor() {}
 
 CatalystAdaptor::CatalystAdaptor(
     GRAMR *a_gr_amr_ptr, const std::vector<std::string> &a_python_scripts,
+    const std::string &a_output_path,
     const std::vector<std::pair<int, VariableType>> &a_vars,
     bool a_abort_on_catalyst_error, int a_verbosity)
 {
-    initialise(a_gr_amr_ptr, a_python_scripts, a_vars,
+    initialise(a_gr_amr_ptr, a_python_scripts, a_output_path, a_vars,
                a_abort_on_catalyst_error, a_verbosity);
 }
 
@@ -29,6 +30,7 @@ CatalystAdaptor::~CatalystAdaptor()
 
 void CatalystAdaptor::initialise(
     GRAMR *a_gr_amr_ptr, const std::vector<std::string> &a_python_scripts,
+    const std::string &a_output_path,
     const std::vector<std::pair<int, VariableType>> &a_vars,
     bool a_abort_on_catalyst_error, int a_verbosity)
 {
@@ -52,10 +54,19 @@ void CatalystAdaptor::initialise(
     if (!m_proc_ptr)
     {
         m_proc_ptr = vtkCPProcessor::New();
-        int m_proc_ptr_intialization_success = m_proc_ptr->Initialize();
+        int m_proc_ptr_initialization_success;
+        if (a_output_path.empty())
+        {
+            m_proc_ptr_initialization_success = m_proc_ptr->Initialize();
+        }
+        else
+        {
+            m_proc_ptr_initialization_success =
+                m_proc_ptr->Initialize(a_output_path.c_str());
+        }
         std::string error_msg = "Failed to initialize vtkCPProcessor in "
                                 "CatalystAdaptor::initialise";
-        catalyst_error_or_warning(m_proc_ptr_intialization_success, error_msg);
+        catalyst_error_or_warning(m_proc_ptr_initialization_success, error_msg);
     }
     else
     {
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index d42ef6a0a..4cf14d206 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -47,6 +47,7 @@ class CatalystAdaptor
     // full constructor (calls initialise)
     CatalystAdaptor(GRAMR *a_gr_amr_ptr,
                     const std::vector<std::string> &a_python_scripts,
+                    const std::string &a_output_path,
                     const std::vector<std::pair<int, VariableType>> &a_vars,
                     bool a_abort_on_catalyst_error, int a_verbosity);
 
@@ -56,6 +57,7 @@ class CatalystAdaptor
     // Initialisation/Finalisation
     void initialise(GRAMR *m_gr_amr_ptr,
                     const std::vector<std::string> &a_python_scripts,
+                    const std::string &a_output_path,
                     const std::vector<std::pair<int, VariableType>> &a_vars,
                     bool a_abort_on_catalyst_error, int a_verbosity);
     void finalise();
diff --git a/Source/Insitu/README.md b/Source/Insitu/README.md
index 067c9c1fe..391247585 100644
--- a/Source/Insitu/README.md
+++ b/Source/Insitu/README.md
@@ -183,11 +183,14 @@ visualization processing which are described below:
  * `catalyst_pout_prefix`: This is the filename prefix for the `catalyst_pout`
  files. They are written to `pout_subpath` and appended by '`.<rank number>`' 
  as for the normal `pout` files. Note that you can set this to the same
- string as 'pout_prefix` to send this output to the same
+ string as `pout_prefix` to send this output to the same
  * `catalyst_scripts_path`: This is the path that contains the Catalyst scripts
- generated as described [above](#generating-paraview-catalyst-scripts).
+ generated as described [above](#generating-paraview-catalyst-scripts). Note
+ that this path is relative to the current working directory and _not_ 
+ `output_path`.
  * `num_catalyst_scripts`: Number of Catalyst scripts to process.
- * `catalyst_scripts`: Filenames of Catalyst scripts
+ * `catalyst_scripts`: Filenames of Catalyst scripts in the 
+ `catalyst_scripts_path` directory.
  * `catalyst_coprocess_level`: This is the level for which the Catalyst
  coprocess routine is called at the end of each timestep.
  * `abort_on_catalyst_error = true/false`: If set to `true`, the code will abort
@@ -201,3 +204,7 @@ only pass specific variables to Catalyst, it is possible to restrict the
 variables passed using the `num_catalyst_vars` and `catalyst_vars` parameters.
 If these are not set, and a script generated by ParaView v5.9 or later is used,
 all variables will be passed to Catalyst.
+
+If the `output_path` parameter is set, then this will be passed to Catalyst as 
+the working directory and the subdirectory for any outputs created by
+visualization pipelines will exist under this path.
\ No newline at end of file
diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.inputs b/Tests/CatalystInsituTest/CatalystInsituTest.inputs
index d483b197b..8f68d2f6d 100644
--- a/Tests/CatalystInsituTest/CatalystInsituTest.inputs
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.inputs
@@ -6,6 +6,8 @@ chk_prefix = TestChk_
 plot_prefix = TestPlt_
 stop_time = 0
 
+output_path = some_path
+
 num_points = 30
 
 #num_ghosts = 3
diff --git a/Tests/CatalystInsituTest/SimulationParameters.hpp b/Tests/CatalystInsituTest/SimulationParameters.hpp
index 4d9ef293f..ed82a5bfe 100644
--- a/Tests/CatalystInsituTest/SimulationParameters.hpp
+++ b/Tests/CatalystInsituTest/SimulationParameters.hpp
@@ -18,6 +18,10 @@ class SimulationParameters : public ChomboParameters
         pp.load("error_threshold", error_threshold, 0.15);
         pp.load("valid_png_file", valid_png_file);
         pp.load("generated_png_file", generated_png_file);
+        if (!output_path.empty())
+        {
+            generated_png_file = output_path + generated_png_file;
+        }
     }
 
     double error_threshold;

From 1b9c294621cf2f3d8e73d661a15ba08184cb358e Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Fri, 11 Mar 2022 17:20:10 +0000
Subject: [PATCH 35/63] Add catalyst_remove_ghosts parameter

This allows copying Chombo arrays to Catalyst without ghost cells (for
debugging purposes).
---
 Source/GRChomboCore/ChomboParameters.hpp |  2 +
 Source/GRChomboCore/GRAMR.cpp            |  5 +-
 Source/GRChomboCore/GRAMR.hpp            |  3 +-
 Source/GRChomboCore/SetupFunctions.hpp   |  2 +-
 Source/Insitu/CatalystAdaptor.cpp        | 78 ++++++++++++++++++++----
 Source/Insitu/CatalystAdaptor.hpp        | 11 +++-
 Source/Insitu/README.md                  |  3 +
 7 files changed, 86 insertions(+), 18 deletions(-)

diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index e636c9b9b..00a3dfd0d 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -162,6 +162,7 @@ class ChomboParameters
                 pp, "catalyst_vars", "num_catalyst_vars", catalyst_vars,
                 num_catalyst_vars);
             pp.load("abort_on_catalyst_error", abort_on_catalyst_error, false);
+            pp.load("catalyst_remove_ghosts", catalyst_remove_ghosts, false);
         }
 #endif
     }
@@ -604,6 +605,7 @@ class ChomboParameters
     int num_catalyst_vars;
     std::vector<std::pair<int, VariableType>> catalyst_vars;
     bool abort_on_catalyst_error;
+    bool catalyst_remove_ghosts;
 #endif
 
   protected:
diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp
index d5ee668e1..da985894b 100644
--- a/Source/GRChomboCore/GRAMR.cpp
+++ b/Source/GRChomboCore/GRAMR.cpp
@@ -68,7 +68,7 @@ void GRAMR::setup_catalyst(
     bool a_activate_catalyst, const std::vector<std::string> &a_python_scripts,
     const std::string &a_output_path,
     const std::vector<std::pair<int, VariableType>> &a_vars,
-    bool a_abort_on_catalyst_error, int a_verbosity)
+    bool a_abort_on_catalyst_error, bool a_remove_ghosts, int a_verbosity)
 {
     m_activate_catalyst = a_activate_catalyst;
     if (m_activate_catalyst)
@@ -76,7 +76,8 @@ void GRAMR::setup_catalyst(
         pout() << "GRAMR::setup_catalyst" << std::endl;
         m_insitu = new CatalystAdaptor;
         m_insitu->initialise(this, a_python_scripts, a_output_path, a_vars,
-                             a_abort_on_catalyst_error, a_verbosity);
+                             a_abort_on_catalyst_error, a_remove_ghosts,
+                             a_verbosity);
     }
 }
 #endif
\ No newline at end of file
diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp
index ac020951f..b48007b5c 100644
--- a/Source/GRChomboCore/GRAMR.hpp
+++ b/Source/GRChomboCore/GRAMR.hpp
@@ -87,7 +87,8 @@ class GRAMR : public AMR
                         const std::vector<std::string> &a_python_scripts,
                         const std::string &a_output_path,
                         const std::vector<std::pair<int, VariableType>> &a_vars,
-                        bool a_abort_on_catalyst_error, int a_verbosity);
+                        bool a_abort_on_catalyst_error, bool a_remove_ghosts,
+                        int a_verbosity);
 #endif
 
 #ifdef USE_CATALYST
diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 1ac4b0505..7a2566ec3 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -214,7 +214,7 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
         chombo_params.activate_catalyst, chombo_params.catalyst_scripts,
         chombo_params.output_path, chombo_params.catalyst_vars,
         chombo_params.abort_on_catalyst_error,
-        chombo_params.catalyst_verbosity);
+        chombo_params.catalyst_remove_ghosts, chombo_params.catalyst_verbosity);
 #endif
 }
 
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 108c5f15d..d35f36e1d 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -14,10 +14,10 @@ CatalystAdaptor::CatalystAdaptor(
     GRAMR *a_gr_amr_ptr, const std::vector<std::string> &a_python_scripts,
     const std::string &a_output_path,
     const std::vector<std::pair<int, VariableType>> &a_vars,
-    bool a_abort_on_catalyst_error, int a_verbosity)
+    bool a_abort_on_catalyst_error, bool a_remove_ghosts, int a_verbosity)
 {
     initialise(a_gr_amr_ptr, a_python_scripts, a_output_path, a_vars,
-               a_abort_on_catalyst_error, a_verbosity);
+               a_abort_on_catalyst_error, a_remove_ghosts, a_verbosity);
 }
 
 CatalystAdaptor::~CatalystAdaptor()
@@ -32,7 +32,7 @@ void CatalystAdaptor::initialise(
     GRAMR *a_gr_amr_ptr, const std::vector<std::string> &a_python_scripts,
     const std::string &a_output_path,
     const std::vector<std::pair<int, VariableType>> &a_vars,
-    bool a_abort_on_catalyst_error, int a_verbosity)
+    bool a_abort_on_catalyst_error, bool a_remove_ghosts, int a_verbosity)
 {
     // don't initalise twice
     if (m_initialised)
@@ -49,6 +49,7 @@ void CatalystAdaptor::initialise(
     m_gr_amr_ptr = a_gr_amr_ptr;
     m_vars = a_vars;
     m_abort_on_catalyst_error = a_abort_on_catalyst_error;
+    m_remove_ghosts = a_remove_ghosts;
 
     // Initialise VTK CP Processor
     if (!m_proc_ptr)
@@ -148,7 +149,7 @@ void CatalystAdaptor::build_vtk_grid()
         gramrlevels[0]->getLevelData().ghostVect();
     const double coarsest_dx = gramrlevels[0]->get_dx();
     RealVect ghosted_origin_global_vect = coarsest_ghost_vect;
-    ghosted_origin_global_vect *= -coarsest_dx;
+    ghosted_origin_global_vect *= (!m_remove_ghosts) ? -coarsest_dx : 0;
     m_vtk_grid_ptr->SetOrigin(ghosted_origin_global_vect.dataPtr());
 
     // now add all the boxes
@@ -174,8 +175,12 @@ void CatalystAdaptor::build_vtk_grid()
             const IntVect &big_end = box.bigEnd();
 
             // now grow the box to make the ghosted box
+            // if remove_ghosts == false
             Box ghosted_box = box;
-            ghosted_box.grow(level_data.ghostVect());
+            if (!m_remove_ghosts)
+            {
+                ghosted_box.grow(level_data.ghostVect());
+            }
             const IntVect &small_ghosted_end = ghosted_box.smallEnd();
             const IntVect &big_ghosted_end = ghosted_box.bigEnd();
 
@@ -319,6 +324,7 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
                 FArrayBox &diagnostic_fab = (NUM_DIAGNOSTIC_VARS > 0)
                                                 ? diagnostic_level_data[dind]
                                                 : evolution_level_data[dind];
+                const Box &unghosted_box = level_box_layout[dind];
 
 #if DEBUG
                 vtkAMRBox vtk_box = m_vtk_grid_ptr->GetAMRBox(ilevel, ibox);
@@ -366,9 +372,19 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
                 {
                     if (requested_evolution_vars[ivar])
                     {
-                        vtkDoubleArray *vtk_double_arr = fab_to_vtk_array(
-                            evolution_fab, ivar,
-                            UserVariables::variable_names[ivar]);
+                        vtkDoubleArray *vtk_double_arr;
+                        if (!m_remove_ghosts)
+                        {
+                            vtk_double_arr = fab_to_vtk_array(
+                                evolution_fab, ivar,
+                                UserVariables::variable_names[ivar]);
+                        }
+                        else
+                        {
+                            vtk_double_arr = fab_to_vtk_array_without_ghosts(
+                                evolution_fab, unghosted_box, ivar,
+                                UserVariables::variable_names[ivar]);
+                        }
                         vtk_uniform_grid_ptr->GetCellData()->AddArray(
                             vtk_double_arr);
                     }
@@ -377,9 +393,19 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
                 {
                     if (requested_diagnostic_vars[ivar])
                     {
-                        vtkDoubleArray *vtk_double_arr = fab_to_vtk_array(
-                            diagnostic_fab, ivar,
-                            DiagnosticVariables::variable_names[ivar]);
+                        vtkDoubleArray *vtk_double_arr;
+                        if (!m_remove_ghosts)
+                        {
+                            vtk_double_arr = fab_to_vtk_array(
+                                diagnostic_fab, ivar,
+                                DiagnosticVariables::variable_names[ivar]);
+                        }
+                        else
+                        {
+                            vtk_double_arr = fab_to_vtk_array_without_ghosts(
+                                diagnostic_fab, unghosted_box, ivar,
+                                DiagnosticVariables::variable_names[ivar]);
+                        }
                         vtk_uniform_grid_ptr->GetCellData()->AddArray(
                             vtk_double_arr);
                     }
@@ -418,7 +444,6 @@ vtkDoubleArray *CatalystAdaptor::fab_to_vtk_array(FArrayBox &a_fab, int a_var,
 {
     vtkDoubleArray *out = vtkDoubleArray::New();
     vtkIdType num_cells = a_fab.size().product();
-    out->SetNumberOfTuples(num_cells);
     out->SetName(a_name.c_str());
     // this prevents Catalyst from deallocating the Chombo
     // data pointers
@@ -427,6 +452,35 @@ vtkDoubleArray *CatalystAdaptor::fab_to_vtk_array(FArrayBox &a_fab, int a_var,
     return out;
 }
 
+vtkDoubleArray *CatalystAdaptor::fab_to_vtk_array_without_ghosts(
+    FArrayBox &a_fab, const Box &a_unghosted_box, int a_var,
+    const std::string &a_name)
+{
+    vtkDoubleArray *out = vtkDoubleArray::New();
+    vtkIdType num_cells = a_unghosted_box.volume();
+    // the following allocates memory in the VTK Array
+    out->SetNumberOfTuples(num_cells);
+    out->SetName(a_name.c_str());
+
+    const IntVect &small_end = a_unghosted_box.smallEnd();
+    const IntVect &big_end = a_unghosted_box.bigEnd();
+
+    vtkIdType vtk_idx = 0;
+    for (int iz = small_end[2]; iz <= big_end[2]; ++iz)
+    {
+        for (int iy = small_end[1]; iy <= big_end[1]; ++iy)
+        {
+            for (int ix = small_end[0]; ix <= big_end[0]; ++ix)
+            {
+                out->SetValue(vtk_idx, a_fab(IntVect(ix, iy, iz), a_var));
+                ++vtk_idx;
+            }
+        }
+    }
+
+    return out;
+}
+
 void CatalystAdaptor::catalyst_error_or_warning(bool a_success,
                                                 std::string a_msg)
 {
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index 4cf14d206..7021abcc8 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -49,7 +49,8 @@ class CatalystAdaptor
                     const std::vector<std::string> &a_python_scripts,
                     const std::string &a_output_path,
                     const std::vector<std::pair<int, VariableType>> &a_vars,
-                    bool a_abort_on_catalyst_error, int a_verbosity);
+                    bool a_abort_on_catalyst_error, bool a_remove_ghosts,
+                    int a_verbosity);
 
     // destructor
     ~CatalystAdaptor();
@@ -59,7 +60,8 @@ class CatalystAdaptor
                     const std::vector<std::string> &a_python_scripts,
                     const std::string &a_output_path,
                     const std::vector<std::pair<int, VariableType>> &a_vars,
-                    bool a_abort_on_catalyst_error, int a_verbosity);
+                    bool a_abort_on_catalyst_error, bool a_remove_ghosts,
+                    int a_verbosity);
     void finalise();
 
     // do Catalyst processing
@@ -75,6 +77,10 @@ class CatalystAdaptor
     vtkDoubleArray *fab_to_vtk_array(FArrayBox &a_fab, int a_var,
                                      const std::string &a_name);
 
+    vtkDoubleArray *fab_to_vtk_array_without_ghosts(FArrayBox &a_fab,
+                                                    const Box &a_box, int a_var,
+                                                    const std::string &a_name);
+
     // if a_success = false, either aborts or prints a warning depending on
     // m_abort_on_catalyst_error
     void catalyst_error_or_warning(bool a_success, std::string a_msg);
@@ -82,6 +88,7 @@ class CatalystAdaptor
     int m_verbosity;
     bool m_initialised = false;
     bool m_abort_on_catalyst_error = false;
+    bool m_remove_ghosts = false;
     GRAMR *m_gr_amr_ptr = nullptr;
     // variables to pass to Catalyst
     std::vector<std::pair<int, VariableType>> m_vars;
diff --git a/Source/Insitu/README.md b/Source/Insitu/README.md
index 391247585..5109f1522 100644
--- a/Source/Insitu/README.md
+++ b/Source/Insitu/README.md
@@ -193,6 +193,9 @@ visualization processing which are described below:
  `catalyst_scripts_path` directory.
  * `catalyst_coprocess_level`: This is the level for which the Catalyst
  coprocess routine is called at the end of each timestep.
+ * `catalyst_remove_ghosts = true/false`: If set to `true`, Chombo data will be
+ copied to new VTK arrays without ghosts rather than passing the Chombo data 
+ pointers directly (for debugging).
  * `abort_on_catalyst_error = true/false`: If set to `true`, the code will abort
  if there is an error in Catalyst. The default is `false`.
  

From 518c653738d914fbe6685801c065178e7e9f9474 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Mon, 14 Mar 2022 14:45:31 +0000
Subject: [PATCH 36/63] Fix dimensions of vtkAMRBox

These should be the same as that of the corresponding vtkUniformGrids.
Thanks to Dave Demarle (@demarle) for finding this.
---
 Source/Insitu/CatalystAdaptor.cpp | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index d35f36e1d..f6e988471 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -171,12 +171,18 @@ void CatalystAdaptor::build_vtk_grid()
         {
             // first get the box without ghosts
             Box box = level_box_layout[lit];
-            const IntVect &small_end = box.smallEnd();
-            const IntVect &big_end = box.bigEnd();
+            // const IntVect &small_end = box.smallEnd();
+            // const IntVect &big_end = box.bigEnd();
 
-            // now grow the box to make the ghosted box
-            // if remove_ghosts == false
+            // now grow the box to make the ghosted box (no change if
+            // m_remove_ghosts == true)
             Box ghosted_box = box;
+            // VTK counts the big end differently to Chombo so modify the Chombo
+            // box so VTK gets what it needs
+            for (int idir = 0; idir < SpaceDim; ++idir)
+            {
+                ghosted_box.growHi(idir, 1);
+            }
             if (!m_remove_ghosts)
             {
                 ghosted_box.grow(level_data.ghostVect());
@@ -206,9 +212,9 @@ void CatalystAdaptor::build_vtk_grid()
                 vtk_uniform_grid_ptr->SetOrigin(origin_global);
                 vtk_uniform_grid_ptr->SetSpacing(dx_arr);
                 vtk_uniform_grid_ptr->SetExtent(
-                    small_ghosted_end[0], big_ghosted_end[0] + 1,
-                    small_ghosted_end[1], big_ghosted_end[1] + 1,
-                    small_ghosted_end[2], big_ghosted_end[2] + 1);
+                    small_ghosted_end[0], big_ghosted_end[0],
+                    small_ghosted_end[1], big_ghosted_end[1],
+                    small_ghosted_end[2], big_ghosted_end[2]);
                 // add the ghost cell information
                 // int no_ghost[6] = {small_end[0], big_end[0] + 1,
                 //    small_end[1], big_end[1] + 1,

From 02d5a3c519a8592924ed4090430d4d23620cd438 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Thu, 17 Mar 2022 16:46:49 +0000
Subject: [PATCH 37/63] Add support for ParaView v5.8

Switch to using a Catalyst v1.0 script for the test so that it will work
with ParaView v5.8. Also add a header file to determine the ParaView
version (not currently used but possibly useful in future).
---
 Source/GRChomboCore/ChomboParameters.hpp      |   4 -
 Source/GRChomboCore/SetupFunctions.hpp        |   3 +
 Source/Insitu/CatalystAdaptor.cpp             |  30 +--
 Source/Insitu/ParaviewVersion.hpp             |  29 +++
 Source/Insitu/README.md                       |   1 +
 .../CatalystInsituTest/CatalystInsituTest.cpp |  17 --
 .../CatalystInsituTest.inputs                 |   6 +-
 Tests/CatalystInsituTest/slice_A_v1.py        | 220 ++++++++++++++++++
 .../{slice_A.py => slice_A_v2.py}             |   0
 9 files changed, 267 insertions(+), 43 deletions(-)
 create mode 100644 Source/Insitu/ParaviewVersion.hpp
 create mode 100644 Tests/CatalystInsituTest/slice_A_v1.py
 rename Tests/CatalystInsituTest/{slice_A.py => slice_A_v2.py} (100%)

diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index 00a3dfd0d..464eee2ee 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -515,10 +515,6 @@ class ChomboParameters
                     "catalyst_scripts[" + std::to_string(iscript) + "]";
                 check_parameter(script_parameter_name, script, script_exists,
                                 "does not exist");
-                bool script_valid = (vtkCPPythonPipeline::DetectScriptVersion(
-                                         script.c_str()) != 0);
-                check_parameter(script_parameter_name, script, script_valid,
-                                "not a valid ParaView Catalyst script");
             }
 
             check_parameter("catalyst_coprocess_level",
diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 7a2566ec3..31fd034d9 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -32,6 +32,7 @@ using std::endl;
 
 #ifdef USE_CATALYST
 #include "vtkLogger.h"
+#include "vtkVersion.h"
 #endif
 
 #ifdef _OPENMP
@@ -187,8 +188,10 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
 #endif
     }
 #ifdef USE_CATALYST
+#if VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 0, 20201030)
     // set vtkLogger internal messages to only appear at verbosity 2
     vtkLogger::SetInternalVerbosityLevel(vtkLogger::VERBOSITY_3);
+#endif
 #ifdef CH_MPI
     vtkLogger::SetStderrVerbosity(vtkLogger::VERBOSITY_ERROR);
     std::string catalyst_log_file = chombo_params.pout_path +
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index f6e988471..3677e6be5 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -77,24 +77,18 @@ void CatalystAdaptor::initialise(
     // Create Python script pipeline and add it to the VTK CP Processor
     for (const std::string &script : a_python_scripts)
     {
-        if (auto pipeline =
-                vtkCPPythonScriptPipeline::CreateAndInitializePipeline(
-                    script.c_str()))
-        {
-            int add_pipeline_success = m_proc_ptr->AddPipeline(pipeline);
-            std::string add_pipeline_fail_msg =
-                "Failed to add pipeline for script: ";
-            add_pipeline_fail_msg += script;
-            catalyst_error_or_warning(add_pipeline_success,
-                                      add_pipeline_fail_msg);
-        }
-        else
-        {
-            std::string pipeline_fail_msg =
-                "Failed to set up pipeline for script: ";
-            pipeline_fail_msg += script;
-            catalyst_error_or_warning(false, pipeline_fail_msg);
-        }
+        vtkNew<vtkCPPythonScriptPipeline> pipeline;
+        int pipeline_init_success = pipeline->Initialize(script.c_str());
+        std::string pipeline_init_fail_msg =
+            "Failed to initialize pipelone for script: ";
+        pipeline_init_fail_msg += script;
+        catalyst_error_or_warning(pipeline_init_success,
+                                  pipeline_init_fail_msg);
+        int add_pipeline_success = m_proc_ptr->AddPipeline(pipeline);
+        std::string add_pipeline_fail_msg =
+            "Failed to add pipeline for script: ";
+        add_pipeline_fail_msg += script;
+        catalyst_error_or_warning(add_pipeline_success, add_pipeline_fail_msg);
     }
 
     m_verbosity = a_verbosity;
diff --git a/Source/Insitu/ParaviewVersion.hpp b/Source/Insitu/ParaviewVersion.hpp
new file mode 100644
index 000000000..904d910d5
--- /dev/null
+++ b/Source/Insitu/ParaviewVersion.hpp
@@ -0,0 +1,29 @@
+#ifndef PARAVIEW_VERSION_HPP_
+#define PARAVIEW_VERSION_HPP_
+
+#ifdef USE_CATALYST
+// ParaView version checking doesn't seem to be very stable between versions
+#ifdef __has_include
+#if __has_include(<vtkPVVersion.h>)
+#include <vtkPVVersion.h>
+#define PARAVIEW_VERSION_KNOWN
+#elif __has_include(<vtkPVConfig.h>)
+#include <vtkPVConfig.h>
+#define PARAVIEW_VERSION_KNOWN
+#endif /* __has_include(<vtk....h>) */
+#endif /* __has_include */
+
+#ifdef PARAVIEW_VERSION_KNOWN
+// there is something similar in the new vtkPVVersion.h header but it uses
+// the build rather than the patch number. It's also not available in
+// vtkPVConfig.h
+#define PARAVIEW_VERSION_TEST(major, minor, patch)                             \
+    10000 * major + 100 * minor + patch
+#define PARAVIEW_VERSION_HERE                                                  \
+    PARAVIEW_VERSION_TEST(PARAVIEW_VERSION_MAJOR, PARAVIEW_VERSION_MINOR,      \
+                          PARAVIEW_VERSION_PATCH)
+#endif /* PARAVIEW_VERSION_KNOWN */
+
+#endif /* USE_CATALYST */
+
+#endif /* PARAVIEW_VERSION_HPP_ */
\ No newline at end of file
diff --git a/Source/Insitu/README.md b/Source/Insitu/README.md
index 5109f1522..89e49bd82 100644
--- a/Source/Insitu/README.md
+++ b/Source/Insitu/README.md
@@ -58,6 +58,7 @@ This in-situ adaptor has been tested with the following versions of ParaView:
 
 | Tested ParaView versions |
 | --- |
+| 5.8.1 |
 | 5.9.1 |
 | 5.10.0 |
 
diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.cpp b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
index b4a3419ac..8fdf96bd6 100644
--- a/Tests/CatalystInsituTest/CatalystInsituTest.cpp
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
@@ -37,16 +37,6 @@
 #include <vtkImageDifference.h>
 #include <vtkNew.h>
 #include <vtkPNGReader.h>
-// ParaView version checking doesn't seem to be very stable between versions
-#ifdef __has_include
-#if __has_include(<vtkPVVersion.h>)
-#include <vtkPVVersion.h>
-#define PARAVIEW_VERSION_KNOWN
-#elif __has_include(<vtkPVConfig.h>)
-#include <vtkPVConfig.h>
-#define PARAVIEW_VERSION_KNOWN
-#endif
-#endif
 #endif
 
 // Chombo namespace
@@ -56,13 +46,6 @@
 int runInsituTest(int argc, char *argv[])
 {
     int status = 0;
-#ifdef PARAVIEW_VERSION_KNOWN
-    // We use a Catalyst v2.0 script that only works with ParaView 5.9 or higher
-    if (10000 * PARAVIEW_VERSION_MAJOR + 100 * PARAVIEW_VERSION_MINOR +
-            PARAVIEW_VERSION_PATCH <
-        50900)
-        return -2;
-#endif
 
     // Load the parameter file and construct the SimulationParameter class
     // To add more parameters edit the SimulationParameters file.
diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.inputs b/Tests/CatalystInsituTest/CatalystInsituTest.inputs
index 8f68d2f6d..317f4d6f4 100644
--- a/Tests/CatalystInsituTest/CatalystInsituTest.inputs
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.inputs
@@ -6,8 +6,6 @@ chk_prefix = TestChk_
 plot_prefix = TestPlt_
 stop_time = 0
 
-output_path = some_path
-
 num_points = 30
 
 #num_ghosts = 3
@@ -27,13 +25,13 @@ lo_boundary = 0 0 0
 # ParaView Catalyst parameters
 
 activate_catalyst = true
-catalyst_verbosity = 0
+catalyst_verbosity = 10
 # catalyst_pout_prefix = catalyst_pout
 # path to directory containing catalyst scripts
 # catalyst_scripts_path = catalyst_scripts
 num_catalyst_scripts = 1
 # list of Catalyst scripts
-catalyst_scripts = slice_A.py
+catalyst_scripts = slice_A_v1.py # slice_A_v2.py
 catalyst_coprocess_level = 0
 # num_catalyst_vars = 1
 # catalyst_vars = A
diff --git a/Tests/CatalystInsituTest/slice_A_v1.py b/Tests/CatalystInsituTest/slice_A_v1.py
new file mode 100644
index 000000000..e9d970bba
--- /dev/null
+++ b/Tests/CatalystInsituTest/slice_A_v1.py
@@ -0,0 +1,220 @@
+
+#--------------------------------------------------------------
+
+# Global timestep output options
+timeStepToStartOutputAt=0
+forceOutputAtFirstCall=False
+
+# Global screenshot output options
+imageFileNamePadding=6
+rescale_lookuptable=False
+
+# Whether or not to request specific arrays from the adaptor.
+requestSpecificArrays=True
+
+# a root directory under which all Catalyst output goes
+rootDirectory='extracts'
+
+# makes a cinema D index table
+make_cinema_table=False
+
+#--------------------------------------------------------------
+# Code generated from cpstate.py to create the CoProcessor.
+# paraview version 5.8.1
+#--------------------------------------------------------------
+
+from paraview.simple import *
+from paraview import coprocessing
+
+# ----------------------- CoProcessor definition -----------------------
+
+def CreateCoProcessor():
+  def _CreatePipeline(coprocessor, datadescription):
+    class Pipeline:
+      # state file generated using paraview version 5.8.1
+
+      # ----------------------------------------------------------------
+      # setup views used in the visualization
+      # ----------------------------------------------------------------
+
+      # trace generated using paraview version 5.8.1
+      #
+      # To ensure correct image size when batch processing, please search 
+      # for and uncomment the line `# renderView*.ViewSize = [*,*]`
+
+      #### disable automatic camera reset on 'Show'
+      paraview.simple._DisableFirstRenderCameraReset()
+
+      # get the material library
+      materialLibrary1 = GetMaterialLibrary()
+
+      # Black background
+      LoadPalette('BlackBackground')
+
+      # Create a new 'Render View'
+      renderView1 = CreateView('RenderView')
+      renderView1.ViewSize = [1092, 778]
+      renderView1.AxesGrid = 'GridAxes3DActor'
+      renderView1.OrientationAxesVisibility = 0
+      renderView1.CenterOfRotation = [8.0, 8.0, 8.0]
+      renderView1.StereoType = 'Crystal Eyes'
+      renderView1.CameraPosition = [-31.513131514425908, 8.0, 8.0]
+      renderView1.CameraFocalPoint = [8.0, 8.0, 8.0]
+      renderView1.CameraViewUp = [0.0, 0.0, 1.0]
+      renderView1.CameraFocalDisk = 1.0
+      renderView1.CameraParallelScale = 12.374368670764582
+      renderView1.BackEnd = 'OSPRay raycaster'
+      renderView1.OSPRayMaterialLibrary = materialLibrary1
+      renderView1.Background = [0.0, 0.0, 0.0]
+
+      # register the view with coprocessor
+      # and provide it with information such as the filename to use,
+      # how frequently to write the images, etc.
+      coprocessor.RegisterView(renderView1,
+          filename='SliceA_%t.png', freq=1, fittoscreen=0, magnification=1, width=800, height=600, cinema={}, compression=5)
+      renderView1.ViewTime = datadescription.GetTime()
+
+      SetActiveView(None)
+
+      # ----------------------------------------------------------------
+      # setup view layouts
+      # ----------------------------------------------------------------
+
+      # create new layout object 'Layout #1'
+      layout1 = CreateLayout(name='Layout #1')
+      layout1.AssignView(0, renderView1)
+
+      # ----------------------------------------------------------------
+      # restore active view
+      SetActiveView(renderView1)
+      # ----------------------------------------------------------------
+
+      # ----------------------------------------------------------------
+      # setup the data processing pipelines
+      # ----------------------------------------------------------------
+
+      # create a new 'VisItChomboReader'
+      # create a producer from a simulation input
+      input = coprocessor.CreateProducer(datadescription, 'input')
+
+      # create a new 'Slice'
+      slice1 = Slice(Input=input)
+      slice1.SliceType = 'Plane'
+      slice1.HyperTreeGridSlicer = 'Plane'
+      slice1.SliceOffsetValues = [0.0]
+
+      # init the 'Plane' selected for 'SliceType'
+      slice1.SliceType.Origin = [8.0, 8.0, 8.0]
+
+      # init the 'Plane' selected for 'HyperTreeGridSlicer'
+      slice1.HyperTreeGridSlicer.Origin = [8.0, 8.0, 8.0]
+
+      # ----------------------------------------------------------------
+      # setup the visualization in view 'renderView1'
+      # ----------------------------------------------------------------
+
+      # show data from slice1
+      slice1Display = Show(slice1, renderView1, 'GeometryRepresentation')
+
+      # get color transfer function/color map for 'A'
+      aLUT = GetColorTransferFunction('A')
+      aLUT.RGBPoints = [42.015869140625, 0.267004, 0.004874, 0.329415, 64.0117780078125, 0.26851, 0.009605, 0.335427, 86.00207853515624, 0.269944, 0.014625, 0.341379, 107.99798740234375, 0.271305, 0.019942, 0.347269, 129.9882879296875, 0.272594, 0.025563, 0.353093, 151.984196796875, 0.273809, 0.031497, 0.358853, 173.97449732421876, 0.274952, 0.037752, 0.364543, 195.97040619140625, 0.276022, 0.044167, 0.370164, 217.96631505859375, 0.277018, 0.050344, 0.375715, 239.9566155859375, 0.277941, 0.056324, 0.381191, 261.952524453125, 0.278791, 0.062145, 0.386592, 283.94282498046874, 0.279566, 0.067836, 0.391917, 305.93873384765624, 0.280267, 0.073417, 0.397163, 327.92903437499996, 0.280894, 0.078907, 0.402329, 349.9249432421875, 0.281446, 0.08432, 0.407414, 371.920852109375, 0.281924, 0.089666, 0.412415, 393.9111526367187, 0.282327, 0.094955, 0.417331, 415.9070615039063, 0.282656, 0.100196, 0.42216, 437.89736203125, 0.28291, 0.105393, 0.426902, 459.89327089843755, 0.283091, 0.110553, 0.431554, 481.88357142578127, 0.283197, 0.11568, 0.436115, 503.8794802929687, 0.283229, 0.120777, 0.440584, 525.8753891601563, 0.283187, 0.125848, 0.44496, 547.8656896875, 0.283072, 0.130895, 0.449241, 569.8615985546875, 0.282884, 0.13592, 0.453427, 591.8518990820313, 0.282623, 0.140926, 0.457517, 613.8478079492187, 0.28229, 0.145912, 0.46151, 635.8381084765625, 0.281887, 0.150881, 0.465405, 657.83401734375, 0.281412, 0.155834, 0.469201, 679.8243178710937, 0.280868, 0.160771, 0.472899, 701.8202267382812, 0.280255, 0.165693, 0.476498, 723.8161356054687, 0.279574, 0.170599, 0.479997, 745.8064361328125, 0.278826, 0.17549, 0.483397, 767.802345, 0.278012, 0.180367, 0.486697, 789.7926455273438, 0.277134, 0.185228, 0.489898, 811.7885543945312, 0.276194, 0.190074, 0.493001, 833.778854921875, 0.275191, 0.194905, 0.496005, 855.7747637890625, 0.274128, 0.199721, 0.498911, 877.7706726562501, 0.273006, 0.20452, 0.501721, 899.7609731835937, 0.271828, 0.209303, 0.504434, 921.7568820507813, 0.270595, 0.214069, 0.507052, 943.747182578125, 0.269308, 0.218818, 0.509577, 965.7430914453124, 0.267968, 0.223549, 0.512008, 987.7333919726562, 0.26658, 0.228262, 0.514349, 1009.7293008398437, 0.265145, 0.232956, 0.516599, 1031.7252097070311, 0.263663, 0.237631, 0.518762, 1053.715510234375, 0.262138, 0.242286, 0.520837, 1075.7114191015626, 0.260571, 0.246922, 0.522828, 1097.7017196289064, 0.258965, 0.251537, 0.524736, 1119.6976284960938, 0.257322, 0.25613, 0.526563, 1141.6879290234376, 0.255645, 0.260703, 0.528312, 1163.683837890625, 0.253935, 0.265254, 0.529983, 1185.6797467578124, 0.252194, 0.269783, 0.531579, 1207.6700472851562, 0.250425, 0.27429, 0.533103, 1229.6659561523438, 0.248629, 0.278775, 0.534556, 1251.6562566796874, 0.246811, 0.283237, 0.535941, 1273.652165546875, 0.244972, 0.287675, 0.53726, 1295.6424660742189, 0.243113, 0.292092, 0.538516, 1317.6383749414063, 0.241237, 0.296485, 0.539709, 1339.6342838085936, 0.239346, 0.300855, 0.540844, 1361.6245843359375, 0.237441, 0.305202, 0.541921, 1383.620493203125, 0.235526, 0.309527, 0.542944, 1405.6107937304687, 0.233603, 0.313828, 0.543914, 1427.6067025976563, 0.231674, 0.318106, 0.544834, 1449.597003125, 0.229739, 0.322361, 0.545706, 1471.5929119921875, 0.227802, 0.326594, 0.546532, 1493.588820859375, 0.225863, 0.330805, 0.547314, 1515.5791213867187, 0.223925, 0.334994, 0.548053, 1537.575030253906, 0.221989, 0.339161, 0.548752, 1559.56533078125, 0.220057, 0.343307, 0.549413, 1581.5612396484373, 0.21813, 0.347432, 0.550038, 1603.5515401757812, 0.21621, 0.351535, 0.550627, 1625.5474490429688, 0.214298, 0.355619, 0.551184, 1647.5433579101561, 0.212395, 0.359683, 0.55171, 1669.5336584375, 0.210503, 0.363727, 0.552206, 1691.5295673046874, 0.208623, 0.367752, 0.552675, 1713.5198678320312, 0.206756, 0.371758, 0.553117, 1735.5157766992186, 0.204903, 0.375746, 0.553533, 1757.5060772265624, 0.203063, 0.379716, 0.553925, 1779.5019860937502, 0.201239, 0.38367, 0.554294, 1801.4922866210936, 0.19943, 0.387607, 0.554642, 1823.4881954882812, 0.197636, 0.391528, 0.554969, 1845.4841043554686, 0.19586, 0.395433, 0.555276, 1867.4744048828125, 0.1941, 0.399323, 0.555565, 1889.4703137499998, 0.192357, 0.403199, 0.555836, 1911.4606142773437, 0.190631, 0.407061, 0.556089, 1933.4565231445315, 0.188923, 0.41091, 0.556326, 1955.4468236718749, 0.187231, 0.414746, 0.556547, 1977.4427325390625, 0.185556, 0.41857, 0.556753, 1999.4386414062499, 0.183898, 0.422383, 0.556944, 2021.4289419335937, 0.182256, 0.426184, 0.55712, 2043.424850800781, 0.180629, 0.429975, 0.557282, 2065.415151328125, 0.179019, 0.433756, 0.55743, 2087.4110601953125, 0.177423, 0.437527, 0.557565, 2109.401360722656, 0.175841, 0.44129, 0.557685, 2131.3972695898437, 0.174274, 0.445044, 0.557792, 2153.3931784570314, 0.172719, 0.448791, 0.557885, 2175.383478984375, 0.171176, 0.45253, 0.557965, 2197.3793878515626, 0.169646, 0.456262, 0.55803, 2219.369688378906, 0.168126, 0.459988, 0.558082, 2241.365597246094, 0.166617, 0.463708, 0.558119, 2263.3558977734374, 0.165117, 0.467423, 0.558141, 2285.351806640625, 0.163625, 0.471133, 0.558148, 2307.3477155078126, 0.162142, 0.474838, 0.55814, 2329.338016035156, 0.160665, 0.47854, 0.558115, 2351.333924902344, 0.159194, 0.482237, 0.558073, 2373.3242254296874, 0.157729, 0.485932, 0.558013, 2395.320134296875, 0.15627, 0.489624, 0.557936, 2417.3104348242186, 0.154815, 0.493313, 0.55784, 2439.3063436914063, 0.153364, 0.497, 0.557724, 2461.302252558594, 0.151918, 0.500685, 0.557587, 2483.2925530859375, 0.150476, 0.504369, 0.55743, 2505.288461953125, 0.149039, 0.508051, 0.55725, 2527.2787624804687, 0.147607, 0.511733, 0.557049, 2549.2746713476563, 0.14618, 0.515413, 0.556823, 2571.264971875, 0.144759, 0.519093, 0.556572, 2593.2608807421875, 0.143343, 0.522773, 0.556295, 2615.256789609375, 0.141935, 0.526453, 0.555991, 2637.2470901367187, 0.140536, 0.530132, 0.555659, 2659.2429990039063, 0.139147, 0.533812, 0.555298, 2681.23329953125, 0.13777, 0.537492, 0.554906, 2703.2292083984375, 0.136408, 0.541173, 0.554483, 2725.219508925781, 0.135066, 0.544853, 0.554029, 2747.2154177929688, 0.133743, 0.548535, 0.553541, 2769.2113266601564, 0.132444, 0.552216, 0.553018, 2791.2016271875, 0.131172, 0.555899, 0.552459, 2813.1975360546876, 0.129933, 0.559582, 0.551864, 2835.187836582031, 0.128729, 0.563265, 0.551229, 2857.183745449219, 0.127568, 0.566949, 0.550556, 2879.174045976563, 0.126453, 0.570633, 0.549841, 2901.16995484375, 0.125394, 0.574318, 0.549086, 2923.1602553710936, 0.124395, 0.578002, 0.548287, 2945.1561642382812, 0.123463, 0.581687, 0.547445, 2967.1520731054684, 0.122606, 0.585371, 0.546557, 2989.1423736328125, 0.121831, 0.589055, 0.545623, 3011.1382825, 0.121148, 0.592739, 0.544641, 3033.1285830273437, 0.120565, 0.596422, 0.543611, 3055.1244918945313, 0.120092, 0.600104, 0.54253, 3077.114792421875, 0.119738, 0.603785, 0.5414, 3099.1107012890625, 0.119512, 0.607464, 0.540218, 3121.1066101562496, 0.119423, 0.611141, 0.538982, 3143.0969106835937, 0.119483, 0.614817, 0.537692, 3165.0928195507813, 0.119699, 0.61849, 0.536347, 3187.083120078125, 0.120081, 0.622161, 0.534946, 3209.0790289453125, 0.120638, 0.625828, 0.533488, 3231.069329472656, 0.12138, 0.629492, 0.531973, 3253.0652383398437, 0.122312, 0.633153, 0.530398, 3275.061147207031, 0.123444, 0.636809, 0.528763, 3297.051447734375, 0.12478, 0.640461, 0.527068, 3319.0473566015626, 0.126326, 0.644107, 0.525311, 3341.037657128906, 0.128087, 0.647749, 0.523491, 3363.033565996094, 0.130067, 0.651384, 0.521608, 3385.0238665234374, 0.132268, 0.655014, 0.519661, 3407.019775390625, 0.134692, 0.658636, 0.517649, 3429.015684257812, 0.137339, 0.662252, 0.515571, 3451.005984785156, 0.14021, 0.665859, 0.513427, 3473.001893652344, 0.143303, 0.669459, 0.511215, 3494.9921941796874, 0.146616, 0.67305, 0.508936, 3516.9881030468755, 0.150148, 0.676631, 0.506589, 3538.9784035742186, 0.153894, 0.680203, 0.504172, 3560.9743124414063, 0.157851, 0.683765, 0.501686, 3582.9702213085934, 0.162016, 0.687316, 0.499129, 3604.9605218359375, 0.166383, 0.690856, 0.496502, 3626.956430703125, 0.170948, 0.694384, 0.493803, 3648.9467312304687, 0.175707, 0.6979, 0.491033, 3670.9426400976567, 0.180653, 0.701402, 0.488189, 3692.932940625, 0.185783, 0.704891, 0.485273, 3714.9288494921875, 0.19109, 0.708366, 0.482284, 3736.9247583593747, 0.196571, 0.711827, 0.479221, 3758.9150588867187, 0.202219, 0.715272, 0.476084, 3780.9109677539063, 0.20803, 0.718701, 0.472873, 3802.90126828125, 0.214, 0.722114, 0.469588, 3824.897177148438, 0.220124, 0.725509, 0.466226, 3846.887477675781, 0.226397, 0.728888, 0.462789, 3868.8833865429688, 0.232815, 0.732247, 0.459277, 3890.879295410156, 0.239374, 0.735588, 0.455688, 3912.8695959375, 0.24607, 0.73891, 0.452024, 3934.8655048046876, 0.252899, 0.742211, 0.448284, 3956.855805332031, 0.259857, 0.745492, 0.444467, 3978.8517141992183, 0.266941, 0.748751, 0.440573, 4000.8420147265624, 0.274149, 0.751988, 0.436601, 4022.83792359375, 0.281477, 0.755203, 0.432552, 4044.828224121094, 0.288921, 0.758394, 0.428426, 4066.8241329882812, 0.296479, 0.761561, 0.424223, 4088.820041855469, 0.304148, 0.764704, 0.419943, 4110.810342382812, 0.311925, 0.767822, 0.415586, 4132.80625125, 0.319809, 0.770914, 0.411152, 4154.796551777344, 0.327796, 0.77398, 0.40664, 4176.792460644531, 0.335885, 0.777018, 0.402049, 4198.782761171875, 0.344074, 0.780029, 0.397381, 4220.7786700390625, 0.35236, 0.783011, 0.392636, 4242.77457890625, 0.360741, 0.785964, 0.387814, 4264.764879433594, 0.369214, 0.788888, 0.382914, 4286.760788300781, 0.377779, 0.791781, 0.377939, 4308.751088828125, 0.386433, 0.794644, 0.372886, 4330.7469976953125, 0.395174, 0.797475, 0.367757, 4352.737298222656, 0.404001, 0.800275, 0.362552, 4374.733207089844, 0.412913, 0.803041, 0.357269, 4396.729115957031, 0.421908, 0.805774, 0.35191, 4418.719416484375, 0.430983, 0.808473, 0.346476, 4440.715325351563, 0.440137, 0.811138, 0.340967, 4462.705625878906, 0.449368, 0.813768, 0.335384, 4484.701534746094, 0.458674, 0.816363, 0.329727, 4506.691835273437, 0.468053, 0.818921, 0.323998, 4528.687744140625, 0.477504, 0.821444, 0.318195, 4550.683653007813, 0.487026, 0.823929, 0.312321, 4572.673953535156, 0.496615, 0.826376, 0.306377, 4594.669862402344, 0.506271, 0.828786, 0.300362, 4616.660162929687, 0.515992, 0.831158, 0.294279, 4638.656071796875, 0.525776, 0.833491, 0.288127, 4660.646372324219, 0.535621, 0.835785, 0.281908, 4682.642281191406, 0.545524, 0.838039, 0.275626, 4704.638190058594, 0.555484, 0.840254, 0.269281, 4726.6284905859375, 0.565498, 0.84243, 0.262877, 4748.624399453125, 0.575563, 0.844566, 0.256415, 4770.614699980469, 0.585678, 0.846661, 0.249897, 4792.610608847656, 0.595839, 0.848717, 0.243329, 4814.600909375, 0.606045, 0.850733, 0.236712, 4836.5968182421875, 0.616293, 0.852709, 0.230052, 4858.592727109375, 0.626579, 0.854645, 0.223353, 4880.583027636719, 0.636902, 0.856542, 0.21662, 4902.578936503906, 0.647257, 0.8584, 0.209861, 4924.56923703125, 0.657642, 0.860219, 0.203082, 4946.565145898438, 0.668054, 0.861999, 0.196293, 4968.555446425781, 0.678489, 0.863742, 0.189503, 4990.551355292969, 0.688944, 0.865448, 0.182725, 5012.547264160156, 0.699415, 0.867117, 0.175971, 5034.5375646875, 0.709898, 0.868751, 0.169257, 5056.533473554688, 0.720391, 0.87035, 0.162603, 5078.523774082031, 0.730889, 0.871916, 0.156029, 5100.519682949219, 0.741388, 0.873449, 0.149561, 5122.509983476562, 0.751884, 0.874951, 0.143228, 5144.50589234375, 0.762373, 0.876424, 0.137064, 5166.496192871094, 0.772852, 0.877868, 0.131109, 5188.492101738281, 0.783315, 0.879285, 0.125405, 5210.488010605469, 0.79376, 0.880678, 0.120005, 5232.478311132812, 0.804182, 0.882046, 0.114965, 5254.47422, 0.814576, 0.883393, 0.110347, 5276.464520527344, 0.82494, 0.88472, 0.106217, 5298.460429394531, 0.83527, 0.886029, 0.102646, 5320.450729921875, 0.845561, 0.887322, 0.099702, 5342.4466387890625, 0.85581, 0.888601, 0.097452, 5364.44254765625, 0.866013, 0.889868, 0.095953, 5386.432848183594, 0.876168, 0.891125, 0.09525, 5408.428757050781, 0.886271, 0.892374, 0.095374, 5430.419057578125, 0.89632, 0.893616, 0.096335, 5452.4149664453125, 0.906311, 0.894855, 0.098125, 5474.405266972656, 0.916242, 0.896091, 0.100717, 5496.401175839844, 0.926106, 0.89733, 0.104071, 5518.397084707031, 0.935904, 0.89857, 0.108131, 5540.387385234375, 0.945636, 0.899815, 0.112838, 5562.383294101563, 0.9553, 0.901065, 0.118128, 5584.373594628906, 0.964894, 0.902323, 0.123941, 5606.369503496094, 0.974417, 0.90359, 0.130215, 5628.359804023437, 0.983868, 0.904867, 0.136897, 5650.355712890625, 0.993248, 0.906157, 0.143936]
+      aLUT.NanColor = [1.0, 0.0, 0.0]
+      aLUT.ScalarRangeInitialized = 1.0
+
+      # trace defaults for the display properties.
+      slice1Display.Representation = 'Surface'
+      slice1Display.ColorArrayName = ['CELLS', 'A']
+      slice1Display.LookupTable = aLUT
+      slice1Display.OSPRayScaleFunction = 'PiecewiseFunction'
+      slice1Display.SelectOrientationVectors = 'None'
+      slice1Display.ScaleFactor = 1.75
+      slice1Display.SelectScaleArray = 'None'
+      slice1Display.GlyphType = 'Arrow'
+      slice1Display.GlyphTableIndexArray = 'None'
+      slice1Display.GaussianRadius = 0.08750000000000001
+      slice1Display.SetScaleArray = [None, '']
+      slice1Display.ScaleTransferFunction = 'PiecewiseFunction'
+      slice1Display.OpacityArray = [None, '']
+      slice1Display.OpacityTransferFunction = 'PiecewiseFunction'
+      slice1Display.DataAxesGrid = 'GridAxesRepresentation'
+      slice1Display.PolarAxes = 'PolarAxesRepresentation'
+
+      # ----------------------------------------------------------------
+      # setup color maps and opacity mapes used in the visualization
+      # note: the Get..() functions create a new object, if needed
+      # ----------------------------------------------------------------
+
+      # get opacity transfer function/opacity map for 'A'
+      aPWF = GetOpacityTransferFunction('A')
+      aPWF.Points = [42.06640625, 0.0, 0.5, 0.0, 7363.00390625, 1.0, 0.5, 0.0]
+      aPWF.ScalarRangeInitialized = 1
+
+      # ----------------------------------------------------------------
+      # finally, restore active source
+      SetActiveSource(slice1)
+      # ----------------------------------------------------------------
+    return Pipeline()
+
+  class CoProcessor(coprocessing.CoProcessor):
+    def CreatePipeline(self, datadescription):
+      self.Pipeline = _CreatePipeline(self, datadescription)
+
+  coprocessor = CoProcessor()
+  # these are the frequencies at which the coprocessor updates.
+  freqs = {'input': [1]}
+  coprocessor.SetUpdateFrequencies(freqs)
+  if requestSpecificArrays:
+    arrays = [['A', 1], ['vtkGhostType', 1]]
+    coprocessor.SetRequestedArrays('input', arrays)
+  coprocessor.SetInitialOutputOptions(timeStepToStartOutputAt,forceOutputAtFirstCall)
+
+  if rootDirectory:
+      coprocessor.SetRootDirectory(rootDirectory)
+
+  if make_cinema_table:
+      coprocessor.EnableCinemaDTable()
+
+  return coprocessor
+
+
+#--------------------------------------------------------------
+# Global variable that will hold the pipeline for each timestep
+# Creating the CoProcessor object, doesn't actually create the ParaView pipeline.
+# It will be automatically setup when coprocessor.UpdateProducers() is called the
+# first time.
+coprocessor = CreateCoProcessor()
+
+#--------------------------------------------------------------
+# Enable Live-Visualizaton with ParaView and the update frequency
+coprocessor.EnableLiveVisualization(False, 1)
+
+# ---------------------- Data Selection method ----------------------
+
+def RequestDataDescription(datadescription):
+    "Callback to populate the request for current timestep"
+    global coprocessor
+
+    # setup requests for all inputs based on the requirements of the
+    # pipeline.
+    coprocessor.LoadRequestedData(datadescription)
+
+# ------------------------ Processing method ------------------------
+
+def DoCoProcessing(datadescription):
+    "Callback to do co-processing for current timestep"
+    global coprocessor
+
+    # Update the coprocessor by providing it the newly generated simulation data.
+    # If the pipeline hasn't been setup yet, this will setup the pipeline.
+    coprocessor.UpdateProducers(datadescription)
+
+    # Write output data, if appropriate.
+    coprocessor.WriteData(datadescription);
+
+    # Write image capture (Last arg: rescale lookup table), if appropriate.
+    coprocessor.WriteImages(datadescription, rescale_lookuptable=rescale_lookuptable,
+        image_quality=0, padding_amount=imageFileNamePadding)
+
+    # Live Visualization, if enabled.
+    coprocessor.DoLiveVisualization(datadescription, "localhost", 22222)
diff --git a/Tests/CatalystInsituTest/slice_A.py b/Tests/CatalystInsituTest/slice_A_v2.py
similarity index 100%
rename from Tests/CatalystInsituTest/slice_A.py
rename to Tests/CatalystInsituTest/slice_A_v2.py

From 468e59e18505d510002af98a798ef8d56473ebf3 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Mon, 21 Mar 2022 16:20:25 +0000
Subject: [PATCH 38/63] Fix issues with Catalyst v2.0 scripts

---
 Source/Insitu/CatalystAdaptor.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 3677e6be5..e0c969a3d 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -5,6 +5,7 @@
 
 #include "CatalystAdaptor.hpp"
 #include "GRAMRLevel.hpp"
+#include "ParaviewVersion.hpp"
 
 #ifdef USE_CATALYST
 
@@ -77,8 +78,14 @@ void CatalystAdaptor::initialise(
     // Create Python script pipeline and add it to the VTK CP Processor
     for (const std::string &script : a_python_scripts)
     {
+#if PARAVIEW_VERSION_HERE >= PARAVIEW_VERSION_TEST(5, 9, 0)
+        auto pipeline =
+            vtkCPPythonPipeline::CreateAndInitializePipeline(script.c_str());
+        bool pipeline_init_success = (pipeline != nullptr);
+#else
         vtkNew<vtkCPPythonScriptPipeline> pipeline;
         int pipeline_init_success = pipeline->Initialize(script.c_str());
+#endif
         std::string pipeline_init_fail_msg =
             "Failed to initialize pipelone for script: ";
         pipeline_init_fail_msg += script;

From 352ee3c7070aca670e6daa492b3cff2df86d7e4e Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Mon, 21 Mar 2022 17:00:27 +0000
Subject: [PATCH 39/63] Make VTK use the same number of threads as GRChombo

If the VTK_SMP_MAX_THREADS environment variable is not set, the number of
VTK SMP threads is set to the same as the number of OpenMP threads.

Also add some small comments to CatalystAdaptor.hpp.
---
 Source/GRChomboCore/SetupFunctions.hpp | 27 ++++++++++++++++++++++++++
 Source/Insitu/CatalystAdaptor.hpp      |  5 ++++-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 31fd034d9..2a5134aa9 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -32,6 +32,7 @@ using std::endl;
 
 #ifdef USE_CATALYST
 #include "vtkLogger.h"
+#include "vtkSMPTools.h"
 #include "vtkVersion.h"
 #endif
 
@@ -94,6 +95,32 @@ void mainSetup(int argc, char *argv[])
         cerr << " usage " << argv[0] << " <input_file_name> " << endl;
         exit(0);
     }
+
+#ifdef USE_CATALYST
+    // Use the VTK_SMP_MAX_THREADS environment variable to set the maximum
+    // number of SMP threads if it exists
+    if (!std::getenv("VTK_SMP_MAX_THREADS"))
+    {
+        // otherwise set it to the same as the number of OpenMP threads
+#ifdef _OPENMP
+        int num_threads = omp_get_max_threads();
+#else
+        int num_threads = 1;
+#endif
+        vtkSMPTools::Initialize(num_threads);
+    }
+    else
+    {
+        // VTK will automatically check the VTK_SMP_MAX_THREADS environment
+        // variable
+        vtkSMPTools::Initialize();
+    }
+    if (rank == 0)
+    {
+        std::cout << " catalyst threads = "
+                  << vtkSMPTools::GetEstimatedNumberOfThreads() << std::endl;
+    }
+#endif
 }
 
 void mainFinalize()
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index 7021abcc8..11e392bef 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -74,11 +74,14 @@ class CatalystAdaptor
     // send variables to catalyst
     void add_vars(vtkCPInputDataDescription *a_input_data_desc);
 
+    // directly passes the FAB pointer to the VTK array
     vtkDoubleArray *fab_to_vtk_array(FArrayBox &a_fab, int a_var,
                                      const std::string &a_name);
 
+    // copies the data in the FAB to the VTK array without the ghosts
     vtkDoubleArray *fab_to_vtk_array_without_ghosts(FArrayBox &a_fab,
-                                                    const Box &a_box, int a_var,
+                                                    const Box &a_unghosted_box,
+                                                    int a_var,
                                                     const std::string &a_name);
 
     // if a_success = false, either aborts or prints a warning depending on

From 6395ba8a7abaf41c7dc691bb8e09b8e64eb29a7f Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 6 Apr 2022 12:33:10 +0100
Subject: [PATCH 40/63] Make preCatalystCoProcess fill ghosts by default

The ghosts are filled for variables requested at the previous CoProcess.
One can also pass a non-positive interval of variables to fillAllGhosts
which will just make it return without doing any ghost filling.
---
 Examples/BinaryBH/BinaryBHLevel.cpp | 11 +++++-
 Source/GRChomboCore/GRAMRLevel.cpp  | 56 +++++++++++++++++++++++++++++
 Source/GRChomboCore/GRAMRLevel.hpp  |  4 ++-
 Source/Insitu/CatalystAdaptor.cpp   | 41 ++++++++++++---------
 Source/Insitu/CatalystAdaptor.hpp   | 12 ++++++-
 5 files changed, 104 insertions(+), 20 deletions(-)

diff --git a/Examples/BinaryBH/BinaryBHLevel.cpp b/Examples/BinaryBH/BinaryBHLevel.cpp
index cf17984d7..413f3429f 100644
--- a/Examples/BinaryBH/BinaryBHLevel.cpp
+++ b/Examples/BinaryBH/BinaryBHLevel.cpp
@@ -138,10 +138,19 @@ void BinaryBHLevel::specificPostTimeStep()
                         // called during setup at t=0 from Main
     // bool first_step = (m_time == m_dt); // if not called in Main
 
-    if (m_p.activate_extraction == 1)
+    if (m_p.activate_extraction
+#ifdef USE_CATALYST
+        || m_p.activate_catalyst
+#endif
+    )
     {
         int min_level = m_p.extraction_params.min_extraction_level();
         bool calculate_weyl = at_level_timestep_multiple(min_level);
+#ifdef USE_CATALYST
+        calculate_weyl |=
+            (m_p.activate_catalyst &&
+             at_level_timestep_multiple(m_p.catalyst_coprocess_level));
+#endif
         if (calculate_weyl)
         {
             // Populate the Weyl Scalar values on the grid
diff --git a/Source/GRChomboCore/GRAMRLevel.cpp b/Source/GRChomboCore/GRAMRLevel.cpp
index f88212927..c7dc4abf2 100644
--- a/Source/GRChomboCore/GRAMRLevel.cpp
+++ b/Source/GRChomboCore/GRAMRLevel.cpp
@@ -191,6 +191,56 @@ void GRAMRLevel::postTimeStep()
 }
 
 #ifdef USE_CATALYST
+void GRAMRLevel::preCatalystCoProcess()
+{
+    // If we're not passing ghosts to Catalyst, there's no point in filling them
+    if (m_p.catalyst_remove_ghosts)
+        return;
+
+    if (std::abs(m_time - m_restart_time) < m_gr_amr.timeEps())
+    {
+        // We don't have any information on the first step so fill all ghosts
+        m_gr_amr.fill_multilevel_ghosts(VariableType::evolution);
+        m_gr_amr.fill_multilevel_ghosts(VariableType::diagnostic);
+    }
+    else
+    {
+        // Now use the last requested variables
+        const auto requested_evolution_vars =
+            m_gr_amr.m_insitu->get_requested_evolution_vars();
+        const auto requested_diagnostic_vars =
+            m_gr_amr.m_insitu->get_requested_diagnostic_vars();
+        int first_requested_evolution_var = 0;
+        int last_requested_evolution_var = NUM_VARS - 1;
+        int first_requested_diagnostic_var = 0;
+        int last_requested_diagnostic_var = NUM_DIAGNOSTIC_VARS - 1;
+
+        while ((!requested_evolution_vars[first_requested_evolution_var]) &&
+               first_requested_evolution_var < NUM_VARS)
+            ++first_requested_evolution_var;
+        while ((!requested_evolution_vars[last_requested_evolution_var]) &&
+               last_requested_evolution_var > -1)
+            --last_requested_evolution_var;
+        while ((!requested_diagnostic_vars[first_requested_diagnostic_var]) &&
+               first_requested_diagnostic_var < NUM_DIAGNOSTIC_VARS)
+            ++first_requested_diagnostic_var;
+        while ((!requested_diagnostic_vars[last_requested_diagnostic_var]) &&
+               last_requested_diagnostic_var > -1)
+            --last_requested_diagnostic_var;
+
+        // For simplicity fill all ghosts in the interval from first to last
+        // for each variable type
+        Interval evolution_vars(first_requested_evolution_var,
+                                last_requested_evolution_var);
+        Interval diagnostic_vars(first_requested_diagnostic_var,
+                                 last_requested_diagnostic_var);
+
+        // These won't do anything if the interval size is <= 0
+        fillAllGhosts(VariableType::evolution, evolution_vars);
+        fillAllGhosts(VariableType::diagnostic, diagnostic_vars);
+    }
+}
+
 void GRAMRLevel::catalystCoProcess()
 {
     if (m_p.activate_catalyst)
@@ -1069,6 +1119,9 @@ void GRAMRLevel::fillAllEvolutionGhosts(const Interval &a_comps)
     if (m_verbosity)
         pout() << "GRAMRLevel::fillAllEvolutionGhosts" << endl;
 
+    if (a_comps.size() <= 0)
+        return;
+
     // If there is a coarser level then interpolate undefined ghost cells
     if (m_coarser_level_ptr != nullptr)
     {
@@ -1085,6 +1138,9 @@ void GRAMRLevel::fillAllDiagnosticsGhosts(const Interval &a_comps)
     if (m_verbosity)
         pout() << "GRAMRLevel::fillAllDiagnosticsGhosts" << endl;
 
+    if (a_comps.size() <= 0)
+        return;
+
     // If there is a coarser level then interpolate undefined ghost cells
     if (m_coarser_level_ptr != nullptr)
     {
diff --git a/Source/GRChomboCore/GRAMRLevel.hpp b/Source/GRChomboCore/GRAMRLevel.hpp
index 1485f25cb..8b915a354 100644
--- a/Source/GRChomboCore/GRAMRLevel.hpp
+++ b/Source/GRChomboCore/GRAMRLevel.hpp
@@ -192,7 +192,9 @@ class GRAMRLevel : public AMRLevel, public InterpSource
     /// Calls Catalyst CoProcess
     void catalystCoProcess();
 
-    virtual void preCatalystCoProcess() {}
+    /// Things to do before calling Catalyst CoProcess (fills ghosts if not
+    /// overriden)
+    virtual void preCatalystCoProcess();
 #endif
 
   protected:
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index e0c969a3d..80e5aa691 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -49,6 +49,8 @@ void CatalystAdaptor::initialise(
     }
     m_gr_amr_ptr = a_gr_amr_ptr;
     m_vars = a_vars;
+    m_requested_evolution_vars.fill(false);
+    m_requested_diagnostic_vars.fill(false);
     m_abort_on_catalyst_error = a_abort_on_catalyst_error;
     m_remove_ghosts = a_remove_ghosts;
 
@@ -251,20 +253,13 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
 {
     if (m_verbosity)
     {
-        pout() << "CatalystAdaptor::add_vars" << std::endl;
-    }
-
-    std::array<bool, NUM_VARS> requested_evolution_vars;
-    std::array<bool, NUM_DIAGNOSTIC_VARS> requested_diagnostic_vars;
-
-    if (m_verbosity)
-    {
-        pout() << "CatalystAdaptor Requested variables: ";
+        pout() << "CatalystAdaptor::add_vars" << std::endl
+               << "CatalystAdaptor Requested variables: ";
     }
 
     for (int ivar = 0; ivar < NUM_VARS; ++ivar)
     {
-        requested_evolution_vars[ivar] = a_input_data_desc->IsFieldNeeded(
+        m_requested_evolution_vars[ivar] = a_input_data_desc->IsFieldNeeded(
             UserVariables::variable_names[ivar].c_str(), vtkDataObject::CELL);
         if (m_vars.size() > 0)
         {
@@ -272,14 +267,14 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
                 !(std::find(m_vars.begin(), m_vars.end(),
                             std::make_pair(ivar, VariableType::evolution)) ==
                   m_vars.end());
-            requested_evolution_vars[ivar] &= pass_var;
+            m_requested_evolution_vars[ivar] &= pass_var;
         }
-        if (m_verbosity && requested_evolution_vars[ivar])
+        if (m_verbosity && m_requested_evolution_vars[ivar])
             pout() << UserVariables::variable_names[ivar] << " ";
     }
     for (int ivar = 0; ivar < NUM_DIAGNOSTIC_VARS; ++ivar)
     {
-        requested_diagnostic_vars[ivar] = a_input_data_desc->IsFieldNeeded(
+        m_requested_diagnostic_vars[ivar] = a_input_data_desc->IsFieldNeeded(
             DiagnosticVariables::variable_names[ivar].c_str(),
             vtkDataObject::CELL);
         if (m_vars.size() > 0)
@@ -288,9 +283,9 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
                 !(std::find(m_vars.begin(), m_vars.end(),
                             std::make_pair(ivar, VariableType::diagnostic)) ==
                   m_vars.end());
-            requested_diagnostic_vars[ivar] &= pass_var;
+            m_requested_diagnostic_vars[ivar] &= pass_var;
         }
-        if (m_verbosity && requested_diagnostic_vars[ivar])
+        if (m_verbosity && m_requested_diagnostic_vars[ivar])
             pout() << DiagnosticVariables::variable_names[ivar] << " ";
     }
     if (m_verbosity)
@@ -377,7 +372,7 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
 
                 for (int ivar = 0; ivar < NUM_VARS; ++ivar)
                 {
-                    if (requested_evolution_vars[ivar])
+                    if (m_requested_evolution_vars[ivar])
                     {
                         vtkDoubleArray *vtk_double_arr;
                         if (!m_remove_ghosts)
@@ -398,7 +393,7 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
                 }
                 for (int ivar = 0; ivar < NUM_DIAGNOSTIC_VARS; ++ivar)
                 {
-                    if (requested_diagnostic_vars[ivar])
+                    if (m_requested_diagnostic_vars[ivar])
                     {
                         vtkDoubleArray *vtk_double_arr;
                         if (!m_remove_ghosts)
@@ -446,6 +441,18 @@ void CatalystAdaptor::coprocess(double a_time, unsigned int a_timestep)
     }
 }
 
+const std::array<bool, NUM_VARS> &
+CatalystAdaptor::get_requested_evolution_vars()
+{
+    return m_requested_evolution_vars;
+}
+
+const std::array<bool, NUM_DIAGNOSTIC_VARS> &
+CatalystAdaptor::get_requested_diagnostic_vars()
+{
+    return m_requested_diagnostic_vars;
+}
+
 vtkDoubleArray *CatalystAdaptor::fab_to_vtk_array(FArrayBox &a_fab, int a_var,
                                                   const std::string &a_name)
 {
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index 11e392bef..1c37bbe2b 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -67,6 +67,11 @@ class CatalystAdaptor
     // do Catalyst processing
     void coprocess(double a_time, unsigned int a_timestep);
 
+    // returns the variables that were last requested/sent to Catalyst
+    const std::array<bool, NUM_VARS> &get_requested_evolution_vars();
+    const std::array<bool, NUM_DIAGNOSTIC_VARS> &
+    get_requested_diagnostic_vars();
+
   private:
     // update the AMR grid (no grid data)
     void build_vtk_grid();
@@ -93,9 +98,14 @@ class CatalystAdaptor
     bool m_abort_on_catalyst_error = false;
     bool m_remove_ghosts = false;
     GRAMR *m_gr_amr_ptr = nullptr;
-    // variables to pass to Catalyst
+
+    // variables to pass to Catalyst set by GRChombo parameter
     std::vector<std::pair<int, VariableType>> m_vars;
 
+    // variables actually passed to Catalyst on last CoProcess
+    std::array<bool, NUM_VARS> m_requested_evolution_vars;
+    std::array<bool, NUM_DIAGNOSTIC_VARS> m_requested_diagnostic_vars;
+
     vtkCPProcessor *m_proc_ptr = nullptr;
     vtkOverlappingAMR *m_vtk_grid_ptr = nullptr;
 };

From 27b8951b1ff93026419959dfd74719762b939b1e Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Mon, 25 Apr 2022 10:37:27 +0100
Subject: [PATCH 41/63] Add GitHub action to run Catalyst Insitu test

---
 .../workflows/run-catalyst-insitu-test.yml    | 82 +++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 .github/workflows/run-catalyst-insitu-test.yml

diff --git a/.github/workflows/run-catalyst-insitu-test.yml b/.github/workflows/run-catalyst-insitu-test.yml
new file mode 100644
index 000000000..c7a2e92e0
--- /dev/null
+++ b/.github/workflows/run-catalyst-insitu-test.yml
@@ -0,0 +1,82 @@
+name: Run Catalyst Insitu test
+
+on: [push]
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-20.04
+    env:
+      CHOMBO_HOME: ${{ github.workspace }}/Chombo/lib
+      PARAVIEW_SUPERBUILD_DIR: ${{ github.workspace }}/build
+      PARAVIEW_DIR: ${{ github.workspace }}/build/install
+      PARAVIEW_VERSION: 5.10.1
+      OMP_NUM_THREADS: 1
+
+    steps:
+    - name: Checkout Chombo
+      uses: actions/checkout@v3
+      with:
+        repository: GRChombo/Chombo
+        path: Chombo
+
+    - name: Checkout GRChombo
+      uses: actions/checkout@v3
+      with:
+        path: GRChombo
+
+    - name: Install Chombo dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get -y --no-install-recommends install csh gfortran-10 g++-10 cpp-10 libhdf5-dev libhdf5-openmpi-dev openmpi-bin libblas-dev liblapack-dev libgetopt-complete-perl
+
+    - name: Set Compilers
+      run: |
+        sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100
+        sudo update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-10 100
+        sudo update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-10 100
+
+    - name: Build Chombo
+      run: |
+        cp $GITHUB_WORKSPACE/GRChombo/InstallNotes/MakeDefsLocalExamples/ubuntu-gcc.Make.defs.local $CHOMBO_HOME/mk/Make.defs.local
+        make -j 4 AMRTimeDependent AMRTools BaseTools BoxTools
+      working-directory: ${{ env.CHOMBO_HOME }}
+
+    - name: Install ParaView dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install build-essential libgl1-mesa-dev \
+        libxt-dev python3-dev python3-numpy libopenmpi-dev libtbb-dev \
+        ninja-build libosmesa6 libosmesa6-dev llvm-12-dev
+        sudo update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-12 12
+
+# There is a problem with the paraview-config script with CMake v3.23 
+# (see https://gitlab.kitware.com/paraview/paraview/-/issues/21361)
+    - name: Downgrade CMake
+      env:
+        CMAKE_VERSION: 3.22.4
+      run: |
+        wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh
+        chmod +x cmake-${CMAKE_VERSION}-linux-x86_64.sh
+        sudo ./cmake-${CMAKE_VERSION}-linux-x86_64.sh --prefix=/usr/local --exclude-subdir
+
+
+    - name: Download pre-built ParaView
+      uses: dsaltares/fetch-gh-release-asset@master
+      with:
+        repo: "GRChombo/paraview-build"
+        version: "tags/v${{ env.PARAVIEW_VERSION }}"
+        file: "paraview-v${{ env.PARAVIEW_VERSION }}.tar.gz"
+
+    - name: Untar ParaView
+      run: |
+        mkdir -p $PARAVIEW_DIR
+        cd $PARAVIEW_DIR
+        tar -xzvf $GITHUB_WORKSPACE/paraview-v${PARAVIEW_VERSION}.tar.gz
+
+    - name: Build Catalyst Insitu test
+      run: make test -j 4
+      working-directory: ${{ github.workspace }}/GRChombo/Tests/CatalystInsituTest
+
+    - name: Run Catalyst Insitu test
+      run: make run
+      working-directory: ${{ github.workspace }}/GRChombo/Tests/CatalystInsituTest

From 1b35751cf18b332d91e5583880adf35ba7b024c7 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 18 May 2022 14:59:37 +0100
Subject: [PATCH 42/63] Fix potential segfault with BinaryBH example

This would occur if Catalyst was used but the extraction parameters were
not set (so minimum extraction_level parameter doesn't exist).
---
 Examples/BinaryBH/BinaryBHLevel.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Examples/BinaryBH/BinaryBHLevel.cpp b/Examples/BinaryBH/BinaryBHLevel.cpp
index 413f3429f..5e684697b 100644
--- a/Examples/BinaryBH/BinaryBHLevel.cpp
+++ b/Examples/BinaryBH/BinaryBHLevel.cpp
@@ -144,7 +144,9 @@ void BinaryBHLevel::specificPostTimeStep()
 #endif
     )
     {
-        int min_level = m_p.extraction_params.min_extraction_level();
+        int min_level = (m_p.activate_extraction)
+                            ? m_p.extraction_params.min_extraction_level()
+                            : -1;
         bool calculate_weyl = at_level_timestep_multiple(min_level);
 #ifdef USE_CATALYST
         calculate_weyl |=

From 8b6ee34f1978e4525986bf489a2db50805dd2cc8 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 18 May 2022 15:00:45 +0100
Subject: [PATCH 43/63] Pass ghost cell info to Catalyst

Cells that are marked as ghosts will no longer be visible in ParaView
(e.g. those in the outer boundary).
---
 Source/Insitu/CatalystAdaptor.cpp             |  40 +++++++++---------
 .../CatalystInsituTest.inputs                 |   3 ++
 .../extracts/SliceA_expected.png              | Bin 10451 -> 5998 bytes
 3 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 80e5aa691..ed5344259 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -174,22 +174,23 @@ void CatalystAdaptor::build_vtk_grid()
         {
             // first get the box without ghosts
             Box box = level_box_layout[lit];
-            // const IntVect &small_end = box.smallEnd();
-            // const IntVect &big_end = box.bigEnd();
-
-            // now grow the box to make the ghosted box (no change if
-            // m_remove_ghosts == true)
-            Box ghosted_box = box;
             // VTK counts the big end differently to Chombo so modify the Chombo
             // box so VTK gets what it needs
             for (int idir = 0; idir < SpaceDim; ++idir)
             {
-                ghosted_box.growHi(idir, 1);
-            }
-            if (!m_remove_ghosts)
-            {
-                ghosted_box.grow(level_data.ghostVect());
+                box.growHi(idir, 1);
             }
+            const IntVect &small_end = box.smallEnd();
+            const IntVect &big_end = box.bigEnd();
+
+            // now grow the box to make the ghosted box (no change if
+            // m_remove_ghosts == true)
+            Box ghosted_box = box;
+
+            // ghost_vect takes into account m_remove_ghosts
+            const IntVect &ghost_vect =
+                (m_remove_ghosts) ? IntVect::Zero : level_data.ghostVect();
+            ghosted_box.grow(ghost_vect);
             const IntVect &small_ghosted_end = ghosted_box.smallEnd();
             const IntVect &big_ghosted_end = ghosted_box.bigEnd();
 
@@ -219,16 +220,13 @@ void CatalystAdaptor::build_vtk_grid()
                     small_ghosted_end[1], big_ghosted_end[1],
                     small_ghosted_end[2], big_ghosted_end[2]);
                 // add the ghost cell information
-                // int no_ghost[6] = {small_end[0], big_end[0] + 1,
-                //    small_end[1], big_end[1] + 1,
-                //    small_end[2], big_end[2] + 1};
-                // bool cell_data = true;
-                // vtk_uniform_grid_ptr->GenerateGhostArray(no_ghost,
-                // cell_data);
-
-                // vtk_uniform_grid_ptr->Initialize(
-                // &vtk_amr_box, origin_global, dx_arr,
-                // level_data.ghostVect().dataPtr());
+                int no_ghost[6] = {small_end[0], big_end[0],   small_end[1],
+                                   big_end[1],   small_end[2], big_end[2]};
+                bool cell_data = true;
+                vtk_uniform_grid_ptr->GenerateGhostArray(no_ghost, cell_data);
+
+                // vtk_uniform_grid_ptr->Initialize(&vtk_amr_box, origin_global,
+                //  dx_arr, ghost_vect.dataPtr());
 
                 m_vtk_grid_ptr->SetDataSet(ilevel, ibox, vtk_uniform_grid_ptr);
             }
diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.inputs b/Tests/CatalystInsituTest/CatalystInsituTest.inputs
index 317f4d6f4..bd6274765 100644
--- a/Tests/CatalystInsituTest/CatalystInsituTest.inputs
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.inputs
@@ -12,6 +12,9 @@ num_points = 30
 max_level = 0
 regrid_interval = 1
 
+max_box_size = 8
+min_box_size = 4
+
 #boundaries and periodicity of grid
 #Periodic directions - 0 = false, 1 = true
 isPeriodic = 0 0 0
diff --git a/Tests/CatalystInsituTest/extracts/SliceA_expected.png b/Tests/CatalystInsituTest/extracts/SliceA_expected.png
index c80d8606e4a2a3ab4c4972511ecdfa3d94404414..2be8f1b537e8adba087d6590e0b7fb6355b746ce 100644
GIT binary patch
literal 5998
zcmdT|X;_nI)_y^$)sCeR6seMESY41oFWE!X2~Yw_#Mr7VMr2Wt9oZ5HXln_Gq>z|^
z5Ks|PMv;9htAb(xn-Lr!f&`JpNLVBxkPyQ6hG}cBZ^r4A&Y#JjTuGih&w20XKIh!`
zNji#k*tp^I4FCXabo>H!3;@>b007mKAFG1jY}VXt2)=1vv^#n6a7ajipO@!Fzu+L?
zsArhxSAITG^vm=w0YGcBBkJ(+=+l!g52tTyi&Xm?^ZO$-&3xp{mU`7h^H=&LjZcf4
z-DZX+@Os<6ybis&bFSIDu>y7kF3T<q()(*!*(NQWKl!W$5BlGDg_s9w4q+D9@r%s4
zU>J6qJ1K}Iy7tHTjaL>qx#0pWD<k|m3XRzoGIo6I){T+fi?b6kn#mN(U~l{g<LvY}
zZjQs|r@9!y;Y6e%4bm__C$dG`p~QnY!oIxYDAaUATnscH{8cPDXznYH<k=TeC{V<{
zLhED-v|*Z8o1B9ktL!#<ciWiF;IHR8?H}M?Fq2MYV;bZAjj!7b=F&KmE$Pnot)p!G
z$V0ARWYo(97lb@VTQp&M?}yte<d)c?X$L;FaWW1mixdY=Gm8k4*D(`Z=Iaih#l`Vu
z#r*R2jgMcuIj9VN13n~K+8Yl>gype};IsWbYYcQ}39eC&V~r(V`*{wTe9V<)cd0fd
zbH*cGsuq&y-P*ZlCYlq?0|?|m+c;6bsCRjxIC#l$kS?U!K>G!8Q#cF|%bcX|QM*c^
zNW2{Yqchf{=m&84uy_&~wdT8X|M+y9#dKMlVA{Xy#D*;p$fVFv>yXM%9|GrSpWICN
zX@@G;^?+Ic+x^r!6sqD8dS$nEs~r#d6)MSn*xF+ylo@a}1zbIm@UgY(Aqcohj|v15
zDmPl!rXsK8VUs>iM1;=?U@-eMEjGchr&H!fuQ?UJO+To|B(L?U>|otfR!%u5($YQ+
zS<zveJRlT!k|f7;%Q!g-+cz4yik6JJLa}?LSdO#l7F%(O8zM6oGUJFpe%pJHgpYd)
ztsW?c*}h|cJ!oK7fqO^Ryo9{N+z9}d8@2$4Ow~RCZf0Co1CBaBPz8(%&Zq#I_!0m}
z-gOcH9%=3c63+h1We^CYfw~FM>%%5dC=nnjPm&Q-xJeH#uXT@fMixV_v0P@ntnV>t
zGnzdsy@AQenRgkvj<600=r;;2XPfL;7NoI`8I0{lE=$kdAQ(>UN7rY#c6FpHFl7I{
zR!m1K&yC#aGPLYH(P*0aod*v9^nN3x4@G{J5+b)&;tqkl3J8J2d*Z!uxdag3)2EQs
zP3GfVp0VE&F$HeIC|)$09TsO}s*JNzC{GgSg&2&Us7{`S9cz<ca(?+7F}8czS;P^)
zxZostNu_j$%gke}GH?>S)#rjNG%}w;5sKMJRw9E?3;=hG{yl;p4mZOxV@twx32PGs
zGGWx56{4Uc6md%+Kw%2wx@Y$Y<>SvyazQvB2`*SkvoZFz+K_%Tw2~H7A!-hhRo^>*
z7)NMEj366~!RrvzwsYgHeP|jJdQB2nW#D+HWrN(Um8Y%CdU>+eD^<d;Z)~NlV|ZNo
zBOqs3JOraI{eMwgl}6nol(D%cgq46l<<ta#oMvF{TN#ZB2y2(nJG8X8V~C}=?{JDk
zV?n!hf&|js=mKd(I)P-oIOTxKE_;QPR+ET1w{h-%{pr=YDes01dbR+7_P^xihfsab
zayZ-|wM1=*6u?HITCFzB7}zW^5OE5zO=;9#f~aLG@5$BJ{rf<cin6gm0vwfs(7_rm
z=CHaf401H*x<q{4MQ68;`!tJ6m*IkZ8U%uWMNc9~dD_>+DOj5VibddDHFr(c$$kLv
zUNr@NVQgutYW@0gK8RqolbJSzJszhS^Q<ahf0x(es%x9fr8q8C`#=j+dI~fBQtDxw
ztA{)x^;Tq#^*WL#UpgrY$~buLAh}<}@zIq2rD=h-R%5T$hjRM2v0JGS5Xdu<<PZsH
zY>90DyAr^AN;2Bjd4pIinxY2+1fJD_^coJemQ9nI>JC2{B;yQ4_#6CDkt3M!tN`gb
zs}1t|m1fBUclc~eL4T7&*(7hI%*PN3m&It?4XG=Be?591JIYzd^ONRk9=6t1*+ROR
z9Y$g0xb3zuWsZy_zMOX%9)o&81*W=0+gR_Yk)#Ie+|CJswG7Pg<4(&)7;e0o*0b3O
zy)vgC@{13g9WI!4A`Y|Ko1VGJOp7!Kmjj)i`fpQ<*6q6c8IZ7jwe{09;+sDMd|&HP
z|KS~>(36X^Hpv6dlRBigGg1m=X{&xvs?3`kPjr4sj_ga)v1Da$rCC4W#&1(2_jWt|
zFem(;cemcE>>Z<V$)c-C)NOKnZ`9TUa997Y@H+<+#Jkr5I~;yjLH`wiC^1Kupt3;t
z8A8e2{$zQO@AQ-0<NWd#i9AE<gy`akBkWP!#BNTKsfb8M^XujAl|pW>{6Jt{^K_Z6
zeMOSi?%O^lp0~a4nA|kx?Yf2_u}Xz`trrSCS0W^;{ecLbS)Ci+Uoogs(zKQ`URZa%
zl9wQ|go2T8GEJw$RAN9FYuIvmwNU+K(&p>9Dl>F_y88IEEy5Ss^>t2}*JH2_RSc9A
zeinR4^5`RAhvRAyvry<LTl`(H5eC)>ImWl<y+gWmzHR8uAN&i_em%4okwflKMCXt&
z1rHsIu5A*fTBm5w*B*zSkm!Rqwc2x6I<!)({iIZs-QsWauM>HjWTxomtohgV28gD)
z3Nc}=!AGnR=HVE3!Q2MmQ0nT_t0)f$Rz6Vz@_Jx0EV=1Fl?(6L^>0<hYL@l5Bvjv8
zviPzNmbZ3>LcSB|uQ{}FA+%=?(5!xBl@|&I32tkC%b2|-ke*C=vt--WI-VHVVf1eJ
zG?S<f^6HHJq9@s=n(-oLY{08ioG*Fv>dIq6Ne#Mc575YsY`?yGj8|li@~K|cd*c$2
zGNX5{R@tyB{?1JE1HRnEejV~#u@`-)_!Sg~EUFy18h>!}kC`<Iv9oln!F&USa=zA+
zB(6z0Qm+y}cJ(R2p~^#L#-j><|HESLw_p#@QH$1)#bBUEU)-j`i-m1Qq1|915NBvG
ziFas%iIj}>Y#M@<!ca(`m*d;Nk;~bY45FhhybG)jUT6C(@@cg!L->pIoGy99SX;^$
zr~SFpHZ~Q7m5*Bf3xFv}c_cpSM=-o|pC<Fc0`H5oPtuyCV;-krBu@-<4T75TgLfm(
zPwfP)j-2K%fVm<_4(fZ5C;W=?bN#I>O~>{}IPg5($bH>9=~5hl;WB)E;GQH&>xKj4
z^c9<mfI8KoI#A@VQh|cvqfgl=#q&7hsH3qUe(%~>y0_<i9KFVo%5x=l>RS7v#4bZo
zlTUm}GSx8Z0r>5CA)CgGy9I}{0@z~QaJSCm6iRILDf^_XQJhXVOS-9Z<HVKW-Ieaq
z(Hex0fY>ahC3!D&evh078IWc9MRr?6J<?F!D3A%N3?C2hAY+7ZB~VjbA&Huy9X{6C
z3k7lFQZswHd1fNoJy>2Vq2)PdG(VS2W{qh1yLA;0`i)6M!50;(vtIWFCimnS9j%k|
z;r@L&@wcMSYh1Xx&aCv&nwh{q#K^sI7MALPizRx>V3UrKiGvPlT7iynL=<YuN_UEu
zU3N{?yhtfaz+Nu#(yVpllW~OcJanUhDJ?~R5nG8j)0U3>kg}=xo}f5IULrNkE%TCy
z*)UKS+|I?9zdiWszXj}yol<i2q0$p$6OafTeb$Hbt*9l()l8fHT*nTDvW<a(cFNqr
zDwA{*a)2X`u>GsteUv60jY69!BD6s(m>GO2c3~IG<{YgJFDaPiEO`*ox~c2|+U$+b
zAd1V6t|pnR0UDo1qESlGIl8K3KPlN74(%7=*dr?W6IS6z<+~30wmSd4spY!^zClgw
z76&t4mzPAI56uIX*9ji+8RBCeV5%78x_5IZ(dc-e>~hX)4~gn52f-Q`WE&#Sl9;3d
z0h8WlR$3M9T^p&^4BBSOQP5wuuPT*Fg_p){syf<oY;IN*7xrCc1kDJB5YDj+&m59Y
z5UpajgWe*ou@7PG+kuc#YViT0sO9k+HA}T2RxxN~w?50A2^I&Ibc@BgmbN;f+6Zki
zF(8b@WPRU(6SQfTHo)}nlj-?fmd}Zu>OsR7)+7X%gN8=wG%Qz@Sq#MIthPvLA8j37
zl8GG*_p(PJpQgCor>dH>$tmz+vABjr?7LV73ZzMg$WBXRl8WiUyN*&^Zu;pisgx=o
zR@Ne4%=9D<BxdTxg=i*b3RcGHNED(xY`tOK5>I-x&muT5Xm47usSpxJALKWg0Si#j
z?cM^LGyOmD^8Z}^1sc6E=li(`(jmcdL+(axi1fU5*NKo6)g~vy{+v|%iUum&+)Y-%
zxERL8`ek=<bXh5soW>>OiLg<^==g4A0BAvIN>Ber;Emdj<IO1cWep7a1izi#X`SB>
z4hf!0Lg~@^+Y~Ne*+wxucs4b1rV7CKwAP0!nH#n$Z@1E>B!qpOHE2>D_l6@3$tS+G
z(uEUc^DSvh3sok0PhgD6t#&~@;(Jp9Qzlq@d0lw)opFChlP3L!??KJKBxu9Sx^*Q{
zJkRQR-pD9hNDqR6b#U^vQE0d{h8{h-;IqvR4!?bFUIXeT(HVL$`eHM%rV0;Vks9pP
z>mj9fn|U|R4Mnqn<z0hRczhp`cu6O(8Pj1{4c3U(T`VCpiLbX*4^ApxRDt*#lL{d`
zZza(Saa7w4a=VJmtQ#43!7)(h+jI}Iw6d4W1MOFI`{}omeJtl`qn~lFu_S|v85t|@
zXS(RY;i)j6$#8;3@fK>puNZ$K4!7*_v6BB``DY-W;F#>}PlK`X<&B@MI2Uke_7Jx(
z0lren;c%JnFSE2?O}{8_m`Tgpe**YF|D7NoQT=7)7A^VJC`}QoZ3k@q$AOU0*iv0U
zvxF6P-S#Rov1(~aRJ{C-z@8tMT(m_LP1aY4Ui9P_v8%FQ{7^CaoY6h7vjCwpg&^&p
zYDl+~dZTcOr0LpESdNc9(!pylco=^l+Mv&DcV5`X8KgPj`MTOl8+-n3o0EO(v)9jM
zWbx9Q{=xaYU<ba0%#-;bbE*}wmW}RJP4Uo;|2q@n=4I|O@L-}yJz&e*ua!e>M>{O)
J{*g16{|`dMgtGtu

literal 10451
zcmb_?d010d`u{~mZDncHs-QrP0%8fs8bBacHrW-l0s;{sAiE%|l7wmts0d|O0>lLp
z5g{5tmIzTnA;dr>EJnbHY>k91tl7x-#!jc5-}IU1H+J~r<Z^QF$vx-YKcDx?Q5QS;
zFO|Op06^aUh_xF4to;T6)_81O1I~QC^=nP=V{43!N6dHU&V>d0`^5xDMgT|sqW!)P
zM#bk|$~^%9-vIX3-yKizof=|9XnS5z{TOLuojr&+;ZvDku_488qs45y``VHcR>K#W
zX!{Z+6Zr7W6#Z{@1bxWz+n55|XrfbI<~D_xewFSKbSPR9gNjX}d)WtS;5Y1=x)7>o
zpwGJouWvYS1HQ=r<j=gEyzPCKNQhmYLI`u_dg;6mgu%gF+~lxee^pgSFTu&3<@D%g
zhoQ>YqF4S(<ouKTt7$hTR=Vn_ryPe)jSG-$(R}T3Yipm6ijtKI>6LaYpA!Px3W3!6
zicKE;1clz|tEg*OdunnKSNlq{HsYOj=<T#LugPAD*lR!drCBq(vJ46h@##pNl1|uK
zdiob!D2h^ZDLxLV)hS=wTPkN2*Bt&b+sU>y&>>~=TE*sagNM!a`_=P`kP!~{=;f&j
zt(${r18W^6h#K`uK!UI_FozAJjm}OPryS=Akz#+Ex?~ZKHApWHuoeyAsLE}%N3E?z
z6;Zp~bc3tK&C^S<b(!Mki1=7arCME-Fsh-cQY~VVvb-26!K^Aux?Dr)vpWd2K;|f3
zOk2T9kh=YDc6xG<2FGDGjMNXsi%o*?ol`pfH=)@hM$n5&X=k1LYJa)BK&EZ-t5L6J
zT6nI3-m$hv@CoOZ?2S5faN17syy29!IP4>3m*g;g>>b;hA2YL@sN%@@le$&zbx=g7
zt|UP8Fu!p*o<7OWi0pTR#9J+z5Xzn06X)nZNERk$uzUpXUPj!uUw-YSbGZqJ-Gf)`
zTW%;Q@<a8W7K)D0GRx2IN_HL7)=}^J@lx!5-@&41lsm!A=^Uk#DPfyWpTLxkwjDQA
zcuI`kR*)TZ_?g+)@QK-~8xGT?&3o>kc8o>BWxgM?bHF&+i~vddgQBz&HMTx*k2;TD
zPkOOczGa`JdO5o!1*qWdQAZH&L;;q0)%CgNTYRs$M-{JH%xLz+)D_^p`2)_$C-_;}
zYVNO10DOn1w?oMnvXjTXefSMJfMl<ZI?)RJ<&!vrgp=IRmM;v5EshqKClrdV{p$`|
zm5Ow_TC*&zYe;DN7dd!uBMbR|d%e#0F^L++6SFr-!b0JWk+ic719W!8ng<=3xuP3a
zU&^c;@BHA#b22PCwX1i0SH!`>l-ETiQL!|Qzh0^H!$hO=ygIl~(Uiv1`~UtzMH%5k
z4{R(4bXjV|3F7WQZh0RWs>?!7Jr2#fC>d+PSK3UA>nZ%<5qN0%#et!f@ufi6*AP7_
zqvR?sp7q-FU6s136K8Z*Pslfe=uMOHh3-on*R+!A(3jaReG55W-L$kq<kChSyQ#*!
zJl{Xha+T9>h%HH{ghk#!7!L-z*0JK*MO9FDfl3pO?EWF{?cj8u+A}!34&kDUNfO#U
zc3I^LC*klGxI}0bBJg=3@whf99C|tNz0ZS`x^332kH=CXeTz;%y%M5))cLOB;b(VF
zFiF~@*rSTG?S<=)Gk?8&^x@gwh0KWj1?regLV)wy`@sv}?enM!X}liV_}#h$r$1&2
z^_}N8z1K66zejkz;r8j*_k%kO8M?`BJ%+amZPk9fWRp{7d!2jaz7bSqeL{@7<k8OB
zxwRIDp56J5Y2nQr+WmASRN38m?N>!QhGz?PopUz5pEJ5%PLSE?vR&DbQ9z){&aM=`
zJkA8x5jV*fT7E&`7z6^UNal?Y<>XtZU*F=auj2H5^{0opUU4HKZzuk_4vRrERl(K{
zRDp6Co1%JyM<HGsQyh?58Dz#X0wYMM&Tott8dHvm7IY4d6BI+QY+z1#eRx*SSe>6t
z(ua0LcwQU)Rv4zgG!M>Z`&JL*?-O%^N;&ppE&LHY#+hz;@rN$_MQ7qdwgM;}yj(8>
zyip%TY@Nz2HXkve4wOA-`?6Ot31VR#ssGZPxYrfyNv?)}z*fmliz9L_6cuG~`5mYM
z9oGG!n3fIalz4Y_5iSj3EQ1t!zGUbU0qF_U%kf_qs1d!5CuAUy^MXD7^X}{4pBc47
zVv3?!TeYEH5e>XQ%@l30u5tQNlq*xe<B#WHTXrK*fQ~I@VXRH7fRT7e+bCqZU_H-1
z+0pH;c7tMhyJkb!m)qo*Pw_9BByY-5OXlD1kUhWomjj!4^Rn-sk8U`PxflJcXr1A!
zOQIy1`U8BK8*2Bx4`|BB<KJfEb%E^~2w>hSZxiA4>vPq4f0_uSm%2aF)-&%o6<w~^
zKfL;Fhj%LSt!Zfau~5%uk;BYO*>isRsD!#9xiXAkf0;t6Lx0@ORlnsk+VmZ<+Vr9R
zX94ki=a;6&pg$4SC1xZ_<IeB)h7m?nobXqBz`MFUS2#-->g14d<zDn>qe3_wA!$MC
zH>vbPmkG<DXh%|OTQASW!RAgsVRV*8;Vx!Dp=aKp3a+EgOo?hwi+rmE@UdXNe;#6$
z-*1R68SQE!xL=@Ai7&Yq5mW<Su8%%cJj7O%Ut75=<cRuoNZ$mBac<>(fo?~i&s?MO
z682d2e9}%yDC~K#J^pRcZMDa2Q1_<thsDz&FxjUkLvp>Sg7*~#xG7!r<=DY~QU^P-
zsvBNVv=cjGWlf*i2F%*+APydTn05dHnCZDR+&vBPg90&UilXhFL7!U#zqXo;Wn8kc
zwl;zQ7Y=;--tqcnd^>QcLK!K(d$<H9yJ`6pV4-v$G{2SWGRakPCk{V@zFPCw7teiv
zB>I<(%;fjGdlJY_sFE``J2pdvW?cU_{ZYtNw94by^RrlFNYdNR#41!(ON8l@A&%P!
z-MP5HE8nJT{jY((DRl#k*+q?CT5fkQwoFYj4rHQ~=o6K<6g4Clgp(EIQ%$xa&g{}t
z$t(Ax4_#hho5e$3qwg{)xG8<iez$n!21L;CP#fck9QG(tL4!@DMQ9<O1e*jA;z5DA
zUua=+r1BEn5EHvwjB(l&6pZJ|VuWUGIDdgvh`tA51vpD^ytlJMUA6-`4r&$ImaFol
zV1YRee-YJx4r>%vJsjL&S#E~HzHzT+KN+403<=(Gb_RM-a}vH`^<k^pLZz7keOmC;
zA2!zzHuN>A!moK~S2JaGt;7Peb52rOmLMlTu}$+sYG&#?N~&l<@1O!;a$gx$g<5S)
zY;Js~;xs*3PeO`et?{@8hB|Tg(7X)Le(WItn7RMgLp5u@++$oYBH~5NEO7kP7SnD_
z9_aT&rZ(g^C<w<YrVxm7qkOMvYCgF~U-zxSQ+pw&Wzj_2s=VcBYP94;#}A{5btIyn
zZ<B2jycscznBNM<^0`m>TYJ+~L{Y)8dsd_Hs>y7|6ZFN!d4&1d3m5C_#DfbFu(_X-
z5(E$P6CNf?76TWkVU0e0(B+qztkH+X{ZQnMYWhnIH9z41QSHzw#ct{6?sE@#Z3s!>
zcvp^=FeAdOtW6I+V7%+&#tnJg!qkgx-*Haopc%e;Y?v;%&UpU@WO)~AvMS+v3(v3G
zG*@JWRiXM`n>|7%ZX%8_rWju9lILUNCw4&~in1$k+=BeK#N8Js0pL*fC0XE?oB!3&
zw9(bZ=i}6L$i7;+cuINy?VdD`kn&pX+MY3Hb0=V~I}<^gevDlm=}*Hk;=<KcuYidY
zW~B`YMsP-CHR<hwj&Mr{N9^_CJ4+=9Q|dMtsg2tj;ArnpXL5d2ZXk@zt)``=B_(uP
zmZt<TRJ%R8X`(`DXiypP7(+6X^wHCKkA_Ra-vrt@w6HTub7GI!?GcxDliaMW1+#7D
zRqE5|K_m(E4@hF4NXz$uO~PLO{MSJ8R@qI!ErZV(CId#{S+dEOy&YU$)_@tmHo^zo
zQ209({3_n^fx2osXEcVAzw&$>y;Nz5el_imz}x_RV@f7x)EJizhPx#pxKq<6ZnnkI
z*iMbRfk=X}kr>F>Eg7QM=dT<Vc#LIYmB_S|`MucbJg)hbwA-aKw4L8ik+fNDo*^x3
zVVvzDXA^Fd6=SLF2R0@!R>3wPdGBZJ<kJC629!#-)AIXvyu~+Qlo4ro&UG<<z<)G6
zbZ{sz9<PRw4jDhE4`D!0fp@WOwG;#6+#jy07pHT-J;B^!iZ|aA?z&$^hcm*SmFlag
z;be!`hfP+-MJhpd*_JNHcB%%mh<MBem)s$6^@q-w<$(0Np(b?z(0=s4w&8zRy^HOI
zEuMhMtFqAmr%)MHTJHdN+pqsfE}EGy2e{MoRu>p)I}qaqpnWa#Xm85mYuJPw;X-_7
znyHG5=46pY=_A7@ZVLsLt43r}uoi26Zwh$m*)F!$hCjJ0PxGqn47=@WhEZ43{P1YO
zd=c7fM29ktsmre~@cG%FUa}4V_G)<mK=J1cnl0OY1w>mKa*NV~VNdroV7Aa0pc5?j
zuudOHiuMK5B8tdzQE#+hC@3e(*Wi=th~m3QN-(+(|861LdFObTrTwbMSPmFVvC>%T
zPeYW4Mc-o)k>g3x-q(`8k7udb%TYj;8@~U*X8QU-SoW4I3m*VD;`X_R{>kaO*fJut
z6oJ&=U!C>DQ);z7#-81@-No9JL>TWUQ%V?@NuQQx1$?847=3D>wKn)-OqytSCC3!!
z3VOG0S~k|Xe+@p@tq-c$Jwr6+5V7e2H>+t;c#9RtG5Cnrs4r$IV}P4!I-beetNDD>
z`v2BX{mH4t?pHPjmE8(d_BN^QdN<J{LXcqEIqFM`+=_SY<OEoAt6lH=u}p;vndI?o
zUxPd2Esk*-#!g*H`XmDWwNmf}H+8goq2(&e+nR>g%(5tR6hzq?nwN{ThGt7TsCSn+
z1;HWp8^?oY>gkrrW~iJN!>Ye;rN<4!K5PO~>vM}TQm(g}-k%dkz<9U(9QoXLGkU}v
z=b3jRGGDWH!L?5mpGhF%eIEO15OU;0qu23AkbP_LJqubsA#uIjyHBbEM%2X%{`Tw%
zBKxSGsvje5AM392NPT|&NkUAtQ>MF%l|XwP@QvNGHNdY<pD~zyUHh18`XzuEaf>#S
z7W$=0Z>nrmwK>bKmaK<mNj{sBc2mO`I;ZAC$JIV&yAafZtb3A7io-k%x~x0tV0O`j
zW>5(z*sOiri?r_9l&kw#sYmA(cZ-3J8y2N{L25ziV4>j1M==#+3tSY+0|dLHW6HP7
z0m<f{?FEnbv`;5|32Xq>IYVlRT54duxxuK(Sq#YrYuY_fs7j|U4ttLia5>*7lLzqb
zn+i*LZPb9sn4O*XlE}HWt=oCSqH0%?XTic9t2o9RVh&_%2c3Y&=->QuZXY=^3;?pX
zY6HogpD{deKQ=Ffu6?>2tnmH4HqCm(3AqDjXzTA1a)K8(hT0bOT3MttK2WdsZcA8n
zTpDkf3h&<PvL6L|9bx*sFsdlE5PANHrl~v&z8e7SzuX2K3i{lkul!!rkKrF>5?uaa
zM31=RqXT#P0%i~QqF^#z@AyxgmB&5pCDyn6Uaz<FG_1@eZyT;D?K%kn(VwyPPbc<(
zIajmu%9?XB;BUT=8g38D^BQ38zv!t*yKhpw6Y&{p`kC7OpB@VXMRTO%fOW1EysM0b
zMZf+?&IfdkCH5g8`_yB9NV}&;vPsz5B}lT+qrnvc(dL3|_uHpsfEdE(4aSwS3EH?v
zqsAO<N1tovpmus$JTHX66l>E%uUu@qaIfuXLG8MleL?5vy&XcPYfWDkx~?8gJx;d0
zo(OBbYl4Fhzi@o~EwG~g2|D^ud;9<J1k&w$7`w-9U-0VRteBDI4I(XRF(%Zo)!XAa
ziLlVVr21XM-xGC0zi?^M^A*>)Mf*><xSJ4%Wkz$X%x)GnLLkR@^l`P2nSkrcb)mae
zU${3v;=4Wek$I|nPRmr~b5&NV?o6*_J2#|vUI5!CAB=@b(VZHOJ+<xIyXm%|Ct`*g
zoA;;}J0UvL^3R!hGHLr7m|D;fF#rArD`Gm>&y3)}WJpP#{_<xR0$fwaU-0v|S`dbj
zCbA0mi2JqZ!J}4O?N^!5#Y=M=$x~kfm>(QzoLL9pZwYqq;)@en4b2C2!SE?e1hXy|
z%}^glsqYln=k3O&tK%MIIfIc$oQ-kg+t9t<=M-YgvNe6Jd$<OT9$<m}y@TL{v*yme
zUAU5Caja})oXKH*$xvGqScIh9E%c3Z&^+&Y|3Xelde5epd)lRHAPcHN%eOM09bs8g
z+kB-9SFby9k<|qPb}0NiA{zfL8SSFE=L{9;Xx^@SFbI#qFx<LBby!Aw?o`QIYB?I8
zVk&+Us}EM(m8=;TU*6O%NR~_fZnUHME-*Z`r?xH3D&b=o<zvt{55tT-cV?FpXQq<O
z$hxq$z5^LD@4qU@X8s*B>LX{FwAG-E)Zv_6@#Pl1d+u=aYulTCPh$p`W*`vZ0$C_6
zRWOX*v7>vu`odVTwYG6AxA{;)Z_MMG!R%hH{bFlj$#GvnZx>#Zzoj;vQ}^x{-opIM
zucEc>B-1io557@b3K0osp$eSiScNW~4FL>lu_;-4{AYKNfpck~)_is<3}$7HkuIHC
zQ_V`yI{OKc-l=PC;c=%HqYi@APoB>b#jf>dWSo=5a@rBqvb2%CIBx$MMF&Q+8<@!4
zEU0b2yuc)J+7ioo2RY+8dVy?rRJ+C=e9=oVun`v2_fW$yj0$d{k$9+Jp%6^WSa;2c
z1;r%t1TDJQ(NYNKaN0|BcKKj-t$q5Pj=HAR-}T6!e5oZSXGj*^===hDhSdj3WK0TE
zyfMz@MZHRyrz@!A(sy6H1?AC3?y*kp&yUfMH?yhPJD)l4HkNt9@+Lx@vOq&zmqHx$
z6W^#cJ7~gLqtSV2H~wf+v1KWQeqt%FFp^B~<QOf(L$4fsSE{+^;^uQP@*_I{g(xjk
zrN85FSmCcsA5Tpd4wjyO2Udknb>*^q&g`njb-B9Jzn`*Nk}*6k-lNUBGsc;yt3{G&
zAnXZdLm-7DIV2d>^%op>rn>ITb~jUkoa8vZ)WCUHRV#jjFJjWf&Vu)LTv9cOlT;1j
z2(Fsdp#-~iZGGn=qP!d8WC3bBSPbp^+=2wGV^DDOi8qfy(#P417!Ot=JyNhW+Y<6I
zS3C`Ys1LnoKJ!1jGVVxsS>%UL6`(+o*dyMUQUYnk36OAM`#4_A3`r1#5646wp%VPf
zOg~9Lz)*wW!V(%fEt8X?F<H*jr0NYJ%lcw{8*XIZX2tRCQZJXpJ29;is}uPo=$nh%
zqGRMQ@BAMKqVl!r<KcmDxpHT-c=x3frwCy`l#nSsb_|f?I>sEx=(~*?n(iLxQjGPy
z271>@agncav9|xx&wc8YKwLH&cpY;*2V@KE1-YGSl6JhvUvRy!WOG!WufZAn)Pl*d
zPuuYgQ0$quSwg*~dN?NeNuptMptO)E*VVa23~Vz5;&_33E`T+kcb8rZI$E2*X(HN9
zBW89>V$$kdT)a`PSBhD#mxGGK^|Z2lFS?tfPlp@bjXqW5d5ZgPDyDW@;(<VTP2}XL
zJS8LWF3KKpiL0|HPtzyfg`iv_L#eKaa=qfB>eL#)_%~oVpH_$wxK@DNi0DxVnDRQ$
z(?IB{3apH@PqJo{ilvfs4Oxa>JhH3{hWRB@-)3Zc>N05pv`<|BfOKR;?2BDgj;dy@
zGTyqGD}ab{?(HY_@5G|6Hn%;^jYo&3$+DZ0>9Zfl&wVXq89`53nLg%LuKldkbO~yX
zU=I#<(jWGa&BG*5dd^6+k{yrDEXArisDS5UrXWA?>O52m(2=NooZ3WzzvoLgA~_vI
zP&Y%)tBjj*LGVw%-;bIurFT}T{mOVL&|FyHx2yVKBULQ|ENeqdJ5A?%F@0F4a?XBF
zCgPS4nce&ftWB$<@FiE=1@}M~!X#n1%+y>_@v9q_;+G5dc?j2THoCz9yPU>1qi|O>
zaDz>+RJ+AL(Bj{$_JSrgy$Hf1bhr>av6JjU5W;SH8`kE>KVCZB+;Q8AH?-iVJl-4H
z8b2bGEI*Ynwj81Qn3DapvHBR3*$EV@^<;mK8e4UTL0PNE$CX&u#5#%~adYyPxv6WJ
zVB}@9pIEF+ASkv0o4#$!Z=!*Gy8Y+*<X3@R6B8icHGnEoTXLZa@248Ch?WF#zA0t-
zx>N-X8I${)WA`D^*INp3hS<Dabl&WVHnO+;{Ik78Cy<VmERB;^BN8q0Q2Xh}$F=gy
zwIPp4EtV;0mF3ts{eGvUL>fZEF8J---@-;_Qu{aLV2?KBqxNySi&8c-0bpZ<v=?Ae
z2NVW=DpauUpm~tOKK`hs@Igupd0H9s>#0vj0gFfsS}k*cbMympU~G$zUhl1Q^EG=x
zn4yh2d@qB=)WuHsI>#~6JgPwW3|s6%(Ck<p17-0^AUp;J#$#1PMuJEYCKi2Aj;_{I
zetDuZl2C{9Tb0U1K?8fRxnVyA6boJ<QZn*`1gH6P^J-MU$PTIsTXkh#AKA9+k#Ip%
zB5e8Ll2PuC4l=)yC|aHR^pAs>$6Jn|@Dt^dB?co8pEL^0Yb0qikqYVJ%d0PX5xYe5
zc;p@vi2SmS_x;;N^e$zfGx@f|K93(gfnW0e`oh{;ZxbNUJN||2wX_`$xz&95BA{8>
zI_w(oz5!%7&QWE5xf0c(B71EkRS)?6a*NEZEhD)HzuX4VSaV48rB*!1ZaJ+>c1d@i
zesgU$8;xCL#@!q@dv(k*T-HZ%>N|>3KFFpN96Z$T1&Vv6)Eb72{P;vkVde<(!+cz)
zf6pWBlU6O?GmBKD<Pq2k5RGtsa#iwZTf=g}7;;sl_`7U%Qe7{X*^2K(!)jmAk1BEv
z#n;|Rk!!{AC|FeF4bYwNszXc8r&^vl%Q{-iRp?^jcgC|eeC5A%=*11+^CWv>-om2}
zHHj$H4At*#%o<I~Tb+i?J6~Yoq?oG3-lgAMQnmkwUU*$-R+m(a3iJY>`?)Rn_E~W+
zuEI}Jeq;!(;<SpxNfO_?D)NWMLqPxhD1fM7M?778HGm=@?sSj??Q0@_7{OI=t-o!Z
zYlfMFct+OYF54$xgUit?DtavXdLK#_$6LHU?o*v|d%j;+Rxh6m%VfiVl1dTqO7Vow
zjY~G^oXn`MuI>Ni`P?6#t!H)o>%<LNK7Y+QKi=8{h~o{XL29u28w3*2al!!Osr$D(
zJ%grfR$tz^mcRF@x>nT67A1@hew3``VO=vAIW{x=PpZ-R8T1YcI!cs#`l3CvS!6Qy
z>7IY(CqWg-cMYexDkWdbQbY5jI#BKpL0W&T<6mde+9N+&u0FS&1GZ};1iX=4dSLa$
ze9oY|1r$ma)<|r`)okx3q~i;a1c~o&shI4EBO=iCCCE&WkXKvo2c3@)y;OCpR5j4L
zwVu9u_v7$>TzXm*4t+1SLxY|R0<QEbC3SHrk7m%lPzs_Z1j;7{zQ@_vw63U|Kf4+T
z<`mcj5`XRZN!Hc#^A{+fCWTasmtM5h_B64bBB|6w3fMif0*A9$1NvR21|YCu-5Lnu
z?D(<B6`;iW5BfTO6OT#;X7~Qaam!v??@vFVoRV_jcV2v}wzA>SQ==i}HRjC(y<S7$
zJ1KRZ6kO;HWRilNjN}i2T>USG$U=?ne|kht-9cb=)$z)aR2fwt29|4k#jLdDt1q`z
z7ETWrPAkZ$x-*kQl+mAN2u@7Vn*T`Yb8=1>TKcd5Gt190uHvkR{AuRneH6TkfNlnU
zUm60Iv<CTXB3ep?ZidYD4<OkQ#_aHj{!0)DVz7DjAxjiE9|79|(lART>A{IH)gZeP
zRxP=A0c<%k-@y#EZ58QVTx;M2g0e3ca@ELe3!db7(-^zFV1ZmaJh9P}Kl5&()GOnJ
zU?mge0P&jh8kF(qVvH?B_{rdk2dN#+qLE24$Tl(pmp=x6l1hYNR?HZXN-85#>en8W
zLA?TXw`&VEeoU+1&(YYdR~ETahylAH+J0||SqC}}?lSAj?2m+e_ThF($6<`}zqOj|
z^B|sySD3j<;Aczh;ALZM9w3VP_ih+2nd&o}OX!{*Sr8rQ_=}bmaGej@n5u&|EeARz
z+@{h$H=F#?;`5jScF`CPO^s=uT#eiER~<eBmGQEFbls@AtG(s5F4YeIc#}C?W}LKH
z@u^yYnigO0>+5T0%Zpx=jovi5%pAK}FHTsGNP1USbbwDg!DN2*|E~N&&ZQve!_tx*
zriwyhDQ0KnQ=0ywWyZRt;l(kuNou?e^FSffph}0e_m7sD-*)j^n-66<XtoM${ov!9
zCjO!eDQqce|LW4rgtQ3?*`%!-(ZO(TXi%*pefW6aJG{d%kFga8_DRKs7<zVNQ4(Uy
zfJBw*<2WRYNRsr=O@c^QD<e?o{=~!4#=14TM5WetRN>?)eV9s>oqb^BaOh|bYeqGE
zIEie(gsME|sG|fMR(7NfJ?m@C&35V=)#&2~^t^p|Io2{Mebg=I;CXh~W7<gxWe%;c
z1i_jG)cxd}pTcOyahGH3NE9^qL8%xB7uR1O$fQq&i_e{6H<7SHaErnB{wH00;3DQJ
a61CIKl@Gi>-jcR20`@j8)>Vg3U-~~R){C0}


From 59a5022722487a3de4513a0f15e3a7d511e36379 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Thu, 19 May 2022 16:21:48 +0100
Subject: [PATCH 44/63] Add ability to write VTK XML files from Catalyst

This writes the created vtkOverlappingAMR object to a file. It's mainly
for debugging but I guess in theory it allows a user to save the state
of the grid at any timestep on any level rather than just a whole
coarsest level timestep.
---
 Source/GRChomboCore/ChomboParameters.hpp |  3 +++
 Source/GRChomboCore/GRAMR.cpp            |  5 ++--
 Source/GRChomboCore/GRAMR.hpp            |  2 +-
 Source/GRChomboCore/SetupFunctions.hpp   |  3 ++-
 Source/Insitu/CatalystAdaptor.cpp        | 32 +++++++++++++++++++++---
 Source/Insitu/CatalystAdaptor.hpp        | 12 +++++++--
 Source/Insitu/Make.insitu                |  2 +-
 7 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index 464eee2ee..e93582433 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -163,6 +163,7 @@ class ChomboParameters
                 num_catalyst_vars);
             pp.load("abort_on_catalyst_error", abort_on_catalyst_error, false);
             pp.load("catalyst_remove_ghosts", catalyst_remove_ghosts, false);
+            pp.load("catalyst_write_files", catalyst_write_files, false);
         }
 #endif
     }
@@ -602,6 +603,8 @@ class ChomboParameters
     std::vector<std::pair<int, VariableType>> catalyst_vars;
     bool abort_on_catalyst_error;
     bool catalyst_remove_ghosts;
+    // write VTK XML files from data passed to Catalyst
+    bool catalyst_write_files;
 #endif
 
   protected:
diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp
index da985894b..f171cb20d 100644
--- a/Source/GRChomboCore/GRAMR.cpp
+++ b/Source/GRChomboCore/GRAMR.cpp
@@ -68,7 +68,8 @@ void GRAMR::setup_catalyst(
     bool a_activate_catalyst, const std::vector<std::string> &a_python_scripts,
     const std::string &a_output_path,
     const std::vector<std::pair<int, VariableType>> &a_vars,
-    bool a_abort_on_catalyst_error, bool a_remove_ghosts, int a_verbosity)
+    bool a_abort_on_catalyst_error, bool a_remove_ghosts, bool a_write_files,
+    int a_verbosity)
 {
     m_activate_catalyst = a_activate_catalyst;
     if (m_activate_catalyst)
@@ -77,7 +78,7 @@ void GRAMR::setup_catalyst(
         m_insitu = new CatalystAdaptor;
         m_insitu->initialise(this, a_python_scripts, a_output_path, a_vars,
                              a_abort_on_catalyst_error, a_remove_ghosts,
-                             a_verbosity);
+                             a_write_files, a_verbosity);
     }
 }
 #endif
\ No newline at end of file
diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp
index b48007b5c..a24278779 100644
--- a/Source/GRChomboCore/GRAMR.hpp
+++ b/Source/GRChomboCore/GRAMR.hpp
@@ -88,7 +88,7 @@ class GRAMR : public AMR
                         const std::string &a_output_path,
                         const std::vector<std::pair<int, VariableType>> &a_vars,
                         bool a_abort_on_catalyst_error, bool a_remove_ghosts,
-                        int a_verbosity);
+                        bool a_write_files, int a_verbosity);
 #endif
 
 #ifdef USE_CATALYST
diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 2a5134aa9..1fc02e50b 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -244,7 +244,8 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
         chombo_params.activate_catalyst, chombo_params.catalyst_scripts,
         chombo_params.output_path, chombo_params.catalyst_vars,
         chombo_params.abort_on_catalyst_error,
-        chombo_params.catalyst_remove_ghosts, chombo_params.catalyst_verbosity);
+        chombo_params.catalyst_remove_ghosts,
+        chombo_params.catalyst_write_files, chombo_params.catalyst_verbosity);
 #endif
 }
 
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index ed5344259..4913e846c 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -15,10 +15,12 @@ CatalystAdaptor::CatalystAdaptor(
     GRAMR *a_gr_amr_ptr, const std::vector<std::string> &a_python_scripts,
     const std::string &a_output_path,
     const std::vector<std::pair<int, VariableType>> &a_vars,
-    bool a_abort_on_catalyst_error, bool a_remove_ghosts, int a_verbosity)
+    bool a_abort_on_catalyst_error, bool a_remove_ghosts, bool a_write_files,
+    int a_verbosity)
 {
     initialise(a_gr_amr_ptr, a_python_scripts, a_output_path, a_vars,
-               a_abort_on_catalyst_error, a_remove_ghosts, a_verbosity);
+               a_abort_on_catalyst_error, a_remove_ghosts, a_write_files,
+               a_verbosity);
 }
 
 CatalystAdaptor::~CatalystAdaptor()
@@ -33,7 +35,8 @@ void CatalystAdaptor::initialise(
     GRAMR *a_gr_amr_ptr, const std::vector<std::string> &a_python_scripts,
     const std::string &a_output_path,
     const std::vector<std::pair<int, VariableType>> &a_vars,
-    bool a_abort_on_catalyst_error, bool a_remove_ghosts, int a_verbosity)
+    bool a_abort_on_catalyst_error, bool a_remove_ghosts, bool a_write_files,
+    int a_verbosity)
 {
     // don't initalise twice
     if (m_initialised)
@@ -53,6 +56,7 @@ void CatalystAdaptor::initialise(
     m_requested_diagnostic_vars.fill(false);
     m_abort_on_catalyst_error = a_abort_on_catalyst_error;
     m_remove_ghosts = a_remove_ghosts;
+    m_write_files = a_write_files;
 
     // Initialise VTK CP Processor
     if (!m_proc_ptr)
@@ -415,6 +419,23 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
     }
 }
 
+void CatalystAdaptor::write_vtk_grid(unsigned int a_timestep)
+{
+    vtkNew<vtkXMLPUniformGridAMRWriter> file_writer;
+
+    // make filename
+    char timestep_cstr[7];
+    std::sprintf(timestep_cstr, "%06d.", a_timestep);
+    std::string filename = m_base_file_name;
+    filename += timestep_cstr;
+    filename += file_writer->GetDefaultFileExtension();
+
+    // set data and write
+    file_writer->SetInputData(m_vtk_grid_ptr);
+    file_writer->SetFileName(filename.c_str());
+    file_writer->Write();
+}
+
 void CatalystAdaptor::coprocess(double a_time, unsigned int a_timestep)
 {
     pout() << "CatalystAdaptor::coprocess at time " << a_time << " and step "
@@ -430,6 +451,11 @@ void CatalystAdaptor::coprocess(double a_time, unsigned int a_timestep)
         auto input_data_description =
             data_description->GetInputDescriptionByName("input");
         add_vars(input_data_description);
+
+        if (m_write_files)
+        {
+            write_vtk_grid(a_timestep);
+        }
         // vtkNew<vtkOverlappingAMR> stripped_vtk_grid;
         // vtkAMRUtilities::StripGhostLayers(m_vtk_grid_ptr, stripped_vtk_grid);
         input_data_description->SetGrid(m_vtk_grid_ptr);
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index 1c37bbe2b..05cba2221 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -30,6 +30,7 @@
 #include <vtkNew.h>
 #include <vtkOverlappingAMR.h>
 #include <vtkUniformGrid.h>
+#include <vtkXMLPUniformGridAMRWriter.h>
 
 // Chombo namespace
 #include "UsingNamespace.H"
@@ -50,7 +51,7 @@ class CatalystAdaptor
                     const std::string &a_output_path,
                     const std::vector<std::pair<int, VariableType>> &a_vars,
                     bool a_abort_on_catalyst_error, bool a_remove_ghosts,
-                    int a_verbosity);
+                    bool a_write_files, int a_verbosity);
 
     // destructor
     ~CatalystAdaptor();
@@ -61,7 +62,7 @@ class CatalystAdaptor
                     const std::string &a_output_path,
                     const std::vector<std::pair<int, VariableType>> &a_vars,
                     bool a_abort_on_catalyst_error, bool a_remove_ghosts,
-                    int a_verbosity);
+                    bool a_write_files, int a_verbosity);
     void finalise();
 
     // do Catalyst processing
@@ -79,6 +80,9 @@ class CatalystAdaptor
     // send variables to catalyst
     void add_vars(vtkCPInputDataDescription *a_input_data_desc);
 
+    // write VTK grid to a file
+    void write_vtk_grid(unsigned int a_timestep);
+
     // directly passes the FAB pointer to the VTK array
     vtkDoubleArray *fab_to_vtk_array(FArrayBox &a_fab, int a_var,
                                      const std::string &a_name);
@@ -99,6 +103,10 @@ class CatalystAdaptor
     bool m_remove_ghosts = false;
     GRAMR *m_gr_amr_ptr = nullptr;
 
+    // file writing parameters
+    bool m_write_files = false;
+    std::string m_base_file_name = "Catalyst_VTK_grid_";
+
     // variables to pass to Catalyst set by GRChombo parameter
     std::vector<std::pair<int, VariableType>> m_vars;
 
diff --git a/Source/Insitu/Make.insitu b/Source/Insitu/Make.insitu
index 5e48cf9c9..dc5c23830 100644
--- a/Source/Insitu/Make.insitu
+++ b/Source/Insitu/Make.insitu
@@ -20,7 +20,7 @@ ifeq ($(USE_CATALYST),TRUE)
   # These components are correct in v5.9 and v5.10 but may differ in other 
   # versions. The paraview-config command may also change
   PARAVIEW_COMPONENT_FLAGS := -c PythonCatalyst ${XTRA_PARAVIEW_COMPONENT_FLAGS}
-  VTK_COMPONENT_FLAGS := -v CommonDataModel -v PythonUsed ${XTRA_VTK_COMPONENT_FLAGS}
+  VTK_COMPONENT_FLAGS := -v CommonDataModel -v PythonUsed -v IOParallelXML ${XTRA_VTK_COMPONENT_FLAGS}
   PARAVIEW_CONFIG_EXEC := $(PARAVIEW_DIR)/bin/paraview-config
   cxxcppflags += $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --cppflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
   XTRALDFLAGS += -rdynamic $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --ldflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))

From 5cedc267daa8b63a9009cfea98aa2c3484a41cf4 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Thu, 19 May 2022 16:25:04 +0100
Subject: [PATCH 45/63] Initialize `vtkUniformGrid`s from `vtkAMRBox`s

This shifts the set `origin` of the `vtkUniformGrid`s from the
`global_origin` at (0,0,0) (in Chombo coordinates) to that of the low
corner of the box. This seems to disagree with what is done in the
ParaView Catalyst CxxOverlappingAMRExample but makes the outputted VTK
XML files appear more sensible when loaded in ParaView.
---
 Source/Insitu/CatalystAdaptor.cpp | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 4913e846c..33ba1fd21 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -217,20 +217,21 @@ void CatalystAdaptor::build_vtk_grid()
             {
                 vtkNew<vtkUniformGrid> vtk_uniform_grid_ptr;
 
-                vtk_uniform_grid_ptr->SetOrigin(origin_global);
-                vtk_uniform_grid_ptr->SetSpacing(dx_arr);
-                vtk_uniform_grid_ptr->SetExtent(
-                    small_ghosted_end[0], big_ghosted_end[0],
-                    small_ghosted_end[1], big_ghosted_end[1],
-                    small_ghosted_end[2], big_ghosted_end[2]);
-                // add the ghost cell information
-                int no_ghost[6] = {small_end[0], big_end[0],   small_end[1],
-                                   big_end[1],   small_end[2], big_end[2]};
-                bool cell_data = true;
-                vtk_uniform_grid_ptr->GenerateGhostArray(no_ghost, cell_data);
-
-                // vtk_uniform_grid_ptr->Initialize(&vtk_amr_box, origin_global,
-                //  dx_arr, ghost_vect.dataPtr());
+                // vtk_uniform_grid_ptr->SetOrigin(origin_global);
+                // vtk_uniform_grid_ptr->SetSpacing(dx_arr);
+                // vtk_uniform_grid_ptr->SetExtent(
+                //     small_ghosted_end[0], big_ghosted_end[0],
+                //     small_ghosted_end[1], big_ghosted_end[1],
+                //     small_ghosted_end[2], big_ghosted_end[2]);
+                // // add the ghost cell information
+                // int no_ghost[6] = {small_end[0], big_end[0],   small_end[1],
+                //                    big_end[1],   small_end[2], big_end[2]};
+                // bool cell_data = true;
+                // vtk_uniform_grid_ptr->GenerateGhostArray(no_ghost,
+                // cell_data);
+
+                vtk_uniform_grid_ptr->Initialize(&vtk_amr_box, origin, dx_arr,
+                                                 ghost_vect.dataPtr());
 
                 m_vtk_grid_ptr->SetDataSet(ilevel, ibox, vtk_uniform_grid_ptr);
             }

From a2bc9fa6472516ce4d684c403395f21adfd1b20f Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 25 May 2022 16:47:31 +0100
Subject: [PATCH 46/63] Refactor Catalyst parameter code

Now, all of the Catalyst parameters used in `CatalystAdaptor` are in a
`CatalystAdaptor::m_p`, a struct of type `CatalystAdaptor::params_t`.
This will make it easier to add/modify parameters. Previously multiple
functions across the `CatalystAdaptor` and `GRAMR` classes as well as a
Setup function needed to be modified every time a Catalyst parameter was
changed. Note that some parameters are used elsewhere (e.g.
`catalyst_coprocess_level` in `GRAMRLevel` and are therefore not part of
`CatalystAdaptor::params_t`.

Some parameters have been renamed in order to make them more consistent
(all Catalyst parameters are now prefixed with `catalyst_`). The Insitu
README has been updated accordingly.
---
 Examples/BinaryBH/BinaryBHLevel.cpp           |  4 +-
 Examples/BinaryBH/params_catalyst.txt         |  8 +-
 Source/GRChomboCore/ChomboParameters.hpp      | 79 +++++++++++--------
 Source/GRChomboCore/GRAMR.cpp                 | 18 ++---
 Source/GRChomboCore/GRAMR.hpp                 | 13 +--
 Source/GRChomboCore/GRAMRLevel.cpp            |  4 +-
 Source/GRChomboCore/SetupFunctions.hpp        | 10 +--
 Source/Insitu/CatalystAdaptor.cpp             | 67 ++++++----------
 Source/Insitu/CatalystAdaptor.hpp             | 47 +++++------
 Source/Insitu/README.md                       | 44 +++++------
 .../CatalystInsituTest.inputs                 |  8 +-
 11 files changed, 137 insertions(+), 165 deletions(-)

diff --git a/Examples/BinaryBH/BinaryBHLevel.cpp b/Examples/BinaryBH/BinaryBHLevel.cpp
index 5e684697b..c5e52b9ae 100644
--- a/Examples/BinaryBH/BinaryBHLevel.cpp
+++ b/Examples/BinaryBH/BinaryBHLevel.cpp
@@ -140,7 +140,7 @@ void BinaryBHLevel::specificPostTimeStep()
 
     if (m_p.activate_extraction
 #ifdef USE_CATALYST
-        || m_p.activate_catalyst
+        || m_p.catalyst_activate
 #endif
     )
     {
@@ -150,7 +150,7 @@ void BinaryBHLevel::specificPostTimeStep()
         bool calculate_weyl = at_level_timestep_multiple(min_level);
 #ifdef USE_CATALYST
         calculate_weyl |=
-            (m_p.activate_catalyst &&
+            (m_p.catalyst_activate &&
              at_level_timestep_multiple(m_p.catalyst_coprocess_level));
 #endif
         if (calculate_weyl)
diff --git a/Examples/BinaryBH/params_catalyst.txt b/Examples/BinaryBH/params_catalyst.txt
index fe55b7480..fb29eeca4 100644
--- a/Examples/BinaryBH/params_catalyst.txt
+++ b/Examples/BinaryBH/params_catalyst.txt
@@ -184,15 +184,15 @@ modes = 2 0 # l m for spherical harmonics
 #################################################
 # ParaView Catalyst parameters
 
-activate_catalyst = true
+catalyst_activate = true
 catalyst_verbosity = 2
 # catalyst_pout_prefix = catalyst_pout
 # path to directory containing catalyst scripts
 catalyst_scripts_path = catalyst_scripts
-num_catalyst_scripts = 1
+catalyst_num_scripts = 1
 # list of Catalyst scripts
 catalyst_scripts = volume_raytrace_resample_chi.py
 catalyst_coprocess_level = 1
-num_catalyst_vars = 1
+catalyst_num_vars = 1
 catalyst_vars = chi
-abort_on_catalyst_error = true
+catalyst_abort_on_error = true
diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index e93582433..851341c99 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -132,38 +132,54 @@ class ChomboParameters
                 false);
 
 #ifdef USE_CATALYST
-        pp.load("activate_catalyst", activate_catalyst, false);
-        if (activate_catalyst)
+        pp.load("catalyst_activate", catalyst_activate, false);
+        if (catalyst_activate)
         {
-            pp.load("catalyst_verbosity", catalyst_verbosity, verbosity);
+            pp.load("catalyst_verbosity", catalyst_params.verbosity, verbosity);
             pp.load("catalyst_pout_prefix", catalyst_pout_prefix,
                     std::string("catalyst_pout"));
             pp.load("catalyst_scripts_path", catalyst_scripts_path,
                     std::string());
-            pp.load("num_catalyst_scripts", num_catalyst_scripts, 1);
-            pp.load("catalyst_scripts", catalyst_scripts, num_catalyst_scripts);
-            // prepend the path to each script
+            pp.load("catalyst_num_scripts", catalyst_num_scripts, 1);
+            pp.load("catalyst_scripts", catalyst_params.python_scripts,
+                    catalyst_num_scripts);
+
+            // this is what the Catalyst working directory will be set to
+            // so extracts end up in the output_path
+            catalyst_params.output_path = output_path;
+
+            // prepend the PWD path to each script
             if (!catalyst_scripts_path.empty() &&
                 catalyst_scripts_path.back() != '/')
                 catalyst_scripts_path += "/";
-            // make each catalyst_script a full path so that Catalyst doesn't
-            // look in its working directory (set to output_path)
-            char cwd[4096];
-            getcwd(cwd, 4096);
-            std::string current_working_directory = cwd;
-            current_working_directory += "/";
-            for (auto &script : catalyst_scripts)
+            // make each catalyst_script a full path (if not already) so that
+            // Catalyst doesn't look in its working directory (set to
+            // output_path)
+            std::string prepend_directory = "";
+            if (catalyst_scripts_path.front() != '/') // not a full path
+            {
+                char cwd[4096];
+                getcwd(cwd, 4096);
+                prepend_directory = cwd;
+                prepend_directory += "/";
+            }
+            for (auto &script : catalyst_params.python_scripts)
             {
-                script =
-                    current_working_directory + catalyst_scripts_path + script;
+                if (script.front() != '/') // not a full path
+                {
+                    script = prepend_directory + catalyst_scripts_path + script;
+                }
             }
+
             pp.load("catalyst_coprocess_level", catalyst_coprocess_level, 0);
             UserVariables::load_vars_to_vector(
-                pp, "catalyst_vars", "num_catalyst_vars", catalyst_vars,
-                num_catalyst_vars);
-            pp.load("abort_on_catalyst_error", abort_on_catalyst_error, false);
-            pp.load("catalyst_remove_ghosts", catalyst_remove_ghosts, false);
-            pp.load("catalyst_write_files", catalyst_write_files, false);
+                pp, "catalyst_vars", "catalyst_num_vars", catalyst_params.vars,
+                catalyst_num_vars);
+            pp.load("catalyst_abort_on_error", catalyst_params.abort_on_error,
+                    false);
+            pp.load("catalyst_remove_ghosts", catalyst_params.remove_ghosts,
+                    false);
+            pp.load("catalyst_write_files", catalyst_params.write_files, false);
         }
 #endif
     }
@@ -506,11 +522,13 @@ class ChomboParameters
         }
 
 #ifdef USE_CATALYST
-        if (activate_catalyst)
+        if (catalyst_activate)
         {
-            for (int iscript = 0; iscript < catalyst_scripts.size(); ++iscript)
+            for (int iscript = 0;
+                 iscript < catalyst_params.python_scripts.size(); ++iscript)
             {
-                const std::string &script = catalyst_scripts[iscript];
+                const std::string &script =
+                    catalyst_params.python_scripts[iscript];
                 bool script_exists = (access(script.c_str(), R_OK) == 0);
                 const std::string script_parameter_name =
                     "catalyst_scripts[" + std::to_string(iscript) + "]";
@@ -586,25 +604,20 @@ class ChomboParameters
     bool print_progress_only_to_rank_0;
 
 #ifdef USE_CATALYST
-    bool activate_catalyst;
-    int catalyst_verbosity;
+    bool catalyst_activate;
     // prefix of filename for Catalyst log output (appended by .<rank>)
     // file will be in pout_path
     std::string catalyst_pout_prefix;
     // ignores output_path
     std::string catalyst_scripts_path;
-    int num_catalyst_scripts;
-    std::vector<std::string> catalyst_scripts;
+    int catalyst_num_scripts;
     int catalyst_coprocess_level;
     // variables to pass to Catalyst
     // only restricts vars if num_catalyst_vars > 0, otherwise all requested
     // are passed
-    int num_catalyst_vars;
-    std::vector<std::pair<int, VariableType>> catalyst_vars;
-    bool abort_on_catalyst_error;
-    bool catalyst_remove_ghosts;
-    // write VTK XML files from data passed to Catalyst
-    bool catalyst_write_files;
+    int catalyst_num_vars;
+
+    CatalystAdaptor::params_t catalyst_params;
 #endif
 
   protected:
diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp
index f171cb20d..282439b20 100644
--- a/Source/GRChomboCore/GRAMR.cpp
+++ b/Source/GRChomboCore/GRAMR.cpp
@@ -12,7 +12,7 @@ void GRAMR::conclude()
 {
     AMR::conclude();
 #ifdef USE_CATALYST
-    if (m_activate_catalyst)
+    if (m_catalyst_activate)
     {
         m_insitu->finalise();
     }
@@ -64,21 +64,15 @@ void GRAMR::fill_multilevel_ghosts(const VariableType a_var_type,
 }
 
 #ifdef USE_CATALYST
-void GRAMR::setup_catalyst(
-    bool a_activate_catalyst, const std::vector<std::string> &a_python_scripts,
-    const std::string &a_output_path,
-    const std::vector<std::pair<int, VariableType>> &a_vars,
-    bool a_abort_on_catalyst_error, bool a_remove_ghosts, bool a_write_files,
-    int a_verbosity)
+void GRAMR::setup_catalyst(bool a_catalyst_activate,
+                           const CatalystAdaptor::params_t &a_catalyst_params)
 {
-    m_activate_catalyst = a_activate_catalyst;
-    if (m_activate_catalyst)
+    m_catalyst_activate = a_catalyst_activate;
+    if (m_catalyst_activate)
     {
         pout() << "GRAMR::setup_catalyst" << std::endl;
         m_insitu = new CatalystAdaptor;
-        m_insitu->initialise(this, a_python_scripts, a_output_path, a_vars,
-                             a_abort_on_catalyst_error, a_remove_ghosts,
-                             a_write_files, a_verbosity);
+        m_insitu->initialise(this, a_catalyst_params);
     }
 }
 #endif
\ No newline at end of file
diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp
index a24278779..be9af3d09 100644
--- a/Source/GRChomboCore/GRAMR.hpp
+++ b/Source/GRChomboCore/GRAMR.hpp
@@ -38,9 +38,6 @@ class GRAMRLevel;
 // Forward declaration for AMRInterpolator
 template <typename InterpAlgo> class AMRInterpolator;
 
-// Forward declaration for CatalystAdaptor
-class CatalystAdaptor;
-
 class GRAMR : public AMR
 {
   private:
@@ -83,16 +80,12 @@ class GRAMR : public AMR
         const int a_max_level = std::numeric_limits<int>::max()) const;
 
 #ifdef USE_CATALYST
-    void setup_catalyst(bool a_activate_catalyst,
-                        const std::vector<std::string> &a_python_scripts,
-                        const std::string &a_output_path,
-                        const std::vector<std::pair<int, VariableType>> &a_vars,
-                        bool a_abort_on_catalyst_error, bool a_remove_ghosts,
-                        bool a_write_files, int a_verbosity);
+    void setup_catalyst(bool a_catalyst_activate,
+                        const CatalystAdaptor::params_t &a_catalyst_params);
 #endif
 
 #ifdef USE_CATALYST
-    bool m_activate_catalyst = false;
+    bool m_catalyst_activate = false;
     CatalystAdaptor *m_insitu;
 #endif
 };
diff --git a/Source/GRChomboCore/GRAMRLevel.cpp b/Source/GRChomboCore/GRAMRLevel.cpp
index c7dc4abf2..c0089692e 100644
--- a/Source/GRChomboCore/GRAMRLevel.cpp
+++ b/Source/GRChomboCore/GRAMRLevel.cpp
@@ -194,7 +194,7 @@ void GRAMRLevel::postTimeStep()
 void GRAMRLevel::preCatalystCoProcess()
 {
     // If we're not passing ghosts to Catalyst, there's no point in filling them
-    if (m_p.catalyst_remove_ghosts)
+    if (m_p.catalyst_params.remove_ghosts)
         return;
 
     if (std::abs(m_time - m_restart_time) < m_gr_amr.timeEps())
@@ -243,7 +243,7 @@ void GRAMRLevel::preCatalystCoProcess()
 
 void GRAMRLevel::catalystCoProcess()
 {
-    if (m_p.activate_catalyst)
+    if (m_p.catalyst_activate)
     {
         if (at_level_timestep_multiple(m_p.catalyst_coprocess_level))
         {
diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 1fc02e50b..582815cfd 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -233,19 +233,15 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
     }
     vtkLogger::LogToFile(
         catalyst_log_file.c_str(), vtk_logger_file_mode,
-        vtkLogger::ConvertToVerbosity(chombo_params.catalyst_verbosity));
+        vtkLogger::ConvertToVerbosity(chombo_params.catalyst_params.verbosity));
     // Only write VTK stderr messages if there is an error
 #else
     vtkLogger::SetStderrVerbosity(
         static_cast<vtkLogger::Verbosity>(chombo_params.verbosity));
 #endif
     vtkLogger::Init();
-    gr_amr.setup_catalyst(
-        chombo_params.activate_catalyst, chombo_params.catalyst_scripts,
-        chombo_params.output_path, chombo_params.catalyst_vars,
-        chombo_params.abort_on_catalyst_error,
-        chombo_params.catalyst_remove_ghosts,
-        chombo_params.catalyst_write_files, chombo_params.catalyst_verbosity);
+    gr_amr.setup_catalyst(chombo_params.catalyst_activate,
+                          chombo_params.catalyst_params);
 #endif
 }
 
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 33ba1fd21..f0462192a 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -11,16 +11,9 @@
 
 CatalystAdaptor::CatalystAdaptor() {}
 
-CatalystAdaptor::CatalystAdaptor(
-    GRAMR *a_gr_amr_ptr, const std::vector<std::string> &a_python_scripts,
-    const std::string &a_output_path,
-    const std::vector<std::pair<int, VariableType>> &a_vars,
-    bool a_abort_on_catalyst_error, bool a_remove_ghosts, bool a_write_files,
-    int a_verbosity)
+CatalystAdaptor::CatalystAdaptor(GRAMR *a_gr_amr_ptr, const params_t &a_params)
 {
-    initialise(a_gr_amr_ptr, a_python_scripts, a_output_path, a_vars,
-               a_abort_on_catalyst_error, a_remove_ghosts, a_write_files,
-               a_verbosity);
+    initialise(a_gr_amr_ptr, a_params);
 }
 
 CatalystAdaptor::~CatalystAdaptor()
@@ -31,12 +24,7 @@ CatalystAdaptor::~CatalystAdaptor()
     }
 }
 
-void CatalystAdaptor::initialise(
-    GRAMR *a_gr_amr_ptr, const std::vector<std::string> &a_python_scripts,
-    const std::string &a_output_path,
-    const std::vector<std::pair<int, VariableType>> &a_vars,
-    bool a_abort_on_catalyst_error, bool a_remove_ghosts, bool a_write_files,
-    int a_verbosity)
+void CatalystAdaptor::initialise(GRAMR *a_gr_amr_ptr, const params_t &a_params)
 {
     // don't initalise twice
     if (m_initialised)
@@ -51,26 +39,23 @@ void CatalystAdaptor::initialise(
         return;
     }
     m_gr_amr_ptr = a_gr_amr_ptr;
-    m_vars = a_vars;
+    m_p = a_params;
     m_requested_evolution_vars.fill(false);
     m_requested_diagnostic_vars.fill(false);
-    m_abort_on_catalyst_error = a_abort_on_catalyst_error;
-    m_remove_ghosts = a_remove_ghosts;
-    m_write_files = a_write_files;
 
     // Initialise VTK CP Processor
     if (!m_proc_ptr)
     {
         m_proc_ptr = vtkCPProcessor::New();
         int m_proc_ptr_initialization_success;
-        if (a_output_path.empty())
+        if (m_p.output_path.empty())
         {
             m_proc_ptr_initialization_success = m_proc_ptr->Initialize();
         }
         else
         {
             m_proc_ptr_initialization_success =
-                m_proc_ptr->Initialize(a_output_path.c_str());
+                m_proc_ptr->Initialize(m_p.output_path.c_str());
         }
         std::string error_msg = "Failed to initialize vtkCPProcessor in "
                                 "CatalystAdaptor::initialise";
@@ -82,7 +67,7 @@ void CatalystAdaptor::initialise(
     }
 
     // Create Python script pipeline and add it to the VTK CP Processor
-    for (const std::string &script : a_python_scripts)
+    for (const std::string &script : m_p.python_scripts)
     {
 #if PARAVIEW_VERSION_HERE >= PARAVIEW_VERSION_TEST(5, 9, 0)
         auto pipeline =
@@ -104,8 +89,6 @@ void CatalystAdaptor::initialise(
         catalyst_error_or_warning(add_pipeline_success, add_pipeline_fail_msg);
     }
 
-    m_verbosity = a_verbosity;
-
     m_initialised = true;
 }
 
@@ -126,7 +109,7 @@ void CatalystAdaptor::finalise()
 
 void CatalystAdaptor::build_vtk_grid()
 {
-    if (m_verbosity)
+    if (m_p.verbosity)
     {
         pout() << "CatalystAdaptor::build_vtk_grid" << std::endl;
     }
@@ -156,7 +139,7 @@ void CatalystAdaptor::build_vtk_grid()
         gramrlevels[0]->getLevelData().ghostVect();
     const double coarsest_dx = gramrlevels[0]->get_dx();
     RealVect ghosted_origin_global_vect = coarsest_ghost_vect;
-    ghosted_origin_global_vect *= (!m_remove_ghosts) ? -coarsest_dx : 0;
+    ghosted_origin_global_vect *= (!m_p.remove_ghosts) ? -coarsest_dx : 0;
     m_vtk_grid_ptr->SetOrigin(ghosted_origin_global_vect.dataPtr());
 
     // now add all the boxes
@@ -193,7 +176,7 @@ void CatalystAdaptor::build_vtk_grid()
 
             // ghost_vect takes into account m_remove_ghosts
             const IntVect &ghost_vect =
-                (m_remove_ghosts) ? IntVect::Zero : level_data.ghostVect();
+                (m_p.remove_ghosts) ? IntVect::Zero : level_data.ghostVect();
             ghosted_box.grow(ghost_vect);
             const IntVect &small_ghosted_end = ghosted_box.smallEnd();
             const IntVect &big_ghosted_end = ghosted_box.bigEnd();
@@ -254,7 +237,7 @@ void CatalystAdaptor::build_vtk_grid()
 
 void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
 {
-    if (m_verbosity)
+    if (m_p.verbosity)
     {
         pout() << "CatalystAdaptor::add_vars" << std::endl
                << "CatalystAdaptor Requested variables: ";
@@ -264,15 +247,15 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
     {
         m_requested_evolution_vars[ivar] = a_input_data_desc->IsFieldNeeded(
             UserVariables::variable_names[ivar].c_str(), vtkDataObject::CELL);
-        if (m_vars.size() > 0)
+        if (m_p.vars.size() > 0)
         {
             bool pass_var =
-                !(std::find(m_vars.begin(), m_vars.end(),
+                !(std::find(m_p.vars.begin(), m_p.vars.end(),
                             std::make_pair(ivar, VariableType::evolution)) ==
-                  m_vars.end());
+                  m_p.vars.end());
             m_requested_evolution_vars[ivar] &= pass_var;
         }
-        if (m_verbosity && m_requested_evolution_vars[ivar])
+        if (m_p.verbosity && m_requested_evolution_vars[ivar])
             pout() << UserVariables::variable_names[ivar] << " ";
     }
     for (int ivar = 0; ivar < NUM_DIAGNOSTIC_VARS; ++ivar)
@@ -280,18 +263,18 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
         m_requested_diagnostic_vars[ivar] = a_input_data_desc->IsFieldNeeded(
             DiagnosticVariables::variable_names[ivar].c_str(),
             vtkDataObject::CELL);
-        if (m_vars.size() > 0)
+        if (m_p.vars.size() > 0)
         {
             bool pass_var =
-                !(std::find(m_vars.begin(), m_vars.end(),
+                !(std::find(m_p.vars.begin(), m_p.vars.end(),
                             std::make_pair(ivar, VariableType::diagnostic)) ==
-                  m_vars.end());
+                  m_p.vars.end());
             m_requested_diagnostic_vars[ivar] &= pass_var;
         }
-        if (m_verbosity && m_requested_diagnostic_vars[ivar])
+        if (m_p.verbosity && m_requested_diagnostic_vars[ivar])
             pout() << DiagnosticVariables::variable_names[ivar] << " ";
     }
-    if (m_verbosity)
+    if (m_p.verbosity)
         pout() << std::endl;
 
     vtkAMRInformation *amr_info = m_vtk_grid_ptr->GetAMRInfo();
@@ -378,7 +361,7 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
                     if (m_requested_evolution_vars[ivar])
                     {
                         vtkDoubleArray *vtk_double_arr;
-                        if (!m_remove_ghosts)
+                        if (!m_p.remove_ghosts)
                         {
                             vtk_double_arr = fab_to_vtk_array(
                                 evolution_fab, ivar,
@@ -399,7 +382,7 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
                     if (m_requested_diagnostic_vars[ivar])
                     {
                         vtkDoubleArray *vtk_double_arr;
-                        if (!m_remove_ghosts)
+                        if (!m_p.remove_ghosts)
                         {
                             vtk_double_arr = fab_to_vtk_array(
                                 diagnostic_fab, ivar,
@@ -427,7 +410,7 @@ void CatalystAdaptor::write_vtk_grid(unsigned int a_timestep)
     // make filename
     char timestep_cstr[7];
     std::sprintf(timestep_cstr, "%06d.", a_timestep);
-    std::string filename = m_base_file_name;
+    std::string filename = m_p.base_file_name;
     filename += timestep_cstr;
     filename += file_writer->GetDefaultFileExtension();
 
@@ -453,7 +436,7 @@ void CatalystAdaptor::coprocess(double a_time, unsigned int a_timestep)
             data_description->GetInputDescriptionByName("input");
         add_vars(input_data_description);
 
-        if (m_write_files)
+        if (m_p.write_files)
         {
             write_vtk_grid(a_timestep);
         }
@@ -526,7 +509,7 @@ void CatalystAdaptor::catalyst_error_or_warning(bool a_success,
     if (a_success)
         return;
 
-    if (m_abort_on_catalyst_error)
+    if (m_p.abort_on_error)
         MayDay::Error(a_msg.c_str());
     else
         MayDay::Warning(a_msg.c_str());
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index 05cba2221..d6d239e4d 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -12,10 +12,11 @@
 #include <string>
 
 // Chombo includes
+#include "Box.H"
+#include "FArrayBox.H"
 #include "MayDay.H"
 
 // GRChombo includes
-#include "GRAMR.hpp"
 #include "UserVariables.hpp"
 
 // ParaView/VTK/Catalyst includes
@@ -42,27 +43,35 @@ class GRAMR;
 class CatalystAdaptor
 {
   public:
+    struct params_t
+    {
+        int verbosity;
+        bool abort_on_error = false;
+        bool remove_ghosts = false;
+
+        // Pipeline parameters
+        std::vector<std::string> python_scripts;
+        std::string output_path;
+
+        // VTK file writing parameters
+        bool write_files = false;
+        std::string base_file_name = "Catalyst_VTK_grid_";
+
+        // variables to pass to Catalyst set by GRChombo parameter
+        std::vector<std::pair<int, VariableType>> vars;
+    };
+
     // empty constructor (doesn't call initialise)
     CatalystAdaptor();
 
     // full constructor (calls initialise)
-    CatalystAdaptor(GRAMR *a_gr_amr_ptr,
-                    const std::vector<std::string> &a_python_scripts,
-                    const std::string &a_output_path,
-                    const std::vector<std::pair<int, VariableType>> &a_vars,
-                    bool a_abort_on_catalyst_error, bool a_remove_ghosts,
-                    bool a_write_files, int a_verbosity);
+    CatalystAdaptor(GRAMR *a_gr_amr_ptr, const params_t &a_params);
 
     // destructor
     ~CatalystAdaptor();
 
     // Initialisation/Finalisation
-    void initialise(GRAMR *m_gr_amr_ptr,
-                    const std::vector<std::string> &a_python_scripts,
-                    const std::string &a_output_path,
-                    const std::vector<std::pair<int, VariableType>> &a_vars,
-                    bool a_abort_on_catalyst_error, bool a_remove_ghosts,
-                    bool a_write_files, int a_verbosity);
+    void initialise(GRAMR *m_gr_amr_ptr, const params_t &a_params);
     void finalise();
 
     // do Catalyst processing
@@ -97,18 +106,10 @@ class CatalystAdaptor
     // m_abort_on_catalyst_error
     void catalyst_error_or_warning(bool a_success, std::string a_msg);
 
-    int m_verbosity;
-    bool m_initialised = false;
-    bool m_abort_on_catalyst_error = false;
-    bool m_remove_ghosts = false;
     GRAMR *m_gr_amr_ptr = nullptr;
+    bool m_initialised = false;
 
-    // file writing parameters
-    bool m_write_files = false;
-    std::string m_base_file_name = "Catalyst_VTK_grid_";
-
-    // variables to pass to Catalyst set by GRChombo parameter
-    std::vector<std::pair<int, VariableType>> m_vars;
+    params_t m_p;
 
     // variables actually passed to Catalyst on last CoProcess
     std::array<bool, NUM_VARS> m_requested_evolution_vars;
diff --git a/Source/Insitu/README.md b/Source/Insitu/README.md
index 89e49bd82..3c4a28bf7 100644
--- a/Source/Insitu/README.md
+++ b/Source/Insitu/README.md
@@ -175,37 +175,29 @@ in order to be able to use it.
 ## Using the in-situ visualization capabilities
 
 There are several Catalyst specific parameters which control the in-situ
-visualization processing which are described below:
- * `activate_catalyst = true/false`: This enables/disable Catalyst coprocessing
- at runtime. Note that if set to `false`, the other parameters are not read in.
- * `catalyst_verbosity = -2,...,10`: This controls the verbosity of information printed
- to `catalyst_pout` files and that printed to the normal `pout` files from
- the `CatalystAdaptor` class which interfaces with ParaView Catalyst.
- * `catalyst_pout_prefix`: This is the filename prefix for the `catalyst_pout`
- files. They are written to `pout_subpath` and appended by '`.<rank number>`' 
- as for the normal `pout` files. Note that you can set this to the same
- string as `pout_prefix` to send this output to the same
- * `catalyst_scripts_path`: This is the path that contains the Catalyst scripts
- generated as described [above](#generating-paraview-catalyst-scripts). Note
- that this path is relative to the current working directory and _not_ 
- `output_path`.
- * `num_catalyst_scripts`: Number of Catalyst scripts to process.
- * `catalyst_scripts`: Filenames of Catalyst scripts in the 
- `catalyst_scripts_path` directory.
- * `catalyst_coprocess_level`: This is the level for which the Catalyst
- coprocess routine is called at the end of each timestep.
- * `catalyst_remove_ghosts = true/false`: If set to `true`, Chombo data will be
- copied to new VTK arrays without ghosts rather than passing the Chombo data 
- pointers directly (for debugging).
- * `abort_on_catalyst_error = true/false`: If set to `true`, the code will abort
- if there is an error in Catalyst. The default is `false`.
+visualization processing which are described in the table below.
+
+ | Parameter Name             | Type             | Possible value [default] | Description |
+ | ---                        | ---              | ---                      | ---         |
+ | `catalyst_activate`        | `bool`           | `true`/[`false`] | Enables/disables Catalyst coprocessing at runtime. If set to `false`, other parameters are not set. |
+ | `catalyst_verbosity`       | `int`            | `-2`,...,[`verbosity`],..,`10` | Controls verbosity in `catalyst_pout` files and that printed to the normal `pout` files from the `CatalystAdaptor` class which interfaces with ParaView Catalyst |
+ | `catalyst_pout_prefix`     | `string`         | [`catalyst_pout`] | Filename prefix for the `catalyst_pout` files. They are written to `pout_subpath` and appended by `. <rank_number>` as for the normal `pout` files. Note that this can be set to the same string as `pout_prefix` to send the output to the normal `pout` files. |
+ | `catalyst_scripts_path`    | `string`         | `""` | Path that contains the Catalyst Python scrips generated as described [above](#generating-paraview-catalyst-scripts). Note that this path should be either an absolute path or relative to the current directory and _not_ `output_path`. |
+ | `catalyst_num_scripts`     | `int`            | [`1`] | Number of Catalyst Python scripts |
+ | `catalyst_scripts`         | `vector<string>` | - | Filenames of Catalyst Python scripts in the `catalyst_scripts_path` directory. |
+ | `catalyst_coprocess_level` | `int`            | [`0`],...,`max_level` | Level for which the Catalyst coprocess routine is called at the end of each timestep |
+ | `catalyst_remove_ghosts`   | `bool`           | `true`/[`false`] | If `true`, Chombo data will be deep-copied to new VTK arrays without ghosts rather than simply passing the pointers. This is for debugging and should not be necessary for most users. |
+ | `catalyst_abort_on_error`  | `bool`           | `true`/[`false`] | If `true`, the code will abort if there is an error in Catalyst. |
+ | `catalyst_num_vars`*        | `int`            | [`0`],...,`NUM_VARS + NUM_DIAGNOSTIC_VARS` | Number of variables in `catalyst_vars` |
+ | `catalyst_vars`*            | `vector<string>` | - | Restrict variables passed to Catalyst to only these ones. No restriction if `catalyst_num_vars == 0`.|
  
-Note that scripts generated with versions of ParaView v5.8 or earlier provide 
+*Note that scripts generated with versions of ParaView v5.8 or earlier provide 
 information to the pipeline about the specific variables they require. Since 
 this feature is not currently provided by ParaView v5.9 or later, scripts
 generated with these versions request all variables by default. In order to
 only pass specific variables to Catalyst, it is possible to restrict the 
-variables passed using the `num_catalyst_vars` and `catalyst_vars` parameters.
+variables passed using the `num_catalyst_vars` and `catalyst_vars` parameters
+(as described above). 
 If these are not set, and a script generated by ParaView v5.9 or later is used,
 all variables will be passed to Catalyst.
 
diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.inputs b/Tests/CatalystInsituTest/CatalystInsituTest.inputs
index bd6274765..d400604ba 100644
--- a/Tests/CatalystInsituTest/CatalystInsituTest.inputs
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.inputs
@@ -27,18 +27,18 @@ lo_boundary = 0 0 0
 #################################################
 # ParaView Catalyst parameters
 
-activate_catalyst = true
+catalyst_activate = true
 catalyst_verbosity = 10
 # catalyst_pout_prefix = catalyst_pout
 # path to directory containing catalyst scripts
 # catalyst_scripts_path = catalyst_scripts
-num_catalyst_scripts = 1
+catalyst_num_scripts = 1
 # list of Catalyst scripts
 catalyst_scripts = slice_A_v1.py # slice_A_v2.py
 catalyst_coprocess_level = 0
-# num_catalyst_vars = 1
+# catalyst_num_vars = 1
 # catalyst_vars = A
-abort_on_catalyst_error = true
+catalyst_abort_on_error = true
 
 # Test parameters
 error_threshold = 0.15 # arbitrary threshold but this is what VTK tests use

From 362ff95b7dfab2da71bfbd06e53c74df2194f1c8 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 25 May 2022 18:02:29 +0100
Subject: [PATCH 47/63] Add VTK XML file parameters

This allows the user to make Catalyst write VTK XML files for the 3D AMR
data that is passed to Catalyst at each Catalyst coprocess and specify
the filename prefix of these files.
---
 Source/GRChomboCore/ChomboParameters.hpp |  5 +++-
 Source/Insitu/CatalystAdaptor.cpp        |  4 ++--
 Source/Insitu/CatalystAdaptor.hpp        |  4 ++--
 Source/Insitu/README.md                  | 30 +++++++++++++-----------
 4 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index 851341c99..f5b6d71cd 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -179,7 +179,10 @@ class ChomboParameters
                     false);
             pp.load("catalyst_remove_ghosts", catalyst_params.remove_ghosts,
                     false);
-            pp.load("catalyst_write_files", catalyst_params.write_files, false);
+            pp.load("catalyst_write_vtk_files", catalyst_params.write_vtk_files,
+                    false);
+            pp.load("catlyst_vtk_file_prefix", catalyst_params.vtk_file_prefix,
+                    std::string("Catalyst_VTK_"));
         }
 #endif
     }
diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index f0462192a..c77d40cdf 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -410,7 +410,7 @@ void CatalystAdaptor::write_vtk_grid(unsigned int a_timestep)
     // make filename
     char timestep_cstr[7];
     std::sprintf(timestep_cstr, "%06d.", a_timestep);
-    std::string filename = m_p.base_file_name;
+    std::string filename = m_p.vtk_file_prefix;
     filename += timestep_cstr;
     filename += file_writer->GetDefaultFileExtension();
 
@@ -436,7 +436,7 @@ void CatalystAdaptor::coprocess(double a_time, unsigned int a_timestep)
             data_description->GetInputDescriptionByName("input");
         add_vars(input_data_description);
 
-        if (m_p.write_files)
+        if (m_p.write_vtk_files)
         {
             write_vtk_grid(a_timestep);
         }
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index d6d239e4d..9aa1b3104 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -54,8 +54,8 @@ class CatalystAdaptor
         std::string output_path;
 
         // VTK file writing parameters
-        bool write_files = false;
-        std::string base_file_name = "Catalyst_VTK_grid_";
+        bool write_vtk_files = false;
+        std::string vtk_file_prefix = "Catalyst_VTK_grid_";
 
         // variables to pass to Catalyst set by GRChombo parameter
         std::vector<std::pair<int, VariableType>> vars;
diff --git a/Source/Insitu/README.md b/Source/Insitu/README.md
index 3c4a28bf7..7c67e000e 100644
--- a/Source/Insitu/README.md
+++ b/Source/Insitu/README.md
@@ -177,20 +177,22 @@ in order to be able to use it.
 There are several Catalyst specific parameters which control the in-situ
 visualization processing which are described in the table below.
 
- | Parameter Name             | Type             | Possible value [default] | Description |
- | ---                        | ---              | ---                      | ---         |
- | `catalyst_activate`        | `bool`           | `true`/[`false`] | Enables/disables Catalyst coprocessing at runtime. If set to `false`, other parameters are not set. |
- | `catalyst_verbosity`       | `int`            | `-2`,...,[`verbosity`],..,`10` | Controls verbosity in `catalyst_pout` files and that printed to the normal `pout` files from the `CatalystAdaptor` class which interfaces with ParaView Catalyst |
- | `catalyst_pout_prefix`     | `string`         | [`catalyst_pout`] | Filename prefix for the `catalyst_pout` files. They are written to `pout_subpath` and appended by `. <rank_number>` as for the normal `pout` files. Note that this can be set to the same string as `pout_prefix` to send the output to the normal `pout` files. |
- | `catalyst_scripts_path`    | `string`         | `""` | Path that contains the Catalyst Python scrips generated as described [above](#generating-paraview-catalyst-scripts). Note that this path should be either an absolute path or relative to the current directory and _not_ `output_path`. |
- | `catalyst_num_scripts`     | `int`            | [`1`] | Number of Catalyst Python scripts |
- | `catalyst_scripts`         | `vector<string>` | - | Filenames of Catalyst Python scripts in the `catalyst_scripts_path` directory. |
- | `catalyst_coprocess_level` | `int`            | [`0`],...,`max_level` | Level for which the Catalyst coprocess routine is called at the end of each timestep |
- | `catalyst_remove_ghosts`   | `bool`           | `true`/[`false`] | If `true`, Chombo data will be deep-copied to new VTK arrays without ghosts rather than simply passing the pointers. This is for debugging and should not be necessary for most users. |
- | `catalyst_abort_on_error`  | `bool`           | `true`/[`false`] | If `true`, the code will abort if there is an error in Catalyst. |
- | `catalyst_num_vars`*        | `int`            | [`0`],...,`NUM_VARS + NUM_DIAGNOSTIC_VARS` | Number of variables in `catalyst_vars` |
- | `catalyst_vars`*            | `vector<string>` | - | Restrict variables passed to Catalyst to only these ones. No restriction if `catalyst_num_vars == 0`.|
- 
+| Parameter Name             | Type             | Possible value [default] | Description |
+| ---                        | ---              | ---                      | ---         |
+| `catalyst_activate`        | `bool`           | `true`/[`false`] | Enables/disables Catalyst coprocessing at runtime. If set to `false`, other parameters are not set. |
+| `catalyst_verbosity`       | `int`            | `-2`,...,[`verbosity`],..,`10` | Controls verbosity in `catalyst_pout` files and that printed to the normal `pout` files from the `CatalystAdaptor` class which interfaces with ParaView Catalyst |
+| `catalyst_pout_prefix`     | `string`         | [`"catalyst_pout"`] | Filename prefix for the `catalyst_pout` files. They are written to `pout_subpath` and appended by `. <rank_number>` as for the normal `pout` files. Note that this can be set to the same string as `pout_prefix` to send the output to the normal `pout` files. |
+| `catalyst_scripts_path`    | `string`         | `""` | Path that contains the Catalyst Python scrips generated as described [above](#generating-paraview-catalyst-scripts). Note that this path should be either an absolute path or relative to the current directory and _not_ `output_path`. |
+| `catalyst_num_scripts`     | `int`            | [`1`] | Number of Catalyst Python scripts |
+| `catalyst_scripts`         | `vector<string>` | - | Filenames of Catalyst Python scripts in the `catalyst_scripts_path` directory. |
+| `catalyst_coprocess_level` | `int`            | [`0`],...,`max_level` | Level for which the Catalyst coprocess routine is called at the end of each timestep |
+| `catalyst_remove_ghosts`   | `bool`           | `true`/[`false`] | If `true`, Chombo data will be deep-copied to new VTK arrays without ghosts rather than simply passing the pointers. This is for debugging and should not be necessary for most users. |
+| `catalyst_abort_on_error`  | `bool`           | `true`/[`false`] | If `true`, the code will abort if there is an error in Catalyst. |
+| `catalyst_num_vars`*        | `int`            | [`0`],...,`NUM_VARS + NUM_DIAGNOSTIC_VARS` | Number of variables in `catalyst_vars` |
+| `catalyst_vars`*            | `vector<string>` | - | Restrict variables passed to Catalyst to only these ones. No restriction if `catalyst_num_vars == 0`.|
+| `catalyst_write_vtk_files`  | `bool`           | `true`/[`false`] | Controls whether VTK XML files containing the full 3D AMR data passed to Catalyst are written or not. These files can be opened in ParaView |
+| `vtk_file_prefix`           | `string`         | [`"Catalyst_VTK_"`] | Filename prefix for VTK XML files. The filenames will be of the form `<vtk_file_prefix>_<6 digit timestep on catalyst_coprocess_level>`. Note there will be multiple files for each timestep: one of type `.vth` and a subdirectory containing `.vti` files for each box. |
+
 *Note that scripts generated with versions of ParaView v5.8 or earlier provide 
 information to the pipeline about the specific variables they require. Since 
 this feature is not currently provided by ParaView v5.9 or later, scripts

From 068c213204250fcc5c62cafec415e4a7dbd0c714 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 25 May 2022 18:04:07 +0100
Subject: [PATCH 48/63] Make CatalystInsituTest check PNG is modified

The test should not pass with a PNG from a previous run if Catalyst
fails to run for whatever reason.
---
 .../CatalystInsituTest/CatalystInsituTest.cpp | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.cpp b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
index 8fdf96bd6..9dfc66b33 100644
--- a/Tests/CatalystInsituTest/CatalystInsituTest.cpp
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
@@ -19,6 +19,7 @@
 // General includes:
 #include <algorithm>
 #include <cmath>
+#include <unistd.h>
 
 #include "GRAMR.hpp"
 
@@ -55,6 +56,12 @@ int runInsituTest(int argc, char *argv[])
     GRParmParse pp(0, argv + argc, NULL, in_file);
     SimulationParameters sim_params(pp);
 
+    // Get the modified time of the generated PNG now to check it's changed by
+    // Catalyst
+    struct stat generated_png_file_stat_before;
+    stat(sim_params.generated_png_file.c_str(),
+         &generated_png_file_stat_before);
+
     GRAMR gr_amr;
     DefaultLevelFactory<InsituTestLevel> insitu_test_level_factory(gr_amr,
                                                                    sim_params);
@@ -72,6 +79,13 @@ int runInsituTest(int argc, char *argv[])
     gr_amr.run(sim_params.stop_time, sim_params.max_steps);
     gr_amr.conclude();
 
+    // Check that the generated PNG was modified after time_before
+    struct stat generated_png_file_stat_after;
+    stat(sim_params.generated_png_file.c_str(), &generated_png_file_stat_after);
+    if (generated_png_file_stat_before.st_mtim.tv_nsec >=
+        generated_png_file_stat_after.st_mtim.tv_nsec)
+        return 2;
+
     // read the newly generated PNG and the expected PNG
     vtkNew<vtkPNGReader> generated_png_reader;
     generated_png_reader->SetFileName(sim_params.generated_png_file.c_str());
@@ -117,6 +131,11 @@ int main(int argc, char *argv[])
         pout() << "Catalyst Insitu test skipped (ParaView version < 5.9)."
                << std::endl;
     }
+    else if (status == 2)
+    {
+        pout() << "Catalyst Insitu test failed (extracted PNG not modified)."
+               << std::endl;
+    }
     else
         pout() << "Catalyst Insitu test failed with return code " << status
                << std::endl;

From 69c37556c271a6d03be018db536bb78e1933540e Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Thu, 26 May 2022 12:46:01 +0100
Subject: [PATCH 49/63] Fix incorrect comparison on file modification time

This was probably obvious but the `tv_nsec` member of a `timespec`
struct is only the number of nanoseconds since the last full second so
it is necessary to compare the `tv_sec` member first.
---
 .../workflows/run-catalyst-insitu-test.yml    |  8 +++++++
 .../CatalystInsituTest/CatalystInsituTest.cpp | 22 +++++++++++++++++--
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-catalyst-insitu-test.yml b/.github/workflows/run-catalyst-insitu-test.yml
index c7a2e92e0..c7b39d39c 100644
--- a/.github/workflows/run-catalyst-insitu-test.yml
+++ b/.github/workflows/run-catalyst-insitu-test.yml
@@ -80,3 +80,11 @@ jobs:
     - name: Run Catalyst Insitu test
       run: make run
       working-directory: ${{ github.workspace }}/GRChombo/Tests/CatalystInsituTest
+      
+    - name: Upload output files if test failed
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+        name: CatalystInsituTest-pouts
+        path: ${{ github.workspace }}/GRChombo/Tests/CatalystInsituTest/pout.*
+        if-no-files-found: warn
\ No newline at end of file
diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.cpp b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
index 9dfc66b33..7dfd8618d 100644
--- a/Tests/CatalystInsituTest/CatalystInsituTest.cpp
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
@@ -19,6 +19,7 @@
 // General includes:
 #include <algorithm>
 #include <cmath>
+#include <ctime>
 #include <unistd.h>
 
 #include "GRAMR.hpp"
@@ -44,6 +45,16 @@
 #include "UsingNamespace.H"
 
 #ifdef USE_CATALYST
+
+// A simple function to compare two timespecs
+bool operator<=(const timespec &lhs, const timespec &rhs)
+{
+    if (lhs.tv_sec == rhs.tv_sec)
+        return lhs.tv_nsec <= rhs.tv_nsec;
+    else
+        return lhs.tv_sec <= rhs.tv_sec;
+}
+
 int runInsituTest(int argc, char *argv[])
 {
     int status = 0;
@@ -82,9 +93,16 @@ int runInsituTest(int argc, char *argv[])
     // Check that the generated PNG was modified after time_before
     struct stat generated_png_file_stat_after;
     stat(sim_params.generated_png_file.c_str(), &generated_png_file_stat_after);
-    if (generated_png_file_stat_before.st_mtim.tv_nsec >=
-        generated_png_file_stat_after.st_mtim.tv_nsec)
+
+    if (generated_png_file_stat_after.st_mtim <=
+        generated_png_file_stat_before.st_mtim)
+    {
+        pout() << "Extract modification time before:  "
+               << std::ctime(&generated_png_file_stat_before.st_mtim.tv_sec);
+        pout() << "Extract modification time after: "
+               << std::ctime(&generated_png_file_stat_after.st_mtim.tv_sec);
         return 2;
+    }
 
     // read the newly generated PNG and the expected PNG
     vtkNew<vtkPNGReader> generated_png_reader;

From 49628def32d72d47a1b73ec287a07ab52c4a8986 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Fri, 27 May 2022 14:11:01 +0100
Subject: [PATCH 50/63] Delete generated PNG at start of CatalystInsituTest

Remove all the code that checks the modified timestamps and simply just
delete the old file at the start. This way, if nothing is generated, the
test will definitely fail.
---
 .../CatalystInsituTest/CatalystInsituTest.cpp | 29 ++-----------------
 1 file changed, 3 insertions(+), 26 deletions(-)

diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.cpp b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
index 7dfd8618d..63ee16e25 100644
--- a/Tests/CatalystInsituTest/CatalystInsituTest.cpp
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
@@ -19,8 +19,7 @@
 // General includes:
 #include <algorithm>
 #include <cmath>
-#include <ctime>
-#include <unistd.h>
+#include <cstdio>
 
 #include "GRAMR.hpp"
 
@@ -67,11 +66,8 @@ int runInsituTest(int argc, char *argv[])
     GRParmParse pp(0, argv + argc, NULL, in_file);
     SimulationParameters sim_params(pp);
 
-    // Get the modified time of the generated PNG now to check it's changed by
-    // Catalyst
-    struct stat generated_png_file_stat_before;
-    stat(sim_params.generated_png_file.c_str(),
-         &generated_png_file_stat_before);
+    // Delete the generated PNG file if it already exists
+    std::remove(sim_params.generated_png_file.c_str());
 
     GRAMR gr_amr;
     DefaultLevelFactory<InsituTestLevel> insitu_test_level_factory(gr_amr,
@@ -90,20 +86,6 @@ int runInsituTest(int argc, char *argv[])
     gr_amr.run(sim_params.stop_time, sim_params.max_steps);
     gr_amr.conclude();
 
-    // Check that the generated PNG was modified after time_before
-    struct stat generated_png_file_stat_after;
-    stat(sim_params.generated_png_file.c_str(), &generated_png_file_stat_after);
-
-    if (generated_png_file_stat_after.st_mtim <=
-        generated_png_file_stat_before.st_mtim)
-    {
-        pout() << "Extract modification time before:  "
-               << std::ctime(&generated_png_file_stat_before.st_mtim.tv_sec);
-        pout() << "Extract modification time after: "
-               << std::ctime(&generated_png_file_stat_after.st_mtim.tv_sec);
-        return 2;
-    }
-
     // read the newly generated PNG and the expected PNG
     vtkNew<vtkPNGReader> generated_png_reader;
     generated_png_reader->SetFileName(sim_params.generated_png_file.c_str());
@@ -149,11 +131,6 @@ int main(int argc, char *argv[])
         pout() << "Catalyst Insitu test skipped (ParaView version < 5.9)."
                << std::endl;
     }
-    else if (status == 2)
-    {
-        pout() << "Catalyst Insitu test failed (extracted PNG not modified)."
-               << std::endl;
-    }
     else
         pout() << "Catalyst Insitu test failed with return code " << status
                << std::endl;

From 7d812328b321cfb3f391038e72b737dd36441ce0 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 20 Sep 2022 12:02:28 +0100
Subject: [PATCH 51/63] Fix slice visualization artifacts

The key step is to apply vtkParallelAMRUtilities::StripGhostLayers
(which removes layers of fine ghost cells that only partially cover
coarse cells) and vtkParallelAMRUtilties::BlankCells (which blanks out
coarser cells that are covered by finer cells on a different rank -- I
thought this was automatic previously) to the vtkOverlappingAMR object
(this is done in the [VisIt Chombo
Reader](https://gitlab.kitware.com/paraview/visitbridge/-/blob/df098f4148a96d62c388861c1d476039e02224ae/databases/AvtAlgorithms/vtkAvtSTMDFileFormatAlgorithm.cxx#L511-L514)).
Since the former might require deep copying (in case stripping happens),
a temporary vtkOverlappingAMR object is instantiated in
CatalystAdaptor::coprocess which is used to create the grid and add the
fields. This is then copied (either shallow or deep as necessary) to the
m_vtk_grid_ptr member which is passed to Catalyst. Note that now, no
cells are manually marked as ghosts in CatalystAdaptor::build_vtk_grid
as I observed gaps when doing so. The only marking of ghosts now comes
from BlankCells.
---
 Source/Insitu/CatalystAdaptor.cpp             |  64 +++++++++++-------
 Source/Insitu/CatalystAdaptor.hpp             |   5 +-
 Source/Insitu/Make.insitu                     |   2 +-
 .../extracts/SliceA_expected.png              | Bin 5998 -> 10371 bytes
 4 files changed, 44 insertions(+), 27 deletions(-)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index c77d40cdf..4f8183206 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -6,6 +6,8 @@
 #include "CatalystAdaptor.hpp"
 #include "GRAMRLevel.hpp"
 #include "ParaviewVersion.hpp"
+#include "vtkMultiProcessController.h"
+#include "vtkParallelAMRUtilities.h"
 
 #ifdef USE_CATALYST
 
@@ -107,16 +109,16 @@ void CatalystAdaptor::finalise()
     m_initialised = false;
 }
 
-void CatalystAdaptor::build_vtk_grid()
+void CatalystAdaptor::build_vtk_grid(vtkOverlappingAMR *a_vtk_grid_ptr)
 {
     if (m_p.verbosity)
     {
         pout() << "CatalystAdaptor::build_vtk_grid" << std::endl;
     }
-    if (m_vtk_grid_ptr != nullptr)
-        m_vtk_grid_ptr->Delete();
+    if (a_vtk_grid_ptr != nullptr)
+        a_vtk_grid_ptr->Delete();
 
-    m_vtk_grid_ptr = vtkOverlappingAMR::New();
+    a_vtk_grid_ptr = vtkOverlappingAMR::New();
 
     const auto gramrlevels = m_gr_amr_ptr->get_gramrlevels();
 
@@ -131,7 +133,7 @@ void CatalystAdaptor::build_vtk_grid()
             level->getLevelData().disjointBoxLayout();
         boxes_per_level[ilevel] = level_box_layout.boxArray().size();
     }
-    m_vtk_grid_ptr->Initialize(num_levels, boxes_per_level);
+    a_vtk_grid_ptr->Initialize(num_levels, boxes_per_level);
 
     // The origin is always at (0, 0, 0) in Chombo
     double origin_global[3] = {0., 0., 0.};
@@ -140,7 +142,7 @@ void CatalystAdaptor::build_vtk_grid()
     const double coarsest_dx = gramrlevels[0]->get_dx();
     RealVect ghosted_origin_global_vect = coarsest_ghost_vect;
     ghosted_origin_global_vect *= (!m_p.remove_ghosts) ? -coarsest_dx : 0;
-    m_vtk_grid_ptr->SetOrigin(ghosted_origin_global_vect.dataPtr());
+    a_vtk_grid_ptr->SetOrigin(ghosted_origin_global_vect.dataPtr());
 
     // now add all the boxes
     for (int ilevel = 0; ilevel < num_levels; ++ilevel)
@@ -150,8 +152,8 @@ void CatalystAdaptor::build_vtk_grid()
         const GRAMRLevel *level = gramrlevels[ilevel];
         const double dx = level->get_dx();
         double dx_arr[3] = {dx, dx, dx};
-        m_vtk_grid_ptr->SetSpacing(ilevel, dx_arr);
-        m_vtk_grid_ptr->SetRefinementRatio(ilevel, level->refRatio());
+        a_vtk_grid_ptr->SetSpacing(ilevel, dx_arr);
+        a_vtk_grid_ptr->SetRefinementRatio(ilevel, level->refRatio());
         const GRLevelData &level_data = level->getLevelData();
         const DisjointBoxLayout &level_box_layout =
             level_data.disjointBoxLayout();
@@ -192,7 +194,7 @@ void CatalystAdaptor::build_vtk_grid()
                                   ghosted_origin_global_vect.dataPtr());
             // vtk_amr_box.Print(pout());
             // pout() << "\n";
-            m_vtk_grid_ptr->SetAMRBox(ilevel, ibox, vtk_amr_box);
+            a_vtk_grid_ptr->SetAMRBox(ilevel, ibox, vtk_amr_box);
 
             bool local_box = (procID() == level_box_layout.procID(lit()));
             // only need to do the following for local boxes
@@ -206,24 +208,23 @@ void CatalystAdaptor::build_vtk_grid()
                 //     small_ghosted_end[0], big_ghosted_end[0],
                 //     small_ghosted_end[1], big_ghosted_end[1],
                 //     small_ghosted_end[2], big_ghosted_end[2]);
-                // // add the ghost cell information
+                // add the ghost cell information
                 // int no_ghost[6] = {small_end[0], big_end[0],   small_end[1],
                 //                    big_end[1],   small_end[2], big_end[2]};
                 // bool cell_data = true;
                 // vtk_uniform_grid_ptr->GenerateGhostArray(no_ghost,
                 // cell_data);
 
-                vtk_uniform_grid_ptr->Initialize(&vtk_amr_box, origin, dx_arr,
-                                                 ghost_vect.dataPtr());
+                vtk_uniform_grid_ptr->Initialize(&vtk_amr_box, origin, dx_arr);
 
-                m_vtk_grid_ptr->SetDataSet(ilevel, ibox, vtk_uniform_grid_ptr);
+                a_vtk_grid_ptr->SetDataSet(ilevel, ibox, vtk_uniform_grid_ptr);
             }
         }
     }
 
-    m_vtk_grid_ptr->Audit();
+    a_vtk_grid_ptr->Audit();
 #if DEBUG
-    const double *vtk_grid_bounds = m_vtk_grid_ptr->GetAMRInfo()->GetBounds();
+    const double *vtk_grid_bounds = a_vtk_grid_ptr->GetAMRInfo()->GetBounds();
     pout() << "VTK Grid Bounds:"
            << "(" << vtk_grid_bounds[0] << "," << vtk_grid_bounds[2] << ","
            << vtk_grid_bounds[4] << "), (" << vtk_grid_bounds[1] << ","
@@ -232,10 +233,11 @@ void CatalystAdaptor::build_vtk_grid()
 #endif
 
     // not sure if this is necessary but it was on the OverlappingAMR example
-    m_vtk_grid_ptr->GenerateParentChildInformation();
+    a_vtk_grid_ptr->GenerateParentChildInformation();
 }
 
-void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
+void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc,
+                               vtkOverlappingAMR *a_vtk_grid_ptr)
 {
     if (m_p.verbosity)
     {
@@ -277,7 +279,7 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
     if (m_p.verbosity)
         pout() << std::endl;
 
-    vtkAMRInformation *amr_info = m_vtk_grid_ptr->GetAMRInfo();
+    vtkAMRInformation *amr_info = a_vtk_grid_ptr->GetAMRInfo();
     auto gramrlevels = m_gr_amr_ptr->get_gramrlevels();
 
     for (int ilevel = 0; ilevel < gramrlevels.size(); ++ilevel)
@@ -304,7 +306,7 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
             if (local_box)
             {
                 vtkUniformGrid *vtk_uniform_grid_ptr =
-                    m_vtk_grid_ptr->GetDataSet(ilevel, ibox);
+                    a_vtk_grid_ptr->GetDataSet(ilevel, ibox);
 
                 // hopefully this promotion works
                 DataIndex dind(lit());
@@ -315,7 +317,7 @@ void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc)
                 const Box &unghosted_box = level_box_layout[dind];
 
 #if DEBUG
-                vtkAMRBox vtk_box = m_vtk_grid_ptr->GetAMRBox(ilevel, ibox);
+                vtkAMRBox vtk_box = a_vtk_grid_ptr->GetAMRBox(ilevel, ibox);
                 // shift to account for different indexing
                 IntVect vtk_origin_offset =
                     -ipow(2, ilevel) * evolution_level_data.ghostVect();
@@ -431,17 +433,31 @@ void CatalystAdaptor::coprocess(double a_time, unsigned int a_timestep)
 
     if (m_proc_ptr->RequestDataDescription(data_description) != 0)
     {
-        build_vtk_grid();
+
+        vtkNew<vtkOverlappingAMR> vtk_ghosted_amr_grid_ptr;
+        build_vtk_grid(vtk_ghosted_amr_grid_ptr);
         auto input_data_description =
             data_description->GetInputDescriptionByName("input");
-        add_vars(input_data_description);
+        add_vars(input_data_description, vtk_ghosted_amr_grid_ptr);
+
+        if (m_vtk_grid_ptr != nullptr)
+        {
+            m_vtk_grid_ptr->Delete();
+        }
+        m_vtk_grid_ptr = vtkOverlappingAMR::New();
+
+        // strip partially overlapping ghost cells
+        vtkParallelAMRUtilities::StripGhostLayers(
+            vtk_ghosted_amr_grid_ptr, m_vtk_grid_ptr,
+            vtkMultiProcessController::GetGlobalController());
+
+        vtkParallelAMRUtilities::BlankCells(
+            m_vtk_grid_ptr, vtkMultiProcessController::GetGlobalController());
 
         if (m_p.write_vtk_files)
         {
             write_vtk_grid(a_timestep);
         }
-        // vtkNew<vtkOverlappingAMR> stripped_vtk_grid;
-        // vtkAMRUtilities::StripGhostLayers(m_vtk_grid_ptr, stripped_vtk_grid);
         input_data_description->SetGrid(m_vtk_grid_ptr);
         int coprocess_success = m_proc_ptr->CoProcess(data_description);
         catalyst_error_or_warning(coprocess_success,
diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp
index 9aa1b3104..06b7ab67c 100644
--- a/Source/Insitu/CatalystAdaptor.hpp
+++ b/Source/Insitu/CatalystAdaptor.hpp
@@ -84,10 +84,11 @@ class CatalystAdaptor
 
   private:
     // update the AMR grid (no grid data)
-    void build_vtk_grid();
+    void build_vtk_grid(vtkOverlappingAMR *a_vtk_grid_ptr);
 
     // send variables to catalyst
-    void add_vars(vtkCPInputDataDescription *a_input_data_desc);
+    void add_vars(vtkCPInputDataDescription *a_input_data_desc,
+                  vtkOverlappingAMR *a_vtk_grid_ptr);
 
     // write VTK grid to a file
     void write_vtk_grid(unsigned int a_timestep);
diff --git a/Source/Insitu/Make.insitu b/Source/Insitu/Make.insitu
index dc5c23830..1a4be1cb1 100644
--- a/Source/Insitu/Make.insitu
+++ b/Source/Insitu/Make.insitu
@@ -20,7 +20,7 @@ ifeq ($(USE_CATALYST),TRUE)
   # These components are correct in v5.9 and v5.10 but may differ in other 
   # versions. The paraview-config command may also change
   PARAVIEW_COMPONENT_FLAGS := -c PythonCatalyst ${XTRA_PARAVIEW_COMPONENT_FLAGS}
-  VTK_COMPONENT_FLAGS := -v CommonDataModel -v PythonUsed -v IOParallelXML ${XTRA_VTK_COMPONENT_FLAGS}
+  VTK_COMPONENT_FLAGS := -v CommonDataModel -v PythonUsed -v ParallelCore -v FiltersAMR -v IOParallelXML ${XTRA_VTK_COMPONENT_FLAGS}
   PARAVIEW_CONFIG_EXEC := $(PARAVIEW_DIR)/bin/paraview-config
   cxxcppflags += $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --cppflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
   XTRALDFLAGS += -rdynamic $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --ldflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
diff --git a/Tests/CatalystInsituTest/extracts/SliceA_expected.png b/Tests/CatalystInsituTest/extracts/SliceA_expected.png
index 2be8f1b537e8adba087d6590e0b7fb6355b746ce..7fa176143794b501f2e871de57ad708656c71914 100644
GIT binary patch
literal 10371
zcmc&)cUY6jy8qDiU|Gc25T&drAWDcJMT($HD1re2m2Q+OHS}f(=<Y%kloB97fLJIZ
zO@fFJ2&@PgLQ#TXq(r0{2t@*sKnVAXdw1`id(OT0IqrF$%O8_4`DVVEdE0M(?|bXK
zg^AeK{aXP55IbuMw+4WXKLEf6yUiQGo;|zwD1slmLyYV~&IAPbdbwT=@k03n=dT7`
zz2xN{o^~_sA^`jVoQ0pc5OH~CsMG&2BU0hR92}lnT7I<pXHAD~&Bwoo)-UwM;;V89
znQKmOltnWM^F6u;jwdoA#(%l;F78Ikhq&Iy2B}#P&%JILkI5um%AwrDIO^n?Gds4T
zq&<v{(Vzd8;H)TI&%fjEgH?JpAU+l&NFws7wxcwO<^+lH7WdjZWjub|#YU4r%5uqW
z_t-^`NbB~o|A?H3gzkZ;c7GP{Z?rQXa-3M{MhQZ|E-U=XG^)|b+z9^C8qaht{Y6gh
zL0<xMyU%?r_R-QuzuBg~D4qwDV^v1bqB$G^e~kL<Uz2eg`>6WvT#ZjL7VCgt7zq>3
zbck45iD>A0E?#8u80_u6;cXpvP?xFcdRF0(%L)1Br52-Gy^?lFr?gbFE~Ky7+32MK
zyp39I^eMQCBt)y?kR&DgVilFBLLHmO4eYa~Ry6XTjitsctr)~FI;{~(katJH703yK
zq~d!`-ur6x&uE5`>xNpw98on(6NQ(Vz2aI+&G9R<%?n6o$NKa--``JC=Ec*!A8rWG
zC}tgnEG@X}vQI9k<^^hMy86{zv~HtQ!XTM3u44lasyB8+ZG@Cj>-N4khY{)_#*S*b
zLpb>#<jm3*38<}}%nJ|MB<hP0MOpuoPk7=bBj3GJ;3#^-&6oF47sq3&$j|FcPu)iA
zHu4$E7-;AmVP$37^pvY~v9r-j^$f2>S?z{(?i>jf@2dOW{`5mUEU(E^H9c-lJ&_ea
zWhRL9jhUEX&KZpYXl2Wfmln<~AGT;okYL1DnW)P*()U21I3H6>!`O?wlvG)pHzxsE
z8t!?z-ma~E#SwwL@5*l%iazI$wHx?vC~Z$l<7eJ(6<M~+ds#M?b^eJ(%Uhf0N3$nr
znR$V{HpD-h<2ib(ZOeuI6vMn2^^X5xl9twsB2BU_Wn&>n?224CyUTAocdi{e4uPol
zvl$nscSX!K*`&xS@l^NR**IURcgvA_;!S{m-~~d_yk+W)L)@F3yg+lEEl-XbSTDF9
z-7osVHF9T7s^`ZoIV#;R)is<ZlM1{jW3pnLBmz*fa7f;u@IfG;pP?9dEK(t_iM4Rf
zSJUU#t|tj`bIFs6wt~Q2S1w}m$J#EaZ*OW_IP5Rq4R5Y(v6A*8{z}P~`LKXHw*LH|
zzAWc1p<txqON}{~oh`9T<dedGuIGo7&Rq;xO^r{=_4UOOSfAV%%FyMtDFyO4XC@3C
z99<3SeJ0bxvZa&E{v=h=zw^3{j`pD2dUP&!UkxM1rHI|LPU*^VK9LbZDrEMiS{V28
zNhZnbQw8IQYN?XN&z3QTJdWF-$GVyi#p~KWdMI~_K##G~D5&@}la{-)pFR;q7HqwY
zbY2gO`aDnr>$?eo7&M|eB_m4}`uT-SBmT6**f30!RW=*xy<Y{2#SX~pK0oB*AF1<Q
zgehX<W3R<CN9>+@ucZ9KPs-k)_oPtdt@_FjyOkX@5Ly4&Qz#0Ljqg>zmu)Qj^G&0Y
z!kFdfUKkO*{Qnwts#T*C_NQlX^6fhrDHnXaRv@Cn?uXfh!lAfA&*z0jhOzPe>V?^V
z1CR0A>qVQ*2;8<0BN}&#3q>|>-75x4BwInEso7`04PxVt92NCz*)X20KM=7S;riGM
zxJm!zZwO8S>&=ee$5=@0SD(>iKJETv5Q%^b*1^;^I42CQwoMx#!4Hqsg?_DK=Z{S0
zW%Tvv+KjsJHBGV=e^EuZwou&XRs$;vTB$YMSN*C1ttQtVW}h&4jO?ox#&K{#ovpB^
zqmB~MKV}bK(9batu)3eJNvmR$-3^5{QD@_errF_fQy$RDnsW@Lz*Fvu-6y+HD4U88
zv=Ra;&X(zmEZa0XP19PpBUNdAWcF?Nf~llZlq}#}^mx4={XolH27B<klLvTj@UlfY
zTALEE8ARx{m6T1foMhQ(USNA<u*{A(YLEq+zkTo>GeKf<f`)TM;ZM)~%uf^fW7wn!
z1iJoXAf_G2xx@cNOS0)A+%$sI4%z;>hZ7s;{E?tZ6a|N|@We;Lx5Z7C@lQxKWl_bO
z_~*wHwYOYbzIB6-7CCu>Cz2`qI7bzHB);s=9f~4iRX-RYfv$5HU?TpgXUEyo&C0L3
zC1OwDWMg^v+a;?QoMO9-eKKRIP7gXlGpC2vw+BmOu1%IY&=!V}!BJW&u_sQz#)oEF
z&QBFJnrS}~;f6`ZRIRxv(*(BCG&Oiw3ERLJgA2Ti_*G}mx>~nt1sHBV67~5lpjK-Q
z;0N6KcKNerZ+~6KEQj~w$vWr?3u!(omDyOk!q<v<wWvvEu@*6)L@{>e>^uFSd_U{x
z;#vm=<99<K&6;Fib`Mp1ZNtQ3_R_%#LDnnT68`7zm_iokaLlYmZjZSjdw*5GAeFKW
z!H_@HgIrsCef)L%ebJ7EA&N$tPI|Lw40ff`cWCYX_Z((FTd(m>3^F0Wp{GdG@^uC3
za)r_C%+&lW*9E$%DZJNV!a?7!7zXnfSHK@Jm#$%!eM0&TOMf5yH<>Y5!O2J-5ej@T
z*;h1Z^#pqy0_dn()IPidxhe;Qxa0(x{3`bX4t#ofa{TU1qn~d9uWtpmOM?<|{wT5>
zsBkoVx3V5eWYO)>&aHc|?gc{J|JN^EIX!8P6tlPA#=bDKh;+_9t6d60)|s7pQqru_
z>MZS#x#HO?-`+NpGl%iRH_y~!!c8U%G9p#C0d-mUg<0;$>x9$zpesf5NM`-}?`dr_
z#JOiTWj>x>Fz}BF(`H|qgCNtji0k$;zuXKxDi~Rj039ULzrr8?*)^r^*hO=xBWLGO
zurS(}>0Y7e)(Gs3g=Ruru!kASRi^=I&PE7D#hPeJ*17h)Q-KDWA^$Q1Q@4?wluf7g
zvCZ!oo>;k8myZ)eu2HXDw_&+n>}hKpIe{Q5@`5XT4+vyjpLfO;DXkVxrA0TDFkr*;
z;U}uqzkeE@l8akfy1`Ut&Nvzt)6a|RZxOdL$S5NtC7RI<40N8}*fe&%CS&BFdX8fb
z7C9tU7bec*t^`3hG#Ngs?x;pr@GyaatC2ysNWFopLw8rk71jc%UZ!8r01&iW8Hnxr
z*Bf6h`e;Ti2E+l+!&ip%r@-iVwcNeDd8pzi#|jg8nLA^>Zgrg_epuur1XW(qdR5X4
zTCjZ;S%3~FPGYgkOG_o4PfCXmQKXpB0?)|ZV1Qqq)QYLwTN_T6o^)Tf%h%6%+>Pr;
z!SvTVe=4jQ1&yaS=<`G~J-+(o@Wk4UH^Hw%8cL03?`Sc`0(2{k^i@6acZU2yvnfKJ
z7L=H9lY0V67upQgvnIN6O4p6Fa(hNIF}?@3pUZDH3+r@P@G0LU_<@|=;7W1449?S8
zu^nyAJO-Zd=o9oiUwuoN1v&^m4_~hf&>40@mKgyNkG-M6P^J$B>DMD(niI!<xDf!(
zZrux<_WYL`P?PtB7Hg3c7O1m%;Rhb~uU+m6U)P1TF<N%V?&8j_xJXS+o%$G5=j?b(
z^ud+f+sK&Jad7*5&DCPW9Eo{9XShP1yFkTe>X*uMHD#y3-$JUs9tCqic}Y~PhT+C=
z$3Y;x`J8AQF>3ay{K|YBPlK3Ts1=&%n!bLHLtU)3ZH@DMHlnF(y<ZS^IJeYLXFp3T
zU63Xx*As6r5eN;O^9ZlqTU&EI-HFrU5G$H$fD{2tt)2jYj?FhT7RH!e_Q1(kWn*sV
zd_-hkEsztyXayn+<Ii6OnrUGHT92jguqSMZr9ie^TfTZZVeo)|4|Tv-4czLxnZBLZ
zgQFPo*Y8c`AdEXP=lr}^Hc1%@O7MZu&$kvU+hy0)+;C!(X9a`D7?C?6hGz|MTBYxi
z_N0oc-Z%+|9@*a4^Jm<VQ~ovfptmE*Lcd9NzX`gJYKoHE2E^B<=OlVTn>N-i`#odd
z6`H?NuCE=ZaAYdCyo%hNq{SGTj6ZbPmF#HFp6LO1QGjg9IdvYCM#@^gca+>D&eC*%
zU>0w_vaE#$K@;e}(@7c2FO$VQ+ZZYNNjAN4whm#e8e-lqcXGzN1O7PVDap}K`@6u!
ztNrUX``d_8T*@$fFR(rMn6@Mxlx)yvP4cFdkOpvmRF&=fh}@xQjA?}=WE6p6L_&jL
z#6-U+lhuKcAiMyy@}g#zTW(dndPbb(Z2{=NjJYSE^4IfTCg&<zfjh(&2ADP5V7OT0
z6|5%u97L5(LdE1&p?n5C`Um~`cYYyl3-V>)03i18rftBz6W`EGyS>{W9cz&4NTp`>
zZO}`mX{8J*W1gRFk#ni<RGb9Wiqtu0BWs9ZIBQC>G+b-I!yj*q!gOA<52MPyWqZCq
z?#Vg3&~}vAAq1-6nM<1s5bij`PRg3vBo1^mU#2uyp|`FIPh6!PG_D8&B1n!@wCyiw
z%vUGTM}5SAj*EOaUXzMbE!!0FgYdRFHVg&}a~<t`ocr%y%sGl7bYysBu!SPW$M|Jx
z^r}W~hip%FUKQ`KSliKDWnOm(blytI9ZJGe^4{k&1*)m;T?%c^+z%%3p+qon@ff<p
z9*v!<OgCK4X4!rJs*C!QKkF;L_D-lSfycC^e^VC~a&a`H6MTt$qZL|Ty9fuVq*03F
z+$Y%s2JPAAQJvUqaTp`(-SJZ#YJE1w#L|>bILeCeS^V5RA!jDgyTw*=?x{HOU<5I4
z$%^wi8e=rqBTMfM*wxWJ97u>332j%1_j3B{MhY(crNnOw7dvn(_`>MwjDuUnRdXJD
zcuutlhp<Iq7<_`milPq~F6aZ@Dg(!N(W!A7+Bf;1VLmsKyjRjxG|1lEi^KM$&#9Qz
z%6uY+E^#5z#O{Qg8TV1^$#VJGbfsJ=`e+(GS$~@d@ao&$qfr_3`rQppo`Yr<ZNjU3
znmBkrYmkp#8^eh?w)a;zDK3Bzf~3*tAt+E`xvhj6!#m@Ji{+dftLIl9WwiD4JstWs
z$hzAm9ubqc0H;!Z&W_o_k~rVWK6poaRmU}_e<!gWM!*z<mTW3dhow9;kG=u`V@h@a
zkozr7(F5bKHk+?LJw)i!O?d(2y^Io;-&9g^skx>k1i}gc57-=`c;IPiWST^>Z70Jd
z-9R9}dT$`WcpO(yDzBd+KFN%Z>a5F`kS6)Gs^lgfX|vN@<Kn0nu%amqPr!&{{SCq9
zYmBlm>YPjX9*BRQo^u!U(T0p!622u>%ow3apMvxyMp(m^_Dzv8KA@#X=`cq8<TF~n
zQqF~ThX+GJKWmJhglRNRBwGaSCrKRq%VuBvDDY>$DUk?^|CH{1-^np|D<8FpLT~}G
zo^vcG1u;%(e&eCnPp<v+P&xaOnJ`54OdbOO?LS%KzXUY;SP_GF;36kC`pXx&IBc6g
zJw+-t0`e+XO5@!w=H%^X{<;D9bngGX@f9wu{Wv_W`T`tY777jO1CIpgvWo0uLXOCX
z{)By`T45Biqk9E?N|l-*3Sh8~)Z+8ssuQ4M5WtX;)@c_+2+1fYH{yEhWL-*n;sK|b
zLq{Dx4p&~DQtfqzia^_Okr68L8eiT^xptbw6dNA3f7n{T_v0D8tb}>C{J<W7zoRCV
ze*cOH5K{OT8ci??93d!?w?&nK;KB7UK15AUPPz+5h$-f|U8ZGg$f}q`^cK?`Zqfr1
zEe*=j2H(}vcXxS3i15aeEfk9U;qWv*iHCo$3$Aa;0W11BQ$my2{_GgAECLGN@Gr>s
zt7N7U5>nY@8Y$x)O>xqf1b0qA%Qg7+f{v)cquJcKhq^eOcx_7&P9rxlRX11efrq$~
z7>K#c(iZh8EVt(DH&1-OXSqcMe^e_J_Y#Co^e4V*?2U7e@1pKEueaa!`3Gy+ed0e1
zlmj0`K|9?}{O;QWl`uA03u)JBG6!7mKqq*h+PS5nAUfoeSXvk3eAP01NyeIzZ|1q?
zD~NFksL;g=aAtWC5UZ-WPrl<x7af#XmKq>}3-*mNVYr)P7$cjhd|TPJ>X(Z9?|Mfu
zlyW;7il^jGNfwJIiYEKn0`nU|la9CC@NGy21VB|n_?XrBE^YAAjJp<i_fB_?B66Lh
zHgdId6S2RcfUqvg8b7-rzu$@EV+?oRhQ5&D`>YUoddlrKNK98DbL+>(r64I583)%)
z^`x$LVqe9#EzV2IhIST@E4+K+t7(5AwYX+3NMD+fl;P^LuMG$Qt%&tktjLF9n&Vm*
ztu3#`lBy{__4d#MUpxrB4(2KkL_!yC4RYro5b`34Jmla41O2GZ0tD*wj9uq&i0oU;
z)1Z6nI6BBsy}9{OZV1#-*cxlH8^^!p(7ITl^|XGCzb7|Fgr`xoD0ZxI;@v!VT7e8w
z(khO0sy4oqN6R_{1}?qWxPu@77b?v+K*-;bXVv1wm=uG%w4)ZGGfD7;%Al$S9w*gs
z9L6p)JQ}0m-}Bttdaoy8$E<Fjv>;Lap-*hRxw!TTx%T=}BW@3>u@E^~+dh|1ux!n2
zC~Qwg80%}_0k1LVSdPYE=*yO1X!xv@z*riG9{7_?LM>prNVjg>xEdbKm0+=6<*7(H
zW281Vo$;4f{41vV)s26(SO3eN;b(~DHkDXAe<6t)Bt-T82h4aUJ(_v$@u`T`<WuJ%
zys^Muw&EUhoh;C19a4Gyk>y|EFcD+8GDd6#@7KLS60hQ6Jye8dt&<z`g1$BqwBMIO
z;H_n%giQ*L>KuCO@bPZKqMMuLhP&lg!95)Oi+fNe-_dRhTYapc#mdnV@+dmhSKx`R
zVH}>9H{)^iD@ttK<Cvf29fkVwjBBoG-{aHAH+Syaz!&LJjFF@ld9UdBnOd;i;7<7{
zNrG=sZ3iV59b};xZp->fwvya|DdfBb`B1!tNB_B`&3DU220uOzn%N3qlEI6_U$F(6
ze?(X1kRar&e~)&9d5hcwfAFemz5;(_!!0p^r-l4vDVlVDBmoB1eLGoGjm!X#Hx*ME
z9M+kt`&sTiH>I<2we9UZci;tUELm`_EhK2akaA!7n1n@W);oD1PwQ(pzETLsGd5PQ
z$*rjxvCF~vISJr2DnxV^`9WU$FJHcQ>;4gxL%_d74jMXbn*1@%xL$?a0j1SqsLiv}
zmJH{qhaUJu>?4++$@FWIe>iJqGQO_VNI!+yhp~VWEHYHDn@*=XS0SjB*lRkE&+;mU
z4@a9#CdG_d3RI)-7Z!nH=})xl85vh2<RIN}0}|kNqEpr~!(e?oiSME-zeX550~dp?
zc1(c7U#80FO4)-X^QV7{jLherccnzHjn{yzi}lp`skZeySO=Olt!+IQA9TvEZ!bCg
zlrw13t?W3iUvB-?{x`l)m07mQLj{zB_7frM4?0+WGvO>PjY3>7H(8Utot%$@SkY1I
zf$14q=?0g0?i`|*zP5YCr)&#j&`!j<@r<!qwbjCBFj9QTmlja*z661h2p#do8{{>}
zQK4jz=&EMiHFRz;ak8jcuYI;#+z{l8-H+Hx3~X#fPVHdUS~;VtHf@XL%r8s@>{(^Y
zq?Fy4a^$`d`KG(`b%_3{ii&cOS$j^NBr!*WXSzk^zZ;wx*F%#kI49M8PjufBrqGZL
zf^O6#-qoJlIAxcC&73~ouhu^@+aD&0(QR1fC&g6wLaRGy$L}(vhnV8F;4QFPYBGn{
zh3PXW94m1{#QYRS#qiXUd{USP!dY1-3zDXCE}vLqwW&m#Qs?q@3oRQ}qpiogaf`M{
zl^*0N7fOc;ncS{WgLD&T%{Gnc`ho{lSp#OFmR~MJ)$FbPvaqxSda{y)nUl)03B7IB
zOOXpj^!VS&!!>62`thWL(X~Nt&2|_~ioyJ<`!M=TIA)dGl&Ms3w*Z5(rZlV#%WnGc
zb3LW~Qe{QOund2#dltfY(O&5;s|I}Y9Rti1&^a8MRt;@AaBh(t-HSIofW}0IRynp?
zH*!iYIo{Zy_tbi7eh3M2whHl&T<u$fzBJP8G7_%pXO5*Q*%f_6m1B5_=u_$Lw<C|z
zE@TvUKq^Z{%eRwt*e-)_M?A7H8ubEx63j|p$o3AnFMC8RMUwjFY|A5+<elq=Ti$|X
zMVLnPO%Tg8q*~6B=DbZ6r=N`wD4{3Hzha4~34_OmxvTd;ZVZY%wVkPBVcE=JCid+y
z1Axsk!difkQXvcd=q`f}i<y0k2<yb!sgPiy;aL`iitV#PH#0j9s&8b2V*-DeMsxx(
zN_1o-I8}vBfz)#a%KQ2ukW7~gwS63CXxrW0riE|zUy}KJF_P10u->c}x44#c=s?@D
z&M;cAD&hX6A&XlgF1odEN9YmG1euC@Cc5>KQmFD0owW*5^qhS_OlIICFyGK`4S^t$
z6k!sqDuF2lN^B02gN^f)=z1IQsdj6uLE?S0BX&RA1FzHnk1udI@5VZ2U+kr?83#Eq
z<?z4cCN}R9xwmUHO?T^Fh{A@`imgiFY@>)S3I$!|+?q6C!1UYHAoc#Iu%k3u!oZy0
z2Ims7asYEqiOKAZT)835ihm?q6HHmHmh2(dbG{Qw7}gf}D%#+)oB+Jr%*oP-F#>{l
zT9W%TXi>b7=}Be@Na~wIIo4A*atmW#lgC>e+)(=b{rY&KX|s^b?yfbIRrLk$?nefi
zgSo+@nyVXJI%FR0qx;ONv!15vU*e4t*UuHPvuO&19N1TbGJ{>O&Ui<l8@l}Fc#i5S
zT&7SigW|*E`$y;3bVBEmZp*wDLg(=qW$W+)Z|p{DBW0zgJBD))>8sKw2Z5M}O$CI2
zl)UM)5`*EdqQVWlL_Le2{6kinp}%a*A^ssM!c_rdV$EUdhz$GEi=(oeR0U01PYotL
zx5auMGXOY-=_d-Wyg9yYfwSpD$C&69%%h;z-OBqt&R{4&P8J{N5Wm=!dDAG7Rh&YE
z0iFtf44HqD{&yV^!``3P*XAeLJH_=Z|40L3Gk(hhGF~N7_$SBfmF(b8`7clZIeYkz
z`9xt4{FS9-Y*or{nM#`{*lSsvc}g1tG#4Ji3@o%!pcX)Qh@ql`y6!-iF=lbRO=+^z
zqubIl$Z`pbeKkuLd|85l7T{>!0%p$q@FNL>VMSoJW5Sl`zh}+WKMW80N3t`w_#P<1
z>!Vy;Lq$w)KnlY6WdtF*805KVX&^V>SuQERR=7Y_%x59Sn|X+EI?+p@R}5kpf9(R!
z2ne;n=-~qDLSqNU^V8HtZ)2mEpgWu{nXc|_sL(_;QqW{DXTLmM!e;wrP3oyK5_ZVB
zoNyC$f+s?+{+_NrVyAv>g>jRgqzvfYE>wN14xIV*!5KRGte~U~080|iX^5RZ|K!?h
zb!4`Tn25|}dZzdO*n3yr++!h+_-8u-;uU{9vIz96*8+cf#0$U7pWp}|G1fpBY~GY`
z3DF&Zu#5-9i@i>3+|mEqWAMvs^+rI(njY)DKYGX3y=B=lquCP@O@`k4&m$g+|M=^J
zi}W<=I(lQ%MxQ_RctxH5bJoA^8siAXi}**n9{v)f%eK(w=$HfRl)bmG%Si)7XOt$=
z-<r_9zOwQ*zH_3alU#kJo=kbyu)p8>OBlHm3?~7OVBJI50O;K^FQ1T`Ah0)f9Cv48
zB<VJoSD2+EV5A2FynKVN-?ILXmjoqR4aFI*>G}d!O?&zzwVqmD>r}|JdO7PYOgG`k
zct+A!+%iah?~A-VR9+w_htY{%UYVM4m>LFMVoca)0!VPs*yD-ruphBl+Ckk958Nkt
zfv+s={;N#o3>g2(+PV;#H@yosn|ZSK*H)}-VrB7vUu$w7ObGd{po(SQ>}E9*y|bJU
z2HsoeNB>(P$?j5{$^m!97l*u~VriD;{85Fb$9z+2Qt>sZ*rf5ZgJyYKHcemfCB8t;
zYIWlI2I!oa!SC$S|M68b&qj1&lx@70w?_)z9RDY!CO?9!t|{LN2u^+_9CY?*X<2@F
z{O&E=pm`T0xHHeWT9a?C2N<aQeL2hn5Pv@^C%(8?=_6?z9MuYC9schtyrmdWr?*Qs
z@5RM8>d1wpxPMmDWBDN@Qb8XbvB9=QMOfBDboysciB0Y`S(-I$j_Fvi`yXp&UU+mD
zeQcy;f#plXjcC>S2&>CZ86~u!NP87c_bMfVQem;Lrs^F=@f)|Z$RsdV3_(^D%tmWd
z5|&vGN9DkL-JOU^&<?TkYXO&h;IjeM^7NjOsI;-v>*iqDSO;e5+O|m?f+Xv6$03M%
zZA-tM+3c2QeFbT5Bn7R?F2#p^anhxn5s(DR9?V(|szYE_kay=&sn2C&m1sRr7~!}f
zr<#FxGgDy_1AYxdeFQ>Ow2i@Fuej_KS)3LV&G}9M-SY3Kx)aVJ^I~1}`Jr}T!yz|{
P@c*+$7Vz>PuiX3(Oa~1Y

literal 5998
zcmdT|X;_nI)_y^$)sCeR6seMESY41oFWE!X2~Yw_#Mr7VMr2Wt9oZ5HXln_Gq>z|^
z5Ks|PMv;9htAb(xn-Lr!f&`JpNLVBxkPyQ6hG}cBZ^r4A&Y#JjTuGih&w20XKIh!`
zNji#k*tp^I4FCXabo>H!3;@>b007mKAFG1jY}VXt2)=1vv^#n6a7ajipO@!Fzu+L?
zsArhxSAITG^vm=w0YGcBBkJ(+=+l!g52tTyi&Xm?^ZO$-&3xp{mU`7h^H=&LjZcf4
z-DZX+@Os<6ybis&bFSIDu>y7kF3T<q()(*!*(NQWKl!W$5BlGDg_s9w4q+D9@r%s4
zU>J6qJ1K}Iy7tHTjaL>qx#0pWD<k|m3XRzoGIo6I){T+fi?b6kn#mN(U~l{g<LvY}
zZjQs|r@9!y;Y6e%4bm__C$dG`p~QnY!oIxYDAaUATnscH{8cPDXznYH<k=TeC{V<{
zLhED-v|*Z8o1B9ktL!#<ciWiF;IHR8?H}M?Fq2MYV;bZAjj!7b=F&KmE$Pnot)p!G
z$V0ARWYo(97lb@VTQp&M?}yte<d)c?X$L;FaWW1mixdY=Gm8k4*D(`Z=Iaih#l`Vu
z#r*R2jgMcuIj9VN13n~K+8Yl>gype};IsWbYYcQ}39eC&V~r(V`*{wTe9V<)cd0fd
zbH*cGsuq&y-P*ZlCYlq?0|?|m+c;6bsCRjxIC#l$kS?U!K>G!8Q#cF|%bcX|QM*c^
zNW2{Yqchf{=m&84uy_&~wdT8X|M+y9#dKMlVA{Xy#D*;p$fVFv>yXM%9|GrSpWICN
zX@@G;^?+Ic+x^r!6sqD8dS$nEs~r#d6)MSn*xF+ylo@a}1zbIm@UgY(Aqcohj|v15
zDmPl!rXsK8VUs>iM1;=?U@-eMEjGchr&H!fuQ?UJO+To|B(L?U>|otfR!%u5($YQ+
zS<zveJRlT!k|f7;%Q!g-+cz4yik6JJLa}?LSdO#l7F%(O8zM6oGUJFpe%pJHgpYd)
ztsW?c*}h|cJ!oK7fqO^Ryo9{N+z9}d8@2$4Ow~RCZf0Co1CBaBPz8(%&Zq#I_!0m}
z-gOcH9%=3c63+h1We^CYfw~FM>%%5dC=nnjPm&Q-xJeH#uXT@fMixV_v0P@ntnV>t
zGnzdsy@AQenRgkvj<600=r;;2XPfL;7NoI`8I0{lE=$kdAQ(>UN7rY#c6FpHFl7I{
zR!m1K&yC#aGPLYH(P*0aod*v9^nN3x4@G{J5+b)&;tqkl3J8J2d*Z!uxdag3)2EQs
zP3GfVp0VE&F$HeIC|)$09TsO}s*JNzC{GgSg&2&Us7{`S9cz<ca(?+7F}8czS;P^)
zxZostNu_j$%gke}GH?>S)#rjNG%}w;5sKMJRw9E?3;=hG{yl;p4mZOxV@twx32PGs
zGGWx56{4Uc6md%+Kw%2wx@Y$Y<>SvyazQvB2`*SkvoZFz+K_%Tw2~H7A!-hhRo^>*
z7)NMEj366~!RrvzwsYgHeP|jJdQB2nW#D+HWrN(Um8Y%CdU>+eD^<d;Z)~NlV|ZNo
zBOqs3JOraI{eMwgl}6nol(D%cgq46l<<ta#oMvF{TN#ZB2y2(nJG8X8V~C}=?{JDk
zV?n!hf&|js=mKd(I)P-oIOTxKE_;QPR+ET1w{h-%{pr=YDes01dbR+7_P^xihfsab
zayZ-|wM1=*6u?HITCFzB7}zW^5OE5zO=;9#f~aLG@5$BJ{rf<cin6gm0vwfs(7_rm
z=CHaf401H*x<q{4MQ68;`!tJ6m*IkZ8U%uWMNc9~dD_>+DOj5VibddDHFr(c$$kLv
zUNr@NVQgutYW@0gK8RqolbJSzJszhS^Q<ahf0x(es%x9fr8q8C`#=j+dI~fBQtDxw
ztA{)x^;Tq#^*WL#UpgrY$~buLAh}<}@zIq2rD=h-R%5T$hjRM2v0JGS5Xdu<<PZsH
zY>90DyAr^AN;2Bjd4pIinxY2+1fJD_^coJemQ9nI>JC2{B;yQ4_#6CDkt3M!tN`gb
zs}1t|m1fBUclc~eL4T7&*(7hI%*PN3m&It?4XG=Be?591JIYzd^ONRk9=6t1*+ROR
z9Y$g0xb3zuWsZy_zMOX%9)o&81*W=0+gR_Yk)#Ie+|CJswG7Pg<4(&)7;e0o*0b3O
zy)vgC@{13g9WI!4A`Y|Ko1VGJOp7!Kmjj)i`fpQ<*6q6c8IZ7jwe{09;+sDMd|&HP
z|KS~>(36X^Hpv6dlRBigGg1m=X{&xvs?3`kPjr4sj_ga)v1Da$rCC4W#&1(2_jWt|
zFem(;cemcE>>Z<V$)c-C)NOKnZ`9TUa997Y@H+<+#Jkr5I~;yjLH`wiC^1Kupt3;t
z8A8e2{$zQO@AQ-0<NWd#i9AE<gy`akBkWP!#BNTKsfb8M^XujAl|pW>{6Jt{^K_Z6
zeMOSi?%O^lp0~a4nA|kx?Yf2_u}Xz`trrSCS0W^;{ecLbS)Ci+Uoogs(zKQ`URZa%
zl9wQ|go2T8GEJw$RAN9FYuIvmwNU+K(&p>9Dl>F_y88IEEy5Ss^>t2}*JH2_RSc9A
zeinR4^5`RAhvRAyvry<LTl`(H5eC)>ImWl<y+gWmzHR8uAN&i_em%4okwflKMCXt&
z1rHsIu5A*fTBm5w*B*zSkm!Rqwc2x6I<!)({iIZs-QsWauM>HjWTxomtohgV28gD)
z3Nc}=!AGnR=HVE3!Q2MmQ0nT_t0)f$Rz6Vz@_Jx0EV=1Fl?(6L^>0<hYL@l5Bvjv8
zviPzNmbZ3>LcSB|uQ{}FA+%=?(5!xBl@|&I32tkC%b2|-ke*C=vt--WI-VHVVf1eJ
zG?S<f^6HHJq9@s=n(-oLY{08ioG*Fv>dIq6Ne#Mc575YsY`?yGj8|li@~K|cd*c$2
zGNX5{R@tyB{?1JE1HRnEejV~#u@`-)_!Sg~EUFy18h>!}kC`<Iv9oln!F&USa=zA+
zB(6z0Qm+y}cJ(R2p~^#L#-j><|HESLw_p#@QH$1)#bBUEU)-j`i-m1Qq1|915NBvG
ziFas%iIj}>Y#M@<!ca(`m*d;Nk;~bY45FhhybG)jUT6C(@@cg!L->pIoGy99SX;^$
zr~SFpHZ~Q7m5*Bf3xFv}c_cpSM=-o|pC<Fc0`H5oPtuyCV;-krBu@-<4T75TgLfm(
zPwfP)j-2K%fVm<_4(fZ5C;W=?bN#I>O~>{}IPg5($bH>9=~5hl;WB)E;GQH&>xKj4
z^c9<mfI8KoI#A@VQh|cvqfgl=#q&7hsH3qUe(%~>y0_<i9KFVo%5x=l>RS7v#4bZo
zlTUm}GSx8Z0r>5CA)CgGy9I}{0@z~QaJSCm6iRILDf^_XQJhXVOS-9Z<HVKW-Ieaq
z(Hex0fY>ahC3!D&evh078IWc9MRr?6J<?F!D3A%N3?C2hAY+7ZB~VjbA&Huy9X{6C
z3k7lFQZswHd1fNoJy>2Vq2)PdG(VS2W{qh1yLA;0`i)6M!50;(vtIWFCimnS9j%k|
z;r@L&@wcMSYh1Xx&aCv&nwh{q#K^sI7MALPizRx>V3UrKiGvPlT7iynL=<YuN_UEu
zU3N{?yhtfaz+Nu#(yVpllW~OcJanUhDJ?~R5nG8j)0U3>kg}=xo}f5IULrNkE%TCy
z*)UKS+|I?9zdiWszXj}yol<i2q0$p$6OafTeb$Hbt*9l()l8fHT*nTDvW<a(cFNqr
zDwA{*a)2X`u>GsteUv60jY69!BD6s(m>GO2c3~IG<{YgJFDaPiEO`*ox~c2|+U$+b
zAd1V6t|pnR0UDo1qESlGIl8K3KPlN74(%7=*dr?W6IS6z<+~30wmSd4spY!^zClgw
z76&t4mzPAI56uIX*9ji+8RBCeV5%78x_5IZ(dc-e>~hX)4~gn52f-Q`WE&#Sl9;3d
z0h8WlR$3M9T^p&^4BBSOQP5wuuPT*Fg_p){syf<oY;IN*7xrCc1kDJB5YDj+&m59Y
z5UpajgWe*ou@7PG+kuc#YViT0sO9k+HA}T2RxxN~w?50A2^I&Ibc@BgmbN;f+6Zki
zF(8b@WPRU(6SQfTHo)}nlj-?fmd}Zu>OsR7)+7X%gN8=wG%Qz@Sq#MIthPvLA8j37
zl8GG*_p(PJpQgCor>dH>$tmz+vABjr?7LV73ZzMg$WBXRl8WiUyN*&^Zu;pisgx=o
zR@Ne4%=9D<BxdTxg=i*b3RcGHNED(xY`tOK5>I-x&muT5Xm47usSpxJALKWg0Si#j
z?cM^LGyOmD^8Z}^1sc6E=li(`(jmcdL+(axi1fU5*NKo6)g~vy{+v|%iUum&+)Y-%
zxERL8`ek=<bXh5soW>>OiLg<^==g4A0BAvIN>Ber;Emdj<IO1cWep7a1izi#X`SB>
z4hf!0Lg~@^+Y~Ne*+wxucs4b1rV7CKwAP0!nH#n$Z@1E>B!qpOHE2>D_l6@3$tS+G
z(uEUc^DSvh3sok0PhgD6t#&~@;(Jp9Qzlq@d0lw)opFChlP3L!??KJKBxu9Sx^*Q{
zJkRQR-pD9hNDqR6b#U^vQE0d{h8{h-;IqvR4!?bFUIXeT(HVL$`eHM%rV0;Vks9pP
z>mj9fn|U|R4Mnqn<z0hRczhp`cu6O(8Pj1{4c3U(T`VCpiLbX*4^ApxRDt*#lL{d`
zZza(Saa7w4a=VJmtQ#43!7)(h+jI}Iw6d4W1MOFI`{}omeJtl`qn~lFu_S|v85t|@
zXS(RY;i)j6$#8;3@fK>puNZ$K4!7*_v6BB``DY-W;F#>}PlK`X<&B@MI2Uke_7Jx(
z0lren;c%JnFSE2?O}{8_m`Tgpe**YF|D7NoQT=7)7A^VJC`}QoZ3k@q$AOU0*iv0U
zvxF6P-S#Rov1(~aRJ{C-z@8tMT(m_LP1aY4Ui9P_v8%FQ{7^CaoY6h7vjCwpg&^&p
zYDl+~dZTcOr0LpESdNc9(!pylco=^l+Mv&DcV5`X8KgPj`MTOl8+-n3o0EO(v)9jM
zWbx9Q{=xaYU<ba0%#-;bbE*}wmW}RJP4Uo;|2q@n=4I|O@L-}yJz&e*ua!e>M>{O)
J{*g16{|`dMgtGtu


From 64cf4ad6ffce2dd054ba0a956b8e9c9d34a56376 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 20 Sep 2022 14:38:01 +0100
Subject: [PATCH 52/63] Fix parameter name typo

---
 Source/GRChomboCore/ChomboParameters.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index f5b6d71cd..a89c4f3c1 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -181,7 +181,7 @@ class ChomboParameters
                     false);
             pp.load("catalyst_write_vtk_files", catalyst_params.write_vtk_files,
                     false);
-            pp.load("catlyst_vtk_file_prefix", catalyst_params.vtk_file_prefix,
+            pp.load("catalyst_vtk_file_prefix", catalyst_params.vtk_file_prefix,
                     std::string("Catalyst_VTK_"));
         }
 #endif

From 5698b5504c89965dd691aa047dfd03528aea0096 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Thu, 27 Oct 2022 17:11:43 +0100
Subject: [PATCH 53/63] Fix segfault due to vtkOverlappingAMR reallocation

---
 Source/Insitu/CatalystAdaptor.cpp | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 4f8183206..2a2f0455b 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -115,10 +115,7 @@ void CatalystAdaptor::build_vtk_grid(vtkOverlappingAMR *a_vtk_grid_ptr)
     {
         pout() << "CatalystAdaptor::build_vtk_grid" << std::endl;
     }
-    if (a_vtk_grid_ptr != nullptr)
-        a_vtk_grid_ptr->Delete();
-
-    a_vtk_grid_ptr = vtkOverlappingAMR::New();
+    CH_assert(a_vtk_grid_ptr);
 
     const auto gramrlevels = m_gr_amr_ptr->get_gramrlevels();
 

From 3a3eed0d25f5a135d1f7d597038fd4c84fca9d6a Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 7 Dec 2022 18:48:16 +0000
Subject: [PATCH 54/63] Make make exit earlier if there are Catalyst problems

Previously the paraview-config command could fail and the build would
continue going which made it difficult for the user to see where the
problem is. Also commands that exit with non-zero exit codes are no
longer cached by the cache.sh script so that the paraview-config error
always appears.
---
 Source/Insitu/Make.insitu | 12 ++++++++++--
 Source/Insitu/cache.sh    |  9 +++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/Source/Insitu/Make.insitu b/Source/Insitu/Make.insitu
index 1a4be1cb1..f99d4565b 100644
--- a/Source/Insitu/Make.insitu
+++ b/Source/Insitu/Make.insitu
@@ -22,8 +22,16 @@ ifeq ($(USE_CATALYST),TRUE)
   PARAVIEW_COMPONENT_FLAGS := -c PythonCatalyst ${XTRA_PARAVIEW_COMPONENT_FLAGS}
   VTK_COMPONENT_FLAGS := -v CommonDataModel -v PythonUsed -v ParallelCore -v FiltersAMR -v IOParallelXML ${XTRA_VTK_COMPONENT_FLAGS}
   PARAVIEW_CONFIG_EXEC := $(PARAVIEW_DIR)/bin/paraview-config
-  cxxcppflags += $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --cppflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
-  XTRALDFLAGS += -rdynamic $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --ldflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
+  CATALYST_CPP_FLAGS := $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --cppflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
+  ifneq ($(.SHELLSTATUS),0)
+    $(error Error in paraview-config script. Is there a new enough version of CMake available?)
+  endif
+  cxxcppflags += $(CATALYST_CPP_FLAGS)
+  CATALYST_LD_FLAGS := $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --ldflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
+  ifneq ($(.SHELLSTATUS),0)
+    $(error Error in paraview-config script. Is there a new enough version of CMake available?)
+  endif
+  XTRALDFLAGS += -rdynamic $(CATALYST_LD_FLAGS)
 endif
 
 ifdef PARAVIEW_DIR
diff --git a/Source/Insitu/cache.sh b/Source/Insitu/cache.sh
index 78cb87a84..2447276d7 100755
--- a/Source/Insitu/cache.sh
+++ b/Source/Insitu/cache.sh
@@ -6,6 +6,7 @@
 
 # hash all arguments
 KEY="$@"
+EXIT_CODE=0
 
 # hash last modified dates of any files
 for arg in "$@"
@@ -30,9 +31,17 @@ done
 if [ -f $FILE ]
 then
   cat $FILE
+  exit $EXIT_CODE
 else
   # lock file
   touch $FILE_LOCK
   $@ | tee $FILE
+  EXIT_CODE=${PIPESTATUS[0]}
+  # delete cached output if command exited with an error
+  if [ "$EXIT_CODE" != "0" ]
+  then
+    rm $FILE
+  fi
   rm -f $FILE_LOCK
+  exit $EXIT_CODE
 fi

From 800deed97686e8a19257c73d0895bab09b4296a0 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Thu, 8 Dec 2022 16:50:13 +0000
Subject: [PATCH 55/63] Fix in-situ building with old Make versions

These don't have the .SHELLSTATUS variable.
---
 Source/Insitu/Make.insitu | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Source/Insitu/Make.insitu b/Source/Insitu/Make.insitu
index f99d4565b..269ec18ef 100644
--- a/Source/Insitu/Make.insitu
+++ b/Source/Insitu/Make.insitu
@@ -23,14 +23,18 @@ ifeq ($(USE_CATALYST),TRUE)
   VTK_COMPONENT_FLAGS := -v CommonDataModel -v PythonUsed -v ParallelCore -v FiltersAMR -v IOParallelXML ${XTRA_VTK_COMPONENT_FLAGS}
   PARAVIEW_CONFIG_EXEC := $(PARAVIEW_DIR)/bin/paraview-config
   CATALYST_CPP_FLAGS := $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --cppflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
+  ifdef .SHELLSTATUS
   ifneq ($(.SHELLSTATUS),0)
     $(error Error in paraview-config script. Is there a new enough version of CMake available?)
   endif
+  endif
   cxxcppflags += $(CATALYST_CPP_FLAGS)
   CATALYST_LD_FLAGS := $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --ldflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS))
+  ifdef .SHELLSTATUS
   ifneq ($(.SHELLSTATUS),0)
     $(error Error in paraview-config script. Is there a new enough version of CMake available?)
   endif
+  endif
   XTRALDFLAGS += -rdynamic $(CATALYST_LD_FLAGS)
 endif
 

From 5b6d28fc75b6e64658de4e8b39e612484fee9ec6 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Fri, 9 Dec 2022 14:38:27 +0000
Subject: [PATCH 56/63] Make GitHub action use latest ParaView version

Also update CMake version to 3.22.6.
---
 .github/workflows/run-catalyst-insitu-test.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/run-catalyst-insitu-test.yml b/.github/workflows/run-catalyst-insitu-test.yml
index c7b39d39c..92d8ec056 100644
--- a/.github/workflows/run-catalyst-insitu-test.yml
+++ b/.github/workflows/run-catalyst-insitu-test.yml
@@ -9,7 +9,6 @@ jobs:
       CHOMBO_HOME: ${{ github.workspace }}/Chombo/lib
       PARAVIEW_SUPERBUILD_DIR: ${{ github.workspace }}/build
       PARAVIEW_DIR: ${{ github.workspace }}/build/install
-      PARAVIEW_VERSION: 5.10.1
       OMP_NUM_THREADS: 1
 
     steps:
@@ -49,29 +48,30 @@ jobs:
         ninja-build libosmesa6 libosmesa6-dev llvm-12-dev
         sudo update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-12 12
 
-# There is a problem with the paraview-config script with CMake v3.23 
+# There is a problem with the paraview-config script with CMake v3.23
 # (see https://gitlab.kitware.com/paraview/paraview/-/issues/21361)
     - name: Downgrade CMake
       env:
-        CMAKE_VERSION: 3.22.4
+        CMAKE_VERSION: 3.22.6
       run: |
         wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh
         chmod +x cmake-${CMAKE_VERSION}-linux-x86_64.sh
         sudo ./cmake-${CMAKE_VERSION}-linux-x86_64.sh --prefix=/usr/local --exclude-subdir
 
-
     - name: Download pre-built ParaView
       uses: dsaltares/fetch-gh-release-asset@master
       with:
         repo: "GRChombo/paraview-build"
-        version: "tags/v${{ env.PARAVIEW_VERSION }}"
-        file: "paraview-v${{ env.PARAVIEW_VERSION }}.tar.gz"
+        version: latest
+        regex: true
+        file: "paraview-v.*.tar.gz"
 
     - name: Untar ParaView
       run: |
         mkdir -p $PARAVIEW_DIR
         cd $PARAVIEW_DIR
-        tar -xzvf $GITHUB_WORKSPACE/paraview-v${PARAVIEW_VERSION}.tar.gz
+        ls $GITHUB_WORKSPACE
+        tar -xzvf $GITHUB_WORKSPACE/paraview-v*.tar.gz
 
     - name: Build Catalyst Insitu test
       run: make test -j 4

From 15851ddc2ca659aa3a1a271f93db73f3e4dbb34d Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Mon, 16 Jan 2023 16:46:57 +0000
Subject: [PATCH 57/63] Update Catalyst README

---
 Source/Insitu/README.md | 43 +++++++++++++++++++++++++----------------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/Source/Insitu/README.md b/Source/Insitu/README.md
index 7c67e000e..56037c2f9 100644
--- a/Source/Insitu/README.md
+++ b/Source/Insitu/README.md
@@ -22,15 +22,15 @@ ParaView Catalyst with a GRChombo example.
 
 You will need a build of ParaView with Catalyst support. Unfortunately, the 
 pre-built binaries available on the ParaView website for various OSs and 
-architecture are not built with Catalyst support at the time of writing 
-(latest version: v5.10.0) so it is necessary to build ParaView from source.
-Given the large number of dependencies, it is probably easiest to use the 
-[superbuild](https://gitlab.kitware.com/paraview/paraview-superbuild/) 
-which downloads and builds all dependencies before building ParaView. When
-configuring the superbuild using CMake, you will want to make sure the following
-options are set
+architecture are not built with Catalyst support so it is necessary to build
+ParaView from source.  Given the large number of dependencies, it is probably
+easiest to use the 
+[superbuild](https://gitlab.kitware.com/paraview/paraview-superbuild/) which
+downloads and builds all dependencies before building ParaView. When configuring
+the superbuild using CMake, you will want to make sure the following options are
+set
 ```cmake
-ENABLE_osmesa = ON # enables rendering without a screen
+ENABLE_osmesa = ON # enables off-screen rendering
 USE_SYSTEM_llvm = ON # a prerequisite for OSMesa above
 USE_SYSTEM_mpi = ON # you will want to use the same MPI as for [GR]Chombo
 PARAVIEW_BUILD_EDITION = CATALYST_RENDERING # This will build without IO
@@ -61,6 +61,8 @@ This in-situ adaptor has been tested with the following versions of ParaView:
 | 5.8.1 |
 | 5.9.1 |
 | 5.10.0 |
+| 5.10.1 |
+| 5.11.0 |
 
 It may work with other versions but these have not been tested.
 
@@ -136,9 +138,17 @@ unset PARAVIEW_DIR
 
 ## Generating ParaView Catalyst scripts
 
-_Note that the procedure to generate Catalyst scripts varies between different
-versions of ParaView. In particular, there are significant differences before
-v5.9. The following instructions apply to v5.9.1 and v5.10.0._
+> **Note** 
+> The procedure to generate Catalyst scripts varies between
+> different versions of ParaView. In particular, there are significant
+> differences before v5.9. The following instructions apply to v5.9 or greater.
+
+First, note that you do not need to use a build of ParaView with Catalyst
+support in order to generate a Catalyst script. For example, you can just use
+the [pre-built binaries from the ParaView
+website](https://www.paraview.org/download/). In fact you probably cannot use
+the same build as you will want a version with the GUI which precludes a version
+with off-screen rendering (which you need for in-situ visualization).
 
 The easiest way to generate a suitable ParaView Catalyst Python script using the
 GUI is to load an HDF5 file that is similar to the configuration that will exist
@@ -148,8 +158,7 @@ initial data of the simulation. Once you have loaded the file into ParaView,
 selecting 'Rename'). This is necessary so that Catalyst can replace the HDF5
 data in the script with that coming from the simulation. 
 
-Next apply filters and adjust the camera view as desired. Note that some filters
-do not currently work in situ (e.g. 'Slice AMR Data').
+Next apply filters and adjust the camera view as desired.
 
 In order to generate a valid Catalyst script, it is then necessary to add an 
 'Extractor' which will, for example, save an image. This can be done using the
@@ -160,9 +169,9 @@ Save Catalyst State. By default, the output directory for Extracts will be
 'dataset'.
 
 Unfortunately, if you have built the `CATALYST_RENDERING` ParaView edition
-as described [above](#prerequisites), since this does not contain any IO 
-libraries, it is necessary to manually edit the generated Catalyst script
-and replace the line
+as described [above](#prerequisites) for running in-situ, since this does not
+contain any IO libraries, it is necessary to manually edit the generated
+Catalyst script and replace the line
 ```python
 input = VisItChomboReader(registrationName='input', FileName=<list of hdf5 files>)
 ```
@@ -190,7 +199,7 @@ visualization processing which are described in the table below.
 | `catalyst_abort_on_error`  | `bool`           | `true`/[`false`] | If `true`, the code will abort if there is an error in Catalyst. |
 | `catalyst_num_vars`*        | `int`            | [`0`],...,`NUM_VARS + NUM_DIAGNOSTIC_VARS` | Number of variables in `catalyst_vars` |
 | `catalyst_vars`*            | `vector<string>` | - | Restrict variables passed to Catalyst to only these ones. No restriction if `catalyst_num_vars == 0`.|
-| `catalyst_write_vtk_files`  | `bool`           | `true`/[`false`] | Controls whether VTK XML files containing the full 3D AMR data passed to Catalyst are written or not. These files can be opened in ParaView |
+| `catalyst_write_vtk_files`  | `bool`           | `true`/[`false`] | Controls whether VTK XML files containing the full 3D AMR data passed to Catalyst are written or not. These files can be opened in ParaView. |
 | `vtk_file_prefix`           | `string`         | [`"Catalyst_VTK_"`] | Filename prefix for VTK XML files. The filenames will be of the form `<vtk_file_prefix>_<6 digit timestep on catalyst_coprocess_level>`. Note there will be multiple files for each timestep: one of type `.vth` and a subdirectory containing `.vti` files for each box. |
 
 *Note that scripts generated with versions of ParaView v5.8 or earlier provide 

From e78dee1c75083b0e7ef7e8fb3700b4d8050496c9 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 17 Jan 2023 16:45:58 +0000
Subject: [PATCH 58/63] Add passing of GRChombo parameters to Catalyst

These are set as environment variables which can be recovered in the
Catalyst Python script. Also update slice_chi.py.
---
 .../BinaryBH/catalyst_scripts/slice_chi.py    | 108 ++++++++++--------
 Source/GRChomboCore/ChomboParameters.hpp      |  22 ++++
 Source/GRChomboCore/SetupFunctions.hpp        |   3 +
 Source/Insitu/README.md                       |  13 ++-
 4 files changed, 98 insertions(+), 48 deletions(-)

diff --git a/Examples/BinaryBH/catalyst_scripts/slice_chi.py b/Examples/BinaryBH/catalyst_scripts/slice_chi.py
index dd0a4ab22..b8346b60f 100644
--- a/Examples/BinaryBH/catalyst_scripts/slice_chi.py
+++ b/Examples/BinaryBH/catalyst_scripts/slice_chi.py
@@ -1,30 +1,39 @@
 # script-version: 2.0
 # Catalyst state generated using paraview version 5.9.1
 
+# Import this to get environment variables
+import os
+
 #### import the simple module from the paraview
 from paraview.simple import *
 #### disable automatic camera reset on 'Show'
 paraview.simple._DisableFirstRenderCameraReset()
 
+# Get center from environment
+grchombo_center_str = os.getenv('GRCHOMBO_PARAM_CENTER')
+grchombo_center = [float(dim) for dim in grchombo_center_str.split(' ')]
+
 # ----------------------------------------------------------------
 # setup views used in the visualization
 # ----------------------------------------------------------------
 
+# black background
+LoadPalette('BlackBackground')
+
 # get the material library
 materialLibrary1 = GetMaterialLibrary()
 
 # Create a new 'Render View'
 renderView1 = CreateView('RenderView')
-renderView1.ViewSize = [885, 774]
+renderView1.ViewSize = [1092, 778]
 renderView1.AxesGrid = 'GridAxes3DActor'
-renderView1.CenterAxesVisibility = 1
-renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
+renderView1.CenterAxesVisibility = 0
+renderView1.CenterOfRotation = grchombo_center
 renderView1.StereoType = 'Crystal Eyes'
-renderView1.CameraPosition = [32.0, 32.0, 223.24355652982143]
-renderView1.CameraFocalPoint = [32.0, 32.0, 32.0]
+renderView1.CameraPosition = [grchombo_center[0], grchombo_center[1], 500.0]
+renderView1.CameraFocalPoint = grchombo_center
 renderView1.CameraFocalDisk = 1.0
-renderView1.CameraParallelScale = 49.49747468305833
-renderView1.CameraParallelProjection = 1
+renderView1.CameraParallelScale = 11.096170510586074
 renderView1.BackEnd = 'OSPRay raycaster'
 renderView1.OSPRayMaterialLibrary = materialLibrary1
 
@@ -40,7 +49,7 @@
 # create new layout object 'Layout #1'
 layout1 = CreateLayout(name='Layout #1')
 layout1.AssignView(0, renderView1)
-layout1.SetSize(885, 774)
+layout1.SetSize(1092, 778)
 
 # ----------------------------------------------------------------
 # restore active view
@@ -52,57 +61,58 @@
 # ----------------------------------------------------------------
 
 # create a new 'VisItChomboReader'
-input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000003.3d.hdf5'])
+input = AMRGaussianPulseSource(registrationName='input')
 input.MeshStatus = ['Mesh']
 input.CellArrayStatus = ['chi']
 
-# create a new 'Slice AMR data'
-sliceAMRdata1 = SliceAMRdata(registrationName='SliceAMRdata1', Input=input)
-sliceAMRdata1.Level = 3
-sliceAMRdata1.OffSet = 35.0
-sliceAMRdata1.Normal = 'Z-Normal'
+# create a new 'Slice'
+slice1 = Slice(registrationName='Slice1', Input=input)
+slice1.SliceType = 'Plane'
+slice1.HyperTreeGridSlicer = 'Plane'
+slice1.Triangulatetheslice = 0
+slice1.SliceOffsetValues = [0.0]
+
+# init the 'Plane' selected for 'SliceType'
+slice1.SliceType.Origin = grchombo_center
+slice1.SliceType.Normal = [0.0, 0.0, 1.0]
+
+# init the 'Plane' selected for 'HyperTreeGridSlicer'
+slice1.HyperTreeGridSlicer.Origin = grchombo_center
 
 # ----------------------------------------------------------------
 # setup the visualization in view 'renderView1'
 # ----------------------------------------------------------------
 
-# show data from sliceAMRdata1
-sliceAMRdata1Display = Show(sliceAMRdata1, renderView1, 'AMRRepresentation')
+# show data from slice1
+slice1Display = Show(slice1, renderView1, 'GeometryRepresentation')
 
 # get color transfer function/color map for 'chi'
 chiLUT = GetColorTransferFunction('chi')
 chiLUT.AutomaticRescaleRangeMode = 'Never'
-chiLUT.RGBPoints = [0.0, 0.267004, 0.004874, 0.329415, 0.003921999999999982, 0.26851, 0.009605, 0.335427, 0.007843000000000046, 0.269944, 0.014625, 0.341379, 0.011765000000000027, 0.271305, 0.019942, 0.347269, 0.01568599999999998, 0.272594, 0.025563, 0.353093, 0.019607999999999962, 0.273809, 0.031497, 0.358853, 0.02352900000000003, 0.274952, 0.037752, 0.364543, 0.027451000000000014, 0.276022, 0.044167, 0.370164, 0.03137299999999999, 0.277018, 0.050344, 0.375715, 0.035294000000000054, 0.277941, 0.056324, 0.381191, 0.039216000000000036, 0.278791, 0.062145, 0.386592, 0.04313699999999999, 0.279566, 0.067836, 0.391917, 0.04705899999999996, 0.280267, 0.073417, 0.397163, 0.05098000000000004, 0.280894, 0.078907, 0.402329, 0.05490200000000001, 0.281446, 0.08432, 0.407414, 0.05882399999999999, 0.281924, 0.089666, 0.412415, 0.06274499999999995, 0.282327, 0.094955, 0.417331, 0.06666700000000005, 0.282656, 0.100196, 0.42216, 0.07058799999999998, 0.28291, 0.105393, 0.426902, 0.07450999999999998, 0.283091, 0.110553, 0.431554, 0.07843100000000004, 0.283197, 0.11568, 0.436115, 0.0823529999999999, 0.283229, 0.120777, 0.440584, 0.08627499999999999, 0.283187, 0.125848, 0.44496, 0.09019599999999996, 0.283072, 0.130895, 0.449241, 0.09411799999999991, 0.282884, 0.13592, 0.453427, 0.098039, 0.282623, 0.140926, 0.457517, 0.10196099999999998, 0.28229, 0.145912, 0.46151, 0.10588199999999992, 0.281887, 0.150881, 0.465405, 0.10980400000000001, 0.281412, 0.155834, 0.469201, 0.11372499999999998, 0.280868, 0.160771, 0.472899, 0.11764699999999993, 0.280255, 0.165693, 0.476498, 0.12156899999999993, 0.279574, 0.170599, 0.479997, 0.12549, 0.278826, 0.17549, 0.483397, 0.12941199999999997, 0.278012, 0.180367, 0.486697, 0.13333300000000006, 0.277134, 0.185228, 0.489898, 0.13725500000000004, 0.276194, 0.190074, 0.493001, 0.141176, 0.275191, 0.194905, 0.496005, 0.14509799999999998, 0.274128, 0.199721, 0.498911, 0.14902000000000007, 0.273006, 0.20452, 0.501721, 0.15294100000000002, 0.271828, 0.209303, 0.504434, 0.1568630000000001, 0.270595, 0.214069, 0.507052, 0.1607840000000001, 0.269308, 0.218818, 0.509577, 0.16470600000000007, 0.267968, 0.223549, 0.512008, 0.168627, 0.26658, 0.228262, 0.514349, 0.17254900000000012, 0.265145, 0.232956, 0.516599, 0.1764710000000001, 0.263663, 0.237631, 0.518762, 0.18039200000000005, 0.262138, 0.242286, 0.520837, 0.1843140000000001, 0.260571, 0.246922, 0.522828, 0.18823500000000007, 0.258965, 0.251537, 0.524736, 0.19215700000000005, 0.257322, 0.25613, 0.526563, 0.1960780000000001, 0.255645, 0.260703, 0.528312, 0.2000000000000001, 0.253935, 0.265254, 0.529983, 0.20392200000000008, 0.252194, 0.269783, 0.531579, 0.20784300000000003, 0.250425, 0.27429, 0.533103, 0.211765, 0.248629, 0.278775, 0.534556, 0.21568599999999996, 0.246811, 0.283237, 0.535941, 0.21960800000000005, 0.244972, 0.287675, 0.53726, 0.22352900000000014, 0.243113, 0.292092, 0.538516, 0.22745099999999996, 0.241237, 0.296485, 0.539709, 0.23137300000000008, 0.239346, 0.300855, 0.540844, 0.23529400000000006, 0.237441, 0.305202, 0.541921, 0.23921600000000004, 0.235526, 0.309527, 0.542944, 0.24313700000000008, 0.233603, 0.313828, 0.543914, 0.24705900000000008, 0.231674, 0.318106, 0.544834, 0.25098000000000004, 0.229739, 0.322361, 0.545706, 0.254902, 0.227802, 0.326594, 0.546532, 0.258824, 0.225863, 0.330805, 0.547314, 0.2627450000000001, 0.223925, 0.334994, 0.548053, 0.26666700000000004, 0.221989, 0.339161, 0.548752, 0.27058800000000016, 0.220057, 0.343307, 0.549413, 0.27451, 0.21813, 0.347432, 0.550038, 0.27843100000000004, 0.21621, 0.351535, 0.550627, 0.282353, 0.214298, 0.355619, 0.551184, 0.2862750000000001, 0.212395, 0.359683, 0.55171, 0.29019600000000007, 0.210503, 0.363727, 0.552206, 0.29411800000000016, 0.208623, 0.367752, 0.552675, 0.298039, 0.206756, 0.371758, 0.553117, 0.3019610000000001, 0.204903, 0.375746, 0.553533, 0.30588200000000004, 0.203063, 0.379716, 0.553925, 0.30980400000000013, 0.201239, 0.38367, 0.554294, 0.3137249999999999, 0.19943, 0.387607, 0.554642, 0.31764700000000007, 0.197636, 0.391528, 0.554969, 0.32156900000000005, 0.19586, 0.395433, 0.555276, 0.32549, 0.1941, 0.399323, 0.555565, 0.329412, 0.192357, 0.403199, 0.555836, 0.33333300000000005, 0.190631, 0.407061, 0.556089, 0.3372550000000002, 0.188923, 0.41091, 0.556326, 0.341176, 0.187231, 0.414746, 0.556547, 0.3450980000000001, 0.185556, 0.41857, 0.556753, 0.34902000000000005, 0.183898, 0.422383, 0.556944, 0.352941, 0.182256, 0.426184, 0.55712, 0.356863, 0.180629, 0.429975, 0.557282, 0.36078400000000005, 0.179019, 0.433756, 0.55743, 0.36470600000000003, 0.177423, 0.437527, 0.557565, 0.368627, 0.175841, 0.44129, 0.557685, 0.3725490000000001, 0.174274, 0.445044, 0.557792, 0.37647100000000006, 0.172719, 0.448791, 0.557885, 0.380392, 0.171176, 0.45253, 0.557965, 0.384314, 0.169646, 0.456262, 0.55803, 0.3882350000000001, 0.168126, 0.459988, 0.558082, 0.39215700000000003, 0.166617, 0.463708, 0.558119, 0.39607800000000015, 0.165117, 0.467423, 0.558141, 0.4000000000000001, 0.163625, 0.471133, 0.558148, 0.4039220000000001, 0.162142, 0.474838, 0.55814, 0.407843, 0.160665, 0.47854, 0.558115, 0.41176500000000016, 0.159194, 0.482237, 0.558073, 0.41568600000000006, 0.157729, 0.485932, 0.558013, 0.4196080000000001, 0.15627, 0.489624, 0.557936, 0.423529, 0.154815, 0.493313, 0.55784, 0.4274510000000001, 0.153364, 0.497, 0.557724, 0.43137300000000006, 0.151918, 0.500685, 0.557587, 0.435294, 0.150476, 0.504369, 0.55743, 0.4392160000000001, 0.149039, 0.508051, 0.55725, 0.44313700000000006, 0.147607, 0.511733, 0.557049, 0.44705900000000004, 0.14618, 0.515413, 0.556823, 0.45098, 0.144759, 0.519093, 0.556572, 0.4549019999999999, 0.143343, 0.522773, 0.556295, 0.45882400000000007, 0.141935, 0.526453, 0.555991, 0.4627450000000002, 0.140536, 0.530132, 0.555659, 0.466667, 0.139147, 0.533812, 0.555298, 0.4705880000000001, 0.13777, 0.537492, 0.554906, 0.47451000000000004, 0.136408, 0.541173, 0.554483, 0.4784310000000001, 0.135066, 0.544853, 0.554029, 0.4823530000000001, 0.133743, 0.548535, 0.553541, 0.4862750000000001, 0.132444, 0.552216, 0.553018, 0.490196, 0.131172, 0.555899, 0.552459, 0.4941180000000001, 0.129933, 0.559582, 0.551864, 0.49803900000000006, 0.128729, 0.563265, 0.551229, 0.501961, 0.127568, 0.566949, 0.550556, 0.505882, 0.126453, 0.570633, 0.549841, 0.509804, 0.125394, 0.574318, 0.549086, 0.5137250000000001, 0.124395, 0.578002, 0.548287, 0.5176470000000002, 0.123463, 0.581687, 0.547445, 0.5215690000000001, 0.122606, 0.585371, 0.546557, 0.5254900000000002, 0.121831, 0.589055, 0.545623, 0.5294120000000004, 0.121148, 0.592739, 0.544641, 0.5333330000000002, 0.120565, 0.596422, 0.543611, 0.5372550000000001, 0.120092, 0.600104, 0.54253, 0.5411760000000003, 0.119738, 0.603785, 0.5414, 0.5450980000000003, 0.119512, 0.607464, 0.540218, 0.5490200000000003, 0.119423, 0.611141, 0.538982, 0.552941, 0.119483, 0.614817, 0.537692, 0.5568630000000002, 0.119699, 0.61849, 0.536347, 0.5607840000000002, 0.120081, 0.622161, 0.534946, 0.5647060000000002, 0.120638, 0.625828, 0.533488, 0.5686270000000001, 0.12138, 0.629492, 0.531973, 0.5725490000000001, 0.122312, 0.633153, 0.530398, 0.5764710000000003, 0.123444, 0.636809, 0.528763, 0.5803920000000002, 0.12478, 0.640461, 0.527068, 0.5843140000000001, 0.126326, 0.644107, 0.525311, 0.5882350000000002, 0.128087, 0.647749, 0.523491, 0.5921570000000003, 0.130067, 0.651384, 0.521608, 0.5960780000000001, 0.132268, 0.655014, 0.519661, 0.6, 0.134692, 0.658636, 0.517649, 0.6039220000000002, 0.137339, 0.662252, 0.515571, 0.6078430000000002, 0.14021, 0.665859, 0.513427, 0.6117650000000003, 0.143303, 0.669459, 0.511215, 0.6156860000000001, 0.146616, 0.67305, 0.508936, 0.6196080000000003, 0.150148, 0.676631, 0.506589, 0.6235290000000002, 0.153894, 0.680203, 0.504172, 0.6274510000000002, 0.157851, 0.683765, 0.501686, 0.6313730000000002, 0.162016, 0.687316, 0.499129, 0.6352940000000001, 0.166383, 0.690856, 0.496502, 0.6392160000000003, 0.170948, 0.694384, 0.493803, 0.6431370000000001, 0.175707, 0.6979, 0.491033, 0.6470590000000003, 0.180653, 0.701402, 0.488189, 0.6509800000000002, 0.185783, 0.704891, 0.485273, 0.6549020000000002, 0.19109, 0.708366, 0.482284, 0.6588240000000003, 0.196571, 0.711827, 0.479221, 0.6627450000000001, 0.202219, 0.715272, 0.476084, 0.6666670000000003, 0.20803, 0.718701, 0.472873, 0.6705880000000001, 0.214, 0.722114, 0.469588, 0.6745100000000003, 0.220124, 0.725509, 0.466226, 0.6784310000000002, 0.226397, 0.728888, 0.462789, 0.6823530000000004, 0.232815, 0.732247, 0.459277, 0.6862750000000001, 0.239374, 0.735588, 0.455688, 0.6901960000000003, 0.24607, 0.73891, 0.452024, 0.6941180000000002, 0.252899, 0.742211, 0.448284, 0.6980390000000002, 0.259857, 0.745492, 0.444467, 0.7019610000000002, 0.266941, 0.748751, 0.440573, 0.7058820000000001, 0.274149, 0.751988, 0.436601, 0.7098040000000001, 0.281477, 0.755203, 0.432552, 0.7137250000000003, 0.288921, 0.758394, 0.428426, 0.7176470000000004, 0.296479, 0.761561, 0.424223, 0.7215690000000002, 0.304148, 0.764704, 0.419943, 0.7254900000000003, 0.311925, 0.767822, 0.415586, 0.7294120000000002, 0.319809, 0.770914, 0.411152, 0.7333330000000003, 0.327796, 0.77398, 0.40664, 0.7372550000000001, 0.335885, 0.777018, 0.402049, 0.7411760000000003, 0.344074, 0.780029, 0.397381, 0.7450980000000003, 0.35236, 0.783011, 0.392636, 0.7490200000000001, 0.360741, 0.785964, 0.387814, 0.7529410000000001, 0.369214, 0.788888, 0.382914, 0.7568630000000001, 0.377779, 0.791781, 0.377939, 0.7607840000000001, 0.386433, 0.794644, 0.372886, 0.7647060000000001, 0.395174, 0.797475, 0.367757, 0.768627, 0.404001, 0.800275, 0.362552, 0.7725490000000002, 0.412913, 0.803041, 0.357269, 0.7764710000000001, 0.421908, 0.805774, 0.35191, 0.7803920000000001, 0.430983, 0.808473, 0.346476, 0.7843140000000001, 0.440137, 0.811138, 0.340967, 0.7882350000000002, 0.449368, 0.813768, 0.335384, 0.7921570000000001, 0.458674, 0.816363, 0.329727, 0.7960780000000001, 0.468053, 0.818921, 0.323998, 0.8000000000000002, 0.477504, 0.821444, 0.318195, 0.803922, 0.487026, 0.823929, 0.312321, 0.8078430000000002, 0.496615, 0.826376, 0.306377, 0.8117650000000002, 0.506271, 0.828786, 0.300362, 0.8156860000000001, 0.515992, 0.831158, 0.294279, 0.819608, 0.525776, 0.833491, 0.288127, 0.8235290000000002, 0.535621, 0.835785, 0.281908, 0.8274510000000003, 0.545524, 0.838039, 0.275626, 0.8313730000000001, 0.555484, 0.840254, 0.269281, 0.835294, 0.565498, 0.84243, 0.262877, 0.8392160000000001, 0.575563, 0.844566, 0.256415, 0.8431370000000002, 0.585678, 0.846661, 0.249897, 0.8470590000000001, 0.595839, 0.848717, 0.243329, 0.8509800000000001, 0.606045, 0.850733, 0.236712, 0.8549020000000003, 0.616293, 0.852709, 0.230052, 0.8588240000000003, 0.626579, 0.854645, 0.223353, 0.8627450000000001, 0.636902, 0.856542, 0.21662, 0.866667, 0.647257, 0.8584, 0.209861, 0.8705880000000001, 0.657642, 0.860219, 0.203082, 0.8745100000000002, 0.668054, 0.861999, 0.196293, 0.8784310000000002, 0.678489, 0.863742, 0.189503, 0.8823530000000003, 0.688944, 0.865448, 0.182725, 0.886275, 0.699415, 0.867117, 0.175971, 0.8901960000000002, 0.709898, 0.868751, 0.169257, 0.8941180000000002, 0.720391, 0.87035, 0.162603, 0.8980390000000001, 0.730889, 0.871916, 0.156029, 0.901961, 0.741388, 0.873449, 0.149561, 0.9058820000000002, 0.751884, 0.874951, 0.143228, 0.9098040000000002, 0.762373, 0.876424, 0.137064, 0.9137250000000001, 0.772852, 0.877868, 0.131109, 0.9176470000000002, 0.783315, 0.879285, 0.125405, 0.9215690000000001, 0.79376, 0.880678, 0.120005, 0.9254900000000004, 0.804182, 0.882046, 0.114965, 0.9294120000000001, 0.814576, 0.883393, 0.110347, 0.9333330000000003, 0.82494, 0.88472, 0.106217, 0.9372550000000002, 0.83527, 0.886029, 0.102646, 0.9411760000000003, 0.845561, 0.887322, 0.099702, 0.945098, 0.85581, 0.888601, 0.097452, 0.94902, 0.866013, 0.889868, 0.095953, 0.9529410000000003, 0.876168, 0.891125, 0.09525, 0.9568630000000002, 0.886271, 0.892374, 0.095374, 0.960784, 0.89632, 0.893616, 0.096335, 0.964706, 0.906311, 0.894855, 0.098125, 0.9686270000000003, 0.916242, 0.896091, 0.100717, 0.9725490000000003, 0.926106, 0.89733, 0.104071, 0.9764710000000002, 0.935904, 0.89857, 0.108131, 0.980392, 0.945636, 0.899815, 0.112838, 0.984314, 0.9553, 0.901065, 0.118128, 0.9882350000000002, 0.964894, 0.902323, 0.123941, 0.9921570000000003, 0.974417, 0.90359, 0.130215, 0.996078, 0.983868, 0.904867, 0.136897, 1.0, 0.993248, 0.906157, 0.143936]
+chiLUT.RGBPoints = [0.0, 0.267004, 0.004874, 0.329415, 0.003922000000000001, 0.26851, 0.009605, 0.335427, 0.007843, 0.269944, 0.014625, 0.341379, 0.011765000000000001, 0.271305, 0.019942, 0.347269, 0.015686, 0.272594, 0.025563, 0.353093, 0.019608, 0.273809, 0.031497, 0.358853, 0.023529, 0.274952, 0.037752, 0.364543, 0.027451000000000003, 0.276022, 0.044167, 0.370164, 0.031373, 0.277018, 0.050344, 0.375715, 0.035294, 0.277941, 0.056324, 0.381191, 0.039216, 0.278791, 0.062145, 0.386592, 0.043137, 0.279566, 0.067836, 0.391917, 0.047059000000000004, 0.280267, 0.073417, 0.397163, 0.050980000000000004, 0.280894, 0.078907, 0.402329, 0.054902, 0.281446, 0.08432, 0.407414, 0.058824, 0.281924, 0.089666, 0.412415, 0.062745, 0.282327, 0.094955, 0.417331, 0.06666699999999999, 0.282656, 0.100196, 0.42216, 0.07058799999999998, 0.28291, 0.105393, 0.426902, 0.07451, 0.283091, 0.110553, 0.431554, 0.078431, 0.283197, 0.11568, 0.436115, 0.08235299999999998, 0.283229, 0.120777, 0.440584, 0.086275, 0.283187, 0.125848, 0.44496, 0.090196, 0.283072, 0.130895, 0.449241, 0.094118, 0.282884, 0.13592, 0.453427, 0.098039, 0.282623, 0.140926, 0.457517, 0.10196099999999998, 0.28229, 0.145912, 0.46151, 0.10588199999999999, 0.281887, 0.150881, 0.465405, 0.109804, 0.281412, 0.155834, 0.469201, 0.113725, 0.280868, 0.160771, 0.472899, 0.11764699999999999, 0.280255, 0.165693, 0.476498, 0.121569, 0.279574, 0.170599, 0.479997, 0.12549, 0.278826, 0.17549, 0.483397, 0.129412, 0.278012, 0.180367, 0.486697, 0.133333, 0.277134, 0.185228, 0.489898, 0.137255, 0.276194, 0.190074, 0.493001, 0.141176, 0.275191, 0.194905, 0.496005, 0.145098, 0.274128, 0.199721, 0.498911, 0.14902000000000004, 0.273006, 0.20452, 0.501721, 0.15294100000000002, 0.271828, 0.209303, 0.504434, 0.15686300000000003, 0.270595, 0.214069, 0.507052, 0.16078400000000004, 0.269308, 0.218818, 0.509577, 0.164706, 0.267968, 0.223549, 0.512008, 0.168627, 0.26658, 0.228262, 0.514349, 0.172549, 0.265145, 0.232956, 0.516599, 0.176471, 0.263663, 0.237631, 0.518762, 0.180392, 0.262138, 0.242286, 0.520837, 0.184314, 0.260571, 0.246922, 0.522828, 0.188235, 0.258965, 0.251537, 0.524736, 0.19215700000000002, 0.257322, 0.25613, 0.526563, 0.19607800000000003, 0.255645, 0.260703, 0.528312, 0.2, 0.253935, 0.265254, 0.529983, 0.203922, 0.252194, 0.269783, 0.531579, 0.207843, 0.250425, 0.27429, 0.533103, 0.211765, 0.248629, 0.278775, 0.534556, 0.215686, 0.246811, 0.283237, 0.535941, 0.219608, 0.244972, 0.287675, 0.53726, 0.223529, 0.243113, 0.292092, 0.538516, 0.22745100000000001, 0.241237, 0.296485, 0.539709, 0.231373, 0.239346, 0.300855, 0.540844, 0.235294, 0.237441, 0.305202, 0.541921, 0.239216, 0.235526, 0.309527, 0.542944, 0.243137, 0.233603, 0.313828, 0.543914, 0.247059, 0.231674, 0.318106, 0.544834, 0.25098, 0.229739, 0.322361, 0.545706, 0.254902, 0.227802, 0.326594, 0.546532, 0.258824, 0.225863, 0.330805, 0.547314, 0.262745, 0.223925, 0.334994, 0.548053, 0.266667, 0.221989, 0.339161, 0.548752, 0.270588, 0.220057, 0.343307, 0.549413, 0.27451, 0.21813, 0.347432, 0.550038, 0.27843100000000004, 0.21621, 0.351535, 0.550627, 0.2823530000000001, 0.214298, 0.355619, 0.551184, 0.286275, 0.212395, 0.359683, 0.55171, 0.290196, 0.210503, 0.363727, 0.552206, 0.294118, 0.208623, 0.367752, 0.552675, 0.298039, 0.206756, 0.371758, 0.553117, 0.301961, 0.204903, 0.375746, 0.553533, 0.305882, 0.203063, 0.379716, 0.553925, 0.309804, 0.201239, 0.38367, 0.554294, 0.313725, 0.19943, 0.387607, 0.554642, 0.317647, 0.197636, 0.391528, 0.554969, 0.321569, 0.19586, 0.395433, 0.555276, 0.32549, 0.1941, 0.399323, 0.555565, 0.329412, 0.192357, 0.403199, 0.555836, 0.333333, 0.190631, 0.407061, 0.556089, 0.337255, 0.188923, 0.41091, 0.556326, 0.341176, 0.187231, 0.414746, 0.556547, 0.345098, 0.185556, 0.41857, 0.556753, 0.34902, 0.183898, 0.422383, 0.556944, 0.35294100000000006, 0.182256, 0.426184, 0.55712, 0.356863, 0.180629, 0.429975, 0.557282, 0.360784, 0.179019, 0.433756, 0.55743, 0.364706, 0.177423, 0.437527, 0.557565, 0.368627, 0.175841, 0.44129, 0.557685, 0.372549, 0.174274, 0.445044, 0.557792, 0.376471, 0.172719, 0.448791, 0.557885, 0.380392, 0.171176, 0.45253, 0.557965, 0.38431400000000004, 0.169646, 0.456262, 0.55803, 0.388235, 0.168126, 0.459988, 0.558082, 0.392157, 0.166617, 0.463708, 0.558119, 0.39607800000000004, 0.165117, 0.467423, 0.558141, 0.4, 0.163625, 0.471133, 0.558148, 0.403922, 0.162142, 0.474838, 0.55814, 0.407843, 0.160665, 0.47854, 0.558115, 0.411765, 0.159194, 0.482237, 0.558073, 0.415686, 0.157729, 0.485932, 0.558013, 0.419608, 0.15627, 0.489624, 0.557936, 0.42352900000000004, 0.154815, 0.493313, 0.55784, 0.4274510000000001, 0.153364, 0.497, 0.557724, 0.431373, 0.151918, 0.500685, 0.557587, 0.435294, 0.150476, 0.504369, 0.55743, 0.439216, 0.149039, 0.508051, 0.55725, 0.443137, 0.147607, 0.511733, 0.557049, 0.447059, 0.14618, 0.515413, 0.556823, 0.45098, 0.144759, 0.519093, 0.556572, 0.45490200000000003, 0.143343, 0.522773, 0.556295, 0.45882400000000007, 0.141935, 0.526453, 0.555991, 0.462745, 0.140536, 0.530132, 0.555659, 0.466667, 0.139147, 0.533812, 0.555298, 0.470588, 0.13777, 0.537492, 0.554906, 0.47451, 0.136408, 0.541173, 0.554483, 0.478431, 0.135066, 0.544853, 0.554029, 0.482353, 0.133743, 0.548535, 0.553541, 0.486275, 0.132444, 0.552216, 0.553018, 0.490196, 0.131172, 0.555899, 0.552459, 0.494118, 0.129933, 0.559582, 0.551864, 0.49803900000000006, 0.128729, 0.563265, 0.551229, 0.501961, 0.127568, 0.566949, 0.550556, 0.505882, 0.126453, 0.570633, 0.549841, 0.509804, 0.125394, 0.574318, 0.549086, 0.513725, 0.124395, 0.578002, 0.548287, 0.517647, 0.123463, 0.581687, 0.547445, 0.5215690000000001, 0.122606, 0.585371, 0.546557, 0.52549, 0.121831, 0.589055, 0.545623, 0.529412, 0.121148, 0.592739, 0.544641, 0.5333330000000001, 0.120565, 0.596422, 0.543611, 0.537255, 0.120092, 0.600104, 0.54253, 0.541176, 0.119738, 0.603785, 0.5414, 0.545098, 0.119512, 0.607464, 0.540218, 0.54902, 0.119423, 0.611141, 0.538982, 0.552941, 0.119483, 0.614817, 0.537692, 0.556863, 0.119699, 0.61849, 0.536347, 0.560784, 0.120081, 0.622161, 0.534946, 0.5647060000000002, 0.120638, 0.625828, 0.533488, 0.568627, 0.12138, 0.629492, 0.531973, 0.572549, 0.122312, 0.633153, 0.530398, 0.576471, 0.123444, 0.636809, 0.528763, 0.580392, 0.12478, 0.640461, 0.527068, 0.584314, 0.126326, 0.644107, 0.525311, 0.588235, 0.128087, 0.647749, 0.523491, 0.592157, 0.130067, 0.651384, 0.521608, 0.596078, 0.132268, 0.655014, 0.519661, 0.6, 0.134692, 0.658636, 0.517649, 0.603922, 0.137339, 0.662252, 0.515571, 0.6078430000000001, 0.14021, 0.665859, 0.513427, 0.611765, 0.143303, 0.669459, 0.511215, 0.615686, 0.146616, 0.67305, 0.508936, 0.619608, 0.150148, 0.676631, 0.506589, 0.623529, 0.153894, 0.680203, 0.504172, 0.627451, 0.157851, 0.683765, 0.501686, 0.631373, 0.162016, 0.687316, 0.499129, 0.635294, 0.166383, 0.690856, 0.496502, 0.6392160000000001, 0.170948, 0.694384, 0.493803, 0.643137, 0.175707, 0.6979, 0.491033, 0.647059, 0.180653, 0.701402, 0.488189, 0.65098, 0.185783, 0.704891, 0.485273, 0.654902, 0.19109, 0.708366, 0.482284, 0.658824, 0.196571, 0.711827, 0.479221, 0.662745, 0.202219, 0.715272, 0.476084, 0.666667, 0.20803, 0.718701, 0.472873, 0.670588, 0.214, 0.722114, 0.469588, 0.67451, 0.220124, 0.725509, 0.466226, 0.678431, 0.226397, 0.728888, 0.462789, 0.682353, 0.232815, 0.732247, 0.459277, 0.686275, 0.239374, 0.735588, 0.455688, 0.690196, 0.24607, 0.73891, 0.452024, 0.694118, 0.252899, 0.742211, 0.448284, 0.698039, 0.259857, 0.745492, 0.444467, 0.701961, 0.266941, 0.748751, 0.440573, 0.705882, 0.274149, 0.751988, 0.436601, 0.709804, 0.281477, 0.755203, 0.432552, 0.713725, 0.288921, 0.758394, 0.428426, 0.717647, 0.296479, 0.761561, 0.424223, 0.721569, 0.304148, 0.764704, 0.419943, 0.72549, 0.311925, 0.767822, 0.415586, 0.729412, 0.319809, 0.770914, 0.411152, 0.733333, 0.327796, 0.77398, 0.40664, 0.737255, 0.335885, 0.777018, 0.402049, 0.741176, 0.344074, 0.780029, 0.397381, 0.745098, 0.35236, 0.783011, 0.392636, 0.74902, 0.360741, 0.785964, 0.387814, 0.752941, 0.369214, 0.788888, 0.382914, 0.756863, 0.377779, 0.791781, 0.377939, 0.760784, 0.386433, 0.794644, 0.372886, 0.764706, 0.395174, 0.797475, 0.367757, 0.768627, 0.404001, 0.800275, 0.362552, 0.772549, 0.412913, 0.803041, 0.357269, 0.776471, 0.421908, 0.805774, 0.35191, 0.7803920000000001, 0.430983, 0.808473, 0.346476, 0.784314, 0.440137, 0.811138, 0.340967, 0.788235, 0.449368, 0.813768, 0.335384, 0.792157, 0.458674, 0.816363, 0.329727, 0.796078, 0.468053, 0.818921, 0.323998, 0.8, 0.477504, 0.821444, 0.318195, 0.803922, 0.487026, 0.823929, 0.312321, 0.807843, 0.496615, 0.826376, 0.306377, 0.8117650000000001, 0.506271, 0.828786, 0.300362, 0.815686, 0.515992, 0.831158, 0.294279, 0.819608, 0.525776, 0.833491, 0.288127, 0.8235290000000001, 0.535621, 0.835785, 0.281908, 0.827451, 0.545524, 0.838039, 0.275626, 0.831373, 0.555484, 0.840254, 0.269281, 0.835294, 0.565498, 0.84243, 0.262877, 0.839216, 0.575563, 0.844566, 0.256415, 0.843137, 0.585678, 0.846661, 0.249897, 0.847059, 0.595839, 0.848717, 0.243329, 0.85098, 0.606045, 0.850733, 0.236712, 0.8549020000000002, 0.616293, 0.852709, 0.230052, 0.858824, 0.626579, 0.854645, 0.223353, 0.862745, 0.636902, 0.856542, 0.21662, 0.866667, 0.647257, 0.8584, 0.209861, 0.870588, 0.657642, 0.860219, 0.203082, 0.87451, 0.668054, 0.861999, 0.196293, 0.878431, 0.678489, 0.863742, 0.189503, 0.882353, 0.688944, 0.865448, 0.182725, 0.8862750000000001, 0.699415, 0.867117, 0.175971, 0.890196, 0.709898, 0.868751, 0.169257, 0.894118, 0.720391, 0.87035, 0.162603, 0.8980390000000001, 0.730889, 0.871916, 0.156029, 0.901961, 0.741388, 0.873449, 0.149561, 0.905882, 0.751884, 0.874951, 0.143228, 0.909804, 0.762373, 0.876424, 0.137064, 0.913725, 0.772852, 0.877868, 0.131109, 0.917647, 0.783315, 0.879285, 0.125405, 0.921569, 0.79376, 0.880678, 0.120005, 0.92549, 0.804182, 0.882046, 0.114965, 0.9294120000000001, 0.814576, 0.883393, 0.110347, 0.933333, 0.82494, 0.88472, 0.106217, 0.937255, 0.83527, 0.886029, 0.102646, 0.941176, 0.845561, 0.887322, 0.099702, 0.945098, 0.85581, 0.888601, 0.097452, 0.94902, 0.866013, 0.889868, 0.095953, 0.952941, 0.876168, 0.891125, 0.09525, 0.956863, 0.886271, 0.892374, 0.095374, 0.960784, 0.89632, 0.893616, 0.096335, 0.964706, 0.906311, 0.894855, 0.098125, 0.968627, 0.916242, 0.896091, 0.100717, 0.972549, 0.926106, 0.89733, 0.104071, 0.976471, 0.935904, 0.89857, 0.108131, 0.980392, 0.945636, 0.899815, 0.112838, 0.984314, 0.9553, 0.901065, 0.118128, 0.988235, 0.964894, 0.902323, 0.123941, 0.992157, 0.974417, 0.90359, 0.130215, 0.9960780000000001, 0.983868, 0.904867, 0.136897, 1.0, 0.993248, 0.906157, 0.143936]
 chiLUT.NanColor = [1.0, 0.0, 0.0]
 chiLUT.ScalarRangeInitialized = 1.0
 
-# get opacity transfer function/opacity map for 'chi'
-chiPWF = GetOpacityTransferFunction('chi')
-chiPWF.Points = [0.0, 1.0, 0.5, 0.0, 1.0, 1.0, 0.5, 0.0]
-chiPWF.ScalarRangeInitialized = 1
-
 # trace defaults for the display properties.
-sliceAMRdata1Display.Representation = 'Surface'
-sliceAMRdata1Display.ColorArrayName = ['CELLS', 'chi']
-sliceAMRdata1Display.LookupTable = chiLUT
-sliceAMRdata1Display.SelectTCoordArray = 'None'
-sliceAMRdata1Display.SelectNormalArray = 'None'
-sliceAMRdata1Display.SelectTangentArray = 'None'
-sliceAMRdata1Display.OSPRayScaleFunction = 'PiecewiseFunction'
-sliceAMRdata1Display.SelectOrientationVectors = 'None'
-sliceAMRdata1Display.ScaleFactor = 7.0
-sliceAMRdata1Display.SelectScaleArray = 'None'
-sliceAMRdata1Display.GlyphType = 'Arrow'
-sliceAMRdata1Display.GlyphTableIndexArray = 'None'
-sliceAMRdata1Display.GaussianRadius = 0.35000000000000003
-sliceAMRdata1Display.SetScaleArray = [None, '']
-sliceAMRdata1Display.ScaleTransferFunction = 'PiecewiseFunction'
-sliceAMRdata1Display.OpacityArray = [None, '']
-sliceAMRdata1Display.OpacityTransferFunction = 'PiecewiseFunction'
-sliceAMRdata1Display.DataAxesGrid = 'GridAxesRepresentation'
-sliceAMRdata1Display.PolarAxes = 'PolarAxesRepresentation'
-sliceAMRdata1Display.ScalarOpacityUnitDistance = 2.3023915422888135
-sliceAMRdata1Display.ScalarOpacityFunction = chiPWF
+slice1Display.Representation = 'Surface With Edges'
+slice1Display.ColorArrayName = ['CELLS', 'chi']
+slice1Display.LookupTable = chiLUT
+slice1Display.SelectTCoordArray = 'None'
+slice1Display.SelectNormalArray = 'None'
+slice1Display.SelectTangentArray = 'None'
+slice1Display.OSPRayScaleFunction = 'PiecewiseFunction'
+slice1Display.SelectOrientationVectors = 'None'
+slice1Display.ScaleFactor = 7.0
+slice1Display.SelectScaleArray = 'None'
+slice1Display.GlyphType = 'Arrow'
+slice1Display.GlyphTableIndexArray = 'None'
+slice1Display.GaussianRadius = 0.35000000000000003
+slice1Display.SetScaleArray = [None, '']
+slice1Display.ScaleTransferFunction = 'PiecewiseFunction'
+slice1Display.OpacityArray = [None, '']
+slice1Display.OpacityTransferFunction = 'PiecewiseFunction'
+slice1Display.DataAxesGrid = 'GridAxesRepresentation'
+slice1Display.PolarAxes = 'PolarAxesRepresentation'
 
 # setup the color legend parameters for each legend in this view
 
@@ -115,13 +125,18 @@
 chiLUTColorBar.Visibility = 1
 
 # show color legend
-sliceAMRdata1Display.SetScalarBarVisibility(renderView1, True)
+slice1Display.SetScalarBarVisibility(renderView1, True)
 
 # ----------------------------------------------------------------
 # setup color maps and opacity mapes used in the visualization
 # note: the Get..() functions create a new object, if needed
 # ----------------------------------------------------------------
 
+# get opacity transfer function/opacity map for 'chi'
+chiPWF = GetOpacityTransferFunction('chi')
+chiPWF.Points = [0.0, 0.005917159840464592, 0.5, 0.0, 1.0, 1.0, 0.5, 0.0]
+chiPWF.ScalarRangeInitialized = 1
+
 # ----------------------------------------------------------------
 # setup extractors
 # ----------------------------------------------------------------
@@ -131,9 +146,9 @@
 # trace defaults for the extractor.
 # init the 'PNG' selected for 'Writer'
 pNG1.Writer.FileName = 'SliceChi_%.6ts%cm.png'
-pNG1.Writer.ImageResolution = [885, 774]
-pNG1.Writer.Format = 'PNG'
+pNG1.Writer.ImageResolution = [1920, 1080]
 pNG1.Writer.ResetDisplay = 1
+pNG1.Writer.Format = 'PNG'
 
 # ----------------------------------------------------------------
 # restore active source
@@ -145,7 +160,6 @@
 from paraview import catalyst
 options = catalyst.Options()
 options.GlobalTrigger = 'TimeStep'
-options.EnableCatalystLive = 1
 options.CatalystLiveTrigger = 'TimeStep'
 
 # ------------------------------------------------------------------------------
diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp
index a89c4f3c1..f281a50d3 100644
--- a/Source/GRChomboCore/ChomboParameters.hpp
+++ b/Source/GRChomboCore/ChomboParameters.hpp
@@ -22,6 +22,7 @@
 #include <string>
 
 #ifdef USE_CATALYST
+#include <cstdlib>
 #include <vtkCPPythonPipeline.h>
 #endif
 
@@ -183,6 +184,27 @@ class ChomboParameters
                     false);
             pp.load("catalyst_vtk_file_prefix", catalyst_params.vtk_file_prefix,
                     std::string("Catalyst_VTK_"));
+
+            // export parameters to environment so Catalyst scripts can use
+            // them
+            std::string center_str = "";
+            FOR1(i)
+            {
+                if (i != 0)
+                    center_str += " ";
+                center_str += std::to_string(center[i]);
+            }
+            setenv("GRCHOMBO_PARAM_CENTER", center_str.c_str(), 1);
+
+            // this isn't really a GRChombo parameter but easiest in this form
+            FOR1(i)
+            {
+                std::string length_env_var =
+                    std::string("GRCHOMBO_PARAM_L") + std::to_string(i + 1);
+                double length = static_cast<double>(ivN[i] + 1) * coarsest_dx;
+                std::string length_str = std::to_string(length);
+                setenv(length_env_var.c_str(), length_str.c_str(), 1);
+            }
         }
 #endif
     }
diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 582815cfd..03f05afc4 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -32,6 +32,7 @@ using std::endl;
 
 #ifdef USE_CATALYST
 #include "vtkLogger.h"
+#include "vtkPythonInterpreter.h"
 #include "vtkSMPTools.h"
 #include "vtkVersion.h"
 #endif
@@ -219,6 +220,8 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
     // set vtkLogger internal messages to only appear at verbosity 2
     vtkLogger::SetInternalVerbosityLevel(vtkLogger::VERBOSITY_3);
 #endif
+    // don't redirect Python interpreter output
+    vtkPythonInterpreter::SetRedirectOutput(false);
 #ifdef CH_MPI
     vtkLogger::SetStderrVerbosity(vtkLogger::VERBOSITY_ERROR);
     std::string catalyst_log_file = chombo_params.pout_path +
diff --git a/Source/Insitu/README.md b/Source/Insitu/README.md
index 56037c2f9..a5650c39b 100644
--- a/Source/Insitu/README.md
+++ b/Source/Insitu/README.md
@@ -138,7 +138,7 @@ unset PARAVIEW_DIR
 
 ## Generating ParaView Catalyst scripts
 
-> **Note** 
+> **Warning** 
 > The procedure to generate Catalyst scripts varies between
 > different versions of ParaView. In particular, there are significant
 > differences before v5.9. The following instructions apply to v5.9 or greater.
@@ -181,6 +181,17 @@ input = AMRGaussianPulseSource(registrationName='input')
 ```
 in order to be able to use it.
 
+> **Note**
+> It is possible to pass information about the GRChombo grid (in particular, the
+> lengths of the domain and the center) to Catalyst pipelines so that they
+> automatically adjust appropriately. These are set as environment variables of
+> the form `GRCHOMBO_PARAM_xxx` in 
+> [ChomboParameters.hpp](../GRChomboCore/ChomboParameters.hpp) and can be 
+> recovered in Catalyst Python scripts using the `os.getenv()` function from the
+> `os` package. See the
+> [slice_chi.py](../../Examples/BinaryBH/catalyst_scripts/slice_chi.py) script
+> as an example.
+
 ## Using the in-situ visualization capabilities
 
 There are several Catalyst specific parameters which control the in-situ

From 7ada53bb1c073e44709cf2d796f266ad92a6dc08 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Fri, 3 Feb 2023 16:35:31 +0000
Subject: [PATCH 59/63] Remove old Catalyst scripts

---
 .../BinaryBH/catalyst_scripts/amr_blocks.py   | 117 --------------
 Examples/BinaryBH/catalyst_scripts/outline.py | 119 --------------
 .../volume_noraytrace_resample_chi.py         | 152 -----------------
 .../volume_raytrace_resample_chi.py           | 153 ------------------
 4 files changed, 541 deletions(-)
 delete mode 100644 Examples/BinaryBH/catalyst_scripts/amr_blocks.py
 delete mode 100644 Examples/BinaryBH/catalyst_scripts/outline.py
 delete mode 100644 Examples/BinaryBH/catalyst_scripts/volume_noraytrace_resample_chi.py
 delete mode 100644 Examples/BinaryBH/catalyst_scripts/volume_raytrace_resample_chi.py

diff --git a/Examples/BinaryBH/catalyst_scripts/amr_blocks.py b/Examples/BinaryBH/catalyst_scripts/amr_blocks.py
deleted file mode 100644
index 64af0e09d..000000000
--- a/Examples/BinaryBH/catalyst_scripts/amr_blocks.py
+++ /dev/null
@@ -1,117 +0,0 @@
-# script-version: 2.0
-# Catalyst state generated using paraview version 5.9.1
-
-#### import the simple module from the paraview
-from paraview.simple import *
-#### disable automatic camera reset on 'Show'
-paraview.simple._DisableFirstRenderCameraReset()
-
-# ----------------------------------------------------------------
-# setup views used in the visualization
-# ----------------------------------------------------------------
-
-# get the material library
-materialLibrary1 = GetMaterialLibrary()
-
-# Create a new 'Render View'
-renderView1 = CreateView('RenderView')
-renderView1.ViewSize = [885, 774]
-renderView1.AxesGrid = 'GridAxes3DActor'
-renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
-renderView1.StereoType = 'Crystal Eyes'
-renderView1.CameraPosition = [158.08172115703206, 212.13325123979374, 112.7252021311981]
-renderView1.CameraFocalPoint = [32.00000000000007, 31.99999999999995, 32.0000000000001]
-renderView1.CameraViewUp = [0.05928269610132273, -0.44249529710522584, 0.8948091830008877]
-renderView1.CameraFocalDisk = 1.0
-renderView1.CameraParallelScale = 60.6217782649107
-renderView1.BackEnd = 'OSPRay raycaster'
-renderView1.OSPRayMaterialLibrary = materialLibrary1
-
-# init the 'GridAxes3DActor' selected for 'AxesGrid'
-renderView1.AxesGrid.Visibility = 1
-
-SetActiveView(None)
-
-# ----------------------------------------------------------------
-# setup view layouts
-# ----------------------------------------------------------------
-
-# create new layout object 'Layout #1'
-layout1 = CreateLayout(name='Layout #1')
-layout1.AssignView(0, renderView1)
-layout1.SetSize(885, 774)
-
-# ----------------------------------------------------------------
-# restore active view
-SetActiveView(renderView1)
-# ----------------------------------------------------------------
-
-# ----------------------------------------------------------------
-# setup the data processing pipelines
-# ----------------------------------------------------------------
-
-# create a new 'VisItChomboReader'
-input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000003.3d.hdf5'])
-input.MeshStatus = ['Mesh']
-input.CellArrayStatus = []
-
-# ----------------------------------------------------------------
-# setup the visualization in view 'renderView1'
-# ----------------------------------------------------------------
-
-# show data from input
-inputDisplay = Show(input, renderView1, 'AMRRepresentation')
-
-# trace defaults for the display properties.
-inputDisplay.Representation = 'AMR Blocks'
-inputDisplay.ColorArrayName = ['POINTS', '']
-inputDisplay.SelectTCoordArray = 'None'
-inputDisplay.SelectNormalArray = 'None'
-inputDisplay.SelectTangentArray = 'None'
-inputDisplay.OSPRayScaleFunction = 'PiecewiseFunction'
-inputDisplay.SelectOrientationVectors = 'None'
-inputDisplay.ScaleFactor = 7.0
-inputDisplay.SelectScaleArray = 'None'
-inputDisplay.GlyphType = 'Arrow'
-inputDisplay.GlyphTableIndexArray = 'None'
-inputDisplay.GaussianRadius = 0.35000000000000003
-inputDisplay.SetScaleArray = [None, '']
-inputDisplay.ScaleTransferFunction = 'PiecewiseFunction'
-inputDisplay.OpacityArray = [None, '']
-inputDisplay.OpacityTransferFunction = 'PiecewiseFunction'
-inputDisplay.DataAxesGrid = 'GridAxesRepresentation'
-inputDisplay.PolarAxes = 'PolarAxesRepresentation'
-inputDisplay.ScalarOpacityUnitDistance = 0.8192401766441247
-
-# ----------------------------------------------------------------
-# setup extractors
-# ----------------------------------------------------------------
-
-# create extractor
-pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
-# trace defaults for the extractor.
-# init the 'PNG' selected for 'Writer'
-pNG1.Writer.FileName = 'AMRBlocks_%.6ts%cm.png'
-pNG1.Writer.ImageResolution = [885, 774]
-pNG1.Writer.Format = 'PNG'
-pNG1.Writer.ResetDisplay = 1
-
-# ----------------------------------------------------------------
-# restore active source
-SetActiveSource(pNG1)
-# ----------------------------------------------------------------
-
-# ------------------------------------------------------------------------------
-# Catalyst options
-from paraview import catalyst
-options = catalyst.Options()
-options.GlobalTrigger = 'TimeStep'
-options.EnableCatalystLive = 1
-options.CatalystLiveTrigger = 'TimeStep'
-
-# ------------------------------------------------------------------------------
-if __name__ == '__main__':
-    from paraview.simple import SaveExtractsUsingCatalystOptions
-    # Code for non in-situ environments; if executing in post-processing
-    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
-    SaveExtractsUsingCatalystOptions(options)
diff --git a/Examples/BinaryBH/catalyst_scripts/outline.py b/Examples/BinaryBH/catalyst_scripts/outline.py
deleted file mode 100644
index 53832a9c0..000000000
--- a/Examples/BinaryBH/catalyst_scripts/outline.py
+++ /dev/null
@@ -1,119 +0,0 @@
-# script-version: 2.0
-# Catalyst state generated using paraview version 5.9.1
-
-#### import the simple module from the paraview
-from paraview.simple import *
-#### disable automatic camera reset on 'Show'
-paraview.simple._DisableFirstRenderCameraReset()
-
-# ----------------------------------------------------------------
-# setup views used in the visualization
-# ----------------------------------------------------------------
-
-# get the material library
-materialLibrary1 = GetMaterialLibrary()
-
-# Create a new 'Render View'
-renderView1 = CreateView('RenderView')
-renderView1.ViewSize = [885, 774]
-renderView1.AxesGrid = 'GridAxes3DActor'
-renderView1.CenterAxesVisibility = 1
-renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
-renderView1.StereoType = 'Crystal Eyes'
-renderView1.CameraPosition = [145.67639666481145, -111.0984129619794, 178.49801336649438]
-renderView1.CameraFocalPoint = [32.000000000000014, 31.999999999999982, 32.00000000000003]
-renderView1.CameraViewUp = [-0.10746516487670288, 0.6682514214076767, 0.7361326484572212]
-renderView1.CameraFocalDisk = 1.0
-renderView1.CameraParallelScale = 60.6217782649107
-renderView1.CameraParallelProjection = 1
-renderView1.BackEnd = 'OSPRay raycaster'
-renderView1.OSPRayMaterialLibrary = materialLibrary1
-
-# init the 'GridAxes3DActor' selected for 'AxesGrid'
-renderView1.AxesGrid.Visibility = 1
-
-SetActiveView(None)
-
-# ----------------------------------------------------------------
-# setup view layouts
-# ----------------------------------------------------------------
-
-# create new layout object 'Layout #1'
-layout1 = CreateLayout(name='Layout #1')
-layout1.AssignView(0, renderView1)
-layout1.SetSize(885, 774)
-
-# ----------------------------------------------------------------
-# restore active view
-SetActiveView(renderView1)
-# ----------------------------------------------------------------
-
-# ----------------------------------------------------------------
-# setup the data processing pipelines
-# ----------------------------------------------------------------
-
-# create a new 'VisItChomboReader'
-input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000003.3d.hdf5'])
-input.MeshStatus = ['Mesh']
-input.CellArrayStatus = []
-
-# ----------------------------------------------------------------
-# setup the visualization in view 'renderView1'
-# ----------------------------------------------------------------
-
-# show data from input
-inputDisplay = Show(input, renderView1, 'AMRRepresentation')
-
-# trace defaults for the display properties.
-inputDisplay.Representation = 'Outline'
-inputDisplay.ColorArrayName = [None, '']
-inputDisplay.SelectTCoordArray = 'None'
-inputDisplay.SelectNormalArray = 'None'
-inputDisplay.SelectTangentArray = 'None'
-inputDisplay.OSPRayScaleFunction = 'PiecewiseFunction'
-inputDisplay.SelectOrientationVectors = 'None'
-inputDisplay.ScaleFactor = 7.0
-inputDisplay.SelectScaleArray = 'None'
-inputDisplay.GlyphType = 'Arrow'
-inputDisplay.GlyphTableIndexArray = 'None'
-inputDisplay.GaussianRadius = 0.35000000000000003
-inputDisplay.SetScaleArray = [None, '']
-inputDisplay.ScaleTransferFunction = 'PiecewiseFunction'
-inputDisplay.OpacityArray = [None, '']
-inputDisplay.OpacityTransferFunction = 'PiecewiseFunction'
-inputDisplay.DataAxesGrid = 'GridAxesRepresentation'
-inputDisplay.PolarAxes = 'PolarAxesRepresentation'
-inputDisplay.ScalarOpacityUnitDistance = 0.8192401766441247
-
-# ----------------------------------------------------------------
-# setup extractors
-# ----------------------------------------------------------------
-
-# create extractor
-pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
-# trace defaults for the extractor.
-# init the 'PNG' selected for 'Writer'
-pNG1.Writer.FileName = 'RenderView1_%.6ts%cm.png'
-pNG1.Writer.ImageResolution = [885, 774]
-pNG1.Writer.Format = 'PNG'
-pNG1.Writer.ResetDisplay = 1
-
-# ----------------------------------------------------------------
-# restore active source
-SetActiveSource(input)
-# ----------------------------------------------------------------
-
-# ------------------------------------------------------------------------------
-# Catalyst options
-from paraview import catalyst
-options = catalyst.Options()
-options.GlobalTrigger = 'TimeStep'
-options.EnableCatalystLive = 1
-options.CatalystLiveTrigger = 'TimeStep'
-
-# ------------------------------------------------------------------------------
-if __name__ == '__main__':
-    from paraview.simple import SaveExtractsUsingCatalystOptions
-    # Code for non in-situ environments; if executing in post-processing
-    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
-    SaveExtractsUsingCatalystOptions(options)
diff --git a/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_resample_chi.py b/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_resample_chi.py
deleted file mode 100644
index 643d4786c..000000000
--- a/Examples/BinaryBH/catalyst_scripts/volume_noraytrace_resample_chi.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# script-version: 2.0
-# Catalyst state generated using paraview version 5.9.1
-
-#### import the simple module from the paraview
-from paraview.simple import *
-#### disable automatic camera reset on 'Show'
-paraview.simple._DisableFirstRenderCameraReset()
-
-# ----------------------------------------------------------------
-# setup views used in the visualization
-# ----------------------------------------------------------------
-
-# get the material library
-materialLibrary1 = GetMaterialLibrary()
-
-# Create a new 'Render View'
-renderView1 = CreateView('RenderView')
-renderView1.ViewSize = [885, 774]
-renderView1.AxesGrid = 'GridAxes3DActor'
-renderView1.CenterAxesVisibility = 1
-renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
-renderView1.StereoType = 'Crystal Eyes'
-renderView1.CameraPosition = [20.770091346798225, 54.44320054580248, 46.07511197528731]
-renderView1.CameraFocalPoint = [32.000000000000014, 31.999999999999986, 31.999999999999993]
-renderView1.CameraViewUp = [0.37830852443531104, -0.3485213720084932, 0.8575637081831853]
-renderView1.CameraFocalDisk = 1.0
-renderView1.CameraParallelScale = 60.6217782649107
-renderView1.BackEnd = 'OSPRay raycaster'
-renderView1.OSPRayMaterialLibrary = materialLibrary1
-renderView1.Background = [0.0, 0.0, 0.0]
-
-SetActiveView(None)
-
-# ----------------------------------------------------------------
-# setup view layouts
-# ----------------------------------------------------------------
-
-# create new layout object 'Layout #1'
-layout1 = CreateLayout(name='Layout #1')
-layout1.AssignView(0, renderView1)
-layout1.SetSize(885, 774)
-
-# ----------------------------------------------------------------
-# restore active view
-SetActiveView(renderView1)
-# ----------------------------------------------------------------
-
-# ----------------------------------------------------------------
-# setup the data processing pipelines
-# ----------------------------------------------------------------
-
-# create a new 'VisItChomboReader'
-input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000002.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000003.3d.hdf5'])
-input.MeshStatus = ['Mesh']
-input.CellArrayStatus = ['chi']
-
-# create a new 'Resample To Image'
-resampleToImage1 = ResampleToImage(registrationName='ResampleToImage1', Input=input)
-resampleToImage1.SamplingBounds = [-3.0, 67.0, -3.0, 67.0, -3.0, 67.0]
-
-# ----------------------------------------------------------------
-# setup the visualization in view 'renderView1'
-# ----------------------------------------------------------------
-
-# show data from resampleToImage1
-resampleToImage1Display = Show(resampleToImage1, renderView1, 'UniformGridRepresentation')
-
-# get color transfer function/color map for 'chi'
-chiLUT = GetColorTransferFunction('chi')
-chiLUT.RGBPoints = [0.006705302783959541, 1.0, 0.988235, 0.968627, 0.025918898344160616, 1.0, 0.952941, 0.878431, 0.05473929168446223, 0.968627, 0.905882, 0.776471, 0.10277328058496492, 0.94902, 0.898039, 0.647059, 0.1508072694854676, 0.901961, 0.878431, 0.556863, 0.1988412583859703, 0.847059, 0.858824, 0.482353, 0.24687524728647298, 0.690196, 0.819608, 0.435294, 0.2949092361869757, 0.513725, 0.768627, 0.384314, 0.3429432250874783, 0.337255, 0.721569, 0.337255, 0.3909772139879811, 0.278431, 0.658824, 0.392157, 0.4390112028884838, 0.231373, 0.639216, 0.435294, 0.4870451917889864, 0.203922, 0.6, 0.486275, 0.5350791806894891, 0.172549, 0.568627, 0.537255, 0.5831131695899918, 0.141176, 0.517647, 0.54902, 0.6311471584904945, 0.133333, 0.458824, 0.541176, 0.6791811473909971, 0.12549, 0.396078, 0.529412, 0.7272151362914999, 0.117647, 0.321569, 0.521569, 0.7752491251920026, 0.121569, 0.258824, 0.509804, 0.8232831140925052, 0.133333, 0.227451, 0.501961, 0.8713171029930079, 0.145098, 0.192157, 0.490196, 0.9193510918935106, 0.188235, 0.164706, 0.470588, 0.9673850807940133, 0.258824, 0.196078, 0.439216]
-chiLUT.ColorSpace = 'RGB'
-chiLUT.NanColor = [0.25, 0.0, 0.0]
-chiLUT.NanOpacity = 0.0
-chiLUT.ScalarRangeInitialized = 1.0
-
-# get opacity transfer function/opacity map for 'chi'
-chiPWF = GetOpacityTransferFunction('chi')
-chiPWF.Points = [0.006705302783959541, 1.0, 0.5, 0.0, 0.055023517459630966, 0.9053254723548889, 0.5, 0.0, 0.1061839833855629, 0.7692307829856873, 0.5, 0.0, 0.21418939530849457, 0.47337278723716736, 0.5, 0.0, 0.37335529923439026, 0.23076923191547394, 0.5, 0.0, 0.5808394117544118, 0.0, 0.5, 0.0, 0.9673850807940133, 0.0, 0.5, 0.0]
-chiPWF.ScalarRangeInitialized = 1
-
-# trace defaults for the display properties.
-resampleToImage1Display.Representation = 'Volume'
-resampleToImage1Display.ColorArrayName = ['POINTS', 'chi']
-resampleToImage1Display.LookupTable = chiLUT
-resampleToImage1Display.SelectTCoordArray = 'None'
-resampleToImage1Display.SelectNormalArray = 'None'
-resampleToImage1Display.SelectTangentArray = 'None'
-resampleToImage1Display.OSPRayScaleArray = 'chi'
-resampleToImage1Display.OSPRayScaleFunction = 'PiecewiseFunction'
-resampleToImage1Display.SelectOrientationVectors = 'None'
-resampleToImage1Display.ScaleFactor = 6.999993000000001
-resampleToImage1Display.SelectScaleArray = 'None'
-resampleToImage1Display.GlyphType = 'Arrow'
-resampleToImage1Display.GlyphTableIndexArray = 'None'
-resampleToImage1Display.GaussianRadius = 0.34999965000000005
-resampleToImage1Display.SetScaleArray = ['POINTS', 'chi']
-resampleToImage1Display.ScaleTransferFunction = 'PiecewiseFunction'
-resampleToImage1Display.OpacityArray = ['POINTS', 'chi']
-resampleToImage1Display.OpacityTransferFunction = 'PiecewiseFunction'
-resampleToImage1Display.DataAxesGrid = 'GridAxesRepresentation'
-resampleToImage1Display.PolarAxes = 'PolarAxesRepresentation'
-resampleToImage1Display.ScalarOpacityUnitDistance = 1.224681164507726
-resampleToImage1Display.ScalarOpacityFunction = chiPWF
-resampleToImage1Display.OpacityArrayName = ['POINTS', 'chi']
-resampleToImage1Display.SliceFunction = 'Plane'
-resampleToImage1Display.Slice = 49
-
-# init the 'PiecewiseFunction' selected for 'ScaleTransferFunction'
-resampleToImage1Display.ScaleTransferFunction.Points = [0.16307053589122933, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
-
-# init the 'PiecewiseFunction' selected for 'OpacityTransferFunction'
-resampleToImage1Display.OpacityTransferFunction.Points = [0.16307053589122933, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
-
-# init the 'Plane' selected for 'SliceFunction'
-resampleToImage1Display.SliceFunction.Origin = [32.0, 32.0, 32.0]
-
-# ----------------------------------------------------------------
-# setup color maps and opacity mapes used in the visualization
-# note: the Get..() functions create a new object, if needed
-# ----------------------------------------------------------------
-
-# ----------------------------------------------------------------
-# setup extractors
-# ----------------------------------------------------------------
-
-# create extractor
-pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
-# trace defaults for the extractor.
-# init the 'PNG' selected for 'Writer'
-pNG1.Writer.FileName = 'VolumeChi_%.6ts%cm.png'
-pNG1.Writer.ImageResolution = [885, 778]
-pNG1.Writer.Format = 'PNG'
-
-# ----------------------------------------------------------------
-# restore active source
-SetActiveSource(pNG1)
-# ----------------------------------------------------------------
-
-# ------------------------------------------------------------------------------
-# Catalyst options
-from paraview import catalyst
-options = catalyst.Options()
-options.GlobalTrigger = 'TimeStep'
-options.EnableCatalystLive = 1
-options.CatalystLiveTrigger = 'TimeStep'
-
-# ------------------------------------------------------------------------------
-if __name__ == '__main__':
-    from paraview.simple import SaveExtractsUsingCatalystOptions
-    # Code for non in-situ environments; if executing in post-processing
-    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
-    SaveExtractsUsingCatalystOptions(options)
diff --git a/Examples/BinaryBH/catalyst_scripts/volume_raytrace_resample_chi.py b/Examples/BinaryBH/catalyst_scripts/volume_raytrace_resample_chi.py
deleted file mode 100644
index 5e08ff80b..000000000
--- a/Examples/BinaryBH/catalyst_scripts/volume_raytrace_resample_chi.py
+++ /dev/null
@@ -1,153 +0,0 @@
-# script-version: 2.0
-# Catalyst state generated using paraview version 5.9.1
-
-#### import the simple module from the paraview
-from paraview.simple import *
-#### disable automatic camera reset on 'Show'
-paraview.simple._DisableFirstRenderCameraReset()
-
-# ----------------------------------------------------------------
-# setup views used in the visualization
-# ----------------------------------------------------------------
-
-# get the material library
-materialLibrary1 = GetMaterialLibrary()
-
-# Create a new 'Render View'
-renderView1 = CreateView('RenderView')
-renderView1.ViewSize = [885, 774]
-renderView1.AxesGrid = 'GridAxes3DActor'
-renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
-renderView1.StereoType = 'Crystal Eyes'
-renderView1.CameraPosition = [20.770091346798225, 54.44320054580248, 46.07511197528731]
-renderView1.CameraFocalPoint = [32.000000000000014, 31.999999999999986, 31.999999999999993]
-renderView1.CameraViewUp = [0.37830852443531104, -0.3485213720084932, 0.8575637081831853]
-renderView1.CameraFocalDisk = 1.0
-renderView1.CameraParallelScale = 60.6217782649107
-renderView1.EnableRayTracing = 1
-renderView1.BackEnd = 'OSPRay raycaster'
-renderView1.SamplesPerPixel = 3
-renderView1.OSPRayMaterialLibrary = materialLibrary1
-renderView1.Background = [0.0, 0.0, 0.0]
-
-SetActiveView(None)
-
-# ----------------------------------------------------------------
-# setup view layouts
-# ----------------------------------------------------------------
-
-# create new layout object 'Layout #1'
-layout1 = CreateLayout(name='Layout #1')
-layout1.AssignView(0, renderView1)
-layout1.SetSize(885, 774)
-
-# ----------------------------------------------------------------
-# restore active view
-SetActiveView(renderView1)
-# ----------------------------------------------------------------
-
-# ----------------------------------------------------------------
-# setup the data processing pipelines
-# ----------------------------------------------------------------
-
-# create a new 'VisItChomboReader'
-input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000002.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBHp_000003.3d.hdf5'])
-input.MeshStatus = ['Mesh']
-input.CellArrayStatus = ['chi']
-
-# create a new 'Resample To Image'
-resampleToImage1 = ResampleToImage(registrationName='ResampleToImage1', Input=input)
-resampleToImage1.SamplingBounds = [-3.0, 67.0, -3.0, 67.0, -3.0, 67.0]
-
-# ----------------------------------------------------------------
-# setup the visualization in view 'renderView1'
-# ----------------------------------------------------------------
-
-# show data from resampleToImage1
-resampleToImage1Display = Show(resampleToImage1, renderView1, 'UniformGridRepresentation')
-
-# get color transfer function/color map for 'chi'
-chiLUT = GetColorTransferFunction('chi')
-chiLUT.RGBPoints = [0.006705302783959541, 1.0, 0.988235, 0.968627, 0.025918898344160616, 1.0, 0.952941, 0.878431, 0.05473929168446223, 0.968627, 0.905882, 0.776471, 0.10277328058496492, 0.94902, 0.898039, 0.647059, 0.1508072694854676, 0.901961, 0.878431, 0.556863, 0.1988412583859703, 0.847059, 0.858824, 0.482353, 0.24687524728647298, 0.690196, 0.819608, 0.435294, 0.2949092361869757, 0.513725, 0.768627, 0.384314, 0.3429432250874783, 0.337255, 0.721569, 0.337255, 0.3909772139879811, 0.278431, 0.658824, 0.392157, 0.4390112028884838, 0.231373, 0.639216, 0.435294, 0.4870451917889864, 0.203922, 0.6, 0.486275, 0.5350791806894891, 0.172549, 0.568627, 0.537255, 0.5831131695899918, 0.141176, 0.517647, 0.54902, 0.6311471584904945, 0.133333, 0.458824, 0.541176, 0.6791811473909971, 0.12549, 0.396078, 0.529412, 0.7272151362914999, 0.117647, 0.321569, 0.521569, 0.7752491251920026, 0.121569, 0.258824, 0.509804, 0.8232831140925052, 0.133333, 0.227451, 0.501961, 0.8713171029930079, 0.145098, 0.192157, 0.490196, 0.9193510918935106, 0.188235, 0.164706, 0.470588, 0.9673850807940133, 0.258824, 0.196078, 0.439216]
-chiLUT.ColorSpace = 'RGB'
-chiLUT.NanColor = [0.25, 0.0, 0.0]
-chiLUT.NanOpacity = 0.0
-chiLUT.ScalarRangeInitialized = 1.0
-
-# get opacity transfer function/opacity map for 'chi'
-chiPWF = GetOpacityTransferFunction('chi')
-chiPWF.Points = [0.006705302783959541, 1.0, 0.5, 0.0, 0.055023517459630966, 0.9053254723548889, 0.5, 0.0, 0.1061839833855629, 0.7692307829856873, 0.5, 0.0, 0.21418939530849457, 0.47337278723716736, 0.5, 0.0, 0.37335529923439026, 0.23076923191547394, 0.5, 0.0, 0.5808394117544118, 0.0, 0.5, 0.0, 0.9673850807940133, 0.0, 0.5, 0.0]
-chiPWF.ScalarRangeInitialized = 1
-
-# trace defaults for the display properties.
-resampleToImage1Display.Representation = 'Volume'
-resampleToImage1Display.ColorArrayName = ['POINTS', 'chi']
-resampleToImage1Display.LookupTable = chiLUT
-resampleToImage1Display.SelectTCoordArray = 'None'
-resampleToImage1Display.SelectNormalArray = 'None'
-resampleToImage1Display.SelectTangentArray = 'None'
-resampleToImage1Display.OSPRayScaleArray = 'chi'
-resampleToImage1Display.OSPRayScaleFunction = 'PiecewiseFunction'
-resampleToImage1Display.SelectOrientationVectors = 'None'
-resampleToImage1Display.ScaleFactor = 6.999993000000001
-resampleToImage1Display.SelectScaleArray = 'None'
-resampleToImage1Display.GlyphType = 'Arrow'
-resampleToImage1Display.GlyphTableIndexArray = 'None'
-resampleToImage1Display.GaussianRadius = 0.34999965000000005
-resampleToImage1Display.SetScaleArray = ['POINTS', 'chi']
-resampleToImage1Display.ScaleTransferFunction = 'PiecewiseFunction'
-resampleToImage1Display.OpacityArray = ['POINTS', 'chi']
-resampleToImage1Display.OpacityTransferFunction = 'PiecewiseFunction'
-resampleToImage1Display.DataAxesGrid = 'GridAxesRepresentation'
-resampleToImage1Display.PolarAxes = 'PolarAxesRepresentation'
-resampleToImage1Display.ScalarOpacityUnitDistance = 1.224681164507726
-resampleToImage1Display.ScalarOpacityFunction = chiPWF
-resampleToImage1Display.OpacityArrayName = ['POINTS', 'chi']
-resampleToImage1Display.SliceFunction = 'Plane'
-resampleToImage1Display.Slice = 49
-
-# init the 'PiecewiseFunction' selected for 'ScaleTransferFunction'
-resampleToImage1Display.ScaleTransferFunction.Points = [0.16307053589122933, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
-
-# init the 'PiecewiseFunction' selected for 'OpacityTransferFunction'
-resampleToImage1Display.OpacityTransferFunction.Points = [0.16307053589122933, 0.0, 0.5, 0.0, 0.9673850807940133, 1.0, 0.5, 0.0]
-
-# init the 'Plane' selected for 'SliceFunction'
-resampleToImage1Display.SliceFunction.Origin = [32.0, 32.0, 32.0]
-
-# ----------------------------------------------------------------
-# setup color maps and opacity mapes used in the visualization
-# note: the Get..() functions create a new object, if needed
-# ----------------------------------------------------------------
-
-# ----------------------------------------------------------------
-# setup extractors
-# ----------------------------------------------------------------
-
-# create extractor
-pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
-# trace defaults for the extractor.
-# init the 'PNG' selected for 'Writer'
-pNG1.Writer.FileName = 'OSPRayVolumeChi_%.6ts%cm.png'
-pNG1.Writer.ImageResolution = [885, 778]
-pNG1.Writer.Format = 'PNG'
-
-# ----------------------------------------------------------------
-# restore active source
-SetActiveSource(resampleToImage1)
-# ----------------------------------------------------------------
-
-# ------------------------------------------------------------------------------
-# Catalyst options
-from paraview import catalyst
-options = catalyst.Options()
-options.GlobalTrigger = 'TimeStep'
-options.EnableCatalystLive = 1
-options.CatalystLiveTrigger = 'TimeStep'
-
-# ------------------------------------------------------------------------------
-if __name__ == '__main__':
-    from paraview.simple import SaveExtractsUsingCatalystOptions
-    # Code for non in-situ environments; if executing in post-processing
-    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
-    SaveExtractsUsingCatalystOptions(options)

From 8974a0c18c0a6fcf968d47bb838ead2fa39f9233 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Fri, 3 Feb 2023 16:39:51 +0000
Subject: [PATCH 60/63] Clean up example Catalyst scripts

---
 ...blocks.py => slice_center_z_amr_blocks.py} | 101 +++++++++++-------
 .../{slice_chi.py => slice_center_z_chi.py}   |   5 +-
 .../catalyst_scripts/slice_center_z_to_vtm.py |  67 ++++++++++++
 Examples/BinaryBH/params_catalyst.txt         |   2 +-
 4 files changed, 133 insertions(+), 42 deletions(-)
 rename Examples/BinaryBH/catalyst_scripts/{slice_amr_blocks.py => slice_center_z_amr_blocks.py} (53%)
 rename Examples/BinaryBH/catalyst_scripts/{slice_chi.py => slice_center_z_chi.py} (99%)
 create mode 100644 Examples/BinaryBH/catalyst_scripts/slice_center_z_to_vtm.py

diff --git a/Examples/BinaryBH/catalyst_scripts/slice_amr_blocks.py b/Examples/BinaryBH/catalyst_scripts/slice_center_z_amr_blocks.py
similarity index 53%
rename from Examples/BinaryBH/catalyst_scripts/slice_amr_blocks.py
rename to Examples/BinaryBH/catalyst_scripts/slice_center_z_amr_blocks.py
index 133925d91..e29e2567e 100644
--- a/Examples/BinaryBH/catalyst_scripts/slice_amr_blocks.py
+++ b/Examples/BinaryBH/catalyst_scripts/slice_center_z_amr_blocks.py
@@ -1,11 +1,21 @@
 # script-version: 2.0
-# Catalyst state generated using paraview version 5.9.1
+# Catalyst state generated using paraview version 5.11.0
+
+# This script takes a slice through the center of the domain of constant z,
+# renders an outline of the blocks in the domain and saves the result to a PNG
+
+# Import this to get environment variables
+import os
 
 #### import the simple module from the paraview
 from paraview.simple import *
 #### disable automatic camera reset on 'Show'
 paraview.simple._DisableFirstRenderCameraReset()
 
+# Get center from environment
+grchombo_center_str = os.getenv('GRCHOMBO_PARAM_CENTER')
+grchombo_center = [float(dim) for dim in grchombo_center_str.split(' ')]
+
 # ----------------------------------------------------------------
 # setup views used in the visualization
 # ----------------------------------------------------------------
@@ -15,18 +25,19 @@
 
 # Create a new 'Render View'
 renderView1 = CreateView('RenderView')
-renderView1.ViewSize = [885, 774]
+renderView1.ViewSize = [928, 789]
 renderView1.AxesGrid = 'GridAxes3DActor'
-renderView1.CenterAxesVisibility = 1
-renderView1.CenterOfRotation = [32.0, 32.0, 32.0]
+renderView1.CenterOfRotation = grchombo_center
 renderView1.StereoType = 'Crystal Eyes'
-renderView1.CameraPosition = [32.0, 32.0, 190.05252605770363]
-renderView1.CameraFocalPoint = [32.0, 32.0, 32.0]
+renderView1.CameraPosition = [grchombo_center[0], grchombo_center[1], 500.0]
+renderView1.CameraFocalPoint = grchombo_center
 renderView1.CameraFocalDisk = 1.0
-renderView1.CameraParallelScale = 49.49747468305833
+renderView1.CameraParallelScale = 49.50757517794625
 renderView1.BackEnd = 'OSPRay raycaster'
 renderView1.OSPRayMaterialLibrary = materialLibrary1
-renderView1.Background = [0.0, 0.0, 0.0]
+
+# init the 'GridAxes3DActor' selected for 'AxesGrid'
+renderView1.AxesGrid.Visibility = 1
 
 SetActiveView(None)
 
@@ -37,7 +48,7 @@
 # create new layout object 'Layout #1'
 layout1 = CreateLayout(name='Layout #1')
 layout1.AssignView(0, renderView1)
-layout1.SetSize(885, 774)
+layout1.SetSize(928, 789)
 
 # ----------------------------------------------------------------
 # restore active view
@@ -48,44 +59,52 @@
 # setup the data processing pipelines
 # ----------------------------------------------------------------
 
-# create a new 'VisItChomboReader'
-input = VisItChomboReader(registrationName='input', FileName=['/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000000.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000001.3d.hdf5', '/home/miren/NR/GRChombo-public/Examples/BinaryBH/hdf5/BinaryBH_000003.3d.hdf5'])
+# create source
+input = AMRGaussianPulseSource(registrationName='input')
 input.MeshStatus = ['Mesh']
 input.CellArrayStatus = []
 
-# create a new 'Slice AMR data'
-sliceAMRdata1 = SliceAMRdata(registrationName='SliceAMRdata1', Input=input)
-sliceAMRdata1.Level = 3
-sliceAMRdata1.OffSet = 35.0
-sliceAMRdata1.Normal = 'Z-Normal'
+# create a new 'Slice'
+slice1 = Slice(registrationName='Slice1', Input=input)
+slice1.SliceType = 'Plane'
+slice1.HyperTreeGridSlicer = 'Plane'
+slice1.SliceOffsetValues = [0.0]
+
+# init the 'Plane' selected for 'SliceType'
+slice1.SliceType.Origin = grchombo_center
+slice1.SliceType.Normal = [0.0, 0.0, 1.0]
+
+# init the 'Plane' selected for 'HyperTreeGridSlicer'
+slice1.HyperTreeGridSlicer.Origin = grchombo_center
 
 # ----------------------------------------------------------------
 # setup the visualization in view 'renderView1'
 # ----------------------------------------------------------------
 
-# show data from sliceAMRdata1
-sliceAMRdata1Display = Show(sliceAMRdata1, renderView1, 'AMRRepresentation')
+# show data from slice1
+slice1Display = Show(slice1, renderView1, 'GeometryRepresentation')
 
 # trace defaults for the display properties.
-sliceAMRdata1Display.Representation = 'Outline'
-sliceAMRdata1Display.ColorArrayName = ['POINTS', '']
-sliceAMRdata1Display.SelectTCoordArray = 'None'
-sliceAMRdata1Display.SelectNormalArray = 'None'
-sliceAMRdata1Display.SelectTangentArray = 'None'
-sliceAMRdata1Display.OSPRayScaleFunction = 'PiecewiseFunction'
-sliceAMRdata1Display.SelectOrientationVectors = 'None'
-sliceAMRdata1Display.ScaleFactor = 7.0
-sliceAMRdata1Display.SelectScaleArray = 'None'
-sliceAMRdata1Display.GlyphType = 'Arrow'
-sliceAMRdata1Display.GlyphTableIndexArray = 'None'
-sliceAMRdata1Display.GaussianRadius = 0.35000000000000003
-sliceAMRdata1Display.SetScaleArray = [None, '']
-sliceAMRdata1Display.ScaleTransferFunction = 'PiecewiseFunction'
-sliceAMRdata1Display.OpacityArray = [None, '']
-sliceAMRdata1Display.OpacityTransferFunction = 'PiecewiseFunction'
-sliceAMRdata1Display.DataAxesGrid = 'GridAxesRepresentation'
-sliceAMRdata1Display.PolarAxes = 'PolarAxesRepresentation'
-sliceAMRdata1Display.ScalarOpacityUnitDistance = 2.3023915422888135
+slice1Display.Representation = 'Outline'
+slice1Display.ColorArrayName = [None, '']
+slice1Display.SelectTCoordArray = 'None'
+slice1Display.SelectNormalArray = 'None'
+slice1Display.SelectTangentArray = 'None'
+slice1Display.OSPRayScaleFunction = 'PiecewiseFunction'
+slice1Display.SelectOrientationVectors = 'None'
+slice1Display.ScaleFactor = 7.0
+slice1Display.SelectScaleArray = 'None'
+slice1Display.GlyphType = 'Arrow'
+slice1Display.GlyphTableIndexArray = 'None'
+slice1Display.GaussianRadius = 0.35000000000000003
+slice1Display.SetScaleArray = [None, '']
+slice1Display.ScaleTransferFunction = 'PiecewiseFunction'
+slice1Display.OpacityArray = [None, '']
+slice1Display.OpacityTransferFunction = 'PiecewiseFunction'
+slice1Display.DataAxesGrid = 'GridAxesRepresentation'
+slice1Display.PolarAxes = 'PolarAxesRepresentation'
+slice1Display.SelectInputVectors = [None, '']
+slice1Display.WriteLog = ''
 
 # ----------------------------------------------------------------
 # setup extractors
@@ -94,10 +113,13 @@
 # create extractor
 pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
 # trace defaults for the extractor.
+pNG1.Trigger = 'TimeStep'
+
 # init the 'PNG' selected for 'Writer'
-pNG1.Writer.FileName = 'AMRBlocksSlice_%.6ts%cm.png'
-pNG1.Writer.ImageResolution = [885, 774]
+pNG1.Writer.FileName = 'SliceAMRBlocks_{timestep:06d}{camera}.png'
+pNG1.Writer.ImageResolution = [2048, 1536]
 pNG1.Writer.Format = 'PNG'
+pNG1.Writer.ResetDisplay = 1
 
 # ----------------------------------------------------------------
 # restore active source
@@ -109,7 +131,6 @@
 from paraview import catalyst
 options = catalyst.Options()
 options.GlobalTrigger = 'TimeStep'
-options.EnableCatalystLive = 1
 options.CatalystLiveTrigger = 'TimeStep'
 
 # ------------------------------------------------------------------------------
diff --git a/Examples/BinaryBH/catalyst_scripts/slice_chi.py b/Examples/BinaryBH/catalyst_scripts/slice_center_z_chi.py
similarity index 99%
rename from Examples/BinaryBH/catalyst_scripts/slice_chi.py
rename to Examples/BinaryBH/catalyst_scripts/slice_center_z_chi.py
index b8346b60f..7eecab3fc 100644
--- a/Examples/BinaryBH/catalyst_scripts/slice_chi.py
+++ b/Examples/BinaryBH/catalyst_scripts/slice_center_z_chi.py
@@ -1,6 +1,9 @@
 # script-version: 2.0
 # Catalyst state generated using paraview version 5.9.1
 
+# This script takes a slice through the center of the domain of constant z, 
+# renders a surface plot of chi and then saves the view to a PNG
+
 # Import this to get environment variables
 import os
 
@@ -60,7 +63,7 @@
 # setup the data processing pipelines
 # ----------------------------------------------------------------
 
-# create a new 'VisItChomboReader'
+# create source
 input = AMRGaussianPulseSource(registrationName='input')
 input.MeshStatus = ['Mesh']
 input.CellArrayStatus = ['chi']
diff --git a/Examples/BinaryBH/catalyst_scripts/slice_center_z_to_vtm.py b/Examples/BinaryBH/catalyst_scripts/slice_center_z_to_vtm.py
new file mode 100644
index 000000000..42f45d323
--- /dev/null
+++ b/Examples/BinaryBH/catalyst_scripts/slice_center_z_to_vtm.py
@@ -0,0 +1,67 @@
+# script-version: 2.0
+# Catalyst state generated using paraview version 5.11.0
+
+# This script takes a slice through the center of the domain of constant z and
+# saves it to a VTM file (+ corresponding folder) that can be visualized in 
+# ParaView (all variables passed to Catalyst are saved)
+
+# Import this to get environment variables
+import os
+
+#### import the simple module from the paraview
+from paraview.simple import *
+#### disable automatic camera reset on 'Show'
+paraview.simple._DisableFirstRenderCameraReset()
+
+
+# Get center from environment
+grchombo_center_str = os.getenv('GRCHOMBO_PARAM_CENTER')
+grchombo_center = [float(dim) for dim in grchombo_center_str.split(' ')]
+
+# ----------------------------------------------------------------
+# setup the data processing pipelines
+# ----------------------------------------------------------------
+
+# create source
+input = AMRGaussianPulseSource(registrationName='input')
+input.MeshStatus = ['Mesh']
+
+# create a new 'Slice With Plane'
+sliceWithPlane1 = SliceWithPlane(registrationName='SliceWithPlane1', Input=input)
+sliceWithPlane1.PlaneType = 'Plane'
+sliceWithPlane1.Level = 3
+
+# init the 'Plane' selected for 'PlaneType'
+sliceWithPlane1.PlaneType.Origin = grchombo_center
+sliceWithPlane1.PlaneType.Normal = [0.0, 0.0, 1.0]
+
+# ----------------------------------------------------------------
+# setup extractors
+# ----------------------------------------------------------------
+
+# create extractor
+vTM1 = CreateExtractor('VTM', sliceWithPlane1, registrationName='VTM1')
+# trace defaults for the extractor.
+vTM1.Trigger = 'TimeStep'
+
+# init the 'VTM' selected for 'Writer'
+vTM1.Writer.FileName = 'CenterZSlice_{timestep:06d}.vtm'
+
+# ----------------------------------------------------------------
+# restore active source
+SetActiveSource(vTM1)
+# ----------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# Catalyst options
+from paraview import catalyst
+options = catalyst.Options()
+options.GlobalTrigger = 'TimeStep'
+options.CatalystLiveTrigger = 'TimeStep'
+
+# ------------------------------------------------------------------------------
+if __name__ == '__main__':
+    from paraview.simple import SaveExtractsUsingCatalystOptions
+    # Code for non in-situ environments; if executing in post-processing
+    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
+    SaveExtractsUsingCatalystOptions(options)
diff --git a/Examples/BinaryBH/params_catalyst.txt b/Examples/BinaryBH/params_catalyst.txt
index fb29eeca4..a5a3a40a4 100644
--- a/Examples/BinaryBH/params_catalyst.txt
+++ b/Examples/BinaryBH/params_catalyst.txt
@@ -191,7 +191,7 @@ catalyst_verbosity = 2
 catalyst_scripts_path = catalyst_scripts
 catalyst_num_scripts = 1
 # list of Catalyst scripts
-catalyst_scripts = volume_raytrace_resample_chi.py
+catalyst_scripts = slice_center_z_chi.py
 catalyst_coprocess_level = 1
 catalyst_num_vars = 1
 catalyst_vars = chi

From d84a86a2eee5e523fcf0b2ca4e46a4be6828df92 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 7 Feb 2023 14:17:03 +0000
Subject: [PATCH 61/63] Update Catalyst insitu action and clang-format

---
 .github/workflows/run-catalyst-insitu-test.yml | 18 +++++++++++-------
 .../CatalystInsituTest/CatalystInsituTest.cpp  |  3 ++-
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/run-catalyst-insitu-test.yml b/.github/workflows/run-catalyst-insitu-test.yml
index 92d8ec056..6f6764b6e 100644
--- a/.github/workflows/run-catalyst-insitu-test.yml
+++ b/.github/workflows/run-catalyst-insitu-test.yml
@@ -4,7 +4,7 @@ on: [push]
 
 jobs:
   build-and-test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       CHOMBO_HOME: ${{ github.workspace }}/Chombo/lib
       PARAVIEW_SUPERBUILD_DIR: ${{ github.workspace }}/build
@@ -25,14 +25,18 @@ jobs:
 
     - name: Install Chombo dependencies
       run: |
+        # First update package repository mirrors
+        sudo gem install apt-spy2
+        sudo apt-spy2 check
+        sudo apt-spy2 fix --commit
         sudo apt-get update
-        sudo apt-get -y --no-install-recommends install csh gfortran-10 g++-10 cpp-10 libhdf5-dev libhdf5-openmpi-dev openmpi-bin libblas-dev liblapack-dev libgetopt-complete-perl
+        sudo apt-get -y --no-install-recommends install csh libhdf5-dev libhdf5-openmpi-dev openmpi-bin libblas-dev liblapack-dev libgetopt-complete-perl
 
     - name: Set Compilers
       run: |
-        sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100
-        sudo update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-10 100
-        sudo update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-10 100
+        sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 120
+        sudo update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-12 120
+        sudo update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-12 120
 
     - name: Build Chombo
       run: |
@@ -45,8 +49,8 @@ jobs:
         sudo apt-get update
         sudo apt-get -y install build-essential libgl1-mesa-dev \
         libxt-dev python3-dev python3-numpy libopenmpi-dev libtbb-dev \
-        ninja-build libosmesa6 libosmesa6-dev llvm-12-dev
-        sudo update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-12 12
+        ninja-build libosmesa6 libosmesa6-dev llvm-14-dev
+        sudo update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 14
 
 # There is a problem with the paraview-config script with CMake v3.23
 # (see https://gitlab.kitware.com/paraview/paraview/-/issues/21361)
diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.cpp b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
index 63ee16e25..84cdd80bc 100644
--- a/Tests/CatalystInsituTest/CatalystInsituTest.cpp
+++ b/Tests/CatalystInsituTest/CatalystInsituTest.cpp
@@ -74,7 +74,8 @@ int runInsituTest(int argc, char *argv[])
                                                                    sim_params);
     setupAMRObject(gr_amr, insitu_test_level_factory);
 
-    auto task = [](GRAMRLevel *a_level) {
+    auto task = [](GRAMRLevel *a_level)
+    {
         if (a_level->time() == 0.0)
         {
             a_level->catalystCoProcess();

From cee303118739bc41d2f7d6676f9a393cfc036e01 Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Tue, 14 Feb 2023 12:25:22 +0000
Subject: [PATCH 62/63] Fix compilation with old VTK versions

The vtkPythonInterpreter::SetRedirectOutput() function was introduced
with VTK version 9.1.0.
---
 Source/GRChomboCore/SetupFunctions.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp
index 03f05afc4..de4da6e6e 100644
--- a/Source/GRChomboCore/SetupFunctions.hpp
+++ b/Source/GRChomboCore/SetupFunctions.hpp
@@ -220,8 +220,10 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
     // set vtkLogger internal messages to only appear at verbosity 2
     vtkLogger::SetInternalVerbosityLevel(vtkLogger::VERBOSITY_3);
 #endif
+#if VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 1, 0)
     // don't redirect Python interpreter output
     vtkPythonInterpreter::SetRedirectOutput(false);
+#endif
 #ifdef CH_MPI
     vtkLogger::SetStderrVerbosity(vtkLogger::VERBOSITY_ERROR);
     std::string catalyst_log_file = chombo_params.pout_path +

From d084a7629f36a5fc2e5c2aaf9130c1eb5202c1cf Mon Sep 17 00:00:00 2001
From: Miren Radia <miren_radia@yahoo.co.uk>
Date: Wed, 15 Feb 2023 10:52:41 +0000
Subject: [PATCH 63/63] Fix buffer overflow warning

---
 Source/Insitu/CatalystAdaptor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp
index 2a2f0455b..9b76a10fd 100644
--- a/Source/Insitu/CatalystAdaptor.cpp
+++ b/Source/Insitu/CatalystAdaptor.cpp
@@ -407,7 +407,7 @@ void CatalystAdaptor::write_vtk_grid(unsigned int a_timestep)
     vtkNew<vtkXMLPUniformGridAMRWriter> file_writer;
 
     // make filename
-    char timestep_cstr[7];
+    char timestep_cstr[100];
     std::sprintf(timestep_cstr, "%06d.", a_timestep);
     std::string filename = m_p.vtk_file_prefix;
     filename += timestep_cstr;