Add ParaView Catalyst in-situ visualization capability

.github/workflows/run-catalyst-insitu-test.yml

@@ -0,0 +1,94 @@
+name: Run Catalyst Insitu test
+
+on: [push]
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-22.04
+    env:
+      CHOMBO_HOME: ${{ github.workspace }}/Chombo/lib
+      PARAVIEW_SUPERBUILD_DIR: ${{ github.workspace }}/build
+      PARAVIEW_DIR: ${{ github.workspace }}/build/install
+      OMP_NUM_THREADS: 1
+
+    steps:
+    - name: Checkout Chombo
+      uses: actions/checkout@v3
+      with:
+        repository: GRChombo/Chombo
+        path: Chombo
+
+    - name: Checkout GRChombo
+      uses: actions/checkout@v3
+      with:
+        path: GRChombo
+
+    - name: Install Chombo dependencies
+      run: |
+        # First update package repository mirrors
+        sudo gem install apt-spy2
+        sudo apt-spy2 check
+        sudo apt-spy2 fix --commit
+        sudo apt-get update
+        sudo apt-get -y --no-install-recommends install csh libhdf5-dev libhdf5-openmpi-dev openmpi-bin libblas-dev liblapack-dev libgetopt-complete-perl
+
+    - name: Set Compilers
+      run: |
+        sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 120
+        sudo update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-12 120
+        sudo update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-12 120
+
+    - name: Build Chombo
+      run: |
+        cp $GITHUB_WORKSPACE/GRChombo/InstallNotes/MakeDefsLocalExamples/ubuntu-gcc.Make.defs.local $CHOMBO_HOME/mk/Make.defs.local
+        make -j 4 AMRTimeDependent AMRTools BaseTools BoxTools
+      working-directory: ${{ env.CHOMBO_HOME }}
+
+    - name: Install ParaView dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install build-essential libgl1-mesa-dev \
+        libxt-dev python3-dev python3-numpy libopenmpi-dev libtbb-dev \
+        ninja-build libosmesa6 libosmesa6-dev llvm-14-dev
+        sudo update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 14
+
+# There is a problem with the paraview-config script with CMake v3.23
+# (see https://gitlab.kitware.com/paraview/paraview/-/issues/21361)
+    - name: Downgrade CMake
+      env:
+        CMAKE_VERSION: 3.22.6
+      run: |
+        wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh
+        chmod +x cmake-${CMAKE_VERSION}-linux-x86_64.sh
+        sudo ./cmake-${CMAKE_VERSION}-linux-x86_64.sh --prefix=/usr/local --exclude-subdir
+
+    - name: Download pre-built ParaView
+      uses: dsaltares/fetch-gh-release-asset@master
+      with:
+        repo: "GRChombo/paraview-build"
+        version: latest
+        regex: true
+        file: "paraview-v.*.tar.gz"
+
+    - name: Untar ParaView
+      run: |
+        mkdir -p $PARAVIEW_DIR
+        cd $PARAVIEW_DIR
+        ls $GITHUB_WORKSPACE
+        tar -xzvf $GITHUB_WORKSPACE/paraview-v*.tar.gz
+
+    - name: Build Catalyst Insitu test
+      run: make test -j 4
+      working-directory: ${{ github.workspace }}/GRChombo/Tests/CatalystInsituTest
+
+    - name: Run Catalyst Insitu test
+      run: make run
+      working-directory: ${{ github.workspace }}/GRChombo/Tests/CatalystInsituTest
+
+    - name: Upload output files if test failed
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+        name: CatalystInsituTest-pouts
+        path: ${{ github.workspace }}/GRChombo/Tests/CatalystInsituTest/pout.*
+        if-no-files-found: warn

.gitignore

@@ -25,3 +25,9 @@ time.table.*
 ipo_out.optrpt
 
 Doxygen/html
+
+# ParaView Catalyst
+catalyst_pout.*
+*.png
+*.jpg
+datasets/*

Examples/BinaryBH/BinaryBHLevel.cpp

@@ -138,10 +138,21 @@ void BinaryBHLevel::specificPostTimeStep()
                         // called during setup at t=0 from Main
     // bool first_step = (m_time == m_dt); // if not called in Main
 
-    if (m_p.activate_extraction == 1)
+    if (m_p.activate_extraction
+#ifdef USE_CATALYST
+        || m_p.catalyst_activate
+#endif
+    )
     {
-        int min_level = m_p.extraction_params.min_extraction_level();
+        int min_level = (m_p.activate_extraction)
+                            ? m_p.extraction_params.min_extraction_level()
+                            : -1;
         bool calculate_weyl = at_level_timestep_multiple(min_level);
+#ifdef USE_CATALYST
+        calculate_weyl |=
+            (m_p.catalyst_activate &&
+             at_level_timestep_multiple(m_p.catalyst_coprocess_level));
+#endif
         if (calculate_weyl)
         {
             // Populate the Weyl Scalar values on the grid

Examples/BinaryBH/GNUmakefile

@@ -41,4 +41,8 @@ ifeq ($(USE_TWOPUNCTURES),TRUE)
   src_dirs += $(TWOPUNCTURES_SOURCE)
 endif
 
+# ParaView Catalyst insitu visualisation
+# Set PARAVIEW_DIR environment variable to use
+include $(GRCHOMBO_SOURCE)/Insitu/Make.insitu
+
 include $(CHOMBO_HOME)/mk/Make.test

Examples/BinaryBH/Main_BinaryBH.cpp

@@ -81,11 +81,17 @@ int runGRChombo(int argc, char *argv[])
 
     std::chrono::time_point<Clock> start_time = Clock::now();
 
-    // Add a scheduler to call specificPostTimeStep on every AMRLevel at t=0
+    // Add a scheduler to call specificPostTimeStep and catalystCoProcess
+    // on every AMRLevel at t=0
     auto task = [](GRAMRLevel *level)
     {
         if (level->time() == 0.)
+        {
             level->specificPostTimeStep();
+#ifdef USE_CATALYST
+            level->catalystCoProcess();
+#endif
+        }
     };
     // call 'now' really now
     MultiLevelTaskPtr<> call_task(task);

Examples/BinaryBH/catalyst_scripts/slice_center_z_amr_blocks.py

@@ -0,0 +1,141 @@
+# script-version: 2.0
+# Catalyst state generated using paraview version 5.11.0
+
+# This script takes a slice through the center of the domain of constant z,
+# renders an outline of the blocks in the domain and saves the result to a PNG
+
+# Import this to get environment variables
+import os
+
+#### import the simple module from the paraview
+from paraview.simple import *
+#### disable automatic camera reset on 'Show'
+paraview.simple._DisableFirstRenderCameraReset()
+
+# Get center from environment
+grchombo_center_str = os.getenv('GRCHOMBO_PARAM_CENTER')
+grchombo_center = [float(dim) for dim in grchombo_center_str.split(' ')]
+
+# ----------------------------------------------------------------
+# setup views used in the visualization
+# ----------------------------------------------------------------
+
+# get the material library
+materialLibrary1 = GetMaterialLibrary()
+
+# Create a new 'Render View'
+renderView1 = CreateView('RenderView')
+renderView1.ViewSize = [928, 789]
+renderView1.AxesGrid = 'GridAxes3DActor'
+renderView1.CenterOfRotation = grchombo_center
+renderView1.StereoType = 'Crystal Eyes'
+renderView1.CameraPosition = [grchombo_center[0], grchombo_center[1], 500.0]
+renderView1.CameraFocalPoint = grchombo_center
+renderView1.CameraFocalDisk = 1.0
+renderView1.CameraParallelScale = 49.50757517794625
+renderView1.BackEnd = 'OSPRay raycaster'
+renderView1.OSPRayMaterialLibrary = materialLibrary1
+
+# init the 'GridAxes3DActor' selected for 'AxesGrid'
+renderView1.AxesGrid.Visibility = 1
+
+SetActiveView(None)
+
+# ----------------------------------------------------------------
+# setup view layouts
+# ----------------------------------------------------------------
+
+# create new layout object 'Layout #1'
+layout1 = CreateLayout(name='Layout #1')
+layout1.AssignView(0, renderView1)
+layout1.SetSize(928, 789)
+
+# ----------------------------------------------------------------
+# restore active view
+SetActiveView(renderView1)
+# ----------------------------------------------------------------
+
+# ----------------------------------------------------------------
+# setup the data processing pipelines
+# ----------------------------------------------------------------
+
+# create source
+input = AMRGaussianPulseSource(registrationName='input')
+input.MeshStatus = ['Mesh']
+input.CellArrayStatus = []
+
+# create a new 'Slice'
+slice1 = Slice(registrationName='Slice1', Input=input)
+slice1.SliceType = 'Plane'
+slice1.HyperTreeGridSlicer = 'Plane'
+slice1.SliceOffsetValues = [0.0]
+
+# init the 'Plane' selected for 'SliceType'
+slice1.SliceType.Origin = grchombo_center
+slice1.SliceType.Normal = [0.0, 0.0, 1.0]
+
+# init the 'Plane' selected for 'HyperTreeGridSlicer'
+slice1.HyperTreeGridSlicer.Origin = grchombo_center
+
+# ----------------------------------------------------------------
+# setup the visualization in view 'renderView1'
+# ----------------------------------------------------------------
+
+# show data from slice1
+slice1Display = Show(slice1, renderView1, 'GeometryRepresentation')
+
+# trace defaults for the display properties.
+slice1Display.Representation = 'Outline'
+slice1Display.ColorArrayName = [None, '']
+slice1Display.SelectTCoordArray = 'None'
+slice1Display.SelectNormalArray = 'None'
+slice1Display.SelectTangentArray = 'None'
+slice1Display.OSPRayScaleFunction = 'PiecewiseFunction'
+slice1Display.SelectOrientationVectors = 'None'
+slice1Display.ScaleFactor = 7.0
+slice1Display.SelectScaleArray = 'None'
+slice1Display.GlyphType = 'Arrow'
+slice1Display.GlyphTableIndexArray = 'None'
+slice1Display.GaussianRadius = 0.35000000000000003
+slice1Display.SetScaleArray = [None, '']
+slice1Display.ScaleTransferFunction = 'PiecewiseFunction'
+slice1Display.OpacityArray = [None, '']
+slice1Display.OpacityTransferFunction = 'PiecewiseFunction'
+slice1Display.DataAxesGrid = 'GridAxesRepresentation'
+slice1Display.PolarAxes = 'PolarAxesRepresentation'
+slice1Display.SelectInputVectors = [None, '']
+slice1Display.WriteLog = ''
+
+# ----------------------------------------------------------------
+# setup extractors
+# ----------------------------------------------------------------
+
+# create extractor
+pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1')
+# trace defaults for the extractor.
+pNG1.Trigger = 'TimeStep'
+
+# init the 'PNG' selected for 'Writer'
+pNG1.Writer.FileName = 'SliceAMRBlocks_{timestep:06d}{camera}.png'
+pNG1.Writer.ImageResolution = [2048, 1536]
+pNG1.Writer.Format = 'PNG'
+pNG1.Writer.ResetDisplay = 1
+
+# ----------------------------------------------------------------
+# restore active source
+SetActiveSource(pNG1)
+# ----------------------------------------------------------------
+
+# ------------------------------------------------------------------------------
+# Catalyst options
+from paraview import catalyst
+options = catalyst.Options()
+options.GlobalTrigger = 'TimeStep'
+options.CatalystLiveTrigger = 'TimeStep'
+
+# ------------------------------------------------------------------------------
+if __name__ == '__main__':
+    from paraview.simple import SaveExtractsUsingCatalystOptions
+    # Code for non in-situ environments; if executing in post-processing
+    # i.e. non-Catalyst mode, let's generate extracts using Catalyst options
+    SaveExtractsUsingCatalystOptions(options)