diff --git a/.github/workflows/run-catalyst-insitu-test.yml b/.github/workflows/run-catalyst-insitu-test.yml new file mode 100644 index 000000000..6f6764b6e --- /dev/null +++ b/.github/workflows/run-catalyst-insitu-test.yml @@ -0,0 +1,94 @@ +name: Run Catalyst Insitu test + +on: [push] + +jobs: + build-and-test: + runs-on: ubuntu-22.04 + env: + CHOMBO_HOME: ${{ github.workspace }}/Chombo/lib + PARAVIEW_SUPERBUILD_DIR: ${{ github.workspace }}/build + PARAVIEW_DIR: ${{ github.workspace }}/build/install + OMP_NUM_THREADS: 1 + + steps: + - name: Checkout Chombo + uses: actions/checkout@v3 + with: + repository: GRChombo/Chombo + path: Chombo + + - name: Checkout GRChombo + uses: actions/checkout@v3 + with: + path: GRChombo + + - name: Install Chombo dependencies + run: | + # First update package repository mirrors + sudo gem install apt-spy2 + sudo apt-spy2 check + sudo apt-spy2 fix --commit + sudo apt-get update + sudo apt-get -y --no-install-recommends install csh libhdf5-dev libhdf5-openmpi-dev openmpi-bin libblas-dev liblapack-dev libgetopt-complete-perl + + - name: Set Compilers + run: | + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 120 + sudo update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-12 120 + sudo update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-12 120 + + - name: Build Chombo + run: | + cp $GITHUB_WORKSPACE/GRChombo/InstallNotes/MakeDefsLocalExamples/ubuntu-gcc.Make.defs.local $CHOMBO_HOME/mk/Make.defs.local + make -j 4 AMRTimeDependent AMRTools BaseTools BoxTools + working-directory: ${{ env.CHOMBO_HOME }} + + - name: Install ParaView dependencies + run: | + sudo apt-get update + sudo apt-get -y install build-essential libgl1-mesa-dev \ + libxt-dev python3-dev python3-numpy libopenmpi-dev libtbb-dev \ + ninja-build libosmesa6 libosmesa6-dev llvm-14-dev + sudo update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-14 14 + +# There is a problem with the paraview-config script with CMake v3.23 +# (see https://gitlab.kitware.com/paraview/paraview/-/issues/21361) + - name: Downgrade CMake + env: + CMAKE_VERSION: 3.22.6 + run: | + wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh + chmod +x cmake-${CMAKE_VERSION}-linux-x86_64.sh + sudo ./cmake-${CMAKE_VERSION}-linux-x86_64.sh --prefix=/usr/local --exclude-subdir + + - name: Download pre-built ParaView + uses: dsaltares/fetch-gh-release-asset@master + with: + repo: "GRChombo/paraview-build" + version: latest + regex: true + file: "paraview-v.*.tar.gz" + + - name: Untar ParaView + run: | + mkdir -p $PARAVIEW_DIR + cd $PARAVIEW_DIR + ls $GITHUB_WORKSPACE + tar -xzvf $GITHUB_WORKSPACE/paraview-v*.tar.gz + + - name: Build Catalyst Insitu test + run: make test -j 4 + working-directory: ${{ github.workspace }}/GRChombo/Tests/CatalystInsituTest + + - name: Run Catalyst Insitu test + run: make run + working-directory: ${{ github.workspace }}/GRChombo/Tests/CatalystInsituTest + + - name: Upload output files if test failed + if: failure() + uses: actions/upload-artifact@v3 + with: + name: CatalystInsituTest-pouts + path: ${{ github.workspace }}/GRChombo/Tests/CatalystInsituTest/pout.* + if-no-files-found: warn \ No newline at end of file diff --git a/.gitignore b/.gitignore index 290526597..1b4095f2b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,9 @@ time.table.* ipo_out.optrpt Doxygen/html + +# ParaView Catalyst +catalyst_pout.* +*.png +*.jpg +datasets/* diff --git a/Examples/BinaryBH/BinaryBHLevel.cpp b/Examples/BinaryBH/BinaryBHLevel.cpp index cf17984d7..c5e52b9ae 100644 --- a/Examples/BinaryBH/BinaryBHLevel.cpp +++ b/Examples/BinaryBH/BinaryBHLevel.cpp @@ -138,10 +138,21 @@ void BinaryBHLevel::specificPostTimeStep() // called during setup at t=0 from Main // bool first_step = (m_time == m_dt); // if not called in Main - if (m_p.activate_extraction == 1) + if (m_p.activate_extraction +#ifdef USE_CATALYST + || m_p.catalyst_activate +#endif + ) { - int min_level = m_p.extraction_params.min_extraction_level(); + int min_level = (m_p.activate_extraction) + ? m_p.extraction_params.min_extraction_level() + : -1; bool calculate_weyl = at_level_timestep_multiple(min_level); +#ifdef USE_CATALYST + calculate_weyl |= + (m_p.catalyst_activate && + at_level_timestep_multiple(m_p.catalyst_coprocess_level)); +#endif if (calculate_weyl) { // Populate the Weyl Scalar values on the grid diff --git a/Examples/BinaryBH/GNUmakefile b/Examples/BinaryBH/GNUmakefile index fe043e5f7..8d2f2a9f8 100644 --- a/Examples/BinaryBH/GNUmakefile +++ b/Examples/BinaryBH/GNUmakefile @@ -41,4 +41,8 @@ ifeq ($(USE_TWOPUNCTURES),TRUE) src_dirs += $(TWOPUNCTURES_SOURCE) endif +# ParaView Catalyst insitu visualisation +# Set PARAVIEW_DIR environment variable to use +include $(GRCHOMBO_SOURCE)/Insitu/Make.insitu + include $(CHOMBO_HOME)/mk/Make.test diff --git a/Examples/BinaryBH/Main_BinaryBH.cpp b/Examples/BinaryBH/Main_BinaryBH.cpp index 120dec3a8..0d9a6fbdd 100644 --- a/Examples/BinaryBH/Main_BinaryBH.cpp +++ b/Examples/BinaryBH/Main_BinaryBH.cpp @@ -81,11 +81,17 @@ int runGRChombo(int argc, char *argv[]) std::chrono::time_point start_time = Clock::now(); - // Add a scheduler to call specificPostTimeStep on every AMRLevel at t=0 + // Add a scheduler to call specificPostTimeStep and catalystCoProcess + // on every AMRLevel at t=0 auto task = [](GRAMRLevel *level) { if (level->time() == 0.) + { level->specificPostTimeStep(); +#ifdef USE_CATALYST + level->catalystCoProcess(); +#endif + } }; // call 'now' really now MultiLevelTaskPtr<> call_task(task); diff --git a/Examples/BinaryBH/catalyst_scripts/slice_center_z_amr_blocks.py b/Examples/BinaryBH/catalyst_scripts/slice_center_z_amr_blocks.py new file mode 100644 index 000000000..e29e2567e --- /dev/null +++ b/Examples/BinaryBH/catalyst_scripts/slice_center_z_amr_blocks.py @@ -0,0 +1,141 @@ +# script-version: 2.0 +# Catalyst state generated using paraview version 5.11.0 + +# This script takes a slice through the center of the domain of constant z, +# renders an outline of the blocks in the domain and saves the result to a PNG + +# Import this to get environment variables +import os + +#### import the simple module from the paraview +from paraview.simple import * +#### disable automatic camera reset on 'Show' +paraview.simple._DisableFirstRenderCameraReset() + +# Get center from environment +grchombo_center_str = os.getenv('GRCHOMBO_PARAM_CENTER') +grchombo_center = [float(dim) for dim in grchombo_center_str.split(' ')] + +# ---------------------------------------------------------------- +# setup views used in the visualization +# ---------------------------------------------------------------- + +# get the material library +materialLibrary1 = GetMaterialLibrary() + +# Create a new 'Render View' +renderView1 = CreateView('RenderView') +renderView1.ViewSize = [928, 789] +renderView1.AxesGrid = 'GridAxes3DActor' +renderView1.CenterOfRotation = grchombo_center +renderView1.StereoType = 'Crystal Eyes' +renderView1.CameraPosition = [grchombo_center[0], grchombo_center[1], 500.0] +renderView1.CameraFocalPoint = grchombo_center +renderView1.CameraFocalDisk = 1.0 +renderView1.CameraParallelScale = 49.50757517794625 +renderView1.BackEnd = 'OSPRay raycaster' +renderView1.OSPRayMaterialLibrary = materialLibrary1 + +# init the 'GridAxes3DActor' selected for 'AxesGrid' +renderView1.AxesGrid.Visibility = 1 + +SetActiveView(None) + +# ---------------------------------------------------------------- +# setup view layouts +# ---------------------------------------------------------------- + +# create new layout object 'Layout #1' +layout1 = CreateLayout(name='Layout #1') +layout1.AssignView(0, renderView1) +layout1.SetSize(928, 789) + +# ---------------------------------------------------------------- +# restore active view +SetActiveView(renderView1) +# ---------------------------------------------------------------- + +# ---------------------------------------------------------------- +# setup the data processing pipelines +# ---------------------------------------------------------------- + +# create source +input = AMRGaussianPulseSource(registrationName='input') +input.MeshStatus = ['Mesh'] +input.CellArrayStatus = [] + +# create a new 'Slice' +slice1 = Slice(registrationName='Slice1', Input=input) +slice1.SliceType = 'Plane' +slice1.HyperTreeGridSlicer = 'Plane' +slice1.SliceOffsetValues = [0.0] + +# init the 'Plane' selected for 'SliceType' +slice1.SliceType.Origin = grchombo_center +slice1.SliceType.Normal = [0.0, 0.0, 1.0] + +# init the 'Plane' selected for 'HyperTreeGridSlicer' +slice1.HyperTreeGridSlicer.Origin = grchombo_center + +# ---------------------------------------------------------------- +# setup the visualization in view 'renderView1' +# ---------------------------------------------------------------- + +# show data from slice1 +slice1Display = Show(slice1, renderView1, 'GeometryRepresentation') + +# trace defaults for the display properties. +slice1Display.Representation = 'Outline' +slice1Display.ColorArrayName = [None, ''] +slice1Display.SelectTCoordArray = 'None' +slice1Display.SelectNormalArray = 'None' +slice1Display.SelectTangentArray = 'None' +slice1Display.OSPRayScaleFunction = 'PiecewiseFunction' +slice1Display.SelectOrientationVectors = 'None' +slice1Display.ScaleFactor = 7.0 +slice1Display.SelectScaleArray = 'None' +slice1Display.GlyphType = 'Arrow' +slice1Display.GlyphTableIndexArray = 'None' +slice1Display.GaussianRadius = 0.35000000000000003 +slice1Display.SetScaleArray = [None, ''] +slice1Display.ScaleTransferFunction = 'PiecewiseFunction' +slice1Display.OpacityArray = [None, ''] +slice1Display.OpacityTransferFunction = 'PiecewiseFunction' +slice1Display.DataAxesGrid = 'GridAxesRepresentation' +slice1Display.PolarAxes = 'PolarAxesRepresentation' +slice1Display.SelectInputVectors = [None, ''] +slice1Display.WriteLog = '' + +# ---------------------------------------------------------------- +# setup extractors +# ---------------------------------------------------------------- + +# create extractor +pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1') +# trace defaults for the extractor. +pNG1.Trigger = 'TimeStep' + +# init the 'PNG' selected for 'Writer' +pNG1.Writer.FileName = 'SliceAMRBlocks_{timestep:06d}{camera}.png' +pNG1.Writer.ImageResolution = [2048, 1536] +pNG1.Writer.Format = 'PNG' +pNG1.Writer.ResetDisplay = 1 + +# ---------------------------------------------------------------- +# restore active source +SetActiveSource(pNG1) +# ---------------------------------------------------------------- + +# ------------------------------------------------------------------------------ +# Catalyst options +from paraview import catalyst +options = catalyst.Options() +options.GlobalTrigger = 'TimeStep' +options.CatalystLiveTrigger = 'TimeStep' + +# ------------------------------------------------------------------------------ +if __name__ == '__main__': + from paraview.simple import SaveExtractsUsingCatalystOptions + # Code for non in-situ environments; if executing in post-processing + # i.e. non-Catalyst mode, let's generate extracts using Catalyst options + SaveExtractsUsingCatalystOptions(options) diff --git a/Examples/BinaryBH/catalyst_scripts/slice_center_z_chi.py b/Examples/BinaryBH/catalyst_scripts/slice_center_z_chi.py new file mode 100644 index 000000000..7eecab3fc --- /dev/null +++ b/Examples/BinaryBH/catalyst_scripts/slice_center_z_chi.py @@ -0,0 +1,173 @@ +# script-version: 2.0 +# Catalyst state generated using paraview version 5.9.1 + +# This script takes a slice through the center of the domain of constant z, +# renders a surface plot of chi and then saves the view to a PNG + +# Import this to get environment variables +import os + +#### import the simple module from the paraview +from paraview.simple import * +#### disable automatic camera reset on 'Show' +paraview.simple._DisableFirstRenderCameraReset() + +# Get center from environment +grchombo_center_str = os.getenv('GRCHOMBO_PARAM_CENTER') +grchombo_center = [float(dim) for dim in grchombo_center_str.split(' ')] + +# ---------------------------------------------------------------- +# setup views used in the visualization +# ---------------------------------------------------------------- + +# black background +LoadPalette('BlackBackground') + +# get the material library +materialLibrary1 = GetMaterialLibrary() + +# Create a new 'Render View' +renderView1 = CreateView('RenderView') +renderView1.ViewSize = [1092, 778] +renderView1.AxesGrid = 'GridAxes3DActor' +renderView1.CenterAxesVisibility = 0 +renderView1.CenterOfRotation = grchombo_center +renderView1.StereoType = 'Crystal Eyes' +renderView1.CameraPosition = [grchombo_center[0], grchombo_center[1], 500.0] +renderView1.CameraFocalPoint = grchombo_center +renderView1.CameraFocalDisk = 1.0 +renderView1.CameraParallelScale = 11.096170510586074 +renderView1.BackEnd = 'OSPRay raycaster' +renderView1.OSPRayMaterialLibrary = materialLibrary1 + +# init the 'GridAxes3DActor' selected for 'AxesGrid' +renderView1.AxesGrid.Visibility = 1 + +SetActiveView(None) + +# ---------------------------------------------------------------- +# setup view layouts +# ---------------------------------------------------------------- + +# create new layout object 'Layout #1' +layout1 = CreateLayout(name='Layout #1') +layout1.AssignView(0, renderView1) +layout1.SetSize(1092, 778) + +# ---------------------------------------------------------------- +# restore active view +SetActiveView(renderView1) +# ---------------------------------------------------------------- + +# ---------------------------------------------------------------- +# setup the data processing pipelines +# ---------------------------------------------------------------- + +# create source +input = AMRGaussianPulseSource(registrationName='input') +input.MeshStatus = ['Mesh'] +input.CellArrayStatus = ['chi'] + +# create a new 'Slice' +slice1 = Slice(registrationName='Slice1', Input=input) +slice1.SliceType = 'Plane' +slice1.HyperTreeGridSlicer = 'Plane' +slice1.Triangulatetheslice = 0 +slice1.SliceOffsetValues = [0.0] + +# init the 'Plane' selected for 'SliceType' +slice1.SliceType.Origin = grchombo_center +slice1.SliceType.Normal = [0.0, 0.0, 1.0] + +# init the 'Plane' selected for 'HyperTreeGridSlicer' +slice1.HyperTreeGridSlicer.Origin = grchombo_center + +# ---------------------------------------------------------------- +# setup the visualization in view 'renderView1' +# ---------------------------------------------------------------- + +# show data from slice1 +slice1Display = Show(slice1, renderView1, 'GeometryRepresentation') + +# get color transfer function/color map for 'chi' +chiLUT = GetColorTransferFunction('chi') +chiLUT.AutomaticRescaleRangeMode = 'Never' +chiLUT.RGBPoints = [0.0, 0.267004, 0.004874, 0.329415, 0.003922000000000001, 0.26851, 0.009605, 0.335427, 0.007843, 0.269944, 0.014625, 0.341379, 0.011765000000000001, 0.271305, 0.019942, 0.347269, 0.015686, 0.272594, 0.025563, 0.353093, 0.019608, 0.273809, 0.031497, 0.358853, 0.023529, 0.274952, 0.037752, 0.364543, 0.027451000000000003, 0.276022, 0.044167, 0.370164, 0.031373, 0.277018, 0.050344, 0.375715, 0.035294, 0.277941, 0.056324, 0.381191, 0.039216, 0.278791, 0.062145, 0.386592, 0.043137, 0.279566, 0.067836, 0.391917, 0.047059000000000004, 0.280267, 0.073417, 0.397163, 0.050980000000000004, 0.280894, 0.078907, 0.402329, 0.054902, 0.281446, 0.08432, 0.407414, 0.058824, 0.281924, 0.089666, 0.412415, 0.062745, 0.282327, 0.094955, 0.417331, 0.06666699999999999, 0.282656, 0.100196, 0.42216, 0.07058799999999998, 0.28291, 0.105393, 0.426902, 0.07451, 0.283091, 0.110553, 0.431554, 0.078431, 0.283197, 0.11568, 0.436115, 0.08235299999999998, 0.283229, 0.120777, 0.440584, 0.086275, 0.283187, 0.125848, 0.44496, 0.090196, 0.283072, 0.130895, 0.449241, 0.094118, 0.282884, 0.13592, 0.453427, 0.098039, 0.282623, 0.140926, 0.457517, 0.10196099999999998, 0.28229, 0.145912, 0.46151, 0.10588199999999999, 0.281887, 0.150881, 0.465405, 0.109804, 0.281412, 0.155834, 0.469201, 0.113725, 0.280868, 0.160771, 0.472899, 0.11764699999999999, 0.280255, 0.165693, 0.476498, 0.121569, 0.279574, 0.170599, 0.479997, 0.12549, 0.278826, 0.17549, 0.483397, 0.129412, 0.278012, 0.180367, 0.486697, 0.133333, 0.277134, 0.185228, 0.489898, 0.137255, 0.276194, 0.190074, 0.493001, 0.141176, 0.275191, 0.194905, 0.496005, 0.145098, 0.274128, 0.199721, 0.498911, 0.14902000000000004, 0.273006, 0.20452, 0.501721, 0.15294100000000002, 0.271828, 0.209303, 0.504434, 0.15686300000000003, 0.270595, 0.214069, 0.507052, 0.16078400000000004, 0.269308, 0.218818, 0.509577, 0.164706, 0.267968, 0.223549, 0.512008, 0.168627, 0.26658, 0.228262, 0.514349, 0.172549, 0.265145, 0.232956, 0.516599, 0.176471, 0.263663, 0.237631, 0.518762, 0.180392, 0.262138, 0.242286, 0.520837, 0.184314, 0.260571, 0.246922, 0.522828, 0.188235, 0.258965, 0.251537, 0.524736, 0.19215700000000002, 0.257322, 0.25613, 0.526563, 0.19607800000000003, 0.255645, 0.260703, 0.528312, 0.2, 0.253935, 0.265254, 0.529983, 0.203922, 0.252194, 0.269783, 0.531579, 0.207843, 0.250425, 0.27429, 0.533103, 0.211765, 0.248629, 0.278775, 0.534556, 0.215686, 0.246811, 0.283237, 0.535941, 0.219608, 0.244972, 0.287675, 0.53726, 0.223529, 0.243113, 0.292092, 0.538516, 0.22745100000000001, 0.241237, 0.296485, 0.539709, 0.231373, 0.239346, 0.300855, 0.540844, 0.235294, 0.237441, 0.305202, 0.541921, 0.239216, 0.235526, 0.309527, 0.542944, 0.243137, 0.233603, 0.313828, 0.543914, 0.247059, 0.231674, 0.318106, 0.544834, 0.25098, 0.229739, 0.322361, 0.545706, 0.254902, 0.227802, 0.326594, 0.546532, 0.258824, 0.225863, 0.330805, 0.547314, 0.262745, 0.223925, 0.334994, 0.548053, 0.266667, 0.221989, 0.339161, 0.548752, 0.270588, 0.220057, 0.343307, 0.549413, 0.27451, 0.21813, 0.347432, 0.550038, 0.27843100000000004, 0.21621, 0.351535, 0.550627, 0.2823530000000001, 0.214298, 0.355619, 0.551184, 0.286275, 0.212395, 0.359683, 0.55171, 0.290196, 0.210503, 0.363727, 0.552206, 0.294118, 0.208623, 0.367752, 0.552675, 0.298039, 0.206756, 0.371758, 0.553117, 0.301961, 0.204903, 0.375746, 0.553533, 0.305882, 0.203063, 0.379716, 0.553925, 0.309804, 0.201239, 0.38367, 0.554294, 0.313725, 0.19943, 0.387607, 0.554642, 0.317647, 0.197636, 0.391528, 0.554969, 0.321569, 0.19586, 0.395433, 0.555276, 0.32549, 0.1941, 0.399323, 0.555565, 0.329412, 0.192357, 0.403199, 0.555836, 0.333333, 0.190631, 0.407061, 0.556089, 0.337255, 0.188923, 0.41091, 0.556326, 0.341176, 0.187231, 0.414746, 0.556547, 0.345098, 0.185556, 0.41857, 0.556753, 0.34902, 0.183898, 0.422383, 0.556944, 0.35294100000000006, 0.182256, 0.426184, 0.55712, 0.356863, 0.180629, 0.429975, 0.557282, 0.360784, 0.179019, 0.433756, 0.55743, 0.364706, 0.177423, 0.437527, 0.557565, 0.368627, 0.175841, 0.44129, 0.557685, 0.372549, 0.174274, 0.445044, 0.557792, 0.376471, 0.172719, 0.448791, 0.557885, 0.380392, 0.171176, 0.45253, 0.557965, 0.38431400000000004, 0.169646, 0.456262, 0.55803, 0.388235, 0.168126, 0.459988, 0.558082, 0.392157, 0.166617, 0.463708, 0.558119, 0.39607800000000004, 0.165117, 0.467423, 0.558141, 0.4, 0.163625, 0.471133, 0.558148, 0.403922, 0.162142, 0.474838, 0.55814, 0.407843, 0.160665, 0.47854, 0.558115, 0.411765, 0.159194, 0.482237, 0.558073, 0.415686, 0.157729, 0.485932, 0.558013, 0.419608, 0.15627, 0.489624, 0.557936, 0.42352900000000004, 0.154815, 0.493313, 0.55784, 0.4274510000000001, 0.153364, 0.497, 0.557724, 0.431373, 0.151918, 0.500685, 0.557587, 0.435294, 0.150476, 0.504369, 0.55743, 0.439216, 0.149039, 0.508051, 0.55725, 0.443137, 0.147607, 0.511733, 0.557049, 0.447059, 0.14618, 0.515413, 0.556823, 0.45098, 0.144759, 0.519093, 0.556572, 0.45490200000000003, 0.143343, 0.522773, 0.556295, 0.45882400000000007, 0.141935, 0.526453, 0.555991, 0.462745, 0.140536, 0.530132, 0.555659, 0.466667, 0.139147, 0.533812, 0.555298, 0.470588, 0.13777, 0.537492, 0.554906, 0.47451, 0.136408, 0.541173, 0.554483, 0.478431, 0.135066, 0.544853, 0.554029, 0.482353, 0.133743, 0.548535, 0.553541, 0.486275, 0.132444, 0.552216, 0.553018, 0.490196, 0.131172, 0.555899, 0.552459, 0.494118, 0.129933, 0.559582, 0.551864, 0.49803900000000006, 0.128729, 0.563265, 0.551229, 0.501961, 0.127568, 0.566949, 0.550556, 0.505882, 0.126453, 0.570633, 0.549841, 0.509804, 0.125394, 0.574318, 0.549086, 0.513725, 0.124395, 0.578002, 0.548287, 0.517647, 0.123463, 0.581687, 0.547445, 0.5215690000000001, 0.122606, 0.585371, 0.546557, 0.52549, 0.121831, 0.589055, 0.545623, 0.529412, 0.121148, 0.592739, 0.544641, 0.5333330000000001, 0.120565, 0.596422, 0.543611, 0.537255, 0.120092, 0.600104, 0.54253, 0.541176, 0.119738, 0.603785, 0.5414, 0.545098, 0.119512, 0.607464, 0.540218, 0.54902, 0.119423, 0.611141, 0.538982, 0.552941, 0.119483, 0.614817, 0.537692, 0.556863, 0.119699, 0.61849, 0.536347, 0.560784, 0.120081, 0.622161, 0.534946, 0.5647060000000002, 0.120638, 0.625828, 0.533488, 0.568627, 0.12138, 0.629492, 0.531973, 0.572549, 0.122312, 0.633153, 0.530398, 0.576471, 0.123444, 0.636809, 0.528763, 0.580392, 0.12478, 0.640461, 0.527068, 0.584314, 0.126326, 0.644107, 0.525311, 0.588235, 0.128087, 0.647749, 0.523491, 0.592157, 0.130067, 0.651384, 0.521608, 0.596078, 0.132268, 0.655014, 0.519661, 0.6, 0.134692, 0.658636, 0.517649, 0.603922, 0.137339, 0.662252, 0.515571, 0.6078430000000001, 0.14021, 0.665859, 0.513427, 0.611765, 0.143303, 0.669459, 0.511215, 0.615686, 0.146616, 0.67305, 0.508936, 0.619608, 0.150148, 0.676631, 0.506589, 0.623529, 0.153894, 0.680203, 0.504172, 0.627451, 0.157851, 0.683765, 0.501686, 0.631373, 0.162016, 0.687316, 0.499129, 0.635294, 0.166383, 0.690856, 0.496502, 0.6392160000000001, 0.170948, 0.694384, 0.493803, 0.643137, 0.175707, 0.6979, 0.491033, 0.647059, 0.180653, 0.701402, 0.488189, 0.65098, 0.185783, 0.704891, 0.485273, 0.654902, 0.19109, 0.708366, 0.482284, 0.658824, 0.196571, 0.711827, 0.479221, 0.662745, 0.202219, 0.715272, 0.476084, 0.666667, 0.20803, 0.718701, 0.472873, 0.670588, 0.214, 0.722114, 0.469588, 0.67451, 0.220124, 0.725509, 0.466226, 0.678431, 0.226397, 0.728888, 0.462789, 0.682353, 0.232815, 0.732247, 0.459277, 0.686275, 0.239374, 0.735588, 0.455688, 0.690196, 0.24607, 0.73891, 0.452024, 0.694118, 0.252899, 0.742211, 0.448284, 0.698039, 0.259857, 0.745492, 0.444467, 0.701961, 0.266941, 0.748751, 0.440573, 0.705882, 0.274149, 0.751988, 0.436601, 0.709804, 0.281477, 0.755203, 0.432552, 0.713725, 0.288921, 0.758394, 0.428426, 0.717647, 0.296479, 0.761561, 0.424223, 0.721569, 0.304148, 0.764704, 0.419943, 0.72549, 0.311925, 0.767822, 0.415586, 0.729412, 0.319809, 0.770914, 0.411152, 0.733333, 0.327796, 0.77398, 0.40664, 0.737255, 0.335885, 0.777018, 0.402049, 0.741176, 0.344074, 0.780029, 0.397381, 0.745098, 0.35236, 0.783011, 0.392636, 0.74902, 0.360741, 0.785964, 0.387814, 0.752941, 0.369214, 0.788888, 0.382914, 0.756863, 0.377779, 0.791781, 0.377939, 0.760784, 0.386433, 0.794644, 0.372886, 0.764706, 0.395174, 0.797475, 0.367757, 0.768627, 0.404001, 0.800275, 0.362552, 0.772549, 0.412913, 0.803041, 0.357269, 0.776471, 0.421908, 0.805774, 0.35191, 0.7803920000000001, 0.430983, 0.808473, 0.346476, 0.784314, 0.440137, 0.811138, 0.340967, 0.788235, 0.449368, 0.813768, 0.335384, 0.792157, 0.458674, 0.816363, 0.329727, 0.796078, 0.468053, 0.818921, 0.323998, 0.8, 0.477504, 0.821444, 0.318195, 0.803922, 0.487026, 0.823929, 0.312321, 0.807843, 0.496615, 0.826376, 0.306377, 0.8117650000000001, 0.506271, 0.828786, 0.300362, 0.815686, 0.515992, 0.831158, 0.294279, 0.819608, 0.525776, 0.833491, 0.288127, 0.8235290000000001, 0.535621, 0.835785, 0.281908, 0.827451, 0.545524, 0.838039, 0.275626, 0.831373, 0.555484, 0.840254, 0.269281, 0.835294, 0.565498, 0.84243, 0.262877, 0.839216, 0.575563, 0.844566, 0.256415, 0.843137, 0.585678, 0.846661, 0.249897, 0.847059, 0.595839, 0.848717, 0.243329, 0.85098, 0.606045, 0.850733, 0.236712, 0.8549020000000002, 0.616293, 0.852709, 0.230052, 0.858824, 0.626579, 0.854645, 0.223353, 0.862745, 0.636902, 0.856542, 0.21662, 0.866667, 0.647257, 0.8584, 0.209861, 0.870588, 0.657642, 0.860219, 0.203082, 0.87451, 0.668054, 0.861999, 0.196293, 0.878431, 0.678489, 0.863742, 0.189503, 0.882353, 0.688944, 0.865448, 0.182725, 0.8862750000000001, 0.699415, 0.867117, 0.175971, 0.890196, 0.709898, 0.868751, 0.169257, 0.894118, 0.720391, 0.87035, 0.162603, 0.8980390000000001, 0.730889, 0.871916, 0.156029, 0.901961, 0.741388, 0.873449, 0.149561, 0.905882, 0.751884, 0.874951, 0.143228, 0.909804, 0.762373, 0.876424, 0.137064, 0.913725, 0.772852, 0.877868, 0.131109, 0.917647, 0.783315, 0.879285, 0.125405, 0.921569, 0.79376, 0.880678, 0.120005, 0.92549, 0.804182, 0.882046, 0.114965, 0.9294120000000001, 0.814576, 0.883393, 0.110347, 0.933333, 0.82494, 0.88472, 0.106217, 0.937255, 0.83527, 0.886029, 0.102646, 0.941176, 0.845561, 0.887322, 0.099702, 0.945098, 0.85581, 0.888601, 0.097452, 0.94902, 0.866013, 0.889868, 0.095953, 0.952941, 0.876168, 0.891125, 0.09525, 0.956863, 0.886271, 0.892374, 0.095374, 0.960784, 0.89632, 0.893616, 0.096335, 0.964706, 0.906311, 0.894855, 0.098125, 0.968627, 0.916242, 0.896091, 0.100717, 0.972549, 0.926106, 0.89733, 0.104071, 0.976471, 0.935904, 0.89857, 0.108131, 0.980392, 0.945636, 0.899815, 0.112838, 0.984314, 0.9553, 0.901065, 0.118128, 0.988235, 0.964894, 0.902323, 0.123941, 0.992157, 0.974417, 0.90359, 0.130215, 0.9960780000000001, 0.983868, 0.904867, 0.136897, 1.0, 0.993248, 0.906157, 0.143936] +chiLUT.NanColor = [1.0, 0.0, 0.0] +chiLUT.ScalarRangeInitialized = 1.0 + +# trace defaults for the display properties. +slice1Display.Representation = 'Surface With Edges' +slice1Display.ColorArrayName = ['CELLS', 'chi'] +slice1Display.LookupTable = chiLUT +slice1Display.SelectTCoordArray = 'None' +slice1Display.SelectNormalArray = 'None' +slice1Display.SelectTangentArray = 'None' +slice1Display.OSPRayScaleFunction = 'PiecewiseFunction' +slice1Display.SelectOrientationVectors = 'None' +slice1Display.ScaleFactor = 7.0 +slice1Display.SelectScaleArray = 'None' +slice1Display.GlyphType = 'Arrow' +slice1Display.GlyphTableIndexArray = 'None' +slice1Display.GaussianRadius = 0.35000000000000003 +slice1Display.SetScaleArray = [None, ''] +slice1Display.ScaleTransferFunction = 'PiecewiseFunction' +slice1Display.OpacityArray = [None, ''] +slice1Display.OpacityTransferFunction = 'PiecewiseFunction' +slice1Display.DataAxesGrid = 'GridAxesRepresentation' +slice1Display.PolarAxes = 'PolarAxesRepresentation' + +# setup the color legend parameters for each legend in this view + +# get color legend/bar for chiLUT in view renderView1 +chiLUTColorBar = GetScalarBar(chiLUT, renderView1) +chiLUTColorBar.Title = 'chi' +chiLUTColorBar.ComponentTitle = '' + +# set color bar visibility +chiLUTColorBar.Visibility = 1 + +# show color legend +slice1Display.SetScalarBarVisibility(renderView1, True) + +# ---------------------------------------------------------------- +# setup color maps and opacity mapes used in the visualization +# note: the Get..() functions create a new object, if needed +# ---------------------------------------------------------------- + +# get opacity transfer function/opacity map for 'chi' +chiPWF = GetOpacityTransferFunction('chi') +chiPWF.Points = [0.0, 0.005917159840464592, 0.5, 0.0, 1.0, 1.0, 0.5, 0.0] +chiPWF.ScalarRangeInitialized = 1 + +# ---------------------------------------------------------------- +# setup extractors +# ---------------------------------------------------------------- + +# create extractor +pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1') +# trace defaults for the extractor. +# init the 'PNG' selected for 'Writer' +pNG1.Writer.FileName = 'SliceChi_%.6ts%cm.png' +pNG1.Writer.ImageResolution = [1920, 1080] +pNG1.Writer.ResetDisplay = 1 +pNG1.Writer.Format = 'PNG' + +# ---------------------------------------------------------------- +# restore active source +SetActiveSource(pNG1) +# ---------------------------------------------------------------- + +# ------------------------------------------------------------------------------ +# Catalyst options +from paraview import catalyst +options = catalyst.Options() +options.GlobalTrigger = 'TimeStep' +options.CatalystLiveTrigger = 'TimeStep' + +# ------------------------------------------------------------------------------ +if __name__ == '__main__': + from paraview.simple import SaveExtractsUsingCatalystOptions + # Code for non in-situ environments; if executing in post-processing + # i.e. non-Catalyst mode, let's generate extracts using Catalyst options + SaveExtractsUsingCatalystOptions(options) diff --git a/Examples/BinaryBH/catalyst_scripts/slice_center_z_to_vtm.py b/Examples/BinaryBH/catalyst_scripts/slice_center_z_to_vtm.py new file mode 100644 index 000000000..42f45d323 --- /dev/null +++ b/Examples/BinaryBH/catalyst_scripts/slice_center_z_to_vtm.py @@ -0,0 +1,67 @@ +# script-version: 2.0 +# Catalyst state generated using paraview version 5.11.0 + +# This script takes a slice through the center of the domain of constant z and +# saves it to a VTM file (+ corresponding folder) that can be visualized in +# ParaView (all variables passed to Catalyst are saved) + +# Import this to get environment variables +import os + +#### import the simple module from the paraview +from paraview.simple import * +#### disable automatic camera reset on 'Show' +paraview.simple._DisableFirstRenderCameraReset() + + +# Get center from environment +grchombo_center_str = os.getenv('GRCHOMBO_PARAM_CENTER') +grchombo_center = [float(dim) for dim in grchombo_center_str.split(' ')] + +# ---------------------------------------------------------------- +# setup the data processing pipelines +# ---------------------------------------------------------------- + +# create source +input = AMRGaussianPulseSource(registrationName='input') +input.MeshStatus = ['Mesh'] + +# create a new 'Slice With Plane' +sliceWithPlane1 = SliceWithPlane(registrationName='SliceWithPlane1', Input=input) +sliceWithPlane1.PlaneType = 'Plane' +sliceWithPlane1.Level = 3 + +# init the 'Plane' selected for 'PlaneType' +sliceWithPlane1.PlaneType.Origin = grchombo_center +sliceWithPlane1.PlaneType.Normal = [0.0, 0.0, 1.0] + +# ---------------------------------------------------------------- +# setup extractors +# ---------------------------------------------------------------- + +# create extractor +vTM1 = CreateExtractor('VTM', sliceWithPlane1, registrationName='VTM1') +# trace defaults for the extractor. +vTM1.Trigger = 'TimeStep' + +# init the 'VTM' selected for 'Writer' +vTM1.Writer.FileName = 'CenterZSlice_{timestep:06d}.vtm' + +# ---------------------------------------------------------------- +# restore active source +SetActiveSource(vTM1) +# ---------------------------------------------------------------- + +# ------------------------------------------------------------------------------ +# Catalyst options +from paraview import catalyst +options = catalyst.Options() +options.GlobalTrigger = 'TimeStep' +options.CatalystLiveTrigger = 'TimeStep' + +# ------------------------------------------------------------------------------ +if __name__ == '__main__': + from paraview.simple import SaveExtractsUsingCatalystOptions + # Code for non in-situ environments; if executing in post-processing + # i.e. non-Catalyst mode, let's generate extracts using Catalyst options + SaveExtractsUsingCatalystOptions(options) diff --git a/Examples/BinaryBH/params_catalyst.txt b/Examples/BinaryBH/params_catalyst.txt new file mode 100644 index 000000000..a5a3a40a4 --- /dev/null +++ b/Examples/BinaryBH/params_catalyst.txt @@ -0,0 +1,198 @@ +################################################# +# Filesystem parameters + +verbosity = 1 + +# location / naming of output files +# output_path = "" # Main path for all files. Must exist! +chk_prefix = BinaryBH_ +plot_prefix = BinaryBHp_ +# restart_file = BinaryBHChk_000000.3d.hdf5 + +# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval +checkpoint_interval = 100 +# set to 0 to turn off plot files (except at t=0 and t=stop_time) +# set to -1 to never ever print plotfiles +plot_interval = 1 +num_plot_vars = 1 +plot_vars = chi + +# subpaths - specific directories for hdf5, pout, extraction data +# (these are created at runtime) +hdf5_subpath = "hdf5" +pout_subpath = "pout" +data_subpath = "data" + +# change the name of output files +# pout_prefix = "pout" +print_progress_only_to_rank_0 = 1 + +# ignore_checkpoint_name_mismatch = 0 +write_plot_ghosts = 1 + +################################################# +# Initial Data parameters + +# provide 'offset' or 'center' + +massA = 0.48847892320123 +massB = 0.48847892320123 + +offsetA = 0.0 6.10679 0.0 +offsetB = 0.0 -6.10679 0.0 +# centerA = 256 232 256 +# centerB = 256 250 256 + +momentumA = -0.0841746 -0.000510846 0.0 +momentumB = 0.0841746 0.000510846 0.0 + +################################################# +# Grid parameters + +# 'N' is the number of subdivisions in each direction of a cubic box +# 'L' is the length of the longest side of the box, dx_coarsest = L/N +# NB - If you use reflective BC and want to specify the subdivisions and side +# of the box were there are no symmetries, specify 'N_full' and 'L_full' instead +# NB - if you have a non-cubic grid, you can specify 'N1' or 'N1_full', +# 'N2' or 'N2_full' and 'N3' or 'N3_full' ( then dx_coarsest = L/N(max) ) +# NB - the N values need to be multiples of the block_factor +N_full = 64 +L_full = 64 + +# Maximum number of times you can regrid above coarsest level +max_level = 3 # There are (max_level+1) grids, so min is zero + +# Frequency of regridding at each level and thresholds on the tagging +# Need one for each level except the top one, ie max_level items +# Generally you do not need to regrid frequently on every level +# Level Regridding: 0 1 2 3 4 5 6 7 8 +regrid_interval = 0 0 0 64 64 64 64 64 64 +regrid_threshold = 0.02 + +# Max and min box sizes +max_box_size = 16 +min_box_size = 16 + +# tag_buffer_size = 3 +# grid_buffer_size = 8 +# fill_ratio = 0.7 +# num_ghosts = 3 +# center = 256.0 256.0 256.0 # defaults to center of the grid + +################################################# +# Boundary Conditions parameters + +# Periodic directions - 0 = false, 1 = true +isPeriodic = 0 0 0 +# if not periodic, then specify the boundary type +# 0 = static, 1 = sommerfeld, 2 = reflective +# (see BoundaryConditions.hpp for details) +hi_boundary = 1 1 1 +lo_boundary = 1 1 1 + +# if reflective boundaries selected, must set +# parity of all vars (in order given by UserVariables.hpp) +# 0 = even +# 1,2,3 = odd x, y, z +# 4,5,6 = odd xy, yz, xz +# 7 = odd xyz +vars_parity = 0 0 4 6 0 5 0 #chi and hij + 0 0 4 6 0 5 0 #K and Aij + 0 1 2 3 #Theta and Gamma + 0 1 2 3 1 2 3 #lapse shift and B +vars_parity_diagnostic = 0 1 2 3 #Ham and Mom + 0 7 #Weyl + +# if sommerfeld boundaries selected, must select +# asymptotic values +num_nonzero_asymptotic_vars = 5 +nonzero_asymptotic_vars = chi h11 h22 h33 lapse +nonzero_asymptotic_values = 1.0 1.0 1.0 1.0 1.0 + +# if you are using extrapolating BC: +# extrapolation_order = 1 +# num_extrapolating_vars = -1 +# extrapolating_vars = + +################################################# +# Evolution parameters + +# dt will be dx*dt_multiplier on each grid level +dt_multiplier = 0.25 +stop_time = 5.0 +# max_steps = 100 + +# Spatial derivative order (only affects CCZ4 RHS) +max_spatial_derivative_order = 4 # can be 4 or 6 + +nan_check = 1 + +# Lapse evolution +lapse_advec_coeff = 1.0 +lapse_coeff = 2.0 +lapse_power = 1.0 + +# Shift evolution +shift_advec_coeff = 0.0 # Usually no advection for beta +shift_Gamma_coeff = 0.75 +eta = 1.0 # eta of gamma driver, should be of order ~1/M_ADM of spacetime + +# CCZ4 parameters +formulation = 0 # 1 for BSSN, 0 for CCZ4 +kappa1 = 0.1 +kappa2 = 0. +kappa3 = 1. +covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse + +# coefficient for KO numerical dissipation +sigma = 1.0 + +track_punctures = 1 +puncture_tracking_level = 3 + +# calculate_constraint_norms = 0 + +# min_chi = 1.e-4 +# min_lapse = 1.e-4 + +################################################# +# Extraction parameters + +# extraction_center = 256 256 256 # defaults to center +activate_extraction = 0 +num_extraction_radii = 0 +extraction_radii = 1.0 1.0 +extraction_levels = 2 1 +num_points_phi = 24 +num_points_theta = 37 +num_modes = 8 +modes = 2 0 # l m for spherical harmonics + 2 1 + 2 2 + 4 0 + 4 1 + 4 2 + 4 3 + 4 4 + +# integral_file_prefix = "Weyl4_mode_" + +# write_extraction = 0 +# extraction_subpath = "data/extraction" # directory for 'write_extraction = 1' +# extraction_file_prefix = "Weyl4_extraction_" + +################################################# +# ParaView Catalyst parameters + +catalyst_activate = true +catalyst_verbosity = 2 +# catalyst_pout_prefix = catalyst_pout +# path to directory containing catalyst scripts +catalyst_scripts_path = catalyst_scripts +catalyst_num_scripts = 1 +# list of Catalyst scripts +catalyst_scripts = slice_center_z_chi.py +catalyst_coprocess_level = 1 +catalyst_num_vars = 1 +catalyst_vars = chi +catalyst_abort_on_error = true diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp index d3ba7cb26..f281a50d3 100644 --- a/Source/GRChomboCore/ChomboParameters.hpp +++ b/Source/GRChomboCore/ChomboParameters.hpp @@ -21,6 +21,11 @@ #include #include +#ifdef USE_CATALYST +#include +#include +#endif + // Chombo namespace #include "UsingNamespace.H" @@ -126,6 +131,82 @@ class ChomboParameters pp.load("print_progress_only_to_rank_0", print_progress_only_to_rank_0, false); + +#ifdef USE_CATALYST + pp.load("catalyst_activate", catalyst_activate, false); + if (catalyst_activate) + { + pp.load("catalyst_verbosity", catalyst_params.verbosity, verbosity); + pp.load("catalyst_pout_prefix", catalyst_pout_prefix, + std::string("catalyst_pout")); + pp.load("catalyst_scripts_path", catalyst_scripts_path, + std::string()); + pp.load("catalyst_num_scripts", catalyst_num_scripts, 1); + pp.load("catalyst_scripts", catalyst_params.python_scripts, + catalyst_num_scripts); + + // this is what the Catalyst working directory will be set to + // so extracts end up in the output_path + catalyst_params.output_path = output_path; + + // prepend the PWD path to each script + if (!catalyst_scripts_path.empty() && + catalyst_scripts_path.back() != '/') + catalyst_scripts_path += "/"; + // make each catalyst_script a full path (if not already) so that + // Catalyst doesn't look in its working directory (set to + // output_path) + std::string prepend_directory = ""; + if (catalyst_scripts_path.front() != '/') // not a full path + { + char cwd[4096]; + getcwd(cwd, 4096); + prepend_directory = cwd; + prepend_directory += "/"; + } + for (auto &script : catalyst_params.python_scripts) + { + if (script.front() != '/') // not a full path + { + script = prepend_directory + catalyst_scripts_path + script; + } + } + + pp.load("catalyst_coprocess_level", catalyst_coprocess_level, 0); + UserVariables::load_vars_to_vector( + pp, "catalyst_vars", "catalyst_num_vars", catalyst_params.vars, + catalyst_num_vars); + pp.load("catalyst_abort_on_error", catalyst_params.abort_on_error, + false); + pp.load("catalyst_remove_ghosts", catalyst_params.remove_ghosts, + false); + pp.load("catalyst_write_vtk_files", catalyst_params.write_vtk_files, + false); + pp.load("catalyst_vtk_file_prefix", catalyst_params.vtk_file_prefix, + std::string("Catalyst_VTK_")); + + // export parameters to environment so Catalyst scripts can use + // them + std::string center_str = ""; + FOR1(i) + { + if (i != 0) + center_str += " "; + center_str += std::to_string(center[i]); + } + setenv("GRCHOMBO_PARAM_CENTER", center_str.c_str(), 1); + + // this isn't really a GRChombo parameter but easiest in this form + FOR1(i) + { + std::string length_env_var = + std::string("GRCHOMBO_PARAM_L") + std::to_string(i + 1); + double length = static_cast(ivN[i] + 1) * coarsest_dx; + std::string length_str = std::to_string(length); + setenv(length_env_var.c_str(), length_str.c_str(), 1); + } + } +#endif } void read_filesystem_params(GRParmParse &pp) @@ -464,6 +545,29 @@ class ChomboParameters "parity type undefined"); } } + +#ifdef USE_CATALYST + if (catalyst_activate) + { + for (int iscript = 0; + iscript < catalyst_params.python_scripts.size(); ++iscript) + { + const std::string &script = + catalyst_params.python_scripts[iscript]; + bool script_exists = (access(script.c_str(), R_OK) == 0); + const std::string script_parameter_name = + "catalyst_scripts[" + std::to_string(iscript) + "]"; + check_parameter(script_parameter_name, script, script_exists, + "does not exist"); + } + + check_parameter("catalyst_coprocess_level", + catalyst_coprocess_level, + catalyst_coprocess_level >= 0 && + catalyst_coprocess_level <= max_level, + "not a valid level"); + } +#endif } // General parameters @@ -524,6 +628,23 @@ class ChomboParameters bool just_check_params = false; bool print_progress_only_to_rank_0; +#ifdef USE_CATALYST + bool catalyst_activate; + // prefix of filename for Catalyst log output (appended by .) + // file will be in pout_path + std::string catalyst_pout_prefix; + // ignores output_path + std::string catalyst_scripts_path; + int catalyst_num_scripts; + int catalyst_coprocess_level; + // variables to pass to Catalyst + // only restricts vars if num_catalyst_vars > 0, otherwise all requested + // are passed + int catalyst_num_vars; + + CatalystAdaptor::params_t catalyst_params; +#endif + protected: // the low and high corners of the domain taking into account reflective BCs // only used in parameter checks hence protected diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp index 090c5ee0c..282439b20 100644 --- a/Source/GRChomboCore/GRAMR.cpp +++ b/Source/GRChomboCore/GRAMR.cpp @@ -8,6 +8,17 @@ GRAMR::GRAMR() : m_interpolator(nullptr) {} +void GRAMR::conclude() +{ + AMR::conclude(); +#ifdef USE_CATALYST + if (m_catalyst_activate) + { + m_insitu->finalise(); + } +#endif +} + // Called after AMR object set up void GRAMR::set_interpolator(AMRInterpolator> *a_interpolator) { @@ -51,3 +62,17 @@ void GRAMR::fill_multilevel_ghosts(const VariableType a_var_type, level.fillAllGhosts(a_var_type, a_comps); } } + +#ifdef USE_CATALYST +void GRAMR::setup_catalyst(bool a_catalyst_activate, + const CatalystAdaptor::params_t &a_catalyst_params) +{ + m_catalyst_activate = a_catalyst_activate; + if (m_catalyst_activate) + { + pout() << "GRAMR::setup_catalyst" << std::endl; + m_insitu = new CatalystAdaptor; + m_insitu->initialise(this, a_catalyst_params); + } +} +#endif \ No newline at end of file diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp index 06f51614e..be9af3d09 100644 --- a/Source/GRChomboCore/GRAMR.hpp +++ b/Source/GRChomboCore/GRAMR.hpp @@ -18,6 +18,10 @@ #include #include +#ifdef USE_CATALYST +#include "CatalystAdaptor.hpp" +#endif + // Chombo namespace #include "UsingNamespace.H" @@ -46,6 +50,9 @@ class GRAMR : public AMR GRAMR(); + // just calls AMR::conclude unless USE_CATALYST == TRUE + void conclude(); + // defined here due to auto return type auto get_walltime() { @@ -71,6 +78,16 @@ class GRAMR : public AMR const Interval &a_comps = Interval(0, std::numeric_limits::max()), const int a_min_level = 0, const int a_max_level = std::numeric_limits::max()) const; + +#ifdef USE_CATALYST + void setup_catalyst(bool a_catalyst_activate, + const CatalystAdaptor::params_t &a_catalyst_params); +#endif + +#ifdef USE_CATALYST + bool m_catalyst_activate = false; + CatalystAdaptor *m_insitu; +#endif }; #endif /* GRAMR_HPP_ */ diff --git a/Source/GRChomboCore/GRAMRLevel.cpp b/Source/GRChomboCore/GRAMRLevel.cpp index 6239f5be9..c0089692e 100644 --- a/Source/GRChomboCore/GRAMRLevel.cpp +++ b/Source/GRChomboCore/GRAMRLevel.cpp @@ -178,6 +178,10 @@ void GRAMRLevel::postTimeStep() specificPostTimeStep(); +#ifdef USE_CATALYST + catalystCoProcess(); +#endif + // enforce solution BCs - this is required after the averaging // and postentially after specificPostTimeStep actions fillBdyGhosts(m_state_new); @@ -186,6 +190,84 @@ void GRAMRLevel::postTimeStep() pout() << "GRAMRLevel::postTimeStep " << m_level << " finished" << endl; } +#ifdef USE_CATALYST +void GRAMRLevel::preCatalystCoProcess() +{ + // If we're not passing ghosts to Catalyst, there's no point in filling them + if (m_p.catalyst_params.remove_ghosts) + return; + + if (std::abs(m_time - m_restart_time) < m_gr_amr.timeEps()) + { + // We don't have any information on the first step so fill all ghosts + m_gr_amr.fill_multilevel_ghosts(VariableType::evolution); + m_gr_amr.fill_multilevel_ghosts(VariableType::diagnostic); + } + else + { + // Now use the last requested variables + const auto requested_evolution_vars = + m_gr_amr.m_insitu->get_requested_evolution_vars(); + const auto requested_diagnostic_vars = + m_gr_amr.m_insitu->get_requested_diagnostic_vars(); + int first_requested_evolution_var = 0; + int last_requested_evolution_var = NUM_VARS - 1; + int first_requested_diagnostic_var = 0; + int last_requested_diagnostic_var = NUM_DIAGNOSTIC_VARS - 1; + + while ((!requested_evolution_vars[first_requested_evolution_var]) && + first_requested_evolution_var < NUM_VARS) + ++first_requested_evolution_var; + while ((!requested_evolution_vars[last_requested_evolution_var]) && + last_requested_evolution_var > -1) + --last_requested_evolution_var; + while ((!requested_diagnostic_vars[first_requested_diagnostic_var]) && + first_requested_diagnostic_var < NUM_DIAGNOSTIC_VARS) + ++first_requested_diagnostic_var; + while ((!requested_diagnostic_vars[last_requested_diagnostic_var]) && + last_requested_diagnostic_var > -1) + --last_requested_diagnostic_var; + + // For simplicity fill all ghosts in the interval from first to last + // for each variable type + Interval evolution_vars(first_requested_evolution_var, + last_requested_evolution_var); + Interval diagnostic_vars(first_requested_diagnostic_var, + last_requested_diagnostic_var); + + // These won't do anything if the interval size is <= 0 + fillAllGhosts(VariableType::evolution, evolution_vars); + fillAllGhosts(VariableType::diagnostic, diagnostic_vars); + } +} + +void GRAMRLevel::catalystCoProcess() +{ + if (m_p.catalyst_activate) + { + if (at_level_timestep_multiple(m_p.catalyst_coprocess_level)) + { + preCatalystCoProcess(); + } + if (m_level == m_p.catalyst_coprocess_level) + { + if (m_verbosity) + { + pout() << "GRAMRLevel::catalystCoProcess" << std::endl; + } + + unsigned int level_timestep = std::round(m_time / m_dt); + m_gr_amr.m_insitu->coprocess(m_time, level_timestep); + + if (m_verbosity) + { + pout() << "GRAMRLevel::catalystCoProcess finished" << std::endl; + } + } + } +} +#endif + // for examples that don't implement a computeTaggingCriterion with diagnostic // variables void GRAMRLevel::computeTaggingCriterion( @@ -1037,6 +1119,9 @@ void GRAMRLevel::fillAllEvolutionGhosts(const Interval &a_comps) if (m_verbosity) pout() << "GRAMRLevel::fillAllEvolutionGhosts" << endl; + if (a_comps.size() <= 0) + return; + // If there is a coarser level then interpolate undefined ghost cells if (m_coarser_level_ptr != nullptr) { @@ -1053,6 +1138,9 @@ void GRAMRLevel::fillAllDiagnosticsGhosts(const Interval &a_comps) if (m_verbosity) pout() << "GRAMRLevel::fillAllDiagnosticsGhosts" << endl; + if (a_comps.size() <= 0) + return; + // If there is a coarser level then interpolate undefined ghost cells if (m_coarser_level_ptr != nullptr) { diff --git a/Source/GRChomboCore/GRAMRLevel.hpp b/Source/GRChomboCore/GRAMRLevel.hpp index e3bad9bbb..8b915a354 100644 --- a/Source/GRChomboCore/GRAMRLevel.hpp +++ b/Source/GRChomboCore/GRAMRLevel.hpp @@ -188,6 +188,15 @@ class GRAMRLevel : public AMRLevel, public InterpSource const VariableType var_type = VariableType::evolution, const Interval &a_comps = Interval(0, std::numeric_limits::max())); +#ifdef USE_CATALYST + /// Calls Catalyst CoProcess + void catalystCoProcess(); + + /// Things to do before calling Catalyst CoProcess (fills ghosts if not + /// overriden) + virtual void preCatalystCoProcess(); +#endif + protected: /// Fill all evolution ghosts cells (i.e. those in m_state_new) virtual void diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp index 7e77141f1..de4da6e6e 100644 --- a/Source/GRChomboCore/SetupFunctions.hpp +++ b/Source/GRChomboCore/SetupFunctions.hpp @@ -13,6 +13,7 @@ #include "parstream.H" //Gives us pout() // Other includes +#include #include using std::cerr; using std::endl; @@ -29,6 +30,13 @@ using std::endl; #include "DebuggingTools.hpp" #endif +#ifdef USE_CATALYST +#include "vtkLogger.h" +#include "vtkPythonInterpreter.h" +#include "vtkSMPTools.h" +#include "vtkVersion.h" +#endif + #ifdef _OPENMP #include #endif @@ -88,6 +96,32 @@ void mainSetup(int argc, char *argv[]) cerr << " usage " << argv[0] << " " << endl; exit(0); } + +#ifdef USE_CATALYST + // Use the VTK_SMP_MAX_THREADS environment variable to set the maximum + // number of SMP threads if it exists + if (!std::getenv("VTK_SMP_MAX_THREADS")) + { + // otherwise set it to the same as the number of OpenMP threads +#ifdef _OPENMP + int num_threads = omp_get_max_threads(); +#else + int num_threads = 1; +#endif + vtkSMPTools::Initialize(num_threads); + } + else + { + // VTK will automatically check the VTK_SMP_MAX_THREADS environment + // variable + vtkSMPTools::Initialize(); + } + if (rank == 0) + { + std::cout << " catalyst threads = " + << vtkSMPTools::GetEstimatedNumberOfThreads() << std::endl; + } +#endif } void mainFinalize() @@ -181,6 +215,39 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory) MayDay::Error("GRChombo restart only defined with hdf5"); #endif } +#ifdef USE_CATALYST +#if VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 0, 20201030) + // set vtkLogger internal messages to only appear at verbosity 2 + vtkLogger::SetInternalVerbosityLevel(vtkLogger::VERBOSITY_3); +#endif +#if VTK_VERSION_NUMBER >= VTK_VERSION_CHECK(9, 1, 0) + // don't redirect Python interpreter output + vtkPythonInterpreter::SetRedirectOutput(false); +#endif +#ifdef CH_MPI + vtkLogger::SetStderrVerbosity(vtkLogger::VERBOSITY_ERROR); + std::string catalyst_log_file = chombo_params.pout_path + + chombo_params.catalyst_pout_prefix + + std::string(".") + std::to_string(procID()); + vtkLogger::FileMode vtk_logger_file_mode = vtkLogger::TRUNCATE; + // if we're writing to the normal pout file we don't want the vtkLogger to + // delete the file at this point + if (chombo_params.catalyst_pout_prefix == chombo_params.pout_prefix) + { + vtk_logger_file_mode = vtkLogger::APPEND; + } + vtkLogger::LogToFile( + catalyst_log_file.c_str(), vtk_logger_file_mode, + vtkLogger::ConvertToVerbosity(chombo_params.catalyst_params.verbosity)); + // Only write VTK stderr messages if there is an error +#else + vtkLogger::SetStderrVerbosity( + static_cast(chombo_params.verbosity)); +#endif + vtkLogger::Init(); + gr_amr.setup_catalyst(chombo_params.catalyst_activate, + chombo_params.catalyst_params); +#endif } #endif /* SETUP_FUNCTIONS_HPP_ */ diff --git a/Source/Insitu/CatalystAdaptor.cpp b/Source/Insitu/CatalystAdaptor.cpp new file mode 100644 index 000000000..9b76a10fd --- /dev/null +++ b/Source/Insitu/CatalystAdaptor.cpp @@ -0,0 +1,531 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#include "CatalystAdaptor.hpp" +#include "GRAMRLevel.hpp" +#include "ParaviewVersion.hpp" +#include "vtkMultiProcessController.h" +#include "vtkParallelAMRUtilities.h" + +#ifdef USE_CATALYST + +CatalystAdaptor::CatalystAdaptor() {} + +CatalystAdaptor::CatalystAdaptor(GRAMR *a_gr_amr_ptr, const params_t &a_params) +{ + initialise(a_gr_amr_ptr, a_params); +} + +CatalystAdaptor::~CatalystAdaptor() +{ + if (m_initialised) + { + finalise(); + } +} + +void CatalystAdaptor::initialise(GRAMR *a_gr_amr_ptr, const params_t &a_params) +{ + // don't initalise twice + if (m_initialised) + return; + + if (!a_gr_amr_ptr) + { + std::cerr << "CatalystAdaptor::initialise: failed to initalise due to" + " invalid GRAMR pointer" + << std::endl; + m_initialised = false; + return; + } + m_gr_amr_ptr = a_gr_amr_ptr; + m_p = a_params; + m_requested_evolution_vars.fill(false); + m_requested_diagnostic_vars.fill(false); + + // Initialise VTK CP Processor + if (!m_proc_ptr) + { + m_proc_ptr = vtkCPProcessor::New(); + int m_proc_ptr_initialization_success; + if (m_p.output_path.empty()) + { + m_proc_ptr_initialization_success = m_proc_ptr->Initialize(); + } + else + { + m_proc_ptr_initialization_success = + m_proc_ptr->Initialize(m_p.output_path.c_str()); + } + std::string error_msg = "Failed to initialize vtkCPProcessor in " + "CatalystAdaptor::initialise"; + catalyst_error_or_warning(m_proc_ptr_initialization_success, error_msg); + } + else + { + m_proc_ptr->RemoveAllPipelines(); + } + + // Create Python script pipeline and add it to the VTK CP Processor + for (const std::string &script : m_p.python_scripts) + { +#if PARAVIEW_VERSION_HERE >= PARAVIEW_VERSION_TEST(5, 9, 0) + auto pipeline = + vtkCPPythonPipeline::CreateAndInitializePipeline(script.c_str()); + bool pipeline_init_success = (pipeline != nullptr); +#else + vtkNew pipeline; + int pipeline_init_success = pipeline->Initialize(script.c_str()); +#endif + std::string pipeline_init_fail_msg = + "Failed to initialize pipelone for script: "; + pipeline_init_fail_msg += script; + catalyst_error_or_warning(pipeline_init_success, + pipeline_init_fail_msg); + int add_pipeline_success = m_proc_ptr->AddPipeline(pipeline); + std::string add_pipeline_fail_msg = + "Failed to add pipeline for script: "; + add_pipeline_fail_msg += script; + catalyst_error_or_warning(add_pipeline_success, add_pipeline_fail_msg); + } + + m_initialised = true; +} + +void CatalystAdaptor::finalise() +{ + if (m_proc_ptr) + { + m_proc_ptr->Delete(); + m_proc_ptr = nullptr; + } + if (m_vtk_grid_ptr) + { + m_vtk_grid_ptr->Delete(); + m_vtk_grid_ptr = nullptr; + } + m_initialised = false; +} + +void CatalystAdaptor::build_vtk_grid(vtkOverlappingAMR *a_vtk_grid_ptr) +{ + if (m_p.verbosity) + { + pout() << "CatalystAdaptor::build_vtk_grid" << std::endl; + } + CH_assert(a_vtk_grid_ptr); + + const auto gramrlevels = m_gr_amr_ptr->get_gramrlevels(); + + // need the number of levels and the number of boxes on each level to + // initialise the vtkOverlappingAMR object + int num_levels = gramrlevels.size(); + int *boxes_per_level = new int[num_levels]; + for (int ilevel = 0; ilevel < num_levels; ++ilevel) + { + const GRAMRLevel *level = gramrlevels[ilevel]; + const DisjointBoxLayout &level_box_layout = + level->getLevelData().disjointBoxLayout(); + boxes_per_level[ilevel] = level_box_layout.boxArray().size(); + } + a_vtk_grid_ptr->Initialize(num_levels, boxes_per_level); + + // The origin is always at (0, 0, 0) in Chombo + double origin_global[3] = {0., 0., 0.}; + const IntVect &coarsest_ghost_vect = + gramrlevels[0]->getLevelData().ghostVect(); + const double coarsest_dx = gramrlevels[0]->get_dx(); + RealVect ghosted_origin_global_vect = coarsest_ghost_vect; + ghosted_origin_global_vect *= (!m_p.remove_ghosts) ? -coarsest_dx : 0; + a_vtk_grid_ptr->SetOrigin(ghosted_origin_global_vect.dataPtr()); + + // now add all the boxes + for (int ilevel = 0; ilevel < num_levels; ++ilevel) + { + // pout() << "========================================\n"; + // pout() << "Level: " << ilevel << std::endl; + const GRAMRLevel *level = gramrlevels[ilevel]; + const double dx = level->get_dx(); + double dx_arr[3] = {dx, dx, dx}; + a_vtk_grid_ptr->SetSpacing(ilevel, dx_arr); + a_vtk_grid_ptr->SetRefinementRatio(ilevel, level->refRatio()); + const GRLevelData &level_data = level->getLevelData(); + const DisjointBoxLayout &level_box_layout = + level_data.disjointBoxLayout(); + LayoutIterator lit = level_box_layout.layoutIterator(); + int ibox; + for (ibox = 0, lit.begin(); lit.ok(); ++lit, ++ibox) + { + // first get the box without ghosts + Box box = level_box_layout[lit]; + // VTK counts the big end differently to Chombo so modify the Chombo + // box so VTK gets what it needs + for (int idir = 0; idir < SpaceDim; ++idir) + { + box.growHi(idir, 1); + } + const IntVect &small_end = box.smallEnd(); + const IntVect &big_end = box.bigEnd(); + + // now grow the box to make the ghosted box (no change if + // m_remove_ghosts == true) + Box ghosted_box = box; + + // ghost_vect takes into account m_remove_ghosts + const IntVect &ghost_vect = + (m_p.remove_ghosts) ? IntVect::Zero : level_data.ghostVect(); + ghosted_box.grow(ghost_vect); + const IntVect &small_ghosted_end = ghosted_box.smallEnd(); + const IntVect &big_ghosted_end = ghosted_box.bigEnd(); + + // vtkAMRBox vtk_amr_box(small_ghosted_end.dataPtr(), + // big_ghosted_end.dataPtr()); + double origin[3] = {dx_arr[0] * small_ghosted_end[0], + dx_arr[1] * small_ghosted_end[1], + dx_arr[2] * small_ghosted_end[2]}; + // RealVect level_origin = -dx * level_data.ghostVect(); + + vtkAMRBox vtk_amr_box(origin, ghosted_box.size().dataPtr(), dx_arr, + ghosted_origin_global_vect.dataPtr()); + // vtk_amr_box.Print(pout()); + // pout() << "\n"; + a_vtk_grid_ptr->SetAMRBox(ilevel, ibox, vtk_amr_box); + + bool local_box = (procID() == level_box_layout.procID(lit())); + // only need to do the following for local boxes + if (local_box) + { + vtkNew vtk_uniform_grid_ptr; + + // vtk_uniform_grid_ptr->SetOrigin(origin_global); + // vtk_uniform_grid_ptr->SetSpacing(dx_arr); + // vtk_uniform_grid_ptr->SetExtent( + // small_ghosted_end[0], big_ghosted_end[0], + // small_ghosted_end[1], big_ghosted_end[1], + // small_ghosted_end[2], big_ghosted_end[2]); + // add the ghost cell information + // int no_ghost[6] = {small_end[0], big_end[0], small_end[1], + // big_end[1], small_end[2], big_end[2]}; + // bool cell_data = true; + // vtk_uniform_grid_ptr->GenerateGhostArray(no_ghost, + // cell_data); + + vtk_uniform_grid_ptr->Initialize(&vtk_amr_box, origin, dx_arr); + + a_vtk_grid_ptr->SetDataSet(ilevel, ibox, vtk_uniform_grid_ptr); + } + } + } + + a_vtk_grid_ptr->Audit(); +#if DEBUG + const double *vtk_grid_bounds = a_vtk_grid_ptr->GetAMRInfo()->GetBounds(); + pout() << "VTK Grid Bounds:" + << "(" << vtk_grid_bounds[0] << "," << vtk_grid_bounds[2] << "," + << vtk_grid_bounds[4] << "), (" << vtk_grid_bounds[1] << "," + << vtk_grid_bounds[3] << "," << vtk_grid_bounds[5] << ")" + << std::endl; +#endif + + // not sure if this is necessary but it was on the OverlappingAMR example + a_vtk_grid_ptr->GenerateParentChildInformation(); +} + +void CatalystAdaptor::add_vars(vtkCPInputDataDescription *a_input_data_desc, + vtkOverlappingAMR *a_vtk_grid_ptr) +{ + if (m_p.verbosity) + { + pout() << "CatalystAdaptor::add_vars" << std::endl + << "CatalystAdaptor Requested variables: "; + } + + for (int ivar = 0; ivar < NUM_VARS; ++ivar) + { + m_requested_evolution_vars[ivar] = a_input_data_desc->IsFieldNeeded( + UserVariables::variable_names[ivar].c_str(), vtkDataObject::CELL); + if (m_p.vars.size() > 0) + { + bool pass_var = + !(std::find(m_p.vars.begin(), m_p.vars.end(), + std::make_pair(ivar, VariableType::evolution)) == + m_p.vars.end()); + m_requested_evolution_vars[ivar] &= pass_var; + } + if (m_p.verbosity && m_requested_evolution_vars[ivar]) + pout() << UserVariables::variable_names[ivar] << " "; + } + for (int ivar = 0; ivar < NUM_DIAGNOSTIC_VARS; ++ivar) + { + m_requested_diagnostic_vars[ivar] = a_input_data_desc->IsFieldNeeded( + DiagnosticVariables::variable_names[ivar].c_str(), + vtkDataObject::CELL); + if (m_p.vars.size() > 0) + { + bool pass_var = + !(std::find(m_p.vars.begin(), m_p.vars.end(), + std::make_pair(ivar, VariableType::diagnostic)) == + m_p.vars.end()); + m_requested_diagnostic_vars[ivar] &= pass_var; + } + if (m_p.verbosity && m_requested_diagnostic_vars[ivar]) + pout() << DiagnosticVariables::variable_names[ivar] << " "; + } + if (m_p.verbosity) + pout() << std::endl; + + vtkAMRInformation *amr_info = a_vtk_grid_ptr->GetAMRInfo(); + auto gramrlevels = m_gr_amr_ptr->get_gramrlevels(); + + for (int ilevel = 0; ilevel < gramrlevels.size(); ++ilevel) + { + GRAMRLevel *level = gramrlevels[ilevel]; + // Unfortunately it doesn't seem possible to pass const pointers + // to Catalyst + GRLevelData &evolution_level_data = const_cast( + level->getLevelData(VariableType::evolution)); + GRLevelData &diagnostic_level_data = + (NUM_DIAGNOSTIC_VARS > 0) + ? const_cast( + level->getLevelData(VariableType::diagnostic)) + : evolution_level_data; + + const DisjointBoxLayout &level_box_layout = + evolution_level_data.disjointBoxLayout(); + LayoutIterator lit = level_box_layout.layoutIterator(); + int ibox; + for (ibox = 0, lit.begin(); lit.ok(); ++lit, ++ibox) + { + // only add data that we have locally + bool local_box = (procID() == level_box_layout.procID(lit())); + if (local_box) + { + vtkUniformGrid *vtk_uniform_grid_ptr = + a_vtk_grid_ptr->GetDataSet(ilevel, ibox); + + // hopefully this promotion works + DataIndex dind(lit()); + FArrayBox &evolution_fab = evolution_level_data[dind]; + FArrayBox &diagnostic_fab = (NUM_DIAGNOSTIC_VARS > 0) + ? diagnostic_level_data[dind] + : evolution_level_data[dind]; + const Box &unghosted_box = level_box_layout[dind]; + +#if DEBUG + vtkAMRBox vtk_box = a_vtk_grid_ptr->GetAMRBox(ilevel, ibox); + // shift to account for different indexing + IntVect vtk_origin_offset = + -ipow(2, ilevel) * evolution_level_data.ghostVect(); + vtk_box.Shift(vtk_origin_offset.dataPtr()); + const int *vtk_box_lo = vtk_box.GetLoCorner(); + const int *vtk_box_hi = vtk_box.GetHiCorner(); + + const Box &evolution_box = evolution_fab.box(); + const IntVect &evolution_box_lo = evolution_box.smallEnd(); + const IntVect &evolution_box_hi = evolution_box.bigEnd(); + + const Box &diagnostic_box = diagnostic_fab.box(); + const IntVect &diagnostic_box_lo = diagnostic_box.smallEnd(); + const IntVect &diagnostic_box_hi = diagnostic_box.bigEnd(); + // add 1 to hi corner of VTK box as this is what is meant by hi + // corner in VTK + bool all_same_boxes = + (vtk_box_lo[0] == evolution_box_lo[0]) && + (vtk_box_lo[1] == evolution_box_lo[1]) && + (vtk_box_lo[2] == evolution_box_lo[2]) && + (vtk_box_hi[0] + 1 == evolution_box_hi[0]) && + (vtk_box_hi[1] + 1 == evolution_box_hi[1]) && + (vtk_box_hi[2] + 1 == evolution_box_hi[2]) && + (vtk_box_lo[0] == diagnostic_box_lo[0]) && + (vtk_box_lo[1] == diagnostic_box_lo[1]) && + (vtk_box_lo[2] == diagnostic_box_lo[2]) && + (vtk_box_hi[0] + 1 == diagnostic_box_hi[0]) && + (vtk_box_hi[1] + 1 == diagnostic_box_hi[1]) && + (vtk_box_hi[2] + 1 == diagnostic_box_hi[2]); + if (!all_same_boxes) + { + pout() << "Boxes do not agree: \n"; + pout() << "vtk_box: "; + vtk_box.Print(pout()); + pout() << "\n"; + pout() << "evolution_box: " << evolution_box << "\n"; + pout() << "diagnostic_box: " << diagnostic_box << std::endl; + } +#endif + + for (int ivar = 0; ivar < NUM_VARS; ++ivar) + { + if (m_requested_evolution_vars[ivar]) + { + vtkDoubleArray *vtk_double_arr; + if (!m_p.remove_ghosts) + { + vtk_double_arr = fab_to_vtk_array( + evolution_fab, ivar, + UserVariables::variable_names[ivar]); + } + else + { + vtk_double_arr = fab_to_vtk_array_without_ghosts( + evolution_fab, unghosted_box, ivar, + UserVariables::variable_names[ivar]); + } + vtk_uniform_grid_ptr->GetCellData()->AddArray( + vtk_double_arr); + } + } + for (int ivar = 0; ivar < NUM_DIAGNOSTIC_VARS; ++ivar) + { + if (m_requested_diagnostic_vars[ivar]) + { + vtkDoubleArray *vtk_double_arr; + if (!m_p.remove_ghosts) + { + vtk_double_arr = fab_to_vtk_array( + diagnostic_fab, ivar, + DiagnosticVariables::variable_names[ivar]); + } + else + { + vtk_double_arr = fab_to_vtk_array_without_ghosts( + diagnostic_fab, unghosted_box, ivar, + DiagnosticVariables::variable_names[ivar]); + } + vtk_uniform_grid_ptr->GetCellData()->AddArray( + vtk_double_arr); + } + } + } + } + } +} + +void CatalystAdaptor::write_vtk_grid(unsigned int a_timestep) +{ + vtkNew file_writer; + + // make filename + char timestep_cstr[100]; + std::sprintf(timestep_cstr, "%06d.", a_timestep); + std::string filename = m_p.vtk_file_prefix; + filename += timestep_cstr; + filename += file_writer->GetDefaultFileExtension(); + + // set data and write + file_writer->SetInputData(m_vtk_grid_ptr); + file_writer->SetFileName(filename.c_str()); + file_writer->Write(); +} + +void CatalystAdaptor::coprocess(double a_time, unsigned int a_timestep) +{ + pout() << "CatalystAdaptor::coprocess at time " << a_time << " and step " + << a_timestep << std::endl; + + vtkNew data_description; + data_description->AddInput("input"); + data_description->SetTimeData(a_time, a_timestep); + + if (m_proc_ptr->RequestDataDescription(data_description) != 0) + { + + vtkNew vtk_ghosted_amr_grid_ptr; + build_vtk_grid(vtk_ghosted_amr_grid_ptr); + auto input_data_description = + data_description->GetInputDescriptionByName("input"); + add_vars(input_data_description, vtk_ghosted_amr_grid_ptr); + + if (m_vtk_grid_ptr != nullptr) + { + m_vtk_grid_ptr->Delete(); + } + m_vtk_grid_ptr = vtkOverlappingAMR::New(); + + // strip partially overlapping ghost cells + vtkParallelAMRUtilities::StripGhostLayers( + vtk_ghosted_amr_grid_ptr, m_vtk_grid_ptr, + vtkMultiProcessController::GetGlobalController()); + + vtkParallelAMRUtilities::BlankCells( + m_vtk_grid_ptr, vtkMultiProcessController::GetGlobalController()); + + if (m_p.write_vtk_files) + { + write_vtk_grid(a_timestep); + } + input_data_description->SetGrid(m_vtk_grid_ptr); + int coprocess_success = m_proc_ptr->CoProcess(data_description); + catalyst_error_or_warning(coprocess_success, + "Error in Catalyst CoProcess"); + } +} + +const std::array & +CatalystAdaptor::get_requested_evolution_vars() +{ + return m_requested_evolution_vars; +} + +const std::array & +CatalystAdaptor::get_requested_diagnostic_vars() +{ + return m_requested_diagnostic_vars; +} + +vtkDoubleArray *CatalystAdaptor::fab_to_vtk_array(FArrayBox &a_fab, int a_var, + const std::string &a_name) +{ + vtkDoubleArray *out = vtkDoubleArray::New(); + vtkIdType num_cells = a_fab.size().product(); + out->SetName(a_name.c_str()); + // this prevents Catalyst from deallocating the Chombo + // data pointers + int save_data = 1; + out->SetArray(a_fab.dataPtr(a_var), num_cells, save_data); + return out; +} + +vtkDoubleArray *CatalystAdaptor::fab_to_vtk_array_without_ghosts( + FArrayBox &a_fab, const Box &a_unghosted_box, int a_var, + const std::string &a_name) +{ + vtkDoubleArray *out = vtkDoubleArray::New(); + vtkIdType num_cells = a_unghosted_box.volume(); + // the following allocates memory in the VTK Array + out->SetNumberOfTuples(num_cells); + out->SetName(a_name.c_str()); + + const IntVect &small_end = a_unghosted_box.smallEnd(); + const IntVect &big_end = a_unghosted_box.bigEnd(); + + vtkIdType vtk_idx = 0; + for (int iz = small_end[2]; iz <= big_end[2]; ++iz) + { + for (int iy = small_end[1]; iy <= big_end[1]; ++iy) + { + for (int ix = small_end[0]; ix <= big_end[0]; ++ix) + { + out->SetValue(vtk_idx, a_fab(IntVect(ix, iy, iz), a_var)); + ++vtk_idx; + } + } + } + + return out; +} + +void CatalystAdaptor::catalyst_error_or_warning(bool a_success, + std::string a_msg) +{ + if (a_success) + return; + + if (m_p.abort_on_error) + MayDay::Error(a_msg.c_str()); + else + MayDay::Warning(a_msg.c_str()); +} + +#endif /* USE_CATALYST */ diff --git a/Source/Insitu/CatalystAdaptor.hpp b/Source/Insitu/CatalystAdaptor.hpp new file mode 100644 index 000000000..06b7ab67c --- /dev/null +++ b/Source/Insitu/CatalystAdaptor.hpp @@ -0,0 +1,124 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef CATALYSTADAPTOR_HPP_ +#define CATALYSTADAPTOR_HPP_ + +#ifdef USE_CATALYST + +// Standard library includes +#include + +// Chombo includes +#include "Box.H" +#include "FArrayBox.H" +#include "MayDay.H" + +// GRChombo includes +#include "UserVariables.hpp" + +// ParaView/VTK/Catalyst includes +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Chombo namespace +#include "UsingNamespace.H" + +// Forward declaration of GRAMR class +class GRAMR; + +// Class that interfaces with ParaView Catalyst for insitu visualisation +class CatalystAdaptor +{ + public: + struct params_t + { + int verbosity; + bool abort_on_error = false; + bool remove_ghosts = false; + + // Pipeline parameters + std::vector python_scripts; + std::string output_path; + + // VTK file writing parameters + bool write_vtk_files = false; + std::string vtk_file_prefix = "Catalyst_VTK_grid_"; + + // variables to pass to Catalyst set by GRChombo parameter + std::vector> vars; + }; + + // empty constructor (doesn't call initialise) + CatalystAdaptor(); + + // full constructor (calls initialise) + CatalystAdaptor(GRAMR *a_gr_amr_ptr, const params_t &a_params); + + // destructor + ~CatalystAdaptor(); + + // Initialisation/Finalisation + void initialise(GRAMR *m_gr_amr_ptr, const params_t &a_params); + void finalise(); + + // do Catalyst processing + void coprocess(double a_time, unsigned int a_timestep); + + // returns the variables that were last requested/sent to Catalyst + const std::array &get_requested_evolution_vars(); + const std::array & + get_requested_diagnostic_vars(); + + private: + // update the AMR grid (no grid data) + void build_vtk_grid(vtkOverlappingAMR *a_vtk_grid_ptr); + + // send variables to catalyst + void add_vars(vtkCPInputDataDescription *a_input_data_desc, + vtkOverlappingAMR *a_vtk_grid_ptr); + + // write VTK grid to a file + void write_vtk_grid(unsigned int a_timestep); + + // directly passes the FAB pointer to the VTK array + vtkDoubleArray *fab_to_vtk_array(FArrayBox &a_fab, int a_var, + const std::string &a_name); + + // copies the data in the FAB to the VTK array without the ghosts + vtkDoubleArray *fab_to_vtk_array_without_ghosts(FArrayBox &a_fab, + const Box &a_unghosted_box, + int a_var, + const std::string &a_name); + + // if a_success = false, either aborts or prints a warning depending on + // m_abort_on_catalyst_error + void catalyst_error_or_warning(bool a_success, std::string a_msg); + + GRAMR *m_gr_amr_ptr = nullptr; + bool m_initialised = false; + + params_t m_p; + + // variables actually passed to Catalyst on last CoProcess + std::array m_requested_evolution_vars; + std::array m_requested_diagnostic_vars; + + vtkCPProcessor *m_proc_ptr = nullptr; + vtkOverlappingAMR *m_vtk_grid_ptr = nullptr; +}; + +#endif /* USE_CATALYST */ +#endif /* CATALYSTADAPTOR_HPP_ */ diff --git a/Source/Insitu/Make.insitu b/Source/Insitu/Make.insitu new file mode 100644 index 000000000..269ec18ef --- /dev/null +++ b/Source/Insitu/Make.insitu @@ -0,0 +1,47 @@ +# In order to build with ParaView Catalyst insitu visualisation, include this +# makefile before mk/Make.test is included in your example and after src_dirs +# are set + +# Decide whether we want to build with ParaView Catalyst by checking whether +# PARAVIEW_DIR environment variable is set +ifdef PARAVIEW_DIR +ifneq (,$(wildcard $(PARAVIEW_DIR)/bin/paraview-config)) + $(info ================== Using ParaView Catalyst ==================) + USE_CATALYST := TRUE +else + $(error No paraview-config in \$\(PARAVIEW_DIR\)/bin/. Has ParaView been built with Catalyst?) +endif +endif + +ifeq ($(USE_CATALYST),TRUE) + src_dirs += $(GRCHOMBO_SOURCE)/Insitu + # This won't work if these make variables are overwritten in Make.defs.local + cxxcppflags += -DUSE_CATALYST + # These components are correct in v5.9 and v5.10 but may differ in other + # versions. The paraview-config command may also change + PARAVIEW_COMPONENT_FLAGS := -c PythonCatalyst ${XTRA_PARAVIEW_COMPONENT_FLAGS} + VTK_COMPONENT_FLAGS := -v CommonDataModel -v PythonUsed -v ParallelCore -v FiltersAMR -v IOParallelXML ${XTRA_VTK_COMPONENT_FLAGS} + PARAVIEW_CONFIG_EXEC := $(PARAVIEW_DIR)/bin/paraview-config + CATALYST_CPP_FLAGS := $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --cppflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS)) + ifdef .SHELLSTATUS + ifneq ($(.SHELLSTATUS),0) + $(error Error in paraview-config script. Is there a new enough version of CMake available?) + endif + endif + cxxcppflags += $(CATALYST_CPP_FLAGS) + CATALYST_LD_FLAGS := $(shell $(GRCHOMBO_SOURCE)/Insitu/cache.sh $(PARAVIEW_CONFIG_EXEC) --ldflags $(PARAVIEW_COMPONENT_FLAGS) $(VTK_COMPONENT_FLAGS)) + ifdef .SHELLSTATUS + ifneq ($(.SHELLSTATUS),0) + $(error Error in paraview-config script. Is there a new enough version of CMake available?) + endif + endif + XTRALDFLAGS += -rdynamic $(CATALYST_LD_FLAGS) +endif + +ifdef PARAVIEW_DIR +insitu: all-test + $(ECHO)mv $(_app_configs) $(ebase)_Insitu.$(config).ex +else +insitu: + @echo "Set PARAVIEW_DIR environment variable to build insitu target."; +endif \ No newline at end of file diff --git a/Source/Insitu/ParaviewVersion.hpp b/Source/Insitu/ParaviewVersion.hpp new file mode 100644 index 000000000..904d910d5 --- /dev/null +++ b/Source/Insitu/ParaviewVersion.hpp @@ -0,0 +1,29 @@ +#ifndef PARAVIEW_VERSION_HPP_ +#define PARAVIEW_VERSION_HPP_ + +#ifdef USE_CATALYST +// ParaView version checking doesn't seem to be very stable between versions +#ifdef __has_include +#if __has_include() +#include +#define PARAVIEW_VERSION_KNOWN +#elif __has_include() +#include +#define PARAVIEW_VERSION_KNOWN +#endif /* __has_include() */ +#endif /* __has_include */ + +#ifdef PARAVIEW_VERSION_KNOWN +// there is something similar in the new vtkPVVersion.h header but it uses +// the build rather than the patch number. It's also not available in +// vtkPVConfig.h +#define PARAVIEW_VERSION_TEST(major, minor, patch) \ + 10000 * major + 100 * minor + patch +#define PARAVIEW_VERSION_HERE \ + PARAVIEW_VERSION_TEST(PARAVIEW_VERSION_MAJOR, PARAVIEW_VERSION_MINOR, \ + PARAVIEW_VERSION_PATCH) +#endif /* PARAVIEW_VERSION_KNOWN */ + +#endif /* USE_CATALYST */ + +#endif /* PARAVIEW_VERSION_HPP_ */ \ No newline at end of file diff --git a/Source/Insitu/README.md b/Source/Insitu/README.md new file mode 100644 index 000000000..a5650c39b --- /dev/null +++ b/Source/Insitu/README.md @@ -0,0 +1,228 @@ +# Using ParaView Catalyst in-situ visualization with GRChombo + +## Why use in-situ visualization? + +Whilst the processing power of supercomputers has grown rapidly over recent +years, the corresponding growth in IO capabilities and data storage has not kept +pace. As a result, it is common to only write checkpoints/plot files relatively +infrequently in order to save on disk space. This means that traditional +post-processed visualization can only be performed at relatively poor +resolutions in time. Alternatively, separate "visualization runs" are sometimes +performed at significantly lower resolutions than that of the production +simulations which takes more time and can also result in inaccurate output. One +solution to this problem is performing the visualization _in-situ_ (i.e. during +the simulation itself). + +In this guide, we describe how to use the in-situ visualization capabilities of +ParaView Catalyst with a GRChombo example. + +![Catalyst Logo](https://www.paraview.org/Wiki/images/8/8a/CatalystLogo.png) + +## Prerequisites + +You will need a build of ParaView with Catalyst support. Unfortunately, the +pre-built binaries available on the ParaView website for various OSs and +architecture are not built with Catalyst support so it is necessary to build +ParaView from source. Given the large number of dependencies, it is probably +easiest to use the +[superbuild](https://gitlab.kitware.com/paraview/paraview-superbuild/) which +downloads and builds all dependencies before building ParaView. When configuring +the superbuild using CMake, you will want to make sure the following options are +set +```cmake +ENABLE_osmesa = ON # enables off-screen rendering +USE_SYSTEM_llvm = ON # a prerequisite for OSMesa above +USE_SYSTEM_mpi = ON # you will want to use the same MPI as for [GR]Chombo +PARAVIEW_BUILD_EDITION = CATALYST_RENDERING # This will build without IO +# libraries and a user interface so will be much easier and quicker +``` +Note that the above setting for `PARAVIEW_BUILD_EDITION` will not build +the usual ParaView graphical user interface as this can take quite a bit longer +and can be more complicated. For more details see +[this Kitware blog post](https://www.kitware.com/paraview-editions/). In order +to [generate ParaView Catalyst scripts](#generating-paraview-catalyst-scripts), +you will probably want to use the GUI but for this part, you can use the +pre-built ParaView binaries available on the [ParaView +website](https://www.paraview.org/download/). Alternatively, you can set +```cmake +PARAVIEW_BUILD_EDITION = CANONICAL +``` + +Since building ParaView can be a little daunting, unless you area already +familiar with building software with CMake, it may be advisable to ask +your HPC system administrator to build a suitable version for you. If so, make +sure you specify that you need a version with Catalyst enabled and with +off-screen rendering. + +This in-situ adaptor has been tested with the following versions of ParaView: + +| Tested ParaView versions | +| --- | +| 5.8.1 | +| 5.9.1 | +| 5.10.0 | +| 5.10.1 | +| 5.11.0 | + +It may work with other versions but these have not been tested. + +## Adding ParaView Catalyst to an existing GRChombo example + +To add ParaView Catalyst to your existing example, make sure your example is +up-to-date with this branch. Then simply add the line +```make +include $(GRCHOMBO_SOURCE)/Insitu/Make.insitu +``` +in the `GNUmakefile` directly after the `src_dirs` variable has been set. See +the [BinaryBH's `GNUmakefile`](../../Examples/BinaryBH/GNUmakefile) for an +example. Note that this script uses the `XTRALDFLAGS` and `cxxcppflags` Chombo +make variables so do *not* set these in your `Make.defs.local` file. + +The in-situ visualization pipelines are called in +`GRAMRLevel::postTimeStep()` after `GRAMRLevel::specificPostTimeStep()`. +If you wish to also process the pipeline before the first timestep, +you can use the [`MultiLevelTask` class](../utils/MultiLevelTask.hpp). See the +BinaryBH's [Main_BinaryBH.cpp](../../Examples/BinaryBH/Main_BinaryBH.cpp) for +an example. + +If you are adding any code to your example that is specific to +Catalyst it is a good idea to conditionally compile it so that you can continue +to compile your example without Catalyst if you wish. For example +```cpp +#ifdef USE_CATALYST + +#endif +``` +If you need to compute diagnostic quantities for visualization, you can override +the virtual function `GRAMRLevel::preCatalystCoProcess()` which is called before +Catalyst CoProcessing. + + +## Building a GRChombo example with ParaView Catalyst + +First, make sure you have updated [Chombo](https://github.com/GRChombo/Chombo) +to the latest commit on `main`. + +To build a GRChombo example with ParaView Catalyst (for example the [BinaryBH +example](../../Examples/BinaryBH/)), next set the `PARAVIEW_DIR` to the path +which contains the build of ParaView as described [above](#prerequisites), for +example +```bash +export PARAVIEW_DIR=/path/to/paraview_build_dir +``` +This directory should contain an executable at `bin/paraview-config` as this is +the script that will be used to determine the compiler/linker flags needed to +build with ParaView Catalyst. Since this script takes a long time to run, the +outputs are cached using the [cache.sh](./cache.sh) script to +`/tmp/command_cache.`. If something messes up and this cached output is +incorrect, it can be helpful to remove these files using e.g. +```bash +rm -f /tmp/command_cache.* +``` + +Finally `make` the `insitu` target e.g. +``` +make insitu -j 4 +``` +An executable binary with a filename containing '`_Insitu`' will be created. +Note that if the `PARAVIEW_DIR` command is set, you can also `make` the +usual `all` target to link with ParaView Catalyst but the executable filename +will be the conventional one (i.e. without '`_Insitu`'). + +If you wish to stop compiling/linking with ParaView Catalyst, unset the +`PARAVIEW_DIR` environment variable with +```bash +unset PARAVIEW_DIR +``` + + +## Generating ParaView Catalyst scripts + +> **Warning** +> The procedure to generate Catalyst scripts varies between +> different versions of ParaView. In particular, there are significant +> differences before v5.9. The following instructions apply to v5.9 or greater. + +First, note that you do not need to use a build of ParaView with Catalyst +support in order to generate a Catalyst script. For example, you can just use +the [pre-built binaries from the ParaView +website](https://www.paraview.org/download/). In fact you probably cannot use +the same build as you will want a version with the GUI which precludes a version +with off-screen rendering (which you need for in-situ visualization). + +The easiest way to generate a suitable ParaView Catalyst Python script using the +GUI is to load an HDF5 file that is similar to the configuration that will exist +in your simulation. For example, you could use a plot/checkpoint file from the +initial data of the simulation. Once you have loaded the file into ParaView, +**make sure you rename the source to 'input'** (by right clicking on it and +selecting 'Rename'). This is necessary so that Catalyst can replace the HDF5 +data in the script with that coming from the simulation. + +Next apply filters and adjust the camera view as desired. + +In order to generate a valid Catalyst script, it is then necessary to add an +'Extractor' which will, for example, save an image. This can be done using the +menu options, Extractors → Image → PNG. + +Finally, to generate the Catalyst script, use the menu options File → +Save Catalyst State. By default, the output directory for Extracts will be +'dataset'. + +Unfortunately, if you have built the `CATALYST_RENDERING` ParaView edition +as described [above](#prerequisites) for running in-situ, since this does not +contain any IO libraries, it is necessary to manually edit the generated +Catalyst script and replace the line +```python +input = VisItChomboReader(registrationName='input', FileName=) +``` +with +```python +input = AMRGaussianPulseSource(registrationName='input') +``` +in order to be able to use it. + +> **Note** +> It is possible to pass information about the GRChombo grid (in particular, the +> lengths of the domain and the center) to Catalyst pipelines so that they +> automatically adjust appropriately. These are set as environment variables of +> the form `GRCHOMBO_PARAM_xxx` in +> [ChomboParameters.hpp](../GRChomboCore/ChomboParameters.hpp) and can be +> recovered in Catalyst Python scripts using the `os.getenv()` function from the +> `os` package. See the +> [slice_chi.py](../../Examples/BinaryBH/catalyst_scripts/slice_chi.py) script +> as an example. + +## Using the in-situ visualization capabilities + +There are several Catalyst specific parameters which control the in-situ +visualization processing which are described in the table below. + +| Parameter Name | Type | Possible value [default] | Description | +| --- | --- | --- | --- | +| `catalyst_activate` | `bool` | `true`/[`false`] | Enables/disables Catalyst coprocessing at runtime. If set to `false`, other parameters are not set. | +| `catalyst_verbosity` | `int` | `-2`,...,[`verbosity`],..,`10` | Controls verbosity in `catalyst_pout` files and that printed to the normal `pout` files from the `CatalystAdaptor` class which interfaces with ParaView Catalyst | +| `catalyst_pout_prefix` | `string` | [`"catalyst_pout"`] | Filename prefix for the `catalyst_pout` files. They are written to `pout_subpath` and appended by `. ` as for the normal `pout` files. Note that this can be set to the same string as `pout_prefix` to send the output to the normal `pout` files. | +| `catalyst_scripts_path` | `string` | `""` | Path that contains the Catalyst Python scrips generated as described [above](#generating-paraview-catalyst-scripts). Note that this path should be either an absolute path or relative to the current directory and _not_ `output_path`. | +| `catalyst_num_scripts` | `int` | [`1`] | Number of Catalyst Python scripts | +| `catalyst_scripts` | `vector` | - | Filenames of Catalyst Python scripts in the `catalyst_scripts_path` directory. | +| `catalyst_coprocess_level` | `int` | [`0`],...,`max_level` | Level for which the Catalyst coprocess routine is called at the end of each timestep | +| `catalyst_remove_ghosts` | `bool` | `true`/[`false`] | If `true`, Chombo data will be deep-copied to new VTK arrays without ghosts rather than simply passing the pointers. This is for debugging and should not be necessary for most users. | +| `catalyst_abort_on_error` | `bool` | `true`/[`false`] | If `true`, the code will abort if there is an error in Catalyst. | +| `catalyst_num_vars`* | `int` | [`0`],...,`NUM_VARS + NUM_DIAGNOSTIC_VARS` | Number of variables in `catalyst_vars` | +| `catalyst_vars`* | `vector` | - | Restrict variables passed to Catalyst to only these ones. No restriction if `catalyst_num_vars == 0`.| +| `catalyst_write_vtk_files` | `bool` | `true`/[`false`] | Controls whether VTK XML files containing the full 3D AMR data passed to Catalyst are written or not. These files can be opened in ParaView. | +| `vtk_file_prefix` | `string` | [`"Catalyst_VTK_"`] | Filename prefix for VTK XML files. The filenames will be of the form `_<6 digit timestep on catalyst_coprocess_level>`. Note there will be multiple files for each timestep: one of type `.vth` and a subdirectory containing `.vti` files for each box. | + +*Note that scripts generated with versions of ParaView v5.8 or earlier provide +information to the pipeline about the specific variables they require. Since +this feature is not currently provided by ParaView v5.9 or later, scripts +generated with these versions request all variables by default. In order to +only pass specific variables to Catalyst, it is possible to restrict the +variables passed using the `num_catalyst_vars` and `catalyst_vars` parameters +(as described above). +If these are not set, and a script generated by ParaView v5.9 or later is used, +all variables will be passed to Catalyst. + +If the `output_path` parameter is set, then this will be passed to Catalyst as +the working directory and the subdirectory for any outputs created by +visualization pipelines will exist under this path. \ No newline at end of file diff --git a/Source/Insitu/cache.sh b/Source/Insitu/cache.sh new file mode 100755 index 000000000..2447276d7 --- /dev/null +++ b/Source/Insitu/cache.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# This script caches the ouput of shell commands when first run and then uses +# the cache if the arguments haven't changed. +# Modified version of https://stackoverflow.com/a/36092050 to avoid race +# condition. + +# hash all arguments +KEY="$@" +EXIT_CODE=0 + +# hash last modified dates of any files +for arg in "$@" +do + if [ -f "$arg" ] + then + KEY+=`date -r "$arg" +\ %s` + fi +done + +# use the hash as a name for temporary file +FILE="/tmp/command_cache.`echo -n "$KEY" | md5sum | cut -c -10`" +FILE_LOCK="${FILE}.lock" + +# check file isn't locked +while [ -f $FILE_LOCK ] +do + sleep 1 +done + +# use cached file or execute the command and cache it +if [ -f $FILE ] +then + cat $FILE + exit $EXIT_CODE +else + # lock file + touch $FILE_LOCK + $@ | tee $FILE + EXIT_CODE=${PIPESTATUS[0]} + # delete cached output if command exited with an error + if [ "$EXIT_CODE" != "0" ] + then + rm $FILE + fi + rm -f $FILE_LOCK + exit $EXIT_CODE +fi diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.cpp b/Tests/CatalystInsituTest/CatalystInsituTest.cpp new file mode 100644 index 000000000..84cdd80bc --- /dev/null +++ b/Tests/CatalystInsituTest/CatalystInsituTest.cpp @@ -0,0 +1,143 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifdef CH_LANG_CC +/* + * _______ __ + * / ___/ / ___ __ _ / / ___ + * / /__/ _ \/ _ \/ V \/ _ \/ _ \ + * \___/_//_/\___/_/_/_/_.__/\___/ + * Please refer to LICENSE, in Chombo's root directory. + */ +#endif + +// Chombo includes +#include "parstream.H" //Gives us pout() + +// General includes: +#include +#include +#include + +#include "GRAMR.hpp" + +#include "GRParmParse.hpp" +#include "MultiLevelTask.hpp" +#include "SetupFunctions.hpp" +#include "SimulationParameters.hpp" + +// Problem specific includes: +#include "DefaultLevelFactory.hpp" +#include "InsituTestLevel.hpp" +#include "UserVariables.hpp" + +// VTK/Paraview includes +#ifdef USE_CATALYST +#include +#include +#include +#endif + +// Chombo namespace +#include "UsingNamespace.H" + +#ifdef USE_CATALYST + +// A simple function to compare two timespecs +bool operator<=(const timespec &lhs, const timespec &rhs) +{ + if (lhs.tv_sec == rhs.tv_sec) + return lhs.tv_nsec <= rhs.tv_nsec; + else + return lhs.tv_sec <= rhs.tv_sec; +} + +int runInsituTest(int argc, char *argv[]) +{ + int status = 0; + + // Load the parameter file and construct the SimulationParameter class + // To add more parameters edit the SimulationParameters file. + std::string in_string = argv[argc - 1]; + pout() << in_string << std::endl; + char const *in_file = argv[argc - 1]; + GRParmParse pp(0, argv + argc, NULL, in_file); + SimulationParameters sim_params(pp); + + // Delete the generated PNG file if it already exists + std::remove(sim_params.generated_png_file.c_str()); + + GRAMR gr_amr; + DefaultLevelFactory insitu_test_level_factory(gr_amr, + sim_params); + setupAMRObject(gr_amr, insitu_test_level_factory); + + auto task = [](GRAMRLevel *a_level) + { + if (a_level->time() == 0.0) + { + a_level->catalystCoProcess(); + } + }; + + MultiLevelTaskPtr<> call_task(task); + call_task.execute(gr_amr); + gr_amr.run(sim_params.stop_time, sim_params.max_steps); + gr_amr.conclude(); + + // read the newly generated PNG and the expected PNG + vtkNew generated_png_reader; + generated_png_reader->SetFileName(sim_params.generated_png_file.c_str()); + vtkNew valid_png_reader; + valid_png_reader->SetFileName(sim_params.valid_png_file.c_str()); + + // Calculate the difference + vtkNew image_difference; + image_difference->SetInputConnection(generated_png_reader->GetOutputPort()); + image_difference->SetImageConnection(valid_png_reader->GetOutputPort()); + image_difference->Update(); + + double error = image_difference->GetThresholdedError(); + + pout() << "error = " << error << "\n"; + pout() << "error_threshold = " << sim_params.error_threshold << std::endl; + + status = (error < sim_params.error_threshold) ? 0 : 1; + + return status; +} +#endif + +int main(int argc, char *argv[]) +{ +#ifdef USE_CATALYST + mainSetup(argc, argv); + + int status = runInsituTest(argc, argv); +#else + int status = -1; +#endif + + if (status == 0) + pout() << "Catalyst Insitu test passed." << std::endl; + else if (status == -1) + { + pout() << "Catalyst Insitu test skipped (USE_CATALYST undefined)." + << std::endl; + } + else if (status == -2) + { + pout() << "Catalyst Insitu test skipped (ParaView version < 5.9)." + << std::endl; + } + else + pout() << "Catalyst Insitu test failed with return code " << status + << std::endl; +#ifdef USE_CATALYST + mainFinalize(); +#endif + + return std::max(status, 0); +} diff --git a/Tests/CatalystInsituTest/CatalystInsituTest.inputs b/Tests/CatalystInsituTest/CatalystInsituTest.inputs new file mode 100644 index 000000000..d400604ba --- /dev/null +++ b/Tests/CatalystInsituTest/CatalystInsituTest.inputs @@ -0,0 +1,46 @@ +verbosity = 0 +N_full = 32 +L_full = 16 + +chk_prefix = TestChk_ +plot_prefix = TestPlt_ +stop_time = 0 + +num_points = 30 + +#num_ghosts = 3 +max_level = 0 +regrid_interval = 1 + +max_box_size = 8 +min_box_size = 4 + +#boundaries and periodicity of grid +#Periodic directions - 0 = false, 1 = true +isPeriodic = 0 0 0 +# if not periodic, then specify the boundary type +# 0 = static, 1 = sommerfeld, 2 = reflective +# (see BoundaryConditions.hpp for details) +hi_boundary = 0 0 0 +lo_boundary = 0 0 0 + +################################################# +# ParaView Catalyst parameters + +catalyst_activate = true +catalyst_verbosity = 10 +# catalyst_pout_prefix = catalyst_pout +# path to directory containing catalyst scripts +# catalyst_scripts_path = catalyst_scripts +catalyst_num_scripts = 1 +# list of Catalyst scripts +catalyst_scripts = slice_A_v1.py # slice_A_v2.py +catalyst_coprocess_level = 0 +# catalyst_num_vars = 1 +# catalyst_vars = A +catalyst_abort_on_error = true + +# Test parameters +error_threshold = 0.15 # arbitrary threshold but this is what VTK tests use +valid_png_file = extracts/SliceA_expected.png +generated_png_file = extracts/SliceA_000000.png \ No newline at end of file diff --git a/Tests/CatalystInsituTest/GNUmakefile b/Tests/CatalystInsituTest/GNUmakefile new file mode 100644 index 000000000..23ef85e54 --- /dev/null +++ b/Tests/CatalystInsituTest/GNUmakefile @@ -0,0 +1,26 @@ +# -*- Mode: Makefile -*- + +### This makefile produces an executable for each name in the `ebase' +### variable using the libraries named in the `LibNames' variable. + +# Included makefiles need an absolute path to the Chombo installation +# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) + +GRCHOMBO_SOURCE = $(shell pwd)/../../Source + +ebase := CatalystInsituTest + +LibNames := AMRTimeDependent AMRTools BoxTools + +src_dirs := $(GRCHOMBO_SOURCE)/utils \ + $(GRCHOMBO_SOURCE)/simd \ + $(GRCHOMBO_SOURCE)/BoxUtils \ + $(GRCHOMBO_SOURCE)/CCZ4 \ + $(GRCHOMBO_SOURCE)/GRChomboCore \ + $(GRCHOMBO_SOURCE)/AMRInterpolator + +XTRA_VTK_COMPONENT_FLAGS = -v ImagingCore -v ImagingSources -v IOImage + +include $(GRCHOMBO_SOURCE)/Insitu/Make.insitu + +include $(CHOMBO_HOME)/mk/Make.test diff --git a/Tests/CatalystInsituTest/InsituTestLevel.hpp b/Tests/CatalystInsituTest/InsituTestLevel.hpp new file mode 100644 index 000000000..0a5669494 --- /dev/null +++ b/Tests/CatalystInsituTest/InsituTestLevel.hpp @@ -0,0 +1,36 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef INSITUTESTLEVEL_HPP_ +#define INSITUTESTLEVEL_HPP_ + +#include "BoxLoops.hpp" +#include "GRAMRLevel.hpp" +#include "Polynomial.hpp" +#include "SetValue.hpp" + +class InsituTestLevel : public GRAMRLevel +{ + friend class DefaultLevelFactory; + // Inherit the contructors from GRAMRLevel + using GRAMRLevel::GRAMRLevel; + + // initialize data + virtual void initialData() + { + BoxLoops::loop(Polynomial(m_p.center, m_dx), m_state_new, m_state_new, + FILL_GHOST_CELLS); + } + + virtual void specificEvalRHS(GRLevelData &a_soln, GRLevelData &a_rhs, + const double a_time) + { + } + + virtual void computeTaggingCriterion(FArrayBox &tagging_criterion, + const FArrayBox ¤t_state){}; +}; + +#endif /* INSITUTESTLEVEL_HPP_ */ diff --git a/Tests/CatalystInsituTest/Polynomial.hpp b/Tests/CatalystInsituTest/Polynomial.hpp new file mode 120000 index 000000000..c2e85b310 --- /dev/null +++ b/Tests/CatalystInsituTest/Polynomial.hpp @@ -0,0 +1 @@ +../AMRInterpolatorTest/Polynomial.hpp \ No newline at end of file diff --git a/Tests/CatalystInsituTest/SimulationParameters.hpp b/Tests/CatalystInsituTest/SimulationParameters.hpp new file mode 100644 index 000000000..ed82a5bfe --- /dev/null +++ b/Tests/CatalystInsituTest/SimulationParameters.hpp @@ -0,0 +1,32 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SIMULATIONPARAMETERS_HPP_ +#define SIMULATIONPARAMETERS_HPP_ + +// General includes +#include "ChomboParameters.hpp" +#include "GRParmParse.hpp" + +class SimulationParameters : public ChomboParameters +{ + public: + SimulationParameters(GRParmParse &pp) : ChomboParameters(pp) + { + pp.load("error_threshold", error_threshold, 0.15); + pp.load("valid_png_file", valid_png_file); + pp.load("generated_png_file", generated_png_file); + if (!output_path.empty()) + { + generated_png_file = output_path + generated_png_file; + } + } + + double error_threshold; + std::string valid_png_file; + std::string generated_png_file; +}; + +#endif /* SIMULATIONPARAMETERS_HPP_ */ diff --git a/Tests/CatalystInsituTest/UserVariables.hpp b/Tests/CatalystInsituTest/UserVariables.hpp new file mode 100644 index 000000000..b1eea0df2 --- /dev/null +++ b/Tests/CatalystInsituTest/UserVariables.hpp @@ -0,0 +1,29 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef USERVARIABLES_HPP +#define USERVARIABLES_HPP + +#include "EmptyDiagnosticVariables.hpp" +#include +#include + +// assign enum to each variable +enum +{ + c_A, + c_B, + + NUM_VARS +}; + +namespace UserVariables +{ +static const std::array variable_names = {"A", "B"}; +} + +#include "UserVariables.inc.hpp" + +#endif /* USERVARIABLES_HPP */ diff --git a/Tests/CatalystInsituTest/extracts/SliceA_expected.png b/Tests/CatalystInsituTest/extracts/SliceA_expected.png new file mode 100644 index 000000000..7fa176143 Binary files /dev/null and b/Tests/CatalystInsituTest/extracts/SliceA_expected.png differ diff --git a/Tests/CatalystInsituTest/slice_A_v1.py b/Tests/CatalystInsituTest/slice_A_v1.py new file mode 100644 index 000000000..e9d970bba --- /dev/null +++ b/Tests/CatalystInsituTest/slice_A_v1.py @@ -0,0 +1,220 @@ + +#-------------------------------------------------------------- + +# Global timestep output options +timeStepToStartOutputAt=0 +forceOutputAtFirstCall=False + +# Global screenshot output options +imageFileNamePadding=6 +rescale_lookuptable=False + +# Whether or not to request specific arrays from the adaptor. +requestSpecificArrays=True + +# a root directory under which all Catalyst output goes +rootDirectory='extracts' + +# makes a cinema D index table +make_cinema_table=False + +#-------------------------------------------------------------- +# Code generated from cpstate.py to create the CoProcessor. +# paraview version 5.8.1 +#-------------------------------------------------------------- + +from paraview.simple import * +from paraview import coprocessing + +# ----------------------- CoProcessor definition ----------------------- + +def CreateCoProcessor(): + def _CreatePipeline(coprocessor, datadescription): + class Pipeline: + # state file generated using paraview version 5.8.1 + + # ---------------------------------------------------------------- + # setup views used in the visualization + # ---------------------------------------------------------------- + + # trace generated using paraview version 5.8.1 + # + # To ensure correct image size when batch processing, please search + # for and uncomment the line `# renderView*.ViewSize = [*,*]` + + #### disable automatic camera reset on 'Show' + paraview.simple._DisableFirstRenderCameraReset() + + # get the material library + materialLibrary1 = GetMaterialLibrary() + + # Black background + LoadPalette('BlackBackground') + + # Create a new 'Render View' + renderView1 = CreateView('RenderView') + renderView1.ViewSize = [1092, 778] + renderView1.AxesGrid = 'GridAxes3DActor' + renderView1.OrientationAxesVisibility = 0 + renderView1.CenterOfRotation = [8.0, 8.0, 8.0] + renderView1.StereoType = 'Crystal Eyes' + renderView1.CameraPosition = [-31.513131514425908, 8.0, 8.0] + renderView1.CameraFocalPoint = [8.0, 8.0, 8.0] + renderView1.CameraViewUp = [0.0, 0.0, 1.0] + renderView1.CameraFocalDisk = 1.0 + renderView1.CameraParallelScale = 12.374368670764582 + renderView1.BackEnd = 'OSPRay raycaster' + renderView1.OSPRayMaterialLibrary = materialLibrary1 + renderView1.Background = [0.0, 0.0, 0.0] + + # register the view with coprocessor + # and provide it with information such as the filename to use, + # how frequently to write the images, etc. + coprocessor.RegisterView(renderView1, + filename='SliceA_%t.png', freq=1, fittoscreen=0, magnification=1, width=800, height=600, cinema={}, compression=5) + renderView1.ViewTime = datadescription.GetTime() + + SetActiveView(None) + + # ---------------------------------------------------------------- + # setup view layouts + # ---------------------------------------------------------------- + + # create new layout object 'Layout #1' + layout1 = CreateLayout(name='Layout #1') + layout1.AssignView(0, renderView1) + + # ---------------------------------------------------------------- + # restore active view + SetActiveView(renderView1) + # ---------------------------------------------------------------- + + # ---------------------------------------------------------------- + # setup the data processing pipelines + # ---------------------------------------------------------------- + + # create a new 'VisItChomboReader' + # create a producer from a simulation input + input = coprocessor.CreateProducer(datadescription, 'input') + + # create a new 'Slice' + slice1 = Slice(Input=input) + slice1.SliceType = 'Plane' + slice1.HyperTreeGridSlicer = 'Plane' + slice1.SliceOffsetValues = [0.0] + + # init the 'Plane' selected for 'SliceType' + slice1.SliceType.Origin = [8.0, 8.0, 8.0] + + # init the 'Plane' selected for 'HyperTreeGridSlicer' + slice1.HyperTreeGridSlicer.Origin = [8.0, 8.0, 8.0] + + # ---------------------------------------------------------------- + # setup the visualization in view 'renderView1' + # ---------------------------------------------------------------- + + # show data from slice1 + slice1Display = Show(slice1, renderView1, 'GeometryRepresentation') + + # get color transfer function/color map for 'A' + aLUT = GetColorTransferFunction('A') + aLUT.RGBPoints = [42.015869140625, 0.267004, 0.004874, 0.329415, 64.0117780078125, 0.26851, 0.009605, 0.335427, 86.00207853515624, 0.269944, 0.014625, 0.341379, 107.99798740234375, 0.271305, 0.019942, 0.347269, 129.9882879296875, 0.272594, 0.025563, 0.353093, 151.984196796875, 0.273809, 0.031497, 0.358853, 173.97449732421876, 0.274952, 0.037752, 0.364543, 195.97040619140625, 0.276022, 0.044167, 0.370164, 217.96631505859375, 0.277018, 0.050344, 0.375715, 239.9566155859375, 0.277941, 0.056324, 0.381191, 261.952524453125, 0.278791, 0.062145, 0.386592, 283.94282498046874, 0.279566, 0.067836, 0.391917, 305.93873384765624, 0.280267, 0.073417, 0.397163, 327.92903437499996, 0.280894, 0.078907, 0.402329, 349.9249432421875, 0.281446, 0.08432, 0.407414, 371.920852109375, 0.281924, 0.089666, 0.412415, 393.9111526367187, 0.282327, 0.094955, 0.417331, 415.9070615039063, 0.282656, 0.100196, 0.42216, 437.89736203125, 0.28291, 0.105393, 0.426902, 459.89327089843755, 0.283091, 0.110553, 0.431554, 481.88357142578127, 0.283197, 0.11568, 0.436115, 503.8794802929687, 0.283229, 0.120777, 0.440584, 525.8753891601563, 0.283187, 0.125848, 0.44496, 547.8656896875, 0.283072, 0.130895, 0.449241, 569.8615985546875, 0.282884, 0.13592, 0.453427, 591.8518990820313, 0.282623, 0.140926, 0.457517, 613.8478079492187, 0.28229, 0.145912, 0.46151, 635.8381084765625, 0.281887, 0.150881, 0.465405, 657.83401734375, 0.281412, 0.155834, 0.469201, 679.8243178710937, 0.280868, 0.160771, 0.472899, 701.8202267382812, 0.280255, 0.165693, 0.476498, 723.8161356054687, 0.279574, 0.170599, 0.479997, 745.8064361328125, 0.278826, 0.17549, 0.483397, 767.802345, 0.278012, 0.180367, 0.486697, 789.7926455273438, 0.277134, 0.185228, 0.489898, 811.7885543945312, 0.276194, 0.190074, 0.493001, 833.778854921875, 0.275191, 0.194905, 0.496005, 855.7747637890625, 0.274128, 0.199721, 0.498911, 877.7706726562501, 0.273006, 0.20452, 0.501721, 899.7609731835937, 0.271828, 0.209303, 0.504434, 921.7568820507813, 0.270595, 0.214069, 0.507052, 943.747182578125, 0.269308, 0.218818, 0.509577, 965.7430914453124, 0.267968, 0.223549, 0.512008, 987.7333919726562, 0.26658, 0.228262, 0.514349, 1009.7293008398437, 0.265145, 0.232956, 0.516599, 1031.7252097070311, 0.263663, 0.237631, 0.518762, 1053.715510234375, 0.262138, 0.242286, 0.520837, 1075.7114191015626, 0.260571, 0.246922, 0.522828, 1097.7017196289064, 0.258965, 0.251537, 0.524736, 1119.6976284960938, 0.257322, 0.25613, 0.526563, 1141.6879290234376, 0.255645, 0.260703, 0.528312, 1163.683837890625, 0.253935, 0.265254, 0.529983, 1185.6797467578124, 0.252194, 0.269783, 0.531579, 1207.6700472851562, 0.250425, 0.27429, 0.533103, 1229.6659561523438, 0.248629, 0.278775, 0.534556, 1251.6562566796874, 0.246811, 0.283237, 0.535941, 1273.652165546875, 0.244972, 0.287675, 0.53726, 1295.6424660742189, 0.243113, 0.292092, 0.538516, 1317.6383749414063, 0.241237, 0.296485, 0.539709, 1339.6342838085936, 0.239346, 0.300855, 0.540844, 1361.6245843359375, 0.237441, 0.305202, 0.541921, 1383.620493203125, 0.235526, 0.309527, 0.542944, 1405.6107937304687, 0.233603, 0.313828, 0.543914, 1427.6067025976563, 0.231674, 0.318106, 0.544834, 1449.597003125, 0.229739, 0.322361, 0.545706, 1471.5929119921875, 0.227802, 0.326594, 0.546532, 1493.588820859375, 0.225863, 0.330805, 0.547314, 1515.5791213867187, 0.223925, 0.334994, 0.548053, 1537.575030253906, 0.221989, 0.339161, 0.548752, 1559.56533078125, 0.220057, 0.343307, 0.549413, 1581.5612396484373, 0.21813, 0.347432, 0.550038, 1603.5515401757812, 0.21621, 0.351535, 0.550627, 1625.5474490429688, 0.214298, 0.355619, 0.551184, 1647.5433579101561, 0.212395, 0.359683, 0.55171, 1669.5336584375, 0.210503, 0.363727, 0.552206, 1691.5295673046874, 0.208623, 0.367752, 0.552675, 1713.5198678320312, 0.206756, 0.371758, 0.553117, 1735.5157766992186, 0.204903, 0.375746, 0.553533, 1757.5060772265624, 0.203063, 0.379716, 0.553925, 1779.5019860937502, 0.201239, 0.38367, 0.554294, 1801.4922866210936, 0.19943, 0.387607, 0.554642, 1823.4881954882812, 0.197636, 0.391528, 0.554969, 1845.4841043554686, 0.19586, 0.395433, 0.555276, 1867.4744048828125, 0.1941, 0.399323, 0.555565, 1889.4703137499998, 0.192357, 0.403199, 0.555836, 1911.4606142773437, 0.190631, 0.407061, 0.556089, 1933.4565231445315, 0.188923, 0.41091, 0.556326, 1955.4468236718749, 0.187231, 0.414746, 0.556547, 1977.4427325390625, 0.185556, 0.41857, 0.556753, 1999.4386414062499, 0.183898, 0.422383, 0.556944, 2021.4289419335937, 0.182256, 0.426184, 0.55712, 2043.424850800781, 0.180629, 0.429975, 0.557282, 2065.415151328125, 0.179019, 0.433756, 0.55743, 2087.4110601953125, 0.177423, 0.437527, 0.557565, 2109.401360722656, 0.175841, 0.44129, 0.557685, 2131.3972695898437, 0.174274, 0.445044, 0.557792, 2153.3931784570314, 0.172719, 0.448791, 0.557885, 2175.383478984375, 0.171176, 0.45253, 0.557965, 2197.3793878515626, 0.169646, 0.456262, 0.55803, 2219.369688378906, 0.168126, 0.459988, 0.558082, 2241.365597246094, 0.166617, 0.463708, 0.558119, 2263.3558977734374, 0.165117, 0.467423, 0.558141, 2285.351806640625, 0.163625, 0.471133, 0.558148, 2307.3477155078126, 0.162142, 0.474838, 0.55814, 2329.338016035156, 0.160665, 0.47854, 0.558115, 2351.333924902344, 0.159194, 0.482237, 0.558073, 2373.3242254296874, 0.157729, 0.485932, 0.558013, 2395.320134296875, 0.15627, 0.489624, 0.557936, 2417.3104348242186, 0.154815, 0.493313, 0.55784, 2439.3063436914063, 0.153364, 0.497, 0.557724, 2461.302252558594, 0.151918, 0.500685, 0.557587, 2483.2925530859375, 0.150476, 0.504369, 0.55743, 2505.288461953125, 0.149039, 0.508051, 0.55725, 2527.2787624804687, 0.147607, 0.511733, 0.557049, 2549.2746713476563, 0.14618, 0.515413, 0.556823, 2571.264971875, 0.144759, 0.519093, 0.556572, 2593.2608807421875, 0.143343, 0.522773, 0.556295, 2615.256789609375, 0.141935, 0.526453, 0.555991, 2637.2470901367187, 0.140536, 0.530132, 0.555659, 2659.2429990039063, 0.139147, 0.533812, 0.555298, 2681.23329953125, 0.13777, 0.537492, 0.554906, 2703.2292083984375, 0.136408, 0.541173, 0.554483, 2725.219508925781, 0.135066, 0.544853, 0.554029, 2747.2154177929688, 0.133743, 0.548535, 0.553541, 2769.2113266601564, 0.132444, 0.552216, 0.553018, 2791.2016271875, 0.131172, 0.555899, 0.552459, 2813.1975360546876, 0.129933, 0.559582, 0.551864, 2835.187836582031, 0.128729, 0.563265, 0.551229, 2857.183745449219, 0.127568, 0.566949, 0.550556, 2879.174045976563, 0.126453, 0.570633, 0.549841, 2901.16995484375, 0.125394, 0.574318, 0.549086, 2923.1602553710936, 0.124395, 0.578002, 0.548287, 2945.1561642382812, 0.123463, 0.581687, 0.547445, 2967.1520731054684, 0.122606, 0.585371, 0.546557, 2989.1423736328125, 0.121831, 0.589055, 0.545623, 3011.1382825, 0.121148, 0.592739, 0.544641, 3033.1285830273437, 0.120565, 0.596422, 0.543611, 3055.1244918945313, 0.120092, 0.600104, 0.54253, 3077.114792421875, 0.119738, 0.603785, 0.5414, 3099.1107012890625, 0.119512, 0.607464, 0.540218, 3121.1066101562496, 0.119423, 0.611141, 0.538982, 3143.0969106835937, 0.119483, 0.614817, 0.537692, 3165.0928195507813, 0.119699, 0.61849, 0.536347, 3187.083120078125, 0.120081, 0.622161, 0.534946, 3209.0790289453125, 0.120638, 0.625828, 0.533488, 3231.069329472656, 0.12138, 0.629492, 0.531973, 3253.0652383398437, 0.122312, 0.633153, 0.530398, 3275.061147207031, 0.123444, 0.636809, 0.528763, 3297.051447734375, 0.12478, 0.640461, 0.527068, 3319.0473566015626, 0.126326, 0.644107, 0.525311, 3341.037657128906, 0.128087, 0.647749, 0.523491, 3363.033565996094, 0.130067, 0.651384, 0.521608, 3385.0238665234374, 0.132268, 0.655014, 0.519661, 3407.019775390625, 0.134692, 0.658636, 0.517649, 3429.015684257812, 0.137339, 0.662252, 0.515571, 3451.005984785156, 0.14021, 0.665859, 0.513427, 3473.001893652344, 0.143303, 0.669459, 0.511215, 3494.9921941796874, 0.146616, 0.67305, 0.508936, 3516.9881030468755, 0.150148, 0.676631, 0.506589, 3538.9784035742186, 0.153894, 0.680203, 0.504172, 3560.9743124414063, 0.157851, 0.683765, 0.501686, 3582.9702213085934, 0.162016, 0.687316, 0.499129, 3604.9605218359375, 0.166383, 0.690856, 0.496502, 3626.956430703125, 0.170948, 0.694384, 0.493803, 3648.9467312304687, 0.175707, 0.6979, 0.491033, 3670.9426400976567, 0.180653, 0.701402, 0.488189, 3692.932940625, 0.185783, 0.704891, 0.485273, 3714.9288494921875, 0.19109, 0.708366, 0.482284, 3736.9247583593747, 0.196571, 0.711827, 0.479221, 3758.9150588867187, 0.202219, 0.715272, 0.476084, 3780.9109677539063, 0.20803, 0.718701, 0.472873, 3802.90126828125, 0.214, 0.722114, 0.469588, 3824.897177148438, 0.220124, 0.725509, 0.466226, 3846.887477675781, 0.226397, 0.728888, 0.462789, 3868.8833865429688, 0.232815, 0.732247, 0.459277, 3890.879295410156, 0.239374, 0.735588, 0.455688, 3912.8695959375, 0.24607, 0.73891, 0.452024, 3934.8655048046876, 0.252899, 0.742211, 0.448284, 3956.855805332031, 0.259857, 0.745492, 0.444467, 3978.8517141992183, 0.266941, 0.748751, 0.440573, 4000.8420147265624, 0.274149, 0.751988, 0.436601, 4022.83792359375, 0.281477, 0.755203, 0.432552, 4044.828224121094, 0.288921, 0.758394, 0.428426, 4066.8241329882812, 0.296479, 0.761561, 0.424223, 4088.820041855469, 0.304148, 0.764704, 0.419943, 4110.810342382812, 0.311925, 0.767822, 0.415586, 4132.80625125, 0.319809, 0.770914, 0.411152, 4154.796551777344, 0.327796, 0.77398, 0.40664, 4176.792460644531, 0.335885, 0.777018, 0.402049, 4198.782761171875, 0.344074, 0.780029, 0.397381, 4220.7786700390625, 0.35236, 0.783011, 0.392636, 4242.77457890625, 0.360741, 0.785964, 0.387814, 4264.764879433594, 0.369214, 0.788888, 0.382914, 4286.760788300781, 0.377779, 0.791781, 0.377939, 4308.751088828125, 0.386433, 0.794644, 0.372886, 4330.7469976953125, 0.395174, 0.797475, 0.367757, 4352.737298222656, 0.404001, 0.800275, 0.362552, 4374.733207089844, 0.412913, 0.803041, 0.357269, 4396.729115957031, 0.421908, 0.805774, 0.35191, 4418.719416484375, 0.430983, 0.808473, 0.346476, 4440.715325351563, 0.440137, 0.811138, 0.340967, 4462.705625878906, 0.449368, 0.813768, 0.335384, 4484.701534746094, 0.458674, 0.816363, 0.329727, 4506.691835273437, 0.468053, 0.818921, 0.323998, 4528.687744140625, 0.477504, 0.821444, 0.318195, 4550.683653007813, 0.487026, 0.823929, 0.312321, 4572.673953535156, 0.496615, 0.826376, 0.306377, 4594.669862402344, 0.506271, 0.828786, 0.300362, 4616.660162929687, 0.515992, 0.831158, 0.294279, 4638.656071796875, 0.525776, 0.833491, 0.288127, 4660.646372324219, 0.535621, 0.835785, 0.281908, 4682.642281191406, 0.545524, 0.838039, 0.275626, 4704.638190058594, 0.555484, 0.840254, 0.269281, 4726.6284905859375, 0.565498, 0.84243, 0.262877, 4748.624399453125, 0.575563, 0.844566, 0.256415, 4770.614699980469, 0.585678, 0.846661, 0.249897, 4792.610608847656, 0.595839, 0.848717, 0.243329, 4814.600909375, 0.606045, 0.850733, 0.236712, 4836.5968182421875, 0.616293, 0.852709, 0.230052, 4858.592727109375, 0.626579, 0.854645, 0.223353, 4880.583027636719, 0.636902, 0.856542, 0.21662, 4902.578936503906, 0.647257, 0.8584, 0.209861, 4924.56923703125, 0.657642, 0.860219, 0.203082, 4946.565145898438, 0.668054, 0.861999, 0.196293, 4968.555446425781, 0.678489, 0.863742, 0.189503, 4990.551355292969, 0.688944, 0.865448, 0.182725, 5012.547264160156, 0.699415, 0.867117, 0.175971, 5034.5375646875, 0.709898, 0.868751, 0.169257, 5056.533473554688, 0.720391, 0.87035, 0.162603, 5078.523774082031, 0.730889, 0.871916, 0.156029, 5100.519682949219, 0.741388, 0.873449, 0.149561, 5122.509983476562, 0.751884, 0.874951, 0.143228, 5144.50589234375, 0.762373, 0.876424, 0.137064, 5166.496192871094, 0.772852, 0.877868, 0.131109, 5188.492101738281, 0.783315, 0.879285, 0.125405, 5210.488010605469, 0.79376, 0.880678, 0.120005, 5232.478311132812, 0.804182, 0.882046, 0.114965, 5254.47422, 0.814576, 0.883393, 0.110347, 5276.464520527344, 0.82494, 0.88472, 0.106217, 5298.460429394531, 0.83527, 0.886029, 0.102646, 5320.450729921875, 0.845561, 0.887322, 0.099702, 5342.4466387890625, 0.85581, 0.888601, 0.097452, 5364.44254765625, 0.866013, 0.889868, 0.095953, 5386.432848183594, 0.876168, 0.891125, 0.09525, 5408.428757050781, 0.886271, 0.892374, 0.095374, 5430.419057578125, 0.89632, 0.893616, 0.096335, 5452.4149664453125, 0.906311, 0.894855, 0.098125, 5474.405266972656, 0.916242, 0.896091, 0.100717, 5496.401175839844, 0.926106, 0.89733, 0.104071, 5518.397084707031, 0.935904, 0.89857, 0.108131, 5540.387385234375, 0.945636, 0.899815, 0.112838, 5562.383294101563, 0.9553, 0.901065, 0.118128, 5584.373594628906, 0.964894, 0.902323, 0.123941, 5606.369503496094, 0.974417, 0.90359, 0.130215, 5628.359804023437, 0.983868, 0.904867, 0.136897, 5650.355712890625, 0.993248, 0.906157, 0.143936] + aLUT.NanColor = [1.0, 0.0, 0.0] + aLUT.ScalarRangeInitialized = 1.0 + + # trace defaults for the display properties. + slice1Display.Representation = 'Surface' + slice1Display.ColorArrayName = ['CELLS', 'A'] + slice1Display.LookupTable = aLUT + slice1Display.OSPRayScaleFunction = 'PiecewiseFunction' + slice1Display.SelectOrientationVectors = 'None' + slice1Display.ScaleFactor = 1.75 + slice1Display.SelectScaleArray = 'None' + slice1Display.GlyphType = 'Arrow' + slice1Display.GlyphTableIndexArray = 'None' + slice1Display.GaussianRadius = 0.08750000000000001 + slice1Display.SetScaleArray = [None, ''] + slice1Display.ScaleTransferFunction = 'PiecewiseFunction' + slice1Display.OpacityArray = [None, ''] + slice1Display.OpacityTransferFunction = 'PiecewiseFunction' + slice1Display.DataAxesGrid = 'GridAxesRepresentation' + slice1Display.PolarAxes = 'PolarAxesRepresentation' + + # ---------------------------------------------------------------- + # setup color maps and opacity mapes used in the visualization + # note: the Get..() functions create a new object, if needed + # ---------------------------------------------------------------- + + # get opacity transfer function/opacity map for 'A' + aPWF = GetOpacityTransferFunction('A') + aPWF.Points = [42.06640625, 0.0, 0.5, 0.0, 7363.00390625, 1.0, 0.5, 0.0] + aPWF.ScalarRangeInitialized = 1 + + # ---------------------------------------------------------------- + # finally, restore active source + SetActiveSource(slice1) + # ---------------------------------------------------------------- + return Pipeline() + + class CoProcessor(coprocessing.CoProcessor): + def CreatePipeline(self, datadescription): + self.Pipeline = _CreatePipeline(self, datadescription) + + coprocessor = CoProcessor() + # these are the frequencies at which the coprocessor updates. + freqs = {'input': [1]} + coprocessor.SetUpdateFrequencies(freqs) + if requestSpecificArrays: + arrays = [['A', 1], ['vtkGhostType', 1]] + coprocessor.SetRequestedArrays('input', arrays) + coprocessor.SetInitialOutputOptions(timeStepToStartOutputAt,forceOutputAtFirstCall) + + if rootDirectory: + coprocessor.SetRootDirectory(rootDirectory) + + if make_cinema_table: + coprocessor.EnableCinemaDTable() + + return coprocessor + + +#-------------------------------------------------------------- +# Global variable that will hold the pipeline for each timestep +# Creating the CoProcessor object, doesn't actually create the ParaView pipeline. +# It will be automatically setup when coprocessor.UpdateProducers() is called the +# first time. +coprocessor = CreateCoProcessor() + +#-------------------------------------------------------------- +# Enable Live-Visualizaton with ParaView and the update frequency +coprocessor.EnableLiveVisualization(False, 1) + +# ---------------------- Data Selection method ---------------------- + +def RequestDataDescription(datadescription): + "Callback to populate the request for current timestep" + global coprocessor + + # setup requests for all inputs based on the requirements of the + # pipeline. + coprocessor.LoadRequestedData(datadescription) + +# ------------------------ Processing method ------------------------ + +def DoCoProcessing(datadescription): + "Callback to do co-processing for current timestep" + global coprocessor + + # Update the coprocessor by providing it the newly generated simulation data. + # If the pipeline hasn't been setup yet, this will setup the pipeline. + coprocessor.UpdateProducers(datadescription) + + # Write output data, if appropriate. + coprocessor.WriteData(datadescription); + + # Write image capture (Last arg: rescale lookup table), if appropriate. + coprocessor.WriteImages(datadescription, rescale_lookuptable=rescale_lookuptable, + image_quality=0, padding_amount=imageFileNamePadding) + + # Live Visualization, if enabled. + coprocessor.DoLiveVisualization(datadescription, "localhost", 22222) diff --git a/Tests/CatalystInsituTest/slice_A_v2.py b/Tests/CatalystInsituTest/slice_A_v2.py new file mode 100644 index 000000000..8b8d16009 --- /dev/null +++ b/Tests/CatalystInsituTest/slice_A_v2.py @@ -0,0 +1,157 @@ +# script-version: 2.0 +# Catalyst state generated using paraview version 5.9.1 + +#### import the simple module from the paraview +from paraview.simple import * +#### disable automatic camera reset on 'Show' +paraview.simple._DisableFirstRenderCameraReset() + +# ---------------------------------------------------------------- +# setup views used in the visualization +# ---------------------------------------------------------------- + +# get the material library +materialLibrary1 = GetMaterialLibrary() + +# Create a new 'Render View' +renderView1 = CreateView('RenderView') +renderView1.ViewSize = [1092, 778] +renderView1.AxesGrid = 'GridAxes3DActor' +renderView1.OrientationAxesVisibility = 0 +renderView1.CenterOfRotation = [8.0, 8.0, 8.0] +renderView1.StereoType = 'Crystal Eyes' +renderView1.CameraPosition = [-31.513131514425908, 8.0, 8.0] +renderView1.CameraFocalPoint = [8.0, 8.0, 8.0] +renderView1.CameraViewUp = [0.0, 0.0, 1.0] +renderView1.CameraFocalDisk = 1.0 +renderView1.CameraParallelScale = 12.374368670764582 +renderView1.BackEnd = 'OSPRay raycaster' +renderView1.OSPRayMaterialLibrary = materialLibrary1 +renderView1.Background = [0.0, 0.0, 0.0] + +SetActiveView(None) + +# ---------------------------------------------------------------- +# setup view layouts +# ---------------------------------------------------------------- + +# create new layout object 'Layout #1' +layout1 = CreateLayout(name='Layout #1') +layout1.AssignView(0, renderView1) +layout1.SetSize(1092, 778) + +# ---------------------------------------------------------------- +# restore active view +SetActiveView(renderView1) +# ---------------------------------------------------------------- + +# ---------------------------------------------------------------- +# setup the data processing pipelines +# ---------------------------------------------------------------- + +# create a new 'AMRGaussianPulseSource' +input = AMRGaussianPulseSource(registrationName='input') +input.MeshStatus = ['Mesh'] +input.CellArrayStatus = ['A'] + +# create a new 'Slice' +slice1 = Slice(registrationName='Slice1', Input=input) +slice1.SliceType = 'Plane' +slice1.HyperTreeGridSlicer = 'Plane' +slice1.SliceOffsetValues = [0.0] + +# init the 'Plane' selected for 'SliceType' +slice1.SliceType.Origin = [8.0, 8.0, 8.0] + +# init the 'Plane' selected for 'HyperTreeGridSlicer' +slice1.HyperTreeGridSlicer.Origin = [8.0, 8.0, 8.0] + +# ---------------------------------------------------------------- +# setup the visualization in view 'renderView1' +# ---------------------------------------------------------------- + +# show data from slice1 +slice1Display = Show(slice1, renderView1, 'GeometryRepresentation') + +# get color transfer function/color map for 'A' +aLUT = GetColorTransferFunction('A') +aLUT.RGBPoints = [42.015869140625, 0.267004, 0.004874, 0.329415, 64.0117780078125, 0.26851, 0.009605, 0.335427, 86.00207853515624, 0.269944, 0.014625, 0.341379, 107.99798740234375, 0.271305, 0.019942, 0.347269, 129.9882879296875, 0.272594, 0.025563, 0.353093, 151.984196796875, 0.273809, 0.031497, 0.358853, 173.97449732421876, 0.274952, 0.037752, 0.364543, 195.97040619140625, 0.276022, 0.044167, 0.370164, 217.96631505859375, 0.277018, 0.050344, 0.375715, 239.9566155859375, 0.277941, 0.056324, 0.381191, 261.952524453125, 0.278791, 0.062145, 0.386592, 283.94282498046874, 0.279566, 0.067836, 0.391917, 305.93873384765624, 0.280267, 0.073417, 0.397163, 327.92903437499996, 0.280894, 0.078907, 0.402329, 349.9249432421875, 0.281446, 0.08432, 0.407414, 371.920852109375, 0.281924, 0.089666, 0.412415, 393.9111526367187, 0.282327, 0.094955, 0.417331, 415.9070615039063, 0.282656, 0.100196, 0.42216, 437.89736203125, 0.28291, 0.105393, 0.426902, 459.89327089843755, 0.283091, 0.110553, 0.431554, 481.88357142578127, 0.283197, 0.11568, 0.436115, 503.8794802929687, 0.283229, 0.120777, 0.440584, 525.8753891601563, 0.283187, 0.125848, 0.44496, 547.8656896875, 0.283072, 0.130895, 0.449241, 569.8615985546875, 0.282884, 0.13592, 0.453427, 591.8518990820313, 0.282623, 0.140926, 0.457517, 613.8478079492187, 0.28229, 0.145912, 0.46151, 635.8381084765625, 0.281887, 0.150881, 0.465405, 657.83401734375, 0.281412, 0.155834, 0.469201, 679.8243178710937, 0.280868, 0.160771, 0.472899, 701.8202267382812, 0.280255, 0.165693, 0.476498, 723.8161356054687, 0.279574, 0.170599, 0.479997, 745.8064361328125, 0.278826, 0.17549, 0.483397, 767.802345, 0.278012, 0.180367, 0.486697, 789.7926455273438, 0.277134, 0.185228, 0.489898, 811.7885543945312, 0.276194, 0.190074, 0.493001, 833.778854921875, 0.275191, 0.194905, 0.496005, 855.7747637890625, 0.274128, 0.199721, 0.498911, 877.7706726562501, 0.273006, 0.20452, 0.501721, 899.7609731835937, 0.271828, 0.209303, 0.504434, 921.7568820507813, 0.270595, 0.214069, 0.507052, 943.747182578125, 0.269308, 0.218818, 0.509577, 965.7430914453124, 0.267968, 0.223549, 0.512008, 987.7333919726562, 0.26658, 0.228262, 0.514349, 1009.7293008398437, 0.265145, 0.232956, 0.516599, 1031.7252097070311, 0.263663, 0.237631, 0.518762, 1053.715510234375, 0.262138, 0.242286, 0.520837, 1075.7114191015626, 0.260571, 0.246922, 0.522828, 1097.7017196289064, 0.258965, 0.251537, 0.524736, 1119.6976284960938, 0.257322, 0.25613, 0.526563, 1141.6879290234376, 0.255645, 0.260703, 0.528312, 1163.683837890625, 0.253935, 0.265254, 0.529983, 1185.6797467578124, 0.252194, 0.269783, 0.531579, 1207.6700472851562, 0.250425, 0.27429, 0.533103, 1229.6659561523438, 0.248629, 0.278775, 0.534556, 1251.6562566796874, 0.246811, 0.283237, 0.535941, 1273.652165546875, 0.244972, 0.287675, 0.53726, 1295.6424660742189, 0.243113, 0.292092, 0.538516, 1317.6383749414063, 0.241237, 0.296485, 0.539709, 1339.6342838085936, 0.239346, 0.300855, 0.540844, 1361.6245843359375, 0.237441, 0.305202, 0.541921, 1383.620493203125, 0.235526, 0.309527, 0.542944, 1405.6107937304687, 0.233603, 0.313828, 0.543914, 1427.6067025976563, 0.231674, 0.318106, 0.544834, 1449.597003125, 0.229739, 0.322361, 0.545706, 1471.5929119921875, 0.227802, 0.326594, 0.546532, 1493.588820859375, 0.225863, 0.330805, 0.547314, 1515.5791213867187, 0.223925, 0.334994, 0.548053, 1537.575030253906, 0.221989, 0.339161, 0.548752, 1559.56533078125, 0.220057, 0.343307, 0.549413, 1581.5612396484373, 0.21813, 0.347432, 0.550038, 1603.5515401757812, 0.21621, 0.351535, 0.550627, 1625.5474490429688, 0.214298, 0.355619, 0.551184, 1647.5433579101561, 0.212395, 0.359683, 0.55171, 1669.5336584375, 0.210503, 0.363727, 0.552206, 1691.5295673046874, 0.208623, 0.367752, 0.552675, 1713.5198678320312, 0.206756, 0.371758, 0.553117, 1735.5157766992186, 0.204903, 0.375746, 0.553533, 1757.5060772265624, 0.203063, 0.379716, 0.553925, 1779.5019860937502, 0.201239, 0.38367, 0.554294, 1801.4922866210936, 0.19943, 0.387607, 0.554642, 1823.4881954882812, 0.197636, 0.391528, 0.554969, 1845.4841043554686, 0.19586, 0.395433, 0.555276, 1867.4744048828125, 0.1941, 0.399323, 0.555565, 1889.4703137499998, 0.192357, 0.403199, 0.555836, 1911.4606142773437, 0.190631, 0.407061, 0.556089, 1933.4565231445315, 0.188923, 0.41091, 0.556326, 1955.4468236718749, 0.187231, 0.414746, 0.556547, 1977.4427325390625, 0.185556, 0.41857, 0.556753, 1999.4386414062499, 0.183898, 0.422383, 0.556944, 2021.4289419335937, 0.182256, 0.426184, 0.55712, 2043.424850800781, 0.180629, 0.429975, 0.557282, 2065.415151328125, 0.179019, 0.433756, 0.55743, 2087.4110601953125, 0.177423, 0.437527, 0.557565, 2109.401360722656, 0.175841, 0.44129, 0.557685, 2131.3972695898437, 0.174274, 0.445044, 0.557792, 2153.3931784570314, 0.172719, 0.448791, 0.557885, 2175.383478984375, 0.171176, 0.45253, 0.557965, 2197.3793878515626, 0.169646, 0.456262, 0.55803, 2219.369688378906, 0.168126, 0.459988, 0.558082, 2241.365597246094, 0.166617, 0.463708, 0.558119, 2263.3558977734374, 0.165117, 0.467423, 0.558141, 2285.351806640625, 0.163625, 0.471133, 0.558148, 2307.3477155078126, 0.162142, 0.474838, 0.55814, 2329.338016035156, 0.160665, 0.47854, 0.558115, 2351.333924902344, 0.159194, 0.482237, 0.558073, 2373.3242254296874, 0.157729, 0.485932, 0.558013, 2395.320134296875, 0.15627, 0.489624, 0.557936, 2417.3104348242186, 0.154815, 0.493313, 0.55784, 2439.3063436914063, 0.153364, 0.497, 0.557724, 2461.302252558594, 0.151918, 0.500685, 0.557587, 2483.2925530859375, 0.150476, 0.504369, 0.55743, 2505.288461953125, 0.149039, 0.508051, 0.55725, 2527.2787624804687, 0.147607, 0.511733, 0.557049, 2549.2746713476563, 0.14618, 0.515413, 0.556823, 2571.264971875, 0.144759, 0.519093, 0.556572, 2593.2608807421875, 0.143343, 0.522773, 0.556295, 2615.256789609375, 0.141935, 0.526453, 0.555991, 2637.2470901367187, 0.140536, 0.530132, 0.555659, 2659.2429990039063, 0.139147, 0.533812, 0.555298, 2681.23329953125, 0.13777, 0.537492, 0.554906, 2703.2292083984375, 0.136408, 0.541173, 0.554483, 2725.219508925781, 0.135066, 0.544853, 0.554029, 2747.2154177929688, 0.133743, 0.548535, 0.553541, 2769.2113266601564, 0.132444, 0.552216, 0.553018, 2791.2016271875, 0.131172, 0.555899, 0.552459, 2813.1975360546876, 0.129933, 0.559582, 0.551864, 2835.187836582031, 0.128729, 0.563265, 0.551229, 2857.183745449219, 0.127568, 0.566949, 0.550556, 2879.174045976563, 0.126453, 0.570633, 0.549841, 2901.16995484375, 0.125394, 0.574318, 0.549086, 2923.1602553710936, 0.124395, 0.578002, 0.548287, 2945.1561642382812, 0.123463, 0.581687, 0.547445, 2967.1520731054684, 0.122606, 0.585371, 0.546557, 2989.1423736328125, 0.121831, 0.589055, 0.545623, 3011.1382825, 0.121148, 0.592739, 0.544641, 3033.1285830273437, 0.120565, 0.596422, 0.543611, 3055.1244918945313, 0.120092, 0.600104, 0.54253, 3077.114792421875, 0.119738, 0.603785, 0.5414, 3099.1107012890625, 0.119512, 0.607464, 0.540218, 3121.1066101562496, 0.119423, 0.611141, 0.538982, 3143.0969106835937, 0.119483, 0.614817, 0.537692, 3165.0928195507813, 0.119699, 0.61849, 0.536347, 3187.083120078125, 0.120081, 0.622161, 0.534946, 3209.0790289453125, 0.120638, 0.625828, 0.533488, 3231.069329472656, 0.12138, 0.629492, 0.531973, 3253.0652383398437, 0.122312, 0.633153, 0.530398, 3275.061147207031, 0.123444, 0.636809, 0.528763, 3297.051447734375, 0.12478, 0.640461, 0.527068, 3319.0473566015626, 0.126326, 0.644107, 0.525311, 3341.037657128906, 0.128087, 0.647749, 0.523491, 3363.033565996094, 0.130067, 0.651384, 0.521608, 3385.0238665234374, 0.132268, 0.655014, 0.519661, 3407.019775390625, 0.134692, 0.658636, 0.517649, 3429.015684257812, 0.137339, 0.662252, 0.515571, 3451.005984785156, 0.14021, 0.665859, 0.513427, 3473.001893652344, 0.143303, 0.669459, 0.511215, 3494.9921941796874, 0.146616, 0.67305, 0.508936, 3516.9881030468755, 0.150148, 0.676631, 0.506589, 3538.9784035742186, 0.153894, 0.680203, 0.504172, 3560.9743124414063, 0.157851, 0.683765, 0.501686, 3582.9702213085934, 0.162016, 0.687316, 0.499129, 3604.9605218359375, 0.166383, 0.690856, 0.496502, 3626.956430703125, 0.170948, 0.694384, 0.493803, 3648.9467312304687, 0.175707, 0.6979, 0.491033, 3670.9426400976567, 0.180653, 0.701402, 0.488189, 3692.932940625, 0.185783, 0.704891, 0.485273, 3714.9288494921875, 0.19109, 0.708366, 0.482284, 3736.9247583593747, 0.196571, 0.711827, 0.479221, 3758.9150588867187, 0.202219, 0.715272, 0.476084, 3780.9109677539063, 0.20803, 0.718701, 0.472873, 3802.90126828125, 0.214, 0.722114, 0.469588, 3824.897177148438, 0.220124, 0.725509, 0.466226, 3846.887477675781, 0.226397, 0.728888, 0.462789, 3868.8833865429688, 0.232815, 0.732247, 0.459277, 3890.879295410156, 0.239374, 0.735588, 0.455688, 3912.8695959375, 0.24607, 0.73891, 0.452024, 3934.8655048046876, 0.252899, 0.742211, 0.448284, 3956.855805332031, 0.259857, 0.745492, 0.444467, 3978.8517141992183, 0.266941, 0.748751, 0.440573, 4000.8420147265624, 0.274149, 0.751988, 0.436601, 4022.83792359375, 0.281477, 0.755203, 0.432552, 4044.828224121094, 0.288921, 0.758394, 0.428426, 4066.8241329882812, 0.296479, 0.761561, 0.424223, 4088.820041855469, 0.304148, 0.764704, 0.419943, 4110.810342382812, 0.311925, 0.767822, 0.415586, 4132.80625125, 0.319809, 0.770914, 0.411152, 4154.796551777344, 0.327796, 0.77398, 0.40664, 4176.792460644531, 0.335885, 0.777018, 0.402049, 4198.782761171875, 0.344074, 0.780029, 0.397381, 4220.7786700390625, 0.35236, 0.783011, 0.392636, 4242.77457890625, 0.360741, 0.785964, 0.387814, 4264.764879433594, 0.369214, 0.788888, 0.382914, 4286.760788300781, 0.377779, 0.791781, 0.377939, 4308.751088828125, 0.386433, 0.794644, 0.372886, 4330.7469976953125, 0.395174, 0.797475, 0.367757, 4352.737298222656, 0.404001, 0.800275, 0.362552, 4374.733207089844, 0.412913, 0.803041, 0.357269, 4396.729115957031, 0.421908, 0.805774, 0.35191, 4418.719416484375, 0.430983, 0.808473, 0.346476, 4440.715325351563, 0.440137, 0.811138, 0.340967, 4462.705625878906, 0.449368, 0.813768, 0.335384, 4484.701534746094, 0.458674, 0.816363, 0.329727, 4506.691835273437, 0.468053, 0.818921, 0.323998, 4528.687744140625, 0.477504, 0.821444, 0.318195, 4550.683653007813, 0.487026, 0.823929, 0.312321, 4572.673953535156, 0.496615, 0.826376, 0.306377, 4594.669862402344, 0.506271, 0.828786, 0.300362, 4616.660162929687, 0.515992, 0.831158, 0.294279, 4638.656071796875, 0.525776, 0.833491, 0.288127, 4660.646372324219, 0.535621, 0.835785, 0.281908, 4682.642281191406, 0.545524, 0.838039, 0.275626, 4704.638190058594, 0.555484, 0.840254, 0.269281, 4726.6284905859375, 0.565498, 0.84243, 0.262877, 4748.624399453125, 0.575563, 0.844566, 0.256415, 4770.614699980469, 0.585678, 0.846661, 0.249897, 4792.610608847656, 0.595839, 0.848717, 0.243329, 4814.600909375, 0.606045, 0.850733, 0.236712, 4836.5968182421875, 0.616293, 0.852709, 0.230052, 4858.592727109375, 0.626579, 0.854645, 0.223353, 4880.583027636719, 0.636902, 0.856542, 0.21662, 4902.578936503906, 0.647257, 0.8584, 0.209861, 4924.56923703125, 0.657642, 0.860219, 0.203082, 4946.565145898438, 0.668054, 0.861999, 0.196293, 4968.555446425781, 0.678489, 0.863742, 0.189503, 4990.551355292969, 0.688944, 0.865448, 0.182725, 5012.547264160156, 0.699415, 0.867117, 0.175971, 5034.5375646875, 0.709898, 0.868751, 0.169257, 5056.533473554688, 0.720391, 0.87035, 0.162603, 5078.523774082031, 0.730889, 0.871916, 0.156029, 5100.519682949219, 0.741388, 0.873449, 0.149561, 5122.509983476562, 0.751884, 0.874951, 0.143228, 5144.50589234375, 0.762373, 0.876424, 0.137064, 5166.496192871094, 0.772852, 0.877868, 0.131109, 5188.492101738281, 0.783315, 0.879285, 0.125405, 5210.488010605469, 0.79376, 0.880678, 0.120005, 5232.478311132812, 0.804182, 0.882046, 0.114965, 5254.47422, 0.814576, 0.883393, 0.110347, 5276.464520527344, 0.82494, 0.88472, 0.106217, 5298.460429394531, 0.83527, 0.886029, 0.102646, 5320.450729921875, 0.845561, 0.887322, 0.099702, 5342.4466387890625, 0.85581, 0.888601, 0.097452, 5364.44254765625, 0.866013, 0.889868, 0.095953, 5386.432848183594, 0.876168, 0.891125, 0.09525, 5408.428757050781, 0.886271, 0.892374, 0.095374, 5430.419057578125, 0.89632, 0.893616, 0.096335, 5452.4149664453125, 0.906311, 0.894855, 0.098125, 5474.405266972656, 0.916242, 0.896091, 0.100717, 5496.401175839844, 0.926106, 0.89733, 0.104071, 5518.397084707031, 0.935904, 0.89857, 0.108131, 5540.387385234375, 0.945636, 0.899815, 0.112838, 5562.383294101563, 0.9553, 0.901065, 0.118128, 5584.373594628906, 0.964894, 0.902323, 0.123941, 5606.369503496094, 0.974417, 0.90359, 0.130215, 5628.359804023437, 0.983868, 0.904867, 0.136897, 5650.355712890625, 0.993248, 0.906157, 0.143936] +aLUT.NanColor = [1.0, 0.0, 0.0] +aLUT.ScalarRangeInitialized = 1.0 + +# trace defaults for the display properties. +slice1Display.Representation = 'Surface' +slice1Display.ColorArrayName = ['CELLS', 'A'] +slice1Display.LookupTable = aLUT +slice1Display.SelectTCoordArray = 'None' +slice1Display.SelectNormalArray = 'None' +slice1Display.SelectTangentArray = 'None' +slice1Display.OSPRayScaleFunction = 'PiecewiseFunction' +slice1Display.SelectOrientationVectors = 'None' +slice1Display.ScaleFactor = 1.75 +slice1Display.SelectScaleArray = 'None' +slice1Display.GlyphType = 'Arrow' +slice1Display.GlyphTableIndexArray = 'None' +slice1Display.GaussianRadius = 0.08750000000000001 +slice1Display.SetScaleArray = [None, ''] +slice1Display.ScaleTransferFunction = 'PiecewiseFunction' +slice1Display.OpacityArray = [None, ''] +slice1Display.OpacityTransferFunction = 'PiecewiseFunction' +slice1Display.DataAxesGrid = 'GridAxesRepresentation' +slice1Display.PolarAxes = 'PolarAxesRepresentation' + +# setup the color legend parameters for each legend in this view + +# get color legend/bar for aLUT in view renderView1 +#aLUTColorBar = GetScalarBar(aLUT, renderView1) +#aLUTColorBar.Title = 'A' +#aLUTColorBar.ComponentTitle = '' + +# set color bar visibility +#aLUTColorBar.Visibility = 1 + +# show color legend +slice1Display.SetScalarBarVisibility(renderView1, False) + +# ---------------------------------------------------------------- +# setup color maps and opacity mapes used in the visualization +# note: the Get..() functions create a new object, if needed +# ---------------------------------------------------------------- + +# get opacity transfer function/opacity map for 'A' +aPWF = GetOpacityTransferFunction('A') +aPWF.Points = [42.015869140625, 1.0, 0.5, 0.0, 5650.35546875, 1.0, 0.5, 0.0] +aPWF.ScalarRangeInitialized = 1 + +# ---------------------------------------------------------------- +# setup extractors +# ---------------------------------------------------------------- + +# create extractor +pNG1 = CreateExtractor('PNG', renderView1, registrationName='PNG1') +# trace defaults for the extractor. +# init the 'PNG' selected for 'Writer' +pNG1.Writer.FileName = 'SliceA_%.6ts%cm.png' +pNG1.Writer.ImageResolution = [800, 600] +pNG1.Writer.OverrideColorPalette = 'BlackBackground' +pNG1.Writer.Format = 'PNG' + +# ---------------------------------------------------------------- +# restore active source +SetActiveSource(slice1) +# ---------------------------------------------------------------- + +# ------------------------------------------------------------------------------ +# Catalyst options +from paraview import catalyst +options = catalyst.Options() +options.ExtractsOutputDirectory = 'extracts' +options.GlobalTrigger = 'TimeStep' +options.CatalystLiveTrigger = 'TimeStep' + +# ------------------------------------------------------------------------------ +if __name__ == '__main__': + from paraview.simple import SaveExtractsUsingCatalystOptions + # Code for non in-situ environments; if executing in post-processing + # i.e. non-Catalyst mode, let's generate extracts using Catalyst options + SaveExtractsUsingCatalystOptions(options)