Benchmark with DaCe cpu and gpu backends

.github/workflows/main_unit_tests.yml

@@ -22,6 +22,10 @@ jobs:
           run: |
             python -m pip install --upgrade pip
             pip install -r requirements_dev.txt
+        - name: Clone datafiles
+          run: |
+            mkdir -p tests/main/input && cd tests/main/input
+            git clone -b store_files https://github.com/mlee03/pace.git tmp && mv tmp/*.nc . && rm -rf tmp
         - name: Run all main tests
           run: |
             pytest -x tests/main

.gitmodules

@@ -1,6 +1,8 @@
 [submodule "external/gt4py"]
 	path = external/gt4py
 	url = https://github.com/gridtools/gt4py.git
-[submodule "external/dace"]
+
+[submodule "dacefix"]
 	path = external/dace
-	url = https://github.com/spcl/dace.git
+	url = https://github.com/FlorianDeconinck/dace.git
+	branch = fix/gcc_dies_on_dacecpu

CONTRIBUTORS.md

@@ -11,7 +11,9 @@ List format (alphabetical order):  Surname, Name. Employer/Affiliation
 * Fuhrer, Oliver. Allen Institute for AI.
 * George, Rhea. Allen Institute for AI.
 * Harris, Lucas. GFDL.
+* Lee, Mi Kyung. GFDL.
 * Kung, Chris. NASA.
+* Malatino, Frank. GFDL
 * McGibbon, Jeremy. Allen Institute for AI.
 * Niedermayr, Yannick. ETH.
 * Savarin, Ajda. University of Washington.

benchmark_README.md

@@ -0,0 +1,96 @@
+# Benchmarking README
+
+The tests contained in this archive are for benchmarking purposes only.  Any
+distribution beyond those personnel performing the tests need explicit approval
+from NOAA/GFDL (Seth Underwood or Rusty Benson).
+
+## Cloning benchmark repository and generating conda environment
+
+Pace requires GCC > 9.2, MPI, and Python 3.8 on your system, and CUDA is required to run with a GPU backend. You will also need the headers of the boost libraries in your `$PATH` (boost itself does not need to be installed).
+
+```shell
+cd BOOST/ROOT
+wget https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.gz
+tar -xzf boost_1_79_0.tar.gz
+mkdir -p boost_1_79_0/include
+mv boost_1_79_0/boost boost_1_79_0/include/
+export BOOST_ROOT=BOOST/ROOT/boost_1_79_0
+```
+
+To clone the benchmark branch use the command:
+
+```shell
+git clone --recursive -b benchmark git@github.com:NOAA-GFDL/pace.git
+```
+
+or if you have already cloned the repository:
+
+```shell
+git submodule update --init --recursive
+```
+
+After cloning, change into the directory containing the clone. To generate the conda environment use the following commands:
+
+```shell
+conda create -y --name <desired_name> python=3.8
+conda activate <desired_name>
+pip3 install --upgrade pip setuptools wheel
+pip3 install -r requirements_dev.txt -c constraints.txt
+```
+
+## Benchmarking configurations
+
+There are four configurations of the PACE application contained within the branch to be used for benchmarking:
+
+```shell
+driver/examples/configs/baroclinic_c384_cpu.yaml
+driver/examples/configs/baroclinic_c384_gpu.yaml
+driver/examples/configs/baroclinic_c3072_cpu.yaml
+driver/examples/configs/baroclinic_c3072_gpu.yaml
+```
+
+## Building
+
+To build with the DaCe backends, set the following environment variables:
+
+```shell
+FV3_DACEMODE=Build
+PACE_FLOAT_PRECISION=64
+PACE_LOGLEVEL=INFO
+PYTHONOPTIMIZE=1
+OMP_NUM_THREAD=1
+```
+
+Adjust the time of the configuration to be built such that the time of the build is for one timestep. For example:
+
+```shell
+dt_atmos: 450
+seconds: 450
+```
+## Running
+To build with the DaCe backends, set the following environment variables:
+
+```shell
+FV3_DACEMODE=Run
+PACE_FLOAT_PRECISION=64
+PACE_LOGLEVEL=INFO
+PYTHONOPTIMIZE=1
+OMP_NUM_THREAD=1
+```
+
+Adjust the time of the configuration to be run to the desired length, example:
+
+```shell
+dt_atmos: 450
+days: 9
+```
+
+The time for the build or run can be set with units of seconds, minutes, hours, or days.
+
+An example command to start the build or run process with MPI using the DaCe CPU backend for the c384 configuration:
+
+```shell
+mpirun -n 1536 python3 -m pace.driver.run driver/examples/configs/baroclinic_c384_cpu.yaml
+```
+
+The build or run requires 1536 ranks, given that layout of 16x16 ranks per tile, and there are 6 tiles.

constraints.txt

@@ -81,7 +81,7 @@ coverage==5.5
     #   pytest-cov
 cytoolz==0.12.1
     # via gt4py
-dace==0.14.4
+dace==0.15.1
     # via
     #   -r requirements_dev.txt
     #   pace-dsl
@@ -184,7 +184,7 @@ googleapis-common-protos==1.53.0
     # via google-api-core
 gprof2dot==2021.2.21
     # via pytest-profiling
-gridtools-cpp==2.3.0
+gridtools-cpp==2.3.1
     # via gt4py
 h5netcdf==0.11.0
     # via -r util/requirements.txt

docs/physics/state.rst

@@ -38,6 +38,6 @@ You can initialize a zero-filled PhysicsState and MicrophysicsState from other P
 
     >>> quantity_factory = QuantityFactory.from_backend(sizer=sizer, backend="numpy")
     >>> physics_state = PhysicsState.init_zeros(
-    ...  quantity_factory=quantity_factory, active_packages=["microphysics"]
+    ...  quantity_factory=quantity_factory, schemes=["GFS_microphysics"]
     ... )
     >>> microphysics_state = physics_state.microphysics

driver/examples/configs/baroclinic_c12.yaml

@@ -94,3 +94,8 @@ physics_config:
   hydrostatic: false
   nwat: 6
   do_qa: true
+
+grid_config:
+  type:  generated
+  config:
+    eta_file: 'tests/main/input/eta79.nc'

driver/examples/configs/baroclinic_c12_explicit_physics.yaml

@@ -0,0 +1,98 @@
+stencil_config:
+  compilation_config:
+    backend: numpy
+    rebuild: false
+    validate_args: true
+    format_source: false
+    device_sync: false
+initialization:
+  type: analytic
+  config:
+    case: baroclinic
+performance_config:
+  collect_performance: true
+  experiment_name: c12_baroclinic
+nx_tile: 12
+nz: 79
+dt_atmos: 225
+minutes: 15
+layout:
+  - 1
+  - 1
+diagnostics_config:
+  path: output
+  output_format: netcdf
+  names:
+    - u
+    - v
+    - ua
+    - va
+    - pt
+    - delp
+    - qvapor
+    - qliquid
+    - qice
+    - qrain
+    - qsnow
+    - qgraupel
+  z_select:
+    - level: 65
+      names:
+        - pt
+dycore_config:
+  a_imp: 1.0
+  beta: 0.
+  consv_te: 0.
+  d2_bg: 0.
+  d2_bg_k1: 0.2
+  d2_bg_k2: 0.1
+  d4_bg: 0.15
+  d_con: 1.0
+  d_ext: 0.0
+  dddmp: 0.5
+  delt_max: 0.002
+  do_sat_adj: true
+  do_vort_damp: true
+  fill: true
+  hord_dp: 6
+  hord_mt: 6
+  hord_tm: 6
+  hord_tr: 8
+  hord_vt: 6
+  hydrostatic: false
+  k_split: 1
+  ke_bg: 0.
+  kord_mt: 9
+  kord_tm: -9
+  kord_tr: 9
+  kord_wz: 9
+  n_split: 1
+  nord: 3
+  nwat: 6
+  p_fac: 0.05
+  rf_cutoff: 3000.
+  rf_fast: true
+  tau: 10.
+  vtdm4: 0.06
+  z_tracer: true
+  do_qa: true
+  tau_i2s: 1000.
+  tau_g2v: 1200.
+  ql_gen: 0.001
+  ql_mlt: 0.002
+  qs_mlt: 0.000001
+  qi_lim: 1.0
+  dw_ocean: 0.1
+  dw_land: 0.15
+  icloud_f: 0
+  tau_l2v: 300.
+  tau_v2l: 90.
+  fv_sg_adj: 0
+  n_sponge: 48
+
+physics_config:
+  hydrostatic: false
+  nwat: 6
+  do_qa: true
+  schemes:
+    - GFS_microphysics

driver/examples/configs/baroclinic_c12_orch_cpu.yaml

@@ -14,10 +14,30 @@ performance_config:
 nx_tile: 12
 nz: 79
 dt_atmos: 225
-minutes: 5
+seconds: 675
 layout:
-  - 1
-  - 1
+  - 2
+  - 2
+diagnostics_config:
+  path: output
+  output_format: netcdf
+  names:
+    - u
+    - v
+    - ua
+    - va
+    - pt
+    - delp
+    - qvapor
+    - qliquid
+    - qice
+    - qrain
+    - qsnow
+    - qgraupel
+  z_select:
+    - level: 65
+      names:
+        - pt
 dycore_config:
   a_imp: 1.0
   beta: 0.

driver/examples/configs/baroclinic_c12_write_restart.yaml

@@ -92,3 +92,8 @@ physics_config:
   hydrostatic: false
   nwat: 6
   do_qa: true
+
+grid_config:
+  type:  generated
+  config:
+    eta_file: "tests/main/input/eta79.nc"

driver/examples/configs/baroclinic_c384_cpu.yaml

@@ -15,11 +15,10 @@ performance_config:
 nx_tile: 384
 nz: 79
 dt_atmos: 450
-minutes: 7
-seconds: 30
+days: 9
 layout:
-  - 1
-  - 1
+  - 16
+  - 16
 diagnostics_config:
   path: output
   output_format: netcdf
@@ -72,7 +71,7 @@ dycore_config:
   nwat: 6
   p_fac: 0.1
   rf_cutoff: 800.
-  rf_fast: false
+  rf_fast: true
   tau: 5.
   vtdm4: 0.06
   z_tracer: true

driver/examples/configs/baroclinic_c384_gpu.yaml

@@ -71,7 +71,7 @@ dycore_config:
   nwat: 6
   p_fac: 0.1
   rf_cutoff: 800.
-  rf_fast: false
+  rf_fast: true
   tau: 5.
   vtdm4: 0.06
   z_tracer: true