add coupler output summary table

.buildkite/benchmarks/pipeline.yml

@@ -0,0 +1,111 @@
+agents:
+  queue: clima
+  slurm_time: 24:00:00
+  modules: common
+
+env:
+  JULIA_CUDA_MEMORY_POOL: none
+  JULIA_MPI_HAS_CUDA: "true"
+  JULIA_NVTX_CALLBACKS: gc
+  JULIA_MAX_NUM_PRECOMPILE_FILES: 100
+  OPENBLAS_NUM_THREADS: 1
+  OMPI_MCA_opal_warn_on_missing_libcuda: 0
+  SLURM_KILL_BAD_EXIT: 1
+  SLURM_GRES_FLAGS: "allow-task-sharing"
+  GPU_DYAMOND_CONFIG_PATH: "config/gpu_configs/gpu_dyamond"
+  GPU_DYAMOND_WS_CONFIG_PATH: "config/gpu_configs/gpu_dyamond_ws"
+  CLIMAATMOS_GC_NSTEPS: 10
+  ATMOS_CONFIG_PATH: "config/atmos_configs"
+  MODEL_CONFIG_PATH: "config/model_configs"
+
+steps:
+  - label: "init :GPU:"
+    key: "init_gpu_env"
+    command:
+      - echo "--- Instantiate experiments/AMIP"
+      - julia --project=experiments/AMIP -e 'using Pkg; Pkg.instantiate(;verbose=true)'
+      - julia --project=experiments/AMIP -e 'using Pkg; Pkg.precompile()'
+      - julia --project=experiments/AMIP -e 'using Pkg; Pkg.status()'
+
+      - echo "--- Instantiate test env"
+      - "julia --project=test/ -e 'using Pkg; Pkg.develop(path=\".\")'"
+      - "julia --project=test/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project=test/ -e 'using Pkg; Pkg.precompile()'"
+      - "julia --project=test/ -e 'using Pkg; Pkg.status()'"
+
+      - echo "--- Download artifacts"
+      - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'"
+      - "julia --project=artifacts -e 'using Pkg; Pkg.status()'"
+      - "julia --project=artifacts artifacts/download_artifacts.jl"
+
+    agents:
+      slurm_gpus: 1
+      slurm_cpus_per_task: 8
+    env:
+      JULIA_NUM_PRECOMPILE_TASKS: 8
+      JULIA_MAX_NUM_PRECOMPILE_FILES: 50
+
+  - wait
+
+  - group: "CPU benchmarks"
+    steps:
+      - label: "CPU aquaplanet with diagnostic EDMF"
+        key: "aquaplanet_diagedmf"
+        command: "srun julia --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $ATMOS_CONFIG_PATH/aquaplanet_diagedmf.yml"
+        artifact_paths: "experiments/AMIP/output/aquaplanet/aquaplanet_diagedmf_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+          CLIMACOMMS_DEVICE: "CPU"
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 4GB
+
+      - label: "CPU AMIP with diagnostic EDMF"
+        key: "amip_diagedmf"
+        command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $MODEL_CONFIG_PATH/amip_diagedmf.yml"
+        artifact_paths: "experiments/AMIP/output/amip/amip_diagedmf_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+          CLIMACOMMS_DEVICE: "CPU"
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 4GB
+
+  - group: "GPU benchmarks"
+    steps:
+      - label: "GPU aquaplanet with diagnostic EDMF"
+        key: "gpu_aquaplanet_diagedmf"
+        command: "srun julia --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $ATMOS_CONFIG_PATH/gpu_aquaplanet_diagedmf.yml"
+        artifact_paths: "experiments/AMIP/output/aquaplanet/gpu_aquaplanet_diagedmf_artifacts/*"
+        agents:
+          slurm_gpus: 1
+          slurm_mem: 16GB
+
+      - label: "GPU AMIP with diagnostic EDMF"
+        key: "gpu_amip_diagedmf"
+        command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $MODEL_CONFIG_PATH/gpu_amip_diagedmf.yml"
+        artifact_paths: "experiments/AMIP/output/amip/gpu_amip_diagedmf_artifacts/*"
+        agents:
+          slurm_gpus: 1
+          slurm_mem: 16GB
+
+  - group: "Generate output table"
+    steps:
+      - label: "Compare AMIP/aquaplanet with diagnostic EDMF"
+        key: "compare_amip_aquaplanet_amip_diagedmf"
+        command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/user_io/benchmarks.jl --cpu_run_name_coupled amip_diagedmf --cpu_run_name_atmos aquaplanet_diagedmf --gpu_run_name_coupled gpu_amip_diagedmf --gpu_run_name_atmos gpu_aquaplanet_diagedmf --mode_name amip"
+        artifact_paths: "experiments/AMIP/output/compare_amip_aquaplanet_amip_diagedmf/*"
+        depends_on:
+          - "aquaplanet_diagedmf"
+          - "amip_diagedmf"
+          - "gpu_aquaplanet_diagedmf"
+          - "gpu_amip_diagedmf"
+
+      - label: ":envelope: Slack report: CPU/GPU AMIP/aquaplanet table"
+        depends_on:
+          - "compare_amip_aquaplanet_amip_diagedmf"
+        command:
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/output/compare_amip_aquaplanet_amip_diagedmf/table.txt -m txt -n compare_amip_aquaplanet_amip_diagedmf_table -x "Coupler CPU/GPU Comparison Table"

.buildkite/longruns/pipeline.yml

@@ -11,7 +11,6 @@ env:
   JULIA_MAX_NUM_PRECOMPILE_FILES: 100
   GKSwstype: 100
   SLURM_KILL_BAD_EXIT: 1
-
   CONFIG_PATH: "config/longrun_configs"
 
 timeout_in_minutes: 1440
@@ -300,11 +299,11 @@ steps:
 
   # DYAMOND AMIP: 1 day (convection resolving)
   - label: "GPU AMIP SUPERFINE: dyamond_target"
-    key: "gpu_dyamond_target"
+    key: "gpu_longrun_amip_dyamond"
     command:
       - echo "--- Run simulation"
-      - "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_dyamond_target.yml"
-    artifact_paths: "experiments/AMIP/output/amip/gpu_dyamond_target_artifacts/*"
+      - "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_longrun_amip_dyamond.yml"
+    artifact_paths: "experiments/AMIP/output/amip/gpu_longrun_amip_dyamond_artifacts/*"
     agents:
       queue: clima
       slurm_mem: 20GB

config/atmos_configs/aquaplanet_diagedmf.yml

@@ -0,0 +1,29 @@
+job_id: "aquaplanet_diagedmf"
+dt_save_state_to_disk: "Inf"
+dt_save_to_sol: "Inf"
+output_default_diagnostics: false
+h_elem: 30
+z_max: 55000.0
+z_elem: 63
+dz_bottom: 30.0
+dz_top: 3000.0
+moist: equil
+surface_setup: DefaultMoninObukhov
+rad: allskywithclear
+idealized_insolation: false
+dt_rad: 1hours
+dt_cloud_fraction: 1hours
+turbconv: diagnostic_edmfx
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+prognostic_tke: true
+edmfx_upwinding: first_order
+edmfx_entr_model: "Generalized"
+edmfx_detr_model: "Generalized"
+edmfx_nh_pressure: true
+edmfx_sgs_mass_flux: true
+edmfx_sgs_diffusive_flux: true
+precip_model: 0M
+dt: 100secs
+t_end: 12hours
+toml: [toml/diagnostic_edmfx_box.toml]

config/atmos_configs/gpu_aquaplanet_diagedmf.yml

@@ -0,0 +1,29 @@
+job_id: "gpu_aquaplanet_diagedmf"
+dt_save_state_to_disk: "Inf"
+dt_save_to_sol: "Inf"
+output_default_diagnostics: false
+h_elem: 30
+z_max: 55000.0
+z_elem: 63
+dz_bottom: 30.0
+dz_top: 3000.0
+moist: equil
+surface_setup: DefaultMoninObukhov
+rad: allskywithclear
+idealized_insolation: false
+dt_rad: 1hours
+dt_cloud_fraction: 1hours
+turbconv: diagnostic_edmfx
+implicit_diffusion: true
+approximate_linear_solve_iters: 2
+prognostic_tke: true
+edmfx_upwinding: first_order
+edmfx_entr_model: "Generalized"
+edmfx_detr_model: "Generalized"
+edmfx_nh_pressure: true
+edmfx_sgs_mass_flux: true
+edmfx_sgs_diffusive_flux: true
+precip_model: 0M
+dt: 100secs
+t_end: 12hours
+toml: [toml/diagnostic_edmfx_box.toml]

config/longrun_configs/gpu_dyamond_target.yml → config/longrun_configs/gpu_longrun_amip_dyamond.yml

@@ -4,12 +4,12 @@ dt_cpl: 50
 dt_save_state_to_disk: "0.5days"
 dt_save_to_sol: "0.5days"
 energy_check: false
-job_id: "gpu_dyamond_target"
+job_id: "gpu_longrun_amip_dyamond"
 land_albedo_type: "map_temporal"
 mode_name: "amip"
 mono_surface: false
 monthly_checkpoint: false
-run_name: "gpu_dyamond_target"
+run_name: "gpu_longrun_amip_dyamond"
 start_date: "19790301"
 t_end: "1days"
 turb_flux_partition: "CombinedStateFluxes"

config/model_configs/amip_diagedmf.yml

@@ -0,0 +1,15 @@
+anim: false
+atmos_config_file: "config/gpu_configs/gpu_aquaplanet_diagedmf.yml"
+dt_cpl: 100
+dt_save_state_to_disk: "12hours"
+dt_save_to_sol: "12hours"
+energy_check: false
+job_id: "amip_diagedmf"
+land_albedo_type: "map_temporal"
+mode_name: "amip"
+mono_surface: false
+monthly_checkpoint: false
+run_name: "amip_diagedmf"
+start_date: "19790301"
+t_end: "12hours"
+turb_flux_partition: "CombinedStateFluxes"

config/model_configs/gpu_amip_diagedmf.yml

@@ -0,0 +1,15 @@
+anim: false
+atmos_config_file: "config/gpu_configs/gpu_aquaplanet_diagedmf.yml"
+dt_cpl: 100
+dt_save_state_to_disk: "12hours"
+dt_save_to_sol: "12hours"
+energy_check: false
+job_id: "gpu_amip_diagedmf"
+land_albedo_type: "map_temporal"
+mode_name: "amip"
+mono_surface: false
+monthly_checkpoint: false
+run_name: "gpu_amip_diagedmf"
+start_date: "19790301"
+t_end: "12hours"
+turb_flux_partition: "CombinedStateFluxes"

experiments/AMIP/Manifest.toml

@@ -2055,6 +2055,12 @@ git-tree-sha1 = "84aa844bd56f62282116b413fbefb45e370e54d6"
 uuid = "58948b4f-47e0-5654-a9ad-f609743f8632"
 version = "901.300.0+1"
 
+[[deps.PackageExtensionCompat]]
+git-tree-sha1 = "fb28e33b8a95c4cee25ce296c817d89cc2e53518"
+uuid = "65ce6f38-6b18-4e1d-a461-8949797d7930"
+version = "1.0.2"
+weakdeps = ["Requires", "TOML"]
+
 [[deps.Packing]]
 deps = ["GeometryBasics"]
 git-tree-sha1 = "ec3edfe723df33528e085e632414499f26650501"

experiments/AMIP/Project.toml

@@ -2,6 +2,7 @@
 ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
 ArtifactWrappers = "a14bc488-3040-4b00-9dc1-f6467924858a"
 AtmosphericProfilesLibrary = "86bc3604-9858-485a-bdbe-831ec50de11d"
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
 ClimaAnalysis = "29b5916a-a76c-4e73-9657-3c8fd22e65e6"
 ClimaAtmos = "b2c96348-7fb7-4fe0-8da9-78d88439e717"