Merge pull request #632 from CliMA/js/gpu-amip-target

add target AMIP runs on GPU

.buildkite/longruns/pipeline.yml

@@ -235,6 +235,29 @@ steps:
           slurm_nodes: 4
           slurm_mem_per_cpu: 20G
 
+  - group: "Current target tests on GPU: AMIP surface"
+
+    steps:
+
+      - label: "GPU AMIP FINE: new target amip: topo"
+        key: "gpu_amip_target_topo"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo.yml"
+        artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_artifacts/*"
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+          slurm_gpus: 1
+
+      - label: "GPU AMIP FINE: new target amip: topo + diagedmf"
+        key: "gpu_amip_target_topo_diagedmf"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf.yml"
+        artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_artifacts/*"
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+          slurm_gpus: 1
 
   - group: "Other AMIP targets"
 

.buildkite/pipeline.yml

@@ -380,10 +380,18 @@ steps:
           slurm_gpus: 1
 
       # GPU RUNS: AMIP
-      - label: "GPU AMIP target: albedo from function"
-        key: "gpu_target_amip_albedo_function"
-        command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_target_amip_albedo_function.yml"
-        artifact_paths: "experiments/AMIP/output/amip/gpu_target_amip_albedo_function_artifacts/*"
+      - label: "GPU AMIP test: albedo from function"
+        key: "gpu_amip_albedo_function"
+        command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_function.yml"
+        artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_function_artifacts/*"
+        agents:
+          slurm_mem: 20GB
+          slurm_gpus: 1
+
+      - label: "GPU AMIP target: topography and diagnostic EDMF"
+        key: "gpu_amip_target_topo_diagedmf_shortrun"
+        command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf_shortrun.yml"
+        artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts/*"
         agents:
           slurm_mem: 20GB
           slurm_gpus: 1

config/longrun_configs/gpu_amip_target_topo.yml

@@ -0,0 +1,27 @@
+FLOAT_TYPE: "Float32"
+anim: false
+apply_limiter: false
+atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_0M_earth.yml"
+dt: "100secs"
+dt_cloud_fraction: "1hours"
+dt_cpl: 100
+dt_rad: "1hours"
+dt_save_state_to_disk: "20days"
+dt_save_to_sol: "10days"
+energy_check: false
+hourly_checkpoint: true
+job_id: "gpu_amip_target_topo"
+land_albedo_type: "map_temporal"
+mode_name: "amip"
+mono_surface: false
+netcdf_interpolate_z_over_msl: true
+rad: "allskywithclear"
+run_name: "gpu_amip_target_topo"
+start_date: "19790301"
+surface_setup: "PrescribedSurface"
+t_end: "100days"
+topo_smoothing: true
+topography: "Earth"
+turb_flux_partition: "CombinedStateFluxes"
+vert_diff: "true"
+use_reference_state: false

config/longrun_configs/gpu_amip_target_topo_diagedmf.yml

@@ -0,0 +1,26 @@
+FLOAT_TYPE: "Float32"
+anim: false
+apply_limiter: false
+atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equil_55km_nz63_allsky_diagedmf_0M.yml"
+dt: "100secs"
+dt_cloud_fraction: "1hours"
+dt_cpl: 100
+dt_rad: "1hours"
+dt_save_state_to_disk: "20days"
+dt_save_to_sol: "10days"
+energy_check: false
+hourly_checkpoint: true
+idealized_insolation: false
+job_id: "gpu_amip_target_topo_diagedmf"
+land_albedo_type: "map_temporal"
+mode_name: "amip"
+mono_surface: false
+netcdf_output_at_levels: true
+run_name: "gpu_amip_target_topo_diagedmf"
+start_date: "19790301"
+surface_setup: "PrescribedSurface"
+t_end: "40days"
+topo_smoothing: true
+topography: "Earth"
+turb_flux_partition: "CombinedStateFluxes"
+use_reference_state: false

config/model_configs/gpu_target_amip_albedo_function.yml → config/model_configs/gpu_amip_albedo_function.yml

@@ -7,15 +7,15 @@ dt_rad: "1hours"
 dt_save_to_sol: "1days"
 dz_bottom: 30
 dz_top: 3000
-h_elem: 16
-job_id: "gpu_target_amip_albedo_function"
+h_elem: 6
+job_id: "gpu_amip_albedo_function"
 land_albedo_type: "function"
 mode_name: "amip"
 moist: "equil"
 precip_model: "0M"
 rad: "gray"
 rayleigh_sponge: true
-run_name: "gpu_target_amip_albedo_function"
+run_name: "gpu_amip_albedo_function"
 t_end: "300secs"
 vert_diff: "true"
 z_elem: 50

config/model_configs/gpu_amip_target_topo_diagedmf_shortrun.yml

@@ -0,0 +1,26 @@
+FLOAT_TYPE: "Float32"
+anim: false
+apply_limiter: false
+atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equil_55km_nz63_allsky_diagedmf_0M.yml"
+dt: "100secs"
+dt_cloud_fraction: "1hours"
+dt_cpl: 100
+dt_rad: "1hours"
+dt_save_state_to_disk: "1days"
+dt_save_to_sol: "1days"
+energy_check: false
+hourly_checkpoint: true
+idealized_insolation: false
+job_id: "gpu_amip_target_topo_diagedmf_shortrun"
+land_albedo_type: "map_temporal"
+mode_name: "amip"
+mono_surface: false
+netcdf_output_at_levels: true
+run_name: "gpu_amip_target_topo_diagedmf_shortrun"
+start_date: "19790301"
+surface_setup: "PrescribedSurface"
+t_end: "200secs"
+topo_smoothing: true
+topography: "Earth"
+turb_flux_partition: "CombinedStateFluxes"
+use_reference_state: false

experiments/AMIP/components/atmosphere/climaatmos_init.jl

@@ -129,7 +129,7 @@ function update_field!(atmos_sim::ClimaAtmosSimulation, ::Val{:co2_gm}, field)
         @warn("Gray radiation model initialized, skipping CO2 update", maxlog = 1)
         return
     else
-        atmos_sim.integrator.p.radiation.radiation_model.volume_mixing_ratio_co2 .= parent(field)[1]
+        atmos_sim.integrator.p.radiation.radiation_model.volume_mixing_ratio_co2 .= mean(parent(field))
     end
 end
 # extensions required by the Interfacer