From 6a32cda64be188182e9c77b4648a0f72578745f9 Mon Sep 17 00:00:00 2001 From: Julia Sloan Date: Fri, 16 Feb 2024 17:01:14 -0800 Subject: [PATCH] add GPU target AMIP runs --- .buildkite/longruns/pipeline.yml | 23 ++++++++++++++++ .buildkite/pipeline.yml | 16 ++++++++--- .../longrun_configs/gpu_amip_target_topo.yml | 27 +++++++++++++++++++ .../gpu_amip_target_topo_diagedmf.yml | 26 ++++++++++++++++++ ...ction.yml => gpu_amip_albedo_function.yml} | 6 ++--- ...gpu_amip_target_topo_diagedmf_shortrun.yml | 26 ++++++++++++++++++ .../components/atmosphere/climaatmos_init.jl | 2 +- 7 files changed, 118 insertions(+), 8 deletions(-) create mode 100644 config/longrun_configs/gpu_amip_target_topo.yml create mode 100644 config/longrun_configs/gpu_amip_target_topo_diagedmf.yml rename config/model_configs/{gpu_target_amip_albedo_function.yml => gpu_amip_albedo_function.yml} (77%) create mode 100644 config/model_configs/gpu_amip_target_topo_diagedmf_shortrun.yml diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml index b86174513b..a11ef938bc 100644 --- a/.buildkite/longruns/pipeline.yml +++ b/.buildkite/longruns/pipeline.yml @@ -235,6 +235,29 @@ steps: slurm_nodes: 4 slurm_mem_per_cpu: 20G + - group: "Current target tests on GPU: AMIP surface" + + steps: + + - label: "GPU AMIP FINE: new target amip: topo" + key: "gpu_amip_target_topo" + command: "mpiexec julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo.yml" + artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_artifacts/*" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + slurm_gpus: 1 + + - label: "GPU AMIP FINE: new target amip: topo + diagedmf" + key: "gpu_amip_target_topo_diagedmf" + command: "mpiexec julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf.yml" + artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_artifacts/*" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + slurm_gpus: 1 - group: "Other AMIP targets" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8446229ac3..4d0e9db8af 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -380,10 +380,18 @@ steps: slurm_gpus: 1 # GPU RUNS: AMIP - - label: "GPU AMIP target: albedo from function" - key: "gpu_target_amip_albedo_function" - command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_target_amip_albedo_function.yml" - artifact_paths: "experiments/AMIP/output/amip/gpu_target_amip_albedo_function_artifacts/*" + - label: "GPU AMIP test: albedo from function" + key: "gpu_amip_albedo_function" + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_albedo_function.yml" + artifact_paths: "experiments/AMIP/output/amip/gpu_amip_albedo_function_artifacts/*" + agents: + slurm_mem: 20GB + slurm_gpus: 1 + + - label: "GPU AMIP target: topography and diagnostic EDMF" + key: "gpu_amip_target_topo_diagedmf_shortrun" + command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf_shortrun.yml" + artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts/*" agents: slurm_mem: 20GB slurm_gpus: 1 diff --git a/config/longrun_configs/gpu_amip_target_topo.yml b/config/longrun_configs/gpu_amip_target_topo.yml new file mode 100644 index 0000000000..f488e3de75 --- /dev/null +++ b/config/longrun_configs/gpu_amip_target_topo.yml @@ -0,0 +1,27 @@ +FLOAT_TYPE: "Float32" +anim: false +apply_limiter: false +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_0M_earth.yml" +dt: "100secs" +dt_cloud_fraction: "1hours" +dt_cpl: 100 +dt_rad: "1hours" +dt_save_state_to_disk: "20days" +dt_save_to_sol: "10days" +energy_check: false +hourly_checkpoint: true +job_id: "gpu_amip_target_topo" +land_albedo_type: "map_temporal" +mode_name: "amip" +mono_surface: false +netcdf_interpolate_z_over_msl: true +rad: "allskywithclear" +run_name: "gpu_amip_target_topo" +start_date: "19790301" +surface_setup: "PrescribedSurface" +t_end: "100days" +topo_smoothing: true +topography: "Earth" +turb_flux_partition: "CombinedStateFluxes" +vert_diff: "true" +use_reference_state: false diff --git a/config/longrun_configs/gpu_amip_target_topo_diagedmf.yml b/config/longrun_configs/gpu_amip_target_topo_diagedmf.yml new file mode 100644 index 0000000000..e3fea7e8ab --- /dev/null +++ b/config/longrun_configs/gpu_amip_target_topo_diagedmf.yml @@ -0,0 +1,26 @@ +FLOAT_TYPE: "Float32" +anim: false +apply_limiter: false +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equil_55km_nz63_allsky_diagedmf_0M.yml" +dt: "100secs" +dt_cloud_fraction: "1hours" +dt_cpl: 100 +dt_rad: "1hours" +dt_save_state_to_disk: "20days" +dt_save_to_sol: "10days" +energy_check: false +hourly_checkpoint: true +idealized_insolation: false +job_id: "gpu_amip_target_topo_diagedmf" +land_albedo_type: "map_temporal" +mode_name: "amip" +mono_surface: false +netcdf_output_at_levels: true +run_name: "gpu_amip_target_topo_diagedmf" +start_date: "19790301" +surface_setup: "PrescribedSurface" +t_end: "40days" +topo_smoothing: true +topography: "Earth" +turb_flux_partition: "CombinedStateFluxes" +use_reference_state: false diff --git a/config/model_configs/gpu_target_amip_albedo_function.yml b/config/model_configs/gpu_amip_albedo_function.yml similarity index 77% rename from config/model_configs/gpu_target_amip_albedo_function.yml rename to config/model_configs/gpu_amip_albedo_function.yml index 04cc3d34d1..e588f58459 100644 --- a/config/model_configs/gpu_target_amip_albedo_function.yml +++ b/config/model_configs/gpu_amip_albedo_function.yml @@ -7,15 +7,15 @@ dt_rad: "1hours" dt_save_to_sol: "1days" dz_bottom: 30 dz_top: 3000 -h_elem: 16 -job_id: "gpu_target_amip_albedo_function" +h_elem: 6 +job_id: "gpu_amip_albedo_function" land_albedo_type: "function" mode_name: "amip" moist: "equil" precip_model: "0M" rad: "gray" rayleigh_sponge: true -run_name: "gpu_target_amip_albedo_function" +run_name: "gpu_amip_albedo_function" t_end: "300secs" vert_diff: "true" z_elem: 50 diff --git a/config/model_configs/gpu_amip_target_topo_diagedmf_shortrun.yml b/config/model_configs/gpu_amip_target_topo_diagedmf_shortrun.yml new file mode 100644 index 0000000000..129e3f7409 --- /dev/null +++ b/config/model_configs/gpu_amip_target_topo_diagedmf_shortrun.yml @@ -0,0 +1,26 @@ +FLOAT_TYPE: "Float32" +anim: false +apply_limiter: false +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equil_55km_nz63_allsky_diagedmf_0M.yml" +dt: "100secs" +dt_cloud_fraction: "1hours" +dt_cpl: 100 +dt_rad: "1hours" +dt_save_state_to_disk: "1days" +dt_save_to_sol: "1days" +energy_check: false +hourly_checkpoint: true +idealized_insolation: false +job_id: "gpu_amip_target_topo_diagedmf_shortrun" +land_albedo_type: "map_temporal" +mode_name: "amip" +mono_surface: false +netcdf_output_at_levels: true +run_name: "gpu_amip_target_topo_diagedmf_shortrun" +start_date: "19790301" +surface_setup: "PrescribedSurface" +t_end: "200secs" +topo_smoothing: true +topography: "Earth" +turb_flux_partition: "CombinedStateFluxes" +use_reference_state: false diff --git a/experiments/AMIP/components/atmosphere/climaatmos_init.jl b/experiments/AMIP/components/atmosphere/climaatmos_init.jl index 724fef4aea..336d4a8fee 100644 --- a/experiments/AMIP/components/atmosphere/climaatmos_init.jl +++ b/experiments/AMIP/components/atmosphere/climaatmos_init.jl @@ -129,7 +129,7 @@ function update_field!(atmos_sim::ClimaAtmosSimulation, ::Val{:co2_gm}, field) @warn("Gray radiation model initialized, skipping CO2 update", maxlog = 1) return else - atmos_sim.integrator.p.radiation.radiation_model.volume_mixing_ratio_co2 .= parent(field)[1] + atmos_sim.integrator.p.radiation.radiation_model.volume_mixing_ratio_co2 .= mean(parent(field)) end end # extensions required by the Interfacer