Skip to content

Commit

Permalink
Remove --device CPUMultiThreaded option; make sure all CI jobs are un…
Browse files Browse the repository at this point in the history 
…threaded
  • Loading branch information
@valeriabarra
valeriabarra committed Jul 13, 2023
1 parent d76b3ed commit 967907a
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 13 deletions.
4 changes: 2 additions & 2 deletions .buildkite/longruns/pipeline.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ steps:

- label: "Moist earth with slab surface - notmono + modular: bulk gray no_sponge idealinsol freq_dt_cpl"
key: "slabplanet"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name default_modular_long --device CPUMultiThreaded --coupled true --moist equil --vert_diff true --rad gray --energy_check true --anim true --mode_name slabplanet --t_end 10days --dt_save_to_sol 1days --dt_cpl 200 --dt 200secs --mono_surface false --h_elem 4 --precip_model 0M"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name default_modular_long --coupled true --moist equil --vert_diff true --rad gray --energy_check true --anim true --mode_name slabplanet --t_end 10days --dt_save_to_sol 1days --dt_cpl 200 --dt 200secs --mono_surface false --h_elem 4 --precip_model 0M"
artifact_paths: "experiments/AMIP/modular/output/slabplanet/default_modular_long_artifacts/total_energy*.png"
env:
BUILD_HISTORY_HANDLE: ""
Expand All @@ -66,7 +66,7 @@ steps:

- label: "MPI AMIP FINE: longrun"
key: "amip_longrun"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name target_amip_n64_longrun --device CPUMultiThreaded --coupled true --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 180days --dt_save_to_sol 10days --mono_surface false --precip_model 0M"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name target_amip_n64_longrun --coupled true --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 180days --dt_save_to_sol 10days --mono_surface false --precip_model 0M"
artifact_paths: "experiments/AMIP/modular/output/amip/target_amip_n64_longrun_artifacts/*"
env:
CLIMACORE_DISTRIBUTED: "MPI"
Expand Down
20 changes: 10 additions & 10 deletions .buildkite/pipeline.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,13 +124,13 @@ steps:
# - diagnostics: check and plot energy conservation, output plots after 9 days
- label: "Slabplanet: default"
key: "slabplanet_default"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name slabplanet_default --FLOAT_TYPE Float64 --device CPUMultiThreaded --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 9days --dt_cpl 200 --dt 200secs --mono_surface true --h_elem 4 --precip_model 0M --anim true --job_id slabplanet_default"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name slabplanet_default --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 9days --dt_cpl 200 --dt 200secs --mono_surface true --h_elem 4 --precip_model 0M --anim true --job_id slabplanet_default"
artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_default_artifacts/*"

# Test: non-monotonous remapping for land mask
- label: "Slabplanet: non-monotonous surface remap"
key: "slabplanet_non-monotonous"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name slabplanet_nonmono --FLOAT_TYPE Float64 --device CPUMultiThreaded --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 9days --dt_cpl 200 --dt 200secs --mono_surface false --h_elem 4 --precip_model 0M --anim true --job_id slabplanet_nonmono"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name slabplanet_nonmono --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 9days --dt_cpl 200 --dt 200secs --mono_surface false --h_elem 4 --precip_model 0M --anim true --job_id slabplanet_nonmono"
artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_nonmono_artifacts/*"


Expand Down Expand Up @@ -172,32 +172,32 @@ steps:
# artifact_paths: "test/*"

- label: "Moist earth with slab surface - default: monin gray no_sponge idealinsol freq_dt_cpl"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --device CPUMultiThreaded --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 200 --dt 200secs --mono_surface true --h_elem 4 --precip_model 0M --run_name default_mono --job_id default_mono"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 200 --dt 200secs --mono_surface true --h_elem 4 --precip_model 0M --run_name default_mono --job_id default_mono"
artifact_paths: "experiments/AMIP/modular/output/slabplanet/default_mono_artifacts/total_energy*.png"

- label: "Moist earth with slab surface - default: monin gray no_sponge idealinsol freq_dt_cpl - bucket using BulkAlbedoFunction"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --device CPUMultiThreaded --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 200 --dt 200secs --mono_surface true --h_elem 4 --precip_model 0M --albedo_from_file false --run_name default_albedo_prescribed --job_id default_albedo_from_file"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 200 --dt 200secs --mono_surface true --h_elem 4 --precip_model 0M --albedo_from_file false --run_name default_albedo_prescribed --job_id default_albedo_from_file"
artifact_paths: "experiments/AMIP/modular/output/slabplanet/default_albedo_from_file_artifacts/total_energy*.png"

- label: "Moist earth with slab surface - notmono: monin gray no_sponge idealinsol freq_dt_cpl notmono"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --device CPUMultiThreaded --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 200 --dt 200secs --mono_surface false --h_elem 4 --precip_model 0M --run_name default_notmono --job_id default_notmono"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 200 --dt 200secs --mono_surface false --h_elem 4 --precip_model 0M --run_name default_notmono --job_id default_notmono"
artifact_paths: "experiments/AMIP/modular/output/slabplanet/default_notmono_artifacts/total_energy*.png"

# Note: this test fails when run with the more realistic albedo from file
- label: "Moist earth with slab surface - target: monin allsky sponge realinsol infreq_dt_cpl - bucket using BulkAlbedoFunction"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --run_name target_params_in_slab --device CPUMultiThreaded --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 3600 --dt 200secs --dt_rad 6hours --idealized_insolation false --mono_surface true --h_elem 6 --precip_model 0M --albedo_from_file false --job_id target_params_in_slab" # idealized albedo breaks
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --run_name target_params_in_slab --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 3600 --dt 200secs --dt_rad 6hours --idealized_insolation false --mono_surface true --h_elem 6 --precip_model 0M --albedo_from_file false --job_id target_params_in_slab" # idealized albedo breaks
artifact_paths: "experiments/AMIP/modular/output/slabplanet/target_params_in_slab_artifacts/total_energy*.png"

# - label: "Moist earth with slab surface - test: monin allsky sponge idealinsol infreq_dt_cpl"
# command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --device CPUMultiThreaded --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 21600 --dt 200secs --dt_rad 6hours --mono_surface true --h_elem 4 --precip_model 0M --run_name target_params_in_slab_test1 --job_id target_params_in_slab_test1" # Unconverged SF (reproduced locally); works with 200s dt_cpl
# artifact_paths: "experiments/AMIP/modular/output/slabplanet/target_params_in_slab_test1_artifacts/total_energy*.png"

- label: "Moist earth with slab surface - test: bulk allsky sponge realinsol infreq_dt_cpl"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --device CPUMultiThreaded --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 3600 --dt 200secs --dt_rad 6hours --idealized_insolation false --mono_surface true --h_elem 6 --precip_model 0M --run_name target_params_in_slab_test2 --job_id target_params_in_slab_test2"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 3600 --dt 200secs --dt_rad 6hours --idealized_insolation false --mono_surface true --h_elem 6 --precip_model 0M --run_name target_params_in_slab_test2 --job_id target_params_in_slab_test2"
artifact_paths: "experiments/AMIP/modular/output/slabplanet/target_params_in_slab_test2_artifacts/total_energy*.png"

- label: "Moist earth with slab surface - test: monin gray sponge realinsol infreq_dt_cpl"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --device CPUMultiThreaded --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 3600 --dt 200secs --dt_rad 6hours --idealized_insolation false --mono_surface true --h_elem 6 --precip_model 0M --run_name target_params_in_slab_test3 --job_id target_params_in_slab_test3"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 3600 --dt 200secs --dt_rad 6hours --idealized_insolation false --mono_surface true --h_elem 6 --precip_model 0M --run_name target_params_in_slab_test3 --job_id target_params_in_slab_test3"
artifact_paths: "experiments/AMIP/modular/output/slabplanet/target_params_in_slab_test3_artifacts/total_energy*.png"

# breaking:
Expand All @@ -207,7 +207,7 @@ steps:

- label: "AMIP - modular Float64"
key: "modular_amip"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name coarse_single_modular --FLOAT_TYPE Float64 --device CPUMultiThreaded --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check false --mode_name amip --anim true --t_end 32days --dt_save_to_sol 1days --dt_cpl 400 --dt 400secs --mono_surface false --h_elem 6 --dt_save_restart 10days --precip_model 0M --job_id coarse_single_modular"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name coarse_single_modular --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check false --mode_name amip --anim true --t_end 32days --dt_save_to_sol 1days --dt_cpl 400 --dt 400secs --mono_surface false --h_elem 6 --dt_save_restart 10days --precip_model 0M --job_id coarse_single_modular"
artifact_paths: "experiments/AMIP/modular/output/amip/coarse_single_modular_artifacts/*"
env:
FLAME_PLOT: ""
Expand All @@ -217,7 +217,7 @@ steps:
slurm_mem: 20GB

- label: "AMIP - modular, Float32 test" # Issue #271
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name coarse_single_modular_ft32 --FLOAT_TYPE Float64 --device CPUMultiThreaded --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check false --mode_name amip --anim true --t_end 10days --dt_save_to_sol 100days --dt_cpl 200 --dt 200secs --mono_surface true --h_elem 6 --dt_save_restart 10days --precip_model 0M --job_id coarse_single_modular_ft32"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name coarse_single_modular_ft32 --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check false --mode_name amip --anim true --t_end 10days --dt_save_to_sol 100days --dt_cpl 200 --dt 200secs --mono_surface true --h_elem 6 --dt_save_restart 10days --precip_model 0M --job_id coarse_single_modular_ft32"
artifact_paths: "experiments/AMIP/modular/output/amip/coarse_single_modular_ft32_artifacts/*"
agents:
slurm_ntasks: 1
Expand Down
2 changes: 1 addition & 1 deletion experiments/AMIP/modular/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ $$

# Distribution
## Julia multithreading
- we can run using multiple threads if the command line argument `--device` is set to `CPUMultiThreaded`.
- we can run using multiple threads if the command line argument `--device` is set to `CPUMultiThreaded` and the number of threads `n` is specified with `--threads n`.

## MPI via ClimaComms.jl
- for AMIP we want the surface columns to be on the same processors as the atmos columns. Since all all surface domains will be on masked spheres, all of them can inherit the same distributed horizontal space from the atmos model.
Expand Down

0 comments on commit 967907a

Please sign in to comment.