From e4bbc3008ab4a6fd0ea91bf184f589070d930849 Mon Sep 17 00:00:00 2001 From: Julia Sloan Date: Tue, 1 Oct 2024 16:56:21 -0700 Subject: [PATCH] make artifacts subdir of output --- .buildkite/amip/pipeline.yml | 8 +-- .buildkite/benchmarks/pipeline.yml | 16 ++--- .buildkite/longruns/pipeline.yml | 38 +++++------ .buildkite/nightly/pipeline.yml | 8 +-- .buildkite/pipeline.yml | 66 ++++++++++---------- experiments/ClimaEarth/run_amip.jl | 2 + experiments/ClimaEarth/user_io/io_helpers.jl | 2 +- 7 files changed, 71 insertions(+), 69 deletions(-) diff --git a/.buildkite/amip/pipeline.yml b/.buildkite/amip/pipeline.yml index 5b35e999d0..c4497aa656 100644 --- a/.buildkite/amip/pipeline.yml +++ b/.buildkite/amip/pipeline.yml @@ -45,7 +45,7 @@ steps: command: - echo "--- Run simulation" - "srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip.yml --job_id amip" - artifact_paths: "experiments/ClimaEarth/output/amip/amip_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/amip/artifacts/*" timeout_in_minutes: 5760 env: CLIMACOMMS_DEVICE: "CUDA" @@ -64,9 +64,9 @@ steps: - label: ":envelope: Slack report: current AMIP" command: - - for file in $(find experiments/ClimaEarth/output/amip/amip_artifacts/ -regextype posix-extended -regex '.*/bias_[A-Z]{3}\.png'); do label=$(basename "$file" | grep -oP '(?<=bias_)[A-Za-z]{3}'); convert "$file" -gravity North -pointsize 50 -annotate +0+0 "$label" "experiments/ClimaEarth/output/amip/amip_artifacts/tmp_bias_$label.png"; done - - convert +append "experiments/ClimaEarth/output/amip/amip_artifacts/tmp_bias_???.png" experiments/ClimaEarth/output/amip/amip_artifacts/bias_all_seasons.png + - for file in $(find experiments/ClimaEarth/output/amip/amip/artifacts/ -regextype posix-extended -regex '.*/bias_[A-Z]{3}\.png'); do label=$(basename "$file" | grep -oP '(?<=bias_)[A-Za-z]{3}'); convert "$file" -gravity North -pointsize 50 -annotate +0+0 "$label" "experiments/ClimaEarth/output/amip/amip/artifacts/tmp_bias_$label.png"; done + - convert +append "experiments/ClimaEarth/output/amip/amip/artifacts/tmp_bias_???.png" experiments/ClimaEarth/output/amip/amip/artifacts/bias_all_seasons.png - | - find experiments/ClimaEarth/output/amip/amip_artifacts/ -type f -name 'bias*.png' -print0 | while IFS= read -r -d '' file; do + find experiments/ClimaEarth/output/amip/amip/artifacts/ -type f -name 'bias*.png' -print0 | while IFS= read -r -d '' file; do slack-upload -c "#coupler-report" -f "$$file" -m png -n "$$(basename "$$file" .png)" -x "$$(basename "$$file" .png)" done diff --git a/.buildkite/benchmarks/pipeline.yml b/.buildkite/benchmarks/pipeline.yml index e14d809f4f..4382d245f9 100644 --- a/.buildkite/benchmarks/pipeline.yml +++ b/.buildkite/benchmarks/pipeline.yml @@ -45,7 +45,7 @@ steps: - label: "CPU ClimaAtmos without diagnostic EDMF" key: "climaatmos" command: "srun julia --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $BENCHMARK_CONFIG_PATH/climaatmos.yml --job_id climaatmos" - artifact_paths: "experiments/ClimaEarth/output/climaatmos/climaatmos_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/climaatmos/climaatmos/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" BUILD_HISTORY_HANDLE: "" @@ -58,7 +58,7 @@ steps: - label: "CPU ClimaAtmos with diagnostic EDMF" key: "climaatmos_diagedmf" command: "srun julia --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $BENCHMARK_CONFIG_PATH/climaatmos_diagedmf.yml --job_id climaatmos_diagedmf" - artifact_paths: "experiments/ClimaEarth/output/climaatmos/climaatmos_diagedmf_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/climaatmos/climaatmos_diagedmf/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" BUILD_HISTORY_HANDLE: "" @@ -71,7 +71,7 @@ steps: - label: "CPU AMIP with diagnostic EDMF" key: "amip_diagedmf" command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf.yml --job_id amip_diagedmf" - artifact_paths: "experiments/ClimaEarth/output/amip/amip_diagedmf_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/amip_diagedmf/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" BUILD_HISTORY_HANDLE: "" @@ -84,7 +84,7 @@ steps: - label: "CPU AMIP with diagnostic EDMF and io" key: "amip_diagedmf_io" command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf_io.yml --job_id amip_diagedmf_io" - artifact_paths: "experiments/ClimaEarth/output/amip/amip_diagedmf_io_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/amip_diagedmf_io/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" BUILD_HISTORY_HANDLE: "" @@ -99,7 +99,7 @@ steps: - label: "GPU ClimaAtmos without diagnostic EDMF" key: "gpu_climaatmos" command: "srun julia --threads=3 --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $BENCHMARK_CONFIG_PATH/climaatmos.yml --job_id gpu_climaatmos" - artifact_paths: "experiments/ClimaEarth/output/climaatmos/gpu_climaatmos_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/climaatmos/gpu_climaatmos/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" CLIMACOMMS_DEVICE: "CUDA" @@ -112,7 +112,7 @@ steps: - label: "GPU ClimaAtmos with diagnostic EDMF" key: "gpu_climaatmos_diagedmf" command: "srun julia --threads=3 --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $BENCHMARK_CONFIG_PATH/climaatmos_diagedmf.yml --job_id gpu_climaatmos_diagedmf" - artifact_paths: "experiments/ClimaEarth/output/climaatmos/gpu_climaatmos_diagedmf_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/climaatmos/gpu_climaatmos_diagedmf/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" CLIMACOMMS_DEVICE: "CUDA" @@ -125,7 +125,7 @@ steps: - label: "GPU AMIP with diagnostic EDMF" key: "gpu_amip_diagedmf" command: "srun julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf.yml --job_id gpu_amip_diagedmf" - artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_diagedmf_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_diagedmf/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" CLIMACOMMS_DEVICE: "CUDA" @@ -138,7 +138,7 @@ steps: - label: "GPU AMIP with diagnostic EDMF and IO" key: "gpu_amip_diagedmf_io" command: "srun julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf_io.yml --job_id gpu_amip_diagedmf_io" - artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_diagedmf_io_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_diagedmf_io/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" CLIMACOMMS_DEVICE: "CUDA" diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml index 0df5d75237..38c54b6461 100644 --- a/.buildkite/longruns/pipeline.yml +++ b/.buildkite/longruns/pipeline.yml @@ -73,7 +73,7 @@ steps: - label: "Slabplanet_aqua: nocouple" key: "slabplanet_aqua_atmos_sf_nocouple" # SF at each Atmos stage, no coupling, prescribed SST from atmos command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml --job_id slabplanet_aqua_atmos_sf_nocouple" - artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple/artifacts/*" env: BUILD_HISTORY_HANDLE: "" agents: @@ -85,7 +85,7 @@ steps: - label: "Slabplanet_aqua: couple" key: "slabplanet_aqua_atmos_sf_couple" # SF at each Atmos stage, coupling, prescribed SST from coupler - identical results to the above confirm 1) initial conditions in Atmos are unchanged compared to the slab, 2) coupling not introducing variability when constant surface command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml --job_id slabplanet_aqua_atmos_sf_couple" - artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple/artifacts/*" env: BUILD_HISTORY_HANDLE: "" agents: @@ -97,7 +97,7 @@ steps: - label: "Slabplanet_aqua: coupler fluxes" key: "slabplanet_aqua_coupler_sf" # SF at each coupler timestep, constant ocean - comparing to the above runs, this tests the sensitivity of less frequent flux calculation command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml --job_id slabplanet_aqua_coupler_sf" - artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_coupler_sf/artifacts/*" env: BUILD_HISTORY_HANDLE: "" agents: @@ -109,7 +109,7 @@ steps: - label: "Slabplanet_aqua: coupler fluxes, evolving ocean" key: "slabplanet_aqua_coupler_sf_evolve_ocn" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving ocean command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml --job_id slabplanet_aqua_coupler_sf_evolve_ocn" - artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn/artifacts/*" env: BUILD_HISTORY_HANDLE: "" agents: @@ -121,7 +121,7 @@ steps: - label: "Slabplanet_terra: coupler fluxes, evolving bucket" key: "slabplanet_terra" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving bucket command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_terra.yml --job_id slabplanet_terra" - artifact_paths: "experiments/ClimaEarth/output/slabplanet_terra/slabplanet_terra_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet_terra/slabplanet_terra/artifacts/*" env: BUILD_HISTORY_HANDLE: "" agents: @@ -133,7 +133,7 @@ steps: - label: "Slabplanet: coupler fluxes, evolving ocean and land" key: "slabplanet_coupler_sf_evolve_ocn" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml --job_id slabplanet_coupler_sf_evolve_ocn" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn/artifacts/*" env: BUILD_HISTORY_HANDLE: "" agents: @@ -162,7 +162,7 @@ steps: - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, nocouple, atmos flux calc" key: "slabplanet_aqua_target_nocouple" command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_nocouple.yml --job_id slabplanet_aqua_target_nocouple" - artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_nocouple_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_nocouple/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" BUILD_HISTORY_HANDLE: "" @@ -175,7 +175,7 @@ steps: - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, coupler flux calc" key: "slabplanet_aqua_target" command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_target.yml --job_id slabplanet_aqua_target" - artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" BUILD_HISTORY_HANDLE: "" @@ -188,7 +188,7 @@ steps: - label: "TARGET IDEALIZED: new target aqua - evolving slab ocean T" key: "slabplanet_aqua_target_evolve_ocn" command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_evolve_ocn.yml --job_id slabplanet_aqua_target_evolve_ocn" - artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_evolve_ocn_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_evolve_ocn/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" BUILD_HISTORY_HANDLE: "" @@ -201,7 +201,7 @@ steps: - label: "TARGET IDEALIZED: new target slab - fixed ocean T, bucket" key: "slabplanet_target" command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_target.yml --job_id slabplanet_target" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_target_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_target/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" BUILD_HISTORY_HANDLE: "" @@ -214,7 +214,7 @@ steps: - label: "TARGET IDEALIZED: new target slab - evolving slab ocean T, bucket" key: "slabplanet_target_evolve_ocn" command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_target_evolve_ocn.yml --job_id slabplanet_target_evolve_ocn" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_target_evolve_ocn_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_target_evolve_ocn/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" BUILD_HISTORY_HANDLE: "" @@ -231,7 +231,7 @@ steps: - label: "MPI AMIP FINE: new target amip" key: "amip_target" command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target.yml --job_id amip_target" - artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/amip_target/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" BUILD_HISTORY_HANDLE: "" @@ -244,7 +244,7 @@ steps: - label: "MPI AMIP FINE: new target amip: topo" key: "amip_target_topo" command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo.yml --job_id amip_target_topo" - artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" BUILD_HISTORY_HANDLE: "" @@ -257,7 +257,7 @@ steps: - label: "MPI AMIP FINE: new target amip: topo + diagedmf" key: "amip_target_topo_diagedmf_cpu" command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf_cpu.yml --job_id amip_target_topo_diagedmf_cpu" - artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_cpu_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_cpu/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" BUILD_HISTORY_HANDLE: "" @@ -274,7 +274,7 @@ steps: - label: "GPU AMIP FINE: new target amip: topo" key: "amip_target_topo_gpu" command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo.yml --job_id amip_target_topo_gpu" - artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_gpu_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_gpu/artifacts/*" env: CLIMACOMMS_CONTEXT: "MPI" CLIMACOMMS_DEVICE: "CUDA" @@ -288,7 +288,7 @@ steps: command: - echo "--- Run simulation" - "srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf_gpu.yml --job_id amip_target_topo_diagedmf_gpu" - artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_gpu_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_gpu/artifacts/*" timeout_in_minutes: 1440 env: CLIMACOMMS_CONTEXT: "MPI" @@ -308,7 +308,7 @@ steps: command: - echo "--- Run simulation" - "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/longrun_amip_dyamond.yml --job_id longrun_amip_dyamond_gpu" - artifact_paths: "experiments/ClimaEarth/output/amip/longrun_amip_dyamond_gpu_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/longrun_amip_dyamond_gpu/artifacts/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: @@ -335,5 +335,5 @@ steps: - label: ":envelope: Slack report: Slabplanet" command: - - slack-upload -c "#coupler-report" -f experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation" - - slack-upload -c "#coupler-report" -f experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation" + - slack-upload -c "#coupler-report" -f experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn/artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation" + - slack-upload -c "#coupler-report" -f experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn/artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation" diff --git a/.buildkite/nightly/pipeline.yml b/.buildkite/nightly/pipeline.yml index bbbd2d2353..2b043f25cf 100644 --- a/.buildkite/nightly/pipeline.yml +++ b/.buildkite/nightly/pipeline.yml @@ -55,7 +55,7 @@ steps: command: - echo "--- Run simulation" - "julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_coarse.yml --job_id amip_coarse" - artifact_paths: "experiments/ClimaEarth/output/amip/amip_coarse_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/amip_coarse/artifacts/*" timeout_in_minutes: 840 env: CLIMACOMMS_DEVICE: "CUDA" @@ -70,7 +70,7 @@ steps: command: - echo "--- Run simulation" - "julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_coarse_random.yml --job_id amip_coarse_random1" - artifact_paths: "experiments/ClimaEarth/output/amip/amip_coarse_random1_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/amip_coarse_random1/artifacts/*" timeout_in_minutes: 1080 env: CLIMACOMMS_DEVICE: "CUDA" @@ -85,7 +85,7 @@ steps: command: - echo "--- Run simulation" - "julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_coarse_random.yml --job_id amip_coarse_random2" - artifact_paths: "experiments/ClimaEarth/output/amip/amip_coarse_random2_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/amip_coarse_random2/artifacts/*" timeout_in_minutes: 1080 env: CLIMACOMMS_DEVICE: "CUDA" @@ -100,7 +100,7 @@ steps: command: - echo "--- Run simulation" - "julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_coarse_random.yml --job_id amip_coarse_random3" - artifact_paths: "experiments/ClimaEarth/output/amip/amip_coarse_random3_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/amip_coarse_random3/artifacts/*" timeout_in_minutes: 1080 env: CLIMACOMMS_DEVICE: "CUDA" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4ebd9c31a6..25b863cb8b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -146,70 +146,70 @@ steps: - label: "Slabplanet: default" key: "slabplanet_default" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_default.yml --job_id slabplanet_default" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_default_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_default/artifacts/*" agents: slurm_mem: 20GB - label: "Slabplanet: dry, no radiation" key: "slabplanet_dry_norad" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_dry_norad.yml --job_id slabplanet_dry_norad" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_dry_norad_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_dry_norad/artifacts/*" agents: slurm_mem: 20GB - label: "Slabplanet: default with Float32" key: "slabplanet_ft32" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_ft32.yml --job_id slabplanet_ft32" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_ft32_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_ft32/artifacts/*" agents: slurm_mem: 20GB - label: "Slabplanet: partitioned turbulent fluxes" key: "slabplanet_partitioned_fluxes" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_partitioned_fluxes.yml --job_id slabplanet_partitioned_fluxes" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_partitioned_fluxes_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_partitioned_fluxes/artifacts/*" agents: slurm_mem: 20GB - label: "Slabplanet: non-monotonous surface remap" key: "slabplanet_nonmono" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_nonmono.yml --job_id slabplanet_nonmono" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_nonmono_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_nonmono/artifacts/*" agents: slurm_mem: 20GB - label: "Slabplanet: albedo from static map" key: "slabplanet_albedo_static_map" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_albedo_static_map.yml --job_id slabplanet_albedo_static_map" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_albedo_static_map_artifacts/total_energy*.png" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_albedo_static_map/artifacts/total_energy*.png" agents: slurm_mem: 20GB - label: "Slabplanet: albedo from temporal map" key: "slabplanet_albedo_temporal_map" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_albedo_temporal_map.yml --job_id slabplanet_albedo_temporal_map" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_albedo_temporal_map_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_albedo_temporal_map/artifacts/*" agents: slurm_mem: 20GB - label: "Slabplanet: albedo from function" key: "slabplanet_albedo_function" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_albedo_function.yml --job_id slabplanet_albedo_function" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_albedo_function_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_albedo_function/artifacts/*" agents: slurm_mem: 20GB - label: "Slabplanet: eisenman sea ice" key: "slabplanet_eisenman" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_eisenman.yml --job_id slabplanet_eisenman" - artifact_paths: "experiments/ClimaEarth/output/slabplanet_eisenman/slabplanet_eisenman_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet_eisenman/slabplanet_eisenman/artifacts/*" agents: slurm_mem: 20GB - label: "Slabplanet: extra atmos diagnostics" key: "slabplanet_atmos_diags" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_atmos_diags.yml --job_id slabplanet_atmos_diags" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_atmos_diags_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_atmos_diags/artifacts/*" agents: slurm_mem: 20GB @@ -224,7 +224,7 @@ steps: - label: ":rocket: Slabplanet: default (unthreaded)" key: "slabplanet_unthreaded" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/default_unthreaded.yml --job_id default_unthreaded" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/default_unthreaded_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/default_unthreaded/artifacts/*" env: FLAME_PLOT: "" BUILD_HISTORY_HANDLE: "" @@ -256,7 +256,7 @@ steps: - label: "heat-diffusion" command: "julia --color=yes --project=experiments/ClimaCore/ experiments/ClimaCore/heat-diffusion/run.jl" - artifact_paths: "experiments/ClimaCore/output/heat-diffusion_artifacts/*" + artifact_paths: "experiments/ClimaCore/output/heat-diffusion/artifacts/*" agents: slurm_mem: 20GB @@ -264,48 +264,48 @@ steps: - label: "Moist earth with slab surface - default: monin gray no_sponge idealinsol freq_dt_cpl" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/default_mono.yml --job_id default_mono" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/default_mono_artifacts/total_energy*.png" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/default_mono/artifacts/total_energy*.png" agents: slurm_mem: 20GB - label: "Moist earth with slab surface - notmono: monin gray no_sponge idealinsol freq_dt_cpl notmono" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/default_notmono.yml --job_id default_notmono" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/default_notmono_artifacts/total_energy*.png" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/default_notmono/artifacts/total_energy*.png" agents: slurm_mem: 20GB # - label: "Moist earth with slab surface - test: monin allsky sponge idealinsol infreq_dt_cpl" # command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 21600 --dt 200secs --dt_rad 6hours --mono_surface true --h_elem 4 --precip_model 0M --job_id target_params_in_slab_test1" # Unconverged SF (reproduced locally); works with 200s dt_cpl - # artifact_paths: "experiments/ClimaEarth/output/slabplanet/target_params_in_slab_test1_artifacts/total_energy*.png" + # artifact_paths: "experiments/ClimaEarth/output/slabplanet/target_params_in_slab_test1/artifacts/total_energy*.png" - label: "Moist earth with slab surface - test: bulk allsky sponge realinsol infreq_dt_cpl" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/target_params_in_slab_test2.yml --job_id target_params_in_slab_test2" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/target_params_in_slab_test2_artifacts/total_energy*.png" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/target_params_in_slab_test2/artifacts/total_energy*.png" agents: slurm_mem: 20GB - label: "Moist earth with slab surface - test: monin gray sponge realinsol infreq_dt_cpl" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/target_params_in_slab_test3.yml --job_id target_params_in_slab_test3" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/target_params_in_slab_test3_artifacts/total_energy*.png" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/target_params_in_slab_test3/artifacts/total_energy*.png" agents: slurm_mem: 20GB # breaking: # - label: "Moist earth with slab surface - monin allsky no_sponge idealinsol infreq_dt_cpl" # command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad allskywithclear --rayleigh_sponge false --energy_check true --mode_name slabplanet --t_end 10days --dt_save_to_sol 3600secs --dt_cpl 21600 --dt 200secs --dt_rad 6hours --idealized_insolation true --mono_surface true --h_elem 4 --precip_model 0M --job_id target_params_in_slab1" - # artifact_paths: "experiments/ClimaEarth/output/slabplanet/target_params_in_slab1_artifacts/total_energy*.png" + # artifact_paths: "experiments/ClimaEarth/output/slabplanet/target_params_in_slab1/artifacts/total_energy*.png" - label: "AMIP target: albedo from function" key: "target_amip_albedo_function" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/target_amip_albedo_function.yml --job_id target_amip_albedo_function" - artifact_paths: "experiments/ClimaEarth/output/amip/target_amip_albedo_function_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/target_amip_albedo_function/artifacts/*" agents: slurm_mem: 20GB - label: "AMIP - Float64 + hourly checkpoint" key: "amip" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/coarse_single_ft64_hourly_checkpoints.yml --job_id coarse_single_ft64_hourly_checkpoints" - artifact_paths: "experiments/ClimaEarth/output/amip/coarse_single_ft64_hourly_checkpoints_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/coarse_single_ft64_hourly_checkpoints/artifacts/*" env: FLAME_PLOT: "" BUILD_HISTORY_HANDLE: "" @@ -315,21 +315,21 @@ steps: - label: "AMIP - Float64 test" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/coarse_single_ft64.yml --job_id coarse_single_ft64" - artifact_paths: "experiments/ClimaEarth/output/amip/coarse_single_ft64_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/coarse_single_ft64/artifacts/*" agents: slurm_ntasks: 1 slurm_mem: 20GB - label: "AMIP - Float32 test" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/coarse_single_ft32.yml --job_id coarse_single_ft32" - artifact_paths: "experiments/ClimaEarth/output/amip/coarse_single_ft32_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/coarse_single_ft32/artifacts/*" agents: slurm_ntasks: 1 slurm_mem: 20GB - label: "MPI AMIP" command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/coarse_mpi_n4.yml --job_id coarse_mpi_n4" - artifact_paths: "experiments/ClimaEarth/output/amip/coarse_mpi_n4_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/coarse_mpi_n4/artifacts/*" timeout_in_minutes: 240 env: CLIMACOMMS_CONTEXT: "MPI" @@ -342,7 +342,7 @@ steps: - label: "Unthreaded AMIP FINE" # also reported by longruns with a flame graph key: "unthreaded_amip_fine" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/target_amip_n1_shortrun.yml --job_id target_amip_n1_shortrun" - artifact_paths: "experiments/ClimaEarth/output/amip/target_amip_n1_shortrun_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/target_amip_n1_shortrun/artifacts/*" env: BUILD_HISTORY_HANDLE: "" agents: @@ -424,7 +424,7 @@ steps: - label: "GPU Slabplanet: albedo from function" key: "gpu_slabplanet_albedo_function" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_albedo_function.yml --job_id gpu_slabplanet_albedo_function" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/gpu_slabplanet_albedo_function_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/gpu_slabplanet_albedo_function/artifacts/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: @@ -434,7 +434,7 @@ steps: - label: "GPU Slabplanet: albedo from static map" key: "gpu_slabplanet_albedo_static_map" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_albedo_static_map.yml --job_id gpu_slabplanet_albedo_static_map" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/gpu_slabplanet_albedo_static_map_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/gpu_slabplanet_albedo_static_map/artifacts/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: @@ -444,7 +444,7 @@ steps: - label: "GPU Slabplanet: albedo from temporal map" key: "gpu_slabplanet_albedo_temporal_map" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_albedo_temporal_map.yml --job_id gpu_slabplanet_albedo_temporal_map" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/gpu_slabplanet_albedo_temporal_map_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/gpu_slabplanet_albedo_temporal_map/artifacts/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: @@ -454,7 +454,7 @@ steps: - label: "GPU Slabplanet: extra atmos diagnostics" key: "gpu_slabplanet_atmos_diags" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_atmos_diags.yml --job_id gpu_slabplanet_atmos_diags" - artifact_paths: "experiments/ClimaEarth/output/slabplanet/gpu_slabplanet_atmos_diags_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/slabplanet/gpu_slabplanet_atmos_diags/artifacts/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: @@ -465,7 +465,7 @@ steps: - label: "GPU AMIP test: albedo from function" key: "gpu_amip_albedo_function" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/gpu_amip_albedo_function.yml --job_id gpu_amip_albedo_function" - artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_albedo_function_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_albedo_function/artifacts/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: @@ -475,7 +475,7 @@ steps: - label: "GPU AMIP target: topography and diagnostic EDMF" key: "gpu_amip_target_topo_diagedmf_shortrun" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf_shortrun.yml --job_id gpu_amip_target_topo_diagedmf_shortrun" - artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_target_topo_diagedmf_shortrun_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_target_topo_diagedmf_shortrun/artifacts/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: @@ -485,7 +485,7 @@ steps: - label: "GPU AMIP: albedo from static map" key: "gpu_amip_albedo_static_map" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/gpu_amip_albedo_static_map.yml --job_id gpu_amip_albedo_static_map" - artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_albedo_static_map_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_albedo_static_map/artifacts/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: @@ -495,7 +495,7 @@ steps: - label: "GPU AMIP: albedo from temporal map + 0M" key: "gpu_amip_albedo_temporal_map" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/gpu_amip_albedo_temporal_map.yml --job_id gpu_amip_albedo_temporal_map" - artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_albedo_temporal_map_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_albedo_temporal_map/artifacts/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: @@ -505,7 +505,7 @@ steps: - label: "GPU AMIP: albedo from temporal map + 1M" key: "gpu_amip_albedo_temporal_map_1M" command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/gpu_amip_albedo_temporal_map_1M.yml --job_id gpu_amip_albedo_temporal_map_1M" - artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_albedo_temporal_map_1M_artifacts/*" + artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_albedo_temporal_map_1M/artifacts/*" env: CLIMACOMMS_DEVICE: "CUDA" agents: diff --git a/experiments/ClimaEarth/run_amip.jl b/experiments/ClimaEarth/run_amip.jl index 746cc92321..ebda26adee 100644 --- a/experiments/ClimaEarth/run_amip.jl +++ b/experiments/ClimaEarth/run_amip.jl @@ -202,6 +202,8 @@ temporary files will be saved. COUPLER_OUTPUT_DIR = joinpath(config_dict["coupler_output_dir"], joinpath(mode_name, job_id)) dir_paths = setup_output_dirs(output_dir = COUPLER_OUTPUT_DIR, comms_ctx = comms_ctx) +@info "Coupler output directory $(dir_paths.output)" +@info "Coupler artifacts directory $(dir_paths.artifacts)" @info(dir_paths.output) config_dict["print_config_dict"] && @info(config_dict) diff --git a/experiments/ClimaEarth/user_io/io_helpers.jl b/experiments/ClimaEarth/user_io/io_helpers.jl index 03642dfe9f..ae50e51380 100644 --- a/experiments/ClimaEarth/user_io/io_helpers.jl +++ b/experiments/ClimaEarth/user_io/io_helpers.jl @@ -21,7 +21,7 @@ function setup_output_dirs(; output_dir = nothing, artifacts_dir = nothing, comm output_dir = "." end if artifacts_dir === nothing - artifacts_dir = output_dir * "_artifacts" + artifacts_dir = joinpath(output_dir, "artifacts") end @info(output_dir)