make artifacts subdir of output

CliMA · Oct 16, 2024 · e4bbc30 · e4bbc30
1 parent a44f9e0
commit e4bbc30
Show file tree

Hide file tree

Showing 7 changed files with 71 additions and 69 deletions.
diff --git a/.buildkite/amip/pipeline.yml b/.buildkite/amip/pipeline.yml
@@ -45,7 +45,7 @@ steps:
         command:
           - echo "--- Run simulation"
           - "srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip.yml --job_id amip"
-        artifact_paths: "experiments/ClimaEarth/output/amip/amip_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/amip/amip/artifacts/*"
         timeout_in_minutes: 5760
         env:
           CLIMACOMMS_DEVICE: "CUDA"
@@ -64,9 +64,9 @@ steps:
 
       - label: ":envelope: Slack report: current AMIP"
         command:
-          - for file in $(find experiments/ClimaEarth/output/amip/amip_artifacts/ -regextype posix-extended -regex '.*/bias_[A-Z]{3}\.png'); do label=$(basename "$file" | grep -oP '(?<=bias_)[A-Za-z]{3}'); convert "$file" -gravity North -pointsize 50 -annotate +0+0 "$label" "experiments/ClimaEarth/output/amip/amip_artifacts/tmp_bias_$label.png"; done
-          - convert +append "experiments/ClimaEarth/output/amip/amip_artifacts/tmp_bias_???.png" experiments/ClimaEarth/output/amip/amip_artifacts/bias_all_seasons.png
+          - for file in $(find experiments/ClimaEarth/output/amip/amip/artifacts/ -regextype posix-extended -regex '.*/bias_[A-Z]{3}\.png'); do label=$(basename "$file" | grep -oP '(?<=bias_)[A-Za-z]{3}'); convert "$file" -gravity North -pointsize 50 -annotate +0+0 "$label" "experiments/ClimaEarth/output/amip/amip/artifacts/tmp_bias_$label.png"; done
+          - convert +append "experiments/ClimaEarth/output/amip/amip/artifacts/tmp_bias_???.png" experiments/ClimaEarth/output/amip/amip/artifacts/bias_all_seasons.png
           - |
-            find experiments/ClimaEarth/output/amip/amip_artifacts/ -type f -name 'bias*.png' -print0 | while IFS= read -r -d '' file; do
+            find experiments/ClimaEarth/output/amip/amip/artifacts/ -type f -name 'bias*.png' -print0 | while IFS= read -r -d '' file; do
               slack-upload -c "#coupler-report" -f "$$file" -m png -n "$$(basename "$$file" .png)" -x "$$(basename "$$file" .png)"
             done
diff --git a/.buildkite/benchmarks/pipeline.yml b/.buildkite/benchmarks/pipeline.yml
@@ -45,7 +45,7 @@ steps:
       - label: "CPU ClimaAtmos without diagnostic EDMF"
         key: "climaatmos"
         command: "srun julia --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $BENCHMARK_CONFIG_PATH/climaatmos.yml --job_id climaatmos"
-        artifact_paths: "experiments/ClimaEarth/output/climaatmos/climaatmos_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/climaatmos/climaatmos/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -58,7 +58,7 @@ steps:
       - label: "CPU ClimaAtmos with diagnostic EDMF"
         key: "climaatmos_diagedmf"
         command: "srun julia --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $BENCHMARK_CONFIG_PATH/climaatmos_diagedmf.yml --job_id climaatmos_diagedmf"
-        artifact_paths: "experiments/ClimaEarth/output/climaatmos/climaatmos_diagedmf_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/climaatmos/climaatmos_diagedmf/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -71,7 +71,7 @@ steps:
       - label: "CPU AMIP with diagnostic EDMF"
         key: "amip_diagedmf"
         command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf.yml --job_id amip_diagedmf"
-        artifact_paths: "experiments/ClimaEarth/output/amip/amip_diagedmf_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/amip/amip_diagedmf/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -84,7 +84,7 @@ steps:
       - label: "CPU AMIP with diagnostic EDMF and io"
         key: "amip_diagedmf_io"
         command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf_io.yml --job_id amip_diagedmf_io"
-        artifact_paths: "experiments/ClimaEarth/output/amip/amip_diagedmf_io_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/amip/amip_diagedmf_io/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -99,7 +99,7 @@ steps:
       - label: "GPU ClimaAtmos without diagnostic EDMF"
         key: "gpu_climaatmos"
         command: "srun julia --threads=3 --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $BENCHMARK_CONFIG_PATH/climaatmos.yml --job_id gpu_climaatmos"
-        artifact_paths: "experiments/ClimaEarth/output/climaatmos/gpu_climaatmos_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/climaatmos/gpu_climaatmos/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           CLIMACOMMS_DEVICE: "CUDA"
@@ -112,7 +112,7 @@ steps:
       - label: "GPU ClimaAtmos with diagnostic EDMF"
         key: "gpu_climaatmos_diagedmf"
         command: "srun julia --threads=3 --color=yes --project=test/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file $BENCHMARK_CONFIG_PATH/climaatmos_diagedmf.yml --job_id gpu_climaatmos_diagedmf"
-        artifact_paths: "experiments/ClimaEarth/output/climaatmos/gpu_climaatmos_diagedmf_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/climaatmos/gpu_climaatmos_diagedmf/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           CLIMACOMMS_DEVICE: "CUDA"
@@ -125,7 +125,7 @@ steps:
       - label: "GPU AMIP with diagnostic EDMF"
         key: "gpu_amip_diagedmf"
         command: "srun julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf.yml --job_id gpu_amip_diagedmf"
-        artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_diagedmf_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_diagedmf/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           CLIMACOMMS_DEVICE: "CUDA"
@@ -138,7 +138,7 @@ steps:
       - label: "GPU AMIP with diagnostic EDMF and IO"
         key: "gpu_amip_diagedmf_io"
         command: "srun julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_diagedmf_io.yml --job_id gpu_amip_diagedmf_io"
-        artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_diagedmf_io_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/amip/gpu_amip_diagedmf_io/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           CLIMACOMMS_DEVICE: "CUDA"

diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml
@@ -73,7 +73,7 @@ steps:
       - label: "Slabplanet_aqua: nocouple"
         key: "slabplanet_aqua_atmos_sf_nocouple" # SF at each Atmos stage, no coupling, prescribed SST from atmos
         command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml --job_id slabplanet_aqua_atmos_sf_nocouple"
-        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple/artifacts/*"
         env:
           BUILD_HISTORY_HANDLE: ""
         agents:
@@ -85,7 +85,7 @@ steps:
       - label: "Slabplanet_aqua: couple"
         key: "slabplanet_aqua_atmos_sf_couple" # SF at each Atmos stage, coupling, prescribed SST from coupler - identical results to the above confirm 1) initial conditions in Atmos are unchanged compared to the slab, 2) coupling not introducing variability when constant surface
         command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml --job_id slabplanet_aqua_atmos_sf_couple"
-        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple/artifacts/*"
         env:
           BUILD_HISTORY_HANDLE: ""
         agents:
@@ -97,7 +97,7 @@ steps:
       - label: "Slabplanet_aqua: coupler fluxes"
         key: "slabplanet_aqua_coupler_sf" # SF at each coupler timestep, constant ocean - comparing to the above runs, this tests the sensitivity of less frequent flux calculation
         command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml --job_id slabplanet_aqua_coupler_sf"
-        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_coupler_sf/artifacts/*"
         env:
           BUILD_HISTORY_HANDLE: ""
         agents:
@@ -109,7 +109,7 @@ steps:
       - label: "Slabplanet_aqua: coupler fluxes, evolving ocean"
         key: "slabplanet_aqua_coupler_sf_evolve_ocn" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving ocean
         command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml --job_id slabplanet_aqua_coupler_sf_evolve_ocn"
-        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn/artifacts/*"
         env:
           BUILD_HISTORY_HANDLE: ""
         agents:
@@ -121,7 +121,7 @@ steps:
       - label: "Slabplanet_terra: coupler fluxes, evolving bucket"
         key: "slabplanet_terra" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving bucket
         command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_terra.yml --job_id slabplanet_terra"
-        artifact_paths: "experiments/ClimaEarth/output/slabplanet_terra/slabplanet_terra_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/slabplanet_terra/slabplanet_terra/artifacts/*"
         env:
           BUILD_HISTORY_HANDLE: ""
         agents:
@@ -133,7 +133,7 @@ steps:
       - label: "Slabplanet: coupler fluxes, evolving ocean and land"
         key: "slabplanet_coupler_sf_evolve_ocn"
         command: "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml --job_id slabplanet_coupler_sf_evolve_ocn"
-        artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn/artifacts/*"
         env:
           BUILD_HISTORY_HANDLE: ""
         agents:
@@ -162,7 +162,7 @@ steps:
       - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, nocouple, atmos flux calc"
         key: "slabplanet_aqua_target_nocouple"
         command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_nocouple.yml --job_id slabplanet_aqua_target_nocouple"
-        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_nocouple_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_nocouple/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -175,7 +175,7 @@ steps:
       - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, coupler flux calc"
         key: "slabplanet_aqua_target"
         command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_target.yml --job_id slabplanet_aqua_target"
-        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -188,7 +188,7 @@ steps:
       - label: "TARGET IDEALIZED: new target aqua - evolving slab ocean T"
         key: "slabplanet_aqua_target_evolve_ocn"
         command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_evolve_ocn.yml --job_id slabplanet_aqua_target_evolve_ocn"
-        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_evolve_ocn_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/slabplanet_aqua/slabplanet_aqua_target_evolve_ocn/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -201,7 +201,7 @@ steps:
       - label: "TARGET IDEALIZED: new target slab - fixed ocean T, bucket"
         key: "slabplanet_target"
         command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_target.yml --job_id slabplanet_target"
-        artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_target_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_target/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -214,7 +214,7 @@ steps:
       - label: "TARGET IDEALIZED: new target slab - evolving slab ocean T, bucket"
         key: "slabplanet_target_evolve_ocn"
         command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/slabplanet_target_evolve_ocn.yml --job_id slabplanet_target_evolve_ocn"
-        artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_target_evolve_ocn_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/slabplanet/slabplanet_target_evolve_ocn/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -231,7 +231,7 @@ steps:
       - label: "MPI AMIP FINE: new target amip"
         key: "amip_target"
         command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target.yml --job_id amip_target"
-        artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/amip/amip_target/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -244,7 +244,7 @@ steps:
       - label: "MPI AMIP FINE: new target amip: topo"
         key: "amip_target_topo"
         command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo.yml --job_id amip_target_topo"
-        artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -257,7 +257,7 @@ steps:
       - label: "MPI AMIP FINE: new target amip: topo + diagedmf"
         key: "amip_target_topo_diagedmf_cpu"
         command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf_cpu.yml --job_id amip_target_topo_diagedmf_cpu"
-        artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_cpu_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_cpu/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -274,7 +274,7 @@ steps:
       - label: "GPU AMIP FINE: new target amip: topo"
         key: "amip_target_topo_gpu"
         command: "srun julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo.yml --job_id amip_target_topo_gpu"
-        artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_gpu_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_gpu/artifacts/*"
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           CLIMACOMMS_DEVICE: "CUDA"
@@ -288,7 +288,7 @@ steps:
         command:
           - echo "--- Run simulation"
           - "srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf_gpu.yml --job_id amip_target_topo_diagedmf_gpu"
-        artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_gpu_artifacts/*"
+        artifact_paths: "experiments/ClimaEarth/output/amip/amip_target_topo_diagedmf_gpu/artifacts/*"
         timeout_in_minutes: 1440
         env:
           CLIMACOMMS_CONTEXT: "MPI"
@@ -308,7 +308,7 @@ steps:
     command:
       - echo "--- Run simulation"
       - "julia --color=yes --project=experiments/ClimaEarth/ experiments/ClimaEarth/run_amip.jl --config_file $CONFIG_PATH/longrun_amip_dyamond.yml --job_id longrun_amip_dyamond_gpu"
-    artifact_paths: "experiments/ClimaEarth/output/amip/longrun_amip_dyamond_gpu_artifacts/*"
+    artifact_paths: "experiments/ClimaEarth/output/amip/longrun_amip_dyamond_gpu/artifacts/*"
     env:
       CLIMACOMMS_DEVICE: "CUDA"
     agents:
@@ -335,5 +335,5 @@ steps:
 
       - label: ":envelope: Slack report: Slabplanet"
         command:
-          - slack-upload -c "#coupler-report" -f experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation"
-          - slack-upload -c "#coupler-report" -f experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation"
+          - slack-upload -c "#coupler-report" -f experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn/artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation"
+          - slack-upload -c "#coupler-report" -f experiments/ClimaEarth/output/slabplanet/slabplanet_coupler_sf_evolve_ocn/artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation"