100km atmos scaling run

switch off coupler diagnostics

rm cpu bind

no atmos diags

no atmos diags

300d

clean

clean

250 d

fix changed atmos albedo flag

wip

runs

try

disable nonideal insol

albedo initial = 0.38

revert reshape

add debug plots

add debug plots

try with gpu debug plots

fix debug plots

try 70d 1 proc

120s

30GB

1 ntask

30GB disable diags and debug

carry over from ln/100km-test (1.082 SYPD w/o diags)

format

rm debug for gpu

debug for CPUSingleThreaded

try sponge setup 50d 1 proc

clean

singleton

clean

clean

120d test

git push

no sponge

edmf params

sponge + edmf params + 200d

205d running

revs

.buildkite/longruns/pipeline copy.yml

@@ -0,0 +1,349 @@
+agents:
+  queue: new-central
+  slurm_time: 24:00:00
+  modules: climacommon/2024_04_05
+
+env:
+  JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
+  OPENBLAS_NUM_THREADS: 1
+  JULIA_NVTX_CALLBACKS: gc
+  OMPI_MCA_opal_warn_on_missing_libcuda: 0
+  JULIA_MAX_NUM_PRECOMPILE_FILES: 100
+  GKSwstype: 100
+  SLURM_KILL_BAD_EXIT: 1
+
+  CONFIG_PATH: "config/longrun_configs"
+
+timeout_in_minutes: 1440
+
+steps:
+  - label: "init :computer:"
+    key: "init_cpu_env"
+    command:
+      - "echo $$JULIA_DEPOT_PATH"
+
+      - echo "--- Instantiate AMIP env"
+      - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.precompile()'"
+      - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.status()'"
+
+      - echo "--- Download artifacts"
+      - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'"
+      - "julia --project=artifacts -e 'using Pkg; Pkg.status()'"
+      - "julia --project=artifacts artifacts/download_artifacts.jl"
+
+    agents:
+      slurm_cpus_per_task: 8
+    env:
+      JULIA_NUM_PRECOMPILE_TASKS: 8
+      JULIA_MAX_NUM_PRECOMPILE_FILES: 50
+
+  - label: "init clima :computer:"
+    key: "init_cpu_env_clima"
+    command:
+      - "echo $$JULIA_DEPOT_PATH"
+
+      - echo "--- Instantiate AMIP env"
+      - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.precompile()'"
+      - "julia --project=experiments/AMIP/ -e 'using Pkg; Pkg.status()'"
+
+      - echo "--- Download artifacts"
+      - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - "julia --project=artifacts -e 'using Pkg; Pkg.precompile()'"
+      - "julia --project=artifacts -e 'using Pkg; Pkg.status()'"
+      - "julia --project=artifacts artifacts/download_artifacts.jl"
+
+    agents:
+      queue: clima
+      modules: common
+    env:
+      JULIA_NUM_PRECOMPILE_TASKS: 8
+      JULIA_MAX_NUM_PRECOMPILE_FILES: 50
+
+  - wait
+
+  - group: "Coupler integration and conservation tests"
+
+    steps:
+
+      # Integration tests - the expected results were also confirmed locally
+      - label: "Slabplanet_aqua: nocouple"
+        key: "slabplanet_aqua_atmos_sf_nocouple" # SF at each Atmos stage, no coupling, prescribed SST from atmos
+        command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml"
+        artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+      - label: "Slabplanet_aqua: couple"
+        key: "slabplanet_aqua_atmos_sf_couple" # SF at each Atmos stage, coupling, prescribed SST from coupler - identical results to the above confirm 1) initial conditions in Atmos are unchanged compared to the slab, 2) coupling not introducing variability when constant surface
+        command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml"
+        artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+      - label: "Slabplanet_aqua: coupler fluxes"
+        key: "slabplanet_aqua_coupler_sf" # SF at each coupler timestep, constant ocean - comparing to the above runs, this tests the sensitivity of less frequent flux calculation
+        command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml"
+        artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+      - label: "Slabplanet_aqua: coupler fluxes, evolving ocean"
+        key: "slabplanet_aqua_coupler_sf_evolve_ocn" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving ocean
+        command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml"
+        artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+      - label: "Slabplanet_terra: coupler fluxes, evolving bucket"
+        key: "slabplanet_terra" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving bucket
+        command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_terra.yml"
+        artifact_paths: "experiments/AMIP/output/slabplanet_terra/slabplanet_terra_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+      - label: "Slabplanet: coupler fluxes, evolving ocean and land"
+        key: "slabplanet_coupler_sf_evolve_ocn"
+        command: "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml"
+        artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+  - group: "Current target tests: idealized surfaces"
+
+    steps:
+
+      - label: "ClimaAtmos standalone target"
+        command:
+          - srun julia --project=experiments/AMIP/ test/component_model_tests/climaatmos_standalone/atmos_driver.jl --config_file test/component_model_tests/climaatmos_standalone/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean.yml
+        artifact_paths: "longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_tvinsol_0M_slabocean/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+          CLIMACORE_DISTRIBUTED: "MPI"
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+      - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, nocouple, atmos flux calc"
+        key: "slabplanet_aqua_target_nocouple"
+        command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_nocouple.yml"
+        artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_target_nocouple_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+      - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, coupler flux calc"
+        key: "slabplanet_aqua_target"
+        command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_target.yml"
+        artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_target_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+      - label: "TARGET IDEALIZED: new target aqua - evolving slab ocean T"
+        key: "slabplanet_aqua_target_evolve_ocn"
+        command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_evolve_ocn.yml"
+        artifact_paths: "experiments/AMIP/output/slabplanet_aqua/slabplanet_aqua_target_evolve_ocn_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+      - label: "TARGET IDEALIZED: new target slab - fixed ocean T, bucket"
+        key: "slabplanet_target"
+        command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_target.yml"
+        artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_target_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+      - label: "TARGET IDEALIZED: new target slab - evolving slab ocean T, bucket"
+        key: "slabplanet_target_evolve_ocn"
+        command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/slabplanet_target_evolve_ocn.yml"
+        artifact_paths: "experiments/AMIP/output/slabplanet/slabplanet_target_evolve_ocn_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+
+  - group: "Current target tests: AMIP surfaces"
+
+    steps:
+
+      - label: "MPI AMIP FINE: new target amip"
+        key: "amip_target"
+        command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_target.yml"
+        artifact_paths: "experiments/AMIP/output/amip/amip_target_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+      - label: "MPI AMIP FINE: new target amip: topo"
+        key: "amip_target_topo"
+        command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_target_topo.yml"
+        artifact_paths: "experiments/AMIP/output/amip/amip_target_topo_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+        soft_fail: true
+
+      - label: "MPI AMIP FINE: new target amip: topo + diagedmf"
+        key: "amip_target_topo_diagedmf"
+        command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/amip_target_topo_diagedmf.yml"
+        artifact_paths: "experiments/AMIP/output/amip/amip_target_topo_diagedmf_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 20G
+        soft_fail: true
+
+  - group: "Current target tests on GPU: AMIP surface"
+
+    steps:
+
+      - label: "GPU AMIP FINE: new target amip: topo"
+        key: "gpu_amip_target_topo"
+        command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo.yml"
+        artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_artifacts/*"
+        agents:
+          slurm_gpus: 1
+          slurm_mem: 16GB
+        soft_fail: true
+
+      - label: "GPU AMIP FINE: new target amip: topo + diagedmf"
+        key: "gpu_amip_target_topo_diagedmf"
+        command:
+          - echo "--- Run simulation"
+          - "srun --cpu-bind=threads --cpus-per-task=4 julia --threads=3 --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf.yml"
+        artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_artifacts/*"
+        timeout_in_minutes: 1440
+        env:
+          CLIMACOMMS_CONTEXT: "MPI"
+        agents:
+          queue: clima
+          slurm_gpus_per_task: 1
+          slurm_cpus_per_task: 4
+          slurm_ntasks: 4
+          slurm_mem: 20GB
+          modules: common
+        soft_fail: true
+
+  # DYAMOND AMIP: 1 day (convection resolving)
+  - label: "GPU AMIP SUPERFINE: dyamond_target"
+    key: "gpu_dyamond_target"
+    command:
+      - echo "--- Run simulation"
+      - "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_dyamond_target.yml"
+    artifact_paths: "experiments/AMIP/output/amip/gpu_dyamond_target_artifacts/*"
+    agents:
+      queue: clima
+      slurm_mem: 20GB
+      slurm_gpus: 1
+      modules: common
+    soft_fail: true
+
+  - wait
+
+  - group: "Job analysis and reporting"
+
+    steps:
+
+      # plot job performance history
+      - label: ":chart_with_downwards_trend: build history"
+        command:
+          - build_history main # name of branch to plot
+        artifact_paths:
+          - "build_history.html"
+
+      - wait
+
+      - label: ":envelope: Slack report: build_history"
+        command:
+          - |
+            slack-upload -c "#coupler-report" -f build_history.html -m html -n build_history -x ":rocket: Interactive overall job performance history (download the attached file and view in browser) :rocket:"
+
+      - label: ":envelope: Slack report: Slabplanet"
+        command:
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation"
+
+      - label: ":envelope: Slack report: target AMIP"
+        command:
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_artifacts/amip_ncep.png -m png -n amip_fine -x "90d Target AMIP v NCEP Last Month Mean"
+          - |
+            find experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_artifacts/ -type f -name 'bias*.png' -print0 | while IFS= read -r -d '' file; do
+              slack-upload -c "#coupler-report" -f "$$file" -m png -n "$$(basename "$$file" .png)" -x "$$(basename "$$file" .png)"
+            done
+        agents:
+          queue: clima
+          modules: common

.buildkite/longruns/pipeline.yml

@@ -294,7 +294,7 @@ steps:
           slurm_gpus_per_task: 1
           slurm_cpus_per_task: 4
           slurm_ntasks: 4
-          slurm_mem: 20GB
+          slurm_mem: 30GB
           modules: common
         soft_fail: true
 
@@ -339,12 +339,11 @@ steps:
 
       - label: ":envelope: Slack report: target AMIP"
         command:
-          - slack-upload -c "#coupler-report" -f experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_artifacts/amip_ncep.png -m png -n amip_fine -x "90d Target AMIP v NCEP Last Month Mean"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_artifacts/amip_ncep.png -m png -n amip_fine -x "250d Target AMIP v NCEP Last Month Mean"
           - |
             find experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_artifacts/ -type f -name 'bias*.png' -print0 | while IFS= read -r -d '' file; do
               slack-upload -c "#coupler-report" -f "$$file" -m png -n "$$(basename "$$file" .png)" -x "$$(basename "$$file" .png)"
             done
         agents:
           queue: clima
           modules: common
-

config/longrun_configs/amip_target_topo.yml

@@ -1,4 +1,5 @@
 FLOAT_TYPE: "Float32"
+albedo_model: "CouplerAlbedo"
 anim: true
 apply_limiter: false
 atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equil_55km_nz63_clearsky_0M_earth.yml"