Aim allocation monitoring at perf targets

Remove threaded allocation table

Increase mem alloc limits

Increase mem request

.buildkite/pipeline.yml

@@ -369,7 +369,7 @@ steps:
         agents:
           slurm_mem: 20GB
 
-      - label: ":rocket: flame graph: threadedd perf target (ρe_tot)"
+      - label: ":rocket: flame graph: threaded perf target (ρe_tot)"
         command: "julia --threads 8 --color=yes --project=perf perf/flame.jl --job_id flame_perf_target_rhoe_threaded --enable_threading true"
         artifact_paths: "flame_perf_target_rhoe_threaded/*"
         agents:
@@ -392,8 +392,10 @@ steps:
       - label: ":rocket: benchmark: baroclinic wave (ρe_tot)"
         command: "julia --color=yes --project=perf perf/benchmark.jl --job_id bm_sphere_baroclinic_wave_rhoe"
 
-      - label: ":mag: Allocations: baroclinic wave (ρe_tot)"
+      - label: ":mag: Allocations: perf target"
         command: "julia --color=yes --project=perf perf/allocs.jl"
+        agents:
+          slurm_mem: 20GB
 
   - wait: ~
     continue_on_failure: true

examples/hybrid/cli_options.jl

@@ -376,3 +376,13 @@ function parsed_args_per_job_id(buildkite_yaml; trigger = "driver.jl")
     end
     return result
 end
+
+function non_default_command_line_flags_parsed_args(parsed_args)
+    (s, default_parsed_args) = parse_commandline()
+    s = ""
+    for k in keys(parsed_args)
+        default_parsed_args[k] == parsed_args[k] && continue
+        s *= "--$k $(parsed_args[k]) "
+    end
+    return rstrip(s)
+end

perf/allocs.jl

@@ -1,5 +1,6 @@
 # Track some important dependencies:
 example_dir = joinpath(dirname(@__DIR__), "examples")
+include(joinpath(example_dir, "hybrid", "cli_options.jl"));
 
 import ClimaCore
 import SciMLBase
@@ -24,9 +25,16 @@ dirs_to_monitor = String.(dirs_to_monitor)
 
 #! format: off
 
+dict = parsed_args_per_job_id(; trigger = "benchmark.jl")
+for k in keys(dict)
+    dict[k]["job_id"] = "allocs_"*dict[k]["job_id"]
+
+    # Lower resolution, since allocation tracking is expensive
+    dict[k]["h_elem"] = 6
+    dict[k]["z_elem"] = 18
+end
 cli_options = [
-    ("--job_id alloc_sphere_baroclinic_wave_rhoe"),
-    ("--vert_diff true --surface_scheme bulk --moist equil --forcing held_suarez --microphy 0M"),
+    non_default_command_line_flags_parsed_args(dict["perf_target_unthreaded"]),
 ]
 #! format: on
 
@@ -35,6 +43,7 @@ import ReportMetrics
 for clio in cli_options
     job_id = first(split(last(split(clio, "--job_id ")), " "))
     clio_in = split(clio, " ")
+    @info "CL options: `$clio_in`"
     ReportMetrics.report_allocs(;
         job_name = string(job_id),
         run_cmd = `$(Base.julia_cmd()) --project=perf/ --track-allocation=all perf/allocs_per_case.jl $clio_in`,

perf/allocs_per_case.jl

@@ -15,5 +15,6 @@ catch err
 end
 
 OrdinaryDiffEq.step!(integrator) # compile first
+Profile.clear()
 Profile.clear_malloc_data()
 OrdinaryDiffEq.step!(integrator)

perf/flame.jl

@@ -68,9 +68,9 @@ allocs = @allocated OrdinaryDiffEq.step!(integrator)
 
 allocs_limit = Dict()
 allocs_limit["flame_perf_target_rhoe"] = 10575616
-allocs_limit["flame_perf_target_rhoe_threaded"] = 12078128
+allocs_limit["flame_perf_target_rhoe_threaded"] = 12078608
 allocs_limit["flame_perf_target_rhoe_callbacks"] = 21740792
-allocs_limit["flame_perf_target_rhoe_ARS343"] = 24021744
+allocs_limit["flame_perf_target_rhoe_ARS343"] = 24056711
 
 if allocs < allocs_limit[job_id] * buffer
     @info "TODO: lower `allocs_limit[$job_id]` to: $(allocs)"