Skip to content

Commit

Permalink
Update deps with latest releases
Browse files Browse the repository at this point in the history
Apply ClimaLSM API soil resistance changes

Apply ClimaComms API changes; remove ClimaCommsMPI

Update ClimaAtmos CLI options

Use ClimaAtmos v0.14.1

ClimaAtmos change: rename Y.f.w -> Y.f.u₃

ClimaLSM API change: use make_exp_tendency in place of make_ode_function

Use ClimaComms.CPUDevice()

Remove simulation.is_distributed from Atmos change PR #1634

WIP: Apply changes from surface fluxes rework CA PR #1447

Rebase onto main after PRs #315 & #320

WIP: update to ClimaAtmos surface rework

Co-authored-by: Lenka Novak <[email protected]>

TEMP ONLY: Test CA surface setup fix dev branch

Adapt flux calculation given CA current Surface Setup

Some WIP fixes w/ Lenka

Co-authored-by: Lenka Novak <[email protected]>

CI: Try Float64 for AMIP modular experiment

Increase alloc limit for perf test

Update to ClimaAtmos v0.15.0

conserving, need Atmos #1803

test ci with Atmos fix

ClimaCore up

add refs to TD and CAP

init with 0 atmos fluxes

Atmos-commit dep

perf project

test project

pip

perf Manifest

flame fix

rm idealinsol true

rm idealized insol

increase allocs

rm debug.jl

clean climaatmos_init.jl
  • Loading branch information
valeriabarra authored and LenkaNovak committed Jun 22, 2023
1 parent 9c35361 commit 81363a7
Show file tree
Hide file tree
Showing 19 changed files with 623 additions and 597 deletions.
36 changes: 18 additions & 18 deletions .buildkite/pipeline.yml

Large diffs are not rendered by default.

8 changes: 6 additions & 2 deletions Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,30 +11,34 @@ ClimaCoreTempestRemap = "d934ef94-cdd4-4710-83d6-720549644b70"
ClimaLSM = "7884a58f-fab6-4fd0-82bb-ecfedb2d8430"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
Insolation = "e98cc03f-d57e-4e3c-b70c-8d51efe9e0d8"
JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
SurfaceFluxes = "49b00bb7-8bd4-4f2b-b78c-51cd0450215f"
TempestRemap_jll = "8573a8c5-1df0-515e-a024-abad257ee284"
TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed"
UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"

[compat]
ClimaAtmos = "0.11"
ClimaComms = "0.3"
ClimaAtmos = "0.15"
ClimaComms = "0.3, 0.4"
ClimaCore = "0.10"
ClimaCoreTempestRemap = "0.3"
ClimaLSM = "0.2"
DocStringExtensions = "0.8, 0.9"
Insolation = "0.6"
JLD2 = "0.4"
NCDatasets = "0.11, 0.12"
OrdinaryDiffEq = "5, 6"
Plots = "1"
PrettyTables = "1, 2"
SciMLBase = "1"
SurfaceFluxes = "0.6"
TempestRemap_jll = "2"
TerminalLoggers = "0.1"
UnPack = "1"
Expand Down
458 changes: 221 additions & 237 deletions experiments/AMIP/modular/Manifest.toml

Large diffs are not rendered by default.

9 changes: 4 additions & 5 deletions experiments/AMIP/modular/Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ AtmosphericProfilesLibrary = "86bc3604-9858-485a-bdbe-831ec50de11d"
CLIMAParameters = "6eacf6c3-8458-43b9-ae03-caf5306d3d53"
ClimaAtmos = "b2c96348-7fb7-4fe0-8da9-78d88439e717"
ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d"
ClimaCommsMPI = "5f86816e-8b66-43b2-912e-75384f99de49"
ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884"
ClimaCorePlots = "cf7c7e5a-b407-4c48-9047-11a94a308626"
ClimaCoreTempestRemap = "d934ef94-cdd4-4710-83d6-720549644b70"
Expand Down Expand Up @@ -51,10 +50,10 @@ UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
ArgParse = "1.1"
ArtifactWrappers = "0.2"
AtmosphericProfilesLibrary = "0.1"
ClimaAtmos = "0.15.1"
CLIMAParameters = "0.4, 0.5, 0.6, 0.7"
ClimaComms = "0.3"
ClimaCommsMPI = "0.3"
ClimaCore = "0.10"
ClimaComms = "0.3, 0.4"
ClimaCore = "0.10.40"
ClimaCorePlots = "0.2"
ClimaCoreTempestRemap = "0.3"
ClimaCoupler = "0.1"
Expand All @@ -70,7 +69,7 @@ ForwardDiff = "0.10"
GR = "0.71"
Glob = "1"
HDF5_jll = "1"
Insolation = "0.3"
Insolation = "0.6"
IntervalSets = "0.5, 0.6, 0.7"
JLD2 = "0.4"
JSON = "0.21"
Expand Down
46 changes: 35 additions & 11 deletions experiments/AMIP/modular/cli_options.jl
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,14 @@ function argparse_settings()
help = "Access land surface albedo information from data file"
arg_type = Bool
default = true
"--surface_setup"
help = "Surface flux scheme [`DefaultExchangeCoefficients` (default), `DefaultMoninObukhov`]"
arg_type = String
default = "PrescribedSurface"
"--coupled" # TODO - remove from Atmos
help = "Coupled simulation [`false` (default), `true`]"
arg_type = Bool
default = true
# ClimaAtmos flags
"--FLOAT_TYPE"
help = "Float type"
Expand Down Expand Up @@ -65,7 +73,7 @@ function argparse_settings()
arg_type = String
default = "sphere"
"--initial_condition"
help = "Initial condition [`DryBaroclinicWave`, `MoistBaroclinicWave`, `DecayingProfile`, `IsothermalProfile`, `Bomex`, `DryDensityCurrentProfile`, `AgnesiHProfile`, `ScharProfile`]"
help = "Initial condition [`DryBaroclinicWave`, `MoistBaroclinicWave`, `DecayingProfile`, `IsothermalProfile`, `Bomex`, `DryDensityCurrentProfile`, `AgnesiHProfile`, `ScharProfile`, `RisingThermalBubbleProfile`]"
arg_type = String
default = "DecayingProfile"
"--moist"
Expand All @@ -91,13 +99,22 @@ function argparse_settings()
help = "EDMFX advection test switches off all velocity tendencies in GM and turbconc [`false` (default), `true`]"
arg_type = Bool
default = false
"--edmfx_entr_detr"
help = "If set to true, it switches on EDMFX entrainment/detrainment closure. [`true`, `false` (default)]"
arg_type = Bool
default = false
"--edmfx_sgs_flux"
help = "If set to true, it switches on EDMFX SGS flux. [`true`, `false` (default)]"
arg_type = Bool
default = false
"--edmfx_nh_pressure"
help = "If set to true, it switches on EDMFX pressure drag closure. [`true`, `false` (default)]"
arg_type = Bool
default = false
"--vert_diff"
help = "Vertical diffusion [`false` (default), `VerticalDiffusion`, `true` (defaults to `VerticalDiffusion`)]"
arg_type = String
default = "false"
"--surface_scheme"
help = "Surface flux scheme [`nothing` (default), `bulk`, `monin_obukhov`]"
arg_type = String
"--surface_thermo_state_type"
help = "Surface thermo state type [`GCMSurfaceThermoState` (default), `PrescribedThermoState`]"
arg_type = String
Expand All @@ -106,10 +123,6 @@ function argparse_settings()
help = "Bulk transfer coefficient"
arg_type = Float64
default = Float64(0.0044)
"--coupled"
help = "Coupled simulation [`false` (default), `true`]"
arg_type = Bool
default = false
"--turbconv"
help = "Turbulence convection scheme [`nothing` (default), `edmf`]"
arg_type = String
Expand Down Expand Up @@ -324,6 +337,14 @@ function argparse_settings()
help = "Define the surface elevation profile [`NoWarp`,`Earth`,`DCMIP200`,`Agnesi`]"
arg_type = String
default = "NoWarp"
"--topo_smoothing"
help = "Choose whether to order-2 smoothing on the LGL mesh"
arg_type = Bool
default = false
"--smoothing_order"
help = "Define the surface smoothing kernel factor (integer) [`3 (default)`]"
arg_type = Int
default = 3
"--apply_limiter"
help = "Apply a horizontal limiter to every tracer [`true` (default), `false`]"
arg_type = Bool
Expand Down Expand Up @@ -361,9 +382,12 @@ function argparse_settings()
arg_type = Bool
default = false
"--orographic_gravity_wave"
help = "Apply parameterization for orographic drag on horizontal mean flow"
arg_type = Bool
default = false
help = "Orographic drag on horizontal mean flow [`nothing` (default), `gfdl_restart`, `raw_topo`]"
arg_type = String
"--device"
help = "Device type to use [`CPUDevice` (default), `CUDADevice`]"
arg_type = String
default = "CPUDevice"
"--perf_summary"
help = "Flag for collecting performance summary information"
arg_type = Bool
Expand Down
135 changes: 85 additions & 50 deletions experiments/AMIP/modular/components/atmosphere/climaatmos_init.jl
Original file line number Diff line number Diff line change
@@ -1,98 +1,133 @@
# atmos_init: for ClimaAtmos pre-AMIP interface
import ClimaAtmos
using ClimaAtmos: RRTMGPI
import ClimaCoupler.FluxCalculator: compute_atmos_turbulent_fluxes!
using ClimaCore: Fields.level, Geometry

driver_file = joinpath(pkgdir(ClimaAtmos), "examples", "hybrid", "driver.jl")
ENV["CI_PERF_SKIP_RUN"] = true
try
include(driver_file)
catch err
if err.error !== :exit_profile
rethrow(err.error)
end
end
# the clima atmos `integrator` is now defined
struct ClimaAtmosSimulation{P, Y, D, I} <: AtmosModelSimulation
params::P
Y_init::Y
domain::D
integrator::I
end
function atmos_init(::Type{FT}, parsed_args::Dict) where {FT}

function atmos_init(::Type{FT}, Y, integrator; params = nothing) where {FT}
atmos_config = ClimaAtmos.AtmosConfig(argparse_settings(); parsed_args)
integrator = ClimaAtmos.get_integrator(atmos_config)
Y = integrator.u
center_space = axes(Y.c.ρe_tot)
face_space = axes(Y.f.w)
face_space = axes(Y.f.u₃)
spaces = (; center_space = center_space, face_space = face_space)
if :ρe_int in propertynames(Y.c)
@warn("Running with ρe_int in coupled mode is not tested yet.")
end

ClimaAtmosSimulation(params, Y, spaces, integrator)
# set initial fluxes to zero
@. integrator.p.sfc_conditions.ρ_flux_h_tot = Geometry.Covariant3Vector(FT(0.0))
@. integrator.p.sfc_conditions.ρ_flux_q_tot = Geometry.Covariant3Vector(FT(0.0))
@. integrator.p.sfc_conditions.ρ_flux_uₕ.components = zeros(axes(integrator.p.sfc_conditions.ρ_flux_uₕ.components))
parent(integrator.p.ᶠradiation_flux) .= parent(zeros(axes(integrator.p.ᶠradiation_flux)))

ClimaAtmosSimulation(integrator.p.params, Y, spaces, integrator)
end

# required by the Interfacer
get_field(sim::ClimaAtmosSimulation, ::Val{:radiative_energy_flux}) = level(sim.integrator.p.ᶠradiation_flux, half)
get_field(sim::ClimaAtmosSimulation, ::Val{:radiative_energy_flux}) =
Fields.level(sim.integrator.p.ᶠradiation_flux, half)
get_field(sim::ClimaAtmosSimulation, ::Val{:liquid_precipitation}) =
sim.integrator.p.col_integrated_rain .+ sim.integrator.p.col_integrated_snow # all fallen snow melts for now
get_field(sim::ClimaAtmosSimulation, ::Val{:snow_precipitation}) = sim.integrator.p.col_integrated_snow .* FT(0)

get_field(sim::ClimaAtmosSimulation, ::Val{:turbulent_energy_flux}) = sim.integrator.p.ρ_dif_flux_h_tot
get_field(sim::ClimaAtmosSimulation, ::Val{:turbulent_moisture_flux}) = sim.integrator.p.ρ_dif_flux_q_tot
get_field(sim::ClimaAtmosSimulation, ::Val{:turbulent_energy_flux}) =
Geometry.WVector.(sim.integrator.p.sfc_conditions.ρ_flux_h_tot)
get_field(sim::ClimaAtmosSimulation, ::Val{:turbulent_moisture_flux}) =
Geometry.WVector.(sim.integrator.p.sfc_conditions.ρ_flux_q_tot)

function update_field!(sim::ClimaAtmosSimulation, ::Val{:surface_temperature}, field)
sim.integrator.p.radiation_model.surface_temperature .= RRTMGPI.field2array(field)
parent(sim.integrator.p.T_sfc) .= parent(field)

end
function update_field!(sim::ClimaAtmosSimulation, ::Val{:albedo}, field)
sim.integrator.p.radiation_model.diffuse_sw_surface_albedo .=
reshape(RRTMGPI.field2array(field), 1, length(parent(field)))
sim.integrator.p.radiation_model.direct_sw_surface_albedo .=
reshape(RRTMGPI.field2array(field), 1, length(parent(field)))
end
function update_field!(sim::ClimaAtmosSimulation, ::Val{:roughness_momentum}, field)
parent(sim.integrator.p.sfc_inputs.z0m) .= parent(field)
end
function update_field!(sim::ClimaAtmosSimulation, ::Val{:roughness_buoyancy}, field)
parent(sim.integrator.p.sfc_inputs.z0b) .= parent(field)
end

step!(sim::ClimaAtmosSimulation, t) = step!(sim.integrator, t - sim.integrator.t, true)

reinit!(sim::ClimaAtmosSimulation) = reinit!(sim.integrator)

"""
update_sim!(atmos_sim::ClimaAtmosSimulation, csf)
Updates the surface fields for temperature and albedo.
# Arguments
- `atmos_sim`: [ClimaAtmosSimulation] containing an atmospheric model simulation object.
- `csf`: [NamedTuple] containing coupler fields.
"""
function update_sim!(atmos_sim::ClimaAtmosSimulation, csf, turbulent_fluxes)
update_field!(atmos_sim, Val(:albedo), csf.albedo)
update_field!(atmos_sim, Val(:surface_temperature), csf.T_S)
end


# flux calculation borrowed from atmos
"""
CoupledMoninObukhov()
A modified version of a Monin-Obukhov surface for the Coupler, see the link below for more information
https://clima.github.io/SurfaceFluxes.jl/dev/SurfaceFluxes/#Monin-Obukhov-Similarity-Theory-(MOST)
"""
struct CoupledMoninObukhov end
"""
coupler_surface_setup(::CoupledMoninObukhov, p, csf_sfc = (; T = nothing, z0m = nothing, z0b = nothing, beta = nothing))
Sets up `surface_setup` as a `Fields.Field` or `SurfaceState`s.
"""
function coupler_surface_setup(
::CoupledMoninObukhov,
p;
csf_sfc = (; T = nothing, z0m = nothing, z0b = nothing, beta = nothing),
)

surface_state(z0m, z0b, T, beta) = ClimaAtmos.SurfaceConditions.SurfaceState(;
parameterization = ClimaAtmos.SurfaceConditions.MoninObukhov(; z0m, z0b),
T,
beta,
)
surface_state_field = @. surface_state(csf_sfc.z0m, csf_sfc.z0b, csf_sfc.T, csf_sfc.beta)
return surface_state_field
end

"""
compute_atmos_turbulent_fluxes!(atmos_sim::ClimaAtmosSimulation, csf)
Computes turbulent surface fluxes using ClimaAtmos's `update_surface_conditions!`. This
requires that we define a new temporary parameter Tuple, `new_p`, and save the new surface state
in it. We do not want `new_p` to live in the atmospheric model permanently, because that would also
trigger flux calculation during Atmos `step!`. We only want to trigger this once per coupling
timestep from ClimaCoupler.
"""
function compute_atmos_turbulent_fluxes!(atmos_sim::ClimaAtmosSimulation, csf)
# TODO: dependence on atmos will be removed

thermo_params = CAP.thermodynamics_params(atmos_sim.integrator.p.params)
p = atmos_sim.integrator.p

# calculate turbulent fluxes on atmos grid and save in atmos cache
update_field!(atmos_sim, Val(:surface_temperature), csf.T_S)
coupler_sfc_setup = coupler_surface_setup(
CoupledMoninObukhov(),
p;
csf_sfc = (; T = csf.T_S, z0m = csf.z0m_S, z0b = csf.z0b_S, beta = csf.beta),
)

if :z0b in propertynames(atmos_sim.integrator.p.surface_scheme)
update_field!(atmos_sim, Val(:roughness_momentum), csf.z0m_S)
update_field!(atmos_sim, Val(:roughness_buoyancy), csf.z0b_S)
end
p_names = propertynames(p)
p_values = map(x -> x == :sfc_setup ? coupler_sfc_setup : getproperty(p, x), p_names)

new_p = (; zip(p_names, p_values)...)

ClimaAtmos.SurfaceConditions.update_surface_conditions!(atmos_sim.integrator.u, new_p, atmos_sim.integrator.t)

p.sfc_conditions.ρ_flux_h_tot .= new_p.sfc_conditions.ρ_flux_h_tot
p.sfc_conditions.ρ_flux_q_tot .= new_p.sfc_conditions.ρ_flux_q_tot
p.sfc_conditions.ρ_flux_uₕ .= new_p.sfc_conditions.ρ_flux_uₕ

Fields.bycolumn(axes(atmos_sim.integrator.p.ts_sfc)) do colidx
ClimaAtmos.set_surface_thermo_state!(
ClimaAtmos.Decoupled(),
atmos_sim.integrator.p.surface_scheme.sfc_thermo_state_type,
atmos_sim.integrator.p.ts_sfc[colidx],
atmos_sim.integrator.p.T_sfc[colidx],
Spaces.level(atmos_sim.integrator.p.ᶜts[colidx], 1),
thermo_params,
atmos_sim.integrator.t,
)

get_surface_fluxes!(
atmos_sim.integrator.u,
atmos_sim.integrator.p,
atmos_sim.integrator.t,
colidx,
atmos_sim.integrator.p.atmos.vert_diff,
)
end
end
4 changes: 2 additions & 2 deletions experiments/AMIP/modular/components/flux_calculator.jl
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ function calculate_surface_fluxes_atmos_grid!(integrator, info_sfc)
Fields.bycolumn(axes(Y.c.uₕ)) do colidx
get_surface_fluxes!(Y, p, colidx)
# corrections (accounting for inhomogeneous surfaces)
@. p.ρ_dif_flux_h_tot[colidx] = # checking right quantity in ClimaAtmos v0.11.0
@. p.ρ_flux_h_tot[colidx] = # checking right quantity in ClimaAtmos v0.11.0
-Geometry.WVector(correct_e_over_ice(p.surface_conditions[colidx], ice_fraction[colidx]))
@. p.ρ_dif_flux_q_tot[colidx] =
@. p.ρ_flux_q_tot[colidx] =
-Geometry.WVector(correct_q_over_ice(p.surface_conditions[colidx], ice_fraction[colidx]))
end
end
Expand Down
6 changes: 3 additions & 3 deletions experiments/AMIP/modular/components/land/bucket_init.jl
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import ClimaLSM.Bucket:
snow_precipitation

using ClimaLSM:
make_ode_function, initialize, obtain_surface_space, make_set_initial_aux_state, surface_evaporative_scaling
make_exp_tendency, initialize, obtain_surface_space, make_set_initial_aux_state, surface_evaporative_scaling

import ClimaCoupler.Interfacer: LandModelSimulation, get_field, update_field!
import ClimaCoupler.FieldExchanger: step!, reinit!
Expand Down Expand Up @@ -245,9 +245,9 @@ function bucket_init(
set_initial_aux_state! = make_set_initial_aux_state(model)
set_initial_aux_state!(p_new, Y, tspan[1])

ode_function! = make_ode_function(model)
exp_tendency! = make_exp_tendency(model)

prob = ODEProblem(ode_function!, Y, tspan, p_new)
prob = ODEProblem(exp_tendency!, Y, tspan, p_new)
integrator = init(prob, stepper; dt = dt, saveat = saveat)

BucketSimulation(model, Y, (; domain = domain, soil_depth = d_soil), integrator, area_fraction)
Expand Down
Loading

0 comments on commit 81363a7

Please sign in to comment.