From acd0d2dfa33c13a54b910ef438bdc3f1b7a229b4 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 6 Aug 2025 14:41:01 -0600 Subject: [PATCH 001/196] Replace most uses of npcropmin/max with is_prognostic_crop(). --- src/biogeochem/CNAllocationMod.F90 | 8 +- src/biogeochem/CNC14DecayMod.F90 | 9 +- src/biogeochem/CNCIsoFluxMod.F90 | 4 +- src/biogeochem/CNCStateUpdate1Mod.F90 | 22 +-- src/biogeochem/CNFUNMod.F90 | 6 +- src/biogeochem/CNFireEmissionsMod.F90 | 4 +- src/biogeochem/CNGRespMod.F90 | 4 +- src/biogeochem/CNGapMortalityMod.F90 | 4 +- src/biogeochem/CNMRespMod.F90 | 4 +- src/biogeochem/CNNStateUpdate1Mod.F90 | 10 +- src/biogeochem/CNPhenologyMod.F90 | 8 +- src/biogeochem/CNVegCarbonFluxType.F90 | 16 +-- src/biogeochem/CNVegCarbonStateType.F90 | 6 +- src/biogeochem/CNVegMatrixMod.F90 | 128 +++++++++--------- src/biogeochem/CNVegNitrogenStateType.F90 | 4 +- src/biogeochem/CNVegStructUpdateMod.F90 | 4 +- src/biogeochem/CropType.F90 | 4 +- src/biogeochem/DryDepVelocity.F90 | 4 +- .../NutrientCompetitionCLM45defaultMod.F90 | 12 +- .../NutrientCompetitionFlexibleCNMod.F90 | 32 ++--- src/biogeophys/PhotosynthesisMod.F90 | 4 +- src/biogeophys/TemperatureType.F90 | 4 +- src/cpl/share_esmf/cropcalStreamMod.F90 | 14 +- src/main/filterMod.F90 | 4 +- src/main/pftconMod.F90 | 32 ++++- src/soilbiogeochem/TillageMod.F90 | 4 +- 26 files changed, 186 insertions(+), 169 deletions(-) diff --git a/src/biogeochem/CNAllocationMod.F90 b/src/biogeochem/CNAllocationMod.F90 index 254e951cfe..78dfadfaee 100644 --- a/src/biogeochem/CNAllocationMod.F90 +++ b/src/biogeochem/CNAllocationMod.F90 @@ -15,7 +15,7 @@ module CNAllocationMod use clm_varcon , only : secspday use clm_varctl , only : use_c13, use_c14, iulog use PatchType , only : patch - use pftconMod , only : pftcon, npcropmin + use pftconMod , only : pftcon, is_prognostic_crop use CropType , only : crop_type use CropType , only : cphase_planted, cphase_leafemerge, cphase_grainfill use PhotosynthesisMod , only : photosyns_type @@ -191,7 +191,7 @@ subroutine calc_gpp_mr_availc(bounds, num_soilp, filter_soilp, & mr = leaf_mr(p) + froot_mr(p) if (woody(ivt(p)) == 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) - else if (ivt(p) >= npcropmin) then + else if (is_prognostic_crop(ivt(p))) then if (croplive(p)) then reproductive_mr_tot = 0._r8 do k = 1, nrepr @@ -500,7 +500,7 @@ subroutine calc_allometry(num_soilp, filter_soilp, & end if f4 = flivewd(ivt(p)) - if (ivt(p) >= npcropmin) then + if (is_prognostic_crop(ivt(p))) then g1 = 0.25_r8 else g1 = grperc(ivt(p)) @@ -521,7 +521,7 @@ subroutine calc_allometry(num_soilp, filter_soilp, & c_allometry(p) = (1._r8+g1a)*(1._r8+f1+f3*(1._r8+f2)) n_allometry(p) = 1._r8/cnl + f1/cnfr + (f3*f4*(1._r8+f2))/cnlw + & (f3*(1._r8-f4)*(1._r8+f2))/cndw - else if (ivt(p) >= npcropmin) then ! skip generic crops + else if (is_prognostic_crop(ivt(p))) then ! skip generic crops cng = graincn(ivt(p)) f1 = aroot(p) / aleaf(p) f3 = astem(p) / aleaf(p) diff --git a/src/biogeochem/CNC14DecayMod.F90 b/src/biogeochem/CNC14DecayMod.F90 index 1679c602e4..8caabfc41d 100644 --- a/src/biogeochem/CNC14DecayMod.F90 +++ b/src/biogeochem/CNC14DecayMod.F90 @@ -11,7 +11,7 @@ module CNC14DecayMod use clm_varctl , only : spinup_state use CNSharedParamsMod , only : use_matrixcn use decompMod , only : bounds_type - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use CNVegCarbonStateType , only : cnveg_carbonstate_type use CNVegCarbonFluxType , only : cnveg_carbonflux_type use SoilBiogeochemDecompCascadeConType , only : decomp_cascade_con, use_soil_matrixcn @@ -205,12 +205,7 @@ subroutine C14Decay( bounds, num_soilc, filter_soilc, num_soilp, filter_soilp, & gresp_xfer(p) = gresp_xfer(p) * (1._r8 - decay_const * dt) pft_ctrunc(p) = pft_ctrunc(p) * (1._r8 - decay_const * dt) - ! NOTE(wjs, 2017-02-02) This isn't a completely robust way to check if this is a - ! prognostic crop patch (at the very least it should also check if <= npcropmax; - ! ideally it should use a prognostic_crop flag that doesn't seem to exist - ! currently). But I'm just being consistent with what's done elsewhere (e.g., in - ! CStateUpdate1). - if (patch%itype(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(patch%itype(p))) then ! skip 2 generic crops cropseedc_deficit(p) = cropseedc_deficit(p) * (1._r8 - decay_const * dt) end if end do diff --git a/src/biogeochem/CNCIsoFluxMod.F90 b/src/biogeochem/CNCIsoFluxMod.F90 index fbe4cd927f..561e9a58ca 100644 --- a/src/biogeochem/CNCIsoFluxMod.F90 +++ b/src/biogeochem/CNCIsoFluxMod.F90 @@ -1357,7 +1357,7 @@ subroutine CNCIsoLitterToColumn (num_soilp, filter_soilp, & ! ! !USES: !DML - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use clm_varctl , only : use_grainproduct !DML @@ -1404,7 +1404,7 @@ subroutine CNCIsoLitterToColumn (num_soilp, filter_soilp, & end do !DML - if (ivt(p) >= npcropmin) then ! add livestemc to litter + if (is_prognostic_crop(ivt(p))) then ! add livestemc to litter ! stem litter carbon fluxes do i = i_litr_min, i_litr_max phenology_c_to_litr_c(c,j,i) = & diff --git a/src/biogeochem/CNCStateUpdate1Mod.F90 b/src/biogeochem/CNCStateUpdate1Mod.F90 index 70f0b86a53..4f5876bc60 100644 --- a/src/biogeochem/CNCStateUpdate1Mod.F90 +++ b/src/biogeochem/CNCStateUpdate1Mod.F90 @@ -10,7 +10,7 @@ module CNCStateUpdate1Mod use clm_time_manager , only : get_step_size_real use clm_varpar , only : i_litr_min, i_litr_max, i_cwd use clm_varpar , only : i_met_lit, i_str_lit, i_phys_som, i_chem_som - use pftconMod , only : npcropmin, nc3crop, pftcon + use pftconMod , only : is_prognostic_crop, nc3crop, pftcon use abortutils , only : endrun use decompMod , only : bounds_type use CNVegCarbonStateType , only : cnveg_carbonstate_type @@ -290,7 +290,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & cs_veg%deadcrootc_patch(p) = cs_veg%deadcrootc_patch(p) + cf_veg%deadcrootc_xfer_to_deadcrootc_patch(p)*dt cs_veg%deadcrootc_xfer_patch(p) = cs_veg%deadcrootc_xfer_patch(p) - cf_veg%deadcrootc_xfer_to_deadcrootc_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero cs_veg%livestemc_patch(p) = cs_veg%livestemc_patch(p) + cf_veg%livestemc_xfer_to_livestemc_patch(p)*dt cs_veg%livestemc_xfer_patch(p) = cs_veg%livestemc_xfer_patch(p) - cf_veg%livestemc_xfer_to_livestemc_patch(p)*dt @@ -313,7 +313,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & cs_veg%livecrootc_patch(p) = cs_veg%livecrootc_patch(p) - cf_veg%livecrootc_to_deadcrootc_patch(p)*dt cs_veg%deadcrootc_patch(p) = cs_veg%deadcrootc_patch(p) + cf_veg%livecrootc_to_deadcrootc_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cs_veg%livestemc_patch(p) = cs_veg%livestemc_patch(p) - cf_veg%livestemc_to_litter_patch(p)*dt cs_veg%livestemc_patch(p) = cs_veg%livestemc_patch(p) - & (cf_veg%livestemc_to_biofuelc_patch(p) + cf_veg%livestemc_to_removedresiduec_patch(p))*dt @@ -337,7 +337,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & ! This part below MUST match exactly the code for the non-matrix part ! above! - if (ivt(p) >= npcropmin) then + if (is_prognostic_crop(ivt(p))) then cs_veg%cropseedc_deficit_patch(p) = cs_veg%cropseedc_deficit_patch(p) & - cf_veg%crop_seedc_to_leaf_patch(p) * dt do k = repr_grain_min, repr_grain_max @@ -359,7 +359,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%livestem_curmr_patch(p)*dt cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%livecroot_curmr_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%livestem_curmr_patch(p)*dt do k = 1, nrepr cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%reproductive_curmr_patch(p,k)*dt @@ -432,7 +432,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & ! NOTE: The equivalent changes for matrix code are in CNPhenology EBK (11/26/2019) end if !not use_matrixcn end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops if (carbon_resp_opt == 1) then cf_veg%cpool_to_livestemc_patch(p) = cf_veg%cpool_to_livestemc_patch(p) - cf_veg%cpool_to_livestemc_resp_patch(p) cf_veg%cpool_to_livestemc_storage_patch(p) = cf_veg%cpool_to_livestemc_storage_patch(p) - & @@ -468,7 +468,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_livecroot_gr_patch(p)*dt cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_deadcroot_gr_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_livestem_gr_patch(p)*dt do k = 1, nrepr cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_reproductive_gr_patch(p,k)*dt @@ -484,7 +484,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & cs_veg%gresp_xfer_patch(p) = cs_veg%gresp_xfer_patch(p) - cf_veg%transfer_livecroot_gr_patch(p)*dt cs_veg%gresp_xfer_patch(p) = cs_veg%gresp_xfer_patch(p) - cf_veg%transfer_deadcroot_gr_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cs_veg%gresp_xfer_patch(p) = cs_veg%gresp_xfer_patch(p) - cf_veg%transfer_livestem_gr_patch(p)*dt do k = 1, nrepr cs_veg%gresp_xfer_patch(p) = cs_veg%gresp_xfer_patch(p) - cf_veg%transfer_reproductive_gr_patch(p,k)*dt @@ -501,7 +501,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_livecroot_storage_gr_patch(p)*dt cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_deadcroot_storage_gr_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_livestem_storage_gr_patch(p)*dt do k = 1, nrepr @@ -539,7 +539,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & ! NOTE: The equivalent changes for matrix code are in CNPhenology EBK (11/26/2019) end if !not use_matrixcn end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero if(.not. use_matrixcn)then ! lines here for consistency; the transfer terms are zero @@ -556,7 +556,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & end if !not use_matrixcn end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cs_veg%xsmrpool_patch(p) = cs_veg%xsmrpool_patch(p) - cf_veg%livestem_xsmr_patch(p)*dt do k = 1, nrepr cs_veg%xsmrpool_patch(p) = cs_veg%xsmrpool_patch(p) - cf_veg%reproductive_xsmr_patch(p,k)*dt diff --git a/src/biogeochem/CNFUNMod.F90 b/src/biogeochem/CNFUNMod.F90 index cdbe7ba71b..e264786c0b 100644 --- a/src/biogeochem/CNFUNMod.F90 +++ b/src/biogeochem/CNFUNMod.F90 @@ -23,7 +23,7 @@ module CNFUNMod use clm_varctl , only : iulog use PatchType , only : patch use ColumnType , only : col - use pftconMod , only : pftcon, npcropmin + use pftconMod , only : pftcon use decompMod , only : bounds_type use clm_varctl , only : use_nitrif_denitrif,use_flexiblecn use CNSharedParamsMod , only : use_matrixcn @@ -284,7 +284,7 @@ subroutine CNFUN(bounds,num_soilc, filter_soilc,num_soilp& use clm_varctl , only : use_nitrif_denitrif use PatchType , only : patch use subgridAveMod , only : p2c - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use CNVegMatrixMod , only : matrix_update_phn ! ! !ARGUMENTS: @@ -1271,7 +1271,7 @@ subroutine CNFUN(bounds,num_soilc, filter_soilc,num_soilp& ! Calculate appropriate degree of retranslocation !------------------------------------------------------------------------------- - if(leafc(p).gt.0.0_r8.and.litterfall_n_step(p,istp)* fixerfrac>0.0_r8.and.ivt(p) 0.0_r8.and. (.not. is_prognostic_crop(ivt(p))))then call fun_retranslocation(p,dt,npp_to_spend,& litterfall_c_step(p,istp)* fixerfrac,& litterfall_n_step(p,istp)* fixerfrac,& diff --git a/src/biogeochem/CNFireEmissionsMod.F90 b/src/biogeochem/CNFireEmissionsMod.F90 index 5a15e138d5..1ee8facfd4 100644 --- a/src/biogeochem/CNFireEmissionsMod.F90 +++ b/src/biogeochem/CNFireEmissionsMod.F90 @@ -347,7 +347,7 @@ function vert_dist_top( veg_type ) result(ztop) use pftconMod , only : nbrdlf_evr_shrub, nbrdlf_dcd_brl_shrub use pftconMod , only : nc3_arctic_grass, nc3_nonarctic_grass use pftconMod , only : nc3crop, nc3irrig - use pftconMod , only : npcropmin, npcropmax + use pftconMod , only : is_prognostic_crop implicit none integer, intent(in) :: veg_type @@ -376,7 +376,7 @@ function vert_dist_top( veg_type ) result(ztop) else if ( veg_type == nc3crop .or. veg_type <= nc3irrig ) then ztop = 1.e3_r8 ! m ! Prognostic crops - else if ( veg_type >= npcropmin .and. veg_type <= npcropmax ) then + else if (is_prognostic_crop(veg_type)) then ztop = 1.e3_r8 ! m else call endrun('ERROR:: undefined veg_type' ) diff --git a/src/biogeochem/CNGRespMod.F90 b/src/biogeochem/CNGRespMod.F90 index de8b145615..29d25487e4 100644 --- a/src/biogeochem/CNGRespMod.F90 +++ b/src/biogeochem/CNGRespMod.F90 @@ -7,7 +7,7 @@ module CNGRespMod ! ! !USES: use shr_kind_mod , only : r8 => shr_kind_r8 - use pftconMod , only : npcropmin, pftcon + use pftconMod , only : is_prognostic_crop, pftcon use CNVegcarbonfluxType , only : cnveg_carbonflux_type use PatchType , only : patch use CanopyStateType , only : canopystate_type @@ -145,7 +145,7 @@ subroutine CNGResp(num_soilp, filter_soilp, cnveg_carbonflux_inst, canopystate_i respfact_livecroot_storage = 1.0_r8 respfact_livestem_storage = 1.0_r8 - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cpool_livestem_gr(p) = cpool_to_livestemc(p) * grperc(ivt(p)) * respfact_livestem cpool_livestem_storage_gr(p) = cpool_to_livestemc_storage(p) * grperc(ivt(p)) * grpnow(ivt(p)) * & diff --git a/src/biogeochem/CNGapMortalityMod.F90 b/src/biogeochem/CNGapMortalityMod.F90 index 24f0f6b145..6b9859265b 100644 --- a/src/biogeochem/CNGapMortalityMod.F90 +++ b/src/biogeochem/CNGapMortalityMod.F90 @@ -121,7 +121,7 @@ subroutine CNGapMortality (bounds, num_soilp, filter_soilp, & use clm_varpar , only: nlevdecomp_full use clm_varcon , only: secspday use clm_varctl , only: use_cndv, spinup_state - use pftconMod , only: npcropmin + use pftconMod , only: is_prognostic_crop ! ! !ARGUMENTS: type(bounds_type) , intent(in) :: bounds @@ -348,7 +348,7 @@ subroutine CNGapMortality (bounds, num_soilp, filter_soilp, & end if !use_matrixcn end if - if (ivt(p) < npcropmin) then + if (.not. is_prognostic_crop(ivt(p))) then if(.not. use_matrixcn)then cnveg_nitrogenflux_inst%m_retransn_to_litter_patch(p) = cnveg_nitrogenstate_inst%retransn_patch(p) * m else diff --git a/src/biogeochem/CNMRespMod.F90 b/src/biogeochem/CNMRespMod.F90 index 33eb0b0e9a..ae0040008b 100644 --- a/src/biogeochem/CNMRespMod.F90 +++ b/src/biogeochem/CNMRespMod.F90 @@ -13,7 +13,7 @@ module CNMRespMod use decompMod , only : bounds_type use abortutils , only : endrun use shr_log_mod , only : errMsg => shr_log_errMsg - use pftconMod , only : npcropmin, pftcon + use pftconMod , only : is_prognostic_crop, pftcon use SoilStateType , only : soilstate_type use CanopyStateType , only : canopystate_type use TemperatureType , only : temperature_type @@ -265,7 +265,7 @@ subroutine CNMResp(bounds, num_soilc, filter_soilc, num_soilp, filter_soilp, & if (woody(ivt(p)) == 1) then livestem_mr(p) = livestemn(p)*br*tc livecroot_mr(p) = livecrootn(p)*br_root*tc - else if (ivt(p) >= npcropmin) then + else if (is_prognostic_crop(ivt(p))) then livestem_mr(p) = livestemn(p)*br*tc do k = 1, nrepr reproductive_mr(p,k) = reproductiven(p,k)*br*tc diff --git a/src/biogeochem/CNNStateUpdate1Mod.F90 b/src/biogeochem/CNNStateUpdate1Mod.F90 index 742afa77dd..7bd346c82a 100644 --- a/src/biogeochem/CNNStateUpdate1Mod.F90 +++ b/src/biogeochem/CNNStateUpdate1Mod.F90 @@ -17,7 +17,7 @@ module CNNStateUpdate1Mod use SoilBiogeochemDecompCascadeConType, only : decomp_method, mimics_decomp, use_soil_matrixcn use CNSharedParamsMod , only : use_matrixcn use clm_varcon , only : nitrif_n2o_loss_frac - use pftconMod , only : npcropmin, pftcon + use pftconMod , only : is_prognostic_crop, pftcon use decompMod , only : bounds_type use CNVegNitrogenStateType , only : cnveg_nitrogenstate_type use CNVegNitrogenFluxType , only : cnveg_nitrogenflux_type @@ -228,7 +228,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & ns_veg%deadcrootn_xfer_patch(p) = ns_veg%deadcrootn_xfer_patch(p) - nf_veg%deadcrootn_xfer_to_deadcrootn_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero ns_veg%livestemn_patch(p) = ns_veg%livestemn_patch(p) + nf_veg%livestemn_xfer_to_livestemn_patch(p)*dt ns_veg%livestemn_xfer_patch(p) = ns_veg%livestemn_xfer_patch(p) - nf_veg%livestemn_xfer_to_livestemn_patch(p)*dt @@ -287,7 +287,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & ! NOTE: The equivalent changes for matrix code are in CNPhenology EBK (11/26/2019) end if !not use_matrixcn end if - if (ivt(p) >= npcropmin) then ! Beth adds retrans from froot + if (is_prognostic_crop(ivt(p))) then ! Beth adds retrans from froot ! ! State update without the matrix solution ! @@ -391,7 +391,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & end if ! not use_matrixcn end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops ns_veg%npool_patch(p) = ns_veg%npool_patch(p) - nf_veg%npool_to_livestemn_patch(p)*dt ns_veg%npool_patch(p) = ns_veg%npool_patch(p) - nf_veg%npool_to_livestemn_storage_patch(p)*dt do k = 1, nrepr @@ -452,7 +452,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & ! NOTE: The equivalent changes for matrix code are in CNPhenology EBK (11/26/2019) end if ! not use_matrixcn - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero ! diff --git a/src/biogeochem/CNPhenologyMod.F90 b/src/biogeochem/CNPhenologyMod.F90 index ff1e5b1b74..f19d57aa03 100644 --- a/src/biogeochem/CNPhenologyMod.F90 +++ b/src/biogeochem/CNPhenologyMod.F90 @@ -3350,7 +3350,7 @@ subroutine CNOffsetLitterfall (num_soilp, filter_soilp, & ! pools during the phenological offset period. ! ! !USES: - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use pftconMod , only : nmiscanthus, nirrig_miscanthus, nswitchgrass, nirrig_switchgrass use CNSharedParamsMod, only : use_fun @@ -3526,7 +3526,7 @@ subroutine CNOffsetLitterfall (num_soilp, filter_soilp, & end if ! use_matrixcn ! this assumes that offset_counter == dt for crops ! if this were ever changed, we'd need to add code to the "else" - if (ivt(p) >= npcropmin) then + if (is_prognostic_crop(ivt(p))) then ! How many harvests have occurred? h = crop_inst%harvest_count(p) @@ -4371,7 +4371,7 @@ subroutine CNLitterToColumn (bounds, num_bgc_vegp, filter_bgc_vegp, & ! ! !USES: use clm_varpar , only : nlevdecomp - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use clm_varctl , only : use_grainproduct ! ! !ARGUMENTS: @@ -4449,7 +4449,7 @@ subroutine CNLitterToColumn (bounds, num_bgc_vegp, filter_bgc_vegp, & ! new ones for now (slevis) ! also for simplicity I've put "food" into the litter pools - if (ivt(p) >= npcropmin) then ! add livestemc to litter + if (is_prognostic_crop(ivt(p))) then ! add livestemc to litter do i = i_litr_min, i_litr_max ! stem litter carbon fluxes phenology_c_to_litr_c(c,j,i) = & diff --git a/src/biogeochem/CNVegCarbonFluxType.F90 b/src/biogeochem/CNVegCarbonFluxType.F90 index b4c581c081..12daf746af 100644 --- a/src/biogeochem/CNVegCarbonFluxType.F90 +++ b/src/biogeochem/CNVegCarbonFluxType.F90 @@ -27,7 +27,7 @@ module CNVegCarbonFluxType use clm_varctl , only : use_grainproduct use clm_varctl , only : iulog use landunit_varcon , only : istsoil, istcrop, istdlak - use pftconMod , only : npcropmin, pftcon + use pftconMod , only : is_prognostic_crop, pftcon use CropReprPoolsMod , only : nrepr, repr_grain_min, repr_grain_max, repr_structure_min, repr_structure_max use CropReprPoolsMod , only : get_repr_hist_fname, get_repr_rest_fname, get_repr_longname use LandunitType , only : lun @@ -4921,7 +4921,7 @@ subroutine Summary_carbonflux(this, & this%livestem_mr_patch(p) + & this%livecroot_mr_patch(p) end if - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%mr_patch(p) = & this%mr_patch(p) + & @@ -4939,7 +4939,7 @@ subroutine Summary_carbonflux(this, & this%cpool_deadstem_gr_patch(p) + & this%cpool_livecroot_gr_patch(p) + & this%cpool_deadcroot_gr_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%current_gr_patch(p) = this%current_gr_patch(p) + & this%cpool_reproductive_gr_patch(p,k) @@ -4955,7 +4955,7 @@ subroutine Summary_carbonflux(this, & this%transfer_deadstem_gr_patch(p) + & this%transfer_livecroot_gr_patch(p) + & this%transfer_deadcroot_gr_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%transfer_gr_patch(p) = this%transfer_gr_patch(p) + & this%transfer_reproductive_gr_patch(p,k) @@ -4971,7 +4971,7 @@ subroutine Summary_carbonflux(this, & this%cpool_livecroot_storage_gr_patch(p) + & this%cpool_deadcroot_storage_gr_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%storage_gr_patch(p) = this%storage_gr_patch(p) + & this%cpool_reproductive_storage_gr_patch(p,k) @@ -4985,7 +4985,7 @@ subroutine Summary_carbonflux(this, & this%storage_gr_patch(p) ! autotrophic respiration (AR) adn - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then this%ar_patch(p) = & this%mr_patch(p) + & this%gr_patch(p) @@ -5045,7 +5045,7 @@ subroutine Summary_carbonflux(this, & this%cpool_to_deadstemc_patch(p) + & this%deadstemc_xfer_to_deadstemc_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%agnpp_patch(p) = & this%agnpp_patch(p) + & @@ -5139,7 +5139,7 @@ subroutine Summary_carbonflux(this, & this%gru_livecrootc_to_litter_patch(p) + & this%gru_deadcrootc_to_litter_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then this%litfall_patch(p) = & this%litfall_patch(p) + & this%livestemc_to_litter_patch(p) diff --git a/src/biogeochem/CNVegCarbonStateType.F90 b/src/biogeochem/CNVegCarbonStateType.F90 index 142578e656..3604821057 100644 --- a/src/biogeochem/CNVegCarbonStateType.F90 +++ b/src/biogeochem/CNVegCarbonStateType.F90 @@ -9,7 +9,7 @@ module CNVegCarbonStateType use shr_infnan_mod , only : nan => shr_infnan_nan, assignment(=) use shr_const_mod , only : SHR_CONST_PDB use shr_log_mod , only : errMsg => shr_log_errMsg - use pftconMod , only : noveg, npcropmin, pftcon, nc3crop, nc3irrig + use pftconMod , only : noveg, is_prognostic_crop, pftcon, nc3crop, nc3irrig use clm_varcon , only : spval, c3_r2, c4_r2, c14ratio use clm_varctl , only : iulog, use_cndv, use_crop use CNSharedParamsMod, only : use_matrixcn @@ -1532,7 +1532,7 @@ subroutine InitCold(this, bounds, ratio, carbon_type, c12_cnveg_carbonstate_inst this%matrix_cap_frootc_patch(p) = cnvegcstate_const%initial_vegC * ratio this%matrix_cap_frootc_storage_patch(p) = 0._r8 end if - else if (patch%itype(p) >= npcropmin) then ! prognostic crop types + else if (is_prognostic_crop(patch%itype(p))) then ! prognostic crop types this%leafc_patch(p) = 0._r8 this%leafc_storage_patch(p) = 0._r8 this%frootc_patch(p) = 0._r8 @@ -4578,7 +4578,7 @@ subroutine Summary_carbonstate(this, bounds, num_bgc_soilc, filter_bgc_soilc, nu this%gresp_storage_patch(p) + & this%gresp_xfer_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%storvegc_patch(p) = & this%storvegc_patch(p) + & diff --git a/src/biogeochem/CNVegMatrixMod.F90 b/src/biogeochem/CNVegMatrixMod.F90 index 5582afeffe..41339d3a2c 100644 --- a/src/biogeochem/CNVegMatrixMod.F90 +++ b/src/biogeochem/CNVegMatrixMod.F90 @@ -35,7 +35,7 @@ module CNVegMatrixMod ncphouttrans,nnphouttrans,ncgmouttrans,nngmouttrans,ncfiouttrans,nnfiouttrans use perf_mod , only : t_startf, t_stopf use PatchType , only : patch - use pftconMod , only : pftcon,npcropmin + use pftconMod , only : pftcon,is_prognostic_crop use CNVegCarbonStateType , only : cnveg_carbonstate_type use CNVegNitrogenStateType , only : cnveg_nitrogenstate_type use CNVegCarbonFluxType , only : cnveg_carbonflux_type !include: callocation,ctransfer, cturnover @@ -1140,7 +1140,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on Xvegc%V(p,igrain) = reproductivec(p,irepr) Xvegc%V(p,igrain_st) = reproductivec_storage(p,irepr) @@ -1173,7 +1173,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on Xveg13c%V(p,igrain) = cs13_veg%reproductivec_patch(p,irepr) Xveg13c%V(p,igrain_st) = cs13_veg%reproductivec_storage_patch(p,irepr) @@ -1207,7 +1207,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on Xveg14c%V(p,igrain) = cs14_veg%reproductivec_patch(p,irepr) Xveg14c%V(p,igrain_st) = cs14_veg%reproductivec_storage_patch(p,irepr) @@ -1241,7 +1241,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then Xvegn%V(p,igrain) = sum(reproductiven(p,:)) Xvegn%V(p,igrain_st) = sum(reproductiven_storage(p,:)) Xvegn%V(p,igrain_xf) = sum(reproductiven_xfer(p,:)) @@ -1279,7 +1279,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on reproc0(p) = max(reproductivec(p,irepr), epsi) reproc0_storage(p) = max(reproductivec_storage(p,irepr), epsi) @@ -1312,7 +1312,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on cs13_veg%reproc0_patch(p) = max(cs13_veg%reproductivec_patch(p,irepr), epsi) cs13_veg%reproc0_storage_patch(p) = max(cs13_veg%reproductivec_storage_patch(p,irepr), epsi) @@ -1346,7 +1346,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on cs14_veg%reproc0_patch(p) = max(cs14_veg%reproductivec_patch(p,irepr), epsi) cs14_veg%reproc0_storage_patch(p) = max(cs14_veg%reproductivec_storage_patch(p,irepr), epsi) @@ -1380,7 +1380,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on repron0(p) = max(reproductiven(p,irepr), epsi) repron0_storage(p) = max(reproductiven_storage(p,irepr), epsi) @@ -1730,7 +1730,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire end do do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_calloc_grain_acc(p) = matrix_calloc_grain_acc(p) + vegmatrixc_input%V(p,igrain) matrix_calloc_grainst_acc(p) = matrix_calloc_grainst_acc(p) + vegmatrixc_input%V(p,igrain_st) if(use_c13)then @@ -2052,7 +2052,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire end do do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_cturnover_grain_acc(p) = matrix_cturnover_grain_acc(p) & + (matrix_phturnover(p,igrain)+matrix_gmturnover(p,igrain)+matrix_fiturnover(p,igrain)) & * reproductivec(p,irepr) @@ -2110,7 +2110,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_nalloc_grain_acc(p) = matrix_nalloc_grain_acc(p) + vegmatrixn_input%V(p,igrain) matrix_nalloc_grainst_acc(p) = matrix_nalloc_grainst_acc(p) + vegmatrixn_input%V(p,igrain_st) end if @@ -2215,7 +2215,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_retransn_to_grain_acc(p) = matrix_ntransfer_retransn_to_grain_acc(p) & + matrix_nphtransfer(p,iretransn_to_igrain_phn) & * dt * retransn(p)!matrix_nphturnover(p,iretransn)*retransn(p) @@ -2287,7 +2287,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire end do do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_nturnover_grain_acc(p) = matrix_nturnover_grain_acc(p) & + (matrix_nphturnover(p,igrain)+matrix_ngmturnover(p,igrain)+matrix_nfiturnover(p,igrain)) & * reproductiven(p,irepr) @@ -2328,7 +2328,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! NOTE: This assumes only a single grain pool! (i.e nrepr is ! fixed at 1)! reproductivec(p,:) = Xvegc%V(p,igrain) @@ -2362,7 +2362,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! NOTE: This assumes only a single grain pool! (i.e nrepr is ! fixed at 1)! cs13_veg%reproductivec_patch(p,:) = Xveg13c%V(p,igrain) @@ -2396,7 +2396,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire end do do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! NOTE: This assumes only a single grain pool! (i.e nrepr is ! fixed at 1)! cs14_veg%reproductivec_patch(p,:) = Xveg14c%V(p,igrain) @@ -2431,7 +2431,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then reproductiven(p,:) = Xvegn%V(p,igrain) reproductiven_storage(p,:) = Xvegn%V(p,igrain_st) reproductiven_xfer(p,:) = Xvegn%V(p,igrain_xf) @@ -2457,7 +2457,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_calloc_acc(ilivecroot_st) = matrix_calloc_livecrootst_acc(p) matrix_calloc_acc(ideadcroot) = matrix_calloc_deadcroot_acc(p) matrix_calloc_acc(ideadcroot_st) = matrix_calloc_deadcrootst_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_calloc_acc(igrain) = matrix_calloc_grain_acc(p) matrix_calloc_acc(igrain_st) = matrix_calloc_grainst_acc(p) end if @@ -2474,7 +2474,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ctransfer_acc(ilivecroot,ilivecroot_xf) = matrix_ctransfer_livecrootxf_to_livecroot_acc(p) matrix_ctransfer_acc(ideadcroot_xf,ideadcroot_st) = matrix_ctransfer_deadcrootst_to_deadcrootxf_acc(p) matrix_ctransfer_acc(ideadcroot,ideadcroot_xf) = matrix_ctransfer_deadcrootxf_to_deadcroot_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ctransfer_acc(igrain_xf,igrain_st) = matrix_ctransfer_grainst_to_grainxf_acc(p) matrix_ctransfer_acc(igrain,igrain_xf) = matrix_ctransfer_grainxf_to_grain_acc(p) end if @@ -2499,7 +2499,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ctransfer_acc(ideadcroot,ideadcroot) = -matrix_cturnover_deadcroot_acc(p) matrix_ctransfer_acc(ideadcroot_st,ideadcroot_st) = -matrix_cturnover_deadcrootst_acc(p) matrix_ctransfer_acc(ideadcroot_xf,ideadcroot_xf) = -matrix_cturnover_deadcrootxf_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ctransfer_acc(igrain,igrain) = -matrix_cturnover_grain_acc(p) matrix_ctransfer_acc(igrain_st,igrain_st) = -matrix_cturnover_grainst_acc(p) matrix_ctransfer_acc(igrain_xf,igrain_xf) = -matrix_cturnover_grainxf_acc(p) @@ -2518,7 +2518,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c13alloc_acc(ilivecroot_st) = cs13_veg%matrix_calloc_livecrootst_acc_patch(p) matrix_c13alloc_acc(ideadcroot) = cs13_veg%matrix_calloc_deadcroot_acc_patch(p) matrix_c13alloc_acc(ideadcroot_st) = cs13_veg%matrix_calloc_deadcrootst_acc_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c13alloc_acc(igrain) = cs13_veg%matrix_calloc_grain_acc_patch(p) matrix_c13alloc_acc(igrain_st) = cs13_veg%matrix_calloc_grainst_acc_patch(p) end if @@ -2535,7 +2535,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c13transfer_acc(ilivecroot,ilivecroot_xf) = cs13_veg%matrix_ctransfer_livecrootxf_to_livecroot_acc_patch(p) matrix_c13transfer_acc(ideadcroot_xf,ideadcroot_st) = cs13_veg%matrix_ctransfer_deadcrootst_to_deadcrootxf_acc_patch(p) matrix_c13transfer_acc(ideadcroot,ideadcroot_xf) = cs13_veg%matrix_ctransfer_deadcrootxf_to_deadcroot_acc_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c13transfer_acc(igrain_xf,igrain_st) = cs13_veg%matrix_ctransfer_grainst_to_grainxf_acc_patch(p) matrix_c13transfer_acc(igrain,igrain_xf) = cs13_veg%matrix_ctransfer_grainxf_to_grain_acc_patch(p) end if @@ -2560,7 +2560,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c13transfer_acc(ideadcroot,ideadcroot) = -cs13_veg%matrix_cturnover_deadcroot_acc_patch(p) matrix_c13transfer_acc(ideadcroot_st,ideadcroot_st) = -cs13_veg%matrix_cturnover_deadcrootst_acc_patch(p) matrix_c13transfer_acc(ideadcroot_xf,ideadcroot_xf) = -cs13_veg%matrix_cturnover_deadcrootxf_acc_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c13transfer_acc(igrain,igrain) = -cs13_veg%matrix_cturnover_grain_acc_patch(p) matrix_c13transfer_acc(igrain_st,igrain_st) = -cs13_veg%matrix_cturnover_grainst_acc_patch(p) matrix_c13transfer_acc(igrain_xf,igrain_xf) = -cs13_veg%matrix_cturnover_grainxf_acc_patch(p) @@ -2580,7 +2580,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c14alloc_acc(ilivecroot_st) = cs14_veg%matrix_calloc_livecrootst_acc_patch(p) matrix_c14alloc_acc(ideadcroot) = cs14_veg%matrix_calloc_deadcroot_acc_patch(p) matrix_c14alloc_acc(ideadcroot_st) = cs14_veg%matrix_calloc_deadcrootst_acc_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c14alloc_acc(igrain) = cs14_veg%matrix_calloc_grain_acc_patch(p) matrix_c14alloc_acc(igrain_st) = cs14_veg%matrix_calloc_grainst_acc_patch(p) end if @@ -2597,7 +2597,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c14transfer_acc(ilivecroot,ilivecroot_xf) = cs14_veg%matrix_ctransfer_livecrootxf_to_livecroot_acc_patch(p) matrix_c14transfer_acc(ideadcroot_xf,ideadcroot_st) = cs14_veg%matrix_ctransfer_deadcrootst_to_deadcrootxf_acc_patch(p) matrix_c14transfer_acc(ideadcroot,ideadcroot_xf) = cs14_veg%matrix_ctransfer_deadcrootxf_to_deadcroot_acc_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c14transfer_acc(igrain_xf,igrain_st) = cs14_veg%matrix_ctransfer_grainst_to_grainxf_acc_patch(p) matrix_c14transfer_acc(igrain,igrain_xf) = cs14_veg%matrix_ctransfer_grainxf_to_grain_acc_patch(p) end if @@ -2622,7 +2622,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c14transfer_acc(ideadcroot,ideadcroot) = -cs14_veg%matrix_cturnover_deadcroot_acc_patch(p) matrix_c14transfer_acc(ideadcroot_st,ideadcroot_st) = -cs14_veg%matrix_cturnover_deadcrootst_acc_patch(p) matrix_c14transfer_acc(ideadcroot_xf,ideadcroot_xf) = -cs14_veg%matrix_cturnover_deadcrootxf_acc_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c14transfer_acc(igrain,igrain) = -cs14_veg%matrix_cturnover_grain_acc_patch(p) matrix_c14transfer_acc(igrain_st,igrain_st) = -cs14_veg%matrix_cturnover_grainst_acc_patch(p) matrix_c14transfer_acc(igrain_xf,igrain_xf) = -cs14_veg%matrix_cturnover_grainxf_acc_patch(p) @@ -2641,7 +2641,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_nalloc_acc(ilivecroot_st) = matrix_nalloc_livecrootst_acc(p) matrix_nalloc_acc(ideadcroot) = matrix_nalloc_deadcroot_acc(p) matrix_nalloc_acc(ideadcroot_st) = matrix_nalloc_deadcrootst_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_nalloc_acc(igrain) = matrix_nalloc_grain_acc(p) matrix_nalloc_acc(igrain_st) = matrix_nalloc_grainst_acc(p) end if @@ -2658,7 +2658,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ntransfer_acc(ilivecroot,ilivecroot_xf) = matrix_ntransfer_livecrootxf_to_livecroot_acc(p) matrix_ntransfer_acc(ideadcroot_xf,ideadcroot_st) = matrix_ntransfer_deadcrootst_to_deadcrootxf_acc(p) matrix_ntransfer_acc(ideadcroot,ideadcroot_xf) = matrix_ntransfer_deadcrootxf_to_deadcroot_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_acc(igrain_xf,igrain_st) = matrix_ntransfer_grainst_to_grainxf_acc(p) matrix_ntransfer_acc(igrain,igrain_xf) = matrix_ntransfer_grainxf_to_grain_acc(p) end if @@ -2677,7 +2677,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ntransfer_acc(ilivecroot_st,iretransn) = matrix_ntransfer_retransn_to_livecrootst_acc(p) matrix_ntransfer_acc(ideadcroot,iretransn) = matrix_ntransfer_retransn_to_deadcroot_acc(p) matrix_ntransfer_acc(ideadcroot_st,iretransn) = matrix_ntransfer_retransn_to_deadcrootst_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_acc(igrain,iretransn) = matrix_ntransfer_retransn_to_grain_acc(p) matrix_ntransfer_acc(igrain_st,iretransn) = matrix_ntransfer_retransn_to_grainst_acc(p) end if @@ -2704,7 +2704,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ntransfer_acc(ideadcroot,ideadcroot) = -matrix_nturnover_deadcroot_acc(p) matrix_ntransfer_acc(ideadcroot_st,ideadcroot_st) = -matrix_nturnover_deadcrootst_acc(p) matrix_ntransfer_acc(ideadcroot_xf,ideadcroot_xf) = -matrix_nturnover_deadcrootxf_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_acc(igrain,igrain) = -matrix_nturnover_grain_acc(p) matrix_ntransfer_acc(igrain_st,igrain_st) = -matrix_nturnover_grainst_acc(p) matrix_ntransfer_acc(igrain_xf,igrain_xf) = -matrix_nturnover_grainxf_acc(p) @@ -2755,7 +2755,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ctransfer_acc(1:nvegcpool,ideadcroot) = matrix_ctransfer_acc(1:nvegcpool,ideadcroot) / deadcrootc0(p) matrix_ctransfer_acc(1:nvegcpool,ideadcroot_st) = matrix_ctransfer_acc(1:nvegcpool,ideadcroot_st) / deadcrootc0_storage(p) matrix_ctransfer_acc(1:nvegcpool,ideadcroot_xf) = matrix_ctransfer_acc(1:nvegcpool,ideadcroot_xf) / deadcrootc0_xfer(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ctransfer_acc(1:nvegcpool,igrain) = matrix_ctransfer_acc(1:nvegcpool,igrain) / reproc0(p) matrix_ctransfer_acc(1:nvegcpool,igrain_st) = matrix_ctransfer_acc(1:nvegcpool,igrain_st) / reproc0_storage(p) matrix_ctransfer_acc(1:nvegcpool,igrain_xf) = matrix_ctransfer_acc(1:nvegcpool,igrain_xf) / reproc0_xfer(p) @@ -2780,7 +2780,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c13transfer_acc(1:nvegcpool,ideadcroot) = matrix_c13transfer_acc(1:nvegcpool,ideadcroot) / cs13_veg%deadcrootc0_patch(p) matrix_c13transfer_acc(1:nvegcpool,ideadcroot_st) = matrix_c13transfer_acc(1:nvegcpool,ideadcroot_st) / cs13_veg%deadcrootc0_storage_patch(p) matrix_c13transfer_acc(1:nvegcpool,ideadcroot_xf) = matrix_c13transfer_acc(1:nvegcpool,ideadcroot_xf) / cs13_veg%deadcrootc0_xfer_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c13transfer_acc(1:nvegcpool,igrain) = matrix_c13transfer_acc(1:nvegcpool,igrain) / cs13_veg%reproc0_patch(p) matrix_c13transfer_acc(1:nvegcpool,igrain_st) = matrix_c13transfer_acc(1:nvegcpool,igrain_st) / cs13_veg%reproc0_storage_patch(p) matrix_c13transfer_acc(1:nvegcpool,igrain_xf) = matrix_c13transfer_acc(1:nvegcpool,igrain_xf) / cs13_veg%reproc0_xfer_patch(p) @@ -2806,7 +2806,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c14transfer_acc(1:nvegcpool,ideadcroot) = matrix_c14transfer_acc(1:nvegcpool,ideadcroot) / cs14_veg%deadcrootc0_patch(p) matrix_c14transfer_acc(1:nvegcpool,ideadcroot_st) = matrix_c14transfer_acc(1:nvegcpool,ideadcroot_st) / cs14_veg%deadcrootc0_storage_patch(p) matrix_c14transfer_acc(1:nvegcpool,ideadcroot_xf) = matrix_c14transfer_acc(1:nvegcpool,ideadcroot_xf) / cs14_veg%deadcrootc0_xfer_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c14transfer_acc(1:nvegcpool,igrain) = matrix_c14transfer_acc(1:nvegcpool,igrain) / cs14_veg%reproc0_patch(p) matrix_c14transfer_acc(1:nvegcpool,igrain_st) = matrix_c14transfer_acc(1:nvegcpool,igrain_st) / cs14_veg%reproc0_storage_patch(p) matrix_c14transfer_acc(1:nvegcpool,igrain_xf) = matrix_c14transfer_acc(1:nvegcpool,igrain_xf) / cs14_veg%reproc0_xfer_patch(p) @@ -2831,7 +2831,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ntransfer_acc(1:nvegnpool,ideadcroot) = matrix_ntransfer_acc(1:nvegnpool,ideadcroot) / deadcrootn0(p) matrix_ntransfer_acc(1:nvegnpool,ideadcroot_st) = matrix_ntransfer_acc(1:nvegnpool,ideadcroot_st) / deadcrootn0_storage(p) matrix_ntransfer_acc(1:nvegnpool,ideadcroot_xf) = matrix_ntransfer_acc(1:nvegnpool,ideadcroot_xf) / deadcrootn0_xfer(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_acc(1:nvegnpool,igrain) = matrix_ntransfer_acc(1:nvegnpool,igrain) / repron0(p) matrix_ntransfer_acc(1:nvegnpool,igrain_st) = matrix_ntransfer_acc(1:nvegnpool,igrain_st) / repron0_storage(p) matrix_ntransfer_acc(1:nvegnpool,igrain_xf) = matrix_ntransfer_acc(1:nvegnpool,igrain_xf) / repron0_xfer(p) @@ -2923,7 +2923,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire deadcrootc_SASUsave(p) = deadcrootc_SASUsave(p) + deadcrootc(p) deadcrootc_storage_SASUsave(p) = deadcrootc_storage_SASUsave(p) + deadcrootc_storage(p) deadcrootc_xfer_SASUsave(p) = deadcrootc_xfer_SASUsave(p) + deadcrootc_xfer(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then grainc_SASUsave(p) = grainc_SASUsave(p) + sum(reproductivec(p,:)) grainc_storage_SASUsave(p) = grainc_storage_SASUsave(p) + sum(reproductivec_storage(p,:)) end if @@ -2946,7 +2946,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%deadcrootc_SASUsave_patch(p) = cs13_veg%deadcrootc_SASUsave_patch(p) + cs13_veg%deadcrootc_patch(p) cs13_veg%deadcrootc_storage_SASUsave_patch(p) = cs13_veg%deadcrootc_storage_SASUsave_patch(p) + cs13_veg%deadcrootc_storage_patch(p) cs13_veg%deadcrootc_xfer_SASUsave_patch(p) = cs13_veg%deadcrootc_xfer_SASUsave_patch(p) + cs13_veg%deadcrootc_xfer_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%grainc_SASUsave_patch(p) = cs13_veg%grainc_SASUsave_patch(p) + cs13_veg%reproductivec_patch(p,irepr) cs13_veg%grainc_storage_SASUsave_patch(p) = cs13_veg%grainc_storage_SASUsave_patch(p) + cs13_veg%reproductivec_storage_patch(p,irepr) end if @@ -2970,7 +2970,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%deadcrootc_SASUsave_patch(p) = cs14_veg%deadcrootc_SASUsave_patch(p) + cs14_veg%deadcrootc_patch(p) cs14_veg%deadcrootc_storage_SASUsave_patch(p) = cs14_veg%deadcrootc_storage_SASUsave_patch(p) + cs14_veg%deadcrootc_storage_patch(p) cs14_veg%deadcrootc_xfer_SASUsave_patch(p) = cs14_veg%deadcrootc_xfer_SASUsave_patch(p) + cs14_veg%deadcrootc_xfer_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%grainc_SASUsave_patch(p) = cs14_veg%grainc_SASUsave_patch(p) + cs14_veg%reproductivec_patch(p,irepr) cs14_veg%grainc_storage_SASUsave_patch(p) = cs14_veg%grainc_storage_SASUsave_patch(p) + cs14_veg%reproductivec_storage_patch(p,irepr) end if @@ -2993,7 +2993,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire deadcrootn_SASUsave(p) = deadcrootn_SASUsave(p) + deadcrootn(p) deadcrootn_storage_SASUsave(p) = deadcrootn_storage_SASUsave(p) + deadcrootn_storage(p) deadcrootn_xfer_SASUsave(p) = deadcrootn_xfer_SASUsave(p) + deadcrootn_xfer(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then grainn_SASUsave(p) = grainn_SASUsave(p) + reproductiven(p,irepr) end if if(iyr .eq. nyr_forcing)then @@ -3015,7 +3015,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire deadcrootc(p) = deadcrootc_SASUsave(p) / (nyr_forcing/nyr_SASU) deadcrootc_storage(p) = deadcrootc_storage_SASUsave(p) / (nyr_forcing/nyr_SASU) deadcrootc_xfer(p) = deadcrootc_xfer_SASUsave(p) / (nyr_forcing/nyr_SASU) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then reproductivec(p,:) = grainc_SASUsave(p) / (nyr_forcing/nyr_SASU) reproductivec_storage(p,:) = grainc_storage_SASUsave(p) / (nyr_forcing/nyr_SASU) end if @@ -3038,7 +3038,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%deadcrootc_patch(p) = cs13_veg%deadcrootc_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) cs13_veg%deadcrootc_storage_patch(p) = cs13_veg%deadcrootc_storage_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) cs13_veg%deadcrootc_xfer_patch(p) = cs13_veg%deadcrootc_xfer_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%reproductivec_patch(p,:) = cs13_veg%grainc_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) cs13_veg%reproductivec_storage_patch(p,:) = cs13_veg%grainc_storage_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) end if @@ -3062,7 +3062,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%deadcrootc_patch(p) = cs14_veg%deadcrootc_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) cs14_veg%deadcrootc_storage_patch(p) = cs14_veg%deadcrootc_storage_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) cs14_veg%deadcrootc_xfer_patch(p) = cs14_veg%deadcrootc_xfer_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%reproductivec_patch(p,:) = cs14_veg%grainc_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) cs14_veg%reproductivec_storage_patch(p,:) = cs14_veg%grainc_storage_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) end if @@ -3085,7 +3085,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire deadcrootn(p) = deadcrootn_SASUsave(p) / (nyr_forcing/nyr_SASU) deadcrootn_storage(p) = deadcrootn_storage_SASUsave(p) / (nyr_forcing/nyr_SASU) deadcrootn_xfer(p) = deadcrootn_xfer_SASUsave(p) / (nyr_forcing/nyr_SASU) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then reproductiven(p,:) = grainn_SASUsave(p) / (nyr_forcing/nyr_SASU) end if leafc_SASUsave(p) = 0 @@ -3106,7 +3106,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire deadcrootc_SASUsave(p) = 0 deadcrootc_storage_SASUsave(p) = 0 deadcrootc_xfer_SASUsave(p) = 0 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then grainc_SASUsave(p) = 0 grainc_storage_SASUsave(p) = 0 end if @@ -3129,7 +3129,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%deadcrootc_SASUsave_patch(p) = 0 cs13_veg%deadcrootc_storage_SASUsave_patch(p) = 0 cs13_veg%deadcrootc_xfer_SASUsave_patch(p) = 0 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%grainc_SASUsave_patch(p) = 0 cs13_veg%grainc_storage_SASUsave_patch(p) = 0 end if @@ -3153,7 +3153,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%deadcrootc_SASUsave_patch(p) = 0 cs14_veg%deadcrootc_storage_SASUsave_patch(p) = 0 cs14_veg%deadcrootc_xfer_SASUsave_patch(p) = 0 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%grainc_SASUsave_patch(p) = 0 cs14_veg%grainc_storage_SASUsave_patch(p) = 0 end if @@ -3176,7 +3176,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire deadcrootn_SASUsave(p) = 0 deadcrootn_storage_SASUsave(p) = 0 deadcrootn_xfer_SASUsave(p) = 0 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then grainn_SASUsave(p) = 0 end if end if @@ -3204,7 +3204,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_cap_deadcrootc(p) = vegmatrixc_rt(ideadcroot) matrix_cap_deadcrootc_storage(p) = vegmatrixc_rt(ideadcroot_st) matrix_cap_deadcrootc_xfer(p) = vegmatrixc_rt(ideadcroot_xf) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_cap_reproc(p) = vegmatrixc_rt(igrain) matrix_cap_reproc_storage(p) = vegmatrixc_rt(igrain_st) matrix_cap_reproc_xfer(p) = vegmatrixc_rt(igrain_xf) @@ -3228,7 +3228,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%matrix_cap_deadcrootc_patch(p) = vegmatrixc13_rt(ideadcroot) cs13_veg%matrix_cap_deadcrootc_storage_patch(p) = vegmatrixc13_rt(ideadcroot_st) cs13_veg%matrix_cap_deadcrootc_xfer_patch(p) = vegmatrixc13_rt(ideadcroot_xf) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%matrix_cap_reproc_patch(p) = vegmatrixc13_rt(igrain) cs13_veg%matrix_cap_reproc_storage_patch(p) = vegmatrixc13_rt(igrain_st) cs13_veg%matrix_cap_reproc_xfer_patch(p) = vegmatrixc13_rt(igrain_xf) @@ -3253,7 +3253,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%matrix_cap_deadcrootc_patch(p) = vegmatrixc14_rt(ideadcroot) cs14_veg%matrix_cap_deadcrootc_storage_patch(p) = vegmatrixc14_rt(ideadcroot_st) cs14_veg%matrix_cap_deadcrootc_xfer_patch(p) = vegmatrixc14_rt(ideadcroot_xf) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%matrix_cap_reproc_patch(p) = vegmatrixc14_rt(igrain) cs14_veg%matrix_cap_reproc_storage_patch(p) = vegmatrixc14_rt(igrain_st) cs14_veg%matrix_cap_reproc_xfer_patch(p) = vegmatrixc14_rt(igrain_xf) @@ -3276,7 +3276,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_cap_livecrootn_xfer(p) = vegmatrixn_rt(ilivecroot_xf) matrix_cap_deadcrootn(p) = vegmatrixn_rt(ideadcroot) matrix_cap_deadcrootn_storage(p) = vegmatrixn_rt(ideadcroot_st) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_cap_repron(p) = vegmatrixn_rt(igrain) matrix_cap_repron_storage(p) = vegmatrixn_rt(igrain_st) matrix_cap_repron_xfer(p) = vegmatrixn_rt(igrain_xf) @@ -3296,7 +3296,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_calloc_livecrootst_acc(p) = 0._r8 matrix_calloc_deadcroot_acc(p) = 0._r8 matrix_calloc_deadcrootst_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_calloc_grain_acc(p) = 0._r8 matrix_calloc_grainst_acc(p) = 0._r8 end if @@ -3313,7 +3313,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ctransfer_livecrootxf_to_livecroot_acc(p) = 0._r8 matrix_ctransfer_deadcrootst_to_deadcrootxf_acc(p) = 0._r8 matrix_ctransfer_deadcrootxf_to_deadcroot_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ctransfer_grainst_to_grainxf_acc(p) = 0._r8 matrix_ctransfer_grainxf_to_grain_acc(p) = 0._r8 end if @@ -3338,7 +3338,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_cturnover_deadcroot_acc(p) = 0._r8 matrix_cturnover_deadcrootst_acc(p) = 0._r8 matrix_cturnover_deadcrootxf_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_cturnover_grain_acc(p) = 0._r8 matrix_cturnover_grainst_acc(p) = 0._r8 matrix_cturnover_grainxf_acc(p) = 0._r8 @@ -3357,7 +3357,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%matrix_calloc_livecrootst_acc_patch(p) = 0._r8 cs13_veg%matrix_calloc_deadcroot_acc_patch(p) = 0._r8 cs13_veg%matrix_calloc_deadcrootst_acc_patch(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%matrix_calloc_grain_acc_patch(p) = 0._r8 cs13_veg%matrix_calloc_grainst_acc_patch(p) = 0._r8 end if @@ -3374,7 +3374,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%matrix_ctransfer_livecrootxf_to_livecroot_acc_patch(p) = 0._r8 cs13_veg%matrix_ctransfer_deadcrootst_to_deadcrootxf_acc_patch(p) = 0._r8 cs13_veg%matrix_ctransfer_deadcrootxf_to_deadcroot_acc_patch(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%matrix_ctransfer_grainst_to_grainxf_acc_patch(p) = 0._r8 cs13_veg%matrix_ctransfer_grainxf_to_grain_acc_patch(p) = 0._r8 end if @@ -3399,7 +3399,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%matrix_cturnover_deadcroot_acc_patch(p) = 0._r8 cs13_veg%matrix_cturnover_deadcrootst_acc_patch(p) = 0._r8 cs13_veg%matrix_cturnover_deadcrootxf_acc_patch(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%matrix_cturnover_grain_acc_patch(p) = 0._r8 cs13_veg%matrix_cturnover_grainst_acc_patch(p) = 0._r8 cs13_veg%matrix_cturnover_grainxf_acc_patch(p) = 0._r8 @@ -3419,7 +3419,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%matrix_calloc_livecrootst_acc_patch(p) = 0._r8 cs14_veg%matrix_calloc_deadcroot_acc_patch(p) = 0._r8 cs14_veg%matrix_calloc_deadcrootst_acc_patch(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%matrix_calloc_grain_acc_patch(p) = 0._r8 cs14_veg%matrix_calloc_grainst_acc_patch(p) = 0._r8 end if @@ -3436,7 +3436,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%matrix_ctransfer_livecrootxf_to_livecroot_acc_patch(p) = 0._r8 cs14_veg%matrix_ctransfer_deadcrootst_to_deadcrootxf_acc_patch(p) = 0._r8 cs14_veg%matrix_ctransfer_deadcrootxf_to_deadcroot_acc_patch(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%matrix_ctransfer_grainst_to_grainxf_acc_patch(p) = 0._r8 cs14_veg%matrix_ctransfer_grainxf_to_grain_acc_patch(p) = 0._r8 end if @@ -3461,7 +3461,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%matrix_cturnover_deadcroot_acc_patch(p) = 0._r8 cs14_veg%matrix_cturnover_deadcrootst_acc_patch(p) = 0._r8 cs14_veg%matrix_cturnover_deadcrootxf_acc_patch(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%matrix_cturnover_grain_acc_patch(p) = 0._r8 cs14_veg%matrix_cturnover_grainst_acc_patch(p) = 0._r8 cs14_veg%matrix_cturnover_grainxf_acc_patch(p) = 0._r8 @@ -3480,7 +3480,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_nalloc_livecrootst_acc(p) = 0._r8 matrix_nalloc_deadcroot_acc(p) = 0._r8 matrix_nalloc_deadcrootst_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_nalloc_grain_acc(p) = 0._r8 matrix_nalloc_grainst_acc(p) = 0._r8 end if @@ -3497,7 +3497,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ntransfer_livecrootxf_to_livecroot_acc(p) = 0._r8 matrix_ntransfer_deadcrootst_to_deadcrootxf_acc(p) = 0._r8 matrix_ntransfer_deadcrootxf_to_deadcroot_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_grainst_to_grainxf_acc(p) = 0._r8 matrix_ntransfer_grainxf_to_grain_acc(p) = 0._r8 end if @@ -3516,7 +3516,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ntransfer_retransn_to_livecrootst_acc(p) = 0._r8 matrix_ntransfer_retransn_to_deadcroot_acc(p) = 0._r8 matrix_ntransfer_retransn_to_deadcrootst_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_retransn_to_grain_acc(p) = 0._r8 matrix_ntransfer_retransn_to_grainst_acc(p) = 0._r8 end if @@ -3543,7 +3543,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_nturnover_deadcroot_acc(p) = 0._r8 matrix_nturnover_deadcrootst_acc(p) = 0._r8 matrix_nturnover_deadcrootxf_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_nturnover_grain_acc(p) = 0._r8 matrix_nturnover_grainst_acc(p) = 0._r8 matrix_nturnover_grainxf_acc(p) = 0._r8 diff --git a/src/biogeochem/CNVegNitrogenStateType.F90 b/src/biogeochem/CNVegNitrogenStateType.F90 index 7ff8e77df9..c343c45aca 100644 --- a/src/biogeochem/CNVegNitrogenStateType.F90 +++ b/src/biogeochem/CNVegNitrogenStateType.F90 @@ -10,7 +10,7 @@ module CNVegNitrogenStateType use clm_varctl , only : use_crop use CNSharedParamsMod , only : use_fun, use_matrixcn use decompMod , only : bounds_type - use pftconMod , only : npcropmin, noveg, pftcon + use pftconMod , only : is_prognostic_crop, noveg, pftcon use abortutils , only : endrun use spmdMod , only : masterproc use LandunitType , only : lun @@ -2251,7 +2251,7 @@ subroutine Summary_nitrogenstate(this, bounds, num_soilc, filter_soilc, num_soil this%npool_patch(p) + & this%retransn_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%dispvegn_patch(p) = & this%dispvegn_patch(p) + & diff --git a/src/biogeochem/CNVegStructUpdateMod.F90 b/src/biogeochem/CNVegStructUpdateMod.F90 index 2e8ed8539b..007478b573 100644 --- a/src/biogeochem/CNVegStructUpdateMod.F90 +++ b/src/biogeochem/CNVegStructUpdateMod.F90 @@ -38,7 +38,7 @@ subroutine CNVegStructUpdate(bounds,num_soilp, filter_soilp, & ! ! !USES: use pftconMod , only : noveg, nc3crop, nc3irrig, nbrdlf_evr_shrub, nbrdlf_dcd_brl_shrub - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use pftconMod , only : ntmp_corn, nirrig_tmp_corn use pftconMod , only : ntrp_corn, nirrig_trp_corn use pftconMod , only : nsugarcane, nirrig_sugarcane @@ -232,7 +232,7 @@ subroutine CNVegStructUpdate(bounds,num_soilp, filter_soilp, & hbot(p) = max(0._r8, min(3._r8, htop(p)-1._r8)) - else if (ivt(p) >= npcropmin) then ! prognostic crops + else if (is_prognostic_crop(ivt(p))) then ! prognostic crops if (tlai(p) >= laimx(ivt(p))) peaklai(p) = 1 ! used in CNAllocation diff --git a/src/biogeochem/CropType.F90 b/src/biogeochem/CropType.F90 index 54395c4668..48f9cb9cad 100644 --- a/src/biogeochem/CropType.F90 +++ b/src/biogeochem/CropType.F90 @@ -528,7 +528,7 @@ subroutine Restart(this, bounds, ncid, cnveg_state_inst, flag) use restUtilMod use ncdio_pio use PatchType, only : patch - use pftconMod, only : npcropmin, npcropmax + use pftconMod, only : is_prognostic_crop use clm_varpar, only : mxsowings, mxharvests ! BACKWARDS_COMPATIBILITY(wjs/ssr, 2023-01-09) use CNVegstateType, only : cnveg_state_type @@ -577,7 +577,7 @@ subroutine Restart(this, bounds, ncid, cnveg_state_inst, flag) interpinic_flag='copy', readvar=readvar, data=restyear) if (readvar) then do p = bounds%begp, bounds%endp - if (patch%itype(p) >= npcropmin .and. patch%itype(p) <= npcropmax .and. & + if (is_prognostic_crop(patch%itype(p)) .and. & patch%active(p)) then this%nyrs_crop_active_patch(p) = restyear end if diff --git a/src/biogeochem/DryDepVelocity.F90 b/src/biogeochem/DryDepVelocity.F90 index f6a3b857da..b13d28c765 100644 --- a/src/biogeochem/DryDepVelocity.F90 +++ b/src/biogeochem/DryDepVelocity.F90 @@ -204,7 +204,7 @@ subroutine depvel_compute( bounds, & use pftconMod , only : nbrdlf_evr_shrub, nbrdlf_dcd_tmp_shrub use pftconMod , only : nbrdlf_dcd_brl_shrub,nc3_arctic_grass use pftconMod , only : nc3_nonarctic_grass, nc4_grass, nc3crop - use pftconMod , only : nc3irrig, npcropmin, npcropmax + use pftconMod , only : nc3irrig, is_prognostic_crop use clm_varcon , only : spval ! @@ -349,7 +349,7 @@ subroutine depvel_compute( bounds, & if (clmveg == nc4_grass ) wesveg = 3 if (clmveg == nc3crop ) wesveg = 2 if (clmveg == nc3irrig ) wesveg = 2 - if (clmveg >= npcropmin .and. clmveg <= npcropmax ) wesveg = 2 + if (is_prognostic_crop(clmveg)) wesveg = 2 if (wesveg == wveg_unset )then write(iulog,*) 'clmveg = ', clmveg, 'lun%itype = ', lun%itype(l) call endrun(subgrid_index=pi, subgrid_level=subgrid_level_patch, & diff --git a/src/biogeochem/NutrientCompetitionCLM45defaultMod.F90 b/src/biogeochem/NutrientCompetitionCLM45defaultMod.F90 index 82dceef664..95f0594a31 100644 --- a/src/biogeochem/NutrientCompetitionCLM45defaultMod.F90 +++ b/src/biogeochem/NutrientCompetitionCLM45defaultMod.F90 @@ -125,7 +125,7 @@ subroutine calc_plant_cn_alloc (this, bounds, num_soilp, filter_soilp, & c14_cnveg_carbonflux_inst, cnveg_nitrogenflux_inst, cnveg_nitrogenstate_inst, fpg_col) ! ! !USES: - use pftconMod , only : pftcon, npcropmin + use pftconMod , only : pftcon, is_prognostic_crop use clm_varctl , only : use_c13, use_c14 use CNVegStateType , only : cnveg_state_type use CropType , only : crop_type @@ -281,7 +281,7 @@ subroutine calc_plant_cn_alloc (this, bounds, num_soilp, filter_soilp, & cndw = deadwdcn(ivt(p)) fcur = fcur2(ivt(p)) - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops if (croplive(p).and.(.not.shr_infnan_isnan(aleaf(p)))) then f1 = aroot(p) / aleaf(p) f3 = astem(p) / aleaf(p) @@ -357,7 +357,7 @@ subroutine calc_plant_cn_alloc (this, bounds, num_soilp, filter_soilp, & cpool_to_deadcrootc(p) = nlc * f2 * f3 * (1._r8 - f4) * fcur cpool_to_deadcrootc_storage(p) = nlc * f2 * f3 * (1._r8 - f4) * (1._r8 - fcur) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cpool_to_livestemc(p) = nlc * f3 * f4 * fcur cpool_to_livestemc_storage(p) = nlc * f3 * f4 * (1._r8 - fcur) cpool_to_deadstemc(p) = nlc * f3 * (1._r8 - f4) * fcur @@ -387,7 +387,7 @@ subroutine calc_plant_cn_alloc (this, bounds, num_soilp, filter_soilp, & npool_to_deadcrootn(p) = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * fcur npool_to_deadcrootn_storage(p) = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * (1._r8 - fcur) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cng = graincn(ivt(p)) npool_to_livestemn(p) = (nlc * f3 * f4 / cnlw) * fcur npool_to_livestemn_storage(p) = (nlc * f3 * f4 / cnlw) * (1._r8 - fcur) @@ -420,7 +420,7 @@ subroutine calc_plant_cn_alloc (this, bounds, num_soilp, filter_soilp, & gresp_storage = gresp_storage + cpool_to_livecrootc_storage(p) gresp_storage = gresp_storage + cpool_to_deadcrootc_storage(p) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops gresp_storage = gresp_storage + cpool_to_livestemc_storage(p) do k = 1, nrepr gresp_storage = gresp_storage + cpool_to_reproductivec_storage(p,k) @@ -505,7 +505,7 @@ subroutine calc_plant_nitrogen_demand(this, bounds, & ! - livestemn_to_retransn ! ! !USES: - use pftconMod , only : npcropmin, pftcon + use pftconMod , only : is_prognostic_crop, pftcon use pftconMod , only : ntmp_soybean, nirrig_tmp_soybean use pftconMod , only : ntrp_soybean, nirrig_trp_soybean use clm_time_manager , only : get_step_size_real diff --git a/src/biogeochem/NutrientCompetitionFlexibleCNMod.F90 b/src/biogeochem/NutrientCompetitionFlexibleCNMod.F90 index 00e8a00b77..d3f8753fd0 100644 --- a/src/biogeochem/NutrientCompetitionFlexibleCNMod.F90 +++ b/src/biogeochem/NutrientCompetitionFlexibleCNMod.F90 @@ -24,7 +24,7 @@ module NutrientCompetitionFlexibleCNMod use LandunitType , only : lun use ColumnType , only : col use PatchType , only : patch - use pftconMod , only : pftcon, npcropmin + use pftconMod , only : pftcon, is_prognostic_crop use NutrientCompetitionMethodMod, only : nutrient_competition_method_type use CropReprPoolsMod , only : nrepr use CNPhenologyMod , only : CropPhase @@ -413,7 +413,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & fcur = 0.0_r8 end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops if (croplive(p)) then f1 = aroot(p) / aleaf(p) f3 = astem(p) / aleaf(p) @@ -497,7 +497,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & + cpool_to_deadcrootc(p) + cpool_to_deadcrootc_storage(p) end if end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cpool_to_livestemc(p) = nlc * f3 * f4 * fcur cpool_to_livestemc_storage(p) = nlc * f3 * f4 * (1._r8 - fcur) cpool_to_deadstemc(p) = nlc * f3 * (1._r8 - f4) * fcur @@ -548,7 +548,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & matrix_alloc(p,ideadcroot_st) = cpool_to_deadcrootc_storage(p) / cpool_to_veg end if end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops if(cpool_to_veg .ne. 0)then matrix_alloc(p,ilivestem) = cpool_to_livestemc(p) / cpool_to_veg matrix_alloc(p,ilivestem_st) = cpool_to_livestemc_storage(p) / cpool_to_veg @@ -586,7 +586,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & gresp_storage = gresp_storage + cpool_to_livecrootc_storage(p) gresp_storage = gresp_storage + cpool_to_deadcrootc_storage(p) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops gresp_storage = gresp_storage + cpool_to_livestemc_storage(p) do k = 1, nrepr gresp_storage = gresp_storage + cpool_to_reproductivec_storage(p,k) @@ -594,7 +594,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & end if cpool_to_gresp_storage(p) = gresp_storage * g1 * (1._r8 - g2) - if (use_crop_agsys .and. ivt(p) >= npcropmin) then + if (use_crop_agsys .and. is_prognostic_crop(ivt(p))) then call calc_npool_to_components_agsys( & ! Inputs npool = npool(p), & @@ -708,7 +708,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & end if end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops if (cnveg_nitrogenstate_inst%livestemn_storage_patch(p) == 0.0_r8) then ! to avoid division by zero, and also to make livestemcn_actual(p) a very large number if livestemc(p) is zero @@ -795,7 +795,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops livewdcn_max = livewdcn(ivt(p)) + 15.0_r8 @@ -876,7 +876,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & + npool_to_deadstemn(p) + npool_to_deadstemn_storage(p) & + npool_to_livecrootn(p) + npool_to_livecrootn_storage(p) & + npool_to_deadcrootn(p) + npool_to_deadcrootn_storage(p) - if (ivt(p) >= npcropmin)then + if (is_prognostic_crop(ivt(p)))then npool_to_veg = npool_to_veg + npool_to_reproductiven(p,1) + npool_to_reproductiven_storage(p,1) end if if(npool_to_veg .ne. 0._r8)then @@ -892,7 +892,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & matrix_nalloc(p,ilivecroot_st ) = npool_to_livecrootn_storage(p) / npool_to_veg matrix_nalloc(p,ideadcroot ) = npool_to_deadcrootn(p) / npool_to_veg matrix_nalloc(p,ideadcroot_st ) = npool_to_deadcrootn_storage(p) / npool_to_veg - if (ivt(p) >= npcropmin)then + if (is_prognostic_crop(ivt(p)))then matrix_nalloc(p,igrain ) = npool_to_reproductiven(p,1) / npool_to_veg matrix_nalloc(p,igrain_st ) = npool_to_reproductiven_storage(p,1) / npool_to_veg end if @@ -916,7 +916,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & tmp = matrix_update_phn(p,iretransn_to_ilivecrootst ,matrix_nalloc(p,ilivecroot_st ) * retransn_to_npool(p) / retransn(p),dt,cnveg_nitrogenflux_inst,matrixcheck_ph,.True.) tmp = matrix_update_phn(p,iretransn_to_ideadcroot ,matrix_nalloc(p,ideadcroot ) * retransn_to_npool(p) / retransn(p),dt,cnveg_nitrogenflux_inst,matrixcheck_ph,.True.) tmp = matrix_update_phn(p,iretransn_to_ideadcrootst ,matrix_nalloc(p,ideadcroot_st ) * retransn_to_npool(p) / retransn(p),dt,cnveg_nitrogenflux_inst,matrixcheck_ph,.True.) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then tmp = matrix_update_phn(p,iretransn_to_igrain ,matrix_nalloc(p,igrain ) * retransn_to_npool(p) / retransn(p),dt,cnveg_nitrogenflux_inst,matrixcheck_ph,.True.) tmp = matrix_update_phn(p,iretransn_to_igrainst ,matrix_nalloc(p,igrain_st ) * retransn_to_npool(p) / retransn(p),dt,cnveg_nitrogenflux_inst,matrixcheck_ph,.True.) end if @@ -1051,7 +1051,7 @@ subroutine calc_npool_to_components_flexiblecn( & npool_to_deadcrootn_demand = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * fcur npool_to_deadcrootn_storage_demand = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * (1._r8 - fcur) end if - if (ivt >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt)) then ! skip 2 generic crops cng = graincn(ivt) npool_to_livestemn_demand = (nlc * f3 * f4 / cnlw) * fcur @@ -1084,7 +1084,7 @@ subroutine calc_npool_to_components_flexiblecn( & npool_to_deadcrootn_storage_demand end if - if (ivt >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt)) then ! skip 2 generic crops npool_to_reproductiven_demand_tot = 0._r8 npool_to_reproductiven_storage_demand_tot = 0._r8 @@ -1122,7 +1122,7 @@ subroutine calc_npool_to_components_flexiblecn( & frNdemand_npool_to_deadcrootn = 0.0_r8 frNdemand_npool_to_deadcrootn_storage = 0.0_r8 end if - if (ivt >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt)) then ! skip 2 generic crops frNdemand_npool_to_livestemn = 0.0_r8 frNdemand_npool_to_livestemn_storage = 0.0_r8 @@ -1155,7 +1155,7 @@ subroutine calc_npool_to_components_flexiblecn( & frNdemand_npool_to_deadcrootn = npool_to_deadcrootn_demand / total_plant_Ndemand frNdemand_npool_to_deadcrootn_storage = npool_to_deadcrootn_storage_demand / total_plant_Ndemand end if - if (ivt >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt)) then ! skip 2 generic crops frNdemand_npool_to_livestemn = npool_to_livestemn_demand / total_plant_Ndemand frNdemand_npool_to_livestemn_storage = npool_to_livestemn_storage_demand / total_plant_Ndemand @@ -1194,7 +1194,7 @@ subroutine calc_npool_to_components_flexiblecn( & npool_to_deadcrootn = frNdemand_npool_to_deadcrootn * npool / dt npool_to_deadcrootn_storage = frNdemand_npool_to_deadcrootn_storage * npool / dt end if - if (ivt >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt)) then ! skip 2 generic crops npool_to_livestemn = frNdemand_npool_to_livestemn * npool / dt npool_to_livestemn_storage = frNdemand_npool_to_livestemn_storage * npool / dt npool_to_deadstemn = frNdemand_npool_to_deadstemn * npool / dt diff --git a/src/biogeophys/PhotosynthesisMod.F90 b/src/biogeophys/PhotosynthesisMod.F90 index b8fd577382..a823bce548 100644 --- a/src/biogeophys/PhotosynthesisMod.F90 +++ b/src/biogeophys/PhotosynthesisMod.F90 @@ -1236,7 +1236,7 @@ subroutine Photosynthesis ( bounds, fn, filterp, & use clm_time_manager , only : get_step_size_real, is_near_local_noon use clm_varctl , only : cnallocate_carbon_only use clm_varctl , only : lnc_opt, reduce_dayl_factor, vcmax_opt - use pftconMod , only : nbrdlf_dcd_tmp_shrub, npcropmin + use pftconMod , only : nbrdlf_dcd_tmp_shrub ! ! !ARGUMENTS: @@ -2727,7 +2727,7 @@ subroutine PhotosynthesisHydraulicStress ( bounds, fn, filterp, & use clm_varctl , only : cnallocate_carbon_only use clm_varctl , only : lnc_opt, reduce_dayl_factor, vcmax_opt use clm_varpar , only : nlevsoi - use pftconMod , only : nbrdlf_dcd_tmp_shrub, npcropmin + use pftconMod , only : nbrdlf_dcd_tmp_shrub use ColumnType , only : col ! diff --git a/src/biogeophys/TemperatureType.F90 b/src/biogeophys/TemperatureType.F90 index 899da6b882..58e4c93e7b 100644 --- a/src/biogeophys/TemperatureType.F90 +++ b/src/biogeophys/TemperatureType.F90 @@ -1414,7 +1414,7 @@ subroutine UpdateAccVars_CropGDDs(this, rbufslp, begp, endp, month, day, secs, d use shr_const_mod , only : SHR_CONST_CDAY, SHR_CONST_TKFRZ use accumulMod , only : update_accum_field, extract_accum_field, markreset_accum_field use clm_time_manager , only : is_doy_in_interval, get_curr_calday - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use CropType, only : crop_type ! ! !ARGUMENTS @@ -1485,7 +1485,7 @@ subroutine UpdateAccVars_CropGDDs(this, rbufslp, begp, endp, month, day, secs, d ((month > 9 .or. month < 4) .and. lat < 0._r8) ! Replace with read-in gdd20 accumulation season, if needed and valid ! (If these aren't being read in or they're invalid, they'll be -1) - if (stream_gdd20_seasons_tt .and. patch%itype(p) >= npcropmin) then + if (stream_gdd20_seasons_tt .and. is_prognostic_crop(patch%itype(p))) then gdd20_season_start = int(gdd20_season_starts(p)) gdd20_season_end = int(gdd20_season_ends(p)) if (gdd20_season_start >= 1 .and. gdd20_season_end >= 1) then diff --git a/src/cpl/share_esmf/cropcalStreamMod.F90 b/src/cpl/share_esmf/cropcalStreamMod.F90 index b19612ca09..e9927a99c7 100644 --- a/src/cpl/share_esmf/cropcalStreamMod.F90 +++ b/src/cpl/share_esmf/cropcalStreamMod.F90 @@ -22,7 +22,7 @@ module cropcalStreamMod use clm_varpar , only : mxsowings use perf_mod , only : t_startf, t_stopf use spmdMod , only : masterproc, mpicom, iam - use pftconMod , only : npcropmin + use pftconMod , only : npcropmin, is_prognostic_crop use CNPhenologyMod , only : generate_crop_gdds ! ! !PUBLIC TYPES: @@ -588,7 +588,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) p = filter_pcropp(fp) ivt = patch%itype(p) ! Will skip generic crops - if (ivt >= npcropmin) then + if (is_prognostic_crop(ivt)) then n = ivt - npcropmin + 1 ! vegetated pft ig = g_to_ig(patch%gridcell(p)) @@ -612,7 +612,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ! Handle invalid sowing window values if (any(swindow_starts(begp:endp,:) < 1 .or. swindow_ends(begp:endp,:) < 1)) then ! Fail if not allowing fallback to paramfile sowing windows - if ((.not. allow_invalid_swindow_inputs) .and. any(all(swindow_starts(begp:endp,:) < 1, dim=2) .and. patch%wtgcell(begp:endp) > 0._r8 .and. patch%itype(begp:endp) >= npcropmin)) then + if ((.not. allow_invalid_swindow_inputs) .and. any(all(swindow_starts(begp:endp,:) < 1, dim=2) .and. patch%wtgcell(begp:endp) > 0._r8 .and. is_prognostic_crop(patch%itype(begp:endp)))) then write(iulog, *) 'At least one crop in one gridcell has invalid prescribed sowing window start date(s). To ignore and fall back to paramfile sowing windows, set allow_invalid_swindow_inputs to .true.' write(iulog, *) 'Affected crops:' do ivt = npcropmin, mxpft @@ -667,7 +667,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ivt = patch%itype(p) ! Will skip generic crops - if (ivt >= npcropmin) then + if (is_prognostic_crop(ivt)) then n = ivt - npcropmin + 1 if (n > ncft) then @@ -718,7 +718,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ivt = patch%itype(p) ! Will skip generic crops - if (ivt >= npcropmin) then + if (is_prognostic_crop(ivt)) then n = ivt - npcropmin + 1 if (n > ncft) then @@ -788,7 +788,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) p = filter_pcropp(fp) ivt = patch%itype(p) ! Will skip generic crops - if (ivt >= npcropmin) then + if (is_prognostic_crop(ivt)) then n = ivt - npcropmin + 1 ! vegetated pft ig = g_to_ig(patch%gridcell(p)) @@ -805,7 +805,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) if (any(gdd20_season_starts(begp:endp) < 1._r8 .or. gdd20_season_ends(begp:endp) < 1._r8)) then ! Fail if not allowing fallback to paramfile sowing windows. Only need to check for ! values < 1 because values outside [1, 366] are set to -1 above. - if ((.not. allow_invalid_gdd20_season_inputs) .and. any(gdd20_season_starts(begp:endp) < 1._r8 .and. patch%wtgcell(begp:endp) > 0._r8 .and. patch%itype(begp:endp) >= npcropmin)) then + if ((.not. allow_invalid_gdd20_season_inputs) .and. any(gdd20_season_starts(begp:endp) < 1._r8 .and. patch%wtgcell(begp:endp) > 0._r8 .and. is_prognostic_crop(patch%itype(begp:endp)))) then write(iulog, *) 'At least one crop in one gridcell has invalid gdd20 season start and/or end date(s). To ignore and fall back to paramfile sowing windows for such crop-gridcells, set allow_invalid_gdd20_season_inputs to .true.' write(iulog, *) 'Affected crops:' do ivt = npcropmin, mxpft diff --git a/src/main/filterMod.F90 b/src/main/filterMod.F90 index 2fb7d23079..6540021923 100644 --- a/src/main/filterMod.F90 +++ b/src/main/filterMod.F90 @@ -316,7 +316,7 @@ subroutine setFiltersOneGroup(bounds, this_filter, include_inactive, glc_behavio ! ! !USES: use decompMod , only : bounds_level_clump - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use landunit_varcon , only : istsoil, istcrop, istice ! ! !ARGUMENTS: @@ -479,7 +479,7 @@ subroutine setFiltersOneGroup(bounds, this_filter, include_inactive, glc_behavio do p = bounds%begp,bounds%endp if(.not.use_fates)then if (patch%active(p) .or. include_inactive) then - if (patch%itype(p) >= npcropmin) then !skips 2 generic crop types + if (is_prognostic_crop(patch%itype(p))) then !skips 2 generic crop types fl = fl + 1 this_filter(nc)%pcropp(fl) = p else diff --git a/src/main/pftconMod.F90 b/src/main/pftconMod.F90 index b48ef92a43..94ca009ae9 100644 --- a/src/main/pftconMod.F90 +++ b/src/main/pftconMod.F90 @@ -2,8 +2,9 @@ module pftconMod !----------------------------------------------------------------------- ! !DESCRIPTION: - ! Module containing vegetation constants and method to - ! read and initialize vegetation (PFT) constants. + ! Module containing vegetation constants, methods to + ! read and initialize vegetation (PFT) constants, and methods to query + ! PFT characteristics ! ! !USES: use shr_kind_mod, only : r8 => shr_kind_r8 @@ -315,6 +316,8 @@ module pftconMod character(len=*), parameter, private :: sourcefile = & __FILE__ + + public :: is_prognostic_crop !----------------------------------------------------------------------- contains @@ -1351,19 +1354,19 @@ subroutine InitRead(this) else call endrun(msg=' ERROR: crop has wrong values'//errMsg(sourcefile, __LINE__)) end if - if ( (i /= noveg) .and. (i < npcropmin) .and. & + if ( (i /= noveg) .and. (.not. is_prognostic_crop(i)) .and. & abs(this%pconv(i) + this%pprod10(i) + this%pprod100(i) - 1.0_r8) > 1.e-7_r8 )then call endrun(msg=' ERROR: pconv+pprod10+pprod100 do NOT sum to one.'//errMsg(sourcefile, __LINE__)) end if if ( this%pprodharv10(i) > 1.0_r8 .or. this%pprodharv10(i) < 0.0_r8 )then call endrun(msg=' ERROR: pprodharv10 outside of range.'//errMsg(sourcefile, __LINE__)) end if - if (i < npcropmin .and. this%biofuel_harvfrac(i) /= 0._r8) then + if ((.not. is_prognostic_crop(i)) .and. this%biofuel_harvfrac(i) /= 0._r8) then call endrun(msg=' ERROR: biofuel_harvfrac non-zero for a non-prognostic crop PFT.'//& errMsg(sourcefile, __LINE__)) end if do k = repr_structure_min, repr_structure_max - if (i < npcropmin .and. this%repr_structure_harvfrac(i,k) /= 0._r8) then + if ((.not. is_prognostic_crop(i)) .and. this%repr_structure_harvfrac(i,k) /= 0._r8) then call endrun(msg=' ERROR: repr_structure_harvfrac non-zero for a non-prognostic crop PFT.'//& errMsg(sourcefile, __LINE__)) end if @@ -1607,5 +1610,24 @@ subroutine Clean(this) deallocate( this%ndays_on) end subroutine Clean + !----------------------------------------------------------------------- + elemental logical function is_prognostic_crop(veg_type) + ! + ! !DESCRIPTION: + ! Given a vegetation type (pft, integer), return whether it's a prognostic crop. Does not + ! include generic crops (those and natural PFTs will return .false.). + ! + ! NOTE(wjs, 2017-02-02) This isn't a completely robust way to check if this is a + ! prognostic crop patch (at the very least it should also check if <= npcropmax; + ! ideally it should use a prognostic_crop flag that doesn't seem to exist + ! currently). + ! + ! !ARGUMENTS + integer, intent(in) :: veg_type + + is_prognostic_crop = veg_type >= npcropmin + + end function is_prognostic_crop + end module pftconMod diff --git a/src/soilbiogeochem/TillageMod.F90 b/src/soilbiogeochem/TillageMod.F90 index 4a24daf4c2..5f94a44777 100644 --- a/src/soilbiogeochem/TillageMod.F90 +++ b/src/soilbiogeochem/TillageMod.F90 @@ -284,7 +284,7 @@ subroutine get_apply_tillage_multipliers(idop, c, j, decomp_k) ! Written by Sam Rabin, based on original code by Michael Graham. ! ! !USES - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use clm_varcon, only : zisoi, dzsoi_decomp use landunit_varcon , only : istcrop use PatchType , only : patch @@ -315,7 +315,7 @@ subroutine get_apply_tillage_multipliers(idop, c, j, decomp_k) sumwt = 0.0_r8 do p = col%patchi(c),col%patchf(c) if (patch%active(p) .and. patch%wtcol(p) /= 0._r8) then - if (patch%itype(p) < npcropmin) then + if (.not. is_prognostic_crop(patch%itype(p))) then ! Do not till generic crops tillage_mults_1patch(:) = 1._r8 else From 85f72b22ee16a1448e26134c2a2dc3733dadbc1c Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 6 Aug 2025 14:59:15 -0600 Subject: [PATCH 002/196] Avoid using npcropmin/max for looping outside pftconMod. --- src/biogeochem/CNPhenologyMod.F90 | 7 ++++--- src/cpl/share_esmf/cropcalStreamMod.F90 | 10 ++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/biogeochem/CNPhenologyMod.F90 b/src/biogeochem/CNPhenologyMod.F90 index f19d57aa03..37748b6e6b 100644 --- a/src/biogeochem/CNPhenologyMod.F90 +++ b/src/biogeochem/CNPhenologyMod.F90 @@ -2691,8 +2691,9 @@ subroutine CropPhenologyInit(bounds) ! initialized, and after pftcon file is read in. ! ! !USES: - use pftconMod , only: npcropmin, npcropmax use clm_time_manager, only: get_calday + use pftconMod, only: is_prognostic_crop + use clm_varpar, only: mxpft ! ! !ARGUMENTS: type(bounds_type), intent(in) :: bounds @@ -2713,8 +2714,8 @@ subroutine CropPhenologyInit(bounds) ! Convert planting dates into julian day minplantjday(:,:) = huge(1) maxplantjday(:,:) = huge(1) - do n = npcropmin, npcropmax - if (pftcon%is_pft_known_to_model(n)) then + do n = 1, mxpft + if (is_prognostic_crop(n) .and. pftcon%is_pft_known_to_model(n)) then minplantjday(n, inNH) = int( get_calday( pftcon%mnNHplantdate(n), 0 ) ) maxplantjday(n, inNH) = int( get_calday( pftcon%mxNHplantdate(n), 0 ) ) diff --git a/src/cpl/share_esmf/cropcalStreamMod.F90 b/src/cpl/share_esmf/cropcalStreamMod.F90 index e9927a99c7..a0e83a5995 100644 --- a/src/cpl/share_esmf/cropcalStreamMod.F90 +++ b/src/cpl/share_esmf/cropcalStreamMod.F90 @@ -615,7 +615,10 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) if ((.not. allow_invalid_swindow_inputs) .and. any(all(swindow_starts(begp:endp,:) < 1, dim=2) .and. patch%wtgcell(begp:endp) > 0._r8 .and. is_prognostic_crop(patch%itype(begp:endp)))) then write(iulog, *) 'At least one crop in one gridcell has invalid prescribed sowing window start date(s). To ignore and fall back to paramfile sowing windows, set allow_invalid_swindow_inputs to .true.' write(iulog, *) 'Affected crops:' - do ivt = npcropmin, mxpft + do ivt = 1, mxpft + if (.not. is_prognostic_crop(ivt)) then + cycle + end if do fp = 1, num_pcropp p = filter_pcropp(fp) if (ivt == patch%itype(p) .and. patch%wtgcell(p) > 0._r8 .and. all(swindow_starts(p,:) < 1)) then @@ -808,7 +811,10 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) if ((.not. allow_invalid_gdd20_season_inputs) .and. any(gdd20_season_starts(begp:endp) < 1._r8 .and. patch%wtgcell(begp:endp) > 0._r8 .and. is_prognostic_crop(patch%itype(begp:endp)))) then write(iulog, *) 'At least one crop in one gridcell has invalid gdd20 season start and/or end date(s). To ignore and fall back to paramfile sowing windows for such crop-gridcells, set allow_invalid_gdd20_season_inputs to .true.' write(iulog, *) 'Affected crops:' - do ivt = npcropmin, mxpft + do ivt = 1, mxpft + if (.not. is_prognostic_crop(ivt)) then + cycle + end if do fp = 1, num_pcropp p = filter_pcropp(fp) if (ivt == patch%itype(p) .and. patch%wtgcell(p) > 0._r8 .and. gdd20_season_starts(p) < 1._r8) then From 25d0ccec96b2e3c64e0810f35db44c5ef5dd9d2f Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 6 Aug 2025 15:48:32 -0600 Subject: [PATCH 003/196] Add get_crop_n_from_veg_type(). --- src/cpl/share_esmf/cropcalStreamMod.F90 | 10 +++++----- src/main/pftconMod.F90 | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/cpl/share_esmf/cropcalStreamMod.F90 b/src/cpl/share_esmf/cropcalStreamMod.F90 index a0e83a5995..1578bd023d 100644 --- a/src/cpl/share_esmf/cropcalStreamMod.F90 +++ b/src/cpl/share_esmf/cropcalStreamMod.F90 @@ -22,7 +22,7 @@ module cropcalStreamMod use clm_varpar , only : mxsowings use perf_mod , only : t_startf, t_stopf use spmdMod , only : masterproc, mpicom, iam - use pftconMod , only : npcropmin, is_prognostic_crop + use pftconMod , only : npcropmin, is_prognostic_crop, get_crop_n_from_veg_type use CNPhenologyMod , only : generate_crop_gdds ! ! !PUBLIC TYPES: @@ -589,7 +589,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ivt = patch%itype(p) ! Will skip generic crops if (is_prognostic_crop(ivt)) then - n = ivt - npcropmin + 1 + n = get_crop_n_from_veg_type(ivt) ! vegetated pft ig = g_to_ig(patch%gridcell(p)) swindow_starts(p,1) = dataptr2d_swindow_start(ig,n) @@ -671,7 +671,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ivt = patch%itype(p) ! Will skip generic crops if (is_prognostic_crop(ivt)) then - n = ivt - npcropmin + 1 + n = get_crop_n_from_veg_type(ivt) if (n > ncft) then write(iulog,'(a,i0,a,i0,a)') 'n (',n,') > ncft (',ncft,')' @@ -722,7 +722,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ivt = patch%itype(p) ! Will skip generic crops if (is_prognostic_crop(ivt)) then - n = ivt - npcropmin + 1 + n = get_crop_n_from_veg_type(ivt) if (n > ncft) then write(iulog,'(a,i0,a,i0,a)') 'n (',n,') > ncft (',ncft,')' @@ -792,7 +792,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ivt = patch%itype(p) ! Will skip generic crops if (is_prognostic_crop(ivt)) then - n = ivt - npcropmin + 1 + n = get_crop_n_from_veg_type(ivt) ! vegetated pft ig = g_to_ig(patch%gridcell(p)) diff --git a/src/main/pftconMod.F90 b/src/main/pftconMod.F90 index 94ca009ae9..8de10b355b 100644 --- a/src/main/pftconMod.F90 +++ b/src/main/pftconMod.F90 @@ -318,6 +318,7 @@ module pftconMod __FILE__ public :: is_prognostic_crop + public :: get_crop_n_from_veg_type !----------------------------------------------------------------------- contains @@ -1629,5 +1630,19 @@ elemental logical function is_prognostic_crop(veg_type) end function is_prognostic_crop + !----------------------------------------------------------------------- + elemental integer function get_crop_n_from_veg_type(veg_type) result(crop_n) + ! + ! !DESCRIPTION: + ! Given a vegetation type (pft, integer), return a 1-indexed number indicating where it would + ! be in a list of all simulated crops. + ! + ! !ARGUMENTS + integer, intent(in) :: veg_type + + crop_n = veg_type - npcropmin + 1 + + end function get_crop_n_from_veg_type + end module pftconMod From 29ae909d302932870f2b638b063ebf71cfa78611 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 6 Aug 2025 15:53:34 -0600 Subject: [PATCH 004/196] Add get_veg_type_from_crop_n(). --- src/cpl/share_esmf/cropcalStreamMod.F90 | 4 ++-- src/main/pftconMod.F90 | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/cpl/share_esmf/cropcalStreamMod.F90 b/src/cpl/share_esmf/cropcalStreamMod.F90 index 1578bd023d..792aafec5f 100644 --- a/src/cpl/share_esmf/cropcalStreamMod.F90 +++ b/src/cpl/share_esmf/cropcalStreamMod.F90 @@ -22,7 +22,7 @@ module cropcalStreamMod use clm_varpar , only : mxsowings use perf_mod , only : t_startf, t_stopf use spmdMod , only : masterproc, mpicom, iam - use pftconMod , only : npcropmin, is_prognostic_crop, get_crop_n_from_veg_type + use pftconMod , only : npcropmin, is_prognostic_crop, get_crop_n_from_veg_type, get_veg_type_from_crop_n use CNPhenologyMod , only : generate_crop_gdds ! ! !PUBLIC TYPES: @@ -144,7 +144,7 @@ subroutine cropcal_init(bounds) allocate(stream_varnames_gdd20_baseline(ncft)) allocate(stream_varnames_gdd20_season_enddate(ncft)) do n = 1,ncft - ivt = npcropmin + n - 1 + ivt = get_veg_type_from_crop_n(n) write(stream_varnames_sdate(n),'(a,i0)') "sdate1_",ivt write(stream_varnames_cultivar_gdds(n),'(a,i0)') "gdd1_",ivt write(stream_varnames_gdd20_baseline(n),'(a,i0)') "gdd20bl_",ivt diff --git a/src/main/pftconMod.F90 b/src/main/pftconMod.F90 index 8de10b355b..9511148be7 100644 --- a/src/main/pftconMod.F90 +++ b/src/main/pftconMod.F90 @@ -319,6 +319,7 @@ module pftconMod public :: is_prognostic_crop public :: get_crop_n_from_veg_type + public :: get_veg_type_from_crop_n !----------------------------------------------------------------------- contains @@ -1644,5 +1645,19 @@ elemental integer function get_crop_n_from_veg_type(veg_type) result(crop_n) end function get_crop_n_from_veg_type + !----------------------------------------------------------------------- + elemental integer function get_veg_type_from_crop_n(crop_n) result(veg_type) + ! + ! !DESCRIPTION: + ! Given a return a 1-indexed number indicating where a PFT would be in a list of all simulated + ! crops, return vegetation type (ivt) + ! + ! !ARGUMENTS + integer, intent(in) :: crop_n + + veg_type = npcropmin + crop_n - 1 + + end function get_veg_type_from_crop_n + end module pftconMod From 05eec125b2fbe7a6ed8bc45fd5eac1c94a7ac91c Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 6 Aug 2025 16:02:43 -0600 Subject: [PATCH 005/196] Add pftconMod public var num_cfts_possible. --- src/cpl/share_esmf/cropcalStreamMod.F90 | 45 ++++++++++++------------- src/main/pftconMod.F90 | 26 ++++++++++++++ 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/src/cpl/share_esmf/cropcalStreamMod.F90 b/src/cpl/share_esmf/cropcalStreamMod.F90 index 792aafec5f..591476e59e 100644 --- a/src/cpl/share_esmf/cropcalStreamMod.F90 +++ b/src/cpl/share_esmf/cropcalStreamMod.F90 @@ -22,7 +22,8 @@ module cropcalStreamMod use clm_varpar , only : mxsowings use perf_mod , only : t_startf, t_stopf use spmdMod , only : masterproc, mpicom, iam - use pftconMod , only : npcropmin, is_prognostic_crop, get_crop_n_from_veg_type, get_veg_type_from_crop_n + use pftconMod , only : is_prognostic_crop, get_crop_n_from_veg_type, get_veg_type_from_crop_n + use pftconMod , only : num_cfts_possible use CNPhenologyMod , only : generate_crop_gdds ! ! !PUBLIC TYPES: @@ -46,7 +47,6 @@ module cropcalStreamMod character(len=CS), allocatable :: stream_varnames_cultivar_gdds(:) character(len=CS), allocatable :: stream_varnames_gdd20_baseline(:) character(len=CS), allocatable :: stream_varnames_gdd20_season_enddate(:) ! start uses stream_varnames_sdate - integer :: ncft ! Number of crop functional types (excl. generic crops) logical :: allow_invalid_swindow_inputs ! Fall back on paramfile sowing windows in cases of invalid values in stream_fldFileName_swindow_start and _end? character(len=FL) :: stream_fldFileName_swindow_start ! sowing window start stream filename to read character(len=FL) :: stream_fldFileName_swindow_end ! sowing window end stream filename to read @@ -138,12 +138,11 @@ subroutine cropcal_init(bounds) stream_fldFileName_gdd20_season_start = '' stream_fldFileName_gdd20_season_end = '' ! Will need modification to work with mxsowings > 1 - ncft = mxpft - npcropmin + 1 ! Ignores generic crops - allocate(stream_varnames_sdate(ncft)) - allocate(stream_varnames_cultivar_gdds(ncft)) - allocate(stream_varnames_gdd20_baseline(ncft)) - allocate(stream_varnames_gdd20_season_enddate(ncft)) - do n = 1,ncft + allocate(stream_varnames_sdate(num_cfts_possible)) + allocate(stream_varnames_cultivar_gdds(num_cfts_possible)) + allocate(stream_varnames_gdd20_baseline(num_cfts_possible)) + allocate(stream_varnames_gdd20_season_enddate(num_cfts_possible)) + do n = 1,num_cfts_possible ivt = get_veg_type_from_crop_n(n) write(stream_varnames_sdate(n),'(a,i0)') "sdate1_",ivt write(stream_varnames_cultivar_gdds(n),'(a,i0)') "gdd1_",ivt @@ -201,7 +200,7 @@ subroutine cropcal_init(bounds) write(iulog,'(a,l1)') ' allow_invalid_gdd20_season_inputs = ',allow_invalid_gdd20_season_inputs write(iulog,'(a,a)' ) ' stream_fldFileName_gdd20_season_start = ',stream_fldFileName_gdd20_season_start write(iulog,'(a,a)' ) ' stream_fldFileName_gdd20_season_end = ',stream_fldFileName_gdd20_season_end - do n = 1,ncft + do n = 1,num_cfts_possible write(iulog,'(a,a)' ) ' stream_varnames_sdate = ',trim(stream_varnames_sdate(n)) write(iulog,'(a,a)' ) ' stream_varnames_cultivar_gdds = ',trim(stream_varnames_cultivar_gdds(n)) write(iulog,'(a,a)' ) ' stream_varnames_gdd20_season_enddate = ',trim(stream_varnames_gdd20_season_enddate(n)) @@ -550,13 +549,13 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) dayspyr = get_curr_days_per_year() ! Read prescribed sowing window start dates from input files - allocate(dataptr2d_swindow_start(begg:endg, ncft)) + allocate(dataptr2d_swindow_start(begg:endg, num_cfts_possible)) dataptr2d_swindow_start(begg:endg,:) = -1._r8 - allocate(dataptr2d_swindow_end (begg:endg, ncft)) + allocate(dataptr2d_swindow_end (begg:endg, num_cfts_possible)) dataptr2d_swindow_end(begg:endg,:) = -1._r8 if (use_cropcal_rx_swindows) then ! Starting with npcropmin will skip generic crops - do n = 1, ncft + do n = 1, num_cfts_possible call dshr_fldbun_getFldPtr(sdat_cropcal_swindow_start%pstrm(1)%fldbun_model, trim(stream_varnames_sdate(n)), & fldptr1=dataptr1d_swindow_start, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) then @@ -640,11 +639,11 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) deallocate(dataptr2d_swindow_start) deallocate(dataptr2d_swindow_end) - allocate(dataptr2d_cultivar_gdds(begg:endg, ncft)) + allocate(dataptr2d_cultivar_gdds(begg:endg, num_cfts_possible)) if (use_cropcal_rx_cultivar_gdds) then ! Read prescribed cultivar GDDs from input files ! Starting with npcropmin will skip generic crops - do n = 1, ncft + do n = 1, num_cfts_possible call dshr_fldbun_getFldPtr(sdat_cropcal_cultivar_gdds%pstrm(1)%fldbun_model, trim(stream_varnames_cultivar_gdds(n)), & fldptr1=dataptr1d_cultivar_gdds, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) then @@ -673,8 +672,8 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) if (is_prognostic_crop(ivt)) then n = get_crop_n_from_veg_type(ivt) - if (n > ncft) then - write(iulog,'(a,i0,a,i0,a)') 'n (',n,') > ncft (',ncft,')' + if (n > num_cfts_possible) then + write(iulog,'(a,i0,a,i0,a)') 'n (',n,') > ncft (',num_cfts_possible,')' call ESMF_Finalize(endflag=ESMF_END_ABORT) end if @@ -697,11 +696,11 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) deallocate(dataptr2d_cultivar_gdds) - allocate(dataptr2d_gdd20_baseline(begg:endg, ncft)) + allocate(dataptr2d_gdd20_baseline(begg:endg, num_cfts_possible)) if (adapt_cropcal_rx_cultivar_gdds) then ! Read GDD20 baselines from input files ! Starting with npcropmin will skip generic crops - do n = 1, ncft + do n = 1, num_cfts_possible call dshr_fldbun_getFldPtr(sdat_cropcal_gdd20_baseline%pstrm(1)%fldbun_model, trim(stream_varnames_gdd20_baseline(n)), & fldptr1=dataptr1d_gdd20_baseline, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) then @@ -724,8 +723,8 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) if (is_prognostic_crop(ivt)) then n = get_crop_n_from_veg_type(ivt) - if (n > ncft) then - write(iulog,'(a,i0,a,i0,a)') 'n (',n,') > ncft (',ncft,')' + if (n > num_cfts_possible) then + write(iulog,'(a,i0,a,i0,a)') 'n (',n,') > ncft (',num_cfts_possible,')' call ESMF_Finalize(endflag=ESMF_END_ABORT) end if @@ -750,13 +749,13 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ! Read prescribed gdd20 season start dates from input files - allocate(dataptr2d_gdd20_season_start(begg:endg, ncft)) + allocate(dataptr2d_gdd20_season_start(begg:endg, num_cfts_possible)) dataptr2d_gdd20_season_start(begg:endg,:) = -1._r8 - allocate(dataptr2d_gdd20_season_end (begg:endg, ncft)) + allocate(dataptr2d_gdd20_season_end (begg:endg, num_cfts_possible)) dataptr2d_gdd20_season_end(begg:endg,:) = -1._r8 if (stream_gdd20_seasons) then ! Starting with npcropmin will skip generic crops - do n = 1, ncft + do n = 1, num_cfts_possible call dshr_fldbun_getFldPtr(sdat_cropcal_gdd20_season_start%pstrm(1)%fldbun_model, trim(stream_varnames_sdate(n)), & fldptr1=dataptr1d_gdd20_season_start, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) then diff --git a/src/main/pftconMod.F90 b/src/main/pftconMod.F90 index 9511148be7..a64edc4488 100644 --- a/src/main/pftconMod.F90 +++ b/src/main/pftconMod.F90 @@ -107,6 +107,9 @@ module pftconMod ! at all, as given by the mergetoclmpft list. integer, public :: num_cfts_known_to_model + ! Number of prognostic crop functional types on the parameter file, even if not actually used + integer, public :: num_cfts_possible + ! !PUBLIC TYPES: type, public :: pftcon_type @@ -295,6 +298,7 @@ module pftconMod procedure, private :: InitRead procedure, private :: set_is_pft_known_to_model ! Set is_pft_known_to_model based on mergetoclmpft procedure, private :: set_num_cfts_known_to_model ! Set the module-level variable, num_cfts_known_to_model + procedure, private :: set_num_cfts_possible ! Set the module-level variable, num_cfts_possible end type pftcon_type @@ -1251,6 +1255,7 @@ subroutine InitRead(this) call this%set_is_pft_known_to_model() call this%set_num_cfts_known_to_model() + call this%set_num_cfts_possible() ! Set vegetation family identifier (tree/shrub/grass) do m = 0,mxpft @@ -1443,6 +1448,27 @@ subroutine set_num_cfts_known_to_model(this) end subroutine set_num_cfts_known_to_model + !----------------------------------------------------------------------- + subroutine set_num_cfts_possible(this) + ! + ! !DESCRIPTION: + ! Set the module-level variable, num_cfts_possible + ! + ! !USES: + ! + ! !ARGUMENTS: + class(pftcon_type), intent(in) :: this + ! + ! !LOCAL VARIABLES: + integer :: m + + character(len=*), parameter :: subname = 'set_num_cfts_possible' + !----------------------------------------------------------------------- + + num_cfts_possible = npcropmax - npcropmin + 1 + + end subroutine set_num_cfts_possible + !----------------------------------------------------------------------- subroutine Clean(this) ! From b50faf2ef0e16e9cb199ae070a47165b3e649bca Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 6 Aug 2025 16:03:58 -0600 Subject: [PATCH 006/196] npcropmin/max are now private to pftconMod. --- src/main/pftconMod.F90 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/pftconMod.F90 b/src/main/pftconMod.F90 index a64edc4488..a39a30b9e2 100644 --- a/src/main/pftconMod.F90 +++ b/src/main/pftconMod.F90 @@ -34,7 +34,6 @@ module pftconMod integer, public :: nc3_arctic_grass ! value for C3 arctic grass integer, public :: nc3_nonarctic_grass ! value for C3 non-arctic grass integer, public :: nc4_grass ! value for C4 grass - integer, public :: npcropmin ! value for first crop integer, public :: ntmp_corn ! value for temperate corn, rain fed (rf) integer, public :: nirrig_tmp_corn ! value for temperate corn, irrigated (ir) integer, public :: nswheat ! value for spring temperate cereal (rf) @@ -97,10 +96,13 @@ module pftconMod integer, public :: nirrig_trp_corn !value for tropical corn (ir) integer, public :: ntrp_soybean !value for tropical soybean (rf) integer, public :: nirrig_trp_soybean !value for tropical soybean (ir) - integer, public :: npcropmax ! value for last prognostic crop in list integer, public :: nc3crop ! value for generic crop (rf) integer, public :: nc3irrig ! value for irrigated generic crop (ir) + ! First and last prognostic crops + integer :: npcropmin ! value for first crop + integer :: npcropmax ! value for last prognostic crop in list + ! Number of crop functional types actually used in the model. This includes each CFT for ! which is_pft_known_to_model is true. Note that this includes irrigated crops even if ! irrigation is turned off in this run: it just excludes crop types that aren't handled From 743098a3c6ba42001ead3bffa7667c4b64df6105 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 7 Aug 2025 17:05:52 -0600 Subject: [PATCH 007/196] Add query_paramfile tool. --- python/ctsm/query_parameters/__init__.py | 0 .../ctsm/query_parameters/query_paramfile.py | 23 +++++++++++++++++++ tools/query_parameters/query_paramfile | 19 +++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 python/ctsm/query_parameters/__init__.py create mode 100644 python/ctsm/query_parameters/query_paramfile.py create mode 100755 tools/query_parameters/query_paramfile diff --git a/python/ctsm/query_parameters/__init__.py b/python/ctsm/query_parameters/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/ctsm/query_parameters/query_paramfile.py b/python/ctsm/query_parameters/query_paramfile.py new file mode 100644 index 0000000000..43794ac6d7 --- /dev/null +++ b/python/ctsm/query_parameters/query_paramfile.py @@ -0,0 +1,23 @@ +import argparse +import xarray as xr + + +def get_arguments(): + parser = argparse.ArgumentParser(description="Print values of a variable from a netCDF file.") + parser.add_argument("-i", "--input", required=True, help="Input netCDF file") + parser.add_argument("variable", help="Name of variable to extract") + args = parser.parse_args() + return args + + +def main(): + args = get_arguments() + + ds = xr.open_dataset(args.input, decode_timedelta=False) + data = ds[args.variable].values + print(f"{args.variable}: {data}") + ds.close() + + +if __name__ == "__main__": + main() diff --git a/tools/query_parameters/query_paramfile b/tools/query_parameters/query_paramfile new file mode 100755 index 0000000000..d7d13b6f52 --- /dev/null +++ b/tools/query_parameters/query_paramfile @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +""" +For description and instructions, please see README. +""" + +import os +import sys + +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), + os.pardir, + os.pardir, + 'python') +sys.path.insert(1, _CTSM_PYTHON) + +from ctsm.query_parameters.query_paramfile import main + +if __name__ == "__main__": + main() + From 0e2e1e3c9a63af9948511fcee58dba76da2376e4 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 7 Aug 2025 17:10:06 -0600 Subject: [PATCH 008/196] query_paramfile: Accept comma-separated variable list. --- .../ctsm/query_parameters/query_paramfile.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/python/ctsm/query_parameters/query_paramfile.py b/python/ctsm/query_parameters/query_paramfile.py index 43794ac6d7..373e7461a1 100644 --- a/python/ctsm/query_parameters/query_paramfile.py +++ b/python/ctsm/query_parameters/query_paramfile.py @@ -3,19 +3,33 @@ def get_arguments(): - parser = argparse.ArgumentParser(description="Print values of a variable from a netCDF file.") + parser = argparse.ArgumentParser( + description="Print values of one or more variables from a netCDF file." + ) parser.add_argument("-i", "--input", required=True, help="Input netCDF file") - parser.add_argument("variable", help="Name of variable to extract") + parser.add_argument("variables", help="Comma-separated list of variable names to extract") args = parser.parse_args() return args +def print_values(ds, var): + data = ds[var].values + print(f"{var}: {data}") + + def main(): args = get_arguments() + variable_names = [v.strip() for v in args.variables.split(",")] + ds = xr.open_dataset(args.input, decode_timedelta=False) - data = ds[args.variable].values - print(f"{args.variable}: {data}") + + for var in variable_names: + if var in ds.variables: + print_values(ds, var) + else: + print(f"Variable '{var}' not found in {args.input}") + ds.close() From 72a27f7eff354755cf77ff6e113fc90d2fb49a20 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 7 Aug 2025 17:18:46 -0600 Subject: [PATCH 009/196] query_paramfile: Print PFT name next to PFT-specific params. --- python/ctsm/query_parameters/query_paramfile.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/ctsm/query_parameters/query_paramfile.py b/python/ctsm/query_parameters/query_paramfile.py index 373e7461a1..9f24adfebd 100644 --- a/python/ctsm/query_parameters/query_paramfile.py +++ b/python/ctsm/query_parameters/query_paramfile.py @@ -1,6 +1,8 @@ import argparse import xarray as xr +PFTNAME_VAR = "pftname" + def get_arguments(): parser = argparse.ArgumentParser( @@ -14,7 +16,13 @@ def get_arguments(): def print_values(ds, var): data = ds[var].values - print(f"{var}: {data}") + if PFTNAME_VAR in ds[var].coords: + print(var + ":") + for p, pft_name in enumerate(ds[PFTNAME_VAR].values): + pft_name = pft_name.decode().strip() + print(f" {pft_name}: {data[p]}") + else: + print(f"{var}: {data}") def main(): From 56aa742c32ca89fdad477edbd5f439ca3b1162dd Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 7 Aug 2025 17:22:29 -0600 Subject: [PATCH 010/196] query_paramfile: Align PFT-specific values. --- python/ctsm/query_parameters/query_paramfile.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/ctsm/query_parameters/query_paramfile.py b/python/ctsm/query_parameters/query_paramfile.py index 9f24adfebd..bb49b651b3 100644 --- a/python/ctsm/query_parameters/query_paramfile.py +++ b/python/ctsm/query_parameters/query_paramfile.py @@ -18,9 +18,10 @@ def print_values(ds, var): data = ds[var].values if PFTNAME_VAR in ds[var].coords: print(var + ":") - for p, pft_name in enumerate(ds[PFTNAME_VAR].values): - pft_name = pft_name.decode().strip() - print(f" {pft_name}: {data[p]}") + pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] + max_name_len = max(len(name) for name in pft_names) + for p, pft_name in enumerate(pft_names): + print(f" {pft_name:<{max_name_len}}: {data[p]}") else: print(f"{var}: {data}") From 8e7004697b44ab427c619f6dc90bb8d28521a916 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 7 Aug 2025 17:24:19 -0600 Subject: [PATCH 011/196] query_paramfile: Add optional -p/--pft option. --- .../ctsm/query_parameters/query_paramfile.py | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/python/ctsm/query_parameters/query_paramfile.py b/python/ctsm/query_parameters/query_paramfile.py index bb49b651b3..25aed36264 100644 --- a/python/ctsm/query_parameters/query_paramfile.py +++ b/python/ctsm/query_parameters/query_paramfile.py @@ -10,18 +10,27 @@ def get_arguments(): ) parser.add_argument("-i", "--input", required=True, help="Input netCDF file") parser.add_argument("variables", help="Comma-separated list of variable names to extract") + parser.add_argument( + "-p", + "--pft", + help="Comma-separated list of PFT names to print (only applies to PFT-specific variables)", + ) args = parser.parse_args() return args -def print_values(ds, var): +def print_values(ds, var, selected_pfts, pft_names): data = ds[var].values if PFTNAME_VAR in ds[var].coords: print(var + ":") - pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] - max_name_len = max(len(name) for name in pft_names) - for p, pft_name in enumerate(pft_names): - print(f" {pft_name:<{max_name_len}}: {data[p]}") + indices = range(len(pft_names)) + if selected_pfts is not None: + indices = [i for i, name in enumerate(pft_names) if name in selected_pfts] + max_name_len = max(len(pft_names[i]) for i in indices) if indices else 0 + else: + max_name_len = max(len(name) for name in pft_names) + for p in indices: + print(f" {pft_names[p]:<{max_name_len}}: {data[p]}") else: print(f"{var}: {data}") @@ -33,9 +42,20 @@ def main(): ds = xr.open_dataset(args.input, decode_timedelta=False) + selected_pfts = None + if args.pft: + selected_pfts = [p.strip() for p in args.pft.split(",")] + pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] + pfts_not_in_file = [] + for pft in selected_pfts: + if pft not in pft_names: + pfts_not_in_file += [pft] + if pfts_not_in_file: + raise KeyError(f"PFT(s) not found in parameter file: {', '.join(pfts_not_in_file)}") + for var in variable_names: if var in ds.variables: - print_values(ds, var) + print_values(ds, var, selected_pfts, pft_names) else: print(f"Variable '{var}' not found in {args.input}") From e9f57b3136617bb5fdeeb6581399ea19a6382910 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 8 Aug 2025 17:03:23 -0600 Subject: [PATCH 012/196] py_env_create: conda rename needs --yes, not --force, as of 25.3.0. --- py_env_create | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/py_env_create b/py_env_create index 5b4515bc75..533ddb02dc 100755 --- a/py_env_create +++ b/py_env_create @@ -143,6 +143,19 @@ if [ ! -f $condafile ]; then exit -1 fi +# Given two version number strings (unsigned integers separated by periods), echo "true" if the +# second one is "greater than or equal to" the first. Otherwise, echo "false". +# Based on https://unix.stackexchange.com/a/285928/208738 +function is_version2_ge_version1 { + requiredver=$1 + currentver=$2 + if [ "$(printf '%s\n' "$requiredver" "$currentver" | sort -V | head -n1)" = "$requiredver" ]; then + echo "true" + else + echo "false" + fi +} + # # Handle an environment that already exists # @@ -155,7 +168,14 @@ function rename_existing_env { exit 1 elif [[ $(conda_env_exists $2) -eq 0 ]]; then echo "Renaming $1 to $2 (this will take a few minutes)..." - yes_tmp=${yes/yes/force} + conda_version=$(conda --version | cut -d" " -f2) + if [[ "$(is_version2_ge_version1 25.3.0 ${conda_version})" == "true" ]]; then + yes_tmp=$yes + else + # Before conda 25.3.0, conda rename needed --force instead of --yes + # https://github.com/conda/conda/blob/408abc3a3826c80750883accb8896005e3ac97b2/CHANGELOG.md?plain=1#L291 + yes_tmp=${yes/yes/force} + fi # Use conda instead of $condamamba because, at least as of mamba 1.5.9 / conda 24.7.1, # rename is not supported through mamba. if [[ "${quiet}" == "--quiet" ]]; then From 8698bfc5c550afbba1f397fb4eaea0478fe0047e Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 11 Aug 2025 10:45:39 -0600 Subject: [PATCH 013/196] test_sys_py_env_create: Empty YML file for mamba. --- python/ctsm/test/test_sys_py_env_create.py | 13 ++++++++++--- python/empty.yml | 6 ++++++ 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 python/empty.yml diff --git a/python/ctsm/test/test_sys_py_env_create.py b/python/ctsm/test/test_sys_py_env_create.py index 43f9d76a56..4da98dcf48 100644 --- a/python/ctsm/test/test_sys_py_env_create.py +++ b/python/ctsm/test/test_sys_py_env_create.py @@ -60,9 +60,11 @@ def setUp(self): self.py_env_create = os.path.join(path_to_ctsm_root(), "py_env_create") assert os.path.exists(self.py_env_create) - # Get path to testing condafile + # Get path to testing conda/mambafile: + # conda needs a completely empty file (# comments okay) as of 25.5.1, but mamba as of 2.3.1 + # needs at least some YML structure. self.empty_condafile = os.path.join(path_to_ctsm_root(), "python", "empty.txt") - assert os.path.exists(self.empty_condafile) + self.empty_mambafile = os.path.join(path_to_ctsm_root(), "python", "empty.yml") # Set up other variables self.env_names = [] @@ -89,7 +91,12 @@ def _create_empty_env(self, check=None, extra_args=None, expect_error=False, new self.env_names.append(get_unique_env_name(5)) # Form and run command - cmd = [self.py_env_create, "-n", self.env_names[-1], "-f", self.empty_condafile, "--yes"] + if extra_args is not None and ("-m" in extra_args or "--mamba" in extra_args): + empty_file = self.empty_mambafile + else: + empty_file = self.empty_condafile + assert os.path.exists(empty_file) + cmd = [self.py_env_create, "-n", self.env_names[-1], "-f", empty_file, "--yes"] if extra_args: cmd += extra_args out = subprocess.run(cmd, capture_output=True, text=True, check=False) diff --git a/python/empty.yml b/python/empty.yml new file mode 100644 index 0000000000..02041ff565 --- /dev/null +++ b/python/empty.yml @@ -0,0 +1,6 @@ +# An empty file for use in testing py_env_create +# +channels: + - conda-forge + - defaults +dependencies: From f591ba59d3fa713e63c2aa88884ace5a606d2ad7 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 11 Aug 2025 10:46:20 -0600 Subject: [PATCH 014/196] test_sys_py_env_create: Better err msg if env not found. --- python/ctsm/test/test_sys_py_env_create.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/ctsm/test/test_sys_py_env_create.py b/python/ctsm/test/test_sys_py_env_create.py index 4da98dcf48..dc15a2f738 100644 --- a/python/ctsm/test/test_sys_py_env_create.py +++ b/python/ctsm/test/test_sys_py_env_create.py @@ -334,7 +334,8 @@ def test_complete_py_env_create(self): raise e env_list = get_conda_envs() for env_name in self.env_names: - assert does_env_exist(env_name, env_list) + if not does_env_exist(env_name, env_list): + raise AssertionError(f"environment not found: {env_name}") def test_complete_py_env_create_mamba(self): """ @@ -365,7 +366,8 @@ def test_complete_py_env_create_mamba(self): raise e env_list = get_conda_envs() for env_name in self.env_names: - assert does_env_exist(env_name, env_list) + if not does_env_exist(env_name, env_list): + raise AssertionError(f"environment not found: {env_name}") if __name__ == "__main__": From b40d87fa2cb45454efda9d8b4b7ebe0f81f37c2b Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 11 Aug 2025 11:08:36 -0600 Subject: [PATCH 015/196] py_env_create: conda_env_exists now always uses conda. At least as of mamba 2.3.1, "mamba env list" doesn't always show environment names. See https://github.com/mamba-org/mamba/issues/4045 --- py_env_create | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/py_env_create b/py_env_create index 533ddb02dc..cf34131e41 100755 --- a/py_env_create +++ b/py_env_create @@ -160,7 +160,9 @@ function is_version2_ge_version1 { # Handle an environment that already exists # function conda_env_exists { - ${condamamba} env list | grep -oE "/$1$" | wc -l + # Not using ${condamamba} because, at least as of mamba 2.3.1, it doesn't always show environment + # names. See https://github.com/mamba-org/mamba/issues/4045 + conda env list | grep -oE "/$1$" | wc -l } function rename_existing_env { if [[ "$CONDA_DEFAULT_ENV" == *"$1" ]]; then From a627c5c43dabed01a13b70f1228f2457d1264156 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 11 Aug 2025 12:26:18 -0600 Subject: [PATCH 016/196] Improve comment in empty.yml. --- python/empty.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/empty.yml b/python/empty.yml index 02041ff565..befe96c419 100644 --- a/python/empty.yml +++ b/python/empty.yml @@ -1,4 +1,5 @@ -# An empty file for use in testing py_env_create +# An empty file for use in testing py_env_create with mamba, which needs some YML structure +# (as opposed to conda, which can use a truly empty file) # channels: - conda-forge From 20331d1dcc3c7ed99e92d58a6dd59cff0d015d41 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 16 Jul 2025 10:56:45 -0600 Subject: [PATCH 017/196] Suppress crop cold temp msgs when generate_crop_gdds true. These get printed a LOT if you put every crop everywhere. --- src/biogeochem/CNPhenologyMod.F90 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/biogeochem/CNPhenologyMod.F90 b/src/biogeochem/CNPhenologyMod.F90 index ff1e5b1b74..5e347e1e9f 100644 --- a/src/biogeochem/CNPhenologyMod.F90 +++ b/src/biogeochem/CNPhenologyMod.F90 @@ -3120,7 +3120,9 @@ subroutine vernalization(p, & tkil = (tbase - 6._r8) - 6._r8 * hdidx(p) if (tkil >= tcrown) then if ((0.95_r8 - 0.02_r8 * (tcrown - tkil)**2) >= 0.02_r8) then - write (iulog,*) 'crop damaged by cold temperatures at p,c =', p,c + if (.not. generate_crop_gdds) then + write (iulog,*) 'crop damaged by cold temperatures at p,c =', p,c + end if else if (tlai(p) > 0._r8) then ! slevis: kill if past phase1 by forcing through harvest ! srabin: do this with force_harvest instead of setting @@ -3129,7 +3131,9 @@ subroutine vernalization(p, & ! on "maturity." This can occur when generate_crop_gdds ! is true. force_harvest = .true. - write (iulog,*) '95% of crop killed by cold temperatures at p,c =', p,c + if (.not. generate_crop_gdds) then + write (iulog,*) '95% of crop killed by cold temperatures at p,c =', p,c + end if end if end if end if From 3a3792320af6dd970b2cfa50bd573b1b63106cf8 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 25 Jul 2025 13:04:49 -0600 Subject: [PATCH 018/196] Move log() and error() from generate_gdds_functions to cropcal_utils. --- python/ctsm/crop_calendars/cropcal_utils.py | 19 +++ python/ctsm/crop_calendars/generate_gdds.py | 25 +-- .../crop_calendars/generate_gdds_functions.py | 155 ++++++++---------- 3 files changed, 101 insertions(+), 98 deletions(-) diff --git a/python/ctsm/crop_calendars/cropcal_utils.py b/python/ctsm/crop_calendars/cropcal_utils.py index c7e8b6ac52..9ddcfb0194 100644 --- a/python/ctsm/crop_calendars/cropcal_utils.py +++ b/python/ctsm/crop_calendars/cropcal_utils.py @@ -9,6 +9,25 @@ from ctsm.utils import is_instantaneous +def log(logger_in, string): + """ + Simultaneously print INFO messages to console and to log file + """ + print(string) + if logger_in: + logger_in.info(string) + + +def error(logger_in, string): + """ + Simultaneously print ERROR messages to console and to log file + """ + print(string) + if logger_in: + logger_in.error(string) + raise RuntimeError(string) + + def define_pftlist(): """ Return list of PFTs used in CLM diff --git a/python/ctsm/crop_calendars/generate_gdds.py b/python/ctsm/crop_calendars/generate_gdds.py index bde28ca80d..0e17428fa1 100644 --- a/python/ctsm/crop_calendars/generate_gdds.py +++ b/python/ctsm/crop_calendars/generate_gdds.py @@ -20,6 +20,7 @@ sys.path.insert(1, _CTSM_PYTHON) import ctsm.crop_calendars.cropcal_module as cc # pylint: disable=wrong-import-position import ctsm.crop_calendars.generate_gdds_functions as gddfn # pylint: disable=wrong-import-position +import ctsm.crop_calendars.cropcal_utils as utils # pylint: disable=wrong-import-position # Functions here were written with too many positional arguments. At some point that should be # fixed. For now, we'll just disable the warning. @@ -85,11 +86,11 @@ def main( raise RuntimeError( "only_make_figs True but not all plotting modules are available" ) from exc - gddfn.log(logger, "Not all plotting modules are available; disabling save_figs") + utils.log(logger, "Not all plotting modules are available; disabling save_figs") save_figs = False # Print some info - gddfn.log(logger, f"Saving to {output_dir}") + utils.log(logger, f"Saving to {output_dir}") # Parse list of crops to skip if "," in skip_crops: @@ -107,7 +108,7 @@ def main( yr_1_import_str = f"{first_season+1}-01-01" yr_n_import_str = f"{last_season+2}-01-01" - gddfn.log( + utils.log( logger, f"Importing netCDF time steps {yr_1_import_str} through {yr_n_import_str} " + "(years are +1 because of CTSM output naming)", @@ -191,7 +192,7 @@ def main( h1_instantaneous, ) - gddfn.log(logger, f" Saving pickle file ({pickle_file})...") + utils.log(logger, f" Saving pickle file ({pickle_file})...") with open(pickle_file, "wb") as file: pickle.dump( [ @@ -219,7 +220,7 @@ def main( [i for i, c in enumerate(gddaccum_yp_list) if not isinstance(c, type(None))] ] - gddfn.log(logger, "Done") + utils.log(logger, "Done") if not h2_ds: h2_ds = xr.open_dataset(h2_ds_file) @@ -236,7 +237,7 @@ def main( "s", sdates_rx, incl_patches1d_itype_veg, mxsowings, logger ) - gddfn.log(logger, "Getting and gridding mean GDDs...") + utils.log(logger, "Getting and gridding mean GDDs...") gdd_maps_ds = gddfn.yp_list_to_ds( gddaccum_yp_list, h2_ds, incl_vegtypes_str, sdates_rx, longname_prefix, logger ) @@ -247,10 +248,10 @@ def main( # Fill NAs with dummy values dummy_fill = -1 gdd_maps_ds = gdd_maps_ds.fillna(dummy_fill) - gddfn.log(logger, "Done getting and gridding means.") + utils.log(logger, "Done getting and gridding means.") # Add dummy variables for crops not actually simulated - gddfn.log(logger, "Adding dummy variables...") + utils.log(logger, "Adding dummy variables...") # Unnecessary? template_ds = xr.open_dataset(sdates_file, decode_times=True) all_vars = [v.replace("sdate", "gdd") for v in template_ds if "sdate" in v] @@ -278,7 +279,7 @@ def make_dummy(this_crop_gridded, addend): for var_index, this_var in enumerate(dummy_vars): if this_var in gdd_maps_ds: - gddfn.error( + utils.error( logger, f"{this_var} is already in gdd_maps_ds. Why overwrite it with dummy?" ) dummy_gridded.name = this_var @@ -294,14 +295,14 @@ def add_lonlat_attrs(this_ds): gdd_maps_ds = add_lonlat_attrs(gdd_maps_ds) gddharv_maps_ds = add_lonlat_attrs(gddharv_maps_ds) - gddfn.log(logger, "Done.") + utils.log(logger, "Done.") ###################### ### Save to netCDF ### ###################### if not only_make_figs: - gddfn.log(logger, "Saving...") + utils.log(logger, "Saving...") # Get output file path datestr = dt.datetime.now().strftime("%Y%m%d_%H%M%S") @@ -336,7 +337,7 @@ def save_gdds(sdates_file, hdates_file, outfile, gdd_maps_ds, sdates_rx): save_gdds(sdates_file, hdates_file, outfile, gdd_maps_ds, sdates_rx) - gddfn.log(logger, "Done saving.") + utils.log(logger, "Done saving.") ######################################## ### Save things needed for mapmaking ### diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index 0489f320b7..6f8f71ea71 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -54,28 +54,11 @@ CAN_PLOT = False -def log(logger, string): - """ - Simultaneously print INFO messages to console and to log file - """ - print(string) - logger.info(string) - - -def error(logger, string): - """ - Simultaneously print ERROR messages to console and to log file - """ - print(string) - logger.error(string) - raise RuntimeError(string) - - def check_sdates(dates_ds, sdates_rx, outdir_figs, logger, verbose=False): """ Checking that input and output sdates match """ - log(logger, " Checking that input and output sdates match...") + utils.log(logger, " Checking that input and output sdates match...") sdates_grid = grid_one_variable(dates_ds, "SDATES") @@ -89,12 +72,12 @@ def check_sdates(dates_ds, sdates_rx, outdir_figs, logger, verbose=False): this_var = f"gs1_{vegtype_int}" if this_var not in sdates_rx: vegtypes_skipped = vegtypes_skipped + [vegtype_str] - # log(logger, f" {vt_str} ({vt}) SKIPPED...") + # utils.log(logger, f" {vt_str} ({vt}) SKIPPED...") continue vegtypes_included = vegtypes_included + [vegtype_str] any_found = True if verbose: - log(logger, f" {vegtype_str} ({vegtype_int})...") + utils.log(logger, f" {vegtype_str} ({vegtype_int})...") in_map = sdates_rx[this_var].squeeze(drop=True) # Output @@ -104,23 +87,23 @@ def check_sdates(dates_ds, sdates_rx, outdir_figs, logger, verbose=False): diff_map = out_map - in_map diff_map_notnan = diff_map.values[np.invert(np.isnan(diff_map.values))] if np.any(diff_map_notnan): - log(logger, f"Difference(s) found in {vegtype_str}") + utils.log(logger, f"Difference(s) found in {vegtype_str}") here = np.where(diff_map_notnan) - log(logger, "in:") + utils.log(logger, "in:") in_map_notnan = in_map.values[np.invert(np.isnan(diff_map.values))] - log(logger, in_map_notnan[here][0:4]) + utils.log(logger, in_map_notnan[here][0:4]) out_map_notnan = out_map.values[np.invert(np.isnan(diff_map.values))] - log(logger, "out:") - log(logger, out_map_notnan[here][0:4]) - log(logger, "diff:") - log(logger, diff_map_notnan[here][0:4]) + utils.log(logger, "out:") + utils.log(logger, out_map_notnan[here][0:4]) + utils.log(logger, "diff:") + utils.log(logger, diff_map_notnan[here][0:4]) first_diff = all_ok all_ok = False if CAN_PLOT: sdate_diffs_dir = os.path.join(outdir_figs, "sdate_diffs") if first_diff: - log(logger, f"Saving sdate difference figures to {sdate_diffs_dir}") + utils.log(logger, f"Saving sdate difference figures to {sdate_diffs_dir}") if not os.path.exists(sdate_diffs_dir): os.makedirs(sdate_diffs_dir) in_map.where(~np.isnan(out_map)).plot() @@ -137,24 +120,24 @@ def check_sdates(dates_ds, sdates_rx, outdir_figs, logger, verbose=False): plt.close() if not any_found: - error(logger, "No matching variables found in sdates_rx!") + utils.error(logger, "No matching variables found in sdates_rx!") # Sanity checks for included vegetation types vegtypes_skipped = np.unique([x.replace("irrigated_", "") for x in vegtypes_skipped]) vegtypes_skipped_weird = [x for x in vegtypes_skipped if x in vegtypes_included] if np.array_equal(vegtypes_included, [x.replace("irrigated_", "") for x in vegtypes_included]): - log(logger, "\nWARNING: No irrigated crops included!!!\n") + utils.log(logger, "\nWARNING: No irrigated crops included!!!\n") elif vegtypes_skipped_weird: - log( + utils.log( logger, "\nWarning: Some crop types had output rainfed patches but no irrigated patches: " + f"{vegtypes_skipped_weird}", ) if all_ok: - log(logger, " ✅ Input and output sdates match!") + utils.log(logger, " ✅ Input and output sdates match!") else: - error(logger, " ❌ Input and output sdates differ.") + utils.error(logger, " ❌ Input and output sdates differ.") def import_rx_dates(s_or_h, date_infile, incl_patches1d_itype_veg, mxsowings, logger): @@ -164,7 +147,7 @@ def import_rx_dates(s_or_h, date_infile, incl_patches1d_itype_veg, mxsowings, lo if isinstance(date_infile, xr.Dataset): return date_infile if not isinstance(date_infile, str): - error( + utils.error( logger, f"Importing {s_or_h}dates_rx: Expected date_infile to be str or DataArray," + f"not {type(date_infile)}", @@ -220,7 +203,7 @@ def yp_list_to_ds(yp_list, daily_ds, incl_vegtypes_str, dates_rx, longname_prefi if isinstance(data, type(None)): continue this_crop_str = incl_vegtypes_str[this_crop_int] - log(logger, f" {this_crop_str}...") + utils.log(logger, f" {this_crop_str}...") new_var = f"gdd1_{utils.ivt_str2int(this_crop_str)}" this_ds = daily_ds.isel( patch=np.where(daily_ds.patches1d_itype_veg_str.values == this_crop_str)[0] @@ -270,8 +253,8 @@ def import_and_process_1yr( Import one year of CLM output data for GDD generation """ save_figs = True - log(logger, f"netCDF year {this_year}...") - log(logger, dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + utils.log(logger, f"netCDF year {this_year}...") + utils.log(logger, dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) # Without dask, this can take a LONG time at resolutions finer than 2-deg if importlib_util.find_spec("dask"): @@ -286,7 +269,7 @@ def import_and_process_1yr( h1_pattern = os.path.join(indir, "*h1i.*.nc.base") h1_filelist = glob.glob(h1_pattern) if not h1_filelist: - error(logger, "No files found matching pattern '*h1i.*.nc(.base)'") + utils.error(logger, "No files found matching pattern '*h1i.*.nc(.base)'") # Get list of crops to include if skip_crops is not None: @@ -315,7 +298,7 @@ def import_and_process_1yr( if dates_ds.dims["time"] > 1: if dates_ds.dims["time"] == 365: if not incorrectly_daily: - log( + utils.log( logger, " ℹ️ You saved SDATES and HDATES daily, but you only needed annual. Fixing.", ) @@ -334,9 +317,9 @@ def import_and_process_1yr( ) n_unmatched_nans = np.sum(sdates_all_nan != hdates_all_nan) if n_unmatched_nans > 0: - error(logger, "Output SDATE and HDATE NaN masks do not match.") + utils.error(logger, "Output SDATE and HDATE NaN masks do not match.") if np.sum(~np.isnan(dates_ds.SDATES.values)) == 0: - error(logger, "All SDATES are NaN!") + utils.error(logger, "All SDATES are NaN!") # Just work with non-NaN patches for now skip_patches_for_isel_nan = np.where(sdates_all_nan)[0] @@ -345,7 +328,7 @@ def import_and_process_1yr( skip_patches_for_isel_nan_last_year, skip_patches_for_isel_nan ) if different_nan_mask: - log(logger, " Different NaN mask than last year") + utils.log(logger, " Different NaN mask than last year") incl_thisyr_but_nan_lastyr = [ dates_ds.patch.values[p] for p in incl_patches_for_isel_nan @@ -355,7 +338,7 @@ def import_and_process_1yr( incl_thisyr_but_nan_lastyr = [] skipping_patches_for_isel_nan = len(skip_patches_for_isel_nan) > 0 if skipping_patches_for_isel_nan: - log( + utils.log( logger, f" Ignoring {len(skip_patches_for_isel_nan)} patches with all-NaN sowing and " + "harvest dates.", @@ -374,14 +357,14 @@ def import_and_process_1yr( if isinstance(incl_vegtypes_str, np.ndarray): incl_vegtypes_str = list(incl_vegtypes_str) if incl_vegtypes_str != list(dates_incl_ds.vegtype_str.values): - error( + utils.error( logger, f"Included veg types differ. Previously {incl_vegtypes_str}, " + f"now {dates_incl_ds.vegtype_str.values}", ) if np.sum(~np.isnan(dates_incl_ds.SDATES.values)) == 0: - error(logger, "All SDATES are NaN after ignoring those patches!") + utils.error(logger, "All SDATES are NaN after ignoring those patches!") # Some patches can have -1 sowing date?? Hopefully just an artifact of me incorrectly saving # SDATES/HDATES daily. @@ -394,14 +377,14 @@ def import_and_process_1yr( dates_incl_ds.HDATES.isel(mxharvests=0, patch=skip_patches_for_isel_sdatelt1).values ) if incorrectly_daily and list(unique_hdates) == [364]: - log( + utils.log( logger, f" ❗ {len(skip_patches_for_isel_sdatelt1)} patches have SDATE < 1, but this" + "might have just been because of incorrectly daily outputs. Setting them to 365.", ) new_sdates_ar = dates_incl_ds.SDATES.values if mxsowings_dim != 0: - error(logger, "Code this up") + utils.error(logger, "Code this up") new_sdates_ar[0, skip_patches_for_isel_sdatelt1] = 365 dates_incl_ds["SDATES"] = xr.DataArray( data=new_sdates_ar, @@ -409,7 +392,7 @@ def import_and_process_1yr( attrs=dates_incl_ds["SDATES"].attrs, ) else: - error( + utils.error( logger, f"{len(skip_patches_for_isel_sdatelt1)} patches have SDATE < 1. " + f"Unique affected hdates: {unique_hdates}", @@ -432,10 +415,10 @@ def import_and_process_1yr( if np.any(hdates_thisyr_where_nan_lastyr < 1): new_hdates = dates_incl_ds.HDATES.values if mxharvests_dim != 0: - error(logger, "Code this up") + utils.error(logger, "Code this up") patch_list = list(hdates_thisyr.patch.values) here = [patch_list.index(x) for x in incl_thisyr_but_nan_lastyr] - log( + utils.log( logger, f" ❗ {len(here)} patches have harvest date -1 because they weren't active last" + "year (and were either never active or were harvested when last active). " @@ -460,7 +443,7 @@ def import_and_process_1yr( dates_incl_ds.SDATES.isel(patch=skip_patches_for_isel_hdatelt1).values ) if incorrectly_daily and list(unique_sdates) == [1]: - log( + utils.log( logger, f" ❗ {len(skip_patches_for_isel_hdatelt1)} patches have HDATE < 1??? Seems like " + "this might have just been because of incorrectly daily outputs; setting them to " @@ -468,7 +451,7 @@ def import_and_process_1yr( ) new_hdates_ar = dates_incl_ds.HDATES.values if mxharvests_dim != 0: - error(logger, "Code this up") + utils.error(logger, "Code this up") new_hdates_ar[0, skip_patches_for_isel_hdatelt1] = 365 dates_incl_ds["HDATES"] = xr.DataArray( data=new_hdates_ar, @@ -476,7 +459,7 @@ def import_and_process_1yr( attrs=dates_incl_ds["HDATES"].attrs, ) else: - error( + utils.error( logger, f"{len(skip_patches_for_isel_hdatelt1)} patches have HDATE < 1. Possible causes:\n" + "* Not using constant crop areas (e.g., flanduse_timeseries from " @@ -492,7 +475,7 @@ def import_and_process_1yr( >= 1 ) if n_extra_harv > 0: - error(logger, f"{n_extra_harv} patches have >1 harvest.") + utils.error(logger, f"{n_extra_harv} patches have >1 harvest.") # Make sure harvest happened the day before sowing sdates_clm = dates_incl_ds.SDATES.values.squeeze() @@ -500,7 +483,7 @@ def import_and_process_1yr( diffdates_clm = sdates_clm - hdates_clm diffdates_clm[(sdates_clm == 1) & (hdates_clm == 365)] = 1 if list(np.unique(diffdates_clm)) != [1]: - error(logger, f"Not all sdates-hdates are 1: {np.unique(diffdates_clm)}") + utils.error(logger, f"Not all sdates-hdates are 1: {np.unique(diffdates_clm)}") # Import expected sowing dates. This will also be used as our template output file. imported_sdates = isinstance(sdates_rx, str) @@ -563,7 +546,7 @@ def import_and_process_1yr( else: hdates_rx = hdates_rx_orig - log(logger, " Importing accumulated GDDs...") + utils.log(logger, " Importing accumulated GDDs...") clm_gdd_var = "GDDACCUM" my_vars = [clm_gdd_var, "GDDHARV"] patterns = [f"*h2i.{this_year-1}-01*.nc", f"*h2i.{this_year-1}-01*.nc.base"] @@ -573,7 +556,7 @@ def import_and_process_1yr( if h2_files: break if not h2_files: - error(logger, f"No files found matching patterns: {patterns}") + utils.error(logger, f"No files found matching patterns: {patterns}") h2_ds = import_ds( h2_files, my_vars=my_vars, @@ -584,13 +567,13 @@ def import_and_process_1yr( # Restrict to patches we're including if skipping_patches_for_isel_nan: if not np.array_equal(dates_ds.patch.values, h2_ds.patch.values): - error(logger, "dates_ds and h2_ds don't have the same patch list!") + utils.error(logger, "dates_ds and h2_ds don't have the same patch list!") h2_incl_ds = h2_ds.isel(patch=incl_patches_for_isel_nan) else: h2_incl_ds = h2_ds if not np.any(h2_incl_ds[clm_gdd_var].values != 0): - error(logger, f"All {clm_gdd_var} values are zero!") + utils.error(logger, f"All {clm_gdd_var} values are zero!") # Get standard datetime axis for outputs n_years = year_n - year_1 + 1 @@ -604,7 +587,7 @@ def import_and_process_1yr( incl_vegtype_indices = [] for var, vegtype_str in enumerate(incl_vegtypes_str): if vegtype_str in skip_crops: - log(logger, f" SKIPPING {vegtype_str}") + utils.log(logger, f" SKIPPING {vegtype_str}") continue vegtype_int = utils.vegtype_str2int(vegtype_str)[0] @@ -619,7 +602,7 @@ def import_and_process_1yr( check_gddharv = True if not this_crop_gddaccum_da.size: continue - log(logger, f" {vegtype_str}...") + utils.log(logger, f" {vegtype_str}...") incl_vegtype_indices = incl_vegtype_indices + [var] # Get prescribed harvest dates for these patches @@ -652,7 +635,7 @@ def import_and_process_1yr( if not np.all( this_crop_gddaccum_da.patch.values[:-1] <= this_crop_gddaccum_da.patch.values[1:] ): - error(logger, "This code depends on DataArray patch list being sorted.") + utils.error(logger, "This code depends on DataArray patch list being sorted.") sortorder = np.argsort(patches) i_patches = list(np.array(i_patches)[np.array(sortorder)]) i_times = list(np.array(i_times)[np.array(sortorder)]) @@ -662,17 +645,17 @@ def import_and_process_1yr( if save_figs: gddharv_atharv_p = this_crop_gddharv_da.values[(i_times, i_patches)] if np.any(np.isnan(gddaccum_atharv_p)): - log( + utils.log( logger, f" ❗ {np.sum(np.isnan(gddaccum_atharv_p))}/{len(gddaccum_atharv_p)} " + "NaN after extracting GDDs accumulated at harvest", ) if save_figs and gddharv_atharv_p is not None and np.any(np.isnan(gddharv_atharv_p)): if np.all(np.isnan(gddharv_atharv_p)): - log(logger, " ❗ All GDDHARV are NaN; should only affect figure") + utils.log(logger, " ❗ All GDDHARV are NaN; should only affect figure") check_gddharv = False else: - log( + utils.log( logger, f" ❗ {np.sum(np.isnan(gddharv_atharv_p))}/{len(gddharv_atharv_p)} " + "NaN after extracting GDDHARV", @@ -699,14 +682,14 @@ def import_and_process_1yr( ] if not np.array_equal(last_year_active_patch_indices, this_year_active_patch_indices): if incorrectly_daily: - log( + utils.log( logger, " ❗ This year's active patch indices differ from last year's. " + "Allowing because this might just be an artifact of incorrectly daily " + "outputs, BUT RESULTS MUST NOT BE TRUSTED.", ) else: - error(logger, "This year's active patch indices differ from last year's.") + utils.error(logger, "This year's active patch indices differ from last year's.") # Make sure we're not about to overwrite any existing values. if np.any( ~np.isnan( @@ -714,28 +697,28 @@ def import_and_process_1yr( ) ): if incorrectly_daily: - log( + utils.log( logger, " ❗ Unexpected non-NaN for last season's GDD accumulation. " + "Allowing because this might just be an artifact of incorrectly daily " + "outputs, BUT RESULTS MUST NOT BE TRUSTED.", ) else: - error(logger, "Unexpected non-NaN for last season's GDD accumulation") + utils.error(logger, "Unexpected non-NaN for last season's GDD accumulation") if save_figs and np.any( ~np.isnan( gddharv_yp_list[var][year_index - 1, active_this_year_where_gs_lastyr_indices] ) ): if incorrectly_daily: - log( + utils.log( logger, " ❗ Unexpected non-NaN for last season's GDDHARV. Allowing " + "because this might just be an artifact of incorrectly daily outputs, " + "BUT RESULTS MUST NOT BE TRUSTED.", ) else: - error(logger, "Unexpected non-NaN for last season's GDDHARV") + utils.error(logger, "Unexpected non-NaN for last season's GDDHARV") # Fill. gddaccum_yp_list[var][year_index - 1, active_this_year_where_gs_lastyr_indices] = ( gddaccum_atharv_p[where_gs_lastyr] @@ -751,14 +734,14 @@ def import_and_process_1yr( ) ): if incorrectly_daily: - log( + utils.log( logger, " ❗ Unexpected NaN for last season's GDD accumulation. Allowing " + "because this might just be an artifact of incorrectly daily outputs, " + "BUT RESULTS MUST NOT BE TRUSTED.", ) else: - error(logger, "Unexpected NaN for last season's GDD accumulation.") + utils.error(logger, "Unexpected NaN for last season's GDD accumulation.") if ( save_figs and check_gddharv @@ -771,14 +754,14 @@ def import_and_process_1yr( ) ): if incorrectly_daily: - log( + utils.log( logger, " ❗ Unexpected NaN for last season's GDDHARV. Allowing because " + "this might just be an artifact of incorrectly daily outputs, BUT " + "RESULTS MUST NOT BE TRUSTED.", ) else: - error(logger, "Unexpected NaN for last season's GDDHARV.") + utils.error(logger, "Unexpected NaN for last season's GDDHARV.") gddaccum_yp_list[var][year_index, this_year_active_patch_indices] = tmp_gddaccum if save_figs: gddharv_yp_list[var][year_index, this_year_active_patch_indices] = tmp_gddharv @@ -794,14 +777,14 @@ def import_and_process_1yr( nanmask_output_gdds_lastyr = np.isnan(gddaccum_yp_list[var][year_index - 1, :]) if not np.array_equal(nanmask_output_gdds_lastyr, nanmask_output_sdates): if incorrectly_daily: - log( + utils.log( logger, " ❗ NaN masks differ between this year's sdates and 'filled-out' " + "GDDs from last year. Allowing because this might just be an artifact of " + "incorrectly daily outputs, BUT RESULTS MUST NOT BE TRUSTED.", ) else: - error( + utils.error( logger, "NaN masks differ between this year's sdates and 'filled-out' GDDs from " + "last year", @@ -811,7 +794,7 @@ def import_and_process_1yr( skip_patches_for_isel_nan_last_year = skip_patches_for_isel_nan # Could save space by only saving variables needed for gridding - log(logger, " Saving h2_ds...") + utils.log(logger, " Saving h2_ds...") h2_ds.to_netcdf(h2_ds_file) return ( @@ -1081,7 +1064,7 @@ def make_figures( """ if not gdd_maps_ds: if not this_dir: - error( + utils.error( logger, "If not providing gdd_maps_ds, you must provide thisDir (location of " + "gdd_maps.nc)", @@ -1089,7 +1072,7 @@ def make_figures( gdd_maps_ds = xr.open_dataset(this_dir + "gdd_maps.nc") if not gddharv_maps_ds: if not this_dir: - error( + utils.error( logger, "If not providing gddharv_maps_ds, you must provide thisDir (location of " + "gddharv_maps.nc)", @@ -1116,7 +1099,7 @@ def make_figures( if land_use_file: year_1_lu = year_1 if first_land_use_year is None else first_land_use_year year_n_lu = year_n if last_land_use_year is None else last_land_use_year - lu_ds = cc.open_lu_ds(land_use_file, year_1_lu, year_n_lu, gdd_maps_ds, ungrid=False) + lu_ds = cc.open_lu_ds(land_use_file, year_1_lu, year_n_lu, gdd_maps_ds, logger, ungrid=False) lu_years_text = f" (masked by {year_1_lu}-{year_n_lu} area)" lu_years_file = f"_mask{year_1_lu}-{year_n_lu}" else: @@ -1152,7 +1135,7 @@ def make_figures( # Maps nplot_y = 3 nplot_x = 1 - log(logger, "Making before/after maps...") + utils.log(logger, "Making before/after maps...") vegtype_list = incl_vegtypes_str if land_use_file: vegtype_list += ["Corn", "Cotton", "Rice", "Soybean", "Sugarcane", "Wheat"] @@ -1213,7 +1196,7 @@ def make_figures( spec = fig.add_gridspec(nrows=3, ncols=2, width_ratios=[0.5, 0.5], wspace=0.2) this_axis = fig.add_subplot(spec[0, 0], projection=ccrs.PlateCarree()) else: - error(logger, f"layout {layout} not recognized") + utils.error(logger, f"layout {layout} not recognized") gddharv_all_nan = np.all(np.isnan(gddharv_map_yx.values)) if gddharv_all_nan: @@ -1238,7 +1221,7 @@ def make_figures( elif layout in ["2x2", "3x2"]: this_axis = fig.add_subplot(spec[1, 0], projection=ccrs.PlateCarree()) else: - error(logger, f"layout {layout} not recognized") + utils.error(logger, f"layout {layout} not recognized") this_min = int(np.round(np.nanmin(gdd_map_yx))) this_max = int(np.round(np.nanmax(gdd_map_yx))) this_title = f"{run2_name} (range {this_min}–{this_max})" @@ -1323,7 +1306,7 @@ def make_figures( elif layout in ["2x2", "3x2"]: this_axis = fig.add_subplot(spec[:, 1]) else: - error(logger, f"layout {layout} not recognized") + utils.error(logger, f"layout {layout} not recognized") # Shift bottom of plot up to make room for legend ax_pos = this_axis.get_position() @@ -1376,4 +1359,4 @@ def make_figures( plt.savefig(outfile, dpi=300, transparent=False, facecolor="white", bbox_inches="tight") plt.close() - log(logger, "Done.") + utils.log(logger, "Done.") From fa14f3ae148800cc8ba63d451940ef72c76383c3 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 31 Jul 2025 14:45:22 -0600 Subject: [PATCH 019/196] Add logging in GDD generation. --- python/ctsm/crop_calendars/cropcal_module.py | 7 ++-- python/ctsm/crop_calendars/generate_gdds.py | 1 + .../crop_calendars/generate_gdds_functions.py | 6 ++-- python/ctsm/crop_calendars/import_ds.py | 35 ++++++++++++++++--- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/python/ctsm/crop_calendars/cropcal_module.py b/python/ctsm/crop_calendars/cropcal_module.py index c7ee9c581a..57dc4600d8 100644 --- a/python/ctsm/crop_calendars/cropcal_module.py +++ b/python/ctsm/crop_calendars/cropcal_module.py @@ -16,7 +16,6 @@ MISSING_RX_GDD_VAL = -1 - def check_and_trim_years(year_1, year_n, ds_in): """ After importing a file, restrict it to years of interest. @@ -49,11 +48,12 @@ def check_and_trim_years(year_1, year_n, ds_in): return ds_in -def open_lu_ds(filename, year_1, year_n, existing_ds, ungrid=True): +def open_lu_ds(filename, year_1, year_n, existing_ds, logger, ungrid=True): """ Open land-use dataset """ # Open and trim to years of interest + utils.log(logger, f"open_lu_ds(): Opening this_ds_gridded: {filename}") this_ds_gridded = xr.open_dataset(filename).sel(time=slice(year_1, year_n)) # Assign actual lon/lat coordinates @@ -347,6 +347,7 @@ def import_output( gdds_rx_ds=None, verbose=False, throw_errors=True, + logger=None, ): """ Import CLM output @@ -354,7 +355,7 @@ def import_output( any_bad = False # Import - this_ds = import_ds(filename, my_vars=my_vars, my_vegtypes=my_vegtypes) + this_ds = import_ds(filename, my_vars=my_vars, my_vegtypes=my_vegtypes, logger=logger) # Trim to years of interest (do not include extra year needed for finishing last growing season) if year_1 and year_n: diff --git a/python/ctsm/crop_calendars/generate_gdds.py b/python/ctsm/crop_calendars/generate_gdds.py index 0e17428fa1..bc0063fabf 100644 --- a/python/ctsm/crop_calendars/generate_gdds.py +++ b/python/ctsm/crop_calendars/generate_gdds.py @@ -223,6 +223,7 @@ def main( utils.log(logger, "Done") if not h2_ds: + utils.log(logger, f"generate_gdds main(): Opening h2_ds: {h2_ds_file}") h2_ds = xr.open_dataset(h2_ds_file) ###################################################### diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index 6f8f71ea71..984059922f 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -291,6 +291,7 @@ def import_and_process_1yr( my_vegtypes=crops_to_read, time_slice=slice(f"{slice_year}-01-01", f"{slice_year}-12-31"), chunks=chunks, + logger=logger, ) for timestep in dates_ds["time"].values: print(timestep) @@ -562,6 +563,7 @@ def import_and_process_1yr( my_vars=my_vars, my_vegtypes=crops_to_read, chunks=chunks, + logger=logger, ) # Restrict to patches we're including @@ -587,7 +589,7 @@ def import_and_process_1yr( incl_vegtype_indices = [] for var, vegtype_str in enumerate(incl_vegtypes_str): if vegtype_str in skip_crops: - utils.log(logger, f" SKIPPING {vegtype_str}") + utils.log(logger, f" import_and_process_1yr(): SKIPPING {vegtype_str}") continue vegtype_int = utils.vegtype_str2int(vegtype_str)[0] @@ -602,7 +604,7 @@ def import_and_process_1yr( check_gddharv = True if not this_crop_gddaccum_da.size: continue - utils.log(logger, f" {vegtype_str}...") + utils.log(logger, f" import_and_process_1yr(): {vegtype_str}...") incl_vegtype_indices = incl_vegtype_indices + [var] # Get prescribed harvest dates for these patches diff --git a/python/ctsm/crop_calendars/import_ds.py b/python/ctsm/crop_calendars/import_ds.py index 71ce28bcce..9d06be99dc 100644 --- a/python/ctsm/crop_calendars/import_ds.py +++ b/python/ctsm/crop_calendars/import_ds.py @@ -16,10 +16,11 @@ from ctsm.crop_calendars.xr_flexsel import xr_flexsel -def compute_derived_vars(ds_in, var): +def compute_derived_vars(ds_in, var, logger=None): """ Compute derived variables """ + utils.log(logger, f"compute_derived_vars(): Getting {var}...") if ( var == "HYEARS" and "HDATES" in ds_in @@ -44,13 +45,14 @@ def compute_derived_vars(ds_in, var): return ds_in -def manual_mfdataset(filelist, my_vars, my_vegtypes, time_slice): +def manual_mfdataset(filelist, my_vars, my_vegtypes, time_slice, logger=None): """ Opening a list of files with Xarray's open_mfdataset requires dask. This function is a workaround for Python environments that don't have dask. """ ds_out = None for filename in filelist: + utils.log(logger, f"manual_mfdataset(): Opening ds_in: {ds_in}") ds_in = xr.open_dataset(filename) ds_in = mfdataset_preproc(ds_in, my_vars, my_vegtypes, time_slice) if ds_out is None: @@ -66,7 +68,7 @@ def manual_mfdataset(filelist, my_vars, my_vegtypes, time_slice): return ds_out -def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice): +def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, logger=None): """ Function to drop unwanted variables in preprocessing of open_mfdataset(). @@ -76,8 +78,11 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice): named like "patch". This can later be reversed, for compatibility with other code, using patch2pft(). """ + utils.log(logger, "mfdataset_preproc(): Start") + # Rename "pft" dimension and variables to "patch", if needed if "pft" in ds_in.dims: + utils.log(logger, 'mfdataset_preproc(): Rename "pft" dimension and variables to "patch"') pattern = re.compile("pft.*1d") matches = [x for x in list(ds_in.keys()) if pattern.search(x) is not None] pft2patch_dict = {"pft": "patch"} @@ -87,6 +92,8 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice): derived_vars = [] if vars_to_import is not None: + utils.log(logger, "mfdataset_preproc(): Getting vars to drop...") + # Split vars_to_import into variables that are vs. aren't already in ds derived_vars = [v for v in vars_to_import if v not in ds_in] present_vars = [v for v in vars_to_import if v in ds_in] @@ -123,10 +130,12 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice): vars_to_drop = list(np.setdiff1d(varlist, vars_to_import)) # Drop them + utils.log(logger, f"mfdataset_preproc(): Dropping variables: {vars_to_drop}") ds_in = ds_in.drop_vars(vars_to_drop) # Add vegetation type info if "patches1d_itype_veg" in list(ds_in): + utils.log(logger, f"mfdataset_preproc(): Adding vegetation type info") this_pftlist = utils.define_pftlist() utils.get_patch_ivts( ds_in, this_pftlist @@ -146,18 +155,25 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice): # Restrict to veg. types of interest, if any if vegtypes_to_import is not None: + utils.log(logger, f"mfdataset_preproc(): Restricting veg types to: {vegtypes_to_import}") ds_in = xr_flexsel(ds_in, vegtype=vegtypes_to_import) # Restrict to time slice, if any if time_slice: + utils.log(logger, f"mfdataset_preproc(): Restricting time slice to: {time_slice}") ds_in = utils.safer_timeslice(ds_in, time_slice) # Finish import + utils.log(logger, "mfdataset_preproc(): decode_cf()...") ds_in = xr.decode_cf(ds_in, decode_times=True) # Compute derived variables + if derived_vars: + utils.log(logger, "mfdataset_preproc(): decode_cf()...") for var in derived_vars: - ds_in = compute_derived_vars(ds_in, var) + ds_in = compute_derived_vars(ds_in, var, logger) + + utils.log(logger, "mfdataset_preproc(): End") return ds_in @@ -201,6 +217,7 @@ def import_ds( my_vars_missing_ok=None, rename_lsmlatlon=False, chunks=None, + logger=None, ): """ Import a dataset that can be spread over multiple files, only including specified variables @@ -209,6 +226,8 @@ def import_ds( - DOES actually read the dataset into memory, but only AFTER dropping unwanted variables and/or vegetation types. """ + utils.log(logger, "import_ds(): Start") + filelist, my_vars, my_vegtypes, my_vars_missing_ok = process_inputs( filelist, my_vars, my_vegtypes, my_vars_missing_ok ) @@ -221,10 +240,12 @@ def import_ds( if time_slice: new_filelist = [] for file in sorted(filelist): + utils.log(logger, f"import_ds(): Getting filetime from file: {file}") filetime = xr.open_dataset(file).time filetime_sel = utils.safer_timeslice(filetime, time_slice) include_this_file = filetime_sel.size if include_this_file: + utils.log(logger, f"import_ds(): Including filetime : {filetime_sel['time'].values}") new_filelist.append(file) # If you found some matching files, but then you find one that doesn't, stop going @@ -250,7 +271,7 @@ def import_ds( warnings.filterwarnings(action="ignore", category=DeprecationWarning) dask_unavailable = find_spec("dask") is None if dask_unavailable: - this_ds = manual_mfdataset(filelist, my_vars, my_vegtypes, time_slice) + this_ds = manual_mfdataset(filelist, my_vars, my_vegtypes, time_slice, logger=logger) else: this_ds = xr.open_mfdataset( sorted(filelist), @@ -263,8 +284,11 @@ def import_ds( chunks=chunks, ) elif isinstance(filelist, str): + utils.log(logger, f"import_ds(): Opening this_ds from filelist: {filelist}") this_ds = xr.open_dataset(filelist, chunks=chunks) + utils.log(logger, "import_ds(): Calling mfdataset_preproc()...") this_ds = mfdataset_preproc(this_ds, my_vars, my_vegtypes, time_slice) + utils.log(logger, "import_ds(): Calling compute()...") this_ds = this_ds.compute() # Warn and/or error about variables that couldn't be imported or derived @@ -289,4 +313,5 @@ def import_ds( if "lsmlon" in this_ds.dims: this_ds = this_ds.rename({"lsmlon": "lon"}) + utils.log(logger, "import_ds(): End") return this_ds From c4127106ee11385700b2392c5f3961165d39f8ba Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 11 Aug 2025 15:13:22 -0600 Subject: [PATCH 020/196] cropcal_utils log(), error(): Prepend datetime string. --- python/ctsm/crop_calendars/cropcal_utils.py | 17 +++++++++++++---- .../crop_calendars/generate_gdds_functions.py | 3 +-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/python/ctsm/crop_calendars/cropcal_utils.py b/python/ctsm/crop_calendars/cropcal_utils.py index 9ddcfb0194..acaf74262d 100644 --- a/python/ctsm/crop_calendars/cropcal_utils.py +++ b/python/ctsm/crop_calendars/cropcal_utils.py @@ -2,29 +2,38 @@ utility functions copied from klindsay, https://github.com/klindsay28/CESM2_coup_carb_cycle_JAMES/blob/master/utils.py """ +from datetime import datetime import numpy as np import xarray as xr from ctsm.utils import is_instantaneous +def leading_datetime_string(): + """ + Return a datetime string like "YYYY-mm-dd HH:MM:SS " + """ + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " " + def log(logger_in, string): """ Simultaneously print INFO messages to console and to log file """ - print(string) + msg = leading_datetime_string() + string + print(msg) if logger_in: - logger_in.info(string) + logger_in.info(msg) def error(logger_in, string): """ Simultaneously print ERROR messages to console and to log file """ - print(string) + msg = leading_datetime_string() + string + print(msg) if logger_in: - logger_in.error(string) + logger_in.error(msg) raise RuntimeError(string) diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index 984059922f..11158eaa72 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -253,8 +253,7 @@ def import_and_process_1yr( Import one year of CLM output data for GDD generation """ save_figs = True - utils.log(logger, f"netCDF year {this_year}...") - utils.log(logger, dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + utils.log(logger, f"import_and_process_1yr(): netCDF year {this_year}...") # Without dask, this can take a LONG time at resolutions finer than 2-deg if importlib_util.find_spec("dask"): From beee2fc1aae29f5285a52296260c4c946d41e15a Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 11 Aug 2025 15:45:22 -0600 Subject: [PATCH 021/196] Reformat with black. --- python/ctsm/crop_calendars/cropcal_module.py | 1 + python/ctsm/crop_calendars/cropcal_utils.py | 2 ++ python/ctsm/crop_calendars/generate_gdds_functions.py | 4 +++- python/ctsm/crop_calendars/import_ds.py | 4 +++- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/python/ctsm/crop_calendars/cropcal_module.py b/python/ctsm/crop_calendars/cropcal_module.py index 57dc4600d8..a3dbda0cce 100644 --- a/python/ctsm/crop_calendars/cropcal_module.py +++ b/python/ctsm/crop_calendars/cropcal_module.py @@ -16,6 +16,7 @@ MISSING_RX_GDD_VAL = -1 + def check_and_trim_years(year_1, year_n, ds_in): """ After importing a file, restrict it to years of interest. diff --git a/python/ctsm/crop_calendars/cropcal_utils.py b/python/ctsm/crop_calendars/cropcal_utils.py index acaf74262d..1d3ff234f2 100644 --- a/python/ctsm/crop_calendars/cropcal_utils.py +++ b/python/ctsm/crop_calendars/cropcal_utils.py @@ -2,6 +2,7 @@ utility functions copied from klindsay, https://github.com/klindsay28/CESM2_coup_carb_cycle_JAMES/blob/master/utils.py """ + from datetime import datetime import numpy as np @@ -9,6 +10,7 @@ from ctsm.utils import is_instantaneous + def leading_datetime_string(): """ Return a datetime string like "YYYY-mm-dd HH:MM:SS " diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index 11158eaa72..dacaf59bc0 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -1100,7 +1100,9 @@ def make_figures( if land_use_file: year_1_lu = year_1 if first_land_use_year is None else first_land_use_year year_n_lu = year_n if last_land_use_year is None else last_land_use_year - lu_ds = cc.open_lu_ds(land_use_file, year_1_lu, year_n_lu, gdd_maps_ds, logger, ungrid=False) + lu_ds = cc.open_lu_ds( + land_use_file, year_1_lu, year_n_lu, gdd_maps_ds, logger, ungrid=False + ) lu_years_text = f" (masked by {year_1_lu}-{year_n_lu} area)" lu_years_file = f"_mask{year_1_lu}-{year_n_lu}" else: diff --git a/python/ctsm/crop_calendars/import_ds.py b/python/ctsm/crop_calendars/import_ds.py index 9d06be99dc..00b1bb7be6 100644 --- a/python/ctsm/crop_calendars/import_ds.py +++ b/python/ctsm/crop_calendars/import_ds.py @@ -245,7 +245,9 @@ def import_ds( filetime_sel = utils.safer_timeslice(filetime, time_slice) include_this_file = filetime_sel.size if include_this_file: - utils.log(logger, f"import_ds(): Including filetime : {filetime_sel['time'].values}") + utils.log( + logger, f"import_ds(): Including filetime : {filetime_sel['time'].values}" + ) new_filelist.append(file) # If you found some matching files, but then you find one that doesn't, stop going From ec194e523d51681251c85e53eab556cfbd1485b4 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 11 Aug 2025 15:45:51 -0600 Subject: [PATCH 022/196] Add previous commit to .git-blame-ignore-revs. --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 203d7b487a..1aeb4a4f37 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -70,3 +70,4 @@ cdf40d265cc82775607a1bf25f5f527bacc97405 4ad46f46de7dde753b4653c15f05326f55116b73 75db098206b064b8b7b2a0604d3f0bf8fdb950cc 84609494b54ea9732f64add43b2f1dd035632b4c +ac03492012837799b7111607188acff9f739044a From 478384de5e609e0c64da53c350eccabcb55151b7 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 11 Aug 2025 15:50:07 -0600 Subject: [PATCH 023/196] Refactoring to satisfy pylint. --- python/ctsm/crop_calendars/cropcal_module.py | 2 +- .../crop_calendars/generate_gdds_functions.py | 3 +-- python/ctsm/crop_calendars/import_ds.py | 24 ++++++++++++------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/python/ctsm/crop_calendars/cropcal_module.py b/python/ctsm/crop_calendars/cropcal_module.py index a3dbda0cce..07b56d0ed2 100644 --- a/python/ctsm/crop_calendars/cropcal_module.py +++ b/python/ctsm/crop_calendars/cropcal_module.py @@ -49,7 +49,7 @@ def check_and_trim_years(year_1, year_n, ds_in): return ds_in -def open_lu_ds(filename, year_1, year_n, existing_ds, logger, ungrid=True): +def open_lu_ds(filename, year_1, year_n, existing_ds, *, logger, ungrid=True): """ Open land-use dataset """ diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index dacaf59bc0..170b65502d 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -6,7 +6,6 @@ import warnings import os import glob -import datetime as dt from importlib import util as importlib_util import numpy as np import xarray as xr @@ -1101,7 +1100,7 @@ def make_figures( year_1_lu = year_1 if first_land_use_year is None else first_land_use_year year_n_lu = year_n if last_land_use_year is None else last_land_use_year lu_ds = cc.open_lu_ds( - land_use_file, year_1_lu, year_n_lu, gdd_maps_ds, logger, ungrid=False + land_use_file, year_1_lu, year_n_lu, gdd_maps_ds, logger=logger, ungrid=False ) lu_years_text = f" (masked by {year_1_lu}-{year_n_lu} area)" lu_years_file = f"_mask{year_1_lu}-{year_n_lu}" diff --git a/python/ctsm/crop_calendars/import_ds.py b/python/ctsm/crop_calendars/import_ds.py index 00b1bb7be6..f844c0a3c3 100644 --- a/python/ctsm/crop_calendars/import_ds.py +++ b/python/ctsm/crop_calendars/import_ds.py @@ -82,13 +82,7 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log # Rename "pft" dimension and variables to "patch", if needed if "pft" in ds_in.dims: - utils.log(logger, 'mfdataset_preproc(): Rename "pft" dimension and variables to "patch"') - pattern = re.compile("pft.*1d") - matches = [x for x in list(ds_in.keys()) if pattern.search(x) is not None] - pft2patch_dict = {"pft": "patch"} - for match in matches: - pft2patch_dict[match] = match.replace("pft", "patch").replace("patchs", "patches") - ds_in = ds_in.rename(pft2patch_dict) + ds_in = rename_pft_to_patch(ds_in, logger) derived_vars = [] if vars_to_import is not None: @@ -135,7 +129,7 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log # Add vegetation type info if "patches1d_itype_veg" in list(ds_in): - utils.log(logger, f"mfdataset_preproc(): Adding vegetation type info") + utils.log(logger, "mfdataset_preproc(): Adding vegetation type info") this_pftlist = utils.define_pftlist() utils.get_patch_ivts( ds_in, this_pftlist @@ -178,6 +172,20 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log return ds_in +def rename_pft_to_patch(ds_in, logger): + """ + Rename "pft" dimension and variables to "patch", if needed + """ + utils.log(logger, 'mfdataset_preproc(): Rename "pft" dimension and variables to "patch"') + pattern = re.compile("pft.*1d") + matches = [x for x in list(ds_in.keys()) if pattern.search(x) is not None] + pft2patch_dict = {"pft": "patch"} + for match in matches: + pft2patch_dict[match] = match.replace("pft", "patch").replace("patchs", "patches") + ds_in = ds_in.rename(pft2patch_dict) + return ds_in + + def process_inputs(filelist, my_vars, my_vegtypes, my_vars_missing_ok): """ Process inputs to import_ds() From debbe36c5dabe96489ab5bb88350af4de3ab33cf Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 10:38:49 -0600 Subject: [PATCH 024/196] Rename query_parameters/ to param_utils/. --- python/ctsm/{query_parameters => param_utils}/__init__.py | 0 .../ctsm/{query_parameters => param_utils}/query_paramfile.py | 0 tools/{query_parameters => param_utils}/query_paramfile | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename python/ctsm/{query_parameters => param_utils}/__init__.py (100%) rename python/ctsm/{query_parameters => param_utils}/query_paramfile.py (100%) rename tools/{query_parameters => param_utils}/query_paramfile (87%) diff --git a/python/ctsm/query_parameters/__init__.py b/python/ctsm/param_utils/__init__.py similarity index 100% rename from python/ctsm/query_parameters/__init__.py rename to python/ctsm/param_utils/__init__.py diff --git a/python/ctsm/query_parameters/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py similarity index 100% rename from python/ctsm/query_parameters/query_paramfile.py rename to python/ctsm/param_utils/query_paramfile.py diff --git a/tools/query_parameters/query_paramfile b/tools/param_utils/query_paramfile similarity index 87% rename from tools/query_parameters/query_paramfile rename to tools/param_utils/query_paramfile index d7d13b6f52..6bde5f09ec 100755 --- a/tools/query_parameters/query_paramfile +++ b/tools/param_utils/query_paramfile @@ -12,7 +12,7 @@ _CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'python') sys.path.insert(1, _CTSM_PYTHON) -from ctsm.query_parameters.query_paramfile import main +from ctsm.param_utils.query_paramfile import main if __name__ == "__main__": main() From e1f29b26c094d56a65a8e11614297ae2814fe2db Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 11:00:05 -0600 Subject: [PATCH 025/196] query_paramfile: Add unit tests of get_arguments(). --- python/ctsm/test/test_unit_query_paramfile.py | 47 +++++++++++++++++++ ...3.041.Nfix_params.v13.c250221_upplim250.nc | 3 ++ 2 files changed, 50 insertions(+) create mode 100755 python/ctsm/test/test_unit_query_paramfile.py create mode 100644 python/ctsm/test/testinputs/ctsm5.3.041.Nfix_params.v13.c250221_upplim250.nc diff --git a/python/ctsm/test/test_unit_query_paramfile.py b/python/ctsm/test/test_unit_query_paramfile.py new file mode 100755 index 0000000000..d748f8e424 --- /dev/null +++ b/python/ctsm/test/test_unit_query_paramfile.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +"""Unit tests for query_paramfile""" + +import unittest +import sys + +from ctsm import unit_testing + +from ctsm.param_utils import query_paramfile as qp + +# Allow names that pylint doesn't like, because otherwise I find it hard +# to make readable unit test names +# pylint: disable=invalid-name + + +class TestQueryParamfile(unittest.TestCase): + """Tests of query_paramfile""" + + def setUp(self): + self.orig_argv = sys.argv + + def tearDown(self): + sys.argv = self.orig_argv + + def test_query_paramfile_args_short(self): + """Test that all arguments can be set correctly with shortnames""" + input_path = "/path/to/input.nc" + sys.argv = ["get_arguments", "-i", input_path, "-p", "pft1,pft2", "var1,var2"] + args = qp.get_arguments() + self.assertEqual(input_path, args.input) + self.assertEqual("pft1,pft2", args.pft) + self.assertEqual("var1,var2", args.variables) + + def test_query_paramfile_args_long(self): + """Test that all arguments can be set correctly with longnames""" + input_path = "/path/to/input.nc" + sys.argv = ["get_arguments", "--input", input_path, "--pft", "pft1,pft2", "var1,var2"] + args = qp.get_arguments() + self.assertEqual(input_path, args.input) + self.assertEqual("pft1,pft2", args.pft) + self.assertEqual("var1,var2", args.variables) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/testinputs/ctsm5.3.041.Nfix_params.v13.c250221_upplim250.nc b/python/ctsm/test/testinputs/ctsm5.3.041.Nfix_params.v13.c250221_upplim250.nc new file mode 100644 index 0000000000..ab47c530f2 --- /dev/null +++ b/python/ctsm/test/testinputs/ctsm5.3.041.Nfix_params.v13.c250221_upplim250.nc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:539679e78631b252a2a9e00fe73024cbc9fdf455e496921e94483750c6cf610c +size 200764 From 71f2affd1ea07ef4b8399f103e8ff44503f2faf3 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 11:41:31 -0600 Subject: [PATCH 026/196] query_paramfile: Add unit tests of print_values(). --- python/ctsm/test/test_unit_query_paramfile.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/python/ctsm/test/test_unit_query_paramfile.py b/python/ctsm/test/test_unit_query_paramfile.py index d748f8e424..c85ce27d3b 100755 --- a/python/ctsm/test/test_unit_query_paramfile.py +++ b/python/ctsm/test/test_unit_query_paramfile.py @@ -4,6 +4,9 @@ import unittest import sys +import io +from contextlib import redirect_stdout +import xarray as xr from ctsm import unit_testing @@ -14,6 +17,26 @@ # pylint: disable=invalid-name +def _setup_pft_parameter_ds(): + """ + Set up a parameter Dataset with a PFT-dimensioned parameter + """ + pft_dimname = "pft" + pft_names_list = ["pft0", "pft1"] + pft_names_da = xr.DataArray( + name=qp.PFTNAME_VAR, + data=pft_names_list, + dims=[pft_dimname], + coords={pft_dimname: pft_names_list}, + ) + var_name = "pft_param" + pft_param_da = xr.DataArray( + data=[1986.0325, 1987.0724], dims=[pft_dimname], coords={qp.PFTNAME_VAR: pft_names_da} + ) + ds = xr.Dataset(data_vars={var_name: pft_param_da, pft_dimname: pft_names_da}) + return ds, var_name, pft_names_list + + class TestQueryParamfile(unittest.TestCase): """Tests of query_paramfile""" @@ -41,6 +64,48 @@ def test_query_paramfile_args_long(self): self.assertEqual("pft1,pft2", args.pft) self.assertEqual("var1,var2", args.variables) + def test_query_paramfile_print_scalar(self): + """Test that print_values works with a scalar parameter""" + scalar_da = xr.DataArray(data=1987.0724) + var_name = "scalar_param" + ds = xr.Dataset(data_vars={var_name: scalar_da}) + + f = io.StringIO() + with redirect_stdout(f): + qp.print_values(ds, var_name, selected_pfts=None, pft_names=None) + out = f.getvalue() + self.assertEqual("scalar_param: 1987.0724\n", out) + + def test_query_paramfile_print_pfts_selectnone(self): + """Test that print_values works with PFT-dimensioned parameter, selecting no PFTs""" + ds, var_name, pft_names_list = _setup_pft_parameter_ds() + + f = io.StringIO() + with redirect_stdout(f): + qp.print_values(ds, var_name, selected_pfts=None, pft_names=pft_names_list) + out = f.getvalue() + self.assertEqual("pft_param:\n pft0: 1986.0325\n pft1: 1987.0724\n", out) + + def test_query_paramfile_print_pfts_selectall(self): + """Test that print_values works with PFT-dimensioned parameter, selecting all PFTs""" + ds, var_name, pft_names_list = _setup_pft_parameter_ds() + + f = io.StringIO() + with redirect_stdout(f): + qp.print_values(ds, var_name, selected_pfts=pft_names_list, pft_names=pft_names_list) + out = f.getvalue() + self.assertEqual("pft_param:\n pft0: 1986.0325\n pft1: 1987.0724\n", out) + + def test_query_paramfile_print_pfts_selectone(self): + """Test that print_values works with PFT-dimensioned parameter, selecting one PFT""" + ds, var_name, pft_names_list = _setup_pft_parameter_ds() + + f = io.StringIO() + with redirect_stdout(f): + qp.print_values(ds, var_name, selected_pfts=["pft0"], pft_names=pft_names_list) + out = f.getvalue() + self.assertEqual("pft_param:\n pft0: 1986.0325\n", out) + if __name__ == "__main__": unit_testing.setup_for_tests() From 99baf017ec8f430d9382b601f4cda538b563e259 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 11:52:40 -0600 Subject: [PATCH 027/196] query_paramfile: Add simple system test; failing. --- python/ctsm/test/test_sys_query_paramfile.py | 47 +++++++++++++++++++ python/ctsm/test/test_unit_query_paramfile.py | 4 +- 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100755 python/ctsm/test/test_sys_query_paramfile.py diff --git a/python/ctsm/test/test_sys_query_paramfile.py b/python/ctsm/test/test_sys_query_paramfile.py new file mode 100755 index 0000000000..03c452dbf7 --- /dev/null +++ b/python/ctsm/test/test_sys_query_paramfile.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +"""System tests for query_paramfile""" + +import unittest +import os +import sys +import io +from contextlib import redirect_stdout + +from ctsm import unit_testing + +from ctsm.param_utils import query_paramfile as qp + +# Allow names that pylint doesn't like, because otherwise I find it hard +# to make readable unit test names +# pylint: disable=invalid-name + +PARAMFILE = os.path.join( + os.path.dirname(__file__), "testinputs", "ctsm5.3.041.Nfix_params.v13.c250221_upplim250.nc" +) + + +class TestSysQueryParamfile(unittest.TestCase): + """System tests of query_paramfile""" + + def setUp(self): + self.orig_argv = sys.argv + + def tearDown(self): + sys.argv = self.orig_argv + + def test_query_paramfile_scalar(self): + """Test that print_values works with scalar parameter""" + + sys.argv = ["get_arguments", "-i", PARAMFILE, "phenology_soil_depth"] + + f = io.StringIO() + with redirect_stdout(f): + qp.main() + out = f.getvalue() + self.assertEqual("phenology_soil_depth: 0.08\n", out) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_unit_query_paramfile.py b/python/ctsm/test/test_unit_query_paramfile.py index c85ce27d3b..d622f4d7a0 100755 --- a/python/ctsm/test/test_unit_query_paramfile.py +++ b/python/ctsm/test/test_unit_query_paramfile.py @@ -37,8 +37,8 @@ def _setup_pft_parameter_ds(): return ds, var_name, pft_names_list -class TestQueryParamfile(unittest.TestCase): - """Tests of query_paramfile""" +class TestUnitQueryParamfile(unittest.TestCase): + """Unit tests of query_paramfile""" def setUp(self): self.orig_argv = sys.argv From bf393253bd4733fb3c36239d50a1980ed500c3a9 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 11:53:10 -0600 Subject: [PATCH 028/196] query_paramfile: Fix call w/o -p/--pft. --- python/ctsm/param_utils/query_paramfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index 25aed36264..7a654f6f30 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -43,6 +43,7 @@ def main(): ds = xr.open_dataset(args.input, decode_timedelta=False) selected_pfts = None + pft_names = None if args.pft: selected_pfts = [p.strip() for p in args.pft.split(",")] pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] From 35bac743955f06185be933d9047109aa49e25fa5 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 12:04:03 -0600 Subject: [PATCH 029/196] query_paramfile: Add more system tests; failing. --- python/ctsm/test/test_sys_query_paramfile.py | 50 +++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/python/ctsm/test/test_sys_query_paramfile.py b/python/ctsm/test/test_sys_query_paramfile.py index 03c452dbf7..87d7ba657c 100755 --- a/python/ctsm/test/test_sys_query_paramfile.py +++ b/python/ctsm/test/test_sys_query_paramfile.py @@ -30,8 +30,8 @@ def setUp(self): def tearDown(self): sys.argv = self.orig_argv - def test_query_paramfile_scalar(self): - """Test that print_values works with scalar parameter""" + def test_query_paramfile_scalar_nopfts(self): + """Test that print_values works with scalar parameter and no PFTs specified""" sys.argv = ["get_arguments", "-i", PARAMFILE, "phenology_soil_depth"] @@ -41,6 +41,52 @@ def test_query_paramfile_scalar(self): out = f.getvalue() self.assertEqual("phenology_soil_depth: 0.08\n", out) + def test_query_paramfile_scalar_ignorepfts(self): + """Test that print_values works with scalar parameter and PFTs specified (ignored)""" + + sys.argv = ["get_arguments", "-i", PARAMFILE, "phenology_soil_depth", "--pft", "c3_crop"] + + f = io.StringIO() + with redirect_stdout(f): + qp.main() + out = f.getvalue() + self.assertEqual("phenology_soil_depth: 0.08\n", out) + + def test_query_paramfile_pft_selectnone(self): + """Test that print_values works with PFT-dim parameter and no PFTs specified""" + + sys.argv = ["get_arguments", "-i", PARAMFILE, "rswf_min"] + + f = io.StringIO() + with redirect_stdout(f): + qp.main() + out = f.getvalue() + self.assertRegex( + out, + r"rswf_min:\n\s+not_vegetated\s*: 0\.25\n\s+needleleaf_evergreen_temperate_tree\s*: 0\.25\n.*", + ) + + def test_query_paramfile_pft_select2(self): + """Test that print_values works with PFT-dim parameter and two PFTs specified""" + + sys.argv = [ + "get_arguments", + "-i", + PARAMFILE, + "--pft", + "not_vegetated,needleleaf_evergreen_temperate_tree", + "rswf_min", + ] + + f = io.StringIO() + with redirect_stdout(f): + qp.main() + out = f.getvalue() + self.assertRegex( + out, + r"rswf_min:\n\s+not_vegetated\s*: 0\.25\n\s+needleleaf_evergreen_temperate_tree\s*: 0\.25\n", + ) + if __name__ == "__main__": unit_testing.setup_for_tests() From b2f08bd5de80563099167b89118b652d7e7d09fa Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 12:04:30 -0600 Subject: [PATCH 030/196] query_paramfile: Always get pft_names if on paramfile. --- python/ctsm/param_utils/query_paramfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index 7a654f6f30..56d00f3b00 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -44,9 +44,10 @@ def main(): selected_pfts = None pft_names = None + if PFTNAME_VAR in ds: + pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] if args.pft: selected_pfts = [p.strip() for p in args.pft.split(",")] - pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] pfts_not_in_file = [] for pft in selected_pfts: if pft not in pft_names: From 6429f005e514f8e67338339fe172fd30957d143e Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 12:42:23 -0600 Subject: [PATCH 031/196] query_parameters: Add docstrings. --- python/ctsm/param_utils/query_paramfile.py | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index 56d00f3b00..8a61a1e79f 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -5,6 +5,17 @@ def get_arguments(): + """ + Parse command-line arguments for querying variables from a netCDF file. + + Returns + ------- + argparse.Namespace + Parsed arguments with attributes: + - input: Path to the netCDF file + - variables: Comma-separated list of variable names to extract + - pft: Optional comma-separated list of PFT names to print + """ parser = argparse.ArgumentParser( description="Print values of one or more variables from a netCDF file." ) @@ -20,6 +31,20 @@ def get_arguments(): def print_values(ds, var, selected_pfts, pft_names): + """ + Print the values of a variable from the dataset, optionally filtered by PFTs. + + Parameters + ---------- + ds : xarray.Dataset + The opened netCDF dataset. + var : str + Variable name to print. + selected_pfts : list or None + List of selected PFT names to print, or None to print all. + pft_names : list or None + List of all PFT names in the file. + """ data = ds[var].values if PFTNAME_VAR in ds[var].coords: print(var + ":") @@ -36,6 +61,10 @@ def print_values(ds, var, selected_pfts, pft_names): def main(): + """ + Main entry point for the script. + Parses arguments, opens the netCDF file, and prints requested variable values. + """ args = get_arguments() variable_names = [v.strip() for v in args.variables.split(",")] From 29ff74d26bde6f90326aad39c55f67f7efbeebbf Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 12:49:43 -0600 Subject: [PATCH 032/196] query_paramfile: Move some parser stuff to paramfile_shared. --- python/ctsm/param_utils/paramfile_shared.py | 15 +++++++++++++++ python/ctsm/param_utils/query_paramfile.py | 11 +++++------ 2 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 python/ctsm/param_utils/paramfile_shared.py diff --git a/python/ctsm/param_utils/paramfile_shared.py b/python/ctsm/param_utils/paramfile_shared.py new file mode 100644 index 0000000000..93635826f3 --- /dev/null +++ b/python/ctsm/param_utils/paramfile_shared.py @@ -0,0 +1,15 @@ +""" +Functions etc. shared among parameter file utilities +""" + +import argparse + + +def paramfile_parser_setup(description): + parser = argparse.ArgumentParser(description=description) + parser.add_argument("-i", "--input", required=True, help="Input netCDF file") + + # Flags that can be used for the PFT argument + pft_flags = ["-p", "--pft"] + + return parser, pft_flags diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index 8a61a1e79f..77adf38fc0 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -1,6 +1,7 @@ -import argparse import xarray as xr +from ctsm.param_utils.paramfile_shared import paramfile_parser_setup + PFTNAME_VAR = "pftname" @@ -16,14 +17,12 @@ def get_arguments(): - variables: Comma-separated list of variable names to extract - pft: Optional comma-separated list of PFT names to print """ - parser = argparse.ArgumentParser( - description="Print values of one or more variables from a netCDF file." + parser, pft_flags = paramfile_parser_setup( + "Print values of one or more variables from a netCDF file." ) - parser.add_argument("-i", "--input", required=True, help="Input netCDF file") parser.add_argument("variables", help="Comma-separated list of variable names to extract") parser.add_argument( - "-p", - "--pft", + *pft_flags, help="Comma-separated list of PFT names to print (only applies to PFT-specific variables)", ) args = parser.parse_args() From 2a178bc5df3f012a6474d268b9ec02e8ed9de89e Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 13:03:26 -0600 Subject: [PATCH 033/196] query_paramfile: Split comma-separated lists in parse_args(). --- python/ctsm/args_utils.py | 9 ++++ python/ctsm/param_utils/query_paramfile.py | 19 ++++---- python/ctsm/test/test_unit_args_utils.py | 43 +++++++++++++++++++ python/ctsm/test/test_unit_query_paramfile.py | 8 ++-- 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/python/ctsm/args_utils.py b/python/ctsm/args_utils.py index 612e3f09a1..ed0e934d32 100644 --- a/python/ctsm/args_utils.py +++ b/python/ctsm/args_utils.py @@ -46,3 +46,12 @@ def plon_type(plon): "ERROR: Longitude should be between 0 and 360 or -180 and 180." ) return plon_float + + +def comma_separated_list(value): + """ + Helper function for argparse to split comma-separated strings into a list. + """ + if value is None: + return None + return [v.strip() for v in value.split(",")] diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index 77adf38fc0..c69628f314 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -1,5 +1,6 @@ import xarray as xr +from ctsm.args_utils import comma_separated_list from ctsm.param_utils.paramfile_shared import paramfile_parser_setup PFTNAME_VAR = "pftname" @@ -15,15 +16,20 @@ def get_arguments(): Parsed arguments with attributes: - input: Path to the netCDF file - variables: Comma-separated list of variable names to extract - - pft: Optional comma-separated list of PFT names to print + - pft: Optional list of PFT names to print """ parser, pft_flags = paramfile_parser_setup( "Print values of one or more variables from a netCDF file." ) - parser.add_argument("variables", help="Comma-separated list of variable names to extract") + parser.add_argument( + "variables", + help="Comma-separated list of variable names to extract", + type=comma_separated_list, + ) parser.add_argument( *pft_flags, help="Comma-separated list of PFT names to print (only applies to PFT-specific variables)", + type=comma_separated_list, ) args = parser.parse_args() return args @@ -66,16 +72,13 @@ def main(): """ args = get_arguments() - variable_names = [v.strip() for v in args.variables.split(",")] - ds = xr.open_dataset(args.input, decode_timedelta=False) - selected_pfts = None + selected_pfts = args.pft pft_names = None if PFTNAME_VAR in ds: pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] - if args.pft: - selected_pfts = [p.strip() for p in args.pft.split(",")] + if selected_pfts: pfts_not_in_file = [] for pft in selected_pfts: if pft not in pft_names: @@ -83,7 +86,7 @@ def main(): if pfts_not_in_file: raise KeyError(f"PFT(s) not found in parameter file: {', '.join(pfts_not_in_file)}") - for var in variable_names: + for var in args.variables: if var in ds.variables: print_values(ds, var, selected_pfts, pft_names) else: diff --git a/python/ctsm/test/test_unit_args_utils.py b/python/ctsm/test/test_unit_args_utils.py index 548e7d9389..b4b101dda6 100755 --- a/python/ctsm/test/test_unit_args_utils.py +++ b/python/ctsm/test/test_unit_args_utils.py @@ -17,6 +17,7 @@ # pylint: disable=wrong-import-position from ctsm.args_utils import plon_type, plat_type +from ctsm.args_utils import comma_separated_list from ctsm import unit_testing # pylint: disable=invalid-name @@ -118,6 +119,48 @@ def test_platType_outOfBounds_negative(self): _ = plat_type(-91) +class TestArgsCommaSeparatedList(unittest.TestCase): + """ + Test comma_separated_list argparse helper + """ + + def setUp(self): + self.orig_argv = sys.argv + + def tearDown(self): + sys.argv = self.orig_argv + + def test_comma_separated_list_1(self): + """ + Test comma_separated_list with one item in list + """ + parser = argparse.ArgumentParser() + parser.add_argument("--list-arg", type=comma_separated_list) + sys.argv = ["scriptname", "--list-arg", "abc"] + args = parser.parse_args() + self.assertEqual(["abc"], args.list_arg) + + def test_comma_separated_list_1plus(self): + """ + Test comma_separated_list with one item in list but a comma too + """ + parser = argparse.ArgumentParser() + parser.add_argument("--list-arg", type=comma_separated_list) + sys.argv = ["scriptname", "--list-arg", "abc,"] + args = parser.parse_args() + self.assertEqual(["abc", ""], args.list_arg) + + def test_comma_separated_list_2(self): + """ + Test comma_separated_list with two items in list + """ + parser = argparse.ArgumentParser() + parser.add_argument("--list-arg", type=comma_separated_list) + sys.argv = ["scriptname", "--list-arg", "abc,def"] + args = parser.parse_args() + self.assertEqual(["abc", "def"], args.list_arg) + + if __name__ == "__main__": unit_testing.setup_for_tests() unittest.main() diff --git a/python/ctsm/test/test_unit_query_paramfile.py b/python/ctsm/test/test_unit_query_paramfile.py index d622f4d7a0..00e8c35a62 100755 --- a/python/ctsm/test/test_unit_query_paramfile.py +++ b/python/ctsm/test/test_unit_query_paramfile.py @@ -52,8 +52,8 @@ def test_query_paramfile_args_short(self): sys.argv = ["get_arguments", "-i", input_path, "-p", "pft1,pft2", "var1,var2"] args = qp.get_arguments() self.assertEqual(input_path, args.input) - self.assertEqual("pft1,pft2", args.pft) - self.assertEqual("var1,var2", args.variables) + self.assertEqual(["pft1", "pft2"], args.pft) + self.assertEqual(["var1", "var2"], args.variables) def test_query_paramfile_args_long(self): """Test that all arguments can be set correctly with longnames""" @@ -61,8 +61,8 @@ def test_query_paramfile_args_long(self): sys.argv = ["get_arguments", "--input", input_path, "--pft", "pft1,pft2", "var1,var2"] args = qp.get_arguments() self.assertEqual(input_path, args.input) - self.assertEqual("pft1,pft2", args.pft) - self.assertEqual("var1,var2", args.variables) + self.assertEqual(["pft1", "pft2"], args.pft) + self.assertEqual(["var1", "var2"], args.variables) def test_query_paramfile_print_scalar(self): """Test that print_values works with a scalar parameter""" From 958aaf26cf2c18b592b7a8a1e825292985de7897 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 13:21:35 -0600 Subject: [PATCH 034/196] Add set_paramfile tool and tests. So far just parses some args. --- python/ctsm/param_utils/set_paramfile.py | 61 ++++++++++++++ python/ctsm/test/test_unit_set_paramfile.py | 90 +++++++++++++++++++++ tools/param_utils/set_paramfile | 19 +++++ 3 files changed, 170 insertions(+) create mode 100644 python/ctsm/param_utils/set_paramfile.py create mode 100755 python/ctsm/test/test_unit_set_paramfile.py create mode 100755 tools/param_utils/set_paramfile diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py new file mode 100644 index 0000000000..e5d19030e3 --- /dev/null +++ b/python/ctsm/param_utils/set_paramfile.py @@ -0,0 +1,61 @@ +""" +Tool for changing parameters on CTSM paramfile +""" +import os + +from ctsm.args_utils import comma_separated_list +from ctsm.param_utils.paramfile_shared import paramfile_parser_setup + +PFTNAME_VAR = "pftname" + + +def check_arguments(args): + """ + Check arguments to set_paramfile + """ + if not os.path.exists(args.input): + raise FileNotFoundError(args.input) + + # Avoid potentially overwriting canonical files + if os.path.exists(args.output): + raise FileExistsError(args.output) + + +def get_arguments(): + """ + Parse command-line arguments for setting variables on a netCDF file. + + Returns + ------- + argparse.Namespace + Parsed arguments with attributes: + - input: Path to the input netCDF file + - output: Path to the output netCDF file + - pft: Optional list of PFT names whose values you want to change + """ + parser, pft_flags = paramfile_parser_setup( + "Print values of one or more variables from a netCDF file." + ) + parser.add_argument("-o", "--output", required=True, help="Output netCDF file") + parser.add_argument( + *pft_flags, + help="Comma-separated list of PFTs whose values you want to change (only applies to PFT-specific variables)", + type=comma_separated_list, + ) + args = parser.parse_args() + + check_arguments(args) + + return args + + +def main(): + """ + Main entry point for the script. + Parses arguments, opens a netCDF file, makes changes, and saves a new netCDF file. + """ + args = get_arguments() + + +if __name__ == "__main__": + main() diff --git a/python/ctsm/test/test_unit_set_paramfile.py b/python/ctsm/test/test_unit_set_paramfile.py new file mode 100755 index 0000000000..c3240bd466 --- /dev/null +++ b/python/ctsm/test/test_unit_set_paramfile.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +"""Unit tests for set_paramfile""" + +import unittest +import os +import sys + +from ctsm import unit_testing + +from ctsm.param_utils import set_paramfile as sp + +# Allow names that pylint doesn't like, because otherwise I find it hard +# to make readable unit test names +# pylint: disable=invalid-name + + +PARAMFILE = os.path.join( + os.path.dirname(__file__), "testinputs", "ctsm5.3.041.Nfix_params.v13.c250221_upplim250.nc" +) + + +class TestUnitSetParamfile(unittest.TestCase): + """Unit tests of set_paramfile""" + + def setUp(self): + self.orig_argv = sys.argv + + def tearDown(self): + sys.argv = self.orig_argv + + def test_set_paramfile_args_short(self): + """Test that all arguments can be set correctly with shortnames""" + output_path = "/path/to/output.nc" + sys.argv = ["get_arguments", "-i", PARAMFILE, "-p", "pft1,pft2", "-o", output_path] + args = sp.get_arguments() + self.assertEqual(PARAMFILE, args.input) + self.assertEqual(["pft1", "pft2"], args.pft) + self.assertEqual(output_path, args.output) + + def test_set_paramfile_args_long(self): + """Test that all arguments can be set correctly with longnames""" + output_path = "/path/to/output.nc" + sys.argv = [ + "get_arguments", + "--input", + PARAMFILE, + "--pft", + "pft1,pft2", + "--output", + output_path, + ] + args = sp.get_arguments() + self.assertEqual(PARAMFILE, args.input) + self.assertEqual(["pft1", "pft2"], args.pft) + self.assertEqual(output_path, args.output) + + def test_set_paramfile_error_missing_input(self): + """Test that it errors if input file doesn't exist""" + output_path = "/path/to/output.nc" + sys.argv = [ + "get_arguments", + "--input", + "nwuefweirbdfdiurbe", + "--pft", + "pft1,pft2", + "--output", + output_path, + ] + with self.assertRaises(FileNotFoundError): + sp.get_arguments() + + def test_set_paramfile_error_existing_output(self): + """Test that it errors if output file already exists""" + sys.argv = [ + "get_arguments", + "--input", + PARAMFILE, + "--pft", + "pft1,pft2", + "--output", + PARAMFILE, + ] + with self.assertRaises(FileExistsError): + sp.get_arguments() + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/tools/param_utils/set_paramfile b/tools/param_utils/set_paramfile new file mode 100755 index 0000000000..cebdb81d30 --- /dev/null +++ b/tools/param_utils/set_paramfile @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +""" +For description and instructions, please see README. +""" + +import os +import sys + +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), + os.pardir, + os.pardir, + 'python') +sys.path.insert(1, _CTSM_PYTHON) + +from ctsm.param_utils.set_paramfile import main + +if __name__ == "__main__": + main() + From 337a072ec9cca5dac7243123cd7cf9b4b6abbd6d Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 15:12:27 -0600 Subject: [PATCH 035/196] set_paramfile: Can now copy a file. --- python/ctsm/netcdf_utils.py | 14 +++++ python/ctsm/param_utils/paramfile_shared.py | 4 ++ python/ctsm/param_utils/set_paramfile.py | 13 ++++- python/ctsm/test/test_sys_set_paramfile.py | 58 +++++++++++++++++++++ python/ctsm/test/test_unit_netcdf_utils.py | 57 ++++++++++++++++++++ 5 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 python/ctsm/netcdf_utils.py create mode 100755 python/ctsm/test/test_sys_set_paramfile.py create mode 100755 python/ctsm/test/test_unit_netcdf_utils.py diff --git a/python/ctsm/netcdf_utils.py b/python/ctsm/netcdf_utils.py new file mode 100644 index 0000000000..64f554dc69 --- /dev/null +++ b/python/ctsm/netcdf_utils.py @@ -0,0 +1,14 @@ +""" +Helper functions for working with netCDF files +""" + +from netCDF4 import Dataset # pylint: disable=no-name-in-module + + +def get_netcdf_format(file_path): + """ + Get format of netCDF file + """ + with Dataset(file_path, "r") as netcdf_file: + netcdf_format = netcdf_file.data_model + return netcdf_format diff --git a/python/ctsm/param_utils/paramfile_shared.py b/python/ctsm/param_utils/paramfile_shared.py index 93635826f3..995df88c57 100644 --- a/python/ctsm/param_utils/paramfile_shared.py +++ b/python/ctsm/param_utils/paramfile_shared.py @@ -3,8 +3,12 @@ """ import argparse +import xarray as xr +def open_paramfile(file_in): + return xr.open_dataset(file_in, decode_timedelta=False) + def paramfile_parser_setup(description): parser = argparse.ArgumentParser(description=description) parser.add_argument("-i", "--input", required=True, help="Input netCDF file") diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index e5d19030e3..f81313c503 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -1,10 +1,12 @@ """ Tool for changing parameters on CTSM paramfile """ + import os from ctsm.args_utils import comma_separated_list -from ctsm.param_utils.paramfile_shared import paramfile_parser_setup +from ctsm.netcdf_utils import get_netcdf_format +from ctsm.param_utils.paramfile_shared import paramfile_parser_setup, open_paramfile PFTNAME_VAR = "pftname" @@ -36,14 +38,16 @@ def get_arguments(): parser, pft_flags = paramfile_parser_setup( "Print values of one or more variables from a netCDF file." ) + parser.add_argument("-o", "--output", required=True, help="Output netCDF file") + parser.add_argument( *pft_flags, help="Comma-separated list of PFTs whose values you want to change (only applies to PFT-specific variables)", type=comma_separated_list, ) - args = parser.parse_args() + args = parser.parse_args() check_arguments(args) return args @@ -56,6 +60,11 @@ def main(): """ args = get_arguments() + ds_in = open_paramfile(args.input) + ds_out = ds_in.copy() + + ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input)) + if __name__ == "__main__": main() diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py new file mode 100755 index 0000000000..b9077a1900 --- /dev/null +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +"""System tests for set_paramfile""" + +import unittest +import os +import sys +import shutil +import tempfile +import xarray as xr +from netCDF4 import Dataset # pylint: disable=no-name-in-module + +from ctsm import unit_testing + +from ctsm.netcdf_utils import get_netcdf_format +from ctsm.param_utils import set_paramfile as sp +from ctsm.param_utils.paramfile_shared import open_paramfile + +# Allow names that pylint doesn't like, because otherwise I find it hard +# to make readable unit test names +# pylint: disable=invalid-name + + +PARAMFILE = os.path.join( + os.path.dirname(__file__), "testinputs", "ctsm5.3.041.Nfix_params.v13.c250221_upplim250.nc" +) + + +class TestSysSetParamfile(unittest.TestCase): + """System tests of set_paramfile""" + + def setUp(self): + self.orig_argv = sys.argv + self.tempdir = tempfile.mkdtemp() + + def tearDown(self): + sys.argv = self.orig_argv + shutil.rmtree(self.tempdir, ignore_errors=True) + + def test_set_paramfile_copyfile(self): + """Test that set_paramfile can straight-up copy to a new file""" + output_path = os.path.join(self.tempdir, "output.nc") + sys.argv = ["set_paramfile", "-i", PARAMFILE, "-o", output_path] + sp.main() + self.assertTrue(os.path.exists(output_path)) + ds_in = open_paramfile(PARAMFILE) + ds_out = open_paramfile(output_path) + + # Check that contents are functionally identical + self.assertEqual(ds_in, ds_out) + + # Check that both are the same kind of netCDF + self.assertEqual(get_netcdf_format(PARAMFILE), get_netcdf_format(output_path)) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() diff --git a/python/ctsm/test/test_unit_netcdf_utils.py b/python/ctsm/test/test_unit_netcdf_utils.py new file mode 100755 index 0000000000..4c904564fb --- /dev/null +++ b/python/ctsm/test/test_unit_netcdf_utils.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +""" +Unit tests for netcdf_utils.py functions +""" + +import os +import sys +import unittest +import shutil +import tempfile +import xarray as xr + +# -- add python/ctsm to path (needed if we want to run the test stand-alone) +_CTSM_PYTHON = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position +from ctsm.netcdf_utils import get_netcdf_format +from ctsm import unit_testing + +# pylint: disable=invalid-name + + +class TestUnitGetNetcdfFormat(unittest.TestCase): + """ + Unit tests for get_netcdf_format + """ + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.outfile = os.path.join(self.tempdir, "file.nc") + da = xr.DataArray(data=[1, 2, 3]) + self.ds = xr.Dataset(data_vars={"var": da}) + + def tearDown(self): + shutil.rmtree(self.tempdir, ignore_errors=True) + + def test_get_netcdf_format_classic(self): + """ + Test that get_netcdf_format() gets "classic" format right + """ + nc_format = "NETCDF3_CLASSIC" + self.ds.to_netcdf(self.outfile, format=nc_format) + self.assertEqual(get_netcdf_format(self.outfile), nc_format) + + def test_get_netcdf_format_netcdf4(self): + """ + Test that get_netcdf_format() gets "netCDF4" format right + """ + nc_format = "NETCDF4" + self.ds.to_netcdf(self.outfile, format=nc_format) + self.assertEqual(get_netcdf_format(self.outfile), nc_format) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() From cea3f1250958788fcbd28b4fb93531a3a426c618 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 15:27:58 -0600 Subject: [PATCH 036/196] set_paramfile: Use -v/--variables to only include some vars. --- python/ctsm/param_utils/set_paramfile.py | 16 +++++++++++ python/ctsm/test/test_sys_set_paramfile.py | 30 +++++++++++++++++++++ python/ctsm/test/test_unit_set_paramfile.py | 16 ++++++++++- 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index f81313c503..35a9e3418e 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -47,6 +47,14 @@ def get_arguments(): type=comma_separated_list, ) + # TODO: Add -x/--exclude argument for variables you DON'T want to extract + parser.add_argument( + "-v", + "--variables", + help="Comma-separated list of variables to extract", + type=comma_separated_list, + ) + args = parser.parse_args() check_arguments(args) @@ -63,6 +71,14 @@ def main(): ds_in = open_paramfile(args.input) ds_out = ds_in.copy() + # If any variables specified, drop others + if args.variables: + vars_to_drop = [] + for var in ds_in: + if var not in args.variables: + vars_to_drop.append(var) + ds_out = ds_out.drop_vars(vars_to_drop) + ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input)) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index b9077a1900..eacb744b1c 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -52,6 +52,36 @@ def test_set_paramfile_copyfile(self): # Check that both are the same kind of netCDF self.assertEqual(get_netcdf_format(PARAMFILE), get_netcdf_format(output_path)) + def test_set_paramfile_extractvars(self): + """Test that set_paramfile can copy to a new file with only some requested variables""" + output_path = os.path.join(self.tempdir, "output.nc") + vars_to_include = ["a_coef", "bgc_cn_s2"] + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "-v", + ",".join(vars_to_include), + ] + sp.main() + self.assertTrue(os.path.exists(output_path)) + ds_in = open_paramfile(PARAMFILE) + ds_out = open_paramfile(output_path) + + # Check that output file variables/coords are what we expect + # (use set() so order doesn't matter) + expected_var_list = set(vars_to_include + list(ds_in.coords)) + self.assertEqual(set(ds_out.variables), expected_var_list) + + # Check that included variables/coords match + for var in vars_to_include: + self.assertEqual(ds_in[var], ds_out[var]) + + # Check that both are the same kind of netCDF + self.assertEqual(get_netcdf_format(PARAMFILE), get_netcdf_format(output_path)) + if __name__ == "__main__": unit_testing.setup_for_tests() diff --git a/python/ctsm/test/test_unit_set_paramfile.py b/python/ctsm/test/test_unit_set_paramfile.py index c3240bd466..6fa0b47002 100755 --- a/python/ctsm/test/test_unit_set_paramfile.py +++ b/python/ctsm/test/test_unit_set_paramfile.py @@ -32,10 +32,21 @@ def tearDown(self): def test_set_paramfile_args_short(self): """Test that all arguments can be set correctly with shortnames""" output_path = "/path/to/output.nc" - sys.argv = ["get_arguments", "-i", PARAMFILE, "-p", "pft1,pft2", "-o", output_path] + sys.argv = [ + "get_arguments", + "-i", + PARAMFILE, + "-p", + "pft1,pft2", + "-o", + output_path, + "-v", + "var1,var2", + ] args = sp.get_arguments() self.assertEqual(PARAMFILE, args.input) self.assertEqual(["pft1", "pft2"], args.pft) + self.assertEqual(["var1", "var2"], args.variables) self.assertEqual(output_path, args.output) def test_set_paramfile_args_long(self): @@ -49,10 +60,13 @@ def test_set_paramfile_args_long(self): "pft1,pft2", "--output", output_path, + "--variables", + "var1,var2", ] args = sp.get_arguments() self.assertEqual(PARAMFILE, args.input) self.assertEqual(["pft1", "pft2"], args.pft) + self.assertEqual(["var1", "var2"], args.variables) self.assertEqual(output_path, args.output) def test_set_paramfile_error_missing_input(self): From 93f29e2f6d8f093643b8ce805dc825dca263bc33 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 15:45:12 -0600 Subject: [PATCH 037/196] set_paramfile: Can now change params. Tested w/ scalars. --- python/ctsm/param_utils/set_paramfile.py | 11 ++++++++ python/ctsm/test/test_sys_set_paramfile.py | 29 +++++++++++++++++++++ python/ctsm/test/test_unit_set_paramfile.py | 6 +++++ 3 files changed, 46 insertions(+) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 35a9e3418e..44dc32c7be 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -55,6 +55,12 @@ def get_arguments(): type=comma_separated_list, ) + parser.add_argument( + "param_changes", + help="Parameter changes to apply. E.g.: param1=new_value1 param2=new_value2", + nargs="*", + ) + args = parser.parse_args() check_arguments(args) @@ -79,6 +85,11 @@ def main(): vars_to_drop.append(var) ds_out = ds_out.drop_vars(vars_to_drop) + # Apply parameter changes, if any + for chg in args.param_changes: + var, new_value = chg.split("=") + ds_out[var].values = new_value + ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input)) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index eacb744b1c..444f92bd1b 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -82,6 +82,35 @@ def test_set_paramfile_extractvars(self): # Check that both are the same kind of netCDF self.assertEqual(get_netcdf_format(PARAMFILE), get_netcdf_format(output_path)) + def test_set_paramfile_changeparams_scalar(self): + """Test that set_paramfile can copy to a new file with some scalar params changed""" + output_path = os.path.join(self.tempdir, "output.nc") + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "a_coef=0.87", + "bgc_cn_s2=87", + ] + sp.main() + self.assertTrue(os.path.exists(output_path)) + ds_in = open_paramfile(PARAMFILE) + ds_out = open_paramfile(output_path) + + # Check that all variables/coords are equal except the ones we changed, which should be set + # to what we asked + for var in ds_in.variables: + if var == "a_coef": + self.assertTrue(ds_in[var].values == 0.13) + self.assertTrue(ds_out[var].values == 0.87) + elif var == "bgc_cn_s2": + self.assertTrue(ds_in[var].values == 11) + self.assertTrue(ds_out[var].values == 87) + else: + self.assertTrue(ds_in[var].equals(ds_out[var])) + if __name__ == "__main__": unit_testing.setup_for_tests() diff --git a/python/ctsm/test/test_unit_set_paramfile.py b/python/ctsm/test/test_unit_set_paramfile.py index 6fa0b47002..9bccb41450 100755 --- a/python/ctsm/test/test_unit_set_paramfile.py +++ b/python/ctsm/test/test_unit_set_paramfile.py @@ -42,12 +42,15 @@ def test_set_paramfile_args_short(self): output_path, "-v", "var1,var2", + "param1=new_value1", + "param2=new_value2", ] args = sp.get_arguments() self.assertEqual(PARAMFILE, args.input) self.assertEqual(["pft1", "pft2"], args.pft) self.assertEqual(["var1", "var2"], args.variables) self.assertEqual(output_path, args.output) + self.assertEqual(["param1=new_value1", "param2=new_value2"], args.param_changes) def test_set_paramfile_args_long(self): """Test that all arguments can be set correctly with longnames""" @@ -62,12 +65,15 @@ def test_set_paramfile_args_long(self): output_path, "--variables", "var1,var2", + "param1=new_value1", + "param2=new_value2", ] args = sp.get_arguments() self.assertEqual(PARAMFILE, args.input) self.assertEqual(["pft1", "pft2"], args.pft) self.assertEqual(["var1", "var2"], args.variables) self.assertEqual(output_path, args.output) + self.assertEqual(["param1=new_value1", "param2=new_value2"], args.param_changes) def test_set_paramfile_error_missing_input(self): """Test that it errors if input file doesn't exist""" From 22bdf2dd27da492daf603486b77e21523cc8f544 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 16:07:34 -0600 Subject: [PATCH 038/196] set_paramfile: Include just some PFTs with -p/--pft. --- python/ctsm/param_utils/paramfile_shared.py | 25 +++++++++++++++++++ python/ctsm/param_utils/query_paramfile.py | 15 +++-------- python/ctsm/param_utils/set_paramfile.py | 13 +++++++--- python/ctsm/test/test_sys_set_paramfile.py | 25 +++++++++++++++++++ python/ctsm/test/test_unit_query_paramfile.py | 5 ++-- 5 files changed, 67 insertions(+), 16 deletions(-) diff --git a/python/ctsm/param_utils/paramfile_shared.py b/python/ctsm/param_utils/paramfile_shared.py index 995df88c57..628745ee9f 100644 --- a/python/ctsm/param_utils/paramfile_shared.py +++ b/python/ctsm/param_utils/paramfile_shared.py @@ -5,10 +5,35 @@ import argparse import xarray as xr +PFTNAME_VAR = "pftname" + + +def check_pfts_in_paramfile(selected_pfts, ds): + """ + Check that given PFTs are in parameter file + """ + if PFTNAME_VAR not in ds: + raise KeyError(f"paramfile missing variable: {PFTNAME_VAR}") + pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] + pfts_not_in_file = [] + for pft in selected_pfts: + if pft not in pft_names: + pfts_not_in_file += [pft] + if pfts_not_in_file: + raise KeyError(f"PFT(s) not found in parameter file: {', '.join(pfts_not_in_file)}") + + return pft_names + + +def get_selected_pft_indices(selected_pfts, pft_names): + indices = [i for i, name in enumerate(pft_names) if name in selected_pfts] + return indices + def open_paramfile(file_in): return xr.open_dataset(file_in, decode_timedelta=False) + def paramfile_parser_setup(description): parser = argparse.ArgumentParser(description=description) parser.add_argument("-i", "--input", required=True, help="Input netCDF file") diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index c69628f314..aa94293d73 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -2,8 +2,8 @@ from ctsm.args_utils import comma_separated_list from ctsm.param_utils.paramfile_shared import paramfile_parser_setup - -PFTNAME_VAR = "pftname" +from ctsm.param_utils.paramfile_shared import PFTNAME_VAR, check_pfts_in_paramfile +from ctsm.param_utils.paramfile_shared import get_selected_pft_indices def get_arguments(): @@ -55,7 +55,7 @@ def print_values(ds, var, selected_pfts, pft_names): print(var + ":") indices = range(len(pft_names)) if selected_pfts is not None: - indices = [i for i, name in enumerate(pft_names) if name in selected_pfts] + indices = get_selected_pft_indices(selected_pfts, pft_names) max_name_len = max(len(pft_names[i]) for i in indices) if indices else 0 else: max_name_len = max(len(name) for name in pft_names) @@ -76,15 +76,8 @@ def main(): selected_pfts = args.pft pft_names = None - if PFTNAME_VAR in ds: - pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] if selected_pfts: - pfts_not_in_file = [] - for pft in selected_pfts: - if pft not in pft_names: - pfts_not_in_file += [pft] - if pfts_not_in_file: - raise KeyError(f"PFT(s) not found in parameter file: {', '.join(pfts_not_in_file)}") + pft_names = check_pfts_in_paramfile(selected_pfts, ds) for var in args.variables: if var in ds.variables: diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 44dc32c7be..666cd6bb71 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -7,8 +7,8 @@ from ctsm.args_utils import comma_separated_list from ctsm.netcdf_utils import get_netcdf_format from ctsm.param_utils.paramfile_shared import paramfile_parser_setup, open_paramfile - -PFTNAME_VAR = "pftname" +from ctsm.param_utils.paramfile_shared import check_pfts_in_paramfile, get_selected_pft_indices +from ctsm.param_utils.paramfile_shared import PFTNAME_VAR def check_arguments(args): @@ -41,9 +41,10 @@ def get_arguments(): parser.add_argument("-o", "--output", required=True, help="Output netCDF file") + # TODO: Add --exclude-pfts argument for PFTs you DON'T want to include parser.add_argument( *pft_flags, - help="Comma-separated list of PFTs whose values you want to change (only applies to PFT-specific variables)", + help="Comma-separated list of PFTs to include (only applies to PFT-specific variables)", type=comma_separated_list, ) @@ -77,6 +78,12 @@ def main(): ds_in = open_paramfile(args.input) ds_out = ds_in.copy() + # If any PFTs were specified, drop others + if args.pft: + pft_names = check_pfts_in_paramfile(args.pft, ds_out) + indices = get_selected_pft_indices(args.pft, pft_names) + ds_out = ds_out.isel({"pft": indices}) + # If any variables specified, drop others if args.variables: vars_to_drop = [] diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 444f92bd1b..c81da80480 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -82,6 +82,31 @@ def test_set_paramfile_extractvars(self): # Check that both are the same kind of netCDF self.assertEqual(get_netcdf_format(PARAMFILE), get_netcdf_format(output_path)) + def test_set_paramfile_extractpfts(self): + """Test that set_paramfile can copy to a new file with only some requested PFTs""" + output_path = os.path.join(self.tempdir, "output.nc") + pfts_to_include = ["not_vegetated", "needleleaf_evergreen_temperate_tree"] + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "-p", + ",".join(pfts_to_include), + ] + sp.main() + self.assertTrue(os.path.exists(output_path)) + ds_in = open_paramfile(PARAMFILE) + ds_out = open_paramfile(output_path) + + # Check that included variables/coords match + for var in ds_in.variables: + if sp.PFTNAME_VAR in ds_in[var].coords: + self.assertTrue(ds_in[var].isel(pft=[0,1]).equals(ds_out[var])) + else: + self.assertTrue(ds_in[var].equals(ds_out[var])) + def test_set_paramfile_changeparams_scalar(self): """Test that set_paramfile can copy to a new file with some scalar params changed""" output_path = os.path.join(self.tempdir, "output.nc") diff --git a/python/ctsm/test/test_unit_query_paramfile.py b/python/ctsm/test/test_unit_query_paramfile.py index 00e8c35a62..651dc16f70 100755 --- a/python/ctsm/test/test_unit_query_paramfile.py +++ b/python/ctsm/test/test_unit_query_paramfile.py @@ -11,6 +11,7 @@ from ctsm import unit_testing from ctsm.param_utils import query_paramfile as qp +from ctsm.param_utils.paramfile_shared import PFTNAME_VAR # Allow names that pylint doesn't like, because otherwise I find it hard # to make readable unit test names @@ -24,14 +25,14 @@ def _setup_pft_parameter_ds(): pft_dimname = "pft" pft_names_list = ["pft0", "pft1"] pft_names_da = xr.DataArray( - name=qp.PFTNAME_VAR, + name=PFTNAME_VAR, data=pft_names_list, dims=[pft_dimname], coords={pft_dimname: pft_names_list}, ) var_name = "pft_param" pft_param_da = xr.DataArray( - data=[1986.0325, 1987.0724], dims=[pft_dimname], coords={qp.PFTNAME_VAR: pft_names_da} + data=[1986.0325, 1987.0724], dims=[pft_dimname], coords={PFTNAME_VAR: pft_names_da} ) ds = xr.Dataset(data_vars={var_name: pft_param_da, pft_dimname: pft_names_da}) return ds, var_name, pft_names_list From 3aeb0407c15828b3e0df23987662b7c4478d6436 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 16:29:23 -0600 Subject: [PATCH 039/196] set_paramfile: Can now change 1-d parameters. --- python/ctsm/param_utils/set_paramfile.py | 9 +++- python/ctsm/test/test_sys_set_paramfile.py | 55 ++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 666cd6bb71..3b82180bfb 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -3,6 +3,7 @@ """ import os +import numpy as np from ctsm.args_utils import comma_separated_list from ctsm.netcdf_utils import get_netcdf_format @@ -58,7 +59,7 @@ def get_arguments(): parser.add_argument( "param_changes", - help="Parameter changes to apply. E.g.: param1=new_value1 param2=new_value2", + help="Parameter changes to apply. E.g.: param1=new_value1 pftparam=pft1_val,pft2_val,...", nargs="*", ) @@ -95,6 +96,12 @@ def main(): # Apply parameter changes, if any for chg in args.param_changes: var, new_value = chg.split("=") + if ds_out[var].ndim == 1: + new_value_list = new_value.split(",") + new_value = np.array(new_value_list, dtype=type(ds_out[var].dtype)) + elif ds_out[var].ndim > 2: + # TODO: Add handling of multi-dimensional parameters + raise NotImplementedError("Can't yet change multi-dimensional parameters") ds_out[var].values = new_value ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input)) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index c81da80480..0c39c690d5 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -7,6 +7,7 @@ import sys import shutil import tempfile +import numpy as np import xarray as xr from netCDF4 import Dataset # pylint: disable=no-name-in-module @@ -136,6 +137,60 @@ def test_set_paramfile_changeparams_scalar(self): else: self.assertTrue(ds_in[var].equals(ds_out[var])) + def test_set_paramfile_extractpfts_changeparam(self): + """ + Test that set_paramfile can (1) copy to a new file with only some requested PFTs and (2) + change the values of parameters of those PFTs + """ + output_path = os.path.join(self.tempdir, "output.nc") + pfts_to_include = ["not_vegetated", "needleleaf_evergreen_temperate_tree"] + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "-p", + ",".join(pfts_to_include), + "xl=0.724,0.87", + ] + sp.main() + self.assertTrue(os.path.exists(output_path)) + ds_in = open_paramfile(PARAMFILE) + ds_out = open_paramfile(output_path) + + # Check that included variables/coords match as expected + for var in ds_in.variables: + if var == "xl": + self.assertTrue(np.array_equal(np.array([0.724, 0.87]), ds_out[var].values)) + elif sp.PFTNAME_VAR in ds_in[var].coords: + self.assertTrue(ds_in[var].isel(pft=[0, 1]).equals(ds_out[var])) + else: + self.assertTrue(ds_in[var].equals(ds_out[var])) + + # TODO: Add test with NaN in existing + + # TODO: Add test changing to NaN + + def test_set_paramfile_changeparam_multidim_errors(self): + """ + Test that set_paramfile errors if requesting change of a multi-dimensional parameter. This + test will obviously need to be replaced once that functionality is added. + """ + output_path = os.path.join(self.tempdir, "output.nc") + pfts_to_include = ["not_vegetated", "needleleaf_evergreen_temperate_tree"] + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "mimics_till_decompk_multipliers=dummy", + ] + + with self.assertRaises(NotImplementedError): + sp.main() + if __name__ == "__main__": unit_testing.setup_for_tests() From a148838492d5ce2f32a5cd9eb963a146fd5072c4 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 16:38:14 -0600 Subject: [PATCH 040/196] query_paramfile: Fix call for PFT param w/o -p/--pft. --- python/ctsm/param_utils/paramfile_shared.py | 6 +++++- python/ctsm/param_utils/query_paramfile.py | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/python/ctsm/param_utils/paramfile_shared.py b/python/ctsm/param_utils/paramfile_shared.py index 628745ee9f..2079957b4b 100644 --- a/python/ctsm/param_utils/paramfile_shared.py +++ b/python/ctsm/param_utils/paramfile_shared.py @@ -14,7 +14,7 @@ def check_pfts_in_paramfile(selected_pfts, ds): """ if PFTNAME_VAR not in ds: raise KeyError(f"paramfile missing variable: {PFTNAME_VAR}") - pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] + pft_names = get_pft_names(ds) pfts_not_in_file = [] for pft in selected_pfts: if pft not in pft_names: @@ -24,6 +24,10 @@ def check_pfts_in_paramfile(selected_pfts, ds): return pft_names +def get_pft_names(ds): + pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] + return pft_names + def get_selected_pft_indices(selected_pfts, pft_names): indices = [i for i, name in enumerate(pft_names) if name in selected_pfts] diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index aa94293d73..52d6fe0c81 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -3,7 +3,7 @@ from ctsm.args_utils import comma_separated_list from ctsm.param_utils.paramfile_shared import paramfile_parser_setup from ctsm.param_utils.paramfile_shared import PFTNAME_VAR, check_pfts_in_paramfile -from ctsm.param_utils.paramfile_shared import get_selected_pft_indices +from ctsm.param_utils.paramfile_shared import get_selected_pft_indices, get_pft_names def get_arguments(): @@ -78,6 +78,8 @@ def main(): pft_names = None if selected_pfts: pft_names = check_pfts_in_paramfile(selected_pfts, ds) + elif PFTNAME_VAR in ds.coords: + pft_names = get_pft_names(ds) for var in args.variables: if var in ds.variables: From bedf08d5cca415c7b47c7006a53e8cabbd124d2a Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 12 Aug 2025 16:43:59 -0600 Subject: [PATCH 041/196] Update _paramfile tool help descriptions. --- python/ctsm/param_utils/query_paramfile.py | 2 +- python/ctsm/param_utils/set_paramfile.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index 52d6fe0c81..b09054c87b 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -19,7 +19,7 @@ def get_arguments(): - pft: Optional list of PFT names to print """ parser, pft_flags = paramfile_parser_setup( - "Print values of one or more variables from a netCDF file." + "Print values of one or more parameters from a CTSM paramfile." ) parser.add_argument( "variables", diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 3b82180bfb..391edd9a70 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -37,7 +37,7 @@ def get_arguments(): - pft: Optional list of PFT names whose values you want to change """ parser, pft_flags = paramfile_parser_setup( - "Print values of one or more variables from a netCDF file." + "Change values of one or more parameters in a CTSM paramfile." ) parser.add_argument("-o", "--output", required=True, help="Output netCDF file") From f0ba3e07532bf094da4d507652e722c8e5538dd7 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 10:59:21 -0600 Subject: [PATCH 042/196] set_paramfile: Check that new value has right ndim. --- python/ctsm/param_utils/set_paramfile.py | 24 ++++++++-- python/ctsm/test/test_sys_set_paramfile.py | 28 +++++++++++ python/ctsm/test/test_unit_set_paramfile.py | 52 +++++++++++++++++++++ 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 391edd9a70..ecdaf16b61 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -69,6 +69,18 @@ def get_arguments(): return args +def check_correct_ndims(da, new_value, throw_error=False): + """ + Check that the new value given for a parameter has the right number of dimensions + """ + expected = da.ndim + actual = np.array(new_value).ndim + is_ndim_correct = expected == actual + if throw_error and not is_ndim_correct: + raise RuntimeError(f"Incorrect N dims: Expected {expected}, got {actual}") + return is_ndim_correct + + def main(): """ Main entry point for the script. @@ -96,12 +108,16 @@ def main(): # Apply parameter changes, if any for chg in args.param_changes: var, new_value = chg.split("=") - if ds_out[var].ndim == 1: + + # TODO: Add handling of multi-dimensional parameters + if ds_out[var].ndim > 1: + raise NotImplementedError("Can't yet change multi-dimensional parameters") + + if "," in new_value: new_value_list = new_value.split(",") new_value = np.array(new_value_list, dtype=type(ds_out[var].dtype)) - elif ds_out[var].ndim > 2: - # TODO: Add handling of multi-dimensional parameters - raise NotImplementedError("Can't yet change multi-dimensional parameters") + + check_correct_ndims(ds_out[var], new_value, throw_error=True) ds_out[var].values = new_value ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input)) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 0c39c690d5..bcdc124fcd 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -108,6 +108,34 @@ def test_set_paramfile_extractpfts(self): else: self.assertTrue(ds_in[var].equals(ds_out[var])) + def test_set_paramfile_changeparams_scalar_errors_given_list(self): + """Test that set_paramfile errors if given a list for a scalar parameter""" + output_path = os.path.join(self.tempdir, "output.nc") + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "a_coef=0.87,0.91", + ] + with self.assertRaisesRegex(RuntimeError, "Incorrect N dims"): + sp.main() + + def test_set_paramfile_changeparam_1d_errors_given_scalar(self): + """Test that set_paramfile errors if given a scalar for a 1-d parameter""" + output_path = os.path.join(self.tempdir, "output.nc") + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "xl=0.724", + ] + with self.assertRaisesRegex(RuntimeError, "Incorrect N dims"): + sp.main() + def test_set_paramfile_changeparams_scalar(self): """Test that set_paramfile can copy to a new file with some scalar params changed""" output_path = os.path.join(self.tempdir, "output.nc") diff --git a/python/ctsm/test/test_unit_set_paramfile.py b/python/ctsm/test/test_unit_set_paramfile.py index 9bccb41450..8ddb074061 100755 --- a/python/ctsm/test/test_unit_set_paramfile.py +++ b/python/ctsm/test/test_unit_set_paramfile.py @@ -5,6 +5,8 @@ import unittest import os import sys +import numpy as np +import xarray as xr from ctsm import unit_testing @@ -20,6 +22,56 @@ ) +class TestUnitCheckCorrectNdims(unittest.TestCase): + """Unit tests of check_correct_ndims""" + + def test_checkcorrectndims_0d_int(self): + """Check True when given a standard int for a 0d parameter""" + da = xr.DataArray(data=1) + self.assertTrue(sp.check_correct_ndims(da, 1)) + + def test_checkcorrectndims_0d_int_np(self): + """Check True when given a numpy int for a 0d parameter""" + da = xr.DataArray(data=1) + self.assertTrue(sp.check_correct_ndims(da, np.int32(1))) + + def test_checkcorrectndims_1d_int(self): + """Check False when given a standard int for a 0d parameter""" + da = xr.DataArray(data=[1, 2]) + self.assertFalse(sp.check_correct_ndims(da, 1)) + + def test_checkcorrectndims_1d_int_np(self): + """Check False when given a numpy int for a 0d parameter""" + da = xr.DataArray(data=[1, 2]) + self.assertFalse(sp.check_correct_ndims(da, np.int32(1))) + + def test_checkcorrectndims_0d_list(self): + """Check False when given a list for a 0d parameter""" + da = xr.DataArray(data=1) + self.assertFalse(sp.check_correct_ndims(da, [1, 2])) + + def test_checkcorrectndims_0d_nparray(self): + """Check False when given a numpy array for a 0d parameter""" + da = xr.DataArray(data=1) + self.assertFalse(sp.check_correct_ndims(da, np.array([1, 2]))) + + def test_checkcorrectndims_0d_nparray_error(self): + """Check for error when given a numpy array for a 0d parameter and requesting throw_error""" + da = xr.DataArray(data=1) + with self.assertRaisesRegex(RuntimeError, "Incorrect N dims: Expected 0, got 1"): + sp.check_correct_ndims(da, np.array([1, 2]), throw_error=True) + + def test_checkcorrectndims_1d_list(self): + """Check True when given a list for a 1d parameter""" + da = xr.DataArray(data=[1, 2]) + self.assertTrue(sp.check_correct_ndims(da, [1, 2])) + + def test_checkcorrectndims_1d_nparray(self): + """Check True when given a numpy array for a 1d parameter""" + da = xr.DataArray(data=[1, 2]) + self.assertTrue(sp.check_correct_ndims(da, np.array([1, 2]))) + + class TestUnitSetParamfile(unittest.TestCase): """Unit tests of set_paramfile""" From 73d0a01fa1e91ad135c540fce056fbde3f0327aa Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 11:28:14 -0600 Subject: [PATCH 043/196] set_paramfile sys tests: Check dtype didn't change. --- python/ctsm/test/test_sys_set_paramfile.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index bcdc124fcd..479156817f 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -153,9 +153,9 @@ def test_set_paramfile_changeparams_scalar(self): ds_in = open_paramfile(PARAMFILE) ds_out = open_paramfile(output_path) - # Check that all variables/coords are equal except the ones we changed, which should be set - # to what we asked for var in ds_in.variables: + # Check that all variables/coords are equal except the ones we changed, which should be + # set to what we asked if var == "a_coef": self.assertTrue(ds_in[var].values == 0.13) self.assertTrue(ds_out[var].values == 0.87) @@ -165,6 +165,9 @@ def test_set_paramfile_changeparams_scalar(self): else: self.assertTrue(ds_in[var].equals(ds_out[var])) + # Check that data type hasn't changed + self.assertTrue(ds_in[var].dtype == ds_out[var].dtype) + def test_set_paramfile_extractpfts_changeparam(self): """ Test that set_paramfile can (1) copy to a new file with only some requested PFTs and (2) From fed3ba4d20bae271b4d93c2a97891455ea795b02 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 11:41:23 -0600 Subject: [PATCH 044/196] set_paramfile: Add is_integer(). Tested but not yet used. --- python/ctsm/param_utils/set_paramfile.py | 12 +++++++ python/ctsm/test/test_unit_set_paramfile.py | 40 +++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index ecdaf16b61..cd0d2bc6f2 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -69,6 +69,18 @@ def get_arguments(): return args +def is_integer(obj): + """ + Given an object, return True if it's (a) any type of integer or (b) a numpy array with an + integer dtype. Note that this will return False for integer types per se. + """ + if isinstance(obj, np.ndarray): + obj_type = obj.dtype + else: + obj_type = type(obj) + return np.issubdtype(obj_type, np.integer) + + def check_correct_ndims(da, new_value, throw_error=False): """ Check that the new value given for a parameter has the right number of dimensions diff --git a/python/ctsm/test/test_unit_set_paramfile.py b/python/ctsm/test/test_unit_set_paramfile.py index 8ddb074061..e38fa73f18 100755 --- a/python/ctsm/test/test_unit_set_paramfile.py +++ b/python/ctsm/test/test_unit_set_paramfile.py @@ -72,6 +72,46 @@ def test_checkcorrectndims_1d_nparray(self): self.assertTrue(sp.check_correct_ndims(da, np.array([1, 2]))) +class TestUnitIsInteger(unittest.TestCase): + """Unit tests of is_integer""" + + def test_isinteger_obj_int(self): + """Check True if given an object of type int""" + self.assertTrue(sp.is_integer(int(1))) + + def test_isinteger_obj_int_np(self): + """Check True if given an object of a numpy integer type""" + self.assertTrue(sp.is_integer(np.int32(1))) + + def test_isinteger_obj_int_array0d_np(self): + """Check True if given an object of a numpy array with integer dtype""" + self.assertTrue(sp.is_integer(np.array(1, dtype=np.int32))) + + def test_isinteger_obj_int_0d_xr(self): + """Check True if given a numpy scalar integer object via xarray""" + da = xr.DataArray(data=1) + self.assertTrue(sp.is_integer(da.values)) + + def test_isinteger_obj_int_1d_xr(self): + """Check True if given a numpy 1-d integer object via xarray""" + da = xr.DataArray(data=[1]) + self.assertTrue(sp.is_integer(da.values)) + da = xr.DataArray(data=[1, 2]) + self.assertTrue(sp.is_integer(da.values)) + + def test_isinteger_obj_float(self): + """Check False if given an object of type float""" + self.assertFalse(sp.is_integer(float(3.14))) + + def test_isinteger_obj_float_np(self): + """Check False if given an object of a numpy float type""" + self.assertFalse(sp.is_integer(np.float32(3.14))) + + def test_isinteger_type_int(self): + """Check False if given a type""" + self.assertFalse(sp.is_integer(int)) + + class TestUnitSetParamfile(unittest.TestCase): """Unit tests of set_paramfile""" From 3056c3699ce9e011ce22828fc4591c4349ab2d0f Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 11:42:26 -0600 Subject: [PATCH 045/196] set_paramfile: Test changing scalar int param. Failing. --- python/ctsm/test/test_sys_set_paramfile.py | 40 ++++++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 479156817f..032f013f07 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -104,7 +104,7 @@ def test_set_paramfile_extractpfts(self): # Check that included variables/coords match for var in ds_in.variables: if sp.PFTNAME_VAR in ds_in[var].coords: - self.assertTrue(ds_in[var].isel(pft=[0,1]).equals(ds_out[var])) + self.assertTrue(ds_in[var].isel(pft=[0, 1]).equals(ds_out[var])) else: self.assertTrue(ds_in[var].equals(ds_out[var])) @@ -136,8 +136,8 @@ def test_set_paramfile_changeparam_1d_errors_given_scalar(self): with self.assertRaisesRegex(RuntimeError, "Incorrect N dims"): sp.main() - def test_set_paramfile_changeparams_scalar(self): - """Test that set_paramfile can copy to a new file with some scalar params changed""" + def test_set_paramfile_changeparams_scalar_double(self): + """Test that set_paramfile can copy to a new file with some scalar double params changed""" output_path = os.path.join(self.tempdir, "output.nc") sys.argv = [ "set_paramfile", @@ -168,6 +168,40 @@ def test_set_paramfile_changeparams_scalar(self): # Check that data type hasn't changed self.assertTrue(ds_in[var].dtype == ds_out[var].dtype) + def test_set_paramfile_changeparams_scalar_int(self): + """Test that set_paramfile can copy to a new file with a scalar integer param changed""" + output_path = os.path.join(self.tempdir, "output.nc") + this_var = "upplim_destruct_metamorph" + new_value = 1987 + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + f"upplim_destruct_metamorph={new_value}", + ] + sp.main() + self.assertTrue(os.path.exists(output_path)) + ds_in = open_paramfile(PARAMFILE) + ds_out = open_paramfile(output_path) + + # Check that the variable in question is actually an integer to begin with + self.assertTrue(sp.is_integer(ds_in[this_var].values)) + # Also check that it actually differs from our new value + self.assertTrue(ds_in[this_var].values != new_value) + + for var in ds_in.variables: + # Check that all variables/coords are equal except the one we changed, which should be + # set to what we asked + if var == this_var: + self.assertTrue(ds_out[var].values == new_value) + else: + self.assertTrue(ds_in[var].equals(ds_out[var])) + + # Check that data type hasn't changed + self.assertTrue(ds_in[var].dtype == ds_out[var].dtype) + def test_set_paramfile_extractpfts_changeparam(self): """ Test that set_paramfile can (1) copy to a new file with only some requested PFTs and (2) From 552f979cda752b273a404c64f32501ac5f191314 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 13:32:42 -0600 Subject: [PATCH 046/196] set_paramfile: Can now change scalar int params. --- python/ctsm/param_utils/set_paramfile.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index cd0d2bc6f2..735a40bb3d 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -130,6 +130,10 @@ def main(): new_value = np.array(new_value_list, dtype=type(ds_out[var].dtype)) check_correct_ndims(ds_out[var], new_value, throw_error=True) + + if is_integer(ds_in[var].values): + new_value = int(new_value) + ds_out[var].values = new_value ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input)) From b1c0c7503602409fa26e142f83944ed166a1f44e Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 13:40:36 -0600 Subject: [PATCH 047/196] Turns out set_paramfile works for 1d int params too. --- python/ctsm/test/test_sys_set_paramfile.py | 40 ++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 032f013f07..1ca8a7a6f8 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -202,10 +202,10 @@ def test_set_paramfile_changeparams_scalar_int(self): # Check that data type hasn't changed self.assertTrue(ds_in[var].dtype == ds_out[var].dtype) - def test_set_paramfile_extractpfts_changeparam(self): + def test_set_paramfile_extractpfts_changeparam_dbl(self): """ Test that set_paramfile can (1) copy to a new file with only some requested PFTs and (2) - change the values of parameters of those PFTs + change the values of double parameters of those PFTs """ output_path = os.path.join(self.tempdir, "output.nc") pfts_to_include = ["not_vegetated", "needleleaf_evergreen_temperate_tree"] @@ -233,6 +233,42 @@ def test_set_paramfile_extractpfts_changeparam(self): else: self.assertTrue(ds_in[var].equals(ds_out[var])) + def test_set_paramfile_extractpfts_changeparam_int(self): + """ + Test that set_paramfile can (1) copy to a new file with only some requested PFTs and (2) + change the values of integer parameters of those PFTs + """ + output_path = os.path.join(self.tempdir, "output.nc") + pfts_to_include = ["not_vegetated", "needleleaf_evergreen_temperate_tree"] + this_var = "max_NH_planting_date" + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "-p", + ",".join(pfts_to_include), + f"{this_var}=1986,1987", + ] + sp.main() + self.assertTrue(os.path.exists(output_path)) + ds_in = open_paramfile(PARAMFILE) + ds_out = open_paramfile(output_path) + + # Check that it actually differs from our new values + self.assertTrue(ds_in[this_var].values[0] != 1986) + self.assertTrue(ds_in[this_var].values[1] != 1987) + + # Check that included variables/coords match as expected + for var in ds_in.variables: + if var == this_var: + self.assertTrue(np.array_equal(np.array([1986, 1987]), ds_out[var].values)) + elif sp.PFTNAME_VAR in ds_in[var].coords: + self.assertTrue(ds_in[var].isel(pft=[0, 1]).equals(ds_out[var])) + else: + self.assertTrue(ds_in[var].equals(ds_out[var])) + # TODO: Add test with NaN in existing # TODO: Add test changing to NaN From 6872d2b5855d9c2ff8f86ad5dc4adacf03fa4337 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 13:47:02 -0600 Subject: [PATCH 048/196] set_paramfile: Don't mask or scale when reading parameter file. --- python/ctsm/param_utils/paramfile_shared.py | 4 ++-- python/ctsm/param_utils/query_paramfile.py | 3 ++- python/ctsm/param_utils/set_paramfile.py | 2 +- python/ctsm/test/test_sys_set_paramfile.py | 4 +++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/python/ctsm/param_utils/paramfile_shared.py b/python/ctsm/param_utils/paramfile_shared.py index 2079957b4b..a2788ac412 100644 --- a/python/ctsm/param_utils/paramfile_shared.py +++ b/python/ctsm/param_utils/paramfile_shared.py @@ -34,8 +34,8 @@ def get_selected_pft_indices(selected_pfts, pft_names): return indices -def open_paramfile(file_in): - return xr.open_dataset(file_in, decode_timedelta=False) +def open_paramfile(file_in, mask_and_scale=False): + return xr.open_dataset(file_in, decode_timedelta=False, mask_and_scale=mask_and_scale) def paramfile_parser_setup(description): diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index b09054c87b..ff5c5fd7bd 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -2,6 +2,7 @@ from ctsm.args_utils import comma_separated_list from ctsm.param_utils.paramfile_shared import paramfile_parser_setup +from ctsm.param_utils.paramfile_shared import open_paramfile from ctsm.param_utils.paramfile_shared import PFTNAME_VAR, check_pfts_in_paramfile from ctsm.param_utils.paramfile_shared import get_selected_pft_indices, get_pft_names @@ -72,7 +73,7 @@ def main(): """ args = get_arguments() - ds = xr.open_dataset(args.input, decode_timedelta=False) + ds = open_paramfile(args.input, mask_and_scale=True) selected_pfts = args.pft pft_names = None diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 735a40bb3d..cf28f373ed 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -132,7 +132,7 @@ def main(): check_correct_ndims(ds_out[var], new_value, throw_error=True) if is_integer(ds_in[var].values): - new_value = int(new_value) + new_value = np.astype(np.array(new_value), ds_in[var].dtype) ds_out[var].values = new_value diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 1ca8a7a6f8..3cba4fa141 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -256,7 +256,9 @@ def test_set_paramfile_extractpfts_changeparam_int(self): ds_in = open_paramfile(PARAMFILE) ds_out = open_paramfile(output_path) - # Check that it actually differs from our new values + # Check that the variable in question is actually an integer to begin with + self.assertTrue(sp.is_integer(ds_in[this_var].values)) + # Also check that it actually differs from our new values self.assertTrue(ds_in[this_var].values[0] != 1986) self.assertTrue(ds_in[this_var].values[1] != 1987) From f37c3017d532f2e224cb6f2e46336f8b8cebf173 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 13:59:39 -0600 Subject: [PATCH 049/196] test_sys_set_paramfile: Check that fill val didn't change. --- python/ctsm/test/test_sys_set_paramfile.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 3cba4fa141..2491a856ca 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -168,6 +168,18 @@ def test_set_paramfile_changeparams_scalar_double(self): # Check that data type hasn't changed self.assertTrue(ds_in[var].dtype == ds_out[var].dtype) + # Check that fill value hasn't changed + if "_FillValue" in ds_in[var].encoding: + fv_in = ds_in[var].encoding["_FillValue"] + fv_out = ds_out[var].encoding["_FillValue"] + if isinstance(fv_in, bytes): + self.assertTrue(isinstance(fv_out, bytes)) + self.assertEqual(fv_in, fv_out) + else: + self.assertEqual(np.isnan(fv_in), np.isnan(fv_out)) + if not np.isnan(fv_in): + self.assertEqual(fv_in, fv_out) + def test_set_paramfile_changeparams_scalar_int(self): """Test that set_paramfile can copy to a new file with a scalar integer param changed""" output_path = os.path.join(self.tempdir, "output.nc") From a8d90d6f5d4099987435b8406428e2e7022690f1 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 14:23:18 -0600 Subject: [PATCH 050/196] set_paramfile: Avoid calling is_integer(). --- python/ctsm/param_utils/set_paramfile.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index cf28f373ed..603041299d 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -127,13 +127,13 @@ def main(): if "," in new_value: new_value_list = new_value.split(",") - new_value = np.array(new_value_list, dtype=type(ds_out[var].dtype)) + new_value = np.array(new_value_list) + else: + new_value = np.array(new_value) + new_value = new_value.astype(type(ds_out[var].dtype)) check_correct_ndims(ds_out[var], new_value, throw_error=True) - if is_integer(ds_in[var].values): - new_value = np.astype(np.array(new_value), ds_in[var].dtype) - ds_out[var].values = new_value ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input)) From dd38d7e2842e6f28c6415c4de88def29defac16b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 29 Jul 2025 13:25:52 -0600 Subject: [PATCH 051/196] Add a PE layout for mpas13p75 --- cime_config/config_pes.xml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/cime_config/config_pes.xml b/cime_config/config_pes.xml index bb10b8019c..d0794339db 100644 --- a/cime_config/config_pes.xml +++ b/cime_config/config_pes.xml @@ -2092,6 +2092,44 @@ + + + + + none + + -1 + -80 + -80 + -80 + -80 + -80 + -80 + -80 + -80 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + From b03835fc59d2c054783d389789f4a31c822d7e00 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 30 Jul 2025 11:47:35 -0600 Subject: [PATCH 052/196] Add a testmod for mpasa3p75 grid --- .../testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm diff --git a/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm new file mode 100644 index 0000000000..bfdcfb115b --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm @@ -0,0 +1,6 @@ +! Settings currently required to run at the mpasa3p75 grid +! urbantv files at that resolution and use a redistribution mapping + +stream_fldfilename_urbantv = '/glade/derecho/scratch/bdobbins/ko/tbuildmax.nc' +stream_meshfile_urbantv = '/glade/derecho/scratch/bdobbins/ko/mesh.nc' +urbantvmapalgo = 'redist' From 52470c4911053e4fdd45ab838991e52e5b6470ae Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 30 Jul 2025 11:53:51 -0600 Subject: [PATCH 053/196] Add decomp initialization test and test list for ultra high resolution (3.75km mpasa) --- cime_config/testdefs/testlist_clm.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 478dc59bdd..3f232e7518 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -14,6 +14,7 @@ matrixcn: Tests exercising the matrix-CN capability aux_clm_mpi_serial: aux_clm tests using mpi-serial. Useful for redoing tests that failed due to https://github.com/ESCOMP/CTSM/issues/2916, after having replaced libraries/mpi-serial with a fresh copy. decomp_init: Initialization tests specifically for examining the PE layout decomposition initialization + uhr_decomp_init: Initialization tests at Ultra High Resolution -- specifically for examining the PE layout decomposition initialization --> @@ -4209,6 +4210,15 @@ + + + + + + + + + From f1a5a4521213ff0818b61296ca41b6947445ab0c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 30 Jul 2025 13:03:20 -0600 Subject: [PATCH 054/196] Fix syntax and correct 3p75 resolution grid name for test --- cime_config/testdefs/testlist_clm.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 3f232e7518..a3dc4d760f 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -14,7 +14,7 @@ matrixcn: Tests exercising the matrix-CN capability aux_clm_mpi_serial: aux_clm tests using mpi-serial. Useful for redoing tests that failed due to https://github.com/ESCOMP/CTSM/issues/2916, after having replaced libraries/mpi-serial with a fresh copy. decomp_init: Initialization tests specifically for examining the PE layout decomposition initialization - uhr_decomp_init: Initialization tests at Ultra High Resolution -- specifically for examining the PE layout decomposition initialization + uhr_decomp_init: Initialization tests at Ultra High Resolution - specifically for examining the PE layout decomposition initialization --> @@ -4210,7 +4210,7 @@ - + From f3ac549aebc04166c32f6503f5b87ba67d6b342a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 30 Jul 2025 13:29:50 -0600 Subject: [PATCH 055/196] Fix name of mpasa3p75 testmod in test --- cime_config/testdefs/testlist_clm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index a3dc4d760f..1a53e25ae9 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -4210,7 +4210,7 @@ - + From 5602a7d78f5f865e6b5178cdc179ff2b5114e3d4 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 14:46:28 -0600 Subject: [PATCH 056/196] test_sys_set_paramfile: Test setting scalar double NaN using 'nan' --- python/ctsm/test/test_sys_set_paramfile.py | 69 +++++++++++++++++++++- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 2491a856ca..01011cc4f6 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -283,9 +283,74 @@ def test_set_paramfile_extractpfts_changeparam_int(self): else: self.assertTrue(ds_in[var].equals(ds_out[var])) - # TODO: Add test with NaN in existing + def test_set_paramfile_setparams_scalar_double_tonan_with_nan(self): + """Test setting scalar double to NaN using 'nan'""" + output_path = os.path.join(self.tempdir, "output.nc") + this_var = "a_coef" + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + f"{this_var}=nan", + ] + + sp.main() + self.assertTrue(os.path.exists(output_path)) + ds_out = open_paramfile(output_path, mask_and_scale=True) + self.assertTrue(np.isnan(ds_out[this_var])) + + def test_set_paramfile_setparams_scalar_double_tonan_with_nancaps(self): + """Test setting scalar double to NaN using 'NaN'""" + output_path = os.path.join(self.tempdir, "output.nc") + this_var = "a_coef" + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + f"{this_var}=NaN", + ] + + sp.main() + self.assertTrue(os.path.exists(output_path)) + ds_out = open_paramfile(output_path, mask_and_scale=True) + self.assertTrue(np.isnan(ds_out[this_var])) + + def test_set_paramfile_setparams_pft_double_tonan_with_nan(self): + """Test setting PFT-dimensioned double to NaN using 'nan'""" + output_path = os.path.join(self.tempdir, "output.nc") + this_var = "xl" + pfts_to_include = ["not_vegetated", "needleleaf_evergreen_temperate_tree"] + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "-p", + ",".join(pfts_to_include), + f"{this_var}=nan,nan", + ] + + sp.main() + self.assertTrue(os.path.exists(output_path)) + ds_out = open_paramfile(output_path, mask_and_scale=True) + self.assertTrue(all(np.isnan(ds_out[this_var]))) + + # TODO: Add test changing scalar int to NaN using nan + # TODO: Add test changing scalar double to NaN using _ + # TODO: Add test changing scalar int to NaN using _ + # TODO: Add test changing vector double to NaN using _ + # TODO: Add test changing vector int to NaN using _ + + # TODO: Add test overwriting existing scalar double NaN using nan (lowercase) + + # TODO: Test changing param when extracting just one PFT - # TODO: Add test changing to NaN + # TODO: Test changing PFT name def test_set_paramfile_changeparam_multidim_errors(self): """ From 89f0c4221fa3beeffb2d664f5a1105bbf34f8114 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 15:19:53 -0600 Subject: [PATCH 057/196] set_paramfile: Don't allow setting NaN if param doesn't have FillValue. --- python/ctsm/param_utils/set_paramfile.py | 13 +++++++++++++ python/ctsm/test/test_sys_set_paramfile.py | 21 ++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 603041299d..278ed2bd83 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -100,6 +100,8 @@ def main(): """ args = get_arguments() + ds_in_masked_scaled = open_paramfile(args.input, mask_and_scale=True) + ds_in = open_paramfile(args.input) ds_out = ds_in.copy() @@ -130,6 +132,17 @@ def main(): new_value = np.array(new_value_list) else: new_value = np.array(new_value) + + # TODO: Add code to set something to NaN if variable doesn't already have fill value + print(ds_out[var].encoding) + if ( + np.any(np.char.lower(new_value) == "nan") + and "_FillValue" not in ds_in_masked_scaled[var].encoding + ): + raise NotImplementedError( + f"Not able to set NaN if parameter doesn't already have fill value: {chg}" + ) + new_value = new_value.astype(type(ds_out[var].dtype)) check_correct_ndims(ds_out[var], new_value, throw_error=True) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 01011cc4f6..7a280640fe 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -340,7 +340,26 @@ def test_set_paramfile_setparams_pft_double_tonan_with_nan(self): ds_out = open_paramfile(output_path, mask_and_scale=True) self.assertTrue(all(np.isnan(ds_out[this_var]))) - # TODO: Add test changing scalar int to NaN using nan + def test_set_paramfile_setparams_nan_but_no_fillvalue(self): + """Test that NotImplementedError is given if trying to set NaN but param has no FillValue""" + output_path = os.path.join(self.tempdir, "output.nc") + this_var = "upplim_destruct_metamorph" + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + f"{this_var}=nan", + ] + + with self.assertRaisesRegex( + NotImplementedError, "Not able to set NaN if parameter doesn't already have fill value:" + ): + sp.main() + + # TODO: Add test changing scalar int to NaN using nan (needs a new input file) + # TODO: Add test changing vector int to NaN using nan # TODO: Add test changing scalar double to NaN using _ # TODO: Add test changing scalar int to NaN using _ # TODO: Add test changing vector double to NaN using _ From c4e1fa46816653470a0ae420a433cb245ff9b00e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 13 Aug 2025 15:21:01 -0600 Subject: [PATCH 058/196] Copy the files from scratch to inputdata just on glade on Derecho --- cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm index bfdcfb115b..69a3a55836 100644 --- a/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm @@ -1,6 +1,6 @@ ! Settings currently required to run at the mpasa3p75 grid ! urbantv files at that resolution and use a redistribution mapping -stream_fldfilename_urbantv = '/glade/derecho/scratch/bdobbins/ko/tbuildmax.nc' -stream_meshfile_urbantv = '/glade/derecho/scratch/bdobbins/ko/mesh.nc' +stream_fldfilename_urbantv = '$CSMDATA/lnd/clm2/urbandata/CTSM52_tbuildmax_OlesonFeddema_2020_mpasa3p75_fromf09_simyr1849-2106_c20240502.nc' +stream_meshfile_urbantv = '$DIN_LOC_ROOT/lnd/clm2/urbandata/CTSM52_tbuildmax_Oleson_2020_mpasa3p75_ESMFmesh_cdf5_c202405021.nc' urbantvmapalgo = 'redist' From 152ac60a80f74d558bbf276696485c1785cd5d8e Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 15:48:42 -0600 Subject: [PATCH 059/196] set_paramfile: Don't allow setting NaN in integer params. --- python/ctsm/param_utils/set_paramfile.py | 21 +++++++------ python/ctsm/test/test_sys_set_paramfile.py | 34 ++++++++++++++++++++-- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 278ed2bd83..b72e5d137a 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -133,15 +133,18 @@ def main(): else: new_value = np.array(new_value) - # TODO: Add code to set something to NaN if variable doesn't already have fill value - print(ds_out[var].encoding) - if ( - np.any(np.char.lower(new_value) == "nan") - and "_FillValue" not in ds_in_masked_scaled[var].encoding - ): - raise NotImplementedError( - f"Not able to set NaN if parameter doesn't already have fill value: {chg}" - ) + # Some NaN handling isn't yet implemented + if np.any(np.char.lower(new_value) == "nan"): + # TODO: Add code to set integer variables to NaN (this might not be possible) + if is_integer(ds_in[var].values): + raise NotImplementedError( + f"Not able to set NaN for integer parameters: {chg}" + ) + # TODO: Add code to set integer variables to NaN (this might not be possible) + if "_FillValue" not in ds_in_masked_scaled[var].encoding: + raise NotImplementedError( + f"Not able to set NaN if parameter doesn't already have fill value: {chg}" + ) new_value = new_value.astype(type(ds_out[var].dtype)) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 7a280640fe..2da477cc42 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -342,6 +342,37 @@ def test_set_paramfile_setparams_pft_double_tonan_with_nan(self): def test_set_paramfile_setparams_nan_but_no_fillvalue(self): """Test that NotImplementedError is given if trying to set NaN but param has no FillValue""" + + # Create paramfile with a double variable without fill value + input_path = os.path.join(self.tempdir, "input.nc") + ds = open_paramfile(PARAMFILE, mask_and_scale=True) + new_param_name = "new_param_abc123" + ds[new_param_name] = xr.DataArray(data=np.array(3.14)) + ds[new_param_name].encoding["_FillValue"] = None + self.assertTrue(new_param_name in ds) + ds.to_netcdf(input_path) + + # Check that it doesn't have fill value + ds_in = open_paramfile(input_path, mask_and_scale=True) + self.assertFalse("_FillValue" in ds_in[new_param_name].encoding) + + # Call set_paramfile, trying to set it to NaN + output_path = os.path.join(self.tempdir, "output.nc") + sys.argv = [ + "set_paramfile", + "-i", + input_path, + "-o", + output_path, + f"{new_param_name}=nan", + ] + with self.assertRaisesRegex( + NotImplementedError, "Not able to set NaN if parameter doesn't already have fill value:" + ): + sp.main() + + def test_set_paramfile_setparams_scalar_int_tonan_with_nan(self): + """Test that NotImplementedError is given if trying to set NaN for an integer""" output_path = os.path.join(self.tempdir, "output.nc") this_var = "upplim_destruct_metamorph" sys.argv = [ @@ -354,11 +385,10 @@ def test_set_paramfile_setparams_nan_but_no_fillvalue(self): ] with self.assertRaisesRegex( - NotImplementedError, "Not able to set NaN if parameter doesn't already have fill value:" + NotImplementedError, "Not able to set NaN for integer parameters:" ): sp.main() - # TODO: Add test changing scalar int to NaN using nan (needs a new input file) # TODO: Add test changing vector int to NaN using nan # TODO: Add test changing scalar double to NaN using _ # TODO: Add test changing scalar int to NaN using _ From cb8fadff805486372d5119cea5f2534eced5b827 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 15:54:36 -0600 Subject: [PATCH 060/196] Resolve or delete some testing TODOs. --- python/ctsm/test/test_sys_set_paramfile.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 2da477cc42..d4b28c5626 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -322,7 +322,6 @@ def test_set_paramfile_setparams_scalar_double_tonan_with_nancaps(self): def test_set_paramfile_setparams_pft_double_tonan_with_nan(self): """Test setting PFT-dimensioned double to NaN using 'nan'""" output_path = os.path.join(self.tempdir, "output.nc") - this_var = "xl" pfts_to_include = ["not_vegetated", "needleleaf_evergreen_temperate_tree"] sys.argv = [ "set_paramfile", @@ -332,13 +331,20 @@ def test_set_paramfile_setparams_pft_double_tonan_with_nan(self): output_path, "-p", ",".join(pfts_to_include), - f"{this_var}=nan,nan", + "xl=nan,nan", + "planting_temp=nan,nan", ] + # Check that planting_temp is already nan and xl isn't + ds_in = open_paramfile(PARAMFILE, mask_and_scale=True) + self.assertTrue(all(np.isnan(ds_in["planting_temp"].isel(pft=[0, 1])))) + self.assertTrue(not any(np.isnan(ds_in["xl"].isel(pft=[0, 1])))) + sp.main() self.assertTrue(os.path.exists(output_path)) ds_out = open_paramfile(output_path, mask_and_scale=True) - self.assertTrue(all(np.isnan(ds_out[this_var]))) + self.assertTrue(all(np.isnan(ds_out["xl"]))) + self.assertTrue(all(np.isnan(ds_out["planting_temp"]))) def test_set_paramfile_setparams_nan_but_no_fillvalue(self): """Test that NotImplementedError is given if trying to set NaN but param has no FillValue""" @@ -389,14 +395,6 @@ def test_set_paramfile_setparams_scalar_int_tonan_with_nan(self): ): sp.main() - # TODO: Add test changing vector int to NaN using nan - # TODO: Add test changing scalar double to NaN using _ - # TODO: Add test changing scalar int to NaN using _ - # TODO: Add test changing vector double to NaN using _ - # TODO: Add test changing vector int to NaN using _ - - # TODO: Add test overwriting existing scalar double NaN using nan (lowercase) - # TODO: Test changing param when extracting just one PFT # TODO: Test changing PFT name From 492bceaf7a1d9d6f703deae4512b392d251cb3a3 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 16:08:44 -0600 Subject: [PATCH 061/196] set_paramfile: Require --drop-other-pfts to actually trim the PFT list. --- python/ctsm/param_utils/set_paramfile.py | 16 +++++++++++++--- python/ctsm/test/test_sys_set_paramfile.py | 4 ++++ python/ctsm/test/test_unit_set_paramfile.py | 15 +++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index b72e5d137a..d1849de226 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -23,6 +23,10 @@ def check_arguments(args): if os.path.exists(args.output): raise FileExistsError(args.output) + # --drop-other-pfts makes no sense without --pfts + if args.drop_other_pfts and not args.pft: + raise RuntimeError("--drop-other-pfts makes no sense without -p/--pft") + def get_arguments(): """ @@ -42,13 +46,19 @@ def get_arguments(): parser.add_argument("-o", "--output", required=True, help="Output netCDF file") - # TODO: Add --exclude-pfts argument for PFTs you DON'T want to include + # TODO: Add mutually-exclusive --exclude-pfts argument for PFTs you DON'T want to include parser.add_argument( *pft_flags, help="Comma-separated list of PFTs to include (only applies to PFT-specific variables)", type=comma_separated_list, ) + parser.add_argument( + "--drop-other-pfts", + help=f"Do not include PFTs other than the ones given in {'/'.join(pft_flags)}", + action="store_true", + ) + # TODO: Add -x/--exclude argument for variables you DON'T want to extract parser.add_argument( "-v", @@ -105,8 +115,8 @@ def main(): ds_in = open_paramfile(args.input) ds_out = ds_in.copy() - # If any PFTs were specified, drop others - if args.pft: + # If --drop-other-pfts was given, drop PFTs not in args.pft + if args.drop_other_pfts: pft_names = check_pfts_in_paramfile(args.pft, ds_out) indices = get_selected_pft_indices(args.pft, pft_names) ds_out = ds_out.isel({"pft": indices}) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index d4b28c5626..a26213156b 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -95,6 +95,7 @@ def test_set_paramfile_extractpfts(self): output_path, "-p", ",".join(pfts_to_include), + "--drop-other-pfts", ] sp.main() self.assertTrue(os.path.exists(output_path)) @@ -229,6 +230,7 @@ def test_set_paramfile_extractpfts_changeparam_dbl(self): output_path, "-p", ",".join(pfts_to_include), + "--drop-other-pfts", "xl=0.724,0.87", ] sp.main() @@ -261,6 +263,7 @@ def test_set_paramfile_extractpfts_changeparam_int(self): output_path, "-p", ",".join(pfts_to_include), + "--drop-other-pfts", f"{this_var}=1986,1987", ] sp.main() @@ -331,6 +334,7 @@ def test_set_paramfile_setparams_pft_double_tonan_with_nan(self): output_path, "-p", ",".join(pfts_to_include), + "--drop-other-pfts", "xl=nan,nan", "planting_temp=nan,nan", ] diff --git a/python/ctsm/test/test_unit_set_paramfile.py b/python/ctsm/test/test_unit_set_paramfile.py index e38fa73f18..5bfb9528c9 100755 --- a/python/ctsm/test/test_unit_set_paramfile.py +++ b/python/ctsm/test/test_unit_set_paramfile.py @@ -153,6 +153,7 @@ def test_set_paramfile_args_long(self): PARAMFILE, "--pft", "pft1,pft2", + "--drop-other-pfts", "--output", output_path, "--variables", @@ -196,6 +197,20 @@ def test_set_paramfile_error_existing_output(self): with self.assertRaises(FileExistsError): sp.get_arguments() + def test_set_paramfile_error_dropotherpfts_without_pft(self): + """Test that it errors if given --drop-other-pfts without --pft""" + output_path = "/path/to/output.nc" + sys.argv = [ + "get_arguments", + "--input", + PARAMFILE, + "--drop-other-pfts", + "--output", + output_path, + ] + with self.assertRaises(RuntimeError): + sp.get_arguments() + if __name__ == "__main__": unit_testing.setup_for_tests() From 50cf969e244c3199ca33fd7b8338f7d8eb021884 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 13 Aug 2025 16:24:27 -0600 Subject: [PATCH 062/196] Initial add of a unit test for endrun --- src/main/test/CMakeLists.txt | 1 + src/main/test/endrun_test/CMakeLists.txt | 8 ++++ src/main/test/endrun_test/test_endrun.pf | 47 ++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 src/main/test/endrun_test/CMakeLists.txt create mode 100644 src/main/test/endrun_test/test_endrun.pf diff --git a/src/main/test/CMakeLists.txt b/src/main/test/CMakeLists.txt index 97bbf081cc..2ade2f6e87 100644 --- a/src/main/test/CMakeLists.txt +++ b/src/main/test/CMakeLists.txt @@ -8,3 +8,4 @@ add_subdirectory(filter_test) add_subdirectory(initVertical_test) add_subdirectory(ncdio_utils_test) add_subdirectory(topo_test) +add_subdirectory(endrun_test) diff --git a/src/main/test/endrun_test/CMakeLists.txt b/src/main/test/endrun_test/CMakeLists.txt new file mode 100644 index 0000000000..45b42a15e7 --- /dev/null +++ b/src/main/test/endrun_test/CMakeLists.txt @@ -0,0 +1,8 @@ +set(pfunit_sources + test_endrun.pf) + +add_pfunit_ctest(endrun + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf + EXTRA_FINALIZE unittest_finalize_esmf + EXTRA_USE unittestInitializeAndFinalize) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf new file mode 100644 index 0000000000..a6b7c19647 --- /dev/null +++ b/src/main/test/endrun_test/test_endrun.pf @@ -0,0 +1,47 @@ +module test_endrun + + ! Tests of abortutils + + use funit + use abortutils + use unittestUtils, only : endrun_msg + + implicit none + + @TestCase + type, extends(TestCase) :: TestAbortUtils + contains + procedure :: setUp + procedure :: tearDown + end type TestAbortUtils + +contains + + ! ======================================================================== + ! Helper routines + ! ======================================================================== + + subroutine setUp(this) + class(TestAbortUtils), intent(inout) :: this + end subroutine setUp + + subroutine tearDown(this) + class(TestAbortUtils), intent(inout) :: this + + end subroutine tearDown + + ! ======================================================================== + ! Begin tests + ! ======================================================================== + + @Test + subroutine endrun_vanilla_aborts(this) + ! Test vanilla operation of endrun + class(TestAbortUtils), intent(inout) :: this + + call endrun() + @assertExceptionRaised(endrun_msg('')) + + end subroutine endrun_vanilla_aborts + +end module test_endrun From 14bb2f8e0746ee64bd1ee23040fab5e9e08ea4f3 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 16:37:44 -0600 Subject: [PATCH 063/196] set_paramfile: Can now change just a few PFT values. --- python/ctsm/param_utils/set_paramfile.py | 8 +++++ python/ctsm/test/test_sys_set_paramfile.py | 42 ++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index d1849de226..9b4e1136df 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -160,6 +160,14 @@ def main(): check_correct_ndims(ds_out[var], new_value, throw_error=True) + # Handle the situation where we're only changing values for some PFTs + if PFTNAME_VAR in ds_out[var].coords and args.pft and not args.drop_other_pfts: + pft_names = check_pfts_in_paramfile(args.pft, ds_out) + indices = get_selected_pft_indices(args.pft, pft_names) + tmp = ds_out[var].values.copy() + tmp[indices] = new_value + new_value = tmp + ds_out[var].values = new_value ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input)) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index a26213156b..24407e5ded 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -247,6 +247,48 @@ def test_set_paramfile_extractpfts_changeparam_dbl(self): else: self.assertTrue(ds_in[var].equals(ds_out[var])) + def test_set_paramfile_changeparam_dbl_onlysomepfts(self): + """ + Test that set_paramfile can (1) copy to a new file with only some requested PFTs and (2) + change the values of double parameters of those PFTs + """ + output_path = os.path.join(self.tempdir, "output.nc") + pfts_to_include = ["not_vegetated", "needleleaf_evergreen_temperate_tree"] + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "-p", + ",".join(pfts_to_include), + "xl=0.724,0.87", + ] + sp.main() + self.assertTrue(os.path.exists(output_path)) + ds_in = open_paramfile(PARAMFILE) + ds_out = open_paramfile(output_path) + + # Check that included variables/coords match as expected + for var in ds_in.variables: + if var == "xl": + print(ds_in[var].values) + print(ds_out[var].values) + + # Changed values (first 2) + this_slice = slice(0, 2) + expected = np.array([0.724, 0.87]) + result = ds_out[var].isel(pft=this_slice).values + self.assertTrue(np.array_equal(expected, result)) + + # Preserved values (everything but the first 2) + this_slice = slice(2, None) + expected = ds_in["xl"].isel(pft=this_slice) + result = ds_out["xl"].isel(pft=this_slice) + self.assertTrue(expected.equals(result)) + else: + self.assertTrue(ds_in[var].equals(ds_out[var])) + def test_set_paramfile_extractpfts_changeparam_int(self): """ Test that set_paramfile can (1) copy to a new file with only some requested PFTs and (2) From f42d816224cc614f447f76e6b183e7e256c06145 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 17:02:00 -0600 Subject: [PATCH 064/196] set_paramfile: Can now change just one PFT without touching others. --- python/ctsm/param_utils/set_paramfile.py | 23 ++++++++++----- python/ctsm/test/test_sys_set_paramfile.py | 33 ++++++++++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 9b4e1136df..f2d9137037 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -147,9 +147,7 @@ def main(): if np.any(np.char.lower(new_value) == "nan"): # TODO: Add code to set integer variables to NaN (this might not be possible) if is_integer(ds_in[var].values): - raise NotImplementedError( - f"Not able to set NaN for integer parameters: {chg}" - ) + raise NotImplementedError(f"Not able to set NaN for integer parameters: {chg}") # TODO: Add code to set integer variables to NaN (this might not be possible) if "_FillValue" not in ds_in_masked_scaled[var].encoding: raise NotImplementedError( @@ -158,12 +156,23 @@ def main(): new_value = new_value.astype(type(ds_out[var].dtype)) - check_correct_ndims(ds_out[var], new_value, throw_error=True) - - # Handle the situation where we're only changing values for some PFTs - if PFTNAME_VAR in ds_out[var].coords and args.pft and not args.drop_other_pfts: + # Are we acting on just some PFTs? If so, we'll need some stuff. + just_some_pfts = PFTNAME_VAR in ds_out[var].coords and args.pft + if just_some_pfts: pft_names = check_pfts_in_paramfile(args.pft, ds_out) indices = get_selected_pft_indices(args.pft, pft_names) + + # Check that correct number of dimensions were given for new values. Special handling needed + # if we're just acting on one PFT. + if just_some_pfts and len(args.pft) == 1: + check_correct_ndims( + ds_out[var].isel(pft=indices).squeeze(), new_value, throw_error=True + ) + else: + check_correct_ndims(ds_out[var], new_value, throw_error=True) + + # Handle the situation where we're only changing values for some PFTs but keeping the others + if just_some_pfts and not args.drop_other_pfts: tmp = ds_out[var].values.copy() tmp[indices] = new_value new_value = tmp diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 24407e5ded..8a5c1938be 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -16,6 +16,7 @@ from ctsm.netcdf_utils import get_netcdf_format from ctsm.param_utils import set_paramfile as sp from ctsm.param_utils.paramfile_shared import open_paramfile +from ctsm.param_utils.paramfile_shared import check_pfts_in_paramfile, get_selected_pft_indices # Allow names that pylint doesn't like, because otherwise I find it hard # to make readable unit test names @@ -464,6 +465,38 @@ def test_set_paramfile_changeparam_multidim_errors(self): with self.assertRaises(NotImplementedError): sp.main() + def test_set_paramfile_setparams_just_one_pft(self): + """Test changing just one PFT's value of something without dropping others""" + output_path = os.path.join(self.tempdir, "output.nc") + pft_to_include = "needleleaf_deciduous_boreal_tree" + this_var = "rswf_max" + new_value = 0.7 + + # Ensure it wasn't new_value before + ds_in = open_paramfile(PARAMFILE) + pft_names = check_pfts_in_paramfile([pft_to_include], ds_in) + pft_index = get_selected_pft_indices([pft_to_include], pft_names)[0] + self.assertFalse(ds_in[this_var].values[pft_index] == new_value) + + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "-p", + pft_to_include, + f"{this_var}={new_value}", + ] + sp.main() + + ds_out = open_paramfile(output_path) + for i, value in enumerate(ds_out[this_var]): + if i == pft_index: + self.assertTrue(value == new_value) + else: + self.assertTrue(value == ds_in[this_var].values[i]) + if __name__ == "__main__": unit_testing.setup_for_tests() From 7eb17f3ef0b9829fb55e0e3d7f02e157b0e41cfb Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 17:09:57 -0600 Subject: [PATCH 065/196] Reformat with black. --- python/ctsm/param_utils/paramfile_shared.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ctsm/param_utils/paramfile_shared.py b/python/ctsm/param_utils/paramfile_shared.py index a2788ac412..e4437b2012 100644 --- a/python/ctsm/param_utils/paramfile_shared.py +++ b/python/ctsm/param_utils/paramfile_shared.py @@ -24,6 +24,7 @@ def check_pfts_in_paramfile(selected_pfts, ds): return pft_names + def get_pft_names(ds): pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] return pft_names From 5462b0ed7656fb7b49b831908e2be62a1079a955 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 17:10:23 -0600 Subject: [PATCH 066/196] Add previous commit to .git-blame-ignore-revs. --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 203d7b487a..41886277ba 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -70,3 +70,4 @@ cdf40d265cc82775607a1bf25f5f527bacc97405 4ad46f46de7dde753b4653c15f05326f55116b73 75db098206b064b8b7b2a0604d3f0bf8fdb950cc 84609494b54ea9732f64add43b2f1dd035632b4c +7eb17f3ef0b9829fb55e0e3d7f02e157b0e41cfb From 187a8e42a88b295343bcd5213eed4a229494cea0 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 13 Aug 2025 17:21:22 -0600 Subject: [PATCH 067/196] Satisfy pylint. --- python/ctsm/param_utils/paramfile_shared.py | 76 +++++++++++++++++++- python/ctsm/param_utils/query_paramfile.py | 4 +- python/ctsm/test/test_sys_query_paramfile.py | 12 +++- python/ctsm/test/test_sys_set_paramfile.py | 2 - 4 files changed, 88 insertions(+), 6 deletions(-) diff --git a/python/ctsm/param_utils/paramfile_shared.py b/python/ctsm/param_utils/paramfile_shared.py index e4437b2012..abc73446c3 100644 --- a/python/ctsm/param_utils/paramfile_shared.py +++ b/python/ctsm/param_utils/paramfile_shared.py @@ -10,7 +10,24 @@ def check_pfts_in_paramfile(selected_pfts, ds): """ - Check that given PFTs are in parameter file + Check that the given PFTs are present in the parameter file. + + Parameters + ---------- + selected_pfts : list of str + List of PFT names to check. + ds : xarray.Dataset + The parameter file dataset. + + Returns + ------- + list of str + List of all PFT names in the file. + + Raises + ------ + KeyError + If any selected PFT is not found in the file, or if PFTNAME_VAR is missing. """ if PFTNAME_VAR not in ds: raise KeyError(f"paramfile missing variable: {PFTNAME_VAR}") @@ -26,20 +43,77 @@ def check_pfts_in_paramfile(selected_pfts, ds): def get_pft_names(ds): + """ + Get the list of PFT names from the parameter file dataset. + + Parameters + ---------- + ds : xarray.Dataset + The parameter file dataset. + + Returns + ------- + list of str + List of PFT names. + """ pft_names = [pft.decode().strip() for pft in ds[PFTNAME_VAR].values] return pft_names def get_selected_pft_indices(selected_pfts, pft_names): + """ + Get indices of selected PFTs in the list of all PFT names. + + Parameters + ---------- + selected_pfts : list of str + List of PFT names to select. + pft_names : list of str + List of all PFT names. + + Returns + ------- + list of int + Indices of selected PFTs. + """ indices = [i for i, name in enumerate(pft_names) if name in selected_pfts] return indices def open_paramfile(file_in, mask_and_scale=False): + """ + Open a parameter file as an xarray.Dataset. + + Parameters + ---------- + file_in : str + Path to the input netCDF file. + mask_and_scale : bool, optional + Whether to apply mask and scale (default: False). + + Returns + ------- + xarray.Dataset + The opened dataset. + """ return xr.open_dataset(file_in, decode_timedelta=False, mask_and_scale=mask_and_scale) def paramfile_parser_setup(description): + """ + Set up an argument parser for parameter file utilities. + + Parameters + ---------- + description : str + Description for the argument parser. + + Returns + ------- + tuple + (parser, pft_flags) where parser is an ArgumentParser and pft_flags is a list of flags for + PFT argument. + """ parser = argparse.ArgumentParser(description=description) parser.add_argument("-i", "--input", required=True, help="Input netCDF file") diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index ff5c5fd7bd..43ecaf3880 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -1,4 +1,6 @@ -import xarray as xr +""" +Query parameters in a CTSM paramfile +""" from ctsm.args_utils import comma_separated_list from ctsm.param_utils.paramfile_shared import paramfile_parser_setup diff --git a/python/ctsm/test/test_sys_query_paramfile.py b/python/ctsm/test/test_sys_query_paramfile.py index 87d7ba657c..dc73ab0edc 100755 --- a/python/ctsm/test/test_sys_query_paramfile.py +++ b/python/ctsm/test/test_sys_query_paramfile.py @@ -63,7 +63,11 @@ def test_query_paramfile_pft_selectnone(self): out = f.getvalue() self.assertRegex( out, - r"rswf_min:\n\s+not_vegetated\s*: 0\.25\n\s+needleleaf_evergreen_temperate_tree\s*: 0\.25\n.*", + ( + r"rswf_min:\n" + r"\s+not_vegetated\s*: 0\.25\n" + r"\s+needleleaf_evergreen_temperate_tree\s*: 0\.25\n.*" + ), ) def test_query_paramfile_pft_select2(self): @@ -84,7 +88,11 @@ def test_query_paramfile_pft_select2(self): out = f.getvalue() self.assertRegex( out, - r"rswf_min:\n\s+not_vegetated\s*: 0\.25\n\s+needleleaf_evergreen_temperate_tree\s*: 0\.25\n", + ( + r"rswf_min:\n" + r"\s+not_vegetated\s*: 0\.25\n" + r"\s+needleleaf_evergreen_temperate_tree\s*: 0\.25\n" + ), ) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 8a5c1938be..a7ace99054 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -9,7 +9,6 @@ import tempfile import numpy as np import xarray as xr -from netCDF4 import Dataset # pylint: disable=no-name-in-module from ctsm import unit_testing @@ -452,7 +451,6 @@ def test_set_paramfile_changeparam_multidim_errors(self): test will obviously need to be replaced once that functionality is added. """ output_path = os.path.join(self.tempdir, "output.nc") - pfts_to_include = ["not_vegetated", "needleleaf_evergreen_temperate_tree"] sys.argv = [ "set_paramfile", "-i", From 4e9d2fec36f2ae7517e4f2c192dd6d101c6982fc Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 14 Aug 2025 12:38:15 -0600 Subject: [PATCH 068/196] query_paramfiles: Use space-separated variables to match set_paramfile. --- python/ctsm/param_utils/query_paramfile.py | 4 ++-- python/ctsm/test/test_unit_query_paramfile.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index 43ecaf3880..81349e75a2 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -26,8 +26,8 @@ def get_arguments(): ) parser.add_argument( "variables", - help="Comma-separated list of variable names to extract", - type=comma_separated_list, + help="Names of variables to query", + nargs="*", ) parser.add_argument( *pft_flags, diff --git a/python/ctsm/test/test_unit_query_paramfile.py b/python/ctsm/test/test_unit_query_paramfile.py index 651dc16f70..b796253ace 100755 --- a/python/ctsm/test/test_unit_query_paramfile.py +++ b/python/ctsm/test/test_unit_query_paramfile.py @@ -50,7 +50,7 @@ def tearDown(self): def test_query_paramfile_args_short(self): """Test that all arguments can be set correctly with shortnames""" input_path = "/path/to/input.nc" - sys.argv = ["get_arguments", "-i", input_path, "-p", "pft1,pft2", "var1,var2"] + sys.argv = ["get_arguments", "-i", input_path, "-p", "pft1,pft2", "var1", "var2"] args = qp.get_arguments() self.assertEqual(input_path, args.input) self.assertEqual(["pft1", "pft2"], args.pft) @@ -59,7 +59,7 @@ def test_query_paramfile_args_short(self): def test_query_paramfile_args_long(self): """Test that all arguments can be set correctly with longnames""" input_path = "/path/to/input.nc" - sys.argv = ["get_arguments", "--input", input_path, "--pft", "pft1,pft2", "var1,var2"] + sys.argv = ["get_arguments", "--input", input_path, "--pft", "pft1,pft2", "var1", "var2"] args = qp.get_arguments() self.assertEqual(input_path, args.input) self.assertEqual(["pft1", "pft2"], args.pft) From c4d6ac65d266eb10e8eedd305049aa0e6db35f0d Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 14 Aug 2025 12:52:43 -0600 Subject: [PATCH 069/196] query_paramfile now prints all variables if none specified. --- python/ctsm/param_utils/query_paramfile.py | 4 +++ python/ctsm/test/test_sys_query_paramfile.py | 35 ++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index 81349e75a2..4791882012 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -77,6 +77,10 @@ def main(): ds = open_paramfile(args.input, mask_and_scale=True) + # If user didn't specify variables, print all + if not args.variables: + args.variables = ds.variables + selected_pfts = args.pft pft_names = None if selected_pfts: diff --git a/python/ctsm/test/test_sys_query_paramfile.py b/python/ctsm/test/test_sys_query_paramfile.py index dc73ab0edc..da89820ac2 100755 --- a/python/ctsm/test/test_sys_query_paramfile.py +++ b/python/ctsm/test/test_sys_query_paramfile.py @@ -7,6 +7,9 @@ import sys import io from contextlib import redirect_stdout +import tempfile +import shutil +import xarray as xr from ctsm import unit_testing @@ -26,9 +29,11 @@ class TestSysQueryParamfile(unittest.TestCase): def setUp(self): self.orig_argv = sys.argv + self.tempdir = tempfile.mkdtemp() def tearDown(self): sys.argv = self.orig_argv + shutil.rmtree(self.tempdir, ignore_errors=True) def test_query_paramfile_scalar_nopfts(self): """Test that print_values works with scalar parameter and no PFTs specified""" @@ -95,6 +100,36 @@ def test_query_paramfile_pft_select2(self): ), ) + def test_query_paramfile_no_variables_fake(self): + """ + Test that print_values prints every variable when no variables are given. Use a small fake + paramfile so we can check that what gets printed is what we expect. + """ + + fake_da1 = xr.DataArray(data=[1, 2, 3]) + fake_da2 = xr.DataArray(data=[4, 5, 6]) + fake_ds = xr.Dataset(data_vars={"fake1": fake_da1, "fake2": fake_da2}) + fake_nc_path = os.path.join(self.tempdir, "fake_da.nc") + fake_ds.to_netcdf(fake_nc_path) + + sys.argv = [ + "query_paramfile", + "-i", + fake_nc_path, + ] + + f = io.StringIO() + with redirect_stdout(f): + qp.main() + out = f.getvalue() + self.assertRegex( + out, + ( + r"fake1: \[1 2 3\]\n" + r"fake2: \[4 5 6\]\n" + ) + ) + if __name__ == "__main__": unit_testing.setup_for_tests() From 4019a5a42c176cc22f9ff95271ff50e18374aba7 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 14 Aug 2025 13:02:20 -0600 Subject: [PATCH 070/196] query_paramfile: Handle multi-dim vars. --- python/ctsm/param_utils/query_paramfile.py | 5 ++++- python/ctsm/test/test_sys_query_paramfile.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/python/ctsm/param_utils/query_paramfile.py b/python/ctsm/param_utils/query_paramfile.py index 4791882012..6c2bcc1440 100644 --- a/python/ctsm/param_utils/query_paramfile.py +++ b/python/ctsm/param_utils/query_paramfile.py @@ -54,7 +54,7 @@ def print_values(ds, var, selected_pfts, pft_names): List of all PFT names in the file. """ data = ds[var].values - if PFTNAME_VAR in ds[var].coords: + if list(ds[var].dims) == ["pft"]: print(var + ":") indices = range(len(pft_names)) if selected_pfts is not None: @@ -64,6 +64,9 @@ def print_values(ds, var, selected_pfts, pft_names): max_name_len = max(len(name) for name in pft_names) for p in indices: print(f" {pft_names[p]:<{max_name_len}}: {data[p]}") + elif ds[var].ndim > 1: + print(f"{var}:") + print(data) else: print(f"{var}: {data}") diff --git a/python/ctsm/test/test_sys_query_paramfile.py b/python/ctsm/test/test_sys_query_paramfile.py index da89820ac2..b2031a8fdd 100755 --- a/python/ctsm/test/test_sys_query_paramfile.py +++ b/python/ctsm/test/test_sys_query_paramfile.py @@ -130,6 +130,23 @@ def test_query_paramfile_no_variables_fake(self): ) ) + def test_query_paramfile_no_variables_real(self): + """ + Test that query_paramfile doesn't error when trying to print every variable from a real + paramfile. Don't actually check that it matches what we expect; that's done in + test_query_paramfile_no_variables_fake. + """ + + sys.argv = [ + "query_paramfile", + "-i", + PARAMFILE, + ] + + f = io.StringIO() + with redirect_stdout(f): + qp.main() + if __name__ == "__main__": unit_testing.setup_for_tests() From ae227035ec6ef6ac79f2196c683f02aff92e0f47 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 14 Aug 2025 13:13:29 -0600 Subject: [PATCH 071/196] set_paramfile sys tests: Explicitly check that a 'nan' is saved as the fill value. --- python/ctsm/test/test_sys_set_paramfile.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index a7ace99054..f1683b8fa2 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -343,8 +343,19 @@ def test_set_paramfile_setparams_scalar_double_tonan_with_nan(self): sp.main() self.assertTrue(os.path.exists(output_path)) + + # Check that it's NaN after considering the fill value ds_out = open_paramfile(output_path, mask_and_scale=True) self.assertTrue(np.isnan(ds_out[this_var])) + fill_value = ds_out[this_var].encoding["_FillValue"] + + # Check that it's literally the fill value + ds_out = open_paramfile(output_path, mask_and_scale=False) + var_value = ds_out[this_var].values + if np.isnan(fill_value): + self.assertTrue(np.isnan(var_value)) + else: + self.assertEqual(var_value, fill_value) def test_set_paramfile_setparams_scalar_double_tonan_with_nancaps(self): """Test setting scalar double to NaN using 'NaN'""" From 79059bcf596351e301414c34ca51828f6343afd5 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 14 Aug 2025 14:25:49 -0600 Subject: [PATCH 072/196] set_paramfile: Fix a comment. --- python/ctsm/param_utils/set_paramfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index f2d9137037..20484b0e40 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -148,7 +148,7 @@ def main(): # TODO: Add code to set integer variables to NaN (this might not be possible) if is_integer(ds_in[var].values): raise NotImplementedError(f"Not able to set NaN for integer parameters: {chg}") - # TODO: Add code to set integer variables to NaN (this might not be possible) + # TODO: Add code to add fill value to parameters without it if "_FillValue" not in ds_in_masked_scaled[var].encoding: raise NotImplementedError( f"Not able to set NaN if parameter doesn't already have fill value: {chg}" From 65daa8543798e2f9dff137fa0bb0830fa5e85b4b Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 14 Aug 2025 14:34:51 -0600 Subject: [PATCH 073/196] set_paramfile now preserves non-NaN _FillValue. --- python/ctsm/param_utils/set_paramfile.py | 24 ++++--- python/ctsm/test/test_sys_set_paramfile.py | 80 ++++++++++++++++++---- 2 files changed, 80 insertions(+), 24 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 20484b0e40..793166f22b 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -143,17 +143,11 @@ def main(): else: new_value = np.array(new_value) - # Some NaN handling isn't yet implemented - if np.any(np.char.lower(new_value) == "nan"): - # TODO: Add code to set integer variables to NaN (this might not be possible) - if is_integer(ds_in[var].values): - raise NotImplementedError(f"Not able to set NaN for integer parameters: {chg}") - # TODO: Add code to add fill value to parameters without it - if "_FillValue" not in ds_in_masked_scaled[var].encoding: - raise NotImplementedError( - f"Not able to set NaN if parameter doesn't already have fill value: {chg}" - ) + # TODO: Add code to set integer variables to NaN (this might not be possible) + if np.any(np.char.lower(new_value) == "nan") and is_integer(ds_in[var].values): + raise NotImplementedError(f"Not able to set NaN for integer parameters: {chg}") + # Convert to the output data type new_value = new_value.astype(type(ds_out[var].dtype)) # Are we acting on just some PFTs? If so, we'll need some stuff. @@ -177,6 +171,16 @@ def main(): tmp[indices] = new_value new_value = tmp + # Ensure that any NaNs are replaced with the fill value + if any(np.isnan(np.atleast_1d(new_value))): + # TODO: Add code to add fill value to parameters without it + if "_FillValue" not in ds_in_masked_scaled[var].encoding: + raise NotImplementedError( + f"Not able to set NaN if parameter doesn't already have fill value: {chg}" + ) + fill_value = ds_in_masked_scaled[var].encoding["_FillValue"] + new_value[np.isnan(new_value)] = fill_value + ds_out[var].values = new_value ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input)) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index f1683b8fa2..4c94846c21 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -328,34 +328,86 @@ def test_set_paramfile_extractpfts_changeparam_int(self): else: self.assertTrue(ds_in[var].equals(ds_out[var])) - def test_set_paramfile_setparams_scalar_double_tonan_with_nan(self): - """Test setting scalar double to NaN using 'nan'""" + def test_set_paramfile_fill_value_scalar_double_nan(self): + """ + Test that setting scalar double to fill value writes a literal NaN if that's the _FillValue + """ + # Create paramfile with a double variable with fill value NaN + input_path = os.path.join(self.tempdir, "input.nc") + ds = open_paramfile(PARAMFILE, mask_and_scale=True) + new_param_name = "new_param_abc123" + ds[new_param_name] = xr.DataArray(data=np.array(3.14)) + ds[new_param_name].encoding["_FillValue"] = np.nan + self.assertTrue(new_param_name in ds) + ds.to_netcdf(input_path) + + # Check that its fill value is NaN + ds_in = open_paramfile(input_path, mask_and_scale=True) + self.assertTrue("_FillValue" in ds_in[new_param_name].encoding) + self.assertTrue(np.isnan(ds_in[new_param_name].encoding["_FillValue"])) + + # Ask to set it to the FillValue output_path = os.path.join(self.tempdir, "output.nc") - this_var = "a_coef" sys.argv = [ "set_paramfile", "-i", - PARAMFILE, + input_path, "-o", output_path, - f"{this_var}=nan", + f"{new_param_name}=nan", ] + sp.main() + self.assertTrue(os.path.exists(output_path)) + + # Ensure it wrote a literal NaN + ds_out = open_paramfile(output_path, mask_and_scale=False) + self.assertTrue(np.isnan(ds_out[new_param_name])) + + # Ensure it preserved NaN FillValue + ds_out = open_paramfile(output_path, mask_and_scale=True) + self.assertTrue(np.isnan(ds_out[new_param_name].encoding["_FillValue"])) + def test_set_paramfile_fill_value_scalar_double_real(self): + """ + Test that setting scalar double to fill value does NOT write NaN if that's not the + _FillValue + """ + # Create paramfile with a double variable with fill value -999.9 + input_path = os.path.join(self.tempdir, "input.nc") + ds = open_paramfile(PARAMFILE, mask_and_scale=True) + new_param_name = "new_param_abc123" + ds[new_param_name] = xr.DataArray(data=np.array(3.14)) + fill_value = -999.9 + ds[new_param_name].encoding["_FillValue"] = fill_value + self.assertTrue(new_param_name in ds) + ds.to_netcdf(input_path) + + # Check that its fill value is what we asked for + ds_in = open_paramfile(input_path, mask_and_scale=True) + self.assertTrue("_FillValue" in ds_in[new_param_name].encoding) + self.assertEqual(fill_value, ds_in[new_param_name].encoding["_FillValue"]) + + # Ask to set it to the FillValue + output_path = os.path.join(self.tempdir, "output.nc") + sys.argv = [ + "set_paramfile", + "-i", + input_path, + "-o", + output_path, + f"{new_param_name}=nan", + ] sp.main() self.assertTrue(os.path.exists(output_path)) - # Check that it's NaN after considering the fill value + # Ensure it preserved FillValue ds_out = open_paramfile(output_path, mask_and_scale=True) - self.assertTrue(np.isnan(ds_out[this_var])) - fill_value = ds_out[this_var].encoding["_FillValue"] + self.assertEqual(fill_value, ds_out[new_param_name].encoding["_FillValue"]) + self.assertTrue(np.isnan(ds_out[new_param_name].values)) - # Check that it's literally the fill value + # Ensure it wrote the FillValue and not a literal NaN ds_out = open_paramfile(output_path, mask_and_scale=False) - var_value = ds_out[this_var].values - if np.isnan(fill_value): - self.assertTrue(np.isnan(var_value)) - else: - self.assertEqual(var_value, fill_value) + self.assertEqual(fill_value, ds_out[new_param_name].values) def test_set_paramfile_setparams_scalar_double_tonan_with_nancaps(self): """Test setting scalar double to NaN using 'NaN'""" From 9f5e4b4d18144a168f18d55eec7da00d79d79bdb Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 14 Aug 2025 15:58:51 -0600 Subject: [PATCH 074/196] Fix subset_data when selecting just 1 pft. --- python/ctsm/param_utils/set_paramfile.py | 20 ++++++- python/ctsm/test/test_sys_set_paramfile.py | 65 ++++++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 793166f22b..aa05c9c25b 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -103,6 +103,13 @@ def check_correct_ndims(da, new_value, throw_error=False): return is_ndim_correct +def drop_other_pfts(selected_pfts, ds): + pft_names = check_pfts_in_paramfile(selected_pfts, ds) + indices = get_selected_pft_indices(selected_pfts, pft_names) + ds = ds.isel({"pft": indices}) + return ds + + def main(): """ Main entry point for the script. @@ -117,9 +124,7 @@ def main(): # If --drop-other-pfts was given, drop PFTs not in args.pft if args.drop_other_pfts: - pft_names = check_pfts_in_paramfile(args.pft, ds_out) - indices = get_selected_pft_indices(args.pft, pft_names) - ds_out = ds_out.isel({"pft": indices}) + ds_out = drop_other_pfts(args.pft, ds_out) # If any variables specified, drop others if args.variables: @@ -155,6 +160,11 @@ def main(): if just_some_pfts: pft_names = check_pfts_in_paramfile(args.pft, ds_out) indices = get_selected_pft_indices(args.pft, pft_names) + else: + # pylint is probably wrong with the possibly-used-before-assignment warning, but do this + # here just to placate it. Make it an invalid index so we get an error if we try to use + # it. + indices = -1 # Check that correct number of dimensions were given for new values. Special handling needed # if we're just acting on one PFT. @@ -181,6 +191,10 @@ def main(): fill_value = ds_in_masked_scaled[var].encoding["_FillValue"] new_value[np.isnan(new_value)] = fill_value + # This can happen if, e.g., you're selecting and changing just one PFT + if ds_in[var].values.ndim > 0 and new_value.ndim == 0: + new_value = np.atleast_1d(new_value) + ds_out[var].values = new_value ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input)) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 4c94846c21..5921f94072 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -558,6 +558,71 @@ def test_set_paramfile_setparams_just_one_pft(self): else: self.assertTrue(value == ds_in[this_var].values[i]) + def test_set_paramfile_setparams_just_one_pft_dropothers_noset(self): + """Test dropping all but one PFT without changing any parameters""" + output_path = os.path.join(self.tempdir, "output.nc") + pft_to_include = "needleleaf_deciduous_boreal_tree" + + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "-p", + pft_to_include, + "--drop-other-pfts", + ] + sp.main() + self.assertTrue(os.path.exists(output_path)) + + # Check that the file is just what you get if you drop all but the one PFT + ds_in = open_paramfile(PARAMFILE) + ds_in_1pft = sp.drop_other_pfts([pft_to_include], ds_in) + ds_out = open_paramfile(output_path) + self.assertTrue(ds_in_1pft.equals(ds_out)) + self.assertEqual(ds_in_1pft.sizes["pft"], 1) + self.assertEqual(ds_out.sizes["pft"], 1) + + + def test_set_paramfile_setparams_just_one_pft_dropothers_doset(self): + """Test dropping all but one PFT, changing one parameter""" + output_path = os.path.join(self.tempdir, "output.nc") + pft_to_include = "needleleaf_deciduous_boreal_tree" + this_var = "rswf_max" + new_value = 0.7 + + # Ensure it wasn't new_value before + ds_in = open_paramfile(PARAMFILE) + pft_names = check_pfts_in_paramfile([pft_to_include], ds_in) + pft_index = get_selected_pft_indices([pft_to_include], pft_names)[0] + self.assertFalse(ds_in[this_var].values[pft_index] == new_value) + + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + "-p", + pft_to_include, + "--drop-other-pfts", + f"{this_var}={new_value}", + ] + sp.main() + self.assertTrue(os.path.exists(output_path)) + + # Check that all variables match except for the one we changed + ds_out = open_paramfile(output_path) + ds_in_1pft = sp.drop_other_pfts([pft_to_include], ds_in) + for var in ds_in.variables: + da_in = ds_in_1pft[var] + da_out = ds_out[var] + if var == this_var: + self.assertFalse(da_in.equals(da_out)) + else: + self.assertTrue(da_in.equals(da_out)) + if __name__ == "__main__": unit_testing.setup_for_tests() From e2edd6b824308247e0ac462462c6ed947869170d Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 14 Aug 2025 16:49:42 -0600 Subject: [PATCH 075/196] Add more tests for more of the options --- src/main/test/endrun_test/test_endrun.pf | 55 +++++++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf index a6b7c19647..3d8f46782e 100644 --- a/src/main/test/endrun_test/test_endrun.pf +++ b/src/main/test/endrun_test/test_endrun.pf @@ -5,6 +5,8 @@ module test_endrun use funit use abortutils use unittestUtils, only : endrun_msg + use shr_kind_mod, only : CL => shr_kind_cl + use clm_varctl, only : iulog implicit none @@ -35,13 +37,62 @@ contains ! ======================================================================== @Test - subroutine endrun_vanilla_aborts(this) + subroutine endrun_plain_vanilla_aborts(this) ! Test vanilla operation of endrun class(TestAbortUtils), intent(inout) :: this call endrun() @assertExceptionRaised(endrun_msg('')) - end subroutine endrun_vanilla_aborts + end subroutine endrun_plain_vanilla_aborts + + @Test + subroutine endrun_msg_vanilla_aborts(this) + ! Test vanilla operation of endrun with a message sent in + class(TestAbortUtils), intent(inout) :: this + character(len=CL) :: msg = "test_message" + + call endrun( msg = msg) + @assertExceptionRaised(endrun_msg(msg)) + + end subroutine endrun_msg_vanilla_aborts + + @Test + subroutine endrun_addmsg_vanilla_aborts(this) + ! Test vanilla operation of endrun with an additional message sent in + class(TestAbortUtils), intent(inout) :: this + character(len=CL) :: msg = "test_message" + character(len=CL) :: add_msg = "additional_test_message" + + call endrun(msg=msg, additional_msg=add_msg) + @assertExceptionRaised(endrun_msg(msg)) + + end subroutine endrun_addmsg_vanilla_aborts + + @Test + subroutine endrun_addmsg_pt_context_aborts(this) + use decompMod, only : subgrid_level_lndgrid, subgrid_level_gridcell + use decompMod, only : subgrid_level_landunit, subgrid_level_column, subgrid_level_patch + use decompMod, only : subgrid_level_cohort + use unittestSimpleSubgridSetupsMod, only : setup_single_veg_patch + ! Test pt_context operation of endrun with an additional message sent in + class(TestAbortUtils), intent(inout) :: this + character(len=CL) :: msg = "test_message" + character(len=CL) :: add_msg = "additional_test_message" + integer :: p = 1, l + integer, parameter :: nlevel = 6 + integer :: subgrid_lvl(nlevel) = (/ subgrid_level_lndgrid, subgrid_level_gridcell, & + subgrid_level_landunit, subgrid_level_column, subgrid_level_patch, & + subgrid_level_cohort /) + + call setup_single_veg_patch(pft_type=1) + ! Loop over all the subgrid level types + do l = 2, nlevel-1 + write(iulog,*) 'level = ', l + call endrun(subgrid_index=p, subgrid_level=subgrid_lvl(l), msg=msg, additional_msg=add_msg) + @assertExceptionRaised(endrun_msg(msg)) + end do + + end subroutine endrun_addmsg_pt_context_aborts end module test_endrun From a6011fbc016203c8974a197b1387d65fd2d4983e Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 10:47:42 -0600 Subject: [PATCH 076/196] set_paramfile: Explain nan in help text. --- python/ctsm/param_utils/paramfile_shared.py | 4 +++- python/ctsm/param_utils/set_paramfile.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/python/ctsm/param_utils/paramfile_shared.py b/python/ctsm/param_utils/paramfile_shared.py index abc73446c3..19529e6850 100644 --- a/python/ctsm/param_utils/paramfile_shared.py +++ b/python/ctsm/param_utils/paramfile_shared.py @@ -114,7 +114,9 @@ def paramfile_parser_setup(description): (parser, pft_flags) where parser is an ArgumentParser and pft_flags is a list of flags for PFT argument. """ - parser = argparse.ArgumentParser(description=description) + parser = argparse.ArgumentParser( + description=description, formatter_class=argparse.RawTextHelpFormatter + ) parser.add_argument("-i", "--input", required=True, help="Input netCDF file") # Flags that can be used for the PFT argument diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index aa05c9c25b..db25011041 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -69,7 +69,10 @@ def get_arguments(): parser.add_argument( "param_changes", - help="Parameter changes to apply. E.g.: param1=new_value1 pftparam=pft1_val,pft2_val,...", + help=( + "Parameter changes to apply. Use nan to set to the fill value. E.g.:\n" + " param1=new_value1 pftparam=pft1_val,nan,... param3=nan" + ), nargs="*", ) From 042f5f4c65c90d8a69bfedcd96b6327d53296847 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 13:12:38 -0600 Subject: [PATCH 077/196] netcdf_utils: Add are_xr_dataarrays_identical(). --- python/ctsm/netcdf_utils.py | 61 ++++++ python/ctsm/test/test_unit_netcdf_utils.py | 215 ++++++++++++++++++++- 2 files changed, 273 insertions(+), 3 deletions(-) diff --git a/python/ctsm/netcdf_utils.py b/python/ctsm/netcdf_utils.py index 64f554dc69..f9f9a927bf 100644 --- a/python/ctsm/netcdf_utils.py +++ b/python/ctsm/netcdf_utils.py @@ -2,6 +2,8 @@ Helper functions for working with netCDF files """ +import numpy as np +import xarray as xr from netCDF4 import Dataset # pylint: disable=no-name-in-module @@ -12,3 +14,62 @@ def get_netcdf_format(file_path): with Dataset(file_path, "r") as netcdf_file: netcdf_format = netcdf_file.data_model return netcdf_format + + +def are_xr_dataarrays_identical(da0: xr.DataArray, da1: xr.DataArray): + """ + Comprehensively check whether two DataArrays are identical + """ + # pylint: disable=too-many-return-statements + + # Check data type + if da0.dtype != da1.dtype: + return False + + # Check encoding + if da0.encoding != da1.encoding: + return False + + # Check attributes + if da0.attrs != da1.attrs: + return False + + # Check name + if da0.name != da1.name: + return False + + # Check dims + if da0.dims != da1.dims: + return False + + # Check sizes + if da0.sizes != da1.sizes: + return False + + # Check coordinates + if bool(da0.coords) or bool(da1.coords): + if not bool(da0.coords) or not bool(da1.coords): + return False + if not da0.coords.equals(da1.coords): + return False + + # Check values ("The array's data converted to numpy.ndarray") + if not np.array_equal(da0.values, da1.values): + # Try-except to avoid TypeError from putting NaN-incapable dtypes through + # np.array_equal(..., equal_nan=True) + try: + if not np.array_equal(da0.values, da1.values, equal_nan=True): + return False + except TypeError: + return False + + # Check data ("The DataArray's data as an array. The underlying array type (e.g. dask, sparse, + # pint) is preserved.") + da0_data_type = type(da0.data) + if not isinstance(da1.data, da0_data_type): + return False + if not isinstance(da0.data, np.ndarray): + raise NotImplementedError(f"Add support for comparing two objects of type {da0_data_type}") + + # Fallback to however xarray defines equality, in case we missed something above + return da0.equals(da1) diff --git a/python/ctsm/test/test_unit_netcdf_utils.py b/python/ctsm/test/test_unit_netcdf_utils.py index 4c904564fb..cd420709aa 100755 --- a/python/ctsm/test/test_unit_netcdf_utils.py +++ b/python/ctsm/test/test_unit_netcdf_utils.py @@ -8,6 +8,8 @@ import unittest import shutil import tempfile +import numpy as np +import pandas as pd import xarray as xr # -- add python/ctsm to path (needed if we want to run the test stand-alone) @@ -15,7 +17,7 @@ sys.path.insert(1, _CTSM_PYTHON) # pylint: disable=wrong-import-position -from ctsm.netcdf_utils import get_netcdf_format +import ctsm.netcdf_utils as nu from ctsm import unit_testing # pylint: disable=invalid-name @@ -41,7 +43,7 @@ def test_get_netcdf_format_classic(self): """ nc_format = "NETCDF3_CLASSIC" self.ds.to_netcdf(self.outfile, format=nc_format) - self.assertEqual(get_netcdf_format(self.outfile), nc_format) + self.assertEqual(nu.get_netcdf_format(self.outfile), nc_format) def test_get_netcdf_format_netcdf4(self): """ @@ -49,7 +51,214 @@ def test_get_netcdf_format_netcdf4(self): """ nc_format = "NETCDF4" self.ds.to_netcdf(self.outfile, format=nc_format) - self.assertEqual(get_netcdf_format(self.outfile), nc_format) + self.assertEqual(nu.get_netcdf_format(self.outfile), nc_format) + + +class TestUnitAreXrDataArraysIdentical(unittest.TestCase): + """ + Unit tests for are_xr_dataarrays_identical + """ + + def test_are_xr_dataarrays_identical_data_types_match(self): + """Should be true if dtypes match""" + da0 = xr.DataArray(int(1)) + da1 = xr.DataArray(int(1)) + self.assertTrue(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_data_types_differ(self): + """Should be false if dtypes differ""" + da0 = xr.DataArray(int(1)) + da1 = xr.DataArray(np.float64(1)) + self.assertFalse(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_data_types_differ_precision(self): + """Should be false if dtypes differ only in their precision""" + da0 = xr.DataArray(np.float64(1)) + da1 = xr.DataArray(np.float32(1)) + self.assertFalse(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_encodings_match(self): + """Should be true if encodings are the exact same""" + da0 = xr.DataArray(int(1)) + da1 = xr.DataArray(int(1)) + da0.encoding["dummy0"] = -999 + da1.encoding["dummy0"] = -999 + da0.encoding["dummy1"] = -999 + da1.encoding["dummy1"] = -999 + self.assertTrue(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_encodings_match_except_order(self): + """Should be true if encodings are the same as long as you don't care about order""" + da0 = xr.DataArray(int(1)) + da1 = xr.DataArray(int(1)) + da0.encoding["dummy0"] = -999 + da0.encoding["dummy1"] = -999 + da1.encoding["dummy1"] = -999 + da1.encoding["dummy0"] = -999 + self.assertTrue(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_encodings_differ_number(self): + """Should be false if encodings have a different number of keys""" + da0 = xr.DataArray(int(1)) + da1 = xr.DataArray(int(1)) + da0.encoding["dummy0"] = -999 + da1.encoding["dummy0"] = -999 + da1.encoding["dummy1"] = -999 + self.assertFalse(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_encodings_differ(self): + """Should be false if encodings have the same number and order of keys but not values""" + da0 = xr.DataArray(int(1)) + da1 = xr.DataArray(int(1)) + da0.encoding["dummy0"] = -999 + da1.encoding["dummy0"] = -999 + da0.encoding["dummy1"] = -998 + da1.encoding["dummy1"] = -999 + self.assertFalse(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_attributes_match(self): + """Should be true if attributes are the exact same""" + da0 = xr.DataArray(int(1)) + da1 = xr.DataArray(int(1)) + da0.attrs["dummy0"] = -999 + da1.attrs["dummy0"] = -999 + da0.attrs["dummy1"] = -999 + da1.attrs["dummy1"] = -999 + self.assertTrue(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_attributes_match_nan(self): + """Should be true if attributes are the exact same and NaN""" + da0 = xr.DataArray(1.0) + da1 = xr.DataArray(1.0) + da0.attrs["_FillValue"] = np.float64(np.nan) + da1.attrs["_FillValue"] = np.float64(np.nan) + self.assertTrue(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_attributes_match_except_order(self): + """Should be true if attributes are the same as long as you don't care about order""" + da0 = xr.DataArray(int(1)) + da1 = xr.DataArray(int(1)) + da0.attrs["dummy0"] = -999 + da0.attrs["dummy1"] = -999 + da1.attrs["dummy1"] = -999 + da1.attrs["dummy0"] = -999 + self.assertTrue(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_attributes_differ_number(self): + """Should be false if attributes have a different number of keys""" + da0 = xr.DataArray(int(1)) + da1 = xr.DataArray(int(1)) + da0.attrs["dummy0"] = -999 + da1.attrs["dummy0"] = -999 + da1.attrs["dummy1"] = -999 + self.assertFalse(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_attributes_differ(self): + """Should be false if attributes have the same number and order of keys but not values""" + da0 = xr.DataArray(int(1)) + da1 = xr.DataArray(int(1)) + da0.attrs["dummy0"] = -999 + da1.attrs["dummy0"] = -999 + da0.attrs["dummy1"] = -998 + da1.attrs["dummy1"] = -999 + self.assertFalse(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_values_ndarrays_match(self): + """Should be true if values match and they're both numpy arrays under the hood""" + da0 = xr.DataArray(np.array(int(1))) + da1 = xr.DataArray(np.array(int(1))) + self.assertTrue(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_values_ndarrays_differ(self): + """Should be false if values differ and they're both numpy arrays under the hood""" + da0 = xr.DataArray(np.array(int(1))) + da1 = xr.DataArray(np.array(int(2))) + self.assertFalse(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_coords_match(self): + """Should be true if coordinates match""" + time = pd.date_range("2000-01-01", periods=3) + da0 = xr.DataArray( + dims=["time"], + coords={"time": time}, + ) + da1 = xr.DataArray( + dims=["time"], + coords={"time": time}, + ) + self.assertTrue(nu.are_xr_dataarrays_identical(da0, da1)) + + # To make sure we don't need a separate "test_are_xr_dataarrays_identical_indexes_match" + self.assertEqual(len(da0.indexes), len(da1.indexes)) + for key in da0.indexes: + self.assertTrue(da0.indexes[key].equals(da1.indexes.get(key))) + + def test_are_xr_dataarrays_identical_coords_onemissing(self): + """Should be false if only one has coords""" + data = [1, 2, 3] + time = pd.date_range("2000-01-01", periods=len(data)) + da0 = xr.DataArray( + data=data, + dims=["time"], + coords={"time": time}, + ) + da1 = xr.DataArray( + data=data, + dims=["time"], + ) + self.assertFalse(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_coords_differ(self): + """Should be false if coordinates differ""" + time0 = pd.date_range("2000-01-01", periods=3) + da0 = xr.DataArray( + dims=["time"], + coords={"time": time0}, + ) + time1 = pd.date_range("1987-01-01", periods=3) + da1 = xr.DataArray( + dims=["time"], + coords={"time": time1}, + ) + self.assertFalse(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_dims_match(self): + """Should be true if dimensions match""" + da0 = xr.DataArray(data=[1], dims=["dim"]) + da1 = xr.DataArray(data=[1], dims=["dim"]) + self.assertTrue(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_dims_differ(self): + """Should be false if dimensions differ""" + da0 = xr.DataArray(data=[1], dims=["dim0"]) + da1 = xr.DataArray(data=[1], dims=["dim1"]) + self.assertFalse(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_names_match(self): + """Should be true if names match""" + da0 = xr.DataArray(data=[1], name="da_name") + da1 = xr.DataArray(data=[1], name="da_name") + self.assertTrue(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_names_differ(self): + """Should be false if names differ""" + da0 = xr.DataArray(data=[1], name="da0_name") + da1 = xr.DataArray(data=[1], name="da1_name") + self.assertFalse(nu.are_xr_dataarrays_identical(da0, da1)) + + def test_are_xr_dataarrays_identical_sizes_differ(self): + """Should be false if sizes differ""" + da0 = xr.DataArray(data=[1]) + da1 = xr.DataArray(data=[1, 1]) + self.assertFalse(nu.are_xr_dataarrays_identical(da0, da1)) + + # Waiting on dask, sparse, or pint to be in ctsm_pylib: + # TODO: False if data types don't match + # TODO: NotImplementedError if data types match but aren't np.ndarray + + # Waiting on dask to be in ctsm_pylib: + # TODO: True if the only difference is chunked or not + # TODO: True if the only difference is chunk sizes if __name__ == "__main__": From 9552803e2895f8cbf12b4fc393d548e711872bf6 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 15:14:40 -0600 Subject: [PATCH 078/196] Use are_xr_dataarrays_identical() in set_paramfile system tests. Failing. --- python/ctsm/test/test_sys_set_paramfile.py | 52 +++++++++++++++++----- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 5921f94072..f59af44620 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -12,7 +12,7 @@ from ctsm import unit_testing -from ctsm.netcdf_utils import get_netcdf_format +from ctsm.netcdf_utils import get_netcdf_format, are_xr_dataarrays_identical from ctsm.param_utils import set_paramfile as sp from ctsm.param_utils.paramfile_shared import open_paramfile from ctsm.param_utils.paramfile_shared import check_pfts_in_paramfile, get_selected_pft_indices @@ -83,6 +83,30 @@ def test_set_paramfile_extractvars(self): # Check that both are the same kind of netCDF self.assertEqual(get_netcdf_format(PARAMFILE), get_netcdf_format(output_path)) + def test_set_paramfile_copy_without_adding_fillvalue(self): + """Test that set_paramfile can copy to a new file without adding _FillValue""" + input_path = os.path.join(self.tempdir, "input.nc") + output_path = os.path.join(self.tempdir, "output.nc") + param_name = "param0" + + # Save a test paramfile without _FillValue + da = xr.DataArray(data=np.float64([1, 2, 3])) + da.encoding["_FillValue"] = None + ds = xr.Dataset(data_vars={param_name: da}) + ds.to_netcdf(input_path, encoding={param_name: {"_FillValue": None}}) + ds_in = open_paramfile(input_path) + self.assertFalse("_FillValue" in ds_in[param_name].encoding) + self.assertFalse("_FillValue" in ds_in[param_name].attrs) + + # Use set_paramfile to copy to new file + sys.argv = ["set_paramfile", "-i", input_path, "-o", output_path] + sp.main() + + # Check that _FillValue wasn't added + ds_out = open_paramfile(output_path) + self.assertFalse("_FillValue" in ds_out[param_name].encoding) + self.assertFalse("_FillValue" in ds_out[param_name].attrs) + def test_set_paramfile_extractpfts(self): """Test that set_paramfile can copy to a new file with only some requested PFTs""" output_path = os.path.join(self.tempdir, "output.nc") @@ -104,10 +128,12 @@ def test_set_paramfile_extractpfts(self): # Check that included variables/coords match for var in ds_in.variables: + actual = ds_out[var] if sp.PFTNAME_VAR in ds_in[var].coords: - self.assertTrue(ds_in[var].isel(pft=[0, 1]).equals(ds_out[var])) + expected = ds_in[var].isel(pft=[0, 1]) else: - self.assertTrue(ds_in[var].equals(ds_out[var])) + expected = ds_in[var] + self.assertTrue(are_xr_dataarrays_identical(expected, actual)) def test_set_paramfile_changeparams_scalar_errors_given_list(self): """Test that set_paramfile errors if given a list for a scalar parameter""" @@ -164,7 +190,7 @@ def test_set_paramfile_changeparams_scalar_double(self): self.assertTrue(ds_in[var].values == 11) self.assertTrue(ds_out[var].values == 87) else: - self.assertTrue(ds_in[var].equals(ds_out[var])) + self.assertTrue(are_xr_dataarrays_identical(ds_in[var], ds_out[var])) # Check that data type hasn't changed self.assertTrue(ds_in[var].dtype == ds_out[var].dtype) @@ -285,9 +311,9 @@ def test_set_paramfile_changeparam_dbl_onlysomepfts(self): this_slice = slice(2, None) expected = ds_in["xl"].isel(pft=this_slice) result = ds_out["xl"].isel(pft=this_slice) - self.assertTrue(expected.equals(result)) + self.assertTrue(are_xr_dataarrays_identical(expected, result)) else: - self.assertTrue(ds_in[var].equals(ds_out[var])) + self.assertTrue(are_xr_dataarrays_identical(ds_in[var], ds_out[var])) def test_set_paramfile_extractpfts_changeparam_int(self): """ @@ -324,9 +350,11 @@ def test_set_paramfile_extractpfts_changeparam_int(self): if var == this_var: self.assertTrue(np.array_equal(np.array([1986, 1987]), ds_out[var].values)) elif sp.PFTNAME_VAR in ds_in[var].coords: - self.assertTrue(ds_in[var].isel(pft=[0, 1]).equals(ds_out[var])) + self.assertTrue( + are_xr_dataarrays_identical(ds_in[var].isel(pft=[0, 1]), ds_out[var]) + ) else: - self.assertTrue(ds_in[var].equals(ds_out[var])) + self.assertTrue(are_xr_dataarrays_identical(ds_in[var], ds_out[var])) def test_set_paramfile_fill_value_scalar_double_nan(self): """ @@ -580,11 +608,13 @@ def test_set_paramfile_setparams_just_one_pft_dropothers_noset(self): ds_in = open_paramfile(PARAMFILE) ds_in_1pft = sp.drop_other_pfts([pft_to_include], ds_in) ds_out = open_paramfile(output_path) + self.assertTrue(set(ds_in_1pft.variables) == set(ds_out.variables)) + for var in ds_in_1pft: + self.assertTrue(are_xr_dataarrays_identical(ds_in_1pft[var], ds_out[var])) self.assertTrue(ds_in_1pft.equals(ds_out)) self.assertEqual(ds_in_1pft.sizes["pft"], 1) self.assertEqual(ds_out.sizes["pft"], 1) - def test_set_paramfile_setparams_just_one_pft_dropothers_doset(self): """Test dropping all but one PFT, changing one parameter""" output_path = os.path.join(self.tempdir, "output.nc") @@ -619,9 +649,9 @@ def test_set_paramfile_setparams_just_one_pft_dropothers_doset(self): da_in = ds_in_1pft[var] da_out = ds_out[var] if var == this_var: - self.assertFalse(da_in.equals(da_out)) + self.assertFalse(are_xr_dataarrays_identical(da_in, da_out)) else: - self.assertTrue(da_in.equals(da_out)) + self.assertTrue(are_xr_dataarrays_identical(da_in, da_out)) if __name__ == "__main__": From 24ba0dee1f795f32055ef48ef8de8c2040149297 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 15:19:55 -0600 Subject: [PATCH 079/196] are_xr_dataarrays_identical(): NaNs equal; ignore source/original_shape. --- python/ctsm/netcdf_utils.py | 8 ++++++-- python/ctsm/utils.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/python/ctsm/netcdf_utils.py b/python/ctsm/netcdf_utils.py index f9f9a927bf..419c5fa7a8 100644 --- a/python/ctsm/netcdf_utils.py +++ b/python/ctsm/netcdf_utils.py @@ -6,6 +6,8 @@ import xarray as xr from netCDF4 import Dataset # pylint: disable=no-name-in-module +from ctsm.utils import are_dicts_identical_nansequal + def get_netcdf_format(file_path): """ @@ -27,11 +29,13 @@ def are_xr_dataarrays_identical(da0: xr.DataArray, da1: xr.DataArray): return False # Check encoding - if da0.encoding != da1.encoding: + if not are_dicts_identical_nansequal( + da0.encoding, da1.encoding, keys_to_ignore=["source", "original_shape"] + ): return False # Check attributes - if da0.attrs != da1.attrs: + if not are_dicts_identical_nansequal(da0.attrs, da1.attrs): return False # Check name diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index 0b529fe282..ed7103f5bc 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -7,6 +7,7 @@ import string import re import pdb +import numpy as np from datetime import date, timedelta from getpass import getuser @@ -267,3 +268,36 @@ def find_one_file_matching_pattern(pattern): f"Expected 1 but found {n_found} files found matching pattern: " + pattern ) return file_list[0] + + +def are_dicts_identical_nansequal(dict0: dict, dict1: dict, keys_to_ignore=None): + """ + Compare two dictionaries, considering NaNs to be equal + """ + # pylint: disable=too-many-return-statements + + if keys_to_ignore is None: + keys_to_ignore = [] + keys_to_ignore = np.array(keys_to_ignore) + + if len(dict0) != len(dict1): + return False + for key, value0 in dict0.items(): + if key in keys_to_ignore: + continue + if key not in dict1: + return False + value1 = dict1[key] + if isinstance(value0, np.ndarray): + if not isinstance(value0, np.ndarray): + return False + if not np.array_equal(value0, value1, equal_nan=True): + return False + elif value1 != value0: + try: + if not (np.isnan(value0) and np.isnan(value1)): + return False + except TypeError: + return False + + return True From ee70147144c86aa6ca96e8bdc7e82c03f1dd32c0 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 15:23:15 -0600 Subject: [PATCH 080/196] set_paramfile: Don't add fill value to parameters without one. --- python/ctsm/param_utils/set_paramfile.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index db25011041..a097459053 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -200,7 +200,14 @@ def main(): ds_out[var].values = new_value - ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input)) + # We don't want to add _FillValue to parameters that didn't already have one. This dict will + # track such parameters and be passed to .to_netcdf(..., encoding=encoding) + encoding = {} + for var in ds_out: + if "_FillValue" not in ds_out[var].encoding: + encoding[var] = {"_FillValue": None} + + ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input), encoding=encoding) if __name__ == "__main__": From 952a355c5cfd7d8b97a4fd64e9a862eef14d7909 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 15:36:25 -0600 Subject: [PATCH 081/196] are_xr_dataarrays_identical(): Refactor for pylint. --- python/ctsm/netcdf_utils.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/python/ctsm/netcdf_utils.py b/python/ctsm/netcdf_utils.py index 419c5fa7a8..105ffccb78 100644 --- a/python/ctsm/netcdf_utils.py +++ b/python/ctsm/netcdf_utils.py @@ -18,11 +18,10 @@ def get_netcdf_format(file_path): return netcdf_format -def are_xr_dataarrays_identical(da0: xr.DataArray, da1: xr.DataArray): +def _is_dataarray_metadata_identical(da0: xr.DataArray, da1: xr.DataArray): """ - Comprehensively check whether two DataArrays are identical + Check whether two DataArrays have identical-enough metadata """ - # pylint: disable=too-many-return-statements # Check data type if da0.dtype != da1.dtype: @@ -46,6 +45,15 @@ def are_xr_dataarrays_identical(da0: xr.DataArray, da1: xr.DataArray): if da0.dims != da1.dims: return False + return True + + +def _is_dataarray_data_identical(da0: xr.DataArray, da1: xr.DataArray): + """ + Check whether two DataArrays have identical data + """ + # pylint: disable=too-many-return-statements + # Check sizes if da0.sizes != da1.sizes: return False @@ -75,5 +83,18 @@ def are_xr_dataarrays_identical(da0: xr.DataArray, da1: xr.DataArray): if not isinstance(da0.data, np.ndarray): raise NotImplementedError(f"Add support for comparing two objects of type {da0_data_type}") + return True + + +def are_xr_dataarrays_identical(da0: xr.DataArray, da1: xr.DataArray): + """ + Comprehensively check whether two DataArrays are identical + """ + if not _is_dataarray_metadata_identical(da0, da1): + return False + + if not _is_dataarray_data_identical(da0, da1): + return False + # Fallback to however xarray defines equality, in case we missed something above return da0.equals(da1) From 5431832c8c6a5f498d73d13125ed2853c34a630a Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 15:48:25 -0600 Subject: [PATCH 082/196] Move subgrid setup and teardown to setup and teardown methods, add explicit tests for subgrid levels not supported: lndgrid and cohort and for unspecified, right now these fail --- src/main/test/endrun_test/test_endrun.pf | 61 +++++++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf index 3d8f46782e..dfea0b6195 100644 --- a/src/main/test/endrun_test/test_endrun.pf +++ b/src/main/test/endrun_test/test_endrun.pf @@ -24,12 +24,21 @@ contains ! ======================================================================== subroutine setUp(this) + use unittestSimpleSubgridSetupsMod, only : setup_single_veg_patch class(TestAbortUtils), intent(inout) :: this + + ! Setup a single gridcell with one vegetated patch + ! So there's only one: gridcell, landunit, column, patch + ! This isn't needed for some tests, but doesn't hurt to do it + call setup_single_veg_patch(pft_type=1) end subroutine setUp subroutine tearDown(this) + use unittestSubgridMod, only : unittest_subgrid_teardown class(TestAbortUtils), intent(inout) :: this + call unittest_subgrid_teardown() + end subroutine tearDown ! ======================================================================== @@ -74,7 +83,6 @@ contains use decompMod, only : subgrid_level_lndgrid, subgrid_level_gridcell use decompMod, only : subgrid_level_landunit, subgrid_level_column, subgrid_level_patch use decompMod, only : subgrid_level_cohort - use unittestSimpleSubgridSetupsMod, only : setup_single_veg_patch ! Test pt_context operation of endrun with an additional message sent in class(TestAbortUtils), intent(inout) :: this character(len=CL) :: msg = "test_message" @@ -85,7 +93,6 @@ contains subgrid_level_landunit, subgrid_level_column, subgrid_level_patch, & subgrid_level_cohort /) - call setup_single_veg_patch(pft_type=1) ! Loop over all the subgrid level types do l = 2, nlevel-1 write(iulog,*) 'level = ', l @@ -95,4 +102,54 @@ contains end subroutine endrun_addmsg_pt_context_aborts + @Test + subroutine endrun_pt_context_lndgrid_aborts(this) + use decompMod, only : subgrid_level_lndgrid + class(TestAbortUtils), intent(inout) :: this + character(len=CL) :: msg = "test_message" + integer :: p = 1 + + ! Also test without an additional msg + call endrun(subgrid_index=p, subgrid_level=subgrid_level_lndgrid, msg=msg) + @assertExceptionRaised(endrun_msg(msg)) + + end subroutine endrun_pt_context_lndgrid_aborts + + @Test + subroutine endrun_nomsg_pt_context_cohort_aborts(this) + use decompMod, only : subgrid_level_cohort + class(TestAbortUtils), intent(inout) :: this + integer :: p = 1 + + ! Also test without either msg or additional msg + call endrun(subgrid_index=p, subgrid_level=subgrid_level_cohort) + @assertExceptionRaised(endrun_msg('')) + + end subroutine endrun_nomsg_pt_context_cohort_aborts + + @Test + subroutine endrun_nomsg_addmsg_pt_context_unspec_aborts(this) + use decompMod, only : subgrid_level_unspecified + class(TestAbortUtils), intent(inout) :: this + integer :: p = 1 + character(len=CL) :: add_msg = "additional_test_message" + + ! Don't use msg but do use additional_msg + call endrun(subgrid_index=p, subgrid_level=subgrid_level_unspecified, additional_msg=add_msg) + @assertExceptionRaised(endrun_msg('')) + + end subroutine endrun_nomsg_addmsg_pt_context_unspec_aborts + + @Test + subroutine endrun_nomsg_pt_context_badlvl_aborts(this) + use decompMod, only : subgrid_level_unspecified + class(TestAbortUtils), intent(inout) :: this + integer :: p = 1 + character(len=CL) :: expected_msg = "subgrid_level not supported" + + call endrun(subgrid_index=p, subgrid_level=subgrid_level_unspecified) + @assertExceptionRaised(endrun_msg(expected_msg)) + + end subroutine endrun_nomsg_pt_context_badlvl_aborts + end module test_endrun From 895a8df874ff4508ae4a1caffb587bd0b04062d1 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 15:51:40 -0600 Subject: [PATCH 083/196] are_xr_dataarrays_identical(): Don't ignore anything by default. --- python/ctsm/netcdf_utils.py | 8 +++---- python/ctsm/test/test_sys_set_paramfile.py | 25 ++++++++++++++-------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/python/ctsm/netcdf_utils.py b/python/ctsm/netcdf_utils.py index 105ffccb78..7ca83d79bd 100644 --- a/python/ctsm/netcdf_utils.py +++ b/python/ctsm/netcdf_utils.py @@ -18,7 +18,7 @@ def get_netcdf_format(file_path): return netcdf_format -def _is_dataarray_metadata_identical(da0: xr.DataArray, da1: xr.DataArray): +def _is_dataarray_metadata_identical(da0: xr.DataArray, da1: xr.DataArray, keys_to_ignore=None): """ Check whether two DataArrays have identical-enough metadata """ @@ -29,7 +29,7 @@ def _is_dataarray_metadata_identical(da0: xr.DataArray, da1: xr.DataArray): # Check encoding if not are_dicts_identical_nansequal( - da0.encoding, da1.encoding, keys_to_ignore=["source", "original_shape"] + da0.encoding, da1.encoding, keys_to_ignore=keys_to_ignore ): return False @@ -86,11 +86,11 @@ def _is_dataarray_data_identical(da0: xr.DataArray, da1: xr.DataArray): return True -def are_xr_dataarrays_identical(da0: xr.DataArray, da1: xr.DataArray): +def are_xr_dataarrays_identical(da0: xr.DataArray, da1: xr.DataArray, keys_to_ignore=None): """ Comprehensively check whether two DataArrays are identical """ - if not _is_dataarray_metadata_identical(da0, da1): + if not _is_dataarray_metadata_identical(da0, da1, keys_to_ignore=keys_to_ignore): return False if not _is_dataarray_data_identical(da0, da1): diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index f59af44620..22045dfd71 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -27,6 +27,13 @@ ) +def are_paramfile_dataarrays_identical(da0: xr.DataArray, da1: xr.DataArray): + """ + Check whether parameter DataArrays are identical enough, ignoring some metadata + """ + return are_xr_dataarrays_identical(da0, da1, keys_to_ignore=["source", "original_shape"]) + + class TestSysSetParamfile(unittest.TestCase): """System tests of set_paramfile""" @@ -133,7 +140,7 @@ def test_set_paramfile_extractpfts(self): expected = ds_in[var].isel(pft=[0, 1]) else: expected = ds_in[var] - self.assertTrue(are_xr_dataarrays_identical(expected, actual)) + self.assertTrue(are_paramfile_dataarrays_identical(expected, actual)) def test_set_paramfile_changeparams_scalar_errors_given_list(self): """Test that set_paramfile errors if given a list for a scalar parameter""" @@ -190,7 +197,7 @@ def test_set_paramfile_changeparams_scalar_double(self): self.assertTrue(ds_in[var].values == 11) self.assertTrue(ds_out[var].values == 87) else: - self.assertTrue(are_xr_dataarrays_identical(ds_in[var], ds_out[var])) + self.assertTrue(are_paramfile_dataarrays_identical(ds_in[var], ds_out[var])) # Check that data type hasn't changed self.assertTrue(ds_in[var].dtype == ds_out[var].dtype) @@ -311,9 +318,9 @@ def test_set_paramfile_changeparam_dbl_onlysomepfts(self): this_slice = slice(2, None) expected = ds_in["xl"].isel(pft=this_slice) result = ds_out["xl"].isel(pft=this_slice) - self.assertTrue(are_xr_dataarrays_identical(expected, result)) + self.assertTrue(are_paramfile_dataarrays_identical(expected, result)) else: - self.assertTrue(are_xr_dataarrays_identical(ds_in[var], ds_out[var])) + self.assertTrue(are_paramfile_dataarrays_identical(ds_in[var], ds_out[var])) def test_set_paramfile_extractpfts_changeparam_int(self): """ @@ -351,10 +358,10 @@ def test_set_paramfile_extractpfts_changeparam_int(self): self.assertTrue(np.array_equal(np.array([1986, 1987]), ds_out[var].values)) elif sp.PFTNAME_VAR in ds_in[var].coords: self.assertTrue( - are_xr_dataarrays_identical(ds_in[var].isel(pft=[0, 1]), ds_out[var]) + are_paramfile_dataarrays_identical(ds_in[var].isel(pft=[0, 1]), ds_out[var]) ) else: - self.assertTrue(are_xr_dataarrays_identical(ds_in[var], ds_out[var])) + self.assertTrue(are_paramfile_dataarrays_identical(ds_in[var], ds_out[var])) def test_set_paramfile_fill_value_scalar_double_nan(self): """ @@ -610,7 +617,7 @@ def test_set_paramfile_setparams_just_one_pft_dropothers_noset(self): ds_out = open_paramfile(output_path) self.assertTrue(set(ds_in_1pft.variables) == set(ds_out.variables)) for var in ds_in_1pft: - self.assertTrue(are_xr_dataarrays_identical(ds_in_1pft[var], ds_out[var])) + self.assertTrue(are_paramfile_dataarrays_identical(ds_in_1pft[var], ds_out[var])) self.assertTrue(ds_in_1pft.equals(ds_out)) self.assertEqual(ds_in_1pft.sizes["pft"], 1) self.assertEqual(ds_out.sizes["pft"], 1) @@ -649,9 +656,9 @@ def test_set_paramfile_setparams_just_one_pft_dropothers_doset(self): da_in = ds_in_1pft[var] da_out = ds_out[var] if var == this_var: - self.assertFalse(are_xr_dataarrays_identical(da_in, da_out)) + self.assertFalse(are_paramfile_dataarrays_identical(da_in, da_out)) else: - self.assertTrue(are_xr_dataarrays_identical(da_in, da_out)) + self.assertTrue(are_paramfile_dataarrays_identical(da_in, da_out)) if __name__ == "__main__": From 5492fffd86f6ec47f266a67c6592811477dcd0da Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 15:55:34 -0600 Subject: [PATCH 084/196] set_paramfile: Add save_paramfile(). --- python/ctsm/param_utils/set_paramfile.py | 25 ++++++++++++++------- python/ctsm/test/test_unit_set_paramfile.py | 22 ++++++++++++++++++ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index a097459053..201579a341 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -4,6 +4,7 @@ import os import numpy as np +import xarray as xr from ctsm.args_utils import comma_separated_list from ctsm.netcdf_utils import get_netcdf_format @@ -113,6 +114,21 @@ def drop_other_pfts(selected_pfts, ds): return ds +def save_paramfile(ds_out: xr.Dataset, output_path, *, nc_format="NETCDF3_CLASSIC"): + """ + Save xarray Dataset to paramfile + """ + + # We don't want to add _FillValue to parameters that didn't already have one. This dict will + # track such parameters and be passed to .to_netcdf(..., encoding=encoding) + encoding = {} + for var in ds_out: + if "_FillValue" not in ds_out[var].encoding: + encoding[var] = {"_FillValue": None} + + ds_out.to_netcdf(output_path, format=nc_format, encoding=encoding) + + def main(): """ Main entry point for the script. @@ -200,14 +216,7 @@ def main(): ds_out[var].values = new_value - # We don't want to add _FillValue to parameters that didn't already have one. This dict will - # track such parameters and be passed to .to_netcdf(..., encoding=encoding) - encoding = {} - for var in ds_out: - if "_FillValue" not in ds_out[var].encoding: - encoding[var] = {"_FillValue": None} - - ds_out.to_netcdf(args.output, format=get_netcdf_format(args.input), encoding=encoding) + save_paramfile(ds_out, args.output, nc_format=get_netcdf_format(args.input)) if __name__ == "__main__": diff --git a/python/ctsm/test/test_unit_set_paramfile.py b/python/ctsm/test/test_unit_set_paramfile.py index 5bfb9528c9..734047f8e1 100755 --- a/python/ctsm/test/test_unit_set_paramfile.py +++ b/python/ctsm/test/test_unit_set_paramfile.py @@ -5,12 +5,16 @@ import unittest import os import sys +import tempfile +import shutil import numpy as np import xarray as xr from ctsm import unit_testing +from ctsm.netcdf_utils import get_netcdf_format from ctsm.param_utils import set_paramfile as sp +from ctsm.param_utils.paramfile_shared import open_paramfile # Allow names that pylint doesn't like, because otherwise I find it hard # to make readable unit test names @@ -211,6 +215,24 @@ def test_set_paramfile_error_dropotherpfts_without_pft(self): with self.assertRaises(RuntimeError): sp.get_arguments() +class TestUnitSaveParamfile(unittest.TestCase): + """Unit tests of save_paramfile""" + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.output_path = os.path.join(self.tempdir, "output.nc") + + def tearDown(self): + shutil.rmtree(self.tempdir, ignore_errors=True) + + def test_save_paramfile (self): + """Test that save_paramfile can save our usual test file to a new file without changes""" + input_path = PARAMFILE + ds_in = open_paramfile(input_path) + sp.save_paramfile(ds_in, self.output_path, nc_format=get_netcdf_format(input_path)) + ds_out = open_paramfile(self.output_path) + self.assertTrue(ds_out.equals(ds_in)) + if __name__ == "__main__": unit_testing.setup_for_tests() From 31894b1c96f705983541d385a1a24715a46284d8 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 16:15:43 -0600 Subject: [PATCH 085/196] set_paramfile: Refactor to reduce branching. --- python/ctsm/param_utils/set_paramfile.py | 41 +++++++++++++----------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 201579a341..03fd417462 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -129,6 +129,22 @@ def save_paramfile(ds_out: xr.Dataset, output_path, *, nc_format="NETCDF3_CLASSI ds_out.to_netcdf(output_path, format=nc_format, encoding=encoding) +def _replace_nans_with_fill(ds_in_masked_scaled, var, chg, new_value): + """ + If there are any NaNs in the new parameter value (array), replace them with the fill value + """ + if any(np.isnan(np.atleast_1d(new_value))): + # TODO: Add code to add fill value to parameters without it + if "_FillValue" not in ds_in_masked_scaled[var].encoding: + raise NotImplementedError( + f"Not able to set NaN if parameter doesn't already have fill value: {chg}" + ) + fill_value = ds_in_masked_scaled[var].encoding["_FillValue"] + new_value[np.isnan(new_value)] = fill_value + + return new_value + + def main(): """ Main entry point for the script. @@ -161,11 +177,8 @@ def main(): if ds_out[var].ndim > 1: raise NotImplementedError("Can't yet change multi-dimensional parameters") - if "," in new_value: - new_value_list = new_value.split(",") - new_value = np.array(new_value_list) - else: - new_value = np.array(new_value) + # Split at commas, if any, and convert to numpy array + new_value = np.array(new_value.split(",")).squeeze() # TODO: Add code to set integer variables to NaN (this might not be possible) if np.any(np.char.lower(new_value) == "nan") and is_integer(ds_in[var].values): @@ -176,14 +189,13 @@ def main(): # Are we acting on just some PFTs? If so, we'll need some stuff. just_some_pfts = PFTNAME_VAR in ds_out[var].coords and args.pft + # pylint is probably wrong with the possibly-used-before-assignment warning, but do this + # here just to placate it. Make it an invalid index so we get an error if we try to use + # it. + indices = -1 if just_some_pfts: pft_names = check_pfts_in_paramfile(args.pft, ds_out) indices = get_selected_pft_indices(args.pft, pft_names) - else: - # pylint is probably wrong with the possibly-used-before-assignment warning, but do this - # here just to placate it. Make it an invalid index so we get an error if we try to use - # it. - indices = -1 # Check that correct number of dimensions were given for new values. Special handling needed # if we're just acting on one PFT. @@ -201,14 +213,7 @@ def main(): new_value = tmp # Ensure that any NaNs are replaced with the fill value - if any(np.isnan(np.atleast_1d(new_value))): - # TODO: Add code to add fill value to parameters without it - if "_FillValue" not in ds_in_masked_scaled[var].encoding: - raise NotImplementedError( - f"Not able to set NaN if parameter doesn't already have fill value: {chg}" - ) - fill_value = ds_in_masked_scaled[var].encoding["_FillValue"] - new_value[np.isnan(new_value)] = fill_value + new_value = _replace_nans_with_fill(ds_in_masked_scaled, var, chg, new_value) # This can happen if, e.g., you're selecting and changing just one PFT if ds_in[var].values.ndim > 0 and new_value.ndim == 0: From 8e3e38211c4ccdb5bd9d97ae9e84df3e822adee9 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 16:27:36 -0600 Subject: [PATCH 086/196] set_paramfile: Improve error when assigning fill for param w/o one. --- python/ctsm/param_utils/set_paramfile.py | 2 +- python/ctsm/test/test_sys_set_paramfile.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 03fd417462..b05d388b70 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -137,7 +137,7 @@ def _replace_nans_with_fill(ds_in_masked_scaled, var, chg, new_value): # TODO: Add code to add fill value to parameters without it if "_FillValue" not in ds_in_masked_scaled[var].encoding: raise NotImplementedError( - f"Not able to set NaN if parameter doesn't already have fill value: {chg}" + f"Can't set parameter to fill value if it doesn't already have one: {chg}" ) fill_value = ds_in_masked_scaled[var].encoding["_FillValue"] new_value[np.isnan(new_value)] = fill_value diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 22045dfd71..59457867e3 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -517,7 +517,7 @@ def test_set_paramfile_setparams_nan_but_no_fillvalue(self): f"{new_param_name}=nan", ] with self.assertRaisesRegex( - NotImplementedError, "Not able to set NaN if parameter doesn't already have fill value:" + NotImplementedError, "Can't set parameter to fill value if it doesn't already have one:" ): sp.main() From 2508e3feccbd0189afc94e40764634c6051fde5f Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 16:43:32 -0600 Subject: [PATCH 087/196] Continue the endrun call even if subgrid_level is bad so that the error messaging gives as much information as possible about the error, also reuse the endrun_vanilla in the pt_context version to remove duplication --- src/main/abortutils.F90 | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/main/abortutils.F90 b/src/main/abortutils.F90 index c93fd761bf..23e36c3e95 100644 --- a/src/main/abortutils.F90 +++ b/src/main/abortutils.F90 @@ -89,13 +89,7 @@ subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additio call write_point_context(subgrid_index, subgrid_level) end if - if (present (additional_msg)) then - write(iulog,*)'ENDRUN: ', additional_msg - else - write(iulog,*)'ENDRUN:' - end if - - call shr_sys_abort(msg) + call endrun_vanilla(msg=msg, additional_msg=additional_msg) end subroutine endrun_write_point_context @@ -188,11 +182,10 @@ subroutine write_point_context(subgrid_index, subgrid_level) else write(iulog,*) 'subgrid_level not supported: ', subgrid_level - call shr_sys_abort('subgrid_level not supported '//errmsg(sourcefile, __LINE__)) + write(iulog,*) errmsg(sourcefile, __LINE__) + write(iulog,*) 'Continuing the endrun without writing point context information' end if - call shr_sys_flush(iulog) - end subroutine write_point_context end module abortutils From 61f5056860bba49c4333c64b261244a606104e57 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 16:46:15 -0600 Subject: [PATCH 088/196] Change to the new expected behavior where the error isn't about the subgrid_level, but the original error sent in, this and the previous commit resolve #3420 --- src/main/test/endrun_test/test_endrun.pf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf index dfea0b6195..3f2844ee20 100644 --- a/src/main/test/endrun_test/test_endrun.pf +++ b/src/main/test/endrun_test/test_endrun.pf @@ -94,8 +94,8 @@ contains subgrid_level_cohort /) ! Loop over all the subgrid level types + ! Skip the first one and the last one which are: lndgrid and cohort do l = 2, nlevel-1 - write(iulog,*) 'level = ', l call endrun(subgrid_index=p, subgrid_level=subgrid_lvl(l), msg=msg, additional_msg=add_msg) @assertExceptionRaised(endrun_msg(msg)) end do @@ -145,10 +145,9 @@ contains use decompMod, only : subgrid_level_unspecified class(TestAbortUtils), intent(inout) :: this integer :: p = 1 - character(len=CL) :: expected_msg = "subgrid_level not supported" - call endrun(subgrid_index=p, subgrid_level=subgrid_level_unspecified) - @assertExceptionRaised(endrun_msg(expected_msg)) + call endrun(subgrid_index=p, subgrid_level=-9999) + @assertExceptionRaised(endrun_msg('')) end subroutine endrun_nomsg_pt_context_badlvl_aborts From e48342f671712f5fd56f9436a91ad16d6a7d133e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 16:57:55 -0600 Subject: [PATCH 089/196] Update to using shr_abort_abort as talked about in #3417 --- src/main/abortutils.F90 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/abortutils.F90 b/src/main/abortutils.F90 index 23e36c3e95..260169a488 100644 --- a/src/main/abortutils.F90 +++ b/src/main/abortutils.F90 @@ -33,7 +33,7 @@ subroutine endrun_vanilla(msg, additional_msg) ! !DESCRIPTION: ! Abort the model for abnormal termination ! - use shr_sys_mod , only: shr_sys_abort + use shr_abort_mod , only: shr_abort_abort use clm_varctl , only: iulog ! ! !ARGUMENTS: @@ -42,8 +42,8 @@ subroutine endrun_vanilla(msg, additional_msg) ! volatile stuff in additional_msg, as in: ! call endrun(msg='Informative message', additional_msg=errmsg(__FILE__, __LINE__)) ! and then just assert against msg. - character(len=*), intent(in), optional :: msg ! string to be passed to shr_sys_abort - character(len=*), intent(in), optional :: additional_msg ! string to be printed, but not passed to shr_sys_abort + character(len=*), intent(in), optional :: msg ! string to be passed to shr_abort + character(len=*), intent(in), optional :: additional_msg ! string to be printed, but not passed to shr_abort !----------------------------------------------------------------------- if (present (additional_msg)) then @@ -52,7 +52,7 @@ subroutine endrun_vanilla(msg, additional_msg) write(iulog,*)'ENDRUN:' end if - call shr_sys_abort(msg) + call shr_abort_abort(msg) end subroutine endrun_vanilla @@ -65,7 +65,6 @@ subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additio ! ! This version also prints additional information about the point causing the error. ! - use shr_sys_mod , only: shr_sys_abort use clm_varctl , only: iulog use decompMod , only: subgrid_level_unspecified ! @@ -78,8 +77,8 @@ subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additio ! volatile stuff in additional_msg, as in: ! call endrun(msg='Informative message', additional_msg=errmsg(__FILE__, __LINE__)) ! and then just assert against msg. - character(len=*), intent(in), optional :: msg ! string to be passed to shr_sys_abort - character(len=*), intent(in), optional :: additional_msg ! string to be printed, but not passed to shr_sys_abort + character(len=*), intent(in), optional :: msg ! string to be passed to shr_abort + character(len=*), intent(in), optional :: additional_msg ! string to be printed, but not passed to shr_abort ! ! Local Variables: integer :: igrc, ilun, icol @@ -101,7 +100,8 @@ subroutine write_point_context(subgrid_index, subgrid_level) ! Write various information giving context for the given index at the given subgrid ! level, including global index information and more. ! - use shr_sys_mod , only : shr_sys_flush, shr_sys_abort + ! NOTE: DO NOT CALL AN ABORT FROM HERE AS THAT WOULD SHORT CIRUIT THE ERROR REPORTING!! + ! use shr_log_mod , only : errMsg => shr_log_errMsg use clm_varctl , only : iulog use decompMod , only : subgrid_level_gridcell, subgrid_level_landunit, subgrid_level_column, subgrid_level_patch From 27b3b544810acdbc2d816f5721fb03212c764865 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 17:00:01 -0600 Subject: [PATCH 090/196] Update to using shr_abort_abort as talked about in #3417 --- src/main/test/endrun_test/test_endrun.pf | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf index 3f2844ee20..7c63cf6781 100644 --- a/src/main/test/endrun_test/test_endrun.pf +++ b/src/main/test/endrun_test/test_endrun.pf @@ -102,6 +102,28 @@ contains end subroutine endrun_addmsg_pt_context_aborts + @Test + subroutine endrun_nomsg_pt_context_bad_pt_aborts(this) + use decompMod, only : subgrid_level_lndgrid, subgrid_level_gridcell + use decompMod, only : subgrid_level_landunit, subgrid_level_column, subgrid_level_patch + use decompMod, only : subgrid_level_cohort + ! Test pt_context operation of endrun with an additional message sent in + class(TestAbortUtils), intent(inout) :: this + integer :: p = 2, l + integer, parameter :: nlevel = 6 + integer :: subgrid_lvl(nlevel) = (/ subgrid_level_lndgrid, subgrid_level_gridcell, & + subgrid_level_landunit, subgrid_level_column, subgrid_level_patch, & + subgrid_level_cohort /) + + ! Loop over all the subgrid level types + ! Skip the first one and the last one which are: lndgrid and cohort + do l = 2, nlevel-1 + call endrun(subgrid_index=p, subgrid_level=subgrid_lvl(l), msg=msg, additional_msg=add_msg) + @assertExceptionRaised(endrun_msg('')) + end do + + end subroutine endrun_nomsg_pt_context_bad_pt_aborts + @Test subroutine endrun_pt_context_lndgrid_aborts(this) use decompMod, only : subgrid_level_lndgrid From 8c25599d7491c3d3c9fafdd8f68c917cd0ebe751 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 17:05:27 -0600 Subject: [PATCH 091/196] Add testing for bad point in the pt_context which fails because of subscript overflow --- src/main/test/endrun_test/test_endrun.pf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf index 7c63cf6781..9b3edb7bc6 100644 --- a/src/main/test/endrun_test/test_endrun.pf +++ b/src/main/test/endrun_test/test_endrun.pf @@ -118,7 +118,7 @@ contains ! Loop over all the subgrid level types ! Skip the first one and the last one which are: lndgrid and cohort do l = 2, nlevel-1 - call endrun(subgrid_index=p, subgrid_level=subgrid_lvl(l), msg=msg, additional_msg=add_msg) + call endrun(subgrid_index=p, subgrid_level=subgrid_lvl(l)) @assertExceptionRaised(endrun_msg('')) end do From b9b94202a50f7904b95e7e986225a55532eaad80 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 17:51:33 -0600 Subject: [PATCH 092/196] save_paramfile(): test saving an integer parameter with a fill value --- python/ctsm/param_utils/paramfile_shared.py | 9 ++++ python/ctsm/test/test_sys_set_paramfile.py | 11 +---- python/ctsm/test/test_unit_set_paramfile.py | 52 ++++++++++++++++++++- 3 files changed, 61 insertions(+), 11 deletions(-) diff --git a/python/ctsm/param_utils/paramfile_shared.py b/python/ctsm/param_utils/paramfile_shared.py index 19529e6850..b9860be0db 100644 --- a/python/ctsm/param_utils/paramfile_shared.py +++ b/python/ctsm/param_utils/paramfile_shared.py @@ -5,9 +5,18 @@ import argparse import xarray as xr +from ctsm.netcdf_utils import are_xr_dataarrays_identical + PFTNAME_VAR = "pftname" +def are_paramfile_dataarrays_identical(da0: xr.DataArray, da1: xr.DataArray): + """ + Check whether parameter DataArrays are identical enough, ignoring some metadata + """ + return are_xr_dataarrays_identical(da0, da1, keys_to_ignore=["source", "original_shape"]) + + def check_pfts_in_paramfile(selected_pfts, ds): """ Check that the given PFTs are present in the parameter file. diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 59457867e3..8f326bfa3d 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -12,9 +12,9 @@ from ctsm import unit_testing -from ctsm.netcdf_utils import get_netcdf_format, are_xr_dataarrays_identical +from ctsm.netcdf_utils import get_netcdf_format from ctsm.param_utils import set_paramfile as sp -from ctsm.param_utils.paramfile_shared import open_paramfile +from ctsm.param_utils.paramfile_shared import open_paramfile, are_paramfile_dataarrays_identical from ctsm.param_utils.paramfile_shared import check_pfts_in_paramfile, get_selected_pft_indices # Allow names that pylint doesn't like, because otherwise I find it hard @@ -27,13 +27,6 @@ ) -def are_paramfile_dataarrays_identical(da0: xr.DataArray, da1: xr.DataArray): - """ - Check whether parameter DataArrays are identical enough, ignoring some metadata - """ - return are_xr_dataarrays_identical(da0, da1, keys_to_ignore=["source", "original_shape"]) - - class TestSysSetParamfile(unittest.TestCase): """System tests of set_paramfile""" diff --git a/python/ctsm/test/test_unit_set_paramfile.py b/python/ctsm/test/test_unit_set_paramfile.py index 734047f8e1..d5cb6c610c 100755 --- a/python/ctsm/test/test_unit_set_paramfile.py +++ b/python/ctsm/test/test_unit_set_paramfile.py @@ -14,7 +14,7 @@ from ctsm.netcdf_utils import get_netcdf_format from ctsm.param_utils import set_paramfile as sp -from ctsm.param_utils.paramfile_shared import open_paramfile +from ctsm.param_utils.paramfile_shared import open_paramfile, are_paramfile_dataarrays_identical # Allow names that pylint doesn't like, because otherwise I find it hard # to make readable unit test names @@ -26,6 +26,31 @@ ) +def save_paramfile_with_integer_that_has_fillvalue(tempdir): + """ + Convenience function for creating a parameter file that has an integer parameter with a fill + value + """ + input_path = os.path.join(tempdir, "input.nc") + new_param_name = "new_param_abc123" + fill_value = -999 + + # Construct the Dataset + data = np.array([1, 2, fill_value, 4, 5], dtype=np.int32) + da = xr.DataArray(data) + da.encoding["_FillValue"] = fill_value + ds = xr.Dataset(data_vars={new_param_name: da}) + + # Check it + assert new_param_name in ds + assert sp.is_integer(ds[new_param_name].values) + + # Save it + sp.save_paramfile(ds, input_path) + + return input_path, new_param_name, fill_value + + class TestUnitCheckCorrectNdims(unittest.TestCase): """Unit tests of check_correct_ndims""" @@ -215,6 +240,7 @@ def test_set_paramfile_error_dropotherpfts_without_pft(self): with self.assertRaises(RuntimeError): sp.get_arguments() + class TestUnitSaveParamfile(unittest.TestCase): """Unit tests of save_paramfile""" @@ -225,13 +251,35 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tempdir, ignore_errors=True) - def test_save_paramfile (self): + def test_save_paramfile(self): """Test that save_paramfile can save our usual test file to a new file without changes""" input_path = PARAMFILE ds_in = open_paramfile(input_path) sp.save_paramfile(ds_in, self.output_path, nc_format=get_netcdf_format(input_path)) ds_out = open_paramfile(self.output_path) self.assertTrue(ds_out.equals(ds_in)) + self.assertTrue(set(ds_in.variables) == set(ds_out.variables)) + for var in ds_in: + self.assertTrue(are_paramfile_dataarrays_identical(ds_in[var], ds_out[var])) + + def test_save_paramfile_integer_with_fillvalue(self): + """Test that save_paramfile can successfully save an integer parameter with a fill value""" + + # Create paramfile with a integer variable with fill value -999 + input_path, new_param_name, fill_value = save_paramfile_with_integer_that_has_fillvalue( + self.tempdir + ) + + # Read it, checking that its fill value is what we asked for. Note: We need to mask, because + # otherwise the fill value won't be read. + ds = xr.open_dataset(input_path, mask_and_scale=True) + self.assertTrue("_FillValue" in ds[new_param_name].encoding) + self.assertEqual(fill_value, ds[new_param_name].encoding["_FillValue"]) + + # Check that the saved variable is an integer type. Note: We need to NOT mask, because + # masking converts fill values to NaN, which forces conversion to float. + ds = xr.open_dataset(input_path, mask_and_scale=False) + self.assertTrue(sp.is_integer(ds[new_param_name].values)) if __name__ == "__main__": From fce72876effb0480cac24afc6ef2a206e9dbd41a Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 17:58:49 -0600 Subject: [PATCH 093/196] set_paramfile(): Improve text about integer fill values. --- python/ctsm/param_utils/set_paramfile.py | 5 +++-- python/ctsm/test/test_sys_set_paramfile.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index b05d388b70..b8a873d030 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -180,9 +180,10 @@ def main(): # Split at commas, if any, and convert to numpy array new_value = np.array(new_value.split(",")).squeeze() - # TODO: Add code to set integer variables to NaN (this might not be possible) + # TODO: Add code to set integer variables to their missing value. This is harder than it + # sounds. if np.any(np.char.lower(new_value) == "nan") and is_integer(ds_in[var].values): - raise NotImplementedError(f"Not able to set NaN for integer parameters: {chg}") + raise NotImplementedError(f"Can't set integer parameter to fill value: {chg}") # Convert to the output data type new_value = new_value.astype(type(ds_out[var].dtype)) diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 8f326bfa3d..d506d6d3ad 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -528,7 +528,7 @@ def test_set_paramfile_setparams_scalar_int_tonan_with_nan(self): ] with self.assertRaisesRegex( - NotImplementedError, "Not able to set NaN for integer parameters:" + NotImplementedError, "Can't set integer parameter to fill value:" ): sp.main() From 62d7711506a0fb9a3ad138ceceffbac1b79a6caa Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 17:59:18 -0600 Subject: [PATCH 094/196] Reformat with black. --- python/ctsm/netcdf_utils.py | 4 +--- python/ctsm/test/test_sys_query_paramfile.py | 8 +------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/python/ctsm/netcdf_utils.py b/python/ctsm/netcdf_utils.py index 7ca83d79bd..93d29a2a5e 100644 --- a/python/ctsm/netcdf_utils.py +++ b/python/ctsm/netcdf_utils.py @@ -28,9 +28,7 @@ def _is_dataarray_metadata_identical(da0: xr.DataArray, da1: xr.DataArray, keys_ return False # Check encoding - if not are_dicts_identical_nansequal( - da0.encoding, da1.encoding, keys_to_ignore=keys_to_ignore - ): + if not are_dicts_identical_nansequal(da0.encoding, da1.encoding, keys_to_ignore=keys_to_ignore): return False # Check attributes diff --git a/python/ctsm/test/test_sys_query_paramfile.py b/python/ctsm/test/test_sys_query_paramfile.py index b2031a8fdd..43692b12f0 100755 --- a/python/ctsm/test/test_sys_query_paramfile.py +++ b/python/ctsm/test/test_sys_query_paramfile.py @@ -122,13 +122,7 @@ def test_query_paramfile_no_variables_fake(self): with redirect_stdout(f): qp.main() out = f.getvalue() - self.assertRegex( - out, - ( - r"fake1: \[1 2 3\]\n" - r"fake2: \[4 5 6\]\n" - ) - ) + self.assertRegex(out, (r"fake1: \[1 2 3\]\n" r"fake2: \[4 5 6\]\n")) def test_query_paramfile_no_variables_real(self): """ From 8246f54fa76308af1da3ad500d887e96e0134f0f Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 17:59:32 -0600 Subject: [PATCH 095/196] Add previous commit to .git-blame-ignore-revs. --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 41886277ba..fb24dc3d5d 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -71,3 +71,4 @@ cdf40d265cc82775607a1bf25f5f527bacc97405 75db098206b064b8b7b2a0604d3f0bf8fdb950cc 84609494b54ea9732f64add43b2f1dd035632b4c 7eb17f3ef0b9829fb55e0e3d7f02e157b0e41cfb +62d7711506a0fb9a3ad138ceceffbac1b79a6caa From eac7e810d01da8a710528e481d6570b93505e6b3 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 18:02:34 -0600 Subject: [PATCH 096/196] Satisfy pylint. --- python/ctsm/param_utils/set_paramfile.py | 3 +++ python/ctsm/test/test_sys_query_paramfile.py | 2 +- python/ctsm/test/test_sys_set_paramfile.py | 2 ++ python/ctsm/test/test_unit_netcdf_utils.py | 2 ++ python/ctsm/utils.py | 3 ++- 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index b8a873d030..2ca2195629 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -108,6 +108,9 @@ def check_correct_ndims(da, new_value, throw_error=False): def drop_other_pfts(selected_pfts, ds): + """ + Drop PFTs other than the selected ones + """ pft_names = check_pfts_in_paramfile(selected_pfts, ds) indices = get_selected_pft_indices(selected_pfts, pft_names) ds = ds.isel({"pft": indices}) diff --git a/python/ctsm/test/test_sys_query_paramfile.py b/python/ctsm/test/test_sys_query_paramfile.py index 43692b12f0..c01e7bc5e8 100755 --- a/python/ctsm/test/test_sys_query_paramfile.py +++ b/python/ctsm/test/test_sys_query_paramfile.py @@ -122,7 +122,7 @@ def test_query_paramfile_no_variables_fake(self): with redirect_stdout(f): qp.main() out = f.getvalue() - self.assertRegex(out, (r"fake1: \[1 2 3\]\n" r"fake2: \[4 5 6\]\n")) + self.assertRegex(out, (r"fake1: \[1 2 3\]\nfake2: \[4 5 6\]\n")) def test_query_paramfile_no_variables_real(self): """ diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index d506d6d3ad..2dba09b788 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -30,6 +30,8 @@ class TestSysSetParamfile(unittest.TestCase): """System tests of set_paramfile""" + # pylint: disable=too-many-public-methods + def setUp(self): self.orig_argv = sys.argv self.tempdir = tempfile.mkdtemp() diff --git a/python/ctsm/test/test_unit_netcdf_utils.py b/python/ctsm/test/test_unit_netcdf_utils.py index cd420709aa..bfbb51f899 100755 --- a/python/ctsm/test/test_unit_netcdf_utils.py +++ b/python/ctsm/test/test_unit_netcdf_utils.py @@ -59,6 +59,8 @@ class TestUnitAreXrDataArraysIdentical(unittest.TestCase): Unit tests for are_xr_dataarrays_identical """ + # pylint: disable=too-many-public-methods + def test_are_xr_dataarrays_identical_data_types_match(self): """Should be true if dtypes match""" da0 = xr.DataArray(int(1)) diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index ed7103f5bc..8c8472c386 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -7,11 +7,12 @@ import string import re import pdb -import numpy as np from datetime import date, timedelta from getpass import getuser +import numpy as np + from ctsm.git_utils import get_ctsm_git_short_hash logger = logging.getLogger(__name__) From ac849e60dbd7d0380c23a8082809e97483f9c4c0 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 15 Aug 2025 18:11:04 -0600 Subject: [PATCH 097/196] save_paramfile: Save calling command to history. --- python/ctsm/param_utils/set_paramfile.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 2ca2195629..f2a64404e3 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -3,6 +3,8 @@ """ import os +import sys +from datetime import datetime import numpy as np import xarray as xr @@ -117,6 +119,18 @@ def drop_other_pfts(selected_pfts, ds): return ds +def _add_cmd_to_history(ds): + """ + Prepends the calling command to the netCDF history + """ + if "history" not in ds.attrs: + ds.attrs["history"] = "" + cmd_items = [f"'{x}'" if " " in x else x for x in sys.argv] + datetime_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + ds.attrs["history"] = f"{datetime_str}: {' '.join(cmd_items)}\n{ds.attrs['history']}" + return ds + + def save_paramfile(ds_out: xr.Dataset, output_path, *, nc_format="NETCDF3_CLASSIC"): """ Save xarray Dataset to paramfile @@ -129,6 +143,8 @@ def save_paramfile(ds_out: xr.Dataset, output_path, *, nc_format="NETCDF3_CLASSI if "_FillValue" not in ds_out[var].encoding: encoding[var] = {"_FillValue": None} + ds_out = _add_cmd_to_history(ds_out) + ds_out.to_netcdf(output_path, format=nc_format, encoding=encoding) From 09f561d793f69d1c41bdb0d3226614018b866379 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 18:46:19 -0600 Subject: [PATCH 098/196] Add handling in case the input point is bad to end_run_pt_context --- src/main/abortutils.F90 | 112 +++++++++++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 30 deletions(-) diff --git a/src/main/abortutils.F90 b/src/main/abortutils.F90 index 260169a488..517b80fffa 100644 --- a/src/main/abortutils.F90 +++ b/src/main/abortutils.F90 @@ -117,43 +117,102 @@ subroutine write_point_context(subgrid_index, subgrid_level) integer , intent(in) :: subgrid_level ! one of the subgrid_level_* constants defined in decompMod ! ! Local Variables: - integer :: igrc, ilun, icol, ipft + integer, parameter :: unset = -9999 ! Unset value for an index + integer :: igrc=unset, ilun=unset, icol=unset, ipft=unset ! Local index for grid-cell, landunit, column, and patch + integer :: ggrc=unset, glun=unset, gcol=unset, gpft=unset ! Global index for grid-cell, landunit, column, and patch + integer :: bad_point = .false. ! Flag to indicate if the point is bad (i.e., global index is -1) !----------------------------------------------------------------------- if (subgrid_level == subgrid_level_gridcell) then igrc = subgrid_index + ggrc = get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell, donot_abort_on_badindex=.true.) + + else if (subgrid_level == subgrid_level_landunit) then + + ilun = subgrid_index + glun = get_global_index(subgrid_index=ilun, subgrid_level=subgrid_level_landunit, donot_abort_on_badindex=.true.) + if ( glun /= -1 ) then + igrc = lun%gridcell(ilun) + ggrc = get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell, donot_abort_on_badindex=.true.) + else + bad_point = .true. + end if + + else if (subgrid_level == subgrid_level_column) then + + icol = subgrid_index + gcol = get_global_index(subgrid_index=icol, subgrid_level=subgrid_level_column, donot_abort_on_badindex=.true.) + if ( gcol /= -1 ) then + ilun = col%landunit(icol) + igrc = col%gridcell(icol) + ggrc = get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell, donot_abort_on_badindex=.true.) + glun = get_global_index(subgrid_index=ilun, subgrid_level=subgrid_level_landunit, donot_abort_on_badindex=.true.) + else + bad_point = .true. + end if + + else if (subgrid_level == subgrid_level_patch) then + + ipft = subgrid_index + gpft = get_global_index(subgrid_index=ipft, subgrid_level=subgrid_level_patch, donot_abort_on_badindex=.true.) + if ( gpft /= -1 ) then + icol = patch%column(ipft) + ilun = patch%landunit(ipft) + igrc = patch%gridcell(ipft) + ggrc = get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell, donot_abort_on_badindex=.true.) + glun = get_global_index(subgrid_index=ilun, subgrid_level=subgrid_level_landunit, donot_abort_on_badindex=.true.) + gcol = get_global_index(subgrid_index=icol, subgrid_level=subgrid_level_column, donot_abort_on_badindex=.true.) + else + bad_point = .true. + end if + + end if + + ! If one of the global indices is -1 then this is a bad point, so flag a bad-point + if ( igrc /= unset) then + if ( ggrc == -1 ) bad_point = .true. + end if + if ( ilun /= unset) then + if ( glun == -1 ) bad_point = .true. + end if + if ( icol /= unset) then + if ( gcol == -1 ) bad_point = .true. + end if + if ( ipft /= unset) then + if ( gpft == -1 ) bad_point = .true. + end if + + if (bad_point) then + write(iulog,*) 'A bad input point was given: subgrid_index = ', subgrid_index, & + ', subgrid_level = ', subgrid_level + write(iulog,*) errmsg(sourcefile, __LINE__) + write(iulog,*) 'Continuing the endrun without writing point context information' + return + end if + + if (subgrid_level == subgrid_level_gridcell) then + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': local gridcell index = ', igrc - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', & - get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell) + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', ggrc write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell longitude = ', grc%londeg(igrc) write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell latitude = ', grc%latdeg(igrc) else if (subgrid_level == subgrid_level_landunit) then - ilun = subgrid_index - igrc = lun%gridcell(ilun) write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': local landunit index = ', ilun - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global landunit index = ', & - get_global_index(subgrid_index=ilun, subgrid_level=subgrid_level_landunit) - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', & - get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell) + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global landunit index = ', glun + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', ggrc write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell longitude = ', grc%londeg(igrc) write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell latitude = ', grc%latdeg(igrc) write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': landunit type = ', lun%itype(subgrid_index) else if (subgrid_level == subgrid_level_column) then - icol = subgrid_index - ilun = col%landunit(icol) - igrc = col%gridcell(icol) write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': local column index = ', icol - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global column index = ', & - get_global_index(subgrid_index=icol, subgrid_level=subgrid_level_column) - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global landunit index = ', & - get_global_index(subgrid_index=ilun, subgrid_level=subgrid_level_landunit) - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', & - get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell) + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global column index = ', gcol + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global landunit index = ', glun + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', ggrc write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell longitude = ', grc%londeg(igrc) write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell latitude = ', grc%latdeg(igrc) write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': column type = ', col%itype(icol) @@ -161,19 +220,11 @@ subroutine write_point_context(subgrid_index, subgrid_level) else if (subgrid_level == subgrid_level_patch) then - ipft = subgrid_index - icol = patch%column(ipft) - ilun = patch%landunit(ipft) - igrc = patch%gridcell(ipft) write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': local patch index = ', ipft - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global patch index = ', & - get_global_index(subgrid_index=ipft, subgrid_level=subgrid_level_patch) - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global column index = ', & - get_global_index(subgrid_index=icol, subgrid_level=subgrid_level_column) - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global landunit index = ', & - get_global_index(subgrid_index=ilun, subgrid_level=subgrid_level_landunit) - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', & - get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell) + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global patch index = ', gpft + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global column index = ', gcol + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global landunit index = ', glun + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', ggrc write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell longitude = ', grc%londeg(igrc) write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell latitude = ', grc%latdeg(igrc) write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': pft type = ', patch%itype(ipft) @@ -184,6 +235,7 @@ subroutine write_point_context(subgrid_index, subgrid_level) write(iulog,*) 'subgrid_level not supported: ', subgrid_level write(iulog,*) errmsg(sourcefile, __LINE__) write(iulog,*) 'Continuing the endrun without writing point context information' + return end if end subroutine write_point_context From 067fe25dd908b3c81d3fa4f79c10284a787941e5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 18:49:34 -0600 Subject: [PATCH 099/196] Add more error checking if a bad point is input to get_global_index, and add an option to it to avoid the abort, which is needed when it's used from an endrun call with pt_context --- src/main/decompMod.F90 | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/main/decompMod.F90 b/src/main/decompMod.F90 index 940ba724bf..3603f12cbf 100644 --- a/src/main/decompMod.F90 +++ b/src/main/decompMod.F90 @@ -361,7 +361,7 @@ integer function get_proc_clumps() end function get_proc_clumps !----------------------------------------------------------------------- - integer function get_global_index(subgrid_index, subgrid_level) + integer function get_global_index(subgrid_index, subgrid_level, donot_abort_on_badindex) !---------------------------------------------------------------- ! Description @@ -373,23 +373,47 @@ integer function get_global_index(subgrid_index, subgrid_level) ! Arguments integer , intent(in) :: subgrid_index ! index of interest (can be at any subgrid level or gridcell level) integer , intent(in) :: subgrid_level ! one of the subgrid_level_* constants defined above + logical , intent(in), optional :: donot_abort_on_badindex ! Don't abort if given a bad index ! ! Local Variables: type(bounds_type) :: bounds_proc ! processor bounds integer :: beg_index ! beginning proc index for subgrid_level + integer :: end_index ! ending proc index for subgrid_level + integer :: index ! index of the point to get integer, pointer :: gindex(:) + logical :: abort_on_badindex = .true. !---------------------------------------------------------------- + if (present(donot_abort_on_badindex)) then + abort_on_badindex = .not. donot_abort_on_badindex + end if call get_proc_bounds(bounds_proc, allow_call_from_threaded_region=.true.) beg_index = get_beg(bounds_proc, subgrid_level) + end_index = get_end(bounds_proc, subgrid_level) if (beg_index == -1) then write(iulog,*) 'get_global_index: subgrid_level not supported: ', subgrid_level - call shr_sys_abort('subgrid_level not supported' // & - errmsg(sourcefile, __LINE__)) + if (abort_on_badindex) then + call shr_sys_abort('subgrid_level not supported') + else + get_global_index = -1 + return + end if end if call get_subgrid_level_gindex(subgrid_level=subgrid_level, gindex=gindex) - get_global_index = gindex(subgrid_index - beg_index + 1) + index = subgrid_index - beg_index + 1 + if ( (index < beg_index) .or. (index > end_index) ) then + if (abort_on_badindex) then + write(iulog,*) 'get_global_index: subgrid_index out of bounds: ', & + 'subgrid_index = ', subgrid_index, ', beg_index = ', beg_index, & + ', end_index = ', end_index, ', subgrid_level = ', subgrid_level + call shr_sys_abort('subgrid_index out of bounds') + else + get_global_index = -1 + return + end if + end if + get_global_index = gindex(index) end function get_global_index @@ -537,7 +561,7 @@ subroutine get_subgrid_level_gindex (subgrid_level, gindex) gindex => gindex_cohort case default write(iulog,*) 'get_subgrid_level_gindex: unknown subgrid_level: ', subgrid_level - call shr_sys_abort() + call shr_sys_abort('bad subgrid_level') end select end subroutine get_subgrid_level_gindex From 323dc8a91ceeb1a738bb84aac8f84f3e5f75bacf Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 20:16:59 -0600 Subject: [PATCH 100/196] Allow passing file and line to endrun, output it through shr_log_errMsg if both are present --- src/main/abortutils.F90 | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/main/abortutils.F90 b/src/main/abortutils.F90 index 517b80fffa..886afccdef 100644 --- a/src/main/abortutils.F90 +++ b/src/main/abortutils.F90 @@ -10,6 +10,8 @@ module abortutils ! in conjunction with aborting the model, or at least issuing a warning. !----------------------------------------------------------------------- + use shr_log_mod , only : errMsg => shr_log_errMsg + implicit none private @@ -27,7 +29,7 @@ module abortutils contains !----------------------------------------------------------------------- - subroutine endrun_vanilla(msg, additional_msg) + subroutine endrun_vanilla(msg, additional_msg, line, file) !----------------------------------------------------------------------- ! !DESCRIPTION: @@ -40,10 +42,12 @@ subroutine endrun_vanilla(msg, additional_msg) ! Generally you want to at least provide msg. The main reason to separate msg from ! additional_msg is to supported expected-exception unit testing: you can put ! volatile stuff in additional_msg, as in: - ! call endrun(msg='Informative message', additional_msg=errmsg(__FILE__, __LINE__)) + ! call endrun(msg='Informative message', additional_msg=datetime ) ! and then just assert against msg. character(len=*), intent(in), optional :: msg ! string to be passed to shr_abort character(len=*), intent(in), optional :: additional_msg ! string to be printed, but not passed to shr_abort + integer , intent(in), optional :: line ! Line number for the endrun call + character(len=*), intent(in), optional :: file ! file for the endrun call !----------------------------------------------------------------------- if (present (additional_msg)) then @@ -52,12 +56,16 @@ subroutine endrun_vanilla(msg, additional_msg) write(iulog,*)'ENDRUN:' end if - call shr_abort_abort(msg) + ! Don't pass file and line to shr_abort_abort since the PFUNIT test version doesn't have those options + if ( present(file) .and. present(line) ) then + write(iulog,*) errMsg(file, line) + end if + call shr_abort_abort(string=msg) end subroutine endrun_vanilla !----------------------------------------------------------------------- - subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additional_msg) + subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additional_msg, line, file) !----------------------------------------------------------------------- ! Description: @@ -71,12 +79,8 @@ subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additio ! Arguments: integer , intent(in) :: subgrid_index ! index of interest (can be at any subgrid level or gridcell level) integer , intent(in) :: subgrid_level ! one of the subgrid_level_* constants defined in decompMod; subgrid_level_unspecified is allowed here, in which case the additional information will not be printed - - ! Generally you want to at least provide msg. The main reason to separate msg from - ! additional_msg is to supported expected-exception unit testing: you can put - ! volatile stuff in additional_msg, as in: - ! call endrun(msg='Informative message', additional_msg=errmsg(__FILE__, __LINE__)) - ! and then just assert against msg. + integer , intent(in), optional :: line ! Line number for the endrun call + character(len=*), intent(in), optional :: file !file for the endrun call character(len=*), intent(in), optional :: msg ! string to be passed to shr_abort character(len=*), intent(in), optional :: additional_msg ! string to be printed, but not passed to shr_abort ! @@ -88,7 +92,7 @@ subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additio call write_point_context(subgrid_index, subgrid_level) end if - call endrun_vanilla(msg=msg, additional_msg=additional_msg) + call endrun_vanilla(msg=msg, additional_msg=additional_msg, line=line, file=file) end subroutine endrun_write_point_context @@ -102,7 +106,6 @@ subroutine write_point_context(subgrid_index, subgrid_level) ! ! NOTE: DO NOT CALL AN ABORT FROM HERE AS THAT WOULD SHORT CIRUIT THE ERROR REPORTING!! ! - use shr_log_mod , only : errMsg => shr_log_errMsg use clm_varctl , only : iulog use decompMod , only : subgrid_level_gridcell, subgrid_level_landunit, subgrid_level_column, subgrid_level_patch use decompMod , only : get_global_index @@ -169,6 +172,9 @@ subroutine write_point_context(subgrid_index, subgrid_level) end if + ! + ! Badpoint should already be determined, but check again in case one of the subsequent + ! calls to get_global_index returns -1 ! If one of the global indices is -1 then this is a bad point, so flag a bad-point if ( igrc /= unset) then if ( ggrc == -1 ) bad_point = .true. @@ -186,7 +192,7 @@ subroutine write_point_context(subgrid_index, subgrid_level) if (bad_point) then write(iulog,*) 'A bad input point was given: subgrid_index = ', subgrid_index, & ', subgrid_level = ', subgrid_level - write(iulog,*) errmsg(sourcefile, __LINE__) + write(iulog,*) errMsg(sourcefile, __LINE__) write(iulog,*) 'Continuing the endrun without writing point context information' return end if @@ -233,7 +239,7 @@ subroutine write_point_context(subgrid_index, subgrid_level) else write(iulog,*) 'subgrid_level not supported: ', subgrid_level - write(iulog,*) errmsg(sourcefile, __LINE__) + write(iulog,*) errMsg(sourcefile, __LINE__) write(iulog,*) 'Continuing the endrun without writing point context information' return end if From 608e44043950e0ea2d43268262ab509d4964ce98 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 20:35:56 -0600 Subject: [PATCH 101/196] Add some tests for sending in file and line and one with just file to show it still works even if only one is present --- src/main/test/endrun_test/test_endrun.pf | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf index 9b3edb7bc6..6dfec9b799 100644 --- a/src/main/test/endrun_test/test_endrun.pf +++ b/src/main/test/endrun_test/test_endrun.pf @@ -25,12 +25,16 @@ contains subroutine setUp(this) use unittestSimpleSubgridSetupsMod, only : setup_single_veg_patch + use GridcellType , only : grc class(TestAbortUtils), intent(inout) :: this ! Setup a single gridcell with one vegetated patch ! So there's only one: gridcell, landunit, column, patch ! This isn't needed for some tests, but doesn't hurt to do it call setup_single_veg_patch(pft_type=1) + ! Set lat and lon for this gridcell, so something is printed in the log + grc%londeg(1) = 255.0 + grc%latdeg(1) = 30.0 end subroutine setUp subroutine tearDown(this) @@ -55,6 +59,26 @@ contains end subroutine endrun_plain_vanilla_aborts + @Test + subroutine endrun_nomsg_file_line_vanilla_aborts(this) + ! Test vanilla operation of endrun with file and line number input + class(TestAbortUtils), intent(inout) :: this + + call endrun(line=1000, file='test_file.F90') + @assertExceptionRaised(endrun_msg('')) + + end subroutine endrun_nomsg_file_line_vanilla_aborts + + @Test + subroutine endrun_nomsg_onlyfile_vanilla_aborts(this) + ! Test vanilla operation of endrun with only file input + class(TestAbortUtils), intent(inout) :: this + + call endrun(file='test_file.F90') + @assertExceptionRaised(endrun_msg('')) + + end subroutine endrun_nomsg_onlyfile_vanilla_aborts + @Test subroutine endrun_msg_vanilla_aborts(this) ! Test vanilla operation of endrun with a message sent in From ccc7b2d0d5fe2ca3c657b9f0188e094a7f6943f4 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 16 Aug 2025 16:27:38 -0600 Subject: [PATCH 102/196] Correct data type of bad_point to logical --- src/main/abortutils.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/abortutils.F90 b/src/main/abortutils.F90 index 886afccdef..8afa4ef195 100644 --- a/src/main/abortutils.F90 +++ b/src/main/abortutils.F90 @@ -123,7 +123,7 @@ subroutine write_point_context(subgrid_index, subgrid_level) integer, parameter :: unset = -9999 ! Unset value for an index integer :: igrc=unset, ilun=unset, icol=unset, ipft=unset ! Local index for grid-cell, landunit, column, and patch integer :: ggrc=unset, glun=unset, gcol=unset, gpft=unset ! Global index for grid-cell, landunit, column, and patch - integer :: bad_point = .false. ! Flag to indicate if the point is bad (i.e., global index is -1) + logical :: bad_point = .false. ! Flag to indicate if the point is bad (i.e., global index is -1) !----------------------------------------------------------------------- if (subgrid_level == subgrid_level_gridcell) then From f1856abe7e200b3eb587ebf508f0ef9a7fabdfe5 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Sat, 16 Aug 2025 17:38:58 -0600 Subject: [PATCH 103/196] set_paramfile: Functionize _convert_to_output_dtype(). --- python/ctsm/param_utils/set_paramfile.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index f2a64404e3..57739a34ca 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -164,6 +164,14 @@ def _replace_nans_with_fill(ds_in_masked_scaled, var, chg, new_value): return new_value +def _convert_to_output_dtype(ds_out, var, new_value): + """ + Convert new_value from np.ndarray of strings to output data type + """ + new_value = new_value.astype(type(ds_out[var].dtype)) + return new_value + + def main(): """ Main entry point for the script. @@ -205,7 +213,7 @@ def main(): raise NotImplementedError(f"Can't set integer parameter to fill value: {chg}") # Convert to the output data type - new_value = new_value.astype(type(ds_out[var].dtype)) + new_value = _convert_to_output_dtype(ds_out, var, new_value) # Are we acting on just some PFTs? If so, we'll need some stuff. just_some_pfts = PFTNAME_VAR in ds_out[var].coords and args.pft From bc2f0681b9e554b2f15545c6115e286943733273 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Sat, 16 Aug 2025 17:52:38 -0600 Subject: [PATCH 104/196] set_paramfile: Improve/test error when given float for integer param. --- python/ctsm/param_utils/set_paramfile.py | 14 +++-- python/ctsm/test/test_sys_set_paramfile.py | 63 ++++++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/python/ctsm/param_utils/set_paramfile.py b/python/ctsm/param_utils/set_paramfile.py index 57739a34ca..4019c4b63c 100644 --- a/python/ctsm/param_utils/set_paramfile.py +++ b/python/ctsm/param_utils/set_paramfile.py @@ -164,11 +164,19 @@ def _replace_nans_with_fill(ds_in_masked_scaled, var, chg, new_value): return new_value -def _convert_to_output_dtype(ds_out, var, new_value): +def _convert_to_output_dtype(ds_out, var, new_value, chg): """ Convert new_value from np.ndarray of strings to output data type """ - new_value = new_value.astype(type(ds_out[var].dtype)) + try: + new_value = new_value.astype(type(ds_out[var].dtype)) + except ValueError as e: + msg = str(e) + if "invalid literal for int() with base 10" in msg: + # Throw a nicer error message including the entire requested change + raise ValueError(f"Invalid assignment to an integer parameter: {chg}") from e + raise e + return new_value @@ -213,7 +221,7 @@ def main(): raise NotImplementedError(f"Can't set integer parameter to fill value: {chg}") # Convert to the output data type - new_value = _convert_to_output_dtype(ds_out, var, new_value) + new_value = _convert_to_output_dtype(ds_out, var, new_value, chg) # Are we acting on just some PFTs? If so, we'll need some stuff. just_some_pfts = PFTNAME_VAR in ds_out[var].coords and args.pft diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 2dba09b788..57f26405ed 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -655,6 +655,69 @@ def test_set_paramfile_setparams_just_one_pft_dropothers_doset(self): else: self.assertTrue(are_paramfile_dataarrays_identical(da_in, da_out)) + def test_set_paramfile_int_errors_given_float_point0(self): + """ + Test that set_paramfile errors if given float value ending in .0 for an int field + """ + input_path = os.path.join(self.tempdir, "input.nc") + output_path = os.path.join(self.tempdir, "output.nc") + param_name = "param0" + + # Save a test paramfile with an int field + da = xr.DataArray(data=np.int32(3)) + ds = xr.Dataset(data_vars={param_name: da}) + ds.to_netcdf(input_path, encoding={param_name: {"_FillValue": None}}) + ds_in = open_paramfile(input_path) + self.assertTrue(sp.is_integer(ds_in[param_name].values)) + + # Try giving it a value ending in .0 + sys.argv = ["set_paramfile", "-i", input_path, "-o", output_path, f"{param_name}=4.0"] + with self.assertRaisesRegex(ValueError, "Invalid assignment to an integer parameter:"): + sp.main() + + def test_set_paramfile_int_errors_given_float_point1(self): + """ + Test that set_paramfile errors if given float value NOT ending in .0 for an int field + """ + input_path = os.path.join(self.tempdir, "input.nc") + output_path = os.path.join(self.tempdir, "output.nc") + param_name = "param0" + + # Save a test paramfile with an int field + da = xr.DataArray(data=np.int32(3)) + ds = xr.Dataset(data_vars={param_name: da}) + ds.to_netcdf(input_path, encoding={param_name: {"_FillValue": None}}) + ds_in = open_paramfile(input_path) + self.assertTrue(sp.is_integer(ds_in[param_name].values)) + + # Try giving it a value ending in .1 + sys.argv = ["set_paramfile", "-i", input_path, "-o", output_path, f"{param_name}=4.1"] + with self.assertRaisesRegex(ValueError, "Invalid assignment to an integer parameter:"): + sp.main() + + def test_set_paramfile_double_ok_given_int(self): + """ + Test that set_paramfile works if given int value for a double field + """ + input_path = os.path.join(self.tempdir, "input.nc") + output_path = os.path.join(self.tempdir, "output.nc") + param_name = "param0" + + # Save a test paramfile with a double field + da = xr.DataArray(data=np.float32(3.14)) + ds = xr.Dataset(data_vars={param_name: da}) + ds.to_netcdf(input_path, encoding={param_name: {"_FillValue": None}}) + ds_in = open_paramfile(input_path) + self.assertFalse(sp.is_integer(ds_in[param_name].values)) + + # Give it an integer + sys.argv = ["set_paramfile", "-i", input_path, "-o", output_path, f"{param_name}=4"] + sp.main() + + # Check that it's still a double after saving + ds_out = open_paramfile(output_path) + self.assertFalse(sp.is_integer(ds_out[param_name].values)) + if __name__ == "__main__": unit_testing.setup_for_tests() From 0207b469b89d11bdb8d461628a724aacc599beab Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Sun, 17 Aug 2025 11:22:40 -0600 Subject: [PATCH 105/196] Move are_dicts_identical_nansequal() to netcdf_utils. --- python/ctsm/netcdf_utils.py | 37 ++++++++++++++++++++++++++++++++++--- python/ctsm/utils.py | 35 ----------------------------------- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/python/ctsm/netcdf_utils.py b/python/ctsm/netcdf_utils.py index 93d29a2a5e..762e1d9433 100644 --- a/python/ctsm/netcdf_utils.py +++ b/python/ctsm/netcdf_utils.py @@ -6,7 +6,38 @@ import xarray as xr from netCDF4 import Dataset # pylint: disable=no-name-in-module -from ctsm.utils import are_dicts_identical_nansequal + +def _are_dicts_identical_nansequal(dict0: dict, dict1: dict, keys_to_ignore=None): + """ + Compare two dictionaries, considering NaNs to be equal + """ + # pylint: disable=too-many-return-statements + + if keys_to_ignore is None: + keys_to_ignore = [] + keys_to_ignore = np.array(keys_to_ignore) + + if len(dict0) != len(dict1): + return False + for key, value0 in dict0.items(): + if key in keys_to_ignore: + continue + if key not in dict1: + return False + value1 = dict1[key] + if isinstance(value0, np.ndarray): + if not isinstance(value0, np.ndarray): + return False + if not np.array_equal(value0, value1, equal_nan=True): + return False + elif value1 != value0: + try: + if not (np.isnan(value0) and np.isnan(value1)): + return False + except TypeError: + return False + + return True def get_netcdf_format(file_path): @@ -28,11 +59,11 @@ def _is_dataarray_metadata_identical(da0: xr.DataArray, da1: xr.DataArray, keys_ return False # Check encoding - if not are_dicts_identical_nansequal(da0.encoding, da1.encoding, keys_to_ignore=keys_to_ignore): + if not _are_dicts_identical_nansequal(da0.encoding, da1.encoding, keys_to_ignore=keys_to_ignore): return False # Check attributes - if not are_dicts_identical_nansequal(da0.attrs, da1.attrs): + if not _are_dicts_identical_nansequal(da0.attrs, da1.attrs): return False # Check name diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index 8c8472c386..0b529fe282 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -11,8 +11,6 @@ from datetime import date, timedelta from getpass import getuser -import numpy as np - from ctsm.git_utils import get_ctsm_git_short_hash logger = logging.getLogger(__name__) @@ -269,36 +267,3 @@ def find_one_file_matching_pattern(pattern): f"Expected 1 but found {n_found} files found matching pattern: " + pattern ) return file_list[0] - - -def are_dicts_identical_nansequal(dict0: dict, dict1: dict, keys_to_ignore=None): - """ - Compare two dictionaries, considering NaNs to be equal - """ - # pylint: disable=too-many-return-statements - - if keys_to_ignore is None: - keys_to_ignore = [] - keys_to_ignore = np.array(keys_to_ignore) - - if len(dict0) != len(dict1): - return False - for key, value0 in dict0.items(): - if key in keys_to_ignore: - continue - if key not in dict1: - return False - value1 = dict1[key] - if isinstance(value0, np.ndarray): - if not isinstance(value0, np.ndarray): - return False - if not np.array_equal(value0, value1, equal_nan=True): - return False - elif value1 != value0: - try: - if not (np.isnan(value0) and np.isnan(value1)): - return False - except TypeError: - return False - - return True From f36f3f9639e6ccbcfb7b435d4497560fa8e65561 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Sun, 17 Aug 2025 11:38:01 -0600 Subject: [PATCH 106/196] Add SETPARAMFILE SystemTest. --- cime_config/SystemTests/setparamfile.py | 95 +++++++++++++++++++++++++ cime_config/config_tests.xml | 10 +++ cime_config/testdefs/testlist_clm.xml | 10 +++ python/ctsm/run_sys_tests.py | 8 +++ 4 files changed, 123 insertions(+) create mode 100644 cime_config/SystemTests/setparamfile.py diff --git a/cime_config/SystemTests/setparamfile.py b/cime_config/SystemTests/setparamfile.py new file mode 100644 index 0000000000..f106a9505a --- /dev/null +++ b/cime_config/SystemTests/setparamfile.py @@ -0,0 +1,95 @@ +""" +CTSM-specific test that first runs the set_paramfile tool and then ensures that CTSM does not fail +using the just-generated parameter file +""" + +import os +import sys +import logging +import re +from CIME.SystemTests.system_tests_common import SystemTestsCommon + +# In case we need to import set_paramfile later +_CTSM_PYTHON = os.path.join( + os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, "python" +) +sys.path.insert(1, _CTSM_PYTHON) + +logger = logging.getLogger(__name__) + + +class SETPARAMFILE(SystemTestsCommon): + def __init__(self, case): + """ + initialize an object interface to the SMS system test + """ + SystemTestsCommon.__init__(self, case) + + # Create out-of-the-box lnd_in to obtain paramfile + case.create_namelists(component="lnd") + + # Find the paramfile to modify + lnd_in_path = os.path.join(self._get_caseroot(), "CaseDocs", "lnd_in") + self._paramfile_in = None + with open(lnd_in_path, "r", encoding="utf-8") as lnd_in: + for line in lnd_in: + paramfile_in = re.match(r" *paramfile *= *'(.*)'", line) + if paramfile_in: + self._paramfile_in = paramfile_in.group(1) + break + if not self._paramfile_in: + raise RuntimeError(f"paramfile not found in {lnd_in_path}") + + # Get the output file + self.paramfile_out = os.path.join(self._get_caseroot(), "paramfile.nc") + + # Define set_paramfile command + self.set_paramfile_cmd = [ + "set_paramfile", + "-i", + self._paramfile_in, + "-o", + self.paramfile_out, + # Change two parameters for one PFT + "-p", + "needleleaf_deciduous_boreal_tree", + "rswf_min=0.35", + "rswf_max=0.7", + ] + + def build_phase(self, sharedlib_only=False, model_only=False): + """ + Run set_paramfile and then build the model + """ + + # Run set_paramfile. + # build_phase gets called twice: + # - once with sharedlib_only = True and + # - once with model_only = True + # Because we only need set_paramfile run once, we only do it for the sharedlib_only call. + # We could also check for the existence of the set_paramfile outputs, but that might lead to + # a situation where the user expects set_paramfile to be called but it's not. Better to run + # unnecessarily (e.g., if you fixed some FORTRAN code and just need to rebuild). + if sharedlib_only: + self._run_set_paramfile() + + # Do the build + self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) + + def _run_set_paramfile(self): + """ + Run set_paramfile + """ + # Import set_paramfile. Do it here rather than at top because otherwise the import will + # be attempted even during RUN phase. + # pylint: disable=wrong-import-position,import-outside-toplevel + from ctsm.param_utils.set_paramfile import main as set_paramfile + + # Run set_paramfile + sys.argv = self.set_paramfile_cmd + set_paramfile() + + # Append + user_nl_clm_path = os.path.join(self._get_caseroot(), "user_nl_clm") + with open(user_nl_clm_path, "a", encoding="utf-8") as user_nl_clm: + user_nl_clm.write(f"paramfile = '{self.paramfile_out}'\n") diff --git a/cime_config/config_tests.xml b/cime_config/config_tests.xml index 12859b9131..6df0f357e4 100644 --- a/cime_config/config_tests.xml +++ b/cime_config/config_tests.xml @@ -155,6 +155,16 @@ This defines various CTSM-specific system tests $STOP_N + + Modify a copy of the paramfile and run with it. + 1 + FALSE + FALSE + never + $STOP_OPTION + $STOP_N + + - + + + + + + + + + + + + + + + + + + + + + + @@ -4174,7 +4195,7 @@ - + @@ -4183,7 +4204,7 @@ - + @@ -4192,7 +4213,7 @@ - + @@ -4201,7 +4222,7 @@ - + @@ -4210,7 +4231,7 @@ - + From f5fe3599e89ded7751f2e56b3e562bf35542aaaa Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 20 Aug 2025 13:22:52 -0600 Subject: [PATCH 143/196] Make decomp_init single grid case run with mpi-serial --- cime_config/testdefs/testlist_clm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 0d4a56ad54..6c3253564a 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -4165,7 +4165,7 @@ - + From fe1a99326a624c9c7a16fe6682cb25bb6e104e1c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 20 Aug 2025 16:59:51 -0600 Subject: [PATCH 144/196] Remove turning on decomp init testing for now --- .../testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm | 1 - 1 file changed, 1 deletion(-) diff --git a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm index ff8468ea50..6187386336 100644 --- a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm @@ -1,2 +1 @@ for_testing_run_ncdiopio_tests = .true. -for_testing_run_decomp_init_tests = .true. From d858665d799690d73b56bcb961684382551193f4 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 20 Aug 2025 17:28:27 -0600 Subject: [PATCH 145/196] Reformat with black. --- python/ctsm/crop_calendars/generate_gdds.py | 4 +--- python/ctsm/crop_calendars/import_ds.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/python/ctsm/crop_calendars/generate_gdds.py b/python/ctsm/crop_calendars/generate_gdds.py index cb69d9e230..308431d003 100644 --- a/python/ctsm/crop_calendars/generate_gdds.py +++ b/python/ctsm/crop_calendars/generate_gdds.py @@ -280,9 +280,7 @@ def make_dummy(this_crop_gridded, addend): for var_index, this_var in enumerate(dummy_vars): if this_var in gdd_maps_ds: - error( - logger, f"{this_var} is already in gdd_maps_ds. Why overwrite it with dummy?" - ) + error(logger, f"{this_var} is already in gdd_maps_ds. Why overwrite it with dummy?") dummy_gridded.name = this_var dummy_gridded.attrs["long_name"] = dummy_longnames[var_index] gdd_maps_ds[this_var] = dummy_gridded diff --git a/python/ctsm/crop_calendars/import_ds.py b/python/ctsm/crop_calendars/import_ds.py index d2ace51ef0..66a0ec9746 100644 --- a/python/ctsm/crop_calendars/import_ds.py +++ b/python/ctsm/crop_calendars/import_ds.py @@ -254,9 +254,7 @@ def import_ds( filetime_sel = utils.safer_timeslice(filetime, time_slice) include_this_file = filetime_sel.size if include_this_file: - log( - logger, f"Including filetime : {filetime_sel['time'].values}" - ) + log(logger, f"Including filetime : {filetime_sel['time'].values}") new_filelist.append(file) # If you found some matching files, but then you find one that doesn't, stop going From b5db20aaf2554defba54b0cb736c54a1f49e2f3e Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 20 Aug 2025 17:28:45 -0600 Subject: [PATCH 146/196] Add previous commit to .git-blame-ignore-revs. --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 1aeb4a4f37..9badbdf673 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -71,3 +71,4 @@ cdf40d265cc82775607a1bf25f5f527bacc97405 75db098206b064b8b7b2a0604d3f0bf8fdb950cc 84609494b54ea9732f64add43b2f1dd035632b4c ac03492012837799b7111607188acff9f739044a +d858665d799690d73b56bcb961684382551193f4 From ac5899bc3692241331d800ac0383164d315baa49 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 20 Aug 2025 17:29:32 -0600 Subject: [PATCH 147/196] Small change for pylint. --- python/ctsm/test/test_unit_ctsm_logging.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ctsm/test/test_unit_ctsm_logging.py b/python/ctsm/test/test_unit_ctsm_logging.py index edfd817a46..d1b4891b17 100755 --- a/python/ctsm/test/test_unit_ctsm_logging.py +++ b/python/ctsm/test/test_unit_ctsm_logging.py @@ -5,7 +5,6 @@ import unittest import io from contextlib import redirect_stdout -from datetime import datetime from ctsm import unit_testing from ctsm.ctsm_logging import log, error From 7b80a4e86e5f6fb5ee6c4f3df4331a6418908092 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 11 Jul 2025 14:33:32 -0600 Subject: [PATCH 148/196] Turn off restarts and history and add some timer options as well as turning off ncdio_pio testing for the purposes of decompInit work Conflicts: cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm --- .../testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm index 6187386336..9e8e0fcd04 100644 --- a/cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/run_self_tests/user_nl_clm @@ -1 +1,5 @@ for_testing_run_ncdiopio_tests = .true. + +! Turn off history, restarts, and output +hist_empty_htapes = .true. +use_noio = .true. From d93d803ffc56408dae6fee25368bf8cf215de7e7 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 20 Aug 2025 19:37:15 -0600 Subject: [PATCH 149/196] Correct for testing namelist items to just the ncdip_pio tests which are on main dev right now --- .../clm/for_testing_fastsetup_bypassrun/user_nl_clm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/user_nl_clm index 573df5c02e..364443edd8 100644 --- a/cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/user_nl_clm @@ -1,5 +1,4 @@ -! Exit early and bypass the run phase -for_testing_exit_after_self_tests = .true. +for_testing_run_ncdiopio_tests = .true. ! Turn off history, restarts, and output hist_empty_htapes = .true. From e57d8f8917563919fc9076348fe75f222078b9fc Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 20 Aug 2025 20:34:48 -0600 Subject: [PATCH 150/196] Remove the self test turning on of ncdio_pio testing, which should just be in the run_self_tests testmod --- .../clm/for_testing_fastsetup_bypassrun/user_nl_clm | 2 -- 1 file changed, 2 deletions(-) diff --git a/cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/user_nl_clm index 364443edd8..c2a2d14793 100644 --- a/cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/user_nl_clm @@ -1,5 +1,3 @@ -for_testing_run_ncdiopio_tests = .true. - ! Turn off history, restarts, and output hist_empty_htapes = .true. use_noio = .true. From 5b5c40c1c500c80c9eee81ab6bca3726d4e3e7d4 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 16 Jul 2025 10:56:45 -0600 Subject: [PATCH 151/196] Suppress crop cold temp msgs when generate_crop_gdds true. These get printed a LOT if you put every crop everywhere. --- src/biogeochem/CNPhenologyMod.F90 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/biogeochem/CNPhenologyMod.F90 b/src/biogeochem/CNPhenologyMod.F90 index ff1e5b1b74..5e347e1e9f 100644 --- a/src/biogeochem/CNPhenologyMod.F90 +++ b/src/biogeochem/CNPhenologyMod.F90 @@ -3120,7 +3120,9 @@ subroutine vernalization(p, & tkil = (tbase - 6._r8) - 6._r8 * hdidx(p) if (tkil >= tcrown) then if ((0.95_r8 - 0.02_r8 * (tcrown - tkil)**2) >= 0.02_r8) then - write (iulog,*) 'crop damaged by cold temperatures at p,c =', p,c + if (.not. generate_crop_gdds) then + write (iulog,*) 'crop damaged by cold temperatures at p,c =', p,c + end if else if (tlai(p) > 0._r8) then ! slevis: kill if past phase1 by forcing through harvest ! srabin: do this with force_harvest instead of setting @@ -3129,7 +3131,9 @@ subroutine vernalization(p, & ! on "maturity." This can occur when generate_crop_gdds ! is true. force_harvest = .true. - write (iulog,*) '95% of crop killed by cold temperatures at p,c =', p,c + if (.not. generate_crop_gdds) then + write (iulog,*) '95% of crop killed by cold temperatures at p,c =', p,c + end if end if end if end if From 14a27f05ff4489b23a6f8955951d315632c03047 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 25 Jul 2025 13:04:49 -0600 Subject: [PATCH 152/196] Move log() and error() from generate_gdds_functions to cropcal_utils. --- python/ctsm/crop_calendars/cropcal_utils.py | 19 +++ python/ctsm/crop_calendars/generate_gdds.py | 25 +-- .../crop_calendars/generate_gdds_functions.py | 155 ++++++++---------- 3 files changed, 101 insertions(+), 98 deletions(-) diff --git a/python/ctsm/crop_calendars/cropcal_utils.py b/python/ctsm/crop_calendars/cropcal_utils.py index c7e8b6ac52..9ddcfb0194 100644 --- a/python/ctsm/crop_calendars/cropcal_utils.py +++ b/python/ctsm/crop_calendars/cropcal_utils.py @@ -9,6 +9,25 @@ from ctsm.utils import is_instantaneous +def log(logger_in, string): + """ + Simultaneously print INFO messages to console and to log file + """ + print(string) + if logger_in: + logger_in.info(string) + + +def error(logger_in, string): + """ + Simultaneously print ERROR messages to console and to log file + """ + print(string) + if logger_in: + logger_in.error(string) + raise RuntimeError(string) + + def define_pftlist(): """ Return list of PFTs used in CLM diff --git a/python/ctsm/crop_calendars/generate_gdds.py b/python/ctsm/crop_calendars/generate_gdds.py index bde28ca80d..0e17428fa1 100644 --- a/python/ctsm/crop_calendars/generate_gdds.py +++ b/python/ctsm/crop_calendars/generate_gdds.py @@ -20,6 +20,7 @@ sys.path.insert(1, _CTSM_PYTHON) import ctsm.crop_calendars.cropcal_module as cc # pylint: disable=wrong-import-position import ctsm.crop_calendars.generate_gdds_functions as gddfn # pylint: disable=wrong-import-position +import ctsm.crop_calendars.cropcal_utils as utils # pylint: disable=wrong-import-position # Functions here were written with too many positional arguments. At some point that should be # fixed. For now, we'll just disable the warning. @@ -85,11 +86,11 @@ def main( raise RuntimeError( "only_make_figs True but not all plotting modules are available" ) from exc - gddfn.log(logger, "Not all plotting modules are available; disabling save_figs") + utils.log(logger, "Not all plotting modules are available; disabling save_figs") save_figs = False # Print some info - gddfn.log(logger, f"Saving to {output_dir}") + utils.log(logger, f"Saving to {output_dir}") # Parse list of crops to skip if "," in skip_crops: @@ -107,7 +108,7 @@ def main( yr_1_import_str = f"{first_season+1}-01-01" yr_n_import_str = f"{last_season+2}-01-01" - gddfn.log( + utils.log( logger, f"Importing netCDF time steps {yr_1_import_str} through {yr_n_import_str} " + "(years are +1 because of CTSM output naming)", @@ -191,7 +192,7 @@ def main( h1_instantaneous, ) - gddfn.log(logger, f" Saving pickle file ({pickle_file})...") + utils.log(logger, f" Saving pickle file ({pickle_file})...") with open(pickle_file, "wb") as file: pickle.dump( [ @@ -219,7 +220,7 @@ def main( [i for i, c in enumerate(gddaccum_yp_list) if not isinstance(c, type(None))] ] - gddfn.log(logger, "Done") + utils.log(logger, "Done") if not h2_ds: h2_ds = xr.open_dataset(h2_ds_file) @@ -236,7 +237,7 @@ def main( "s", sdates_rx, incl_patches1d_itype_veg, mxsowings, logger ) - gddfn.log(logger, "Getting and gridding mean GDDs...") + utils.log(logger, "Getting and gridding mean GDDs...") gdd_maps_ds = gddfn.yp_list_to_ds( gddaccum_yp_list, h2_ds, incl_vegtypes_str, sdates_rx, longname_prefix, logger ) @@ -247,10 +248,10 @@ def main( # Fill NAs with dummy values dummy_fill = -1 gdd_maps_ds = gdd_maps_ds.fillna(dummy_fill) - gddfn.log(logger, "Done getting and gridding means.") + utils.log(logger, "Done getting and gridding means.") # Add dummy variables for crops not actually simulated - gddfn.log(logger, "Adding dummy variables...") + utils.log(logger, "Adding dummy variables...") # Unnecessary? template_ds = xr.open_dataset(sdates_file, decode_times=True) all_vars = [v.replace("sdate", "gdd") for v in template_ds if "sdate" in v] @@ -278,7 +279,7 @@ def make_dummy(this_crop_gridded, addend): for var_index, this_var in enumerate(dummy_vars): if this_var in gdd_maps_ds: - gddfn.error( + utils.error( logger, f"{this_var} is already in gdd_maps_ds. Why overwrite it with dummy?" ) dummy_gridded.name = this_var @@ -294,14 +295,14 @@ def add_lonlat_attrs(this_ds): gdd_maps_ds = add_lonlat_attrs(gdd_maps_ds) gddharv_maps_ds = add_lonlat_attrs(gddharv_maps_ds) - gddfn.log(logger, "Done.") + utils.log(logger, "Done.") ###################### ### Save to netCDF ### ###################### if not only_make_figs: - gddfn.log(logger, "Saving...") + utils.log(logger, "Saving...") # Get output file path datestr = dt.datetime.now().strftime("%Y%m%d_%H%M%S") @@ -336,7 +337,7 @@ def save_gdds(sdates_file, hdates_file, outfile, gdd_maps_ds, sdates_rx): save_gdds(sdates_file, hdates_file, outfile, gdd_maps_ds, sdates_rx) - gddfn.log(logger, "Done saving.") + utils.log(logger, "Done saving.") ######################################## ### Save things needed for mapmaking ### diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index 0489f320b7..6f8f71ea71 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -54,28 +54,11 @@ CAN_PLOT = False -def log(logger, string): - """ - Simultaneously print INFO messages to console and to log file - """ - print(string) - logger.info(string) - - -def error(logger, string): - """ - Simultaneously print ERROR messages to console and to log file - """ - print(string) - logger.error(string) - raise RuntimeError(string) - - def check_sdates(dates_ds, sdates_rx, outdir_figs, logger, verbose=False): """ Checking that input and output sdates match """ - log(logger, " Checking that input and output sdates match...") + utils.log(logger, " Checking that input and output sdates match...") sdates_grid = grid_one_variable(dates_ds, "SDATES") @@ -89,12 +72,12 @@ def check_sdates(dates_ds, sdates_rx, outdir_figs, logger, verbose=False): this_var = f"gs1_{vegtype_int}" if this_var not in sdates_rx: vegtypes_skipped = vegtypes_skipped + [vegtype_str] - # log(logger, f" {vt_str} ({vt}) SKIPPED...") + # utils.log(logger, f" {vt_str} ({vt}) SKIPPED...") continue vegtypes_included = vegtypes_included + [vegtype_str] any_found = True if verbose: - log(logger, f" {vegtype_str} ({vegtype_int})...") + utils.log(logger, f" {vegtype_str} ({vegtype_int})...") in_map = sdates_rx[this_var].squeeze(drop=True) # Output @@ -104,23 +87,23 @@ def check_sdates(dates_ds, sdates_rx, outdir_figs, logger, verbose=False): diff_map = out_map - in_map diff_map_notnan = diff_map.values[np.invert(np.isnan(diff_map.values))] if np.any(diff_map_notnan): - log(logger, f"Difference(s) found in {vegtype_str}") + utils.log(logger, f"Difference(s) found in {vegtype_str}") here = np.where(diff_map_notnan) - log(logger, "in:") + utils.log(logger, "in:") in_map_notnan = in_map.values[np.invert(np.isnan(diff_map.values))] - log(logger, in_map_notnan[here][0:4]) + utils.log(logger, in_map_notnan[here][0:4]) out_map_notnan = out_map.values[np.invert(np.isnan(diff_map.values))] - log(logger, "out:") - log(logger, out_map_notnan[here][0:4]) - log(logger, "diff:") - log(logger, diff_map_notnan[here][0:4]) + utils.log(logger, "out:") + utils.log(logger, out_map_notnan[here][0:4]) + utils.log(logger, "diff:") + utils.log(logger, diff_map_notnan[here][0:4]) first_diff = all_ok all_ok = False if CAN_PLOT: sdate_diffs_dir = os.path.join(outdir_figs, "sdate_diffs") if first_diff: - log(logger, f"Saving sdate difference figures to {sdate_diffs_dir}") + utils.log(logger, f"Saving sdate difference figures to {sdate_diffs_dir}") if not os.path.exists(sdate_diffs_dir): os.makedirs(sdate_diffs_dir) in_map.where(~np.isnan(out_map)).plot() @@ -137,24 +120,24 @@ def check_sdates(dates_ds, sdates_rx, outdir_figs, logger, verbose=False): plt.close() if not any_found: - error(logger, "No matching variables found in sdates_rx!") + utils.error(logger, "No matching variables found in sdates_rx!") # Sanity checks for included vegetation types vegtypes_skipped = np.unique([x.replace("irrigated_", "") for x in vegtypes_skipped]) vegtypes_skipped_weird = [x for x in vegtypes_skipped if x in vegtypes_included] if np.array_equal(vegtypes_included, [x.replace("irrigated_", "") for x in vegtypes_included]): - log(logger, "\nWARNING: No irrigated crops included!!!\n") + utils.log(logger, "\nWARNING: No irrigated crops included!!!\n") elif vegtypes_skipped_weird: - log( + utils.log( logger, "\nWarning: Some crop types had output rainfed patches but no irrigated patches: " + f"{vegtypes_skipped_weird}", ) if all_ok: - log(logger, " ✅ Input and output sdates match!") + utils.log(logger, " ✅ Input and output sdates match!") else: - error(logger, " ❌ Input and output sdates differ.") + utils.error(logger, " ❌ Input and output sdates differ.") def import_rx_dates(s_or_h, date_infile, incl_patches1d_itype_veg, mxsowings, logger): @@ -164,7 +147,7 @@ def import_rx_dates(s_or_h, date_infile, incl_patches1d_itype_veg, mxsowings, lo if isinstance(date_infile, xr.Dataset): return date_infile if not isinstance(date_infile, str): - error( + utils.error( logger, f"Importing {s_or_h}dates_rx: Expected date_infile to be str or DataArray," + f"not {type(date_infile)}", @@ -220,7 +203,7 @@ def yp_list_to_ds(yp_list, daily_ds, incl_vegtypes_str, dates_rx, longname_prefi if isinstance(data, type(None)): continue this_crop_str = incl_vegtypes_str[this_crop_int] - log(logger, f" {this_crop_str}...") + utils.log(logger, f" {this_crop_str}...") new_var = f"gdd1_{utils.ivt_str2int(this_crop_str)}" this_ds = daily_ds.isel( patch=np.where(daily_ds.patches1d_itype_veg_str.values == this_crop_str)[0] @@ -270,8 +253,8 @@ def import_and_process_1yr( Import one year of CLM output data for GDD generation """ save_figs = True - log(logger, f"netCDF year {this_year}...") - log(logger, dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + utils.log(logger, f"netCDF year {this_year}...") + utils.log(logger, dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) # Without dask, this can take a LONG time at resolutions finer than 2-deg if importlib_util.find_spec("dask"): @@ -286,7 +269,7 @@ def import_and_process_1yr( h1_pattern = os.path.join(indir, "*h1i.*.nc.base") h1_filelist = glob.glob(h1_pattern) if not h1_filelist: - error(logger, "No files found matching pattern '*h1i.*.nc(.base)'") + utils.error(logger, "No files found matching pattern '*h1i.*.nc(.base)'") # Get list of crops to include if skip_crops is not None: @@ -315,7 +298,7 @@ def import_and_process_1yr( if dates_ds.dims["time"] > 1: if dates_ds.dims["time"] == 365: if not incorrectly_daily: - log( + utils.log( logger, " ℹ️ You saved SDATES and HDATES daily, but you only needed annual. Fixing.", ) @@ -334,9 +317,9 @@ def import_and_process_1yr( ) n_unmatched_nans = np.sum(sdates_all_nan != hdates_all_nan) if n_unmatched_nans > 0: - error(logger, "Output SDATE and HDATE NaN masks do not match.") + utils.error(logger, "Output SDATE and HDATE NaN masks do not match.") if np.sum(~np.isnan(dates_ds.SDATES.values)) == 0: - error(logger, "All SDATES are NaN!") + utils.error(logger, "All SDATES are NaN!") # Just work with non-NaN patches for now skip_patches_for_isel_nan = np.where(sdates_all_nan)[0] @@ -345,7 +328,7 @@ def import_and_process_1yr( skip_patches_for_isel_nan_last_year, skip_patches_for_isel_nan ) if different_nan_mask: - log(logger, " Different NaN mask than last year") + utils.log(logger, " Different NaN mask than last year") incl_thisyr_but_nan_lastyr = [ dates_ds.patch.values[p] for p in incl_patches_for_isel_nan @@ -355,7 +338,7 @@ def import_and_process_1yr( incl_thisyr_but_nan_lastyr = [] skipping_patches_for_isel_nan = len(skip_patches_for_isel_nan) > 0 if skipping_patches_for_isel_nan: - log( + utils.log( logger, f" Ignoring {len(skip_patches_for_isel_nan)} patches with all-NaN sowing and " + "harvest dates.", @@ -374,14 +357,14 @@ def import_and_process_1yr( if isinstance(incl_vegtypes_str, np.ndarray): incl_vegtypes_str = list(incl_vegtypes_str) if incl_vegtypes_str != list(dates_incl_ds.vegtype_str.values): - error( + utils.error( logger, f"Included veg types differ. Previously {incl_vegtypes_str}, " + f"now {dates_incl_ds.vegtype_str.values}", ) if np.sum(~np.isnan(dates_incl_ds.SDATES.values)) == 0: - error(logger, "All SDATES are NaN after ignoring those patches!") + utils.error(logger, "All SDATES are NaN after ignoring those patches!") # Some patches can have -1 sowing date?? Hopefully just an artifact of me incorrectly saving # SDATES/HDATES daily. @@ -394,14 +377,14 @@ def import_and_process_1yr( dates_incl_ds.HDATES.isel(mxharvests=0, patch=skip_patches_for_isel_sdatelt1).values ) if incorrectly_daily and list(unique_hdates) == [364]: - log( + utils.log( logger, f" ❗ {len(skip_patches_for_isel_sdatelt1)} patches have SDATE < 1, but this" + "might have just been because of incorrectly daily outputs. Setting them to 365.", ) new_sdates_ar = dates_incl_ds.SDATES.values if mxsowings_dim != 0: - error(logger, "Code this up") + utils.error(logger, "Code this up") new_sdates_ar[0, skip_patches_for_isel_sdatelt1] = 365 dates_incl_ds["SDATES"] = xr.DataArray( data=new_sdates_ar, @@ -409,7 +392,7 @@ def import_and_process_1yr( attrs=dates_incl_ds["SDATES"].attrs, ) else: - error( + utils.error( logger, f"{len(skip_patches_for_isel_sdatelt1)} patches have SDATE < 1. " + f"Unique affected hdates: {unique_hdates}", @@ -432,10 +415,10 @@ def import_and_process_1yr( if np.any(hdates_thisyr_where_nan_lastyr < 1): new_hdates = dates_incl_ds.HDATES.values if mxharvests_dim != 0: - error(logger, "Code this up") + utils.error(logger, "Code this up") patch_list = list(hdates_thisyr.patch.values) here = [patch_list.index(x) for x in incl_thisyr_but_nan_lastyr] - log( + utils.log( logger, f" ❗ {len(here)} patches have harvest date -1 because they weren't active last" + "year (and were either never active or were harvested when last active). " @@ -460,7 +443,7 @@ def import_and_process_1yr( dates_incl_ds.SDATES.isel(patch=skip_patches_for_isel_hdatelt1).values ) if incorrectly_daily and list(unique_sdates) == [1]: - log( + utils.log( logger, f" ❗ {len(skip_patches_for_isel_hdatelt1)} patches have HDATE < 1??? Seems like " + "this might have just been because of incorrectly daily outputs; setting them to " @@ -468,7 +451,7 @@ def import_and_process_1yr( ) new_hdates_ar = dates_incl_ds.HDATES.values if mxharvests_dim != 0: - error(logger, "Code this up") + utils.error(logger, "Code this up") new_hdates_ar[0, skip_patches_for_isel_hdatelt1] = 365 dates_incl_ds["HDATES"] = xr.DataArray( data=new_hdates_ar, @@ -476,7 +459,7 @@ def import_and_process_1yr( attrs=dates_incl_ds["HDATES"].attrs, ) else: - error( + utils.error( logger, f"{len(skip_patches_for_isel_hdatelt1)} patches have HDATE < 1. Possible causes:\n" + "* Not using constant crop areas (e.g., flanduse_timeseries from " @@ -492,7 +475,7 @@ def import_and_process_1yr( >= 1 ) if n_extra_harv > 0: - error(logger, f"{n_extra_harv} patches have >1 harvest.") + utils.error(logger, f"{n_extra_harv} patches have >1 harvest.") # Make sure harvest happened the day before sowing sdates_clm = dates_incl_ds.SDATES.values.squeeze() @@ -500,7 +483,7 @@ def import_and_process_1yr( diffdates_clm = sdates_clm - hdates_clm diffdates_clm[(sdates_clm == 1) & (hdates_clm == 365)] = 1 if list(np.unique(diffdates_clm)) != [1]: - error(logger, f"Not all sdates-hdates are 1: {np.unique(diffdates_clm)}") + utils.error(logger, f"Not all sdates-hdates are 1: {np.unique(diffdates_clm)}") # Import expected sowing dates. This will also be used as our template output file. imported_sdates = isinstance(sdates_rx, str) @@ -563,7 +546,7 @@ def import_and_process_1yr( else: hdates_rx = hdates_rx_orig - log(logger, " Importing accumulated GDDs...") + utils.log(logger, " Importing accumulated GDDs...") clm_gdd_var = "GDDACCUM" my_vars = [clm_gdd_var, "GDDHARV"] patterns = [f"*h2i.{this_year-1}-01*.nc", f"*h2i.{this_year-1}-01*.nc.base"] @@ -573,7 +556,7 @@ def import_and_process_1yr( if h2_files: break if not h2_files: - error(logger, f"No files found matching patterns: {patterns}") + utils.error(logger, f"No files found matching patterns: {patterns}") h2_ds = import_ds( h2_files, my_vars=my_vars, @@ -584,13 +567,13 @@ def import_and_process_1yr( # Restrict to patches we're including if skipping_patches_for_isel_nan: if not np.array_equal(dates_ds.patch.values, h2_ds.patch.values): - error(logger, "dates_ds and h2_ds don't have the same patch list!") + utils.error(logger, "dates_ds and h2_ds don't have the same patch list!") h2_incl_ds = h2_ds.isel(patch=incl_patches_for_isel_nan) else: h2_incl_ds = h2_ds if not np.any(h2_incl_ds[clm_gdd_var].values != 0): - error(logger, f"All {clm_gdd_var} values are zero!") + utils.error(logger, f"All {clm_gdd_var} values are zero!") # Get standard datetime axis for outputs n_years = year_n - year_1 + 1 @@ -604,7 +587,7 @@ def import_and_process_1yr( incl_vegtype_indices = [] for var, vegtype_str in enumerate(incl_vegtypes_str): if vegtype_str in skip_crops: - log(logger, f" SKIPPING {vegtype_str}") + utils.log(logger, f" SKIPPING {vegtype_str}") continue vegtype_int = utils.vegtype_str2int(vegtype_str)[0] @@ -619,7 +602,7 @@ def import_and_process_1yr( check_gddharv = True if not this_crop_gddaccum_da.size: continue - log(logger, f" {vegtype_str}...") + utils.log(logger, f" {vegtype_str}...") incl_vegtype_indices = incl_vegtype_indices + [var] # Get prescribed harvest dates for these patches @@ -652,7 +635,7 @@ def import_and_process_1yr( if not np.all( this_crop_gddaccum_da.patch.values[:-1] <= this_crop_gddaccum_da.patch.values[1:] ): - error(logger, "This code depends on DataArray patch list being sorted.") + utils.error(logger, "This code depends on DataArray patch list being sorted.") sortorder = np.argsort(patches) i_patches = list(np.array(i_patches)[np.array(sortorder)]) i_times = list(np.array(i_times)[np.array(sortorder)]) @@ -662,17 +645,17 @@ def import_and_process_1yr( if save_figs: gddharv_atharv_p = this_crop_gddharv_da.values[(i_times, i_patches)] if np.any(np.isnan(gddaccum_atharv_p)): - log( + utils.log( logger, f" ❗ {np.sum(np.isnan(gddaccum_atharv_p))}/{len(gddaccum_atharv_p)} " + "NaN after extracting GDDs accumulated at harvest", ) if save_figs and gddharv_atharv_p is not None and np.any(np.isnan(gddharv_atharv_p)): if np.all(np.isnan(gddharv_atharv_p)): - log(logger, " ❗ All GDDHARV are NaN; should only affect figure") + utils.log(logger, " ❗ All GDDHARV are NaN; should only affect figure") check_gddharv = False else: - log( + utils.log( logger, f" ❗ {np.sum(np.isnan(gddharv_atharv_p))}/{len(gddharv_atharv_p)} " + "NaN after extracting GDDHARV", @@ -699,14 +682,14 @@ def import_and_process_1yr( ] if not np.array_equal(last_year_active_patch_indices, this_year_active_patch_indices): if incorrectly_daily: - log( + utils.log( logger, " ❗ This year's active patch indices differ from last year's. " + "Allowing because this might just be an artifact of incorrectly daily " + "outputs, BUT RESULTS MUST NOT BE TRUSTED.", ) else: - error(logger, "This year's active patch indices differ from last year's.") + utils.error(logger, "This year's active patch indices differ from last year's.") # Make sure we're not about to overwrite any existing values. if np.any( ~np.isnan( @@ -714,28 +697,28 @@ def import_and_process_1yr( ) ): if incorrectly_daily: - log( + utils.log( logger, " ❗ Unexpected non-NaN for last season's GDD accumulation. " + "Allowing because this might just be an artifact of incorrectly daily " + "outputs, BUT RESULTS MUST NOT BE TRUSTED.", ) else: - error(logger, "Unexpected non-NaN for last season's GDD accumulation") + utils.error(logger, "Unexpected non-NaN for last season's GDD accumulation") if save_figs and np.any( ~np.isnan( gddharv_yp_list[var][year_index - 1, active_this_year_where_gs_lastyr_indices] ) ): if incorrectly_daily: - log( + utils.log( logger, " ❗ Unexpected non-NaN for last season's GDDHARV. Allowing " + "because this might just be an artifact of incorrectly daily outputs, " + "BUT RESULTS MUST NOT BE TRUSTED.", ) else: - error(logger, "Unexpected non-NaN for last season's GDDHARV") + utils.error(logger, "Unexpected non-NaN for last season's GDDHARV") # Fill. gddaccum_yp_list[var][year_index - 1, active_this_year_where_gs_lastyr_indices] = ( gddaccum_atharv_p[where_gs_lastyr] @@ -751,14 +734,14 @@ def import_and_process_1yr( ) ): if incorrectly_daily: - log( + utils.log( logger, " ❗ Unexpected NaN for last season's GDD accumulation. Allowing " + "because this might just be an artifact of incorrectly daily outputs, " + "BUT RESULTS MUST NOT BE TRUSTED.", ) else: - error(logger, "Unexpected NaN for last season's GDD accumulation.") + utils.error(logger, "Unexpected NaN for last season's GDD accumulation.") if ( save_figs and check_gddharv @@ -771,14 +754,14 @@ def import_and_process_1yr( ) ): if incorrectly_daily: - log( + utils.log( logger, " ❗ Unexpected NaN for last season's GDDHARV. Allowing because " + "this might just be an artifact of incorrectly daily outputs, BUT " + "RESULTS MUST NOT BE TRUSTED.", ) else: - error(logger, "Unexpected NaN for last season's GDDHARV.") + utils.error(logger, "Unexpected NaN for last season's GDDHARV.") gddaccum_yp_list[var][year_index, this_year_active_patch_indices] = tmp_gddaccum if save_figs: gddharv_yp_list[var][year_index, this_year_active_patch_indices] = tmp_gddharv @@ -794,14 +777,14 @@ def import_and_process_1yr( nanmask_output_gdds_lastyr = np.isnan(gddaccum_yp_list[var][year_index - 1, :]) if not np.array_equal(nanmask_output_gdds_lastyr, nanmask_output_sdates): if incorrectly_daily: - log( + utils.log( logger, " ❗ NaN masks differ between this year's sdates and 'filled-out' " + "GDDs from last year. Allowing because this might just be an artifact of " + "incorrectly daily outputs, BUT RESULTS MUST NOT BE TRUSTED.", ) else: - error( + utils.error( logger, "NaN masks differ between this year's sdates and 'filled-out' GDDs from " + "last year", @@ -811,7 +794,7 @@ def import_and_process_1yr( skip_patches_for_isel_nan_last_year = skip_patches_for_isel_nan # Could save space by only saving variables needed for gridding - log(logger, " Saving h2_ds...") + utils.log(logger, " Saving h2_ds...") h2_ds.to_netcdf(h2_ds_file) return ( @@ -1081,7 +1064,7 @@ def make_figures( """ if not gdd_maps_ds: if not this_dir: - error( + utils.error( logger, "If not providing gdd_maps_ds, you must provide thisDir (location of " + "gdd_maps.nc)", @@ -1089,7 +1072,7 @@ def make_figures( gdd_maps_ds = xr.open_dataset(this_dir + "gdd_maps.nc") if not gddharv_maps_ds: if not this_dir: - error( + utils.error( logger, "If not providing gddharv_maps_ds, you must provide thisDir (location of " + "gddharv_maps.nc)", @@ -1116,7 +1099,7 @@ def make_figures( if land_use_file: year_1_lu = year_1 if first_land_use_year is None else first_land_use_year year_n_lu = year_n if last_land_use_year is None else last_land_use_year - lu_ds = cc.open_lu_ds(land_use_file, year_1_lu, year_n_lu, gdd_maps_ds, ungrid=False) + lu_ds = cc.open_lu_ds(land_use_file, year_1_lu, year_n_lu, gdd_maps_ds, logger, ungrid=False) lu_years_text = f" (masked by {year_1_lu}-{year_n_lu} area)" lu_years_file = f"_mask{year_1_lu}-{year_n_lu}" else: @@ -1152,7 +1135,7 @@ def make_figures( # Maps nplot_y = 3 nplot_x = 1 - log(logger, "Making before/after maps...") + utils.log(logger, "Making before/after maps...") vegtype_list = incl_vegtypes_str if land_use_file: vegtype_list += ["Corn", "Cotton", "Rice", "Soybean", "Sugarcane", "Wheat"] @@ -1213,7 +1196,7 @@ def make_figures( spec = fig.add_gridspec(nrows=3, ncols=2, width_ratios=[0.5, 0.5], wspace=0.2) this_axis = fig.add_subplot(spec[0, 0], projection=ccrs.PlateCarree()) else: - error(logger, f"layout {layout} not recognized") + utils.error(logger, f"layout {layout} not recognized") gddharv_all_nan = np.all(np.isnan(gddharv_map_yx.values)) if gddharv_all_nan: @@ -1238,7 +1221,7 @@ def make_figures( elif layout in ["2x2", "3x2"]: this_axis = fig.add_subplot(spec[1, 0], projection=ccrs.PlateCarree()) else: - error(logger, f"layout {layout} not recognized") + utils.error(logger, f"layout {layout} not recognized") this_min = int(np.round(np.nanmin(gdd_map_yx))) this_max = int(np.round(np.nanmax(gdd_map_yx))) this_title = f"{run2_name} (range {this_min}–{this_max})" @@ -1323,7 +1306,7 @@ def make_figures( elif layout in ["2x2", "3x2"]: this_axis = fig.add_subplot(spec[:, 1]) else: - error(logger, f"layout {layout} not recognized") + utils.error(logger, f"layout {layout} not recognized") # Shift bottom of plot up to make room for legend ax_pos = this_axis.get_position() @@ -1376,4 +1359,4 @@ def make_figures( plt.savefig(outfile, dpi=300, transparent=False, facecolor="white", bbox_inches="tight") plt.close() - log(logger, "Done.") + utils.log(logger, "Done.") From 444fa5440de015a95d62db7ae773b8d37ac7c3e5 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 31 Jul 2025 14:45:22 -0600 Subject: [PATCH 153/196] Add logging in GDD generation. --- python/ctsm/crop_calendars/cropcal_module.py | 7 ++-- python/ctsm/crop_calendars/generate_gdds.py | 1 + .../crop_calendars/generate_gdds_functions.py | 6 ++-- python/ctsm/crop_calendars/import_ds.py | 35 ++++++++++++++++--- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/python/ctsm/crop_calendars/cropcal_module.py b/python/ctsm/crop_calendars/cropcal_module.py index c7ee9c581a..57dc4600d8 100644 --- a/python/ctsm/crop_calendars/cropcal_module.py +++ b/python/ctsm/crop_calendars/cropcal_module.py @@ -16,7 +16,6 @@ MISSING_RX_GDD_VAL = -1 - def check_and_trim_years(year_1, year_n, ds_in): """ After importing a file, restrict it to years of interest. @@ -49,11 +48,12 @@ def check_and_trim_years(year_1, year_n, ds_in): return ds_in -def open_lu_ds(filename, year_1, year_n, existing_ds, ungrid=True): +def open_lu_ds(filename, year_1, year_n, existing_ds, logger, ungrid=True): """ Open land-use dataset """ # Open and trim to years of interest + utils.log(logger, f"open_lu_ds(): Opening this_ds_gridded: {filename}") this_ds_gridded = xr.open_dataset(filename).sel(time=slice(year_1, year_n)) # Assign actual lon/lat coordinates @@ -347,6 +347,7 @@ def import_output( gdds_rx_ds=None, verbose=False, throw_errors=True, + logger=None, ): """ Import CLM output @@ -354,7 +355,7 @@ def import_output( any_bad = False # Import - this_ds = import_ds(filename, my_vars=my_vars, my_vegtypes=my_vegtypes) + this_ds = import_ds(filename, my_vars=my_vars, my_vegtypes=my_vegtypes, logger=logger) # Trim to years of interest (do not include extra year needed for finishing last growing season) if year_1 and year_n: diff --git a/python/ctsm/crop_calendars/generate_gdds.py b/python/ctsm/crop_calendars/generate_gdds.py index 0e17428fa1..bc0063fabf 100644 --- a/python/ctsm/crop_calendars/generate_gdds.py +++ b/python/ctsm/crop_calendars/generate_gdds.py @@ -223,6 +223,7 @@ def main( utils.log(logger, "Done") if not h2_ds: + utils.log(logger, f"generate_gdds main(): Opening h2_ds: {h2_ds_file}") h2_ds = xr.open_dataset(h2_ds_file) ###################################################### diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index 6f8f71ea71..984059922f 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -291,6 +291,7 @@ def import_and_process_1yr( my_vegtypes=crops_to_read, time_slice=slice(f"{slice_year}-01-01", f"{slice_year}-12-31"), chunks=chunks, + logger=logger, ) for timestep in dates_ds["time"].values: print(timestep) @@ -562,6 +563,7 @@ def import_and_process_1yr( my_vars=my_vars, my_vegtypes=crops_to_read, chunks=chunks, + logger=logger, ) # Restrict to patches we're including @@ -587,7 +589,7 @@ def import_and_process_1yr( incl_vegtype_indices = [] for var, vegtype_str in enumerate(incl_vegtypes_str): if vegtype_str in skip_crops: - utils.log(logger, f" SKIPPING {vegtype_str}") + utils.log(logger, f" import_and_process_1yr(): SKIPPING {vegtype_str}") continue vegtype_int = utils.vegtype_str2int(vegtype_str)[0] @@ -602,7 +604,7 @@ def import_and_process_1yr( check_gddharv = True if not this_crop_gddaccum_da.size: continue - utils.log(logger, f" {vegtype_str}...") + utils.log(logger, f" import_and_process_1yr(): {vegtype_str}...") incl_vegtype_indices = incl_vegtype_indices + [var] # Get prescribed harvest dates for these patches diff --git a/python/ctsm/crop_calendars/import_ds.py b/python/ctsm/crop_calendars/import_ds.py index 71ce28bcce..9d06be99dc 100644 --- a/python/ctsm/crop_calendars/import_ds.py +++ b/python/ctsm/crop_calendars/import_ds.py @@ -16,10 +16,11 @@ from ctsm.crop_calendars.xr_flexsel import xr_flexsel -def compute_derived_vars(ds_in, var): +def compute_derived_vars(ds_in, var, logger=None): """ Compute derived variables """ + utils.log(logger, f"compute_derived_vars(): Getting {var}...") if ( var == "HYEARS" and "HDATES" in ds_in @@ -44,13 +45,14 @@ def compute_derived_vars(ds_in, var): return ds_in -def manual_mfdataset(filelist, my_vars, my_vegtypes, time_slice): +def manual_mfdataset(filelist, my_vars, my_vegtypes, time_slice, logger=None): """ Opening a list of files with Xarray's open_mfdataset requires dask. This function is a workaround for Python environments that don't have dask. """ ds_out = None for filename in filelist: + utils.log(logger, f"manual_mfdataset(): Opening ds_in: {ds_in}") ds_in = xr.open_dataset(filename) ds_in = mfdataset_preproc(ds_in, my_vars, my_vegtypes, time_slice) if ds_out is None: @@ -66,7 +68,7 @@ def manual_mfdataset(filelist, my_vars, my_vegtypes, time_slice): return ds_out -def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice): +def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, logger=None): """ Function to drop unwanted variables in preprocessing of open_mfdataset(). @@ -76,8 +78,11 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice): named like "patch". This can later be reversed, for compatibility with other code, using patch2pft(). """ + utils.log(logger, "mfdataset_preproc(): Start") + # Rename "pft" dimension and variables to "patch", if needed if "pft" in ds_in.dims: + utils.log(logger, 'mfdataset_preproc(): Rename "pft" dimension and variables to "patch"') pattern = re.compile("pft.*1d") matches = [x for x in list(ds_in.keys()) if pattern.search(x) is not None] pft2patch_dict = {"pft": "patch"} @@ -87,6 +92,8 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice): derived_vars = [] if vars_to_import is not None: + utils.log(logger, "mfdataset_preproc(): Getting vars to drop...") + # Split vars_to_import into variables that are vs. aren't already in ds derived_vars = [v for v in vars_to_import if v not in ds_in] present_vars = [v for v in vars_to_import if v in ds_in] @@ -123,10 +130,12 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice): vars_to_drop = list(np.setdiff1d(varlist, vars_to_import)) # Drop them + utils.log(logger, f"mfdataset_preproc(): Dropping variables: {vars_to_drop}") ds_in = ds_in.drop_vars(vars_to_drop) # Add vegetation type info if "patches1d_itype_veg" in list(ds_in): + utils.log(logger, f"mfdataset_preproc(): Adding vegetation type info") this_pftlist = utils.define_pftlist() utils.get_patch_ivts( ds_in, this_pftlist @@ -146,18 +155,25 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice): # Restrict to veg. types of interest, if any if vegtypes_to_import is not None: + utils.log(logger, f"mfdataset_preproc(): Restricting veg types to: {vegtypes_to_import}") ds_in = xr_flexsel(ds_in, vegtype=vegtypes_to_import) # Restrict to time slice, if any if time_slice: + utils.log(logger, f"mfdataset_preproc(): Restricting time slice to: {time_slice}") ds_in = utils.safer_timeslice(ds_in, time_slice) # Finish import + utils.log(logger, "mfdataset_preproc(): decode_cf()...") ds_in = xr.decode_cf(ds_in, decode_times=True) # Compute derived variables + if derived_vars: + utils.log(logger, "mfdataset_preproc(): decode_cf()...") for var in derived_vars: - ds_in = compute_derived_vars(ds_in, var) + ds_in = compute_derived_vars(ds_in, var, logger) + + utils.log(logger, "mfdataset_preproc(): End") return ds_in @@ -201,6 +217,7 @@ def import_ds( my_vars_missing_ok=None, rename_lsmlatlon=False, chunks=None, + logger=None, ): """ Import a dataset that can be spread over multiple files, only including specified variables @@ -209,6 +226,8 @@ def import_ds( - DOES actually read the dataset into memory, but only AFTER dropping unwanted variables and/or vegetation types. """ + utils.log(logger, "import_ds(): Start") + filelist, my_vars, my_vegtypes, my_vars_missing_ok = process_inputs( filelist, my_vars, my_vegtypes, my_vars_missing_ok ) @@ -221,10 +240,12 @@ def import_ds( if time_slice: new_filelist = [] for file in sorted(filelist): + utils.log(logger, f"import_ds(): Getting filetime from file: {file}") filetime = xr.open_dataset(file).time filetime_sel = utils.safer_timeslice(filetime, time_slice) include_this_file = filetime_sel.size if include_this_file: + utils.log(logger, f"import_ds(): Including filetime : {filetime_sel['time'].values}") new_filelist.append(file) # If you found some matching files, but then you find one that doesn't, stop going @@ -250,7 +271,7 @@ def import_ds( warnings.filterwarnings(action="ignore", category=DeprecationWarning) dask_unavailable = find_spec("dask") is None if dask_unavailable: - this_ds = manual_mfdataset(filelist, my_vars, my_vegtypes, time_slice) + this_ds = manual_mfdataset(filelist, my_vars, my_vegtypes, time_slice, logger=logger) else: this_ds = xr.open_mfdataset( sorted(filelist), @@ -263,8 +284,11 @@ def import_ds( chunks=chunks, ) elif isinstance(filelist, str): + utils.log(logger, f"import_ds(): Opening this_ds from filelist: {filelist}") this_ds = xr.open_dataset(filelist, chunks=chunks) + utils.log(logger, "import_ds(): Calling mfdataset_preproc()...") this_ds = mfdataset_preproc(this_ds, my_vars, my_vegtypes, time_slice) + utils.log(logger, "import_ds(): Calling compute()...") this_ds = this_ds.compute() # Warn and/or error about variables that couldn't be imported or derived @@ -289,4 +313,5 @@ def import_ds( if "lsmlon" in this_ds.dims: this_ds = this_ds.rename({"lsmlon": "lon"}) + utils.log(logger, "import_ds(): End") return this_ds From 8b5c90dd5d50f08c1dd35f1a90b0a140ba86df90 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 11 Aug 2025 15:13:22 -0600 Subject: [PATCH 154/196] cropcal_utils log(), error(): Prepend datetime string. --- python/ctsm/crop_calendars/cropcal_utils.py | 17 +++++++++++++---- .../crop_calendars/generate_gdds_functions.py | 3 +-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/python/ctsm/crop_calendars/cropcal_utils.py b/python/ctsm/crop_calendars/cropcal_utils.py index 9ddcfb0194..acaf74262d 100644 --- a/python/ctsm/crop_calendars/cropcal_utils.py +++ b/python/ctsm/crop_calendars/cropcal_utils.py @@ -2,29 +2,38 @@ utility functions copied from klindsay, https://github.com/klindsay28/CESM2_coup_carb_cycle_JAMES/blob/master/utils.py """ +from datetime import datetime import numpy as np import xarray as xr from ctsm.utils import is_instantaneous +def leading_datetime_string(): + """ + Return a datetime string like "YYYY-mm-dd HH:MM:SS " + """ + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " " + def log(logger_in, string): """ Simultaneously print INFO messages to console and to log file """ - print(string) + msg = leading_datetime_string() + string + print(msg) if logger_in: - logger_in.info(string) + logger_in.info(msg) def error(logger_in, string): """ Simultaneously print ERROR messages to console and to log file """ - print(string) + msg = leading_datetime_string() + string + print(msg) if logger_in: - logger_in.error(string) + logger_in.error(msg) raise RuntimeError(string) diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index 984059922f..11158eaa72 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -253,8 +253,7 @@ def import_and_process_1yr( Import one year of CLM output data for GDD generation """ save_figs = True - utils.log(logger, f"netCDF year {this_year}...") - utils.log(logger, dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + utils.log(logger, f"import_and_process_1yr(): netCDF year {this_year}...") # Without dask, this can take a LONG time at resolutions finer than 2-deg if importlib_util.find_spec("dask"): From 2132750855058f903b5e114b2374499c5507a014 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 11 Aug 2025 15:45:22 -0600 Subject: [PATCH 155/196] Reformat with black. --- python/ctsm/crop_calendars/cropcal_module.py | 1 + python/ctsm/crop_calendars/cropcal_utils.py | 2 ++ python/ctsm/crop_calendars/generate_gdds_functions.py | 4 +++- python/ctsm/crop_calendars/import_ds.py | 4 +++- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/python/ctsm/crop_calendars/cropcal_module.py b/python/ctsm/crop_calendars/cropcal_module.py index 57dc4600d8..a3dbda0cce 100644 --- a/python/ctsm/crop_calendars/cropcal_module.py +++ b/python/ctsm/crop_calendars/cropcal_module.py @@ -16,6 +16,7 @@ MISSING_RX_GDD_VAL = -1 + def check_and_trim_years(year_1, year_n, ds_in): """ After importing a file, restrict it to years of interest. diff --git a/python/ctsm/crop_calendars/cropcal_utils.py b/python/ctsm/crop_calendars/cropcal_utils.py index acaf74262d..1d3ff234f2 100644 --- a/python/ctsm/crop_calendars/cropcal_utils.py +++ b/python/ctsm/crop_calendars/cropcal_utils.py @@ -2,6 +2,7 @@ utility functions copied from klindsay, https://github.com/klindsay28/CESM2_coup_carb_cycle_JAMES/blob/master/utils.py """ + from datetime import datetime import numpy as np @@ -9,6 +10,7 @@ from ctsm.utils import is_instantaneous + def leading_datetime_string(): """ Return a datetime string like "YYYY-mm-dd HH:MM:SS " diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index 11158eaa72..dacaf59bc0 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -1100,7 +1100,9 @@ def make_figures( if land_use_file: year_1_lu = year_1 if first_land_use_year is None else first_land_use_year year_n_lu = year_n if last_land_use_year is None else last_land_use_year - lu_ds = cc.open_lu_ds(land_use_file, year_1_lu, year_n_lu, gdd_maps_ds, logger, ungrid=False) + lu_ds = cc.open_lu_ds( + land_use_file, year_1_lu, year_n_lu, gdd_maps_ds, logger, ungrid=False + ) lu_years_text = f" (masked by {year_1_lu}-{year_n_lu} area)" lu_years_file = f"_mask{year_1_lu}-{year_n_lu}" else: diff --git a/python/ctsm/crop_calendars/import_ds.py b/python/ctsm/crop_calendars/import_ds.py index 9d06be99dc..00b1bb7be6 100644 --- a/python/ctsm/crop_calendars/import_ds.py +++ b/python/ctsm/crop_calendars/import_ds.py @@ -245,7 +245,9 @@ def import_ds( filetime_sel = utils.safer_timeslice(filetime, time_slice) include_this_file = filetime_sel.size if include_this_file: - utils.log(logger, f"import_ds(): Including filetime : {filetime_sel['time'].values}") + utils.log( + logger, f"import_ds(): Including filetime : {filetime_sel['time'].values}" + ) new_filelist.append(file) # If you found some matching files, but then you find one that doesn't, stop going From 97e2bfe28caae79907d2689f360240676b307adc Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 11 Aug 2025 15:45:51 -0600 Subject: [PATCH 156/196] Add previous commit to .git-blame-ignore-revs. --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 5b1f2f8c3e..ae6a568078 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -73,3 +73,4 @@ cdf40d265cc82775607a1bf25f5f527bacc97405 7eb17f3ef0b9829fb55e0e3d7f02e157b0e41cfb 62d7711506a0fb9a3ad138ceceffbac1b79a6caa 49ad0f7ebe0b07459abc00a5c33c55a646f1e7e0 +ac03492012837799b7111607188acff9f739044a From 4113ece06b6077dd430ab79981c7eee3b1a31b95 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 11 Aug 2025 15:50:07 -0600 Subject: [PATCH 157/196] Refactoring to satisfy pylint. --- python/ctsm/crop_calendars/cropcal_module.py | 2 +- .../crop_calendars/generate_gdds_functions.py | 3 +-- python/ctsm/crop_calendars/import_ds.py | 24 ++++++++++++------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/python/ctsm/crop_calendars/cropcal_module.py b/python/ctsm/crop_calendars/cropcal_module.py index a3dbda0cce..07b56d0ed2 100644 --- a/python/ctsm/crop_calendars/cropcal_module.py +++ b/python/ctsm/crop_calendars/cropcal_module.py @@ -49,7 +49,7 @@ def check_and_trim_years(year_1, year_n, ds_in): return ds_in -def open_lu_ds(filename, year_1, year_n, existing_ds, logger, ungrid=True): +def open_lu_ds(filename, year_1, year_n, existing_ds, *, logger, ungrid=True): """ Open land-use dataset """ diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index dacaf59bc0..170b65502d 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -6,7 +6,6 @@ import warnings import os import glob -import datetime as dt from importlib import util as importlib_util import numpy as np import xarray as xr @@ -1101,7 +1100,7 @@ def make_figures( year_1_lu = year_1 if first_land_use_year is None else first_land_use_year year_n_lu = year_n if last_land_use_year is None else last_land_use_year lu_ds = cc.open_lu_ds( - land_use_file, year_1_lu, year_n_lu, gdd_maps_ds, logger, ungrid=False + land_use_file, year_1_lu, year_n_lu, gdd_maps_ds, logger=logger, ungrid=False ) lu_years_text = f" (masked by {year_1_lu}-{year_n_lu} area)" lu_years_file = f"_mask{year_1_lu}-{year_n_lu}" diff --git a/python/ctsm/crop_calendars/import_ds.py b/python/ctsm/crop_calendars/import_ds.py index 00b1bb7be6..f844c0a3c3 100644 --- a/python/ctsm/crop_calendars/import_ds.py +++ b/python/ctsm/crop_calendars/import_ds.py @@ -82,13 +82,7 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log # Rename "pft" dimension and variables to "patch", if needed if "pft" in ds_in.dims: - utils.log(logger, 'mfdataset_preproc(): Rename "pft" dimension and variables to "patch"') - pattern = re.compile("pft.*1d") - matches = [x for x in list(ds_in.keys()) if pattern.search(x) is not None] - pft2patch_dict = {"pft": "patch"} - for match in matches: - pft2patch_dict[match] = match.replace("pft", "patch").replace("patchs", "patches") - ds_in = ds_in.rename(pft2patch_dict) + ds_in = rename_pft_to_patch(ds_in, logger) derived_vars = [] if vars_to_import is not None: @@ -135,7 +129,7 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log # Add vegetation type info if "patches1d_itype_veg" in list(ds_in): - utils.log(logger, f"mfdataset_preproc(): Adding vegetation type info") + utils.log(logger, "mfdataset_preproc(): Adding vegetation type info") this_pftlist = utils.define_pftlist() utils.get_patch_ivts( ds_in, this_pftlist @@ -178,6 +172,20 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log return ds_in +def rename_pft_to_patch(ds_in, logger): + """ + Rename "pft" dimension and variables to "patch", if needed + """ + utils.log(logger, 'mfdataset_preproc(): Rename "pft" dimension and variables to "patch"') + pattern = re.compile("pft.*1d") + matches = [x for x in list(ds_in.keys()) if pattern.search(x) is not None] + pft2patch_dict = {"pft": "patch"} + for match in matches: + pft2patch_dict[match] = match.replace("pft", "patch").replace("patchs", "patches") + ds_in = ds_in.rename(pft2patch_dict) + return ds_in + + def process_inputs(filelist, my_vars, my_vegtypes, my_vars_missing_ok): """ Process inputs to import_ds() From ebef8c7a629bfc35e9adbd90253f4cd8f3450cb5 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 19 Aug 2025 13:50:56 -0600 Subject: [PATCH 158/196] Move 3 functions from cropcal_utils. * leading_datetime_string() to utils.py; renamed to datetime_string(). * log() and error() to ctsm_logging.py --- python/ctsm/crop_calendars/cropcal_module.py | 3 +- python/ctsm/crop_calendars/cropcal_utils.py | 30 ---- python/ctsm/crop_calendars/generate_gdds.py | 28 ++-- .../crop_calendars/generate_gdds_functions.py | 135 +++++++++--------- python/ctsm/crop_calendars/import_ds.py | 39 ++--- python/ctsm/ctsm_logging.py | 27 ++++ python/ctsm/utils.py | 9 +- 7 files changed, 139 insertions(+), 132 deletions(-) diff --git a/python/ctsm/crop_calendars/cropcal_module.py b/python/ctsm/crop_calendars/cropcal_module.py index 07b56d0ed2..d6fe3f6fc6 100644 --- a/python/ctsm/crop_calendars/cropcal_module.py +++ b/python/ctsm/crop_calendars/cropcal_module.py @@ -13,6 +13,7 @@ from ctsm.crop_calendars.cropcal_constants import DEFAULT_GDD_MIN from ctsm.crop_calendars.import_ds import import_ds from ctsm.utils import is_instantaneous +from ctsm.ctsm_logging import log MISSING_RX_GDD_VAL = -1 @@ -54,7 +55,7 @@ def open_lu_ds(filename, year_1, year_n, existing_ds, *, logger, ungrid=True): Open land-use dataset """ # Open and trim to years of interest - utils.log(logger, f"open_lu_ds(): Opening this_ds_gridded: {filename}") + log(logger, f"open_lu_ds(): Opening this_ds_gridded: {filename}") this_ds_gridded = xr.open_dataset(filename).sel(time=slice(year_1, year_n)) # Assign actual lon/lat coordinates diff --git a/python/ctsm/crop_calendars/cropcal_utils.py b/python/ctsm/crop_calendars/cropcal_utils.py index 1d3ff234f2..c7e8b6ac52 100644 --- a/python/ctsm/crop_calendars/cropcal_utils.py +++ b/python/ctsm/crop_calendars/cropcal_utils.py @@ -3,42 +3,12 @@ copied from klindsay, https://github.com/klindsay28/CESM2_coup_carb_cycle_JAMES/blob/master/utils.py """ -from datetime import datetime - import numpy as np import xarray as xr from ctsm.utils import is_instantaneous -def leading_datetime_string(): - """ - Return a datetime string like "YYYY-mm-dd HH:MM:SS " - """ - return datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " " - - -def log(logger_in, string): - """ - Simultaneously print INFO messages to console and to log file - """ - msg = leading_datetime_string() + string - print(msg) - if logger_in: - logger_in.info(msg) - - -def error(logger_in, string): - """ - Simultaneously print ERROR messages to console and to log file - """ - msg = leading_datetime_string() + string - print(msg) - if logger_in: - logger_in.error(msg) - raise RuntimeError(string) - - def define_pftlist(): """ Return list of PFTs used in CLM diff --git a/python/ctsm/crop_calendars/generate_gdds.py b/python/ctsm/crop_calendars/generate_gdds.py index bc0063fabf..233fbc0866 100644 --- a/python/ctsm/crop_calendars/generate_gdds.py +++ b/python/ctsm/crop_calendars/generate_gdds.py @@ -18,9 +18,9 @@ os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir, "python" ) sys.path.insert(1, _CTSM_PYTHON) +from ctsm.ctsm_logging import log, error # pylint: disable=wrong-import-position import ctsm.crop_calendars.cropcal_module as cc # pylint: disable=wrong-import-position import ctsm.crop_calendars.generate_gdds_functions as gddfn # pylint: disable=wrong-import-position -import ctsm.crop_calendars.cropcal_utils as utils # pylint: disable=wrong-import-position # Functions here were written with too many positional arguments. At some point that should be # fixed. For now, we'll just disable the warning. @@ -86,11 +86,11 @@ def main( raise RuntimeError( "only_make_figs True but not all plotting modules are available" ) from exc - utils.log(logger, "Not all plotting modules are available; disabling save_figs") + log(logger, "Not all plotting modules are available; disabling save_figs") save_figs = False # Print some info - utils.log(logger, f"Saving to {output_dir}") + log(logger, f"Saving to {output_dir}") # Parse list of crops to skip if "," in skip_crops: @@ -108,7 +108,7 @@ def main( yr_1_import_str = f"{first_season+1}-01-01" yr_n_import_str = f"{last_season+2}-01-01" - utils.log( + log( logger, f"Importing netCDF time steps {yr_1_import_str} through {yr_n_import_str} " + "(years are +1 because of CTSM output naming)", @@ -192,7 +192,7 @@ def main( h1_instantaneous, ) - utils.log(logger, f" Saving pickle file ({pickle_file})...") + log(logger, f" Saving pickle file ({pickle_file})...") with open(pickle_file, "wb") as file: pickle.dump( [ @@ -220,10 +220,10 @@ def main( [i for i, c in enumerate(gddaccum_yp_list) if not isinstance(c, type(None))] ] - utils.log(logger, "Done") + log(logger, "Done") if not h2_ds: - utils.log(logger, f"generate_gdds main(): Opening h2_ds: {h2_ds_file}") + log(logger, f"generate_gdds main(): Opening h2_ds: {h2_ds_file}") h2_ds = xr.open_dataset(h2_ds_file) ###################################################### @@ -238,7 +238,7 @@ def main( "s", sdates_rx, incl_patches1d_itype_veg, mxsowings, logger ) - utils.log(logger, "Getting and gridding mean GDDs...") + log(logger, "Getting and gridding mean GDDs...") gdd_maps_ds = gddfn.yp_list_to_ds( gddaccum_yp_list, h2_ds, incl_vegtypes_str, sdates_rx, longname_prefix, logger ) @@ -249,10 +249,10 @@ def main( # Fill NAs with dummy values dummy_fill = -1 gdd_maps_ds = gdd_maps_ds.fillna(dummy_fill) - utils.log(logger, "Done getting and gridding means.") + log(logger, "Done getting and gridding means.") # Add dummy variables for crops not actually simulated - utils.log(logger, "Adding dummy variables...") + log(logger, "Adding dummy variables...") # Unnecessary? template_ds = xr.open_dataset(sdates_file, decode_times=True) all_vars = [v.replace("sdate", "gdd") for v in template_ds if "sdate" in v] @@ -280,7 +280,7 @@ def make_dummy(this_crop_gridded, addend): for var_index, this_var in enumerate(dummy_vars): if this_var in gdd_maps_ds: - utils.error( + error( logger, f"{this_var} is already in gdd_maps_ds. Why overwrite it with dummy?" ) dummy_gridded.name = this_var @@ -296,14 +296,14 @@ def add_lonlat_attrs(this_ds): gdd_maps_ds = add_lonlat_attrs(gdd_maps_ds) gddharv_maps_ds = add_lonlat_attrs(gddharv_maps_ds) - utils.log(logger, "Done.") + log(logger, "Done.") ###################### ### Save to netCDF ### ###################### if not only_make_figs: - utils.log(logger, "Saving...") + log(logger, "Saving...") # Get output file path datestr = dt.datetime.now().strftime("%Y%m%d_%H%M%S") @@ -338,7 +338,7 @@ def save_gdds(sdates_file, hdates_file, outfile, gdd_maps_ds, sdates_rx): save_gdds(sdates_file, hdates_file, outfile, gdd_maps_ds, sdates_rx) - utils.log(logger, "Done saving.") + log(logger, "Done saving.") ######################################## ### Save things needed for mapmaking ### diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index 170b65502d..c27ebdfb57 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -11,6 +11,7 @@ import xarray as xr from ctsm.utils import is_instantaneous +from ctsm.ctsm_logging import log, error import ctsm.crop_calendars.cropcal_utils as utils import ctsm.crop_calendars.cropcal_module as cc from ctsm.crop_calendars.xr_flexsel import xr_flexsel @@ -57,7 +58,7 @@ def check_sdates(dates_ds, sdates_rx, outdir_figs, logger, verbose=False): """ Checking that input and output sdates match """ - utils.log(logger, " Checking that input and output sdates match...") + log(logger, " Checking that input and output sdates match...") sdates_grid = grid_one_variable(dates_ds, "SDATES") @@ -71,12 +72,12 @@ def check_sdates(dates_ds, sdates_rx, outdir_figs, logger, verbose=False): this_var = f"gs1_{vegtype_int}" if this_var not in sdates_rx: vegtypes_skipped = vegtypes_skipped + [vegtype_str] - # utils.log(logger, f" {vt_str} ({vt}) SKIPPED...") + # log(logger, f" {vt_str} ({vt}) SKIPPED...") continue vegtypes_included = vegtypes_included + [vegtype_str] any_found = True if verbose: - utils.log(logger, f" {vegtype_str} ({vegtype_int})...") + log(logger, f" {vegtype_str} ({vegtype_int})...") in_map = sdates_rx[this_var].squeeze(drop=True) # Output @@ -86,23 +87,23 @@ def check_sdates(dates_ds, sdates_rx, outdir_figs, logger, verbose=False): diff_map = out_map - in_map diff_map_notnan = diff_map.values[np.invert(np.isnan(diff_map.values))] if np.any(diff_map_notnan): - utils.log(logger, f"Difference(s) found in {vegtype_str}") + log(logger, f"Difference(s) found in {vegtype_str}") here = np.where(diff_map_notnan) - utils.log(logger, "in:") + log(logger, "in:") in_map_notnan = in_map.values[np.invert(np.isnan(diff_map.values))] - utils.log(logger, in_map_notnan[here][0:4]) + log(logger, in_map_notnan[here][0:4]) out_map_notnan = out_map.values[np.invert(np.isnan(diff_map.values))] - utils.log(logger, "out:") - utils.log(logger, out_map_notnan[here][0:4]) - utils.log(logger, "diff:") - utils.log(logger, diff_map_notnan[here][0:4]) + log(logger, "out:") + log(logger, out_map_notnan[here][0:4]) + log(logger, "diff:") + log(logger, diff_map_notnan[here][0:4]) first_diff = all_ok all_ok = False if CAN_PLOT: sdate_diffs_dir = os.path.join(outdir_figs, "sdate_diffs") if first_diff: - utils.log(logger, f"Saving sdate difference figures to {sdate_diffs_dir}") + log(logger, f"Saving sdate difference figures to {sdate_diffs_dir}") if not os.path.exists(sdate_diffs_dir): os.makedirs(sdate_diffs_dir) in_map.where(~np.isnan(out_map)).plot() @@ -119,24 +120,24 @@ def check_sdates(dates_ds, sdates_rx, outdir_figs, logger, verbose=False): plt.close() if not any_found: - utils.error(logger, "No matching variables found in sdates_rx!") + error(logger, "No matching variables found in sdates_rx!") # Sanity checks for included vegetation types vegtypes_skipped = np.unique([x.replace("irrigated_", "") for x in vegtypes_skipped]) vegtypes_skipped_weird = [x for x in vegtypes_skipped if x in vegtypes_included] if np.array_equal(vegtypes_included, [x.replace("irrigated_", "") for x in vegtypes_included]): - utils.log(logger, "\nWARNING: No irrigated crops included!!!\n") + log(logger, "\nWARNING: No irrigated crops included!!!\n") elif vegtypes_skipped_weird: - utils.log( + log( logger, "\nWarning: Some crop types had output rainfed patches but no irrigated patches: " + f"{vegtypes_skipped_weird}", ) if all_ok: - utils.log(logger, " ✅ Input and output sdates match!") + log(logger, " ✅ Input and output sdates match!") else: - utils.error(logger, " ❌ Input and output sdates differ.") + error(logger, " ❌ Input and output sdates differ.") def import_rx_dates(s_or_h, date_infile, incl_patches1d_itype_veg, mxsowings, logger): @@ -146,7 +147,7 @@ def import_rx_dates(s_or_h, date_infile, incl_patches1d_itype_veg, mxsowings, lo if isinstance(date_infile, xr.Dataset): return date_infile if not isinstance(date_infile, str): - utils.error( + error( logger, f"Importing {s_or_h}dates_rx: Expected date_infile to be str or DataArray," + f"not {type(date_infile)}", @@ -202,7 +203,7 @@ def yp_list_to_ds(yp_list, daily_ds, incl_vegtypes_str, dates_rx, longname_prefi if isinstance(data, type(None)): continue this_crop_str = incl_vegtypes_str[this_crop_int] - utils.log(logger, f" {this_crop_str}...") + log(logger, f" {this_crop_str}...") new_var = f"gdd1_{utils.ivt_str2int(this_crop_str)}" this_ds = daily_ds.isel( patch=np.where(daily_ds.patches1d_itype_veg_str.values == this_crop_str)[0] @@ -252,7 +253,7 @@ def import_and_process_1yr( Import one year of CLM output data for GDD generation """ save_figs = True - utils.log(logger, f"import_and_process_1yr(): netCDF year {this_year}...") + log(logger, f"import_and_process_1yr(): netCDF year {this_year}...") # Without dask, this can take a LONG time at resolutions finer than 2-deg if importlib_util.find_spec("dask"): @@ -267,7 +268,7 @@ def import_and_process_1yr( h1_pattern = os.path.join(indir, "*h1i.*.nc.base") h1_filelist = glob.glob(h1_pattern) if not h1_filelist: - utils.error(logger, "No files found matching pattern '*h1i.*.nc(.base)'") + error(logger, "No files found matching pattern '*h1i.*.nc(.base)'") # Get list of crops to include if skip_crops is not None: @@ -297,7 +298,7 @@ def import_and_process_1yr( if dates_ds.dims["time"] > 1: if dates_ds.dims["time"] == 365: if not incorrectly_daily: - utils.log( + log( logger, " ℹ️ You saved SDATES and HDATES daily, but you only needed annual. Fixing.", ) @@ -316,9 +317,9 @@ def import_and_process_1yr( ) n_unmatched_nans = np.sum(sdates_all_nan != hdates_all_nan) if n_unmatched_nans > 0: - utils.error(logger, "Output SDATE and HDATE NaN masks do not match.") + error(logger, "Output SDATE and HDATE NaN masks do not match.") if np.sum(~np.isnan(dates_ds.SDATES.values)) == 0: - utils.error(logger, "All SDATES are NaN!") + error(logger, "All SDATES are NaN!") # Just work with non-NaN patches for now skip_patches_for_isel_nan = np.where(sdates_all_nan)[0] @@ -327,7 +328,7 @@ def import_and_process_1yr( skip_patches_for_isel_nan_last_year, skip_patches_for_isel_nan ) if different_nan_mask: - utils.log(logger, " Different NaN mask than last year") + log(logger, " Different NaN mask than last year") incl_thisyr_but_nan_lastyr = [ dates_ds.patch.values[p] for p in incl_patches_for_isel_nan @@ -337,7 +338,7 @@ def import_and_process_1yr( incl_thisyr_but_nan_lastyr = [] skipping_patches_for_isel_nan = len(skip_patches_for_isel_nan) > 0 if skipping_patches_for_isel_nan: - utils.log( + log( logger, f" Ignoring {len(skip_patches_for_isel_nan)} patches with all-NaN sowing and " + "harvest dates.", @@ -356,14 +357,14 @@ def import_and_process_1yr( if isinstance(incl_vegtypes_str, np.ndarray): incl_vegtypes_str = list(incl_vegtypes_str) if incl_vegtypes_str != list(dates_incl_ds.vegtype_str.values): - utils.error( + error( logger, f"Included veg types differ. Previously {incl_vegtypes_str}, " + f"now {dates_incl_ds.vegtype_str.values}", ) if np.sum(~np.isnan(dates_incl_ds.SDATES.values)) == 0: - utils.error(logger, "All SDATES are NaN after ignoring those patches!") + error(logger, "All SDATES are NaN after ignoring those patches!") # Some patches can have -1 sowing date?? Hopefully just an artifact of me incorrectly saving # SDATES/HDATES daily. @@ -376,14 +377,14 @@ def import_and_process_1yr( dates_incl_ds.HDATES.isel(mxharvests=0, patch=skip_patches_for_isel_sdatelt1).values ) if incorrectly_daily and list(unique_hdates) == [364]: - utils.log( + log( logger, f" ❗ {len(skip_patches_for_isel_sdatelt1)} patches have SDATE < 1, but this" + "might have just been because of incorrectly daily outputs. Setting them to 365.", ) new_sdates_ar = dates_incl_ds.SDATES.values if mxsowings_dim != 0: - utils.error(logger, "Code this up") + error(logger, "Code this up") new_sdates_ar[0, skip_patches_for_isel_sdatelt1] = 365 dates_incl_ds["SDATES"] = xr.DataArray( data=new_sdates_ar, @@ -391,7 +392,7 @@ def import_and_process_1yr( attrs=dates_incl_ds["SDATES"].attrs, ) else: - utils.error( + error( logger, f"{len(skip_patches_for_isel_sdatelt1)} patches have SDATE < 1. " + f"Unique affected hdates: {unique_hdates}", @@ -414,10 +415,10 @@ def import_and_process_1yr( if np.any(hdates_thisyr_where_nan_lastyr < 1): new_hdates = dates_incl_ds.HDATES.values if mxharvests_dim != 0: - utils.error(logger, "Code this up") + error(logger, "Code this up") patch_list = list(hdates_thisyr.patch.values) here = [patch_list.index(x) for x in incl_thisyr_but_nan_lastyr] - utils.log( + log( logger, f" ❗ {len(here)} patches have harvest date -1 because they weren't active last" + "year (and were either never active or were harvested when last active). " @@ -442,7 +443,7 @@ def import_and_process_1yr( dates_incl_ds.SDATES.isel(patch=skip_patches_for_isel_hdatelt1).values ) if incorrectly_daily and list(unique_sdates) == [1]: - utils.log( + log( logger, f" ❗ {len(skip_patches_for_isel_hdatelt1)} patches have HDATE < 1??? Seems like " + "this might have just been because of incorrectly daily outputs; setting them to " @@ -450,7 +451,7 @@ def import_and_process_1yr( ) new_hdates_ar = dates_incl_ds.HDATES.values if mxharvests_dim != 0: - utils.error(logger, "Code this up") + error(logger, "Code this up") new_hdates_ar[0, skip_patches_for_isel_hdatelt1] = 365 dates_incl_ds["HDATES"] = xr.DataArray( data=new_hdates_ar, @@ -458,7 +459,7 @@ def import_and_process_1yr( attrs=dates_incl_ds["HDATES"].attrs, ) else: - utils.error( + error( logger, f"{len(skip_patches_for_isel_hdatelt1)} patches have HDATE < 1. Possible causes:\n" + "* Not using constant crop areas (e.g., flanduse_timeseries from " @@ -474,7 +475,7 @@ def import_and_process_1yr( >= 1 ) if n_extra_harv > 0: - utils.error(logger, f"{n_extra_harv} patches have >1 harvest.") + error(logger, f"{n_extra_harv} patches have >1 harvest.") # Make sure harvest happened the day before sowing sdates_clm = dates_incl_ds.SDATES.values.squeeze() @@ -482,7 +483,7 @@ def import_and_process_1yr( diffdates_clm = sdates_clm - hdates_clm diffdates_clm[(sdates_clm == 1) & (hdates_clm == 365)] = 1 if list(np.unique(diffdates_clm)) != [1]: - utils.error(logger, f"Not all sdates-hdates are 1: {np.unique(diffdates_clm)}") + error(logger, f"Not all sdates-hdates are 1: {np.unique(diffdates_clm)}") # Import expected sowing dates. This will also be used as our template output file. imported_sdates = isinstance(sdates_rx, str) @@ -545,7 +546,7 @@ def import_and_process_1yr( else: hdates_rx = hdates_rx_orig - utils.log(logger, " Importing accumulated GDDs...") + log(logger, " Importing accumulated GDDs...") clm_gdd_var = "GDDACCUM" my_vars = [clm_gdd_var, "GDDHARV"] patterns = [f"*h2i.{this_year-1}-01*.nc", f"*h2i.{this_year-1}-01*.nc.base"] @@ -555,7 +556,7 @@ def import_and_process_1yr( if h2_files: break if not h2_files: - utils.error(logger, f"No files found matching patterns: {patterns}") + error(logger, f"No files found matching patterns: {patterns}") h2_ds = import_ds( h2_files, my_vars=my_vars, @@ -567,13 +568,13 @@ def import_and_process_1yr( # Restrict to patches we're including if skipping_patches_for_isel_nan: if not np.array_equal(dates_ds.patch.values, h2_ds.patch.values): - utils.error(logger, "dates_ds and h2_ds don't have the same patch list!") + error(logger, "dates_ds and h2_ds don't have the same patch list!") h2_incl_ds = h2_ds.isel(patch=incl_patches_for_isel_nan) else: h2_incl_ds = h2_ds if not np.any(h2_incl_ds[clm_gdd_var].values != 0): - utils.error(logger, f"All {clm_gdd_var} values are zero!") + error(logger, f"All {clm_gdd_var} values are zero!") # Get standard datetime axis for outputs n_years = year_n - year_1 + 1 @@ -587,7 +588,7 @@ def import_and_process_1yr( incl_vegtype_indices = [] for var, vegtype_str in enumerate(incl_vegtypes_str): if vegtype_str in skip_crops: - utils.log(logger, f" import_and_process_1yr(): SKIPPING {vegtype_str}") + log(logger, f" import_and_process_1yr(): SKIPPING {vegtype_str}") continue vegtype_int = utils.vegtype_str2int(vegtype_str)[0] @@ -602,7 +603,7 @@ def import_and_process_1yr( check_gddharv = True if not this_crop_gddaccum_da.size: continue - utils.log(logger, f" import_and_process_1yr(): {vegtype_str}...") + log(logger, f" import_and_process_1yr(): {vegtype_str}...") incl_vegtype_indices = incl_vegtype_indices + [var] # Get prescribed harvest dates for these patches @@ -635,7 +636,7 @@ def import_and_process_1yr( if not np.all( this_crop_gddaccum_da.patch.values[:-1] <= this_crop_gddaccum_da.patch.values[1:] ): - utils.error(logger, "This code depends on DataArray patch list being sorted.") + error(logger, "This code depends on DataArray patch list being sorted.") sortorder = np.argsort(patches) i_patches = list(np.array(i_patches)[np.array(sortorder)]) i_times = list(np.array(i_times)[np.array(sortorder)]) @@ -645,17 +646,17 @@ def import_and_process_1yr( if save_figs: gddharv_atharv_p = this_crop_gddharv_da.values[(i_times, i_patches)] if np.any(np.isnan(gddaccum_atharv_p)): - utils.log( + log( logger, f" ❗ {np.sum(np.isnan(gddaccum_atharv_p))}/{len(gddaccum_atharv_p)} " + "NaN after extracting GDDs accumulated at harvest", ) if save_figs and gddharv_atharv_p is not None and np.any(np.isnan(gddharv_atharv_p)): if np.all(np.isnan(gddharv_atharv_p)): - utils.log(logger, " ❗ All GDDHARV are NaN; should only affect figure") + log(logger, " ❗ All GDDHARV are NaN; should only affect figure") check_gddharv = False else: - utils.log( + log( logger, f" ❗ {np.sum(np.isnan(gddharv_atharv_p))}/{len(gddharv_atharv_p)} " + "NaN after extracting GDDHARV", @@ -682,14 +683,14 @@ def import_and_process_1yr( ] if not np.array_equal(last_year_active_patch_indices, this_year_active_patch_indices): if incorrectly_daily: - utils.log( + log( logger, " ❗ This year's active patch indices differ from last year's. " + "Allowing because this might just be an artifact of incorrectly daily " + "outputs, BUT RESULTS MUST NOT BE TRUSTED.", ) else: - utils.error(logger, "This year's active patch indices differ from last year's.") + error(logger, "This year's active patch indices differ from last year's.") # Make sure we're not about to overwrite any existing values. if np.any( ~np.isnan( @@ -697,28 +698,28 @@ def import_and_process_1yr( ) ): if incorrectly_daily: - utils.log( + log( logger, " ❗ Unexpected non-NaN for last season's GDD accumulation. " + "Allowing because this might just be an artifact of incorrectly daily " + "outputs, BUT RESULTS MUST NOT BE TRUSTED.", ) else: - utils.error(logger, "Unexpected non-NaN for last season's GDD accumulation") + error(logger, "Unexpected non-NaN for last season's GDD accumulation") if save_figs and np.any( ~np.isnan( gddharv_yp_list[var][year_index - 1, active_this_year_where_gs_lastyr_indices] ) ): if incorrectly_daily: - utils.log( + log( logger, " ❗ Unexpected non-NaN for last season's GDDHARV. Allowing " + "because this might just be an artifact of incorrectly daily outputs, " + "BUT RESULTS MUST NOT BE TRUSTED.", ) else: - utils.error(logger, "Unexpected non-NaN for last season's GDDHARV") + error(logger, "Unexpected non-NaN for last season's GDDHARV") # Fill. gddaccum_yp_list[var][year_index - 1, active_this_year_where_gs_lastyr_indices] = ( gddaccum_atharv_p[where_gs_lastyr] @@ -734,14 +735,14 @@ def import_and_process_1yr( ) ): if incorrectly_daily: - utils.log( + log( logger, " ❗ Unexpected NaN for last season's GDD accumulation. Allowing " + "because this might just be an artifact of incorrectly daily outputs, " + "BUT RESULTS MUST NOT BE TRUSTED.", ) else: - utils.error(logger, "Unexpected NaN for last season's GDD accumulation.") + error(logger, "Unexpected NaN for last season's GDD accumulation.") if ( save_figs and check_gddharv @@ -754,14 +755,14 @@ def import_and_process_1yr( ) ): if incorrectly_daily: - utils.log( + log( logger, " ❗ Unexpected NaN for last season's GDDHARV. Allowing because " + "this might just be an artifact of incorrectly daily outputs, BUT " + "RESULTS MUST NOT BE TRUSTED.", ) else: - utils.error(logger, "Unexpected NaN for last season's GDDHARV.") + error(logger, "Unexpected NaN for last season's GDDHARV.") gddaccum_yp_list[var][year_index, this_year_active_patch_indices] = tmp_gddaccum if save_figs: gddharv_yp_list[var][year_index, this_year_active_patch_indices] = tmp_gddharv @@ -777,14 +778,14 @@ def import_and_process_1yr( nanmask_output_gdds_lastyr = np.isnan(gddaccum_yp_list[var][year_index - 1, :]) if not np.array_equal(nanmask_output_gdds_lastyr, nanmask_output_sdates): if incorrectly_daily: - utils.log( + log( logger, " ❗ NaN masks differ between this year's sdates and 'filled-out' " + "GDDs from last year. Allowing because this might just be an artifact of " + "incorrectly daily outputs, BUT RESULTS MUST NOT BE TRUSTED.", ) else: - utils.error( + error( logger, "NaN masks differ between this year's sdates and 'filled-out' GDDs from " + "last year", @@ -794,7 +795,7 @@ def import_and_process_1yr( skip_patches_for_isel_nan_last_year = skip_patches_for_isel_nan # Could save space by only saving variables needed for gridding - utils.log(logger, " Saving h2_ds...") + log(logger, " Saving h2_ds...") h2_ds.to_netcdf(h2_ds_file) return ( @@ -1064,7 +1065,7 @@ def make_figures( """ if not gdd_maps_ds: if not this_dir: - utils.error( + error( logger, "If not providing gdd_maps_ds, you must provide thisDir (location of " + "gdd_maps.nc)", @@ -1072,7 +1073,7 @@ def make_figures( gdd_maps_ds = xr.open_dataset(this_dir + "gdd_maps.nc") if not gddharv_maps_ds: if not this_dir: - utils.error( + error( logger, "If not providing gddharv_maps_ds, you must provide thisDir (location of " + "gddharv_maps.nc)", @@ -1137,7 +1138,7 @@ def make_figures( # Maps nplot_y = 3 nplot_x = 1 - utils.log(logger, "Making before/after maps...") + log(logger, "Making before/after maps...") vegtype_list = incl_vegtypes_str if land_use_file: vegtype_list += ["Corn", "Cotton", "Rice", "Soybean", "Sugarcane", "Wheat"] @@ -1198,7 +1199,7 @@ def make_figures( spec = fig.add_gridspec(nrows=3, ncols=2, width_ratios=[0.5, 0.5], wspace=0.2) this_axis = fig.add_subplot(spec[0, 0], projection=ccrs.PlateCarree()) else: - utils.error(logger, f"layout {layout} not recognized") + error(logger, f"layout {layout} not recognized") gddharv_all_nan = np.all(np.isnan(gddharv_map_yx.values)) if gddharv_all_nan: @@ -1223,7 +1224,7 @@ def make_figures( elif layout in ["2x2", "3x2"]: this_axis = fig.add_subplot(spec[1, 0], projection=ccrs.PlateCarree()) else: - utils.error(logger, f"layout {layout} not recognized") + error(logger, f"layout {layout} not recognized") this_min = int(np.round(np.nanmin(gdd_map_yx))) this_max = int(np.round(np.nanmax(gdd_map_yx))) this_title = f"{run2_name} (range {this_min}–{this_max})" @@ -1308,7 +1309,7 @@ def make_figures( elif layout in ["2x2", "3x2"]: this_axis = fig.add_subplot(spec[:, 1]) else: - utils.error(logger, f"layout {layout} not recognized") + error(logger, f"layout {layout} not recognized") # Shift bottom of plot up to make room for legend ax_pos = this_axis.get_position() @@ -1361,4 +1362,4 @@ def make_figures( plt.savefig(outfile, dpi=300, transparent=False, facecolor="white", bbox_inches="tight") plt.close() - utils.log(logger, "Done.") + log(logger, "Done.") diff --git a/python/ctsm/crop_calendars/import_ds.py b/python/ctsm/crop_calendars/import_ds.py index f844c0a3c3..ccf712fe4b 100644 --- a/python/ctsm/crop_calendars/import_ds.py +++ b/python/ctsm/crop_calendars/import_ds.py @@ -12,6 +12,7 @@ import numpy as np import xarray as xr from ctsm.utils import is_instantaneous +from ctsm.ctsm_logging import log import ctsm.crop_calendars.cropcal_utils as utils from ctsm.crop_calendars.xr_flexsel import xr_flexsel @@ -20,7 +21,7 @@ def compute_derived_vars(ds_in, var, logger=None): """ Compute derived variables """ - utils.log(logger, f"compute_derived_vars(): Getting {var}...") + log(logger, f"compute_derived_vars(): Getting {var}...") if ( var == "HYEARS" and "HDATES" in ds_in @@ -52,7 +53,7 @@ def manual_mfdataset(filelist, my_vars, my_vegtypes, time_slice, logger=None): """ ds_out = None for filename in filelist: - utils.log(logger, f"manual_mfdataset(): Opening ds_in: {ds_in}") + log(logger, f"manual_mfdataset(): Opening ds_in: {ds_in}") ds_in = xr.open_dataset(filename) ds_in = mfdataset_preproc(ds_in, my_vars, my_vegtypes, time_slice) if ds_out is None: @@ -78,7 +79,7 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log named like "patch". This can later be reversed, for compatibility with other code, using patch2pft(). """ - utils.log(logger, "mfdataset_preproc(): Start") + log(logger, "mfdataset_preproc(): Start") # Rename "pft" dimension and variables to "patch", if needed if "pft" in ds_in.dims: @@ -86,7 +87,7 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log derived_vars = [] if vars_to_import is not None: - utils.log(logger, "mfdataset_preproc(): Getting vars to drop...") + log(logger, "mfdataset_preproc(): Getting vars to drop...") # Split vars_to_import into variables that are vs. aren't already in ds derived_vars = [v for v in vars_to_import if v not in ds_in] @@ -124,12 +125,12 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log vars_to_drop = list(np.setdiff1d(varlist, vars_to_import)) # Drop them - utils.log(logger, f"mfdataset_preproc(): Dropping variables: {vars_to_drop}") + log(logger, f"mfdataset_preproc(): Dropping variables: {vars_to_drop}") ds_in = ds_in.drop_vars(vars_to_drop) # Add vegetation type info if "patches1d_itype_veg" in list(ds_in): - utils.log(logger, "mfdataset_preproc(): Adding vegetation type info") + log(logger, "mfdataset_preproc(): Adding vegetation type info") this_pftlist = utils.define_pftlist() utils.get_patch_ivts( ds_in, this_pftlist @@ -149,25 +150,25 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log # Restrict to veg. types of interest, if any if vegtypes_to_import is not None: - utils.log(logger, f"mfdataset_preproc(): Restricting veg types to: {vegtypes_to_import}") + log(logger, f"mfdataset_preproc(): Restricting veg types to: {vegtypes_to_import}") ds_in = xr_flexsel(ds_in, vegtype=vegtypes_to_import) # Restrict to time slice, if any if time_slice: - utils.log(logger, f"mfdataset_preproc(): Restricting time slice to: {time_slice}") + log(logger, f"mfdataset_preproc(): Restricting time slice to: {time_slice}") ds_in = utils.safer_timeslice(ds_in, time_slice) # Finish import - utils.log(logger, "mfdataset_preproc(): decode_cf()...") + log(logger, "mfdataset_preproc(): decode_cf()...") ds_in = xr.decode_cf(ds_in, decode_times=True) # Compute derived variables if derived_vars: - utils.log(logger, "mfdataset_preproc(): decode_cf()...") + log(logger, "mfdataset_preproc(): decode_cf()...") for var in derived_vars: ds_in = compute_derived_vars(ds_in, var, logger) - utils.log(logger, "mfdataset_preproc(): End") + log(logger, "mfdataset_preproc(): End") return ds_in @@ -176,7 +177,7 @@ def rename_pft_to_patch(ds_in, logger): """ Rename "pft" dimension and variables to "patch", if needed """ - utils.log(logger, 'mfdataset_preproc(): Rename "pft" dimension and variables to "patch"') + log(logger, 'mfdataset_preproc(): Rename "pft" dimension and variables to "patch"') pattern = re.compile("pft.*1d") matches = [x for x in list(ds_in.keys()) if pattern.search(x) is not None] pft2patch_dict = {"pft": "patch"} @@ -234,7 +235,7 @@ def import_ds( - DOES actually read the dataset into memory, but only AFTER dropping unwanted variables and/or vegetation types. """ - utils.log(logger, "import_ds(): Start") + log(logger, "import_ds(): Start") filelist, my_vars, my_vegtypes, my_vars_missing_ok = process_inputs( filelist, my_vars, my_vegtypes, my_vars_missing_ok @@ -248,12 +249,12 @@ def import_ds( if time_slice: new_filelist = [] for file in sorted(filelist): - utils.log(logger, f"import_ds(): Getting filetime from file: {file}") + log(logger, f"import_ds(): Getting filetime from file: {file}") filetime = xr.open_dataset(file).time filetime_sel = utils.safer_timeslice(filetime, time_slice) include_this_file = filetime_sel.size if include_this_file: - utils.log( + log( logger, f"import_ds(): Including filetime : {filetime_sel['time'].values}" ) new_filelist.append(file) @@ -294,11 +295,11 @@ def import_ds( chunks=chunks, ) elif isinstance(filelist, str): - utils.log(logger, f"import_ds(): Opening this_ds from filelist: {filelist}") + log(logger, f"import_ds(): Opening this_ds from filelist: {filelist}") this_ds = xr.open_dataset(filelist, chunks=chunks) - utils.log(logger, "import_ds(): Calling mfdataset_preproc()...") + log(logger, "import_ds(): Calling mfdataset_preproc()...") this_ds = mfdataset_preproc(this_ds, my_vars, my_vegtypes, time_slice) - utils.log(logger, "import_ds(): Calling compute()...") + log(logger, "import_ds(): Calling compute()...") this_ds = this_ds.compute() # Warn and/or error about variables that couldn't be imported or derived @@ -323,5 +324,5 @@ def import_ds( if "lsmlon" in this_ds.dims: this_ds = this_ds.rename({"lsmlon": "lon"}) - utils.log(logger, "import_ds(): End") + log(logger, "import_ds(): End") return this_ds diff --git a/python/ctsm/ctsm_logging.py b/python/ctsm/ctsm_logging.py index e14ec2754c..d11d59fe75 100644 --- a/python/ctsm/ctsm_logging.py +++ b/python/ctsm/ctsm_logging.py @@ -29,8 +29,14 @@ import logging +from ctsm.utils import datetime_string + logger = logging.getLogger(__name__) +# In logfile lines, what should be used as spacing between the leading datetime string and the +# message text? +LOG_SPACING_AFTER_DATETIME_STR = " " * 4 + def setup_logging_pre_config(): """Setup logging for a script / application @@ -99,3 +105,24 @@ def output_to_file(file_path, message, log_to_logger=False): log_file.write(message) if log_to_logger: logger.info(message) + + +def log(logger_in, string): + """ + Simultaneously print INFO messages to console and to log file + """ + msg = datetime_string() + LOG_SPACING_AFTER_DATETIME_STR + string + print(msg) + if logger_in: + logger_in.info(msg) + + +def error(logger_in, string): + """ + Simultaneously print ERROR messages to console and to log file + """ + msg = datetime_string() + LOG_SPACING_AFTER_DATETIME_STR + string + print(msg) + if logger_in: + logger_in.error(msg) + raise RuntimeError(string) diff --git a/python/ctsm/utils.py b/python/ctsm/utils.py index 0b529fe282..df2b78ab7c 100644 --- a/python/ctsm/utils.py +++ b/python/ctsm/utils.py @@ -8,7 +8,7 @@ import re import pdb -from datetime import date, timedelta +from datetime import date, timedelta, datetime from getpass import getuser from ctsm.git_utils import get_ctsm_git_short_hash @@ -267,3 +267,10 @@ def find_one_file_matching_pattern(pattern): f"Expected 1 but found {n_found} files found matching pattern: " + pattern ) return file_list[0] + + +def datetime_string(): + """ + Return a datetime string like "YYYY-mm-dd HH:MM:SS" + """ + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") From c057113cb40af6875d83282bca8be734db2c0ff1 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 19 Aug 2025 13:53:44 -0600 Subject: [PATCH 159/196] ctsm_logging: Add error_type option to error(). --- python/ctsm/ctsm_logging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ctsm/ctsm_logging.py b/python/ctsm/ctsm_logging.py index d11d59fe75..d455a3790e 100644 --- a/python/ctsm/ctsm_logging.py +++ b/python/ctsm/ctsm_logging.py @@ -117,7 +117,7 @@ def log(logger_in, string): logger_in.info(msg) -def error(logger_in, string): +def error(logger_in, string, *, error_type=RuntimeError): """ Simultaneously print ERROR messages to console and to log file """ @@ -125,4 +125,4 @@ def error(logger_in, string): print(msg) if logger_in: logger_in.error(msg) - raise RuntimeError(string) + raise error_type(string) From 191e1190b107422085600829f5554d3cd3b87dd3 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 19 Aug 2025 16:08:38 -0600 Subject: [PATCH 160/196] Try adding caller name to log() and error() msgs. --- python/ctsm/ctsm_logging.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/python/ctsm/ctsm_logging.py b/python/ctsm/ctsm_logging.py index d455a3790e..4411ccef6a 100644 --- a/python/ctsm/ctsm_logging.py +++ b/python/ctsm/ctsm_logging.py @@ -27,6 +27,7 @@ setup_logging_for_tests (this is typically done via unit_testing.setup_for_tests) """ +import inspect import logging from ctsm.utils import datetime_string @@ -107,11 +108,17 @@ def output_to_file(file_path, message, log_to_logger=False): logger.info(message) +def _get_caller_name_for_logging(): + return inspect.stack()[2][3] + + def log(logger_in, string): """ Simultaneously print INFO messages to console and to log file """ - msg = datetime_string() + LOG_SPACING_AFTER_DATETIME_STR + string + msg = ( + _get_caller_name_for_logging() + datetime_string() + LOG_SPACING_AFTER_DATETIME_STR + string + ) print(msg) if logger_in: logger_in.info(msg) @@ -121,7 +128,9 @@ def error(logger_in, string, *, error_type=RuntimeError): """ Simultaneously print ERROR messages to console and to log file """ - msg = datetime_string() + LOG_SPACING_AFTER_DATETIME_STR + string + msg = ( + _get_caller_name_for_logging() + datetime_string() + LOG_SPACING_AFTER_DATETIME_STR + string + ) print(msg) if logger_in: logger_in.error(msg) From e277c9219fa8a42d21bab670d9c99d979e80696b Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 20 Aug 2025 09:42:36 -0600 Subject: [PATCH 161/196] Finalize caller name in log()/error() msgs. --- python/ctsm/crop_calendars/cropcal_module.py | 2 +- python/ctsm/crop_calendars/generate_gdds.py | 2 +- .../crop_calendars/generate_gdds_functions.py | 6 ++-- python/ctsm/crop_calendars/import_ds.py | 36 +++++++++---------- python/ctsm/ctsm_logging.py | 6 ++-- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/python/ctsm/crop_calendars/cropcal_module.py b/python/ctsm/crop_calendars/cropcal_module.py index d6fe3f6fc6..393e3a9cd5 100644 --- a/python/ctsm/crop_calendars/cropcal_module.py +++ b/python/ctsm/crop_calendars/cropcal_module.py @@ -55,7 +55,7 @@ def open_lu_ds(filename, year_1, year_n, existing_ds, *, logger, ungrid=True): Open land-use dataset """ # Open and trim to years of interest - log(logger, f"open_lu_ds(): Opening this_ds_gridded: {filename}") + log(logger, f"Opening this_ds_gridded: {filename}") this_ds_gridded = xr.open_dataset(filename).sel(time=slice(year_1, year_n)) # Assign actual lon/lat coordinates diff --git a/python/ctsm/crop_calendars/generate_gdds.py b/python/ctsm/crop_calendars/generate_gdds.py index 233fbc0866..cb69d9e230 100644 --- a/python/ctsm/crop_calendars/generate_gdds.py +++ b/python/ctsm/crop_calendars/generate_gdds.py @@ -223,7 +223,7 @@ def main( log(logger, "Done") if not h2_ds: - log(logger, f"generate_gdds main(): Opening h2_ds: {h2_ds_file}") + log(logger, f"Opening h2_ds: {h2_ds_file}") h2_ds = xr.open_dataset(h2_ds_file) ###################################################### diff --git a/python/ctsm/crop_calendars/generate_gdds_functions.py b/python/ctsm/crop_calendars/generate_gdds_functions.py index c27ebdfb57..be724659dd 100644 --- a/python/ctsm/crop_calendars/generate_gdds_functions.py +++ b/python/ctsm/crop_calendars/generate_gdds_functions.py @@ -253,7 +253,7 @@ def import_and_process_1yr( Import one year of CLM output data for GDD generation """ save_figs = True - log(logger, f"import_and_process_1yr(): netCDF year {this_year}...") + log(logger, f"netCDF year {this_year}...") # Without dask, this can take a LONG time at resolutions finer than 2-deg if importlib_util.find_spec("dask"): @@ -588,7 +588,7 @@ def import_and_process_1yr( incl_vegtype_indices = [] for var, vegtype_str in enumerate(incl_vegtypes_str): if vegtype_str in skip_crops: - log(logger, f" import_and_process_1yr(): SKIPPING {vegtype_str}") + log(logger, f"SKIPPING {vegtype_str}") continue vegtype_int = utils.vegtype_str2int(vegtype_str)[0] @@ -603,7 +603,7 @@ def import_and_process_1yr( check_gddharv = True if not this_crop_gddaccum_da.size: continue - log(logger, f" import_and_process_1yr(): {vegtype_str}...") + log(logger, f"{vegtype_str}...") incl_vegtype_indices = incl_vegtype_indices + [var] # Get prescribed harvest dates for these patches diff --git a/python/ctsm/crop_calendars/import_ds.py b/python/ctsm/crop_calendars/import_ds.py index ccf712fe4b..d2ace51ef0 100644 --- a/python/ctsm/crop_calendars/import_ds.py +++ b/python/ctsm/crop_calendars/import_ds.py @@ -21,7 +21,7 @@ def compute_derived_vars(ds_in, var, logger=None): """ Compute derived variables """ - log(logger, f"compute_derived_vars(): Getting {var}...") + log(logger, f"Getting {var}...") if ( var == "HYEARS" and "HDATES" in ds_in @@ -53,7 +53,7 @@ def manual_mfdataset(filelist, my_vars, my_vegtypes, time_slice, logger=None): """ ds_out = None for filename in filelist: - log(logger, f"manual_mfdataset(): Opening ds_in: {ds_in}") + log(logger, f"Opening ds_in: {ds_in}") ds_in = xr.open_dataset(filename) ds_in = mfdataset_preproc(ds_in, my_vars, my_vegtypes, time_slice) if ds_out is None: @@ -79,7 +79,7 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log named like "patch". This can later be reversed, for compatibility with other code, using patch2pft(). """ - log(logger, "mfdataset_preproc(): Start") + log(logger, "Start") # Rename "pft" dimension and variables to "patch", if needed if "pft" in ds_in.dims: @@ -87,7 +87,7 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log derived_vars = [] if vars_to_import is not None: - log(logger, "mfdataset_preproc(): Getting vars to drop...") + log(logger, "Getting vars to drop...") # Split vars_to_import into variables that are vs. aren't already in ds derived_vars = [v for v in vars_to_import if v not in ds_in] @@ -125,12 +125,12 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log vars_to_drop = list(np.setdiff1d(varlist, vars_to_import)) # Drop them - log(logger, f"mfdataset_preproc(): Dropping variables: {vars_to_drop}") + log(logger, f"Dropping variables: {vars_to_drop}") ds_in = ds_in.drop_vars(vars_to_drop) # Add vegetation type info if "patches1d_itype_veg" in list(ds_in): - log(logger, "mfdataset_preproc(): Adding vegetation type info") + log(logger, "Adding vegetation type info") this_pftlist = utils.define_pftlist() utils.get_patch_ivts( ds_in, this_pftlist @@ -150,25 +150,25 @@ def mfdataset_preproc(ds_in, vars_to_import, vegtypes_to_import, time_slice, log # Restrict to veg. types of interest, if any if vegtypes_to_import is not None: - log(logger, f"mfdataset_preproc(): Restricting veg types to: {vegtypes_to_import}") + log(logger, f"Restricting veg types to: {vegtypes_to_import}") ds_in = xr_flexsel(ds_in, vegtype=vegtypes_to_import) # Restrict to time slice, if any if time_slice: - log(logger, f"mfdataset_preproc(): Restricting time slice to: {time_slice}") + log(logger, f"Restricting time slice to: {time_slice}") ds_in = utils.safer_timeslice(ds_in, time_slice) # Finish import - log(logger, "mfdataset_preproc(): decode_cf()...") + log(logger, "decode_cf()...") ds_in = xr.decode_cf(ds_in, decode_times=True) # Compute derived variables if derived_vars: - log(logger, "mfdataset_preproc(): decode_cf()...") + log(logger, "decode_cf()...") for var in derived_vars: ds_in = compute_derived_vars(ds_in, var, logger) - log(logger, "mfdataset_preproc(): End") + log(logger, "End") return ds_in @@ -235,7 +235,7 @@ def import_ds( - DOES actually read the dataset into memory, but only AFTER dropping unwanted variables and/or vegetation types. """ - log(logger, "import_ds(): Start") + log(logger, "Start") filelist, my_vars, my_vegtypes, my_vars_missing_ok = process_inputs( filelist, my_vars, my_vegtypes, my_vars_missing_ok @@ -249,13 +249,13 @@ def import_ds( if time_slice: new_filelist = [] for file in sorted(filelist): - log(logger, f"import_ds(): Getting filetime from file: {file}") + log(logger, f"Getting filetime from file: {file}") filetime = xr.open_dataset(file).time filetime_sel = utils.safer_timeslice(filetime, time_slice) include_this_file = filetime_sel.size if include_this_file: log( - logger, f"import_ds(): Including filetime : {filetime_sel['time'].values}" + logger, f"Including filetime : {filetime_sel['time'].values}" ) new_filelist.append(file) @@ -295,11 +295,11 @@ def import_ds( chunks=chunks, ) elif isinstance(filelist, str): - log(logger, f"import_ds(): Opening this_ds from filelist: {filelist}") + log(logger, f"Opening this_ds from filelist: {filelist}") this_ds = xr.open_dataset(filelist, chunks=chunks) - log(logger, "import_ds(): Calling mfdataset_preproc()...") + log(logger, "Calling mfdataset_preproc()...") this_ds = mfdataset_preproc(this_ds, my_vars, my_vegtypes, time_slice) - log(logger, "import_ds(): Calling compute()...") + log(logger, "Calling compute()...") this_ds = this_ds.compute() # Warn and/or error about variables that couldn't be imported or derived @@ -324,5 +324,5 @@ def import_ds( if "lsmlon" in this_ds.dims: this_ds = this_ds.rename({"lsmlon": "lon"}) - log(logger, "import_ds(): End") + log(logger, "End") return this_ds diff --git a/python/ctsm/ctsm_logging.py b/python/ctsm/ctsm_logging.py index 4411ccef6a..780048ac90 100644 --- a/python/ctsm/ctsm_logging.py +++ b/python/ctsm/ctsm_logging.py @@ -36,7 +36,7 @@ # In logfile lines, what should be used as spacing between the leading datetime string and the # message text? -LOG_SPACING_AFTER_DATETIME_STR = " " * 4 +LOG_SPACING = " " * 4 def setup_logging_pre_config(): @@ -117,7 +117,7 @@ def log(logger_in, string): Simultaneously print INFO messages to console and to log file """ msg = ( - _get_caller_name_for_logging() + datetime_string() + LOG_SPACING_AFTER_DATETIME_STR + string + datetime_string() + LOG_SPACING + _get_caller_name_for_logging() + LOG_SPACING + string ) print(msg) if logger_in: @@ -129,7 +129,7 @@ def error(logger_in, string, *, error_type=RuntimeError): Simultaneously print ERROR messages to console and to log file """ msg = ( - _get_caller_name_for_logging() + datetime_string() + LOG_SPACING_AFTER_DATETIME_STR + string + datetime_string() + LOG_SPACING + _get_caller_name_for_logging() + LOG_SPACING + string ) print(msg) if logger_in: From 037ae1b134b308675b3cdb5a6eef3afef804f7dc Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 20 Aug 2025 10:01:11 -0600 Subject: [PATCH 162/196] ctsm_logging: Functionize _compose_log_msg(). --- python/ctsm/ctsm_logging.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/python/ctsm/ctsm_logging.py b/python/ctsm/ctsm_logging.py index 780048ac90..e6aa01a254 100644 --- a/python/ctsm/ctsm_logging.py +++ b/python/ctsm/ctsm_logging.py @@ -108,17 +108,21 @@ def output_to_file(file_path, message, log_to_logger=False): logger.info(message) -def _get_caller_name_for_logging(): - return inspect.stack()[2][3] +def _compose_log_msg(string, frame_record=2): + """ + Prepend the log/error string with reference information + """ + # Get name of the function that called log() or error() + caller_name = inspect.stack()[frame_record][3] + + return datetime_string() + LOG_SPACING + caller_name + LOG_SPACING + string def log(logger_in, string): """ Simultaneously print INFO messages to console and to log file """ - msg = ( - datetime_string() + LOG_SPACING + _get_caller_name_for_logging() + LOG_SPACING + string - ) + msg = _compose_log_msg(string) print(msg) if logger_in: logger_in.info(msg) @@ -128,9 +132,7 @@ def error(logger_in, string, *, error_type=RuntimeError): """ Simultaneously print ERROR messages to console and to log file """ - msg = ( - datetime_string() + LOG_SPACING + _get_caller_name_for_logging() + LOG_SPACING + string - ) + msg = _compose_log_msg(string) print(msg) if logger_in: logger_in.error(msg) From dcb39b4e83e75b2706e9c23114bd79c741a0d01d Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 20 Aug 2025 10:53:30 -0600 Subject: [PATCH 163/196] Add unit tests of log() and error(). --- python/ctsm/test/test_unit_ctsm_logging.py | 109 +++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100755 python/ctsm/test/test_unit_ctsm_logging.py diff --git a/python/ctsm/test/test_unit_ctsm_logging.py b/python/ctsm/test/test_unit_ctsm_logging.py new file mode 100755 index 0000000000..edfd817a46 --- /dev/null +++ b/python/ctsm/test/test_unit_ctsm_logging.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +"""Unit tests for functions in ctsm_logging""" + +import unittest +import io +from contextlib import redirect_stdout +from datetime import datetime + +from ctsm import unit_testing +from ctsm.ctsm_logging import log, error +from ctsm.utils import datetime_string + +# Allow names that pylint doesn't like, because otherwise I find it hard +# to make readable unit test names +# pylint: disable=invalid-name + + +DATETIME_STR_PATTERN = r"\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}" + + +class TestLog(unittest.TestCase): + """ + Tests of log() function + + Currently not testing ability to write to file, because unittest makes that difficult. So just + testing write to stdout. + """ + + def test_datetime_str_pattern(self): + """Test that our regex matches output of datetime_str""" + self.assertRegex(datetime_string(), expected_regex=DATETIME_STR_PATTERN) + + def test_log_without_logger(self): + """ + Tests the log() function without providing a logger for writing to file + """ + msg = "abc123" + f = io.StringIO() + with redirect_stdout(f): + log(None, msg) + + # Check that stdout matches what we expect + stdout = f.getvalue() + expected_regex = DATETIME_STR_PATTERN + r"\s+test_log_without_logger\s+" + msg + self.assertRegex(stdout, expected_regex=expected_regex) + + +class TestError(unittest.TestCase): + """ + Tests of error() function + + Currently not testing ability to write to file, because unittest makes that difficult. So just + testing write to stdout and error raising. + """ + + def test_error_without_logger(self): + """ + Tests the error() function without providing a logger for writing to file and without + specifying a custom error type + """ + msg = "abc123" + f = io.StringIO() + error_raised = None + try: + with redirect_stdout(f): + error(None, msg) + except Exception as e: # pylint: disable=broad-exception-caught + error_raised = e + + # Check that stdout matches what we expect + stdout = f.getvalue() + expected_regex = DATETIME_STR_PATTERN + r"\s+test_error_without_logger\s+" + msg + self.assertRegex(stdout, expected_regex=expected_regex) + + # Check that error is correct + self.assertFalse(error_raised is None) + self.assertIsInstance(error_raised, RuntimeError) + self.assertEqual(msg, str(error_raised)) + + def test_error_without_logger_custom_err(self): + """ + Tests the error() function without providing a logger for writing to file and + specifying a custom error type + """ + msg = "abc123" + f = io.StringIO() + error_raised = None + error_type = ValueError + try: + with redirect_stdout(f): + error(None, msg, error_type=error_type) + except Exception as e: # pylint: disable=broad-exception-caught + error_raised = e + + # Check that stdout matches what we expect + stdout = f.getvalue() + expected_regex = DATETIME_STR_PATTERN + r"\s+test_error_without_logger_custom_err\s+" + msg + self.assertRegex(stdout, expected_regex=expected_regex) + + # Check that error is correct + self.assertFalse(error_raised is None) + self.assertIsInstance(error_raised, error_type) + self.assertEqual(msg, str(error_raised)) + + +if __name__ == "__main__": + unit_testing.setup_for_tests() + unittest.main() From 75c67945afbe9a62edd41fbbe060468942a0bb40 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 13 Aug 2025 16:24:27 -0600 Subject: [PATCH 164/196] Initial add of a unit test for endrun --- src/main/test/CMakeLists.txt | 1 + src/main/test/endrun_test/CMakeLists.txt | 8 ++++ src/main/test/endrun_test/test_endrun.pf | 47 ++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 src/main/test/endrun_test/CMakeLists.txt create mode 100644 src/main/test/endrun_test/test_endrun.pf diff --git a/src/main/test/CMakeLists.txt b/src/main/test/CMakeLists.txt index 97bbf081cc..2ade2f6e87 100644 --- a/src/main/test/CMakeLists.txt +++ b/src/main/test/CMakeLists.txt @@ -8,3 +8,4 @@ add_subdirectory(filter_test) add_subdirectory(initVertical_test) add_subdirectory(ncdio_utils_test) add_subdirectory(topo_test) +add_subdirectory(endrun_test) diff --git a/src/main/test/endrun_test/CMakeLists.txt b/src/main/test/endrun_test/CMakeLists.txt new file mode 100644 index 0000000000..45b42a15e7 --- /dev/null +++ b/src/main/test/endrun_test/CMakeLists.txt @@ -0,0 +1,8 @@ +set(pfunit_sources + test_endrun.pf) + +add_pfunit_ctest(endrun + TEST_SOURCES "${pfunit_sources}" + LINK_LIBRARIES clm csm_share esmf + EXTRA_FINALIZE unittest_finalize_esmf + EXTRA_USE unittestInitializeAndFinalize) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf new file mode 100644 index 0000000000..a6b7c19647 --- /dev/null +++ b/src/main/test/endrun_test/test_endrun.pf @@ -0,0 +1,47 @@ +module test_endrun + + ! Tests of abortutils + + use funit + use abortutils + use unittestUtils, only : endrun_msg + + implicit none + + @TestCase + type, extends(TestCase) :: TestAbortUtils + contains + procedure :: setUp + procedure :: tearDown + end type TestAbortUtils + +contains + + ! ======================================================================== + ! Helper routines + ! ======================================================================== + + subroutine setUp(this) + class(TestAbortUtils), intent(inout) :: this + end subroutine setUp + + subroutine tearDown(this) + class(TestAbortUtils), intent(inout) :: this + + end subroutine tearDown + + ! ======================================================================== + ! Begin tests + ! ======================================================================== + + @Test + subroutine endrun_vanilla_aborts(this) + ! Test vanilla operation of endrun + class(TestAbortUtils), intent(inout) :: this + + call endrun() + @assertExceptionRaised(endrun_msg('')) + + end subroutine endrun_vanilla_aborts + +end module test_endrun From 4053c0ac18133c0b44509cc694d05bdde1364b45 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 14 Aug 2025 16:49:42 -0600 Subject: [PATCH 165/196] Add more tests for more of the options --- src/main/test/endrun_test/test_endrun.pf | 55 +++++++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf index a6b7c19647..3d8f46782e 100644 --- a/src/main/test/endrun_test/test_endrun.pf +++ b/src/main/test/endrun_test/test_endrun.pf @@ -5,6 +5,8 @@ module test_endrun use funit use abortutils use unittestUtils, only : endrun_msg + use shr_kind_mod, only : CL => shr_kind_cl + use clm_varctl, only : iulog implicit none @@ -35,13 +37,62 @@ contains ! ======================================================================== @Test - subroutine endrun_vanilla_aborts(this) + subroutine endrun_plain_vanilla_aborts(this) ! Test vanilla operation of endrun class(TestAbortUtils), intent(inout) :: this call endrun() @assertExceptionRaised(endrun_msg('')) - end subroutine endrun_vanilla_aborts + end subroutine endrun_plain_vanilla_aborts + + @Test + subroutine endrun_msg_vanilla_aborts(this) + ! Test vanilla operation of endrun with a message sent in + class(TestAbortUtils), intent(inout) :: this + character(len=CL) :: msg = "test_message" + + call endrun( msg = msg) + @assertExceptionRaised(endrun_msg(msg)) + + end subroutine endrun_msg_vanilla_aborts + + @Test + subroutine endrun_addmsg_vanilla_aborts(this) + ! Test vanilla operation of endrun with an additional message sent in + class(TestAbortUtils), intent(inout) :: this + character(len=CL) :: msg = "test_message" + character(len=CL) :: add_msg = "additional_test_message" + + call endrun(msg=msg, additional_msg=add_msg) + @assertExceptionRaised(endrun_msg(msg)) + + end subroutine endrun_addmsg_vanilla_aborts + + @Test + subroutine endrun_addmsg_pt_context_aborts(this) + use decompMod, only : subgrid_level_lndgrid, subgrid_level_gridcell + use decompMod, only : subgrid_level_landunit, subgrid_level_column, subgrid_level_patch + use decompMod, only : subgrid_level_cohort + use unittestSimpleSubgridSetupsMod, only : setup_single_veg_patch + ! Test pt_context operation of endrun with an additional message sent in + class(TestAbortUtils), intent(inout) :: this + character(len=CL) :: msg = "test_message" + character(len=CL) :: add_msg = "additional_test_message" + integer :: p = 1, l + integer, parameter :: nlevel = 6 + integer :: subgrid_lvl(nlevel) = (/ subgrid_level_lndgrid, subgrid_level_gridcell, & + subgrid_level_landunit, subgrid_level_column, subgrid_level_patch, & + subgrid_level_cohort /) + + call setup_single_veg_patch(pft_type=1) + ! Loop over all the subgrid level types + do l = 2, nlevel-1 + write(iulog,*) 'level = ', l + call endrun(subgrid_index=p, subgrid_level=subgrid_lvl(l), msg=msg, additional_msg=add_msg) + @assertExceptionRaised(endrun_msg(msg)) + end do + + end subroutine endrun_addmsg_pt_context_aborts end module test_endrun From 4dc49a97b9427e788dc2ad11cec662ed2756e028 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 15:48:25 -0600 Subject: [PATCH 166/196] Move subgrid setup and teardown to setup and teardown methods, add explicit tests for subgrid levels not supported: lndgrid and cohort and for unspecified, right now these fail --- src/main/test/endrun_test/test_endrun.pf | 61 +++++++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf index 3d8f46782e..dfea0b6195 100644 --- a/src/main/test/endrun_test/test_endrun.pf +++ b/src/main/test/endrun_test/test_endrun.pf @@ -24,12 +24,21 @@ contains ! ======================================================================== subroutine setUp(this) + use unittestSimpleSubgridSetupsMod, only : setup_single_veg_patch class(TestAbortUtils), intent(inout) :: this + + ! Setup a single gridcell with one vegetated patch + ! So there's only one: gridcell, landunit, column, patch + ! This isn't needed for some tests, but doesn't hurt to do it + call setup_single_veg_patch(pft_type=1) end subroutine setUp subroutine tearDown(this) + use unittestSubgridMod, only : unittest_subgrid_teardown class(TestAbortUtils), intent(inout) :: this + call unittest_subgrid_teardown() + end subroutine tearDown ! ======================================================================== @@ -74,7 +83,6 @@ contains use decompMod, only : subgrid_level_lndgrid, subgrid_level_gridcell use decompMod, only : subgrid_level_landunit, subgrid_level_column, subgrid_level_patch use decompMod, only : subgrid_level_cohort - use unittestSimpleSubgridSetupsMod, only : setup_single_veg_patch ! Test pt_context operation of endrun with an additional message sent in class(TestAbortUtils), intent(inout) :: this character(len=CL) :: msg = "test_message" @@ -85,7 +93,6 @@ contains subgrid_level_landunit, subgrid_level_column, subgrid_level_patch, & subgrid_level_cohort /) - call setup_single_veg_patch(pft_type=1) ! Loop over all the subgrid level types do l = 2, nlevel-1 write(iulog,*) 'level = ', l @@ -95,4 +102,54 @@ contains end subroutine endrun_addmsg_pt_context_aborts + @Test + subroutine endrun_pt_context_lndgrid_aborts(this) + use decompMod, only : subgrid_level_lndgrid + class(TestAbortUtils), intent(inout) :: this + character(len=CL) :: msg = "test_message" + integer :: p = 1 + + ! Also test without an additional msg + call endrun(subgrid_index=p, subgrid_level=subgrid_level_lndgrid, msg=msg) + @assertExceptionRaised(endrun_msg(msg)) + + end subroutine endrun_pt_context_lndgrid_aborts + + @Test + subroutine endrun_nomsg_pt_context_cohort_aborts(this) + use decompMod, only : subgrid_level_cohort + class(TestAbortUtils), intent(inout) :: this + integer :: p = 1 + + ! Also test without either msg or additional msg + call endrun(subgrid_index=p, subgrid_level=subgrid_level_cohort) + @assertExceptionRaised(endrun_msg('')) + + end subroutine endrun_nomsg_pt_context_cohort_aborts + + @Test + subroutine endrun_nomsg_addmsg_pt_context_unspec_aborts(this) + use decompMod, only : subgrid_level_unspecified + class(TestAbortUtils), intent(inout) :: this + integer :: p = 1 + character(len=CL) :: add_msg = "additional_test_message" + + ! Don't use msg but do use additional_msg + call endrun(subgrid_index=p, subgrid_level=subgrid_level_unspecified, additional_msg=add_msg) + @assertExceptionRaised(endrun_msg('')) + + end subroutine endrun_nomsg_addmsg_pt_context_unspec_aborts + + @Test + subroutine endrun_nomsg_pt_context_badlvl_aborts(this) + use decompMod, only : subgrid_level_unspecified + class(TestAbortUtils), intent(inout) :: this + integer :: p = 1 + character(len=CL) :: expected_msg = "subgrid_level not supported" + + call endrun(subgrid_index=p, subgrid_level=subgrid_level_unspecified) + @assertExceptionRaised(endrun_msg(expected_msg)) + + end subroutine endrun_nomsg_pt_context_badlvl_aborts + end module test_endrun From 1586175f0a125827cb5f65288b301a0346ceb9cf Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 16:43:32 -0600 Subject: [PATCH 167/196] Continue the endrun call even if subgrid_level is bad so that the error messaging gives as much information as possible about the error, also reuse the endrun_vanilla in the pt_context version to remove duplication --- src/main/abortutils.F90 | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/main/abortutils.F90 b/src/main/abortutils.F90 index c93fd761bf..23e36c3e95 100644 --- a/src/main/abortutils.F90 +++ b/src/main/abortutils.F90 @@ -89,13 +89,7 @@ subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additio call write_point_context(subgrid_index, subgrid_level) end if - if (present (additional_msg)) then - write(iulog,*)'ENDRUN: ', additional_msg - else - write(iulog,*)'ENDRUN:' - end if - - call shr_sys_abort(msg) + call endrun_vanilla(msg=msg, additional_msg=additional_msg) end subroutine endrun_write_point_context @@ -188,11 +182,10 @@ subroutine write_point_context(subgrid_index, subgrid_level) else write(iulog,*) 'subgrid_level not supported: ', subgrid_level - call shr_sys_abort('subgrid_level not supported '//errmsg(sourcefile, __LINE__)) + write(iulog,*) errmsg(sourcefile, __LINE__) + write(iulog,*) 'Continuing the endrun without writing point context information' end if - call shr_sys_flush(iulog) - end subroutine write_point_context end module abortutils From 6c38d62d8cca86e3d52edf9ca831f63dcbc6769f Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 16:46:15 -0600 Subject: [PATCH 168/196] Change to the new expected behavior where the error isn't about the subgrid_level, but the original error sent in, this and the previous commit resolve #3420 --- src/main/test/endrun_test/test_endrun.pf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf index dfea0b6195..3f2844ee20 100644 --- a/src/main/test/endrun_test/test_endrun.pf +++ b/src/main/test/endrun_test/test_endrun.pf @@ -94,8 +94,8 @@ contains subgrid_level_cohort /) ! Loop over all the subgrid level types + ! Skip the first one and the last one which are: lndgrid and cohort do l = 2, nlevel-1 - write(iulog,*) 'level = ', l call endrun(subgrid_index=p, subgrid_level=subgrid_lvl(l), msg=msg, additional_msg=add_msg) @assertExceptionRaised(endrun_msg(msg)) end do @@ -145,10 +145,9 @@ contains use decompMod, only : subgrid_level_unspecified class(TestAbortUtils), intent(inout) :: this integer :: p = 1 - character(len=CL) :: expected_msg = "subgrid_level not supported" - call endrun(subgrid_index=p, subgrid_level=subgrid_level_unspecified) - @assertExceptionRaised(endrun_msg(expected_msg)) + call endrun(subgrid_index=p, subgrid_level=-9999) + @assertExceptionRaised(endrun_msg('')) end subroutine endrun_nomsg_pt_context_badlvl_aborts From 7d25eba11273ab5311a3b3827ad6cfc1cdc6fe76 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 16:57:55 -0600 Subject: [PATCH 169/196] Update to using shr_abort_abort as talked about in #3417 --- src/main/abortutils.F90 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/abortutils.F90 b/src/main/abortutils.F90 index 23e36c3e95..260169a488 100644 --- a/src/main/abortutils.F90 +++ b/src/main/abortutils.F90 @@ -33,7 +33,7 @@ subroutine endrun_vanilla(msg, additional_msg) ! !DESCRIPTION: ! Abort the model for abnormal termination ! - use shr_sys_mod , only: shr_sys_abort + use shr_abort_mod , only: shr_abort_abort use clm_varctl , only: iulog ! ! !ARGUMENTS: @@ -42,8 +42,8 @@ subroutine endrun_vanilla(msg, additional_msg) ! volatile stuff in additional_msg, as in: ! call endrun(msg='Informative message', additional_msg=errmsg(__FILE__, __LINE__)) ! and then just assert against msg. - character(len=*), intent(in), optional :: msg ! string to be passed to shr_sys_abort - character(len=*), intent(in), optional :: additional_msg ! string to be printed, but not passed to shr_sys_abort + character(len=*), intent(in), optional :: msg ! string to be passed to shr_abort + character(len=*), intent(in), optional :: additional_msg ! string to be printed, but not passed to shr_abort !----------------------------------------------------------------------- if (present (additional_msg)) then @@ -52,7 +52,7 @@ subroutine endrun_vanilla(msg, additional_msg) write(iulog,*)'ENDRUN:' end if - call shr_sys_abort(msg) + call shr_abort_abort(msg) end subroutine endrun_vanilla @@ -65,7 +65,6 @@ subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additio ! ! This version also prints additional information about the point causing the error. ! - use shr_sys_mod , only: shr_sys_abort use clm_varctl , only: iulog use decompMod , only: subgrid_level_unspecified ! @@ -78,8 +77,8 @@ subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additio ! volatile stuff in additional_msg, as in: ! call endrun(msg='Informative message', additional_msg=errmsg(__FILE__, __LINE__)) ! and then just assert against msg. - character(len=*), intent(in), optional :: msg ! string to be passed to shr_sys_abort - character(len=*), intent(in), optional :: additional_msg ! string to be printed, but not passed to shr_sys_abort + character(len=*), intent(in), optional :: msg ! string to be passed to shr_abort + character(len=*), intent(in), optional :: additional_msg ! string to be printed, but not passed to shr_abort ! ! Local Variables: integer :: igrc, ilun, icol @@ -101,7 +100,8 @@ subroutine write_point_context(subgrid_index, subgrid_level) ! Write various information giving context for the given index at the given subgrid ! level, including global index information and more. ! - use shr_sys_mod , only : shr_sys_flush, shr_sys_abort + ! NOTE: DO NOT CALL AN ABORT FROM HERE AS THAT WOULD SHORT CIRUIT THE ERROR REPORTING!! + ! use shr_log_mod , only : errMsg => shr_log_errMsg use clm_varctl , only : iulog use decompMod , only : subgrid_level_gridcell, subgrid_level_landunit, subgrid_level_column, subgrid_level_patch From a13dcdb7087adb0dabf3cfc9e07d8f3710848d81 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 17:00:01 -0600 Subject: [PATCH 170/196] Update to using shr_abort_abort as talked about in #3417 --- src/main/test/endrun_test/test_endrun.pf | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf index 3f2844ee20..7c63cf6781 100644 --- a/src/main/test/endrun_test/test_endrun.pf +++ b/src/main/test/endrun_test/test_endrun.pf @@ -102,6 +102,28 @@ contains end subroutine endrun_addmsg_pt_context_aborts + @Test + subroutine endrun_nomsg_pt_context_bad_pt_aborts(this) + use decompMod, only : subgrid_level_lndgrid, subgrid_level_gridcell + use decompMod, only : subgrid_level_landunit, subgrid_level_column, subgrid_level_patch + use decompMod, only : subgrid_level_cohort + ! Test pt_context operation of endrun with an additional message sent in + class(TestAbortUtils), intent(inout) :: this + integer :: p = 2, l + integer, parameter :: nlevel = 6 + integer :: subgrid_lvl(nlevel) = (/ subgrid_level_lndgrid, subgrid_level_gridcell, & + subgrid_level_landunit, subgrid_level_column, subgrid_level_patch, & + subgrid_level_cohort /) + + ! Loop over all the subgrid level types + ! Skip the first one and the last one which are: lndgrid and cohort + do l = 2, nlevel-1 + call endrun(subgrid_index=p, subgrid_level=subgrid_lvl(l), msg=msg, additional_msg=add_msg) + @assertExceptionRaised(endrun_msg('')) + end do + + end subroutine endrun_nomsg_pt_context_bad_pt_aborts + @Test subroutine endrun_pt_context_lndgrid_aborts(this) use decompMod, only : subgrid_level_lndgrid From 33a8e000051ec85e6d453c72555931fefe346bbf Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 17:05:27 -0600 Subject: [PATCH 171/196] Add testing for bad point in the pt_context which fails because of subscript overflow --- src/main/test/endrun_test/test_endrun.pf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf index 7c63cf6781..9b3edb7bc6 100644 --- a/src/main/test/endrun_test/test_endrun.pf +++ b/src/main/test/endrun_test/test_endrun.pf @@ -118,7 +118,7 @@ contains ! Loop over all the subgrid level types ! Skip the first one and the last one which are: lndgrid and cohort do l = 2, nlevel-1 - call endrun(subgrid_index=p, subgrid_level=subgrid_lvl(l), msg=msg, additional_msg=add_msg) + call endrun(subgrid_index=p, subgrid_level=subgrid_lvl(l)) @assertExceptionRaised(endrun_msg('')) end do From ef54137fda4da0233e14464487afa3a295a02bf7 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 18:46:19 -0600 Subject: [PATCH 172/196] Add handling in case the input point is bad to end_run_pt_context --- src/main/abortutils.F90 | 112 +++++++++++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 30 deletions(-) diff --git a/src/main/abortutils.F90 b/src/main/abortutils.F90 index 260169a488..517b80fffa 100644 --- a/src/main/abortutils.F90 +++ b/src/main/abortutils.F90 @@ -117,43 +117,102 @@ subroutine write_point_context(subgrid_index, subgrid_level) integer , intent(in) :: subgrid_level ! one of the subgrid_level_* constants defined in decompMod ! ! Local Variables: - integer :: igrc, ilun, icol, ipft + integer, parameter :: unset = -9999 ! Unset value for an index + integer :: igrc=unset, ilun=unset, icol=unset, ipft=unset ! Local index for grid-cell, landunit, column, and patch + integer :: ggrc=unset, glun=unset, gcol=unset, gpft=unset ! Global index for grid-cell, landunit, column, and patch + integer :: bad_point = .false. ! Flag to indicate if the point is bad (i.e., global index is -1) !----------------------------------------------------------------------- if (subgrid_level == subgrid_level_gridcell) then igrc = subgrid_index + ggrc = get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell, donot_abort_on_badindex=.true.) + + else if (subgrid_level == subgrid_level_landunit) then + + ilun = subgrid_index + glun = get_global_index(subgrid_index=ilun, subgrid_level=subgrid_level_landunit, donot_abort_on_badindex=.true.) + if ( glun /= -1 ) then + igrc = lun%gridcell(ilun) + ggrc = get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell, donot_abort_on_badindex=.true.) + else + bad_point = .true. + end if + + else if (subgrid_level == subgrid_level_column) then + + icol = subgrid_index + gcol = get_global_index(subgrid_index=icol, subgrid_level=subgrid_level_column, donot_abort_on_badindex=.true.) + if ( gcol /= -1 ) then + ilun = col%landunit(icol) + igrc = col%gridcell(icol) + ggrc = get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell, donot_abort_on_badindex=.true.) + glun = get_global_index(subgrid_index=ilun, subgrid_level=subgrid_level_landunit, donot_abort_on_badindex=.true.) + else + bad_point = .true. + end if + + else if (subgrid_level == subgrid_level_patch) then + + ipft = subgrid_index + gpft = get_global_index(subgrid_index=ipft, subgrid_level=subgrid_level_patch, donot_abort_on_badindex=.true.) + if ( gpft /= -1 ) then + icol = patch%column(ipft) + ilun = patch%landunit(ipft) + igrc = patch%gridcell(ipft) + ggrc = get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell, donot_abort_on_badindex=.true.) + glun = get_global_index(subgrid_index=ilun, subgrid_level=subgrid_level_landunit, donot_abort_on_badindex=.true.) + gcol = get_global_index(subgrid_index=icol, subgrid_level=subgrid_level_column, donot_abort_on_badindex=.true.) + else + bad_point = .true. + end if + + end if + + ! If one of the global indices is -1 then this is a bad point, so flag a bad-point + if ( igrc /= unset) then + if ( ggrc == -1 ) bad_point = .true. + end if + if ( ilun /= unset) then + if ( glun == -1 ) bad_point = .true. + end if + if ( icol /= unset) then + if ( gcol == -1 ) bad_point = .true. + end if + if ( ipft /= unset) then + if ( gpft == -1 ) bad_point = .true. + end if + + if (bad_point) then + write(iulog,*) 'A bad input point was given: subgrid_index = ', subgrid_index, & + ', subgrid_level = ', subgrid_level + write(iulog,*) errmsg(sourcefile, __LINE__) + write(iulog,*) 'Continuing the endrun without writing point context information' + return + end if + + if (subgrid_level == subgrid_level_gridcell) then + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': local gridcell index = ', igrc - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', & - get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell) + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', ggrc write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell longitude = ', grc%londeg(igrc) write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell latitude = ', grc%latdeg(igrc) else if (subgrid_level == subgrid_level_landunit) then - ilun = subgrid_index - igrc = lun%gridcell(ilun) write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': local landunit index = ', ilun - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global landunit index = ', & - get_global_index(subgrid_index=ilun, subgrid_level=subgrid_level_landunit) - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', & - get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell) + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global landunit index = ', glun + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', ggrc write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell longitude = ', grc%londeg(igrc) write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell latitude = ', grc%latdeg(igrc) write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': landunit type = ', lun%itype(subgrid_index) else if (subgrid_level == subgrid_level_column) then - icol = subgrid_index - ilun = col%landunit(icol) - igrc = col%gridcell(icol) write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': local column index = ', icol - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global column index = ', & - get_global_index(subgrid_index=icol, subgrid_level=subgrid_level_column) - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global landunit index = ', & - get_global_index(subgrid_index=ilun, subgrid_level=subgrid_level_landunit) - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', & - get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell) + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global column index = ', gcol + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global landunit index = ', glun + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', ggrc write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell longitude = ', grc%londeg(igrc) write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell latitude = ', grc%latdeg(igrc) write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': column type = ', col%itype(icol) @@ -161,19 +220,11 @@ subroutine write_point_context(subgrid_index, subgrid_level) else if (subgrid_level == subgrid_level_patch) then - ipft = subgrid_index - icol = patch%column(ipft) - ilun = patch%landunit(ipft) - igrc = patch%gridcell(ipft) write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': local patch index = ', ipft - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global patch index = ', & - get_global_index(subgrid_index=ipft, subgrid_level=subgrid_level_patch) - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global column index = ', & - get_global_index(subgrid_index=icol, subgrid_level=subgrid_level_column) - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global landunit index = ', & - get_global_index(subgrid_index=ilun, subgrid_level=subgrid_level_landunit) - write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', & - get_global_index(subgrid_index=igrc, subgrid_level=subgrid_level_gridcell) + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global patch index = ', gpft + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global column index = ', gcol + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global landunit index = ', glun + write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': global gridcell index = ', ggrc write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell longitude = ', grc%londeg(igrc) write(iulog,'(a, i0, a, f12.7)') 'iam = ', iam, ': gridcell latitude = ', grc%latdeg(igrc) write(iulog,'(a, i0, a, i0)') 'iam = ', iam, ': pft type = ', patch%itype(ipft) @@ -184,6 +235,7 @@ subroutine write_point_context(subgrid_index, subgrid_level) write(iulog,*) 'subgrid_level not supported: ', subgrid_level write(iulog,*) errmsg(sourcefile, __LINE__) write(iulog,*) 'Continuing the endrun without writing point context information' + return end if end subroutine write_point_context From 33907d78618bfb9dd6465172008e7730f3ccbe57 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 18:49:34 -0600 Subject: [PATCH 173/196] Add more error checking if a bad point is input to get_global_index, and add an option to it to avoid the abort, which is needed when it's used from an endrun call with pt_context --- src/main/decompMod.F90 | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/main/decompMod.F90 b/src/main/decompMod.F90 index 940ba724bf..3603f12cbf 100644 --- a/src/main/decompMod.F90 +++ b/src/main/decompMod.F90 @@ -361,7 +361,7 @@ integer function get_proc_clumps() end function get_proc_clumps !----------------------------------------------------------------------- - integer function get_global_index(subgrid_index, subgrid_level) + integer function get_global_index(subgrid_index, subgrid_level, donot_abort_on_badindex) !---------------------------------------------------------------- ! Description @@ -373,23 +373,47 @@ integer function get_global_index(subgrid_index, subgrid_level) ! Arguments integer , intent(in) :: subgrid_index ! index of interest (can be at any subgrid level or gridcell level) integer , intent(in) :: subgrid_level ! one of the subgrid_level_* constants defined above + logical , intent(in), optional :: donot_abort_on_badindex ! Don't abort if given a bad index ! ! Local Variables: type(bounds_type) :: bounds_proc ! processor bounds integer :: beg_index ! beginning proc index for subgrid_level + integer :: end_index ! ending proc index for subgrid_level + integer :: index ! index of the point to get integer, pointer :: gindex(:) + logical :: abort_on_badindex = .true. !---------------------------------------------------------------- + if (present(donot_abort_on_badindex)) then + abort_on_badindex = .not. donot_abort_on_badindex + end if call get_proc_bounds(bounds_proc, allow_call_from_threaded_region=.true.) beg_index = get_beg(bounds_proc, subgrid_level) + end_index = get_end(bounds_proc, subgrid_level) if (beg_index == -1) then write(iulog,*) 'get_global_index: subgrid_level not supported: ', subgrid_level - call shr_sys_abort('subgrid_level not supported' // & - errmsg(sourcefile, __LINE__)) + if (abort_on_badindex) then + call shr_sys_abort('subgrid_level not supported') + else + get_global_index = -1 + return + end if end if call get_subgrid_level_gindex(subgrid_level=subgrid_level, gindex=gindex) - get_global_index = gindex(subgrid_index - beg_index + 1) + index = subgrid_index - beg_index + 1 + if ( (index < beg_index) .or. (index > end_index) ) then + if (abort_on_badindex) then + write(iulog,*) 'get_global_index: subgrid_index out of bounds: ', & + 'subgrid_index = ', subgrid_index, ', beg_index = ', beg_index, & + ', end_index = ', end_index, ', subgrid_level = ', subgrid_level + call shr_sys_abort('subgrid_index out of bounds') + else + get_global_index = -1 + return + end if + end if + get_global_index = gindex(index) end function get_global_index @@ -537,7 +561,7 @@ subroutine get_subgrid_level_gindex (subgrid_level, gindex) gindex => gindex_cohort case default write(iulog,*) 'get_subgrid_level_gindex: unknown subgrid_level: ', subgrid_level - call shr_sys_abort() + call shr_sys_abort('bad subgrid_level') end select end subroutine get_subgrid_level_gindex From 7df48b940f73da1a7aba743554248b7e7edaa52b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 20:16:59 -0600 Subject: [PATCH 174/196] Allow passing file and line to endrun, output it through shr_log_errMsg if both are present --- src/main/abortutils.F90 | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/main/abortutils.F90 b/src/main/abortutils.F90 index 517b80fffa..886afccdef 100644 --- a/src/main/abortutils.F90 +++ b/src/main/abortutils.F90 @@ -10,6 +10,8 @@ module abortutils ! in conjunction with aborting the model, or at least issuing a warning. !----------------------------------------------------------------------- + use shr_log_mod , only : errMsg => shr_log_errMsg + implicit none private @@ -27,7 +29,7 @@ module abortutils contains !----------------------------------------------------------------------- - subroutine endrun_vanilla(msg, additional_msg) + subroutine endrun_vanilla(msg, additional_msg, line, file) !----------------------------------------------------------------------- ! !DESCRIPTION: @@ -40,10 +42,12 @@ subroutine endrun_vanilla(msg, additional_msg) ! Generally you want to at least provide msg. The main reason to separate msg from ! additional_msg is to supported expected-exception unit testing: you can put ! volatile stuff in additional_msg, as in: - ! call endrun(msg='Informative message', additional_msg=errmsg(__FILE__, __LINE__)) + ! call endrun(msg='Informative message', additional_msg=datetime ) ! and then just assert against msg. character(len=*), intent(in), optional :: msg ! string to be passed to shr_abort character(len=*), intent(in), optional :: additional_msg ! string to be printed, but not passed to shr_abort + integer , intent(in), optional :: line ! Line number for the endrun call + character(len=*), intent(in), optional :: file ! file for the endrun call !----------------------------------------------------------------------- if (present (additional_msg)) then @@ -52,12 +56,16 @@ subroutine endrun_vanilla(msg, additional_msg) write(iulog,*)'ENDRUN:' end if - call shr_abort_abort(msg) + ! Don't pass file and line to shr_abort_abort since the PFUNIT test version doesn't have those options + if ( present(file) .and. present(line) ) then + write(iulog,*) errMsg(file, line) + end if + call shr_abort_abort(string=msg) end subroutine endrun_vanilla !----------------------------------------------------------------------- - subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additional_msg) + subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additional_msg, line, file) !----------------------------------------------------------------------- ! Description: @@ -71,12 +79,8 @@ subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additio ! Arguments: integer , intent(in) :: subgrid_index ! index of interest (can be at any subgrid level or gridcell level) integer , intent(in) :: subgrid_level ! one of the subgrid_level_* constants defined in decompMod; subgrid_level_unspecified is allowed here, in which case the additional information will not be printed - - ! Generally you want to at least provide msg. The main reason to separate msg from - ! additional_msg is to supported expected-exception unit testing: you can put - ! volatile stuff in additional_msg, as in: - ! call endrun(msg='Informative message', additional_msg=errmsg(__FILE__, __LINE__)) - ! and then just assert against msg. + integer , intent(in), optional :: line ! Line number for the endrun call + character(len=*), intent(in), optional :: file !file for the endrun call character(len=*), intent(in), optional :: msg ! string to be passed to shr_abort character(len=*), intent(in), optional :: additional_msg ! string to be printed, but not passed to shr_abort ! @@ -88,7 +92,7 @@ subroutine endrun_write_point_context(subgrid_index, subgrid_level, msg, additio call write_point_context(subgrid_index, subgrid_level) end if - call endrun_vanilla(msg=msg, additional_msg=additional_msg) + call endrun_vanilla(msg=msg, additional_msg=additional_msg, line=line, file=file) end subroutine endrun_write_point_context @@ -102,7 +106,6 @@ subroutine write_point_context(subgrid_index, subgrid_level) ! ! NOTE: DO NOT CALL AN ABORT FROM HERE AS THAT WOULD SHORT CIRUIT THE ERROR REPORTING!! ! - use shr_log_mod , only : errMsg => shr_log_errMsg use clm_varctl , only : iulog use decompMod , only : subgrid_level_gridcell, subgrid_level_landunit, subgrid_level_column, subgrid_level_patch use decompMod , only : get_global_index @@ -169,6 +172,9 @@ subroutine write_point_context(subgrid_index, subgrid_level) end if + ! + ! Badpoint should already be determined, but check again in case one of the subsequent + ! calls to get_global_index returns -1 ! If one of the global indices is -1 then this is a bad point, so flag a bad-point if ( igrc /= unset) then if ( ggrc == -1 ) bad_point = .true. @@ -186,7 +192,7 @@ subroutine write_point_context(subgrid_index, subgrid_level) if (bad_point) then write(iulog,*) 'A bad input point was given: subgrid_index = ', subgrid_index, & ', subgrid_level = ', subgrid_level - write(iulog,*) errmsg(sourcefile, __LINE__) + write(iulog,*) errMsg(sourcefile, __LINE__) write(iulog,*) 'Continuing the endrun without writing point context information' return end if @@ -233,7 +239,7 @@ subroutine write_point_context(subgrid_index, subgrid_level) else write(iulog,*) 'subgrid_level not supported: ', subgrid_level - write(iulog,*) errmsg(sourcefile, __LINE__) + write(iulog,*) errMsg(sourcefile, __LINE__) write(iulog,*) 'Continuing the endrun without writing point context information' return end if From 986fe312268f69a2a2c62320ed5f75746b2e3ad9 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 15 Aug 2025 20:35:56 -0600 Subject: [PATCH 175/196] Add some tests for sending in file and line and one with just file to show it still works even if only one is present --- src/main/test/endrun_test/test_endrun.pf | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/endrun_test/test_endrun.pf index 9b3edb7bc6..6dfec9b799 100644 --- a/src/main/test/endrun_test/test_endrun.pf +++ b/src/main/test/endrun_test/test_endrun.pf @@ -25,12 +25,16 @@ contains subroutine setUp(this) use unittestSimpleSubgridSetupsMod, only : setup_single_veg_patch + use GridcellType , only : grc class(TestAbortUtils), intent(inout) :: this ! Setup a single gridcell with one vegetated patch ! So there's only one: gridcell, landunit, column, patch ! This isn't needed for some tests, but doesn't hurt to do it call setup_single_veg_patch(pft_type=1) + ! Set lat and lon for this gridcell, so something is printed in the log + grc%londeg(1) = 255.0 + grc%latdeg(1) = 30.0 end subroutine setUp subroutine tearDown(this) @@ -55,6 +59,26 @@ contains end subroutine endrun_plain_vanilla_aborts + @Test + subroutine endrun_nomsg_file_line_vanilla_aborts(this) + ! Test vanilla operation of endrun with file and line number input + class(TestAbortUtils), intent(inout) :: this + + call endrun(line=1000, file='test_file.F90') + @assertExceptionRaised(endrun_msg('')) + + end subroutine endrun_nomsg_file_line_vanilla_aborts + + @Test + subroutine endrun_nomsg_onlyfile_vanilla_aborts(this) + ! Test vanilla operation of endrun with only file input + class(TestAbortUtils), intent(inout) :: this + + call endrun(file='test_file.F90') + @assertExceptionRaised(endrun_msg('')) + + end subroutine endrun_nomsg_onlyfile_vanilla_aborts + @Test subroutine endrun_msg_vanilla_aborts(this) ! Test vanilla operation of endrun with a message sent in From 9a6bf539fc085c5b7bb67e340e9bcc1b8a5ed940 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Sat, 16 Aug 2025 16:27:38 -0600 Subject: [PATCH 176/196] Correct data type of bad_point to logical --- src/main/abortutils.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/abortutils.F90 b/src/main/abortutils.F90 index 886afccdef..8afa4ef195 100644 --- a/src/main/abortutils.F90 +++ b/src/main/abortutils.F90 @@ -123,7 +123,7 @@ subroutine write_point_context(subgrid_index, subgrid_level) integer, parameter :: unset = -9999 ! Unset value for an index integer :: igrc=unset, ilun=unset, icol=unset, ipft=unset ! Local index for grid-cell, landunit, column, and patch integer :: ggrc=unset, glun=unset, gcol=unset, gpft=unset ! Global index for grid-cell, landunit, column, and patch - integer :: bad_point = .false. ! Flag to indicate if the point is bad (i.e., global index is -1) + logical :: bad_point = .false. ! Flag to indicate if the point is bad (i.e., global index is -1) !----------------------------------------------------------------------- if (subgrid_level == subgrid_level_gridcell) then From fc884eb987b713ec856a635e4a7ff1cb2344acaf Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 6 Aug 2025 14:41:01 -0600 Subject: [PATCH 177/196] Replace most uses of npcropmin/max with is_prognostic_crop(). --- src/biogeochem/CNAllocationMod.F90 | 8 +- src/biogeochem/CNC14DecayMod.F90 | 9 +- src/biogeochem/CNCIsoFluxMod.F90 | 4 +- src/biogeochem/CNCStateUpdate1Mod.F90 | 22 +-- src/biogeochem/CNFUNMod.F90 | 6 +- src/biogeochem/CNFireEmissionsMod.F90 | 4 +- src/biogeochem/CNGRespMod.F90 | 4 +- src/biogeochem/CNGapMortalityMod.F90 | 4 +- src/biogeochem/CNMRespMod.F90 | 4 +- src/biogeochem/CNNStateUpdate1Mod.F90 | 10 +- src/biogeochem/CNPhenologyMod.F90 | 8 +- src/biogeochem/CNVegCarbonFluxType.F90 | 16 +-- src/biogeochem/CNVegCarbonStateType.F90 | 6 +- src/biogeochem/CNVegMatrixMod.F90 | 128 +++++++++--------- src/biogeochem/CNVegNitrogenStateType.F90 | 4 +- src/biogeochem/CNVegStructUpdateMod.F90 | 4 +- src/biogeochem/CropType.F90 | 4 +- src/biogeochem/DryDepVelocity.F90 | 4 +- .../NutrientCompetitionCLM45defaultMod.F90 | 12 +- .../NutrientCompetitionFlexibleCNMod.F90 | 32 ++--- src/biogeophys/PhotosynthesisMod.F90 | 4 +- src/biogeophys/TemperatureType.F90 | 4 +- src/cpl/share_esmf/cropcalStreamMod.F90 | 14 +- src/main/filterMod.F90 | 4 +- src/main/pftconMod.F90 | 32 ++++- src/soilbiogeochem/TillageMod.F90 | 4 +- 26 files changed, 186 insertions(+), 169 deletions(-) diff --git a/src/biogeochem/CNAllocationMod.F90 b/src/biogeochem/CNAllocationMod.F90 index 254e951cfe..78dfadfaee 100644 --- a/src/biogeochem/CNAllocationMod.F90 +++ b/src/biogeochem/CNAllocationMod.F90 @@ -15,7 +15,7 @@ module CNAllocationMod use clm_varcon , only : secspday use clm_varctl , only : use_c13, use_c14, iulog use PatchType , only : patch - use pftconMod , only : pftcon, npcropmin + use pftconMod , only : pftcon, is_prognostic_crop use CropType , only : crop_type use CropType , only : cphase_planted, cphase_leafemerge, cphase_grainfill use PhotosynthesisMod , only : photosyns_type @@ -191,7 +191,7 @@ subroutine calc_gpp_mr_availc(bounds, num_soilp, filter_soilp, & mr = leaf_mr(p) + froot_mr(p) if (woody(ivt(p)) == 1.0_r8) then mr = mr + livestem_mr(p) + livecroot_mr(p) - else if (ivt(p) >= npcropmin) then + else if (is_prognostic_crop(ivt(p))) then if (croplive(p)) then reproductive_mr_tot = 0._r8 do k = 1, nrepr @@ -500,7 +500,7 @@ subroutine calc_allometry(num_soilp, filter_soilp, & end if f4 = flivewd(ivt(p)) - if (ivt(p) >= npcropmin) then + if (is_prognostic_crop(ivt(p))) then g1 = 0.25_r8 else g1 = grperc(ivt(p)) @@ -521,7 +521,7 @@ subroutine calc_allometry(num_soilp, filter_soilp, & c_allometry(p) = (1._r8+g1a)*(1._r8+f1+f3*(1._r8+f2)) n_allometry(p) = 1._r8/cnl + f1/cnfr + (f3*f4*(1._r8+f2))/cnlw + & (f3*(1._r8-f4)*(1._r8+f2))/cndw - else if (ivt(p) >= npcropmin) then ! skip generic crops + else if (is_prognostic_crop(ivt(p))) then ! skip generic crops cng = graincn(ivt(p)) f1 = aroot(p) / aleaf(p) f3 = astem(p) / aleaf(p) diff --git a/src/biogeochem/CNC14DecayMod.F90 b/src/biogeochem/CNC14DecayMod.F90 index 1679c602e4..8caabfc41d 100644 --- a/src/biogeochem/CNC14DecayMod.F90 +++ b/src/biogeochem/CNC14DecayMod.F90 @@ -11,7 +11,7 @@ module CNC14DecayMod use clm_varctl , only : spinup_state use CNSharedParamsMod , only : use_matrixcn use decompMod , only : bounds_type - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use CNVegCarbonStateType , only : cnveg_carbonstate_type use CNVegCarbonFluxType , only : cnveg_carbonflux_type use SoilBiogeochemDecompCascadeConType , only : decomp_cascade_con, use_soil_matrixcn @@ -205,12 +205,7 @@ subroutine C14Decay( bounds, num_soilc, filter_soilc, num_soilp, filter_soilp, & gresp_xfer(p) = gresp_xfer(p) * (1._r8 - decay_const * dt) pft_ctrunc(p) = pft_ctrunc(p) * (1._r8 - decay_const * dt) - ! NOTE(wjs, 2017-02-02) This isn't a completely robust way to check if this is a - ! prognostic crop patch (at the very least it should also check if <= npcropmax; - ! ideally it should use a prognostic_crop flag that doesn't seem to exist - ! currently). But I'm just being consistent with what's done elsewhere (e.g., in - ! CStateUpdate1). - if (patch%itype(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(patch%itype(p))) then ! skip 2 generic crops cropseedc_deficit(p) = cropseedc_deficit(p) * (1._r8 - decay_const * dt) end if end do diff --git a/src/biogeochem/CNCIsoFluxMod.F90 b/src/biogeochem/CNCIsoFluxMod.F90 index fbe4cd927f..561e9a58ca 100644 --- a/src/biogeochem/CNCIsoFluxMod.F90 +++ b/src/biogeochem/CNCIsoFluxMod.F90 @@ -1357,7 +1357,7 @@ subroutine CNCIsoLitterToColumn (num_soilp, filter_soilp, & ! ! !USES: !DML - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use clm_varctl , only : use_grainproduct !DML @@ -1404,7 +1404,7 @@ subroutine CNCIsoLitterToColumn (num_soilp, filter_soilp, & end do !DML - if (ivt(p) >= npcropmin) then ! add livestemc to litter + if (is_prognostic_crop(ivt(p))) then ! add livestemc to litter ! stem litter carbon fluxes do i = i_litr_min, i_litr_max phenology_c_to_litr_c(c,j,i) = & diff --git a/src/biogeochem/CNCStateUpdate1Mod.F90 b/src/biogeochem/CNCStateUpdate1Mod.F90 index 70f0b86a53..4f5876bc60 100644 --- a/src/biogeochem/CNCStateUpdate1Mod.F90 +++ b/src/biogeochem/CNCStateUpdate1Mod.F90 @@ -10,7 +10,7 @@ module CNCStateUpdate1Mod use clm_time_manager , only : get_step_size_real use clm_varpar , only : i_litr_min, i_litr_max, i_cwd use clm_varpar , only : i_met_lit, i_str_lit, i_phys_som, i_chem_som - use pftconMod , only : npcropmin, nc3crop, pftcon + use pftconMod , only : is_prognostic_crop, nc3crop, pftcon use abortutils , only : endrun use decompMod , only : bounds_type use CNVegCarbonStateType , only : cnveg_carbonstate_type @@ -290,7 +290,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & cs_veg%deadcrootc_patch(p) = cs_veg%deadcrootc_patch(p) + cf_veg%deadcrootc_xfer_to_deadcrootc_patch(p)*dt cs_veg%deadcrootc_xfer_patch(p) = cs_veg%deadcrootc_xfer_patch(p) - cf_veg%deadcrootc_xfer_to_deadcrootc_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero cs_veg%livestemc_patch(p) = cs_veg%livestemc_patch(p) + cf_veg%livestemc_xfer_to_livestemc_patch(p)*dt cs_veg%livestemc_xfer_patch(p) = cs_veg%livestemc_xfer_patch(p) - cf_veg%livestemc_xfer_to_livestemc_patch(p)*dt @@ -313,7 +313,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & cs_veg%livecrootc_patch(p) = cs_veg%livecrootc_patch(p) - cf_veg%livecrootc_to_deadcrootc_patch(p)*dt cs_veg%deadcrootc_patch(p) = cs_veg%deadcrootc_patch(p) + cf_veg%livecrootc_to_deadcrootc_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cs_veg%livestemc_patch(p) = cs_veg%livestemc_patch(p) - cf_veg%livestemc_to_litter_patch(p)*dt cs_veg%livestemc_patch(p) = cs_veg%livestemc_patch(p) - & (cf_veg%livestemc_to_biofuelc_patch(p) + cf_veg%livestemc_to_removedresiduec_patch(p))*dt @@ -337,7 +337,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & ! This part below MUST match exactly the code for the non-matrix part ! above! - if (ivt(p) >= npcropmin) then + if (is_prognostic_crop(ivt(p))) then cs_veg%cropseedc_deficit_patch(p) = cs_veg%cropseedc_deficit_patch(p) & - cf_veg%crop_seedc_to_leaf_patch(p) * dt do k = repr_grain_min, repr_grain_max @@ -359,7 +359,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%livestem_curmr_patch(p)*dt cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%livecroot_curmr_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%livestem_curmr_patch(p)*dt do k = 1, nrepr cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%reproductive_curmr_patch(p,k)*dt @@ -432,7 +432,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & ! NOTE: The equivalent changes for matrix code are in CNPhenology EBK (11/26/2019) end if !not use_matrixcn end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops if (carbon_resp_opt == 1) then cf_veg%cpool_to_livestemc_patch(p) = cf_veg%cpool_to_livestemc_patch(p) - cf_veg%cpool_to_livestemc_resp_patch(p) cf_veg%cpool_to_livestemc_storage_patch(p) = cf_veg%cpool_to_livestemc_storage_patch(p) - & @@ -468,7 +468,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_livecroot_gr_patch(p)*dt cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_deadcroot_gr_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_livestem_gr_patch(p)*dt do k = 1, nrepr cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_reproductive_gr_patch(p,k)*dt @@ -484,7 +484,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & cs_veg%gresp_xfer_patch(p) = cs_veg%gresp_xfer_patch(p) - cf_veg%transfer_livecroot_gr_patch(p)*dt cs_veg%gresp_xfer_patch(p) = cs_veg%gresp_xfer_patch(p) - cf_veg%transfer_deadcroot_gr_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cs_veg%gresp_xfer_patch(p) = cs_veg%gresp_xfer_patch(p) - cf_veg%transfer_livestem_gr_patch(p)*dt do k = 1, nrepr cs_veg%gresp_xfer_patch(p) = cs_veg%gresp_xfer_patch(p) - cf_veg%transfer_reproductive_gr_patch(p,k)*dt @@ -501,7 +501,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_livecroot_storage_gr_patch(p)*dt cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_deadcroot_storage_gr_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cs_veg%cpool_patch(p) = cs_veg%cpool_patch(p) - cf_veg%cpool_livestem_storage_gr_patch(p)*dt do k = 1, nrepr @@ -539,7 +539,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & ! NOTE: The equivalent changes for matrix code are in CNPhenology EBK (11/26/2019) end if !not use_matrixcn end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero if(.not. use_matrixcn)then ! lines here for consistency; the transfer terms are zero @@ -556,7 +556,7 @@ subroutine CStateUpdate1( num_soilc, filter_soilc, num_soilp, filter_soilp, & end if !not use_matrixcn end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cs_veg%xsmrpool_patch(p) = cs_veg%xsmrpool_patch(p) - cf_veg%livestem_xsmr_patch(p)*dt do k = 1, nrepr cs_veg%xsmrpool_patch(p) = cs_veg%xsmrpool_patch(p) - cf_veg%reproductive_xsmr_patch(p,k)*dt diff --git a/src/biogeochem/CNFUNMod.F90 b/src/biogeochem/CNFUNMod.F90 index cdbe7ba71b..e264786c0b 100644 --- a/src/biogeochem/CNFUNMod.F90 +++ b/src/biogeochem/CNFUNMod.F90 @@ -23,7 +23,7 @@ module CNFUNMod use clm_varctl , only : iulog use PatchType , only : patch use ColumnType , only : col - use pftconMod , only : pftcon, npcropmin + use pftconMod , only : pftcon use decompMod , only : bounds_type use clm_varctl , only : use_nitrif_denitrif,use_flexiblecn use CNSharedParamsMod , only : use_matrixcn @@ -284,7 +284,7 @@ subroutine CNFUN(bounds,num_soilc, filter_soilc,num_soilp& use clm_varctl , only : use_nitrif_denitrif use PatchType , only : patch use subgridAveMod , only : p2c - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use CNVegMatrixMod , only : matrix_update_phn ! ! !ARGUMENTS: @@ -1271,7 +1271,7 @@ subroutine CNFUN(bounds,num_soilc, filter_soilc,num_soilp& ! Calculate appropriate degree of retranslocation !------------------------------------------------------------------------------- - if(leafc(p).gt.0.0_r8.and.litterfall_n_step(p,istp)* fixerfrac>0.0_r8.and.ivt(p) 0.0_r8.and. (.not. is_prognostic_crop(ivt(p))))then call fun_retranslocation(p,dt,npp_to_spend,& litterfall_c_step(p,istp)* fixerfrac,& litterfall_n_step(p,istp)* fixerfrac,& diff --git a/src/biogeochem/CNFireEmissionsMod.F90 b/src/biogeochem/CNFireEmissionsMod.F90 index 5a15e138d5..1ee8facfd4 100644 --- a/src/biogeochem/CNFireEmissionsMod.F90 +++ b/src/biogeochem/CNFireEmissionsMod.F90 @@ -347,7 +347,7 @@ function vert_dist_top( veg_type ) result(ztop) use pftconMod , only : nbrdlf_evr_shrub, nbrdlf_dcd_brl_shrub use pftconMod , only : nc3_arctic_grass, nc3_nonarctic_grass use pftconMod , only : nc3crop, nc3irrig - use pftconMod , only : npcropmin, npcropmax + use pftconMod , only : is_prognostic_crop implicit none integer, intent(in) :: veg_type @@ -376,7 +376,7 @@ function vert_dist_top( veg_type ) result(ztop) else if ( veg_type == nc3crop .or. veg_type <= nc3irrig ) then ztop = 1.e3_r8 ! m ! Prognostic crops - else if ( veg_type >= npcropmin .and. veg_type <= npcropmax ) then + else if (is_prognostic_crop(veg_type)) then ztop = 1.e3_r8 ! m else call endrun('ERROR:: undefined veg_type' ) diff --git a/src/biogeochem/CNGRespMod.F90 b/src/biogeochem/CNGRespMod.F90 index de8b145615..29d25487e4 100644 --- a/src/biogeochem/CNGRespMod.F90 +++ b/src/biogeochem/CNGRespMod.F90 @@ -7,7 +7,7 @@ module CNGRespMod ! ! !USES: use shr_kind_mod , only : r8 => shr_kind_r8 - use pftconMod , only : npcropmin, pftcon + use pftconMod , only : is_prognostic_crop, pftcon use CNVegcarbonfluxType , only : cnveg_carbonflux_type use PatchType , only : patch use CanopyStateType , only : canopystate_type @@ -145,7 +145,7 @@ subroutine CNGResp(num_soilp, filter_soilp, cnveg_carbonflux_inst, canopystate_i respfact_livecroot_storage = 1.0_r8 respfact_livestem_storage = 1.0_r8 - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cpool_livestem_gr(p) = cpool_to_livestemc(p) * grperc(ivt(p)) * respfact_livestem cpool_livestem_storage_gr(p) = cpool_to_livestemc_storage(p) * grperc(ivt(p)) * grpnow(ivt(p)) * & diff --git a/src/biogeochem/CNGapMortalityMod.F90 b/src/biogeochem/CNGapMortalityMod.F90 index 24f0f6b145..6b9859265b 100644 --- a/src/biogeochem/CNGapMortalityMod.F90 +++ b/src/biogeochem/CNGapMortalityMod.F90 @@ -121,7 +121,7 @@ subroutine CNGapMortality (bounds, num_soilp, filter_soilp, & use clm_varpar , only: nlevdecomp_full use clm_varcon , only: secspday use clm_varctl , only: use_cndv, spinup_state - use pftconMod , only: npcropmin + use pftconMod , only: is_prognostic_crop ! ! !ARGUMENTS: type(bounds_type) , intent(in) :: bounds @@ -348,7 +348,7 @@ subroutine CNGapMortality (bounds, num_soilp, filter_soilp, & end if !use_matrixcn end if - if (ivt(p) < npcropmin) then + if (.not. is_prognostic_crop(ivt(p))) then if(.not. use_matrixcn)then cnveg_nitrogenflux_inst%m_retransn_to_litter_patch(p) = cnveg_nitrogenstate_inst%retransn_patch(p) * m else diff --git a/src/biogeochem/CNMRespMod.F90 b/src/biogeochem/CNMRespMod.F90 index 33eb0b0e9a..ae0040008b 100644 --- a/src/biogeochem/CNMRespMod.F90 +++ b/src/biogeochem/CNMRespMod.F90 @@ -13,7 +13,7 @@ module CNMRespMod use decompMod , only : bounds_type use abortutils , only : endrun use shr_log_mod , only : errMsg => shr_log_errMsg - use pftconMod , only : npcropmin, pftcon + use pftconMod , only : is_prognostic_crop, pftcon use SoilStateType , only : soilstate_type use CanopyStateType , only : canopystate_type use TemperatureType , only : temperature_type @@ -265,7 +265,7 @@ subroutine CNMResp(bounds, num_soilc, filter_soilc, num_soilp, filter_soilp, & if (woody(ivt(p)) == 1) then livestem_mr(p) = livestemn(p)*br*tc livecroot_mr(p) = livecrootn(p)*br_root*tc - else if (ivt(p) >= npcropmin) then + else if (is_prognostic_crop(ivt(p))) then livestem_mr(p) = livestemn(p)*br*tc do k = 1, nrepr reproductive_mr(p,k) = reproductiven(p,k)*br*tc diff --git a/src/biogeochem/CNNStateUpdate1Mod.F90 b/src/biogeochem/CNNStateUpdate1Mod.F90 index 742afa77dd..7bd346c82a 100644 --- a/src/biogeochem/CNNStateUpdate1Mod.F90 +++ b/src/biogeochem/CNNStateUpdate1Mod.F90 @@ -17,7 +17,7 @@ module CNNStateUpdate1Mod use SoilBiogeochemDecompCascadeConType, only : decomp_method, mimics_decomp, use_soil_matrixcn use CNSharedParamsMod , only : use_matrixcn use clm_varcon , only : nitrif_n2o_loss_frac - use pftconMod , only : npcropmin, pftcon + use pftconMod , only : is_prognostic_crop, pftcon use decompMod , only : bounds_type use CNVegNitrogenStateType , only : cnveg_nitrogenstate_type use CNVegNitrogenFluxType , only : cnveg_nitrogenflux_type @@ -228,7 +228,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & ns_veg%deadcrootn_xfer_patch(p) = ns_veg%deadcrootn_xfer_patch(p) - nf_veg%deadcrootn_xfer_to_deadcrootn_patch(p)*dt end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero ns_veg%livestemn_patch(p) = ns_veg%livestemn_patch(p) + nf_veg%livestemn_xfer_to_livestemn_patch(p)*dt ns_veg%livestemn_xfer_patch(p) = ns_veg%livestemn_xfer_patch(p) - nf_veg%livestemn_xfer_to_livestemn_patch(p)*dt @@ -287,7 +287,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & ! NOTE: The equivalent changes for matrix code are in CNPhenology EBK (11/26/2019) end if !not use_matrixcn end if - if (ivt(p) >= npcropmin) then ! Beth adds retrans from froot + if (is_prognostic_crop(ivt(p))) then ! Beth adds retrans from froot ! ! State update without the matrix solution ! @@ -391,7 +391,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & end if ! not use_matrixcn end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops ns_veg%npool_patch(p) = ns_veg%npool_patch(p) - nf_veg%npool_to_livestemn_patch(p)*dt ns_veg%npool_patch(p) = ns_veg%npool_patch(p) - nf_veg%npool_to_livestemn_storage_patch(p)*dt do k = 1, nrepr @@ -452,7 +452,7 @@ subroutine NStateUpdate1(num_soilc, filter_soilc, num_soilp, filter_soilp, & ! NOTE: The equivalent changes for matrix code are in CNPhenology EBK (11/26/2019) end if ! not use_matrixcn - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops ! lines here for consistency; the transfer terms are zero ! diff --git a/src/biogeochem/CNPhenologyMod.F90 b/src/biogeochem/CNPhenologyMod.F90 index 5e347e1e9f..a907a689c9 100644 --- a/src/biogeochem/CNPhenologyMod.F90 +++ b/src/biogeochem/CNPhenologyMod.F90 @@ -3354,7 +3354,7 @@ subroutine CNOffsetLitterfall (num_soilp, filter_soilp, & ! pools during the phenological offset period. ! ! !USES: - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use pftconMod , only : nmiscanthus, nirrig_miscanthus, nswitchgrass, nirrig_switchgrass use CNSharedParamsMod, only : use_fun @@ -3530,7 +3530,7 @@ subroutine CNOffsetLitterfall (num_soilp, filter_soilp, & end if ! use_matrixcn ! this assumes that offset_counter == dt for crops ! if this were ever changed, we'd need to add code to the "else" - if (ivt(p) >= npcropmin) then + if (is_prognostic_crop(ivt(p))) then ! How many harvests have occurred? h = crop_inst%harvest_count(p) @@ -4375,7 +4375,7 @@ subroutine CNLitterToColumn (bounds, num_bgc_vegp, filter_bgc_vegp, & ! ! !USES: use clm_varpar , only : nlevdecomp - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use clm_varctl , only : use_grainproduct ! ! !ARGUMENTS: @@ -4453,7 +4453,7 @@ subroutine CNLitterToColumn (bounds, num_bgc_vegp, filter_bgc_vegp, & ! new ones for now (slevis) ! also for simplicity I've put "food" into the litter pools - if (ivt(p) >= npcropmin) then ! add livestemc to litter + if (is_prognostic_crop(ivt(p))) then ! add livestemc to litter do i = i_litr_min, i_litr_max ! stem litter carbon fluxes phenology_c_to_litr_c(c,j,i) = & diff --git a/src/biogeochem/CNVegCarbonFluxType.F90 b/src/biogeochem/CNVegCarbonFluxType.F90 index b4c581c081..12daf746af 100644 --- a/src/biogeochem/CNVegCarbonFluxType.F90 +++ b/src/biogeochem/CNVegCarbonFluxType.F90 @@ -27,7 +27,7 @@ module CNVegCarbonFluxType use clm_varctl , only : use_grainproduct use clm_varctl , only : iulog use landunit_varcon , only : istsoil, istcrop, istdlak - use pftconMod , only : npcropmin, pftcon + use pftconMod , only : is_prognostic_crop, pftcon use CropReprPoolsMod , only : nrepr, repr_grain_min, repr_grain_max, repr_structure_min, repr_structure_max use CropReprPoolsMod , only : get_repr_hist_fname, get_repr_rest_fname, get_repr_longname use LandunitType , only : lun @@ -4921,7 +4921,7 @@ subroutine Summary_carbonflux(this, & this%livestem_mr_patch(p) + & this%livecroot_mr_patch(p) end if - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%mr_patch(p) = & this%mr_patch(p) + & @@ -4939,7 +4939,7 @@ subroutine Summary_carbonflux(this, & this%cpool_deadstem_gr_patch(p) + & this%cpool_livecroot_gr_patch(p) + & this%cpool_deadcroot_gr_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%current_gr_patch(p) = this%current_gr_patch(p) + & this%cpool_reproductive_gr_patch(p,k) @@ -4955,7 +4955,7 @@ subroutine Summary_carbonflux(this, & this%transfer_deadstem_gr_patch(p) + & this%transfer_livecroot_gr_patch(p) + & this%transfer_deadcroot_gr_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%transfer_gr_patch(p) = this%transfer_gr_patch(p) + & this%transfer_reproductive_gr_patch(p,k) @@ -4971,7 +4971,7 @@ subroutine Summary_carbonflux(this, & this%cpool_livecroot_storage_gr_patch(p) + & this%cpool_deadcroot_storage_gr_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%storage_gr_patch(p) = this%storage_gr_patch(p) + & this%cpool_reproductive_storage_gr_patch(p,k) @@ -4985,7 +4985,7 @@ subroutine Summary_carbonflux(this, & this%storage_gr_patch(p) ! autotrophic respiration (AR) adn - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then this%ar_patch(p) = & this%mr_patch(p) + & this%gr_patch(p) @@ -5045,7 +5045,7 @@ subroutine Summary_carbonflux(this, & this%cpool_to_deadstemc_patch(p) + & this%deadstemc_xfer_to_deadstemc_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%agnpp_patch(p) = & this%agnpp_patch(p) + & @@ -5139,7 +5139,7 @@ subroutine Summary_carbonflux(this, & this%gru_livecrootc_to_litter_patch(p) + & this%gru_deadcrootc_to_litter_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then this%litfall_patch(p) = & this%litfall_patch(p) + & this%livestemc_to_litter_patch(p) diff --git a/src/biogeochem/CNVegCarbonStateType.F90 b/src/biogeochem/CNVegCarbonStateType.F90 index 142578e656..3604821057 100644 --- a/src/biogeochem/CNVegCarbonStateType.F90 +++ b/src/biogeochem/CNVegCarbonStateType.F90 @@ -9,7 +9,7 @@ module CNVegCarbonStateType use shr_infnan_mod , only : nan => shr_infnan_nan, assignment(=) use shr_const_mod , only : SHR_CONST_PDB use shr_log_mod , only : errMsg => shr_log_errMsg - use pftconMod , only : noveg, npcropmin, pftcon, nc3crop, nc3irrig + use pftconMod , only : noveg, is_prognostic_crop, pftcon, nc3crop, nc3irrig use clm_varcon , only : spval, c3_r2, c4_r2, c14ratio use clm_varctl , only : iulog, use_cndv, use_crop use CNSharedParamsMod, only : use_matrixcn @@ -1532,7 +1532,7 @@ subroutine InitCold(this, bounds, ratio, carbon_type, c12_cnveg_carbonstate_inst this%matrix_cap_frootc_patch(p) = cnvegcstate_const%initial_vegC * ratio this%matrix_cap_frootc_storage_patch(p) = 0._r8 end if - else if (patch%itype(p) >= npcropmin) then ! prognostic crop types + else if (is_prognostic_crop(patch%itype(p))) then ! prognostic crop types this%leafc_patch(p) = 0._r8 this%leafc_storage_patch(p) = 0._r8 this%frootc_patch(p) = 0._r8 @@ -4578,7 +4578,7 @@ subroutine Summary_carbonstate(this, bounds, num_bgc_soilc, filter_bgc_soilc, nu this%gresp_storage_patch(p) + & this%gresp_xfer_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%storvegc_patch(p) = & this%storvegc_patch(p) + & diff --git a/src/biogeochem/CNVegMatrixMod.F90 b/src/biogeochem/CNVegMatrixMod.F90 index 5582afeffe..41339d3a2c 100644 --- a/src/biogeochem/CNVegMatrixMod.F90 +++ b/src/biogeochem/CNVegMatrixMod.F90 @@ -35,7 +35,7 @@ module CNVegMatrixMod ncphouttrans,nnphouttrans,ncgmouttrans,nngmouttrans,ncfiouttrans,nnfiouttrans use perf_mod , only : t_startf, t_stopf use PatchType , only : patch - use pftconMod , only : pftcon,npcropmin + use pftconMod , only : pftcon,is_prognostic_crop use CNVegCarbonStateType , only : cnveg_carbonstate_type use CNVegNitrogenStateType , only : cnveg_nitrogenstate_type use CNVegCarbonFluxType , only : cnveg_carbonflux_type !include: callocation,ctransfer, cturnover @@ -1140,7 +1140,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on Xvegc%V(p,igrain) = reproductivec(p,irepr) Xvegc%V(p,igrain_st) = reproductivec_storage(p,irepr) @@ -1173,7 +1173,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on Xveg13c%V(p,igrain) = cs13_veg%reproductivec_patch(p,irepr) Xveg13c%V(p,igrain_st) = cs13_veg%reproductivec_storage_patch(p,irepr) @@ -1207,7 +1207,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on Xveg14c%V(p,igrain) = cs14_veg%reproductivec_patch(p,irepr) Xveg14c%V(p,igrain_st) = cs14_veg%reproductivec_storage_patch(p,irepr) @@ -1241,7 +1241,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then Xvegn%V(p,igrain) = sum(reproductiven(p,:)) Xvegn%V(p,igrain_st) = sum(reproductiven_storage(p,:)) Xvegn%V(p,igrain_xf) = sum(reproductiven_xfer(p,:)) @@ -1279,7 +1279,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on reproc0(p) = max(reproductivec(p,irepr), epsi) reproc0_storage(p) = max(reproductivec_storage(p,irepr), epsi) @@ -1312,7 +1312,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on cs13_veg%reproc0_patch(p) = max(cs13_veg%reproductivec_patch(p,irepr), epsi) cs13_veg%reproc0_storage_patch(p) = max(cs13_veg%reproductivec_storage_patch(p,irepr), epsi) @@ -1346,7 +1346,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on cs14_veg%reproc0_patch(p) = max(cs14_veg%reproductivec_patch(p,irepr), epsi) cs14_veg%reproc0_storage_patch(p) = max(cs14_veg%reproductivec_storage_patch(p,irepr), epsi) @@ -1380,7 +1380,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! Use one index of the grain reproductive pools to operate on repron0(p) = max(reproductiven(p,irepr), epsi) repron0_storage(p) = max(reproductiven_storage(p,irepr), epsi) @@ -1730,7 +1730,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire end do do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_calloc_grain_acc(p) = matrix_calloc_grain_acc(p) + vegmatrixc_input%V(p,igrain) matrix_calloc_grainst_acc(p) = matrix_calloc_grainst_acc(p) + vegmatrixc_input%V(p,igrain_st) if(use_c13)then @@ -2052,7 +2052,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire end do do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_cturnover_grain_acc(p) = matrix_cturnover_grain_acc(p) & + (matrix_phturnover(p,igrain)+matrix_gmturnover(p,igrain)+matrix_fiturnover(p,igrain)) & * reproductivec(p,irepr) @@ -2110,7 +2110,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_nalloc_grain_acc(p) = matrix_nalloc_grain_acc(p) + vegmatrixn_input%V(p,igrain) matrix_nalloc_grainst_acc(p) = matrix_nalloc_grainst_acc(p) + vegmatrixn_input%V(p,igrain_st) end if @@ -2215,7 +2215,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_retransn_to_grain_acc(p) = matrix_ntransfer_retransn_to_grain_acc(p) & + matrix_nphtransfer(p,iretransn_to_igrain_phn) & * dt * retransn(p)!matrix_nphturnover(p,iretransn)*retransn(p) @@ -2287,7 +2287,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire end do do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_nturnover_grain_acc(p) = matrix_nturnover_grain_acc(p) & + (matrix_nphturnover(p,igrain)+matrix_ngmturnover(p,igrain)+matrix_nfiturnover(p,igrain)) & * reproductiven(p,irepr) @@ -2328,7 +2328,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! NOTE: This assumes only a single grain pool! (i.e nrepr is ! fixed at 1)! reproductivec(p,:) = Xvegc%V(p,igrain) @@ -2362,7 +2362,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! NOTE: This assumes only a single grain pool! (i.e nrepr is ! fixed at 1)! cs13_veg%reproductivec_patch(p,:) = Xveg13c%V(p,igrain) @@ -2396,7 +2396,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire end do do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then ! NOTE: This assumes only a single grain pool! (i.e nrepr is ! fixed at 1)! cs14_veg%reproductivec_patch(p,:) = Xveg14c%V(p,igrain) @@ -2431,7 +2431,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire do fp = 1,num_soilp p = filter_soilp(fp) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then reproductiven(p,:) = Xvegn%V(p,igrain) reproductiven_storage(p,:) = Xvegn%V(p,igrain_st) reproductiven_xfer(p,:) = Xvegn%V(p,igrain_xf) @@ -2457,7 +2457,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_calloc_acc(ilivecroot_st) = matrix_calloc_livecrootst_acc(p) matrix_calloc_acc(ideadcroot) = matrix_calloc_deadcroot_acc(p) matrix_calloc_acc(ideadcroot_st) = matrix_calloc_deadcrootst_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_calloc_acc(igrain) = matrix_calloc_grain_acc(p) matrix_calloc_acc(igrain_st) = matrix_calloc_grainst_acc(p) end if @@ -2474,7 +2474,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ctransfer_acc(ilivecroot,ilivecroot_xf) = matrix_ctransfer_livecrootxf_to_livecroot_acc(p) matrix_ctransfer_acc(ideadcroot_xf,ideadcroot_st) = matrix_ctransfer_deadcrootst_to_deadcrootxf_acc(p) matrix_ctransfer_acc(ideadcroot,ideadcroot_xf) = matrix_ctransfer_deadcrootxf_to_deadcroot_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ctransfer_acc(igrain_xf,igrain_st) = matrix_ctransfer_grainst_to_grainxf_acc(p) matrix_ctransfer_acc(igrain,igrain_xf) = matrix_ctransfer_grainxf_to_grain_acc(p) end if @@ -2499,7 +2499,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ctransfer_acc(ideadcroot,ideadcroot) = -matrix_cturnover_deadcroot_acc(p) matrix_ctransfer_acc(ideadcroot_st,ideadcroot_st) = -matrix_cturnover_deadcrootst_acc(p) matrix_ctransfer_acc(ideadcroot_xf,ideadcroot_xf) = -matrix_cturnover_deadcrootxf_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ctransfer_acc(igrain,igrain) = -matrix_cturnover_grain_acc(p) matrix_ctransfer_acc(igrain_st,igrain_st) = -matrix_cturnover_grainst_acc(p) matrix_ctransfer_acc(igrain_xf,igrain_xf) = -matrix_cturnover_grainxf_acc(p) @@ -2518,7 +2518,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c13alloc_acc(ilivecroot_st) = cs13_veg%matrix_calloc_livecrootst_acc_patch(p) matrix_c13alloc_acc(ideadcroot) = cs13_veg%matrix_calloc_deadcroot_acc_patch(p) matrix_c13alloc_acc(ideadcroot_st) = cs13_veg%matrix_calloc_deadcrootst_acc_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c13alloc_acc(igrain) = cs13_veg%matrix_calloc_grain_acc_patch(p) matrix_c13alloc_acc(igrain_st) = cs13_veg%matrix_calloc_grainst_acc_patch(p) end if @@ -2535,7 +2535,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c13transfer_acc(ilivecroot,ilivecroot_xf) = cs13_veg%matrix_ctransfer_livecrootxf_to_livecroot_acc_patch(p) matrix_c13transfer_acc(ideadcroot_xf,ideadcroot_st) = cs13_veg%matrix_ctransfer_deadcrootst_to_deadcrootxf_acc_patch(p) matrix_c13transfer_acc(ideadcroot,ideadcroot_xf) = cs13_veg%matrix_ctransfer_deadcrootxf_to_deadcroot_acc_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c13transfer_acc(igrain_xf,igrain_st) = cs13_veg%matrix_ctransfer_grainst_to_grainxf_acc_patch(p) matrix_c13transfer_acc(igrain,igrain_xf) = cs13_veg%matrix_ctransfer_grainxf_to_grain_acc_patch(p) end if @@ -2560,7 +2560,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c13transfer_acc(ideadcroot,ideadcroot) = -cs13_veg%matrix_cturnover_deadcroot_acc_patch(p) matrix_c13transfer_acc(ideadcroot_st,ideadcroot_st) = -cs13_veg%matrix_cturnover_deadcrootst_acc_patch(p) matrix_c13transfer_acc(ideadcroot_xf,ideadcroot_xf) = -cs13_veg%matrix_cturnover_deadcrootxf_acc_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c13transfer_acc(igrain,igrain) = -cs13_veg%matrix_cturnover_grain_acc_patch(p) matrix_c13transfer_acc(igrain_st,igrain_st) = -cs13_veg%matrix_cturnover_grainst_acc_patch(p) matrix_c13transfer_acc(igrain_xf,igrain_xf) = -cs13_veg%matrix_cturnover_grainxf_acc_patch(p) @@ -2580,7 +2580,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c14alloc_acc(ilivecroot_st) = cs14_veg%matrix_calloc_livecrootst_acc_patch(p) matrix_c14alloc_acc(ideadcroot) = cs14_veg%matrix_calloc_deadcroot_acc_patch(p) matrix_c14alloc_acc(ideadcroot_st) = cs14_veg%matrix_calloc_deadcrootst_acc_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c14alloc_acc(igrain) = cs14_veg%matrix_calloc_grain_acc_patch(p) matrix_c14alloc_acc(igrain_st) = cs14_veg%matrix_calloc_grainst_acc_patch(p) end if @@ -2597,7 +2597,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c14transfer_acc(ilivecroot,ilivecroot_xf) = cs14_veg%matrix_ctransfer_livecrootxf_to_livecroot_acc_patch(p) matrix_c14transfer_acc(ideadcroot_xf,ideadcroot_st) = cs14_veg%matrix_ctransfer_deadcrootst_to_deadcrootxf_acc_patch(p) matrix_c14transfer_acc(ideadcroot,ideadcroot_xf) = cs14_veg%matrix_ctransfer_deadcrootxf_to_deadcroot_acc_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c14transfer_acc(igrain_xf,igrain_st) = cs14_veg%matrix_ctransfer_grainst_to_grainxf_acc_patch(p) matrix_c14transfer_acc(igrain,igrain_xf) = cs14_veg%matrix_ctransfer_grainxf_to_grain_acc_patch(p) end if @@ -2622,7 +2622,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c14transfer_acc(ideadcroot,ideadcroot) = -cs14_veg%matrix_cturnover_deadcroot_acc_patch(p) matrix_c14transfer_acc(ideadcroot_st,ideadcroot_st) = -cs14_veg%matrix_cturnover_deadcrootst_acc_patch(p) matrix_c14transfer_acc(ideadcroot_xf,ideadcroot_xf) = -cs14_veg%matrix_cturnover_deadcrootxf_acc_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c14transfer_acc(igrain,igrain) = -cs14_veg%matrix_cturnover_grain_acc_patch(p) matrix_c14transfer_acc(igrain_st,igrain_st) = -cs14_veg%matrix_cturnover_grainst_acc_patch(p) matrix_c14transfer_acc(igrain_xf,igrain_xf) = -cs14_veg%matrix_cturnover_grainxf_acc_patch(p) @@ -2641,7 +2641,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_nalloc_acc(ilivecroot_st) = matrix_nalloc_livecrootst_acc(p) matrix_nalloc_acc(ideadcroot) = matrix_nalloc_deadcroot_acc(p) matrix_nalloc_acc(ideadcroot_st) = matrix_nalloc_deadcrootst_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_nalloc_acc(igrain) = matrix_nalloc_grain_acc(p) matrix_nalloc_acc(igrain_st) = matrix_nalloc_grainst_acc(p) end if @@ -2658,7 +2658,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ntransfer_acc(ilivecroot,ilivecroot_xf) = matrix_ntransfer_livecrootxf_to_livecroot_acc(p) matrix_ntransfer_acc(ideadcroot_xf,ideadcroot_st) = matrix_ntransfer_deadcrootst_to_deadcrootxf_acc(p) matrix_ntransfer_acc(ideadcroot,ideadcroot_xf) = matrix_ntransfer_deadcrootxf_to_deadcroot_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_acc(igrain_xf,igrain_st) = matrix_ntransfer_grainst_to_grainxf_acc(p) matrix_ntransfer_acc(igrain,igrain_xf) = matrix_ntransfer_grainxf_to_grain_acc(p) end if @@ -2677,7 +2677,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ntransfer_acc(ilivecroot_st,iretransn) = matrix_ntransfer_retransn_to_livecrootst_acc(p) matrix_ntransfer_acc(ideadcroot,iretransn) = matrix_ntransfer_retransn_to_deadcroot_acc(p) matrix_ntransfer_acc(ideadcroot_st,iretransn) = matrix_ntransfer_retransn_to_deadcrootst_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_acc(igrain,iretransn) = matrix_ntransfer_retransn_to_grain_acc(p) matrix_ntransfer_acc(igrain_st,iretransn) = matrix_ntransfer_retransn_to_grainst_acc(p) end if @@ -2704,7 +2704,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ntransfer_acc(ideadcroot,ideadcroot) = -matrix_nturnover_deadcroot_acc(p) matrix_ntransfer_acc(ideadcroot_st,ideadcroot_st) = -matrix_nturnover_deadcrootst_acc(p) matrix_ntransfer_acc(ideadcroot_xf,ideadcroot_xf) = -matrix_nturnover_deadcrootxf_acc(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_acc(igrain,igrain) = -matrix_nturnover_grain_acc(p) matrix_ntransfer_acc(igrain_st,igrain_st) = -matrix_nturnover_grainst_acc(p) matrix_ntransfer_acc(igrain_xf,igrain_xf) = -matrix_nturnover_grainxf_acc(p) @@ -2755,7 +2755,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ctransfer_acc(1:nvegcpool,ideadcroot) = matrix_ctransfer_acc(1:nvegcpool,ideadcroot) / deadcrootc0(p) matrix_ctransfer_acc(1:nvegcpool,ideadcroot_st) = matrix_ctransfer_acc(1:nvegcpool,ideadcroot_st) / deadcrootc0_storage(p) matrix_ctransfer_acc(1:nvegcpool,ideadcroot_xf) = matrix_ctransfer_acc(1:nvegcpool,ideadcroot_xf) / deadcrootc0_xfer(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ctransfer_acc(1:nvegcpool,igrain) = matrix_ctransfer_acc(1:nvegcpool,igrain) / reproc0(p) matrix_ctransfer_acc(1:nvegcpool,igrain_st) = matrix_ctransfer_acc(1:nvegcpool,igrain_st) / reproc0_storage(p) matrix_ctransfer_acc(1:nvegcpool,igrain_xf) = matrix_ctransfer_acc(1:nvegcpool,igrain_xf) / reproc0_xfer(p) @@ -2780,7 +2780,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c13transfer_acc(1:nvegcpool,ideadcroot) = matrix_c13transfer_acc(1:nvegcpool,ideadcroot) / cs13_veg%deadcrootc0_patch(p) matrix_c13transfer_acc(1:nvegcpool,ideadcroot_st) = matrix_c13transfer_acc(1:nvegcpool,ideadcroot_st) / cs13_veg%deadcrootc0_storage_patch(p) matrix_c13transfer_acc(1:nvegcpool,ideadcroot_xf) = matrix_c13transfer_acc(1:nvegcpool,ideadcroot_xf) / cs13_veg%deadcrootc0_xfer_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c13transfer_acc(1:nvegcpool,igrain) = matrix_c13transfer_acc(1:nvegcpool,igrain) / cs13_veg%reproc0_patch(p) matrix_c13transfer_acc(1:nvegcpool,igrain_st) = matrix_c13transfer_acc(1:nvegcpool,igrain_st) / cs13_veg%reproc0_storage_patch(p) matrix_c13transfer_acc(1:nvegcpool,igrain_xf) = matrix_c13transfer_acc(1:nvegcpool,igrain_xf) / cs13_veg%reproc0_xfer_patch(p) @@ -2806,7 +2806,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_c14transfer_acc(1:nvegcpool,ideadcroot) = matrix_c14transfer_acc(1:nvegcpool,ideadcroot) / cs14_veg%deadcrootc0_patch(p) matrix_c14transfer_acc(1:nvegcpool,ideadcroot_st) = matrix_c14transfer_acc(1:nvegcpool,ideadcroot_st) / cs14_veg%deadcrootc0_storage_patch(p) matrix_c14transfer_acc(1:nvegcpool,ideadcroot_xf) = matrix_c14transfer_acc(1:nvegcpool,ideadcroot_xf) / cs14_veg%deadcrootc0_xfer_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_c14transfer_acc(1:nvegcpool,igrain) = matrix_c14transfer_acc(1:nvegcpool,igrain) / cs14_veg%reproc0_patch(p) matrix_c14transfer_acc(1:nvegcpool,igrain_st) = matrix_c14transfer_acc(1:nvegcpool,igrain_st) / cs14_veg%reproc0_storage_patch(p) matrix_c14transfer_acc(1:nvegcpool,igrain_xf) = matrix_c14transfer_acc(1:nvegcpool,igrain_xf) / cs14_veg%reproc0_xfer_patch(p) @@ -2831,7 +2831,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ntransfer_acc(1:nvegnpool,ideadcroot) = matrix_ntransfer_acc(1:nvegnpool,ideadcroot) / deadcrootn0(p) matrix_ntransfer_acc(1:nvegnpool,ideadcroot_st) = matrix_ntransfer_acc(1:nvegnpool,ideadcroot_st) / deadcrootn0_storage(p) matrix_ntransfer_acc(1:nvegnpool,ideadcroot_xf) = matrix_ntransfer_acc(1:nvegnpool,ideadcroot_xf) / deadcrootn0_xfer(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_acc(1:nvegnpool,igrain) = matrix_ntransfer_acc(1:nvegnpool,igrain) / repron0(p) matrix_ntransfer_acc(1:nvegnpool,igrain_st) = matrix_ntransfer_acc(1:nvegnpool,igrain_st) / repron0_storage(p) matrix_ntransfer_acc(1:nvegnpool,igrain_xf) = matrix_ntransfer_acc(1:nvegnpool,igrain_xf) / repron0_xfer(p) @@ -2923,7 +2923,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire deadcrootc_SASUsave(p) = deadcrootc_SASUsave(p) + deadcrootc(p) deadcrootc_storage_SASUsave(p) = deadcrootc_storage_SASUsave(p) + deadcrootc_storage(p) deadcrootc_xfer_SASUsave(p) = deadcrootc_xfer_SASUsave(p) + deadcrootc_xfer(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then grainc_SASUsave(p) = grainc_SASUsave(p) + sum(reproductivec(p,:)) grainc_storage_SASUsave(p) = grainc_storage_SASUsave(p) + sum(reproductivec_storage(p,:)) end if @@ -2946,7 +2946,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%deadcrootc_SASUsave_patch(p) = cs13_veg%deadcrootc_SASUsave_patch(p) + cs13_veg%deadcrootc_patch(p) cs13_veg%deadcrootc_storage_SASUsave_patch(p) = cs13_veg%deadcrootc_storage_SASUsave_patch(p) + cs13_veg%deadcrootc_storage_patch(p) cs13_veg%deadcrootc_xfer_SASUsave_patch(p) = cs13_veg%deadcrootc_xfer_SASUsave_patch(p) + cs13_veg%deadcrootc_xfer_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%grainc_SASUsave_patch(p) = cs13_veg%grainc_SASUsave_patch(p) + cs13_veg%reproductivec_patch(p,irepr) cs13_veg%grainc_storage_SASUsave_patch(p) = cs13_veg%grainc_storage_SASUsave_patch(p) + cs13_veg%reproductivec_storage_patch(p,irepr) end if @@ -2970,7 +2970,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%deadcrootc_SASUsave_patch(p) = cs14_veg%deadcrootc_SASUsave_patch(p) + cs14_veg%deadcrootc_patch(p) cs14_veg%deadcrootc_storage_SASUsave_patch(p) = cs14_veg%deadcrootc_storage_SASUsave_patch(p) + cs14_veg%deadcrootc_storage_patch(p) cs14_veg%deadcrootc_xfer_SASUsave_patch(p) = cs14_veg%deadcrootc_xfer_SASUsave_patch(p) + cs14_veg%deadcrootc_xfer_patch(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%grainc_SASUsave_patch(p) = cs14_veg%grainc_SASUsave_patch(p) + cs14_veg%reproductivec_patch(p,irepr) cs14_veg%grainc_storage_SASUsave_patch(p) = cs14_veg%grainc_storage_SASUsave_patch(p) + cs14_veg%reproductivec_storage_patch(p,irepr) end if @@ -2993,7 +2993,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire deadcrootn_SASUsave(p) = deadcrootn_SASUsave(p) + deadcrootn(p) deadcrootn_storage_SASUsave(p) = deadcrootn_storage_SASUsave(p) + deadcrootn_storage(p) deadcrootn_xfer_SASUsave(p) = deadcrootn_xfer_SASUsave(p) + deadcrootn_xfer(p) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then grainn_SASUsave(p) = grainn_SASUsave(p) + reproductiven(p,irepr) end if if(iyr .eq. nyr_forcing)then @@ -3015,7 +3015,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire deadcrootc(p) = deadcrootc_SASUsave(p) / (nyr_forcing/nyr_SASU) deadcrootc_storage(p) = deadcrootc_storage_SASUsave(p) / (nyr_forcing/nyr_SASU) deadcrootc_xfer(p) = deadcrootc_xfer_SASUsave(p) / (nyr_forcing/nyr_SASU) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then reproductivec(p,:) = grainc_SASUsave(p) / (nyr_forcing/nyr_SASU) reproductivec_storage(p,:) = grainc_storage_SASUsave(p) / (nyr_forcing/nyr_SASU) end if @@ -3038,7 +3038,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%deadcrootc_patch(p) = cs13_veg%deadcrootc_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) cs13_veg%deadcrootc_storage_patch(p) = cs13_veg%deadcrootc_storage_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) cs13_veg%deadcrootc_xfer_patch(p) = cs13_veg%deadcrootc_xfer_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%reproductivec_patch(p,:) = cs13_veg%grainc_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) cs13_veg%reproductivec_storage_patch(p,:) = cs13_veg%grainc_storage_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) end if @@ -3062,7 +3062,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%deadcrootc_patch(p) = cs14_veg%deadcrootc_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) cs14_veg%deadcrootc_storage_patch(p) = cs14_veg%deadcrootc_storage_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) cs14_veg%deadcrootc_xfer_patch(p) = cs14_veg%deadcrootc_xfer_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%reproductivec_patch(p,:) = cs14_veg%grainc_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) cs14_veg%reproductivec_storage_patch(p,:) = cs14_veg%grainc_storage_SASUsave_patch(p) / (nyr_forcing/nyr_SASU) end if @@ -3085,7 +3085,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire deadcrootn(p) = deadcrootn_SASUsave(p) / (nyr_forcing/nyr_SASU) deadcrootn_storage(p) = deadcrootn_storage_SASUsave(p) / (nyr_forcing/nyr_SASU) deadcrootn_xfer(p) = deadcrootn_xfer_SASUsave(p) / (nyr_forcing/nyr_SASU) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then reproductiven(p,:) = grainn_SASUsave(p) / (nyr_forcing/nyr_SASU) end if leafc_SASUsave(p) = 0 @@ -3106,7 +3106,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire deadcrootc_SASUsave(p) = 0 deadcrootc_storage_SASUsave(p) = 0 deadcrootc_xfer_SASUsave(p) = 0 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then grainc_SASUsave(p) = 0 grainc_storage_SASUsave(p) = 0 end if @@ -3129,7 +3129,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%deadcrootc_SASUsave_patch(p) = 0 cs13_veg%deadcrootc_storage_SASUsave_patch(p) = 0 cs13_veg%deadcrootc_xfer_SASUsave_patch(p) = 0 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%grainc_SASUsave_patch(p) = 0 cs13_veg%grainc_storage_SASUsave_patch(p) = 0 end if @@ -3153,7 +3153,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%deadcrootc_SASUsave_patch(p) = 0 cs14_veg%deadcrootc_storage_SASUsave_patch(p) = 0 cs14_veg%deadcrootc_xfer_SASUsave_patch(p) = 0 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%grainc_SASUsave_patch(p) = 0 cs14_veg%grainc_storage_SASUsave_patch(p) = 0 end if @@ -3176,7 +3176,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire deadcrootn_SASUsave(p) = 0 deadcrootn_storage_SASUsave(p) = 0 deadcrootn_xfer_SASUsave(p) = 0 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then grainn_SASUsave(p) = 0 end if end if @@ -3204,7 +3204,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_cap_deadcrootc(p) = vegmatrixc_rt(ideadcroot) matrix_cap_deadcrootc_storage(p) = vegmatrixc_rt(ideadcroot_st) matrix_cap_deadcrootc_xfer(p) = vegmatrixc_rt(ideadcroot_xf) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_cap_reproc(p) = vegmatrixc_rt(igrain) matrix_cap_reproc_storage(p) = vegmatrixc_rt(igrain_st) matrix_cap_reproc_xfer(p) = vegmatrixc_rt(igrain_xf) @@ -3228,7 +3228,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%matrix_cap_deadcrootc_patch(p) = vegmatrixc13_rt(ideadcroot) cs13_veg%matrix_cap_deadcrootc_storage_patch(p) = vegmatrixc13_rt(ideadcroot_st) cs13_veg%matrix_cap_deadcrootc_xfer_patch(p) = vegmatrixc13_rt(ideadcroot_xf) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%matrix_cap_reproc_patch(p) = vegmatrixc13_rt(igrain) cs13_veg%matrix_cap_reproc_storage_patch(p) = vegmatrixc13_rt(igrain_st) cs13_veg%matrix_cap_reproc_xfer_patch(p) = vegmatrixc13_rt(igrain_xf) @@ -3253,7 +3253,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%matrix_cap_deadcrootc_patch(p) = vegmatrixc14_rt(ideadcroot) cs14_veg%matrix_cap_deadcrootc_storage_patch(p) = vegmatrixc14_rt(ideadcroot_st) cs14_veg%matrix_cap_deadcrootc_xfer_patch(p) = vegmatrixc14_rt(ideadcroot_xf) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%matrix_cap_reproc_patch(p) = vegmatrixc14_rt(igrain) cs14_veg%matrix_cap_reproc_storage_patch(p) = vegmatrixc14_rt(igrain_st) cs14_veg%matrix_cap_reproc_xfer_patch(p) = vegmatrixc14_rt(igrain_xf) @@ -3276,7 +3276,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_cap_livecrootn_xfer(p) = vegmatrixn_rt(ilivecroot_xf) matrix_cap_deadcrootn(p) = vegmatrixn_rt(ideadcroot) matrix_cap_deadcrootn_storage(p) = vegmatrixn_rt(ideadcroot_st) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_cap_repron(p) = vegmatrixn_rt(igrain) matrix_cap_repron_storage(p) = vegmatrixn_rt(igrain_st) matrix_cap_repron_xfer(p) = vegmatrixn_rt(igrain_xf) @@ -3296,7 +3296,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_calloc_livecrootst_acc(p) = 0._r8 matrix_calloc_deadcroot_acc(p) = 0._r8 matrix_calloc_deadcrootst_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_calloc_grain_acc(p) = 0._r8 matrix_calloc_grainst_acc(p) = 0._r8 end if @@ -3313,7 +3313,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ctransfer_livecrootxf_to_livecroot_acc(p) = 0._r8 matrix_ctransfer_deadcrootst_to_deadcrootxf_acc(p) = 0._r8 matrix_ctransfer_deadcrootxf_to_deadcroot_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ctransfer_grainst_to_grainxf_acc(p) = 0._r8 matrix_ctransfer_grainxf_to_grain_acc(p) = 0._r8 end if @@ -3338,7 +3338,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_cturnover_deadcroot_acc(p) = 0._r8 matrix_cturnover_deadcrootst_acc(p) = 0._r8 matrix_cturnover_deadcrootxf_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_cturnover_grain_acc(p) = 0._r8 matrix_cturnover_grainst_acc(p) = 0._r8 matrix_cturnover_grainxf_acc(p) = 0._r8 @@ -3357,7 +3357,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%matrix_calloc_livecrootst_acc_patch(p) = 0._r8 cs13_veg%matrix_calloc_deadcroot_acc_patch(p) = 0._r8 cs13_veg%matrix_calloc_deadcrootst_acc_patch(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%matrix_calloc_grain_acc_patch(p) = 0._r8 cs13_veg%matrix_calloc_grainst_acc_patch(p) = 0._r8 end if @@ -3374,7 +3374,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%matrix_ctransfer_livecrootxf_to_livecroot_acc_patch(p) = 0._r8 cs13_veg%matrix_ctransfer_deadcrootst_to_deadcrootxf_acc_patch(p) = 0._r8 cs13_veg%matrix_ctransfer_deadcrootxf_to_deadcroot_acc_patch(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%matrix_ctransfer_grainst_to_grainxf_acc_patch(p) = 0._r8 cs13_veg%matrix_ctransfer_grainxf_to_grain_acc_patch(p) = 0._r8 end if @@ -3399,7 +3399,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs13_veg%matrix_cturnover_deadcroot_acc_patch(p) = 0._r8 cs13_veg%matrix_cturnover_deadcrootst_acc_patch(p) = 0._r8 cs13_veg%matrix_cturnover_deadcrootxf_acc_patch(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs13_veg%matrix_cturnover_grain_acc_patch(p) = 0._r8 cs13_veg%matrix_cturnover_grainst_acc_patch(p) = 0._r8 cs13_veg%matrix_cturnover_grainxf_acc_patch(p) = 0._r8 @@ -3419,7 +3419,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%matrix_calloc_livecrootst_acc_patch(p) = 0._r8 cs14_veg%matrix_calloc_deadcroot_acc_patch(p) = 0._r8 cs14_veg%matrix_calloc_deadcrootst_acc_patch(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%matrix_calloc_grain_acc_patch(p) = 0._r8 cs14_veg%matrix_calloc_grainst_acc_patch(p) = 0._r8 end if @@ -3436,7 +3436,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%matrix_ctransfer_livecrootxf_to_livecroot_acc_patch(p) = 0._r8 cs14_veg%matrix_ctransfer_deadcrootst_to_deadcrootxf_acc_patch(p) = 0._r8 cs14_veg%matrix_ctransfer_deadcrootxf_to_deadcroot_acc_patch(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%matrix_ctransfer_grainst_to_grainxf_acc_patch(p) = 0._r8 cs14_veg%matrix_ctransfer_grainxf_to_grain_acc_patch(p) = 0._r8 end if @@ -3461,7 +3461,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire cs14_veg%matrix_cturnover_deadcroot_acc_patch(p) = 0._r8 cs14_veg%matrix_cturnover_deadcrootst_acc_patch(p) = 0._r8 cs14_veg%matrix_cturnover_deadcrootxf_acc_patch(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then cs14_veg%matrix_cturnover_grain_acc_patch(p) = 0._r8 cs14_veg%matrix_cturnover_grainst_acc_patch(p) = 0._r8 cs14_veg%matrix_cturnover_grainxf_acc_patch(p) = 0._r8 @@ -3480,7 +3480,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_nalloc_livecrootst_acc(p) = 0._r8 matrix_nalloc_deadcroot_acc(p) = 0._r8 matrix_nalloc_deadcrootst_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_nalloc_grain_acc(p) = 0._r8 matrix_nalloc_grainst_acc(p) = 0._r8 end if @@ -3497,7 +3497,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ntransfer_livecrootxf_to_livecroot_acc(p) = 0._r8 matrix_ntransfer_deadcrootst_to_deadcrootxf_acc(p) = 0._r8 matrix_ntransfer_deadcrootxf_to_deadcroot_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_grainst_to_grainxf_acc(p) = 0._r8 matrix_ntransfer_grainxf_to_grain_acc(p) = 0._r8 end if @@ -3516,7 +3516,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_ntransfer_retransn_to_livecrootst_acc(p) = 0._r8 matrix_ntransfer_retransn_to_deadcroot_acc(p) = 0._r8 matrix_ntransfer_retransn_to_deadcrootst_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_ntransfer_retransn_to_grain_acc(p) = 0._r8 matrix_ntransfer_retransn_to_grainst_acc(p) = 0._r8 end if @@ -3543,7 +3543,7 @@ subroutine CNVegMatrix(bounds,num_soilp,filter_soilp,num_actfirep,filter_actfire matrix_nturnover_deadcroot_acc(p) = 0._r8 matrix_nturnover_deadcrootst_acc(p) = 0._r8 matrix_nturnover_deadcrootxf_acc(p) = 0._r8 - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then matrix_nturnover_grain_acc(p) = 0._r8 matrix_nturnover_grainst_acc(p) = 0._r8 matrix_nturnover_grainxf_acc(p) = 0._r8 diff --git a/src/biogeochem/CNVegNitrogenStateType.F90 b/src/biogeochem/CNVegNitrogenStateType.F90 index 7ff8e77df9..c343c45aca 100644 --- a/src/biogeochem/CNVegNitrogenStateType.F90 +++ b/src/biogeochem/CNVegNitrogenStateType.F90 @@ -10,7 +10,7 @@ module CNVegNitrogenStateType use clm_varctl , only : use_crop use CNSharedParamsMod , only : use_fun, use_matrixcn use decompMod , only : bounds_type - use pftconMod , only : npcropmin, noveg, pftcon + use pftconMod , only : is_prognostic_crop, noveg, pftcon use abortutils , only : endrun use spmdMod , only : masterproc use LandunitType , only : lun @@ -2251,7 +2251,7 @@ subroutine Summary_nitrogenstate(this, bounds, num_soilc, filter_soilc, num_soil this%npool_patch(p) + & this%retransn_patch(p) - if ( use_crop .and. patch%itype(p) >= npcropmin )then + if ( use_crop .and. is_prognostic_crop(patch%itype(p)) )then do k = 1, nrepr this%dispvegn_patch(p) = & this%dispvegn_patch(p) + & diff --git a/src/biogeochem/CNVegStructUpdateMod.F90 b/src/biogeochem/CNVegStructUpdateMod.F90 index 2e8ed8539b..007478b573 100644 --- a/src/biogeochem/CNVegStructUpdateMod.F90 +++ b/src/biogeochem/CNVegStructUpdateMod.F90 @@ -38,7 +38,7 @@ subroutine CNVegStructUpdate(bounds,num_soilp, filter_soilp, & ! ! !USES: use pftconMod , only : noveg, nc3crop, nc3irrig, nbrdlf_evr_shrub, nbrdlf_dcd_brl_shrub - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use pftconMod , only : ntmp_corn, nirrig_tmp_corn use pftconMod , only : ntrp_corn, nirrig_trp_corn use pftconMod , only : nsugarcane, nirrig_sugarcane @@ -232,7 +232,7 @@ subroutine CNVegStructUpdate(bounds,num_soilp, filter_soilp, & hbot(p) = max(0._r8, min(3._r8, htop(p)-1._r8)) - else if (ivt(p) >= npcropmin) then ! prognostic crops + else if (is_prognostic_crop(ivt(p))) then ! prognostic crops if (tlai(p) >= laimx(ivt(p))) peaklai(p) = 1 ! used in CNAllocation diff --git a/src/biogeochem/CropType.F90 b/src/biogeochem/CropType.F90 index 54395c4668..48f9cb9cad 100644 --- a/src/biogeochem/CropType.F90 +++ b/src/biogeochem/CropType.F90 @@ -528,7 +528,7 @@ subroutine Restart(this, bounds, ncid, cnveg_state_inst, flag) use restUtilMod use ncdio_pio use PatchType, only : patch - use pftconMod, only : npcropmin, npcropmax + use pftconMod, only : is_prognostic_crop use clm_varpar, only : mxsowings, mxharvests ! BACKWARDS_COMPATIBILITY(wjs/ssr, 2023-01-09) use CNVegstateType, only : cnveg_state_type @@ -577,7 +577,7 @@ subroutine Restart(this, bounds, ncid, cnveg_state_inst, flag) interpinic_flag='copy', readvar=readvar, data=restyear) if (readvar) then do p = bounds%begp, bounds%endp - if (patch%itype(p) >= npcropmin .and. patch%itype(p) <= npcropmax .and. & + if (is_prognostic_crop(patch%itype(p)) .and. & patch%active(p)) then this%nyrs_crop_active_patch(p) = restyear end if diff --git a/src/biogeochem/DryDepVelocity.F90 b/src/biogeochem/DryDepVelocity.F90 index f6a3b857da..b13d28c765 100644 --- a/src/biogeochem/DryDepVelocity.F90 +++ b/src/biogeochem/DryDepVelocity.F90 @@ -204,7 +204,7 @@ subroutine depvel_compute( bounds, & use pftconMod , only : nbrdlf_evr_shrub, nbrdlf_dcd_tmp_shrub use pftconMod , only : nbrdlf_dcd_brl_shrub,nc3_arctic_grass use pftconMod , only : nc3_nonarctic_grass, nc4_grass, nc3crop - use pftconMod , only : nc3irrig, npcropmin, npcropmax + use pftconMod , only : nc3irrig, is_prognostic_crop use clm_varcon , only : spval ! @@ -349,7 +349,7 @@ subroutine depvel_compute( bounds, & if (clmveg == nc4_grass ) wesveg = 3 if (clmveg == nc3crop ) wesveg = 2 if (clmveg == nc3irrig ) wesveg = 2 - if (clmveg >= npcropmin .and. clmveg <= npcropmax ) wesveg = 2 + if (is_prognostic_crop(clmveg)) wesveg = 2 if (wesveg == wveg_unset )then write(iulog,*) 'clmveg = ', clmveg, 'lun%itype = ', lun%itype(l) call endrun(subgrid_index=pi, subgrid_level=subgrid_level_patch, & diff --git a/src/biogeochem/NutrientCompetitionCLM45defaultMod.F90 b/src/biogeochem/NutrientCompetitionCLM45defaultMod.F90 index 82dceef664..95f0594a31 100644 --- a/src/biogeochem/NutrientCompetitionCLM45defaultMod.F90 +++ b/src/biogeochem/NutrientCompetitionCLM45defaultMod.F90 @@ -125,7 +125,7 @@ subroutine calc_plant_cn_alloc (this, bounds, num_soilp, filter_soilp, & c14_cnveg_carbonflux_inst, cnveg_nitrogenflux_inst, cnveg_nitrogenstate_inst, fpg_col) ! ! !USES: - use pftconMod , only : pftcon, npcropmin + use pftconMod , only : pftcon, is_prognostic_crop use clm_varctl , only : use_c13, use_c14 use CNVegStateType , only : cnveg_state_type use CropType , only : crop_type @@ -281,7 +281,7 @@ subroutine calc_plant_cn_alloc (this, bounds, num_soilp, filter_soilp, & cndw = deadwdcn(ivt(p)) fcur = fcur2(ivt(p)) - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops if (croplive(p).and.(.not.shr_infnan_isnan(aleaf(p)))) then f1 = aroot(p) / aleaf(p) f3 = astem(p) / aleaf(p) @@ -357,7 +357,7 @@ subroutine calc_plant_cn_alloc (this, bounds, num_soilp, filter_soilp, & cpool_to_deadcrootc(p) = nlc * f2 * f3 * (1._r8 - f4) * fcur cpool_to_deadcrootc_storage(p) = nlc * f2 * f3 * (1._r8 - f4) * (1._r8 - fcur) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cpool_to_livestemc(p) = nlc * f3 * f4 * fcur cpool_to_livestemc_storage(p) = nlc * f3 * f4 * (1._r8 - fcur) cpool_to_deadstemc(p) = nlc * f3 * (1._r8 - f4) * fcur @@ -387,7 +387,7 @@ subroutine calc_plant_cn_alloc (this, bounds, num_soilp, filter_soilp, & npool_to_deadcrootn(p) = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * fcur npool_to_deadcrootn_storage(p) = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * (1._r8 - fcur) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cng = graincn(ivt(p)) npool_to_livestemn(p) = (nlc * f3 * f4 / cnlw) * fcur npool_to_livestemn_storage(p) = (nlc * f3 * f4 / cnlw) * (1._r8 - fcur) @@ -420,7 +420,7 @@ subroutine calc_plant_cn_alloc (this, bounds, num_soilp, filter_soilp, & gresp_storage = gresp_storage + cpool_to_livecrootc_storage(p) gresp_storage = gresp_storage + cpool_to_deadcrootc_storage(p) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops gresp_storage = gresp_storage + cpool_to_livestemc_storage(p) do k = 1, nrepr gresp_storage = gresp_storage + cpool_to_reproductivec_storage(p,k) @@ -505,7 +505,7 @@ subroutine calc_plant_nitrogen_demand(this, bounds, & ! - livestemn_to_retransn ! ! !USES: - use pftconMod , only : npcropmin, pftcon + use pftconMod , only : is_prognostic_crop, pftcon use pftconMod , only : ntmp_soybean, nirrig_tmp_soybean use pftconMod , only : ntrp_soybean, nirrig_trp_soybean use clm_time_manager , only : get_step_size_real diff --git a/src/biogeochem/NutrientCompetitionFlexibleCNMod.F90 b/src/biogeochem/NutrientCompetitionFlexibleCNMod.F90 index 00e8a00b77..d3f8753fd0 100644 --- a/src/biogeochem/NutrientCompetitionFlexibleCNMod.F90 +++ b/src/biogeochem/NutrientCompetitionFlexibleCNMod.F90 @@ -24,7 +24,7 @@ module NutrientCompetitionFlexibleCNMod use LandunitType , only : lun use ColumnType , only : col use PatchType , only : patch - use pftconMod , only : pftcon, npcropmin + use pftconMod , only : pftcon, is_prognostic_crop use NutrientCompetitionMethodMod, only : nutrient_competition_method_type use CropReprPoolsMod , only : nrepr use CNPhenologyMod , only : CropPhase @@ -413,7 +413,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & fcur = 0.0_r8 end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops if (croplive(p)) then f1 = aroot(p) / aleaf(p) f3 = astem(p) / aleaf(p) @@ -497,7 +497,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & + cpool_to_deadcrootc(p) + cpool_to_deadcrootc_storage(p) end if end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops cpool_to_livestemc(p) = nlc * f3 * f4 * fcur cpool_to_livestemc_storage(p) = nlc * f3 * f4 * (1._r8 - fcur) cpool_to_deadstemc(p) = nlc * f3 * (1._r8 - f4) * fcur @@ -548,7 +548,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & matrix_alloc(p,ideadcroot_st) = cpool_to_deadcrootc_storage(p) / cpool_to_veg end if end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops if(cpool_to_veg .ne. 0)then matrix_alloc(p,ilivestem) = cpool_to_livestemc(p) / cpool_to_veg matrix_alloc(p,ilivestem_st) = cpool_to_livestemc_storage(p) / cpool_to_veg @@ -586,7 +586,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & gresp_storage = gresp_storage + cpool_to_livecrootc_storage(p) gresp_storage = gresp_storage + cpool_to_deadcrootc_storage(p) end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops gresp_storage = gresp_storage + cpool_to_livestemc_storage(p) do k = 1, nrepr gresp_storage = gresp_storage + cpool_to_reproductivec_storage(p,k) @@ -594,7 +594,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & end if cpool_to_gresp_storage(p) = gresp_storage * g1 * (1._r8 - g2) - if (use_crop_agsys .and. ivt(p) >= npcropmin) then + if (use_crop_agsys .and. is_prognostic_crop(ivt(p))) then call calc_npool_to_components_agsys( & ! Inputs npool = npool(p), & @@ -708,7 +708,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & end if end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops if (cnveg_nitrogenstate_inst%livestemn_storage_patch(p) == 0.0_r8) then ! to avoid division by zero, and also to make livestemcn_actual(p) a very large number if livestemc(p) is zero @@ -795,7 +795,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & end if - if (ivt(p) >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt(p))) then ! skip 2 generic crops livewdcn_max = livewdcn(ivt(p)) + 15.0_r8 @@ -876,7 +876,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & + npool_to_deadstemn(p) + npool_to_deadstemn_storage(p) & + npool_to_livecrootn(p) + npool_to_livecrootn_storage(p) & + npool_to_deadcrootn(p) + npool_to_deadcrootn_storage(p) - if (ivt(p) >= npcropmin)then + if (is_prognostic_crop(ivt(p)))then npool_to_veg = npool_to_veg + npool_to_reproductiven(p,1) + npool_to_reproductiven_storage(p,1) end if if(npool_to_veg .ne. 0._r8)then @@ -892,7 +892,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & matrix_nalloc(p,ilivecroot_st ) = npool_to_livecrootn_storage(p) / npool_to_veg matrix_nalloc(p,ideadcroot ) = npool_to_deadcrootn(p) / npool_to_veg matrix_nalloc(p,ideadcroot_st ) = npool_to_deadcrootn_storage(p) / npool_to_veg - if (ivt(p) >= npcropmin)then + if (is_prognostic_crop(ivt(p)))then matrix_nalloc(p,igrain ) = npool_to_reproductiven(p,1) / npool_to_veg matrix_nalloc(p,igrain_st ) = npool_to_reproductiven_storage(p,1) / npool_to_veg end if @@ -916,7 +916,7 @@ subroutine calc_plant_cn_alloc(this, bounds, num_soilp, filter_soilp, & tmp = matrix_update_phn(p,iretransn_to_ilivecrootst ,matrix_nalloc(p,ilivecroot_st ) * retransn_to_npool(p) / retransn(p),dt,cnveg_nitrogenflux_inst,matrixcheck_ph,.True.) tmp = matrix_update_phn(p,iretransn_to_ideadcroot ,matrix_nalloc(p,ideadcroot ) * retransn_to_npool(p) / retransn(p),dt,cnveg_nitrogenflux_inst,matrixcheck_ph,.True.) tmp = matrix_update_phn(p,iretransn_to_ideadcrootst ,matrix_nalloc(p,ideadcroot_st ) * retransn_to_npool(p) / retransn(p),dt,cnveg_nitrogenflux_inst,matrixcheck_ph,.True.) - if(ivt(p) >= npcropmin)then + if(is_prognostic_crop(ivt(p)))then tmp = matrix_update_phn(p,iretransn_to_igrain ,matrix_nalloc(p,igrain ) * retransn_to_npool(p) / retransn(p),dt,cnveg_nitrogenflux_inst,matrixcheck_ph,.True.) tmp = matrix_update_phn(p,iretransn_to_igrainst ,matrix_nalloc(p,igrain_st ) * retransn_to_npool(p) / retransn(p),dt,cnveg_nitrogenflux_inst,matrixcheck_ph,.True.) end if @@ -1051,7 +1051,7 @@ subroutine calc_npool_to_components_flexiblecn( & npool_to_deadcrootn_demand = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * fcur npool_to_deadcrootn_storage_demand = (nlc * f2 * f3 * (1._r8 - f4) / cndw) * (1._r8 - fcur) end if - if (ivt >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt)) then ! skip 2 generic crops cng = graincn(ivt) npool_to_livestemn_demand = (nlc * f3 * f4 / cnlw) * fcur @@ -1084,7 +1084,7 @@ subroutine calc_npool_to_components_flexiblecn( & npool_to_deadcrootn_storage_demand end if - if (ivt >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt)) then ! skip 2 generic crops npool_to_reproductiven_demand_tot = 0._r8 npool_to_reproductiven_storage_demand_tot = 0._r8 @@ -1122,7 +1122,7 @@ subroutine calc_npool_to_components_flexiblecn( & frNdemand_npool_to_deadcrootn = 0.0_r8 frNdemand_npool_to_deadcrootn_storage = 0.0_r8 end if - if (ivt >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt)) then ! skip 2 generic crops frNdemand_npool_to_livestemn = 0.0_r8 frNdemand_npool_to_livestemn_storage = 0.0_r8 @@ -1155,7 +1155,7 @@ subroutine calc_npool_to_components_flexiblecn( & frNdemand_npool_to_deadcrootn = npool_to_deadcrootn_demand / total_plant_Ndemand frNdemand_npool_to_deadcrootn_storage = npool_to_deadcrootn_storage_demand / total_plant_Ndemand end if - if (ivt >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt)) then ! skip 2 generic crops frNdemand_npool_to_livestemn = npool_to_livestemn_demand / total_plant_Ndemand frNdemand_npool_to_livestemn_storage = npool_to_livestemn_storage_demand / total_plant_Ndemand @@ -1194,7 +1194,7 @@ subroutine calc_npool_to_components_flexiblecn( & npool_to_deadcrootn = frNdemand_npool_to_deadcrootn * npool / dt npool_to_deadcrootn_storage = frNdemand_npool_to_deadcrootn_storage * npool / dt end if - if (ivt >= npcropmin) then ! skip 2 generic crops + if (is_prognostic_crop(ivt)) then ! skip 2 generic crops npool_to_livestemn = frNdemand_npool_to_livestemn * npool / dt npool_to_livestemn_storage = frNdemand_npool_to_livestemn_storage * npool / dt npool_to_deadstemn = frNdemand_npool_to_deadstemn * npool / dt diff --git a/src/biogeophys/PhotosynthesisMod.F90 b/src/biogeophys/PhotosynthesisMod.F90 index b8fd577382..a823bce548 100644 --- a/src/biogeophys/PhotosynthesisMod.F90 +++ b/src/biogeophys/PhotosynthesisMod.F90 @@ -1236,7 +1236,7 @@ subroutine Photosynthesis ( bounds, fn, filterp, & use clm_time_manager , only : get_step_size_real, is_near_local_noon use clm_varctl , only : cnallocate_carbon_only use clm_varctl , only : lnc_opt, reduce_dayl_factor, vcmax_opt - use pftconMod , only : nbrdlf_dcd_tmp_shrub, npcropmin + use pftconMod , only : nbrdlf_dcd_tmp_shrub ! ! !ARGUMENTS: @@ -2727,7 +2727,7 @@ subroutine PhotosynthesisHydraulicStress ( bounds, fn, filterp, & use clm_varctl , only : cnallocate_carbon_only use clm_varctl , only : lnc_opt, reduce_dayl_factor, vcmax_opt use clm_varpar , only : nlevsoi - use pftconMod , only : nbrdlf_dcd_tmp_shrub, npcropmin + use pftconMod , only : nbrdlf_dcd_tmp_shrub use ColumnType , only : col ! diff --git a/src/biogeophys/TemperatureType.F90 b/src/biogeophys/TemperatureType.F90 index 899da6b882..58e4c93e7b 100644 --- a/src/biogeophys/TemperatureType.F90 +++ b/src/biogeophys/TemperatureType.F90 @@ -1414,7 +1414,7 @@ subroutine UpdateAccVars_CropGDDs(this, rbufslp, begp, endp, month, day, secs, d use shr_const_mod , only : SHR_CONST_CDAY, SHR_CONST_TKFRZ use accumulMod , only : update_accum_field, extract_accum_field, markreset_accum_field use clm_time_manager , only : is_doy_in_interval, get_curr_calday - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use CropType, only : crop_type ! ! !ARGUMENTS @@ -1485,7 +1485,7 @@ subroutine UpdateAccVars_CropGDDs(this, rbufslp, begp, endp, month, day, secs, d ((month > 9 .or. month < 4) .and. lat < 0._r8) ! Replace with read-in gdd20 accumulation season, if needed and valid ! (If these aren't being read in or they're invalid, they'll be -1) - if (stream_gdd20_seasons_tt .and. patch%itype(p) >= npcropmin) then + if (stream_gdd20_seasons_tt .and. is_prognostic_crop(patch%itype(p))) then gdd20_season_start = int(gdd20_season_starts(p)) gdd20_season_end = int(gdd20_season_ends(p)) if (gdd20_season_start >= 1 .and. gdd20_season_end >= 1) then diff --git a/src/cpl/share_esmf/cropcalStreamMod.F90 b/src/cpl/share_esmf/cropcalStreamMod.F90 index b19612ca09..e9927a99c7 100644 --- a/src/cpl/share_esmf/cropcalStreamMod.F90 +++ b/src/cpl/share_esmf/cropcalStreamMod.F90 @@ -22,7 +22,7 @@ module cropcalStreamMod use clm_varpar , only : mxsowings use perf_mod , only : t_startf, t_stopf use spmdMod , only : masterproc, mpicom, iam - use pftconMod , only : npcropmin + use pftconMod , only : npcropmin, is_prognostic_crop use CNPhenologyMod , only : generate_crop_gdds ! ! !PUBLIC TYPES: @@ -588,7 +588,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) p = filter_pcropp(fp) ivt = patch%itype(p) ! Will skip generic crops - if (ivt >= npcropmin) then + if (is_prognostic_crop(ivt)) then n = ivt - npcropmin + 1 ! vegetated pft ig = g_to_ig(patch%gridcell(p)) @@ -612,7 +612,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ! Handle invalid sowing window values if (any(swindow_starts(begp:endp,:) < 1 .or. swindow_ends(begp:endp,:) < 1)) then ! Fail if not allowing fallback to paramfile sowing windows - if ((.not. allow_invalid_swindow_inputs) .and. any(all(swindow_starts(begp:endp,:) < 1, dim=2) .and. patch%wtgcell(begp:endp) > 0._r8 .and. patch%itype(begp:endp) >= npcropmin)) then + if ((.not. allow_invalid_swindow_inputs) .and. any(all(swindow_starts(begp:endp,:) < 1, dim=2) .and. patch%wtgcell(begp:endp) > 0._r8 .and. is_prognostic_crop(patch%itype(begp:endp)))) then write(iulog, *) 'At least one crop in one gridcell has invalid prescribed sowing window start date(s). To ignore and fall back to paramfile sowing windows, set allow_invalid_swindow_inputs to .true.' write(iulog, *) 'Affected crops:' do ivt = npcropmin, mxpft @@ -667,7 +667,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ivt = patch%itype(p) ! Will skip generic crops - if (ivt >= npcropmin) then + if (is_prognostic_crop(ivt)) then n = ivt - npcropmin + 1 if (n > ncft) then @@ -718,7 +718,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ivt = patch%itype(p) ! Will skip generic crops - if (ivt >= npcropmin) then + if (is_prognostic_crop(ivt)) then n = ivt - npcropmin + 1 if (n > ncft) then @@ -788,7 +788,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) p = filter_pcropp(fp) ivt = patch%itype(p) ! Will skip generic crops - if (ivt >= npcropmin) then + if (is_prognostic_crop(ivt)) then n = ivt - npcropmin + 1 ! vegetated pft ig = g_to_ig(patch%gridcell(p)) @@ -805,7 +805,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) if (any(gdd20_season_starts(begp:endp) < 1._r8 .or. gdd20_season_ends(begp:endp) < 1._r8)) then ! Fail if not allowing fallback to paramfile sowing windows. Only need to check for ! values < 1 because values outside [1, 366] are set to -1 above. - if ((.not. allow_invalid_gdd20_season_inputs) .and. any(gdd20_season_starts(begp:endp) < 1._r8 .and. patch%wtgcell(begp:endp) > 0._r8 .and. patch%itype(begp:endp) >= npcropmin)) then + if ((.not. allow_invalid_gdd20_season_inputs) .and. any(gdd20_season_starts(begp:endp) < 1._r8 .and. patch%wtgcell(begp:endp) > 0._r8 .and. is_prognostic_crop(patch%itype(begp:endp)))) then write(iulog, *) 'At least one crop in one gridcell has invalid gdd20 season start and/or end date(s). To ignore and fall back to paramfile sowing windows for such crop-gridcells, set allow_invalid_gdd20_season_inputs to .true.' write(iulog, *) 'Affected crops:' do ivt = npcropmin, mxpft diff --git a/src/main/filterMod.F90 b/src/main/filterMod.F90 index 2fb7d23079..6540021923 100644 --- a/src/main/filterMod.F90 +++ b/src/main/filterMod.F90 @@ -316,7 +316,7 @@ subroutine setFiltersOneGroup(bounds, this_filter, include_inactive, glc_behavio ! ! !USES: use decompMod , only : bounds_level_clump - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use landunit_varcon , only : istsoil, istcrop, istice ! ! !ARGUMENTS: @@ -479,7 +479,7 @@ subroutine setFiltersOneGroup(bounds, this_filter, include_inactive, glc_behavio do p = bounds%begp,bounds%endp if(.not.use_fates)then if (patch%active(p) .or. include_inactive) then - if (patch%itype(p) >= npcropmin) then !skips 2 generic crop types + if (is_prognostic_crop(patch%itype(p))) then !skips 2 generic crop types fl = fl + 1 this_filter(nc)%pcropp(fl) = p else diff --git a/src/main/pftconMod.F90 b/src/main/pftconMod.F90 index b48ef92a43..94ca009ae9 100644 --- a/src/main/pftconMod.F90 +++ b/src/main/pftconMod.F90 @@ -2,8 +2,9 @@ module pftconMod !----------------------------------------------------------------------- ! !DESCRIPTION: - ! Module containing vegetation constants and method to - ! read and initialize vegetation (PFT) constants. + ! Module containing vegetation constants, methods to + ! read and initialize vegetation (PFT) constants, and methods to query + ! PFT characteristics ! ! !USES: use shr_kind_mod, only : r8 => shr_kind_r8 @@ -315,6 +316,8 @@ module pftconMod character(len=*), parameter, private :: sourcefile = & __FILE__ + + public :: is_prognostic_crop !----------------------------------------------------------------------- contains @@ -1351,19 +1354,19 @@ subroutine InitRead(this) else call endrun(msg=' ERROR: crop has wrong values'//errMsg(sourcefile, __LINE__)) end if - if ( (i /= noveg) .and. (i < npcropmin) .and. & + if ( (i /= noveg) .and. (.not. is_prognostic_crop(i)) .and. & abs(this%pconv(i) + this%pprod10(i) + this%pprod100(i) - 1.0_r8) > 1.e-7_r8 )then call endrun(msg=' ERROR: pconv+pprod10+pprod100 do NOT sum to one.'//errMsg(sourcefile, __LINE__)) end if if ( this%pprodharv10(i) > 1.0_r8 .or. this%pprodharv10(i) < 0.0_r8 )then call endrun(msg=' ERROR: pprodharv10 outside of range.'//errMsg(sourcefile, __LINE__)) end if - if (i < npcropmin .and. this%biofuel_harvfrac(i) /= 0._r8) then + if ((.not. is_prognostic_crop(i)) .and. this%biofuel_harvfrac(i) /= 0._r8) then call endrun(msg=' ERROR: biofuel_harvfrac non-zero for a non-prognostic crop PFT.'//& errMsg(sourcefile, __LINE__)) end if do k = repr_structure_min, repr_structure_max - if (i < npcropmin .and. this%repr_structure_harvfrac(i,k) /= 0._r8) then + if ((.not. is_prognostic_crop(i)) .and. this%repr_structure_harvfrac(i,k) /= 0._r8) then call endrun(msg=' ERROR: repr_structure_harvfrac non-zero for a non-prognostic crop PFT.'//& errMsg(sourcefile, __LINE__)) end if @@ -1607,5 +1610,24 @@ subroutine Clean(this) deallocate( this%ndays_on) end subroutine Clean + !----------------------------------------------------------------------- + elemental logical function is_prognostic_crop(veg_type) + ! + ! !DESCRIPTION: + ! Given a vegetation type (pft, integer), return whether it's a prognostic crop. Does not + ! include generic crops (those and natural PFTs will return .false.). + ! + ! NOTE(wjs, 2017-02-02) This isn't a completely robust way to check if this is a + ! prognostic crop patch (at the very least it should also check if <= npcropmax; + ! ideally it should use a prognostic_crop flag that doesn't seem to exist + ! currently). + ! + ! !ARGUMENTS + integer, intent(in) :: veg_type + + is_prognostic_crop = veg_type >= npcropmin + + end function is_prognostic_crop + end module pftconMod diff --git a/src/soilbiogeochem/TillageMod.F90 b/src/soilbiogeochem/TillageMod.F90 index 4a24daf4c2..5f94a44777 100644 --- a/src/soilbiogeochem/TillageMod.F90 +++ b/src/soilbiogeochem/TillageMod.F90 @@ -284,7 +284,7 @@ subroutine get_apply_tillage_multipliers(idop, c, j, decomp_k) ! Written by Sam Rabin, based on original code by Michael Graham. ! ! !USES - use pftconMod , only : npcropmin + use pftconMod , only : is_prognostic_crop use clm_varcon, only : zisoi, dzsoi_decomp use landunit_varcon , only : istcrop use PatchType , only : patch @@ -315,7 +315,7 @@ subroutine get_apply_tillage_multipliers(idop, c, j, decomp_k) sumwt = 0.0_r8 do p = col%patchi(c),col%patchf(c) if (patch%active(p) .and. patch%wtcol(p) /= 0._r8) then - if (patch%itype(p) < npcropmin) then + if (.not. is_prognostic_crop(patch%itype(p))) then ! Do not till generic crops tillage_mults_1patch(:) = 1._r8 else From 8ca7af5c79fd7fdc7390253407fe8e2f8aebe5f3 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 6 Aug 2025 14:59:15 -0600 Subject: [PATCH 178/196] Avoid using npcropmin/max for looping outside pftconMod. --- src/biogeochem/CNPhenologyMod.F90 | 7 ++++--- src/cpl/share_esmf/cropcalStreamMod.F90 | 10 ++++++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/biogeochem/CNPhenologyMod.F90 b/src/biogeochem/CNPhenologyMod.F90 index a907a689c9..fb43c37804 100644 --- a/src/biogeochem/CNPhenologyMod.F90 +++ b/src/biogeochem/CNPhenologyMod.F90 @@ -2691,8 +2691,9 @@ subroutine CropPhenologyInit(bounds) ! initialized, and after pftcon file is read in. ! ! !USES: - use pftconMod , only: npcropmin, npcropmax use clm_time_manager, only: get_calday + use pftconMod, only: is_prognostic_crop + use clm_varpar, only: mxpft ! ! !ARGUMENTS: type(bounds_type), intent(in) :: bounds @@ -2713,8 +2714,8 @@ subroutine CropPhenologyInit(bounds) ! Convert planting dates into julian day minplantjday(:,:) = huge(1) maxplantjday(:,:) = huge(1) - do n = npcropmin, npcropmax - if (pftcon%is_pft_known_to_model(n)) then + do n = 1, mxpft + if (is_prognostic_crop(n) .and. pftcon%is_pft_known_to_model(n)) then minplantjday(n, inNH) = int( get_calday( pftcon%mnNHplantdate(n), 0 ) ) maxplantjday(n, inNH) = int( get_calday( pftcon%mxNHplantdate(n), 0 ) ) diff --git a/src/cpl/share_esmf/cropcalStreamMod.F90 b/src/cpl/share_esmf/cropcalStreamMod.F90 index e9927a99c7..a0e83a5995 100644 --- a/src/cpl/share_esmf/cropcalStreamMod.F90 +++ b/src/cpl/share_esmf/cropcalStreamMod.F90 @@ -615,7 +615,10 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) if ((.not. allow_invalid_swindow_inputs) .and. any(all(swindow_starts(begp:endp,:) < 1, dim=2) .and. patch%wtgcell(begp:endp) > 0._r8 .and. is_prognostic_crop(patch%itype(begp:endp)))) then write(iulog, *) 'At least one crop in one gridcell has invalid prescribed sowing window start date(s). To ignore and fall back to paramfile sowing windows, set allow_invalid_swindow_inputs to .true.' write(iulog, *) 'Affected crops:' - do ivt = npcropmin, mxpft + do ivt = 1, mxpft + if (.not. is_prognostic_crop(ivt)) then + cycle + end if do fp = 1, num_pcropp p = filter_pcropp(fp) if (ivt == patch%itype(p) .and. patch%wtgcell(p) > 0._r8 .and. all(swindow_starts(p,:) < 1)) then @@ -808,7 +811,10 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) if ((.not. allow_invalid_gdd20_season_inputs) .and. any(gdd20_season_starts(begp:endp) < 1._r8 .and. patch%wtgcell(begp:endp) > 0._r8 .and. is_prognostic_crop(patch%itype(begp:endp)))) then write(iulog, *) 'At least one crop in one gridcell has invalid gdd20 season start and/or end date(s). To ignore and fall back to paramfile sowing windows for such crop-gridcells, set allow_invalid_gdd20_season_inputs to .true.' write(iulog, *) 'Affected crops:' - do ivt = npcropmin, mxpft + do ivt = 1, mxpft + if (.not. is_prognostic_crop(ivt)) then + cycle + end if do fp = 1, num_pcropp p = filter_pcropp(fp) if (ivt == patch%itype(p) .and. patch%wtgcell(p) > 0._r8 .and. gdd20_season_starts(p) < 1._r8) then From 0dfa907065f3f3b0911f80f113d6ed23dd4cd665 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 6 Aug 2025 15:48:32 -0600 Subject: [PATCH 179/196] Add get_crop_n_from_veg_type(). --- src/cpl/share_esmf/cropcalStreamMod.F90 | 10 +++++----- src/main/pftconMod.F90 | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/cpl/share_esmf/cropcalStreamMod.F90 b/src/cpl/share_esmf/cropcalStreamMod.F90 index a0e83a5995..1578bd023d 100644 --- a/src/cpl/share_esmf/cropcalStreamMod.F90 +++ b/src/cpl/share_esmf/cropcalStreamMod.F90 @@ -22,7 +22,7 @@ module cropcalStreamMod use clm_varpar , only : mxsowings use perf_mod , only : t_startf, t_stopf use spmdMod , only : masterproc, mpicom, iam - use pftconMod , only : npcropmin, is_prognostic_crop + use pftconMod , only : npcropmin, is_prognostic_crop, get_crop_n_from_veg_type use CNPhenologyMod , only : generate_crop_gdds ! ! !PUBLIC TYPES: @@ -589,7 +589,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ivt = patch%itype(p) ! Will skip generic crops if (is_prognostic_crop(ivt)) then - n = ivt - npcropmin + 1 + n = get_crop_n_from_veg_type(ivt) ! vegetated pft ig = g_to_ig(patch%gridcell(p)) swindow_starts(p,1) = dataptr2d_swindow_start(ig,n) @@ -671,7 +671,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ivt = patch%itype(p) ! Will skip generic crops if (is_prognostic_crop(ivt)) then - n = ivt - npcropmin + 1 + n = get_crop_n_from_veg_type(ivt) if (n > ncft) then write(iulog,'(a,i0,a,i0,a)') 'n (',n,') > ncft (',ncft,')' @@ -722,7 +722,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ivt = patch%itype(p) ! Will skip generic crops if (is_prognostic_crop(ivt)) then - n = ivt - npcropmin + 1 + n = get_crop_n_from_veg_type(ivt) if (n > ncft) then write(iulog,'(a,i0,a,i0,a)') 'n (',n,') > ncft (',ncft,')' @@ -792,7 +792,7 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ivt = patch%itype(p) ! Will skip generic crops if (is_prognostic_crop(ivt)) then - n = ivt - npcropmin + 1 + n = get_crop_n_from_veg_type(ivt) ! vegetated pft ig = g_to_ig(patch%gridcell(p)) diff --git a/src/main/pftconMod.F90 b/src/main/pftconMod.F90 index 94ca009ae9..8de10b355b 100644 --- a/src/main/pftconMod.F90 +++ b/src/main/pftconMod.F90 @@ -318,6 +318,7 @@ module pftconMod __FILE__ public :: is_prognostic_crop + public :: get_crop_n_from_veg_type !----------------------------------------------------------------------- contains @@ -1629,5 +1630,19 @@ elemental logical function is_prognostic_crop(veg_type) end function is_prognostic_crop + !----------------------------------------------------------------------- + elemental integer function get_crop_n_from_veg_type(veg_type) result(crop_n) + ! + ! !DESCRIPTION: + ! Given a vegetation type (pft, integer), return a 1-indexed number indicating where it would + ! be in a list of all simulated crops. + ! + ! !ARGUMENTS + integer, intent(in) :: veg_type + + crop_n = veg_type - npcropmin + 1 + + end function get_crop_n_from_veg_type + end module pftconMod From 0901503ecd38b81c261a6bc3cc7e7586a15141c6 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 6 Aug 2025 15:53:34 -0600 Subject: [PATCH 180/196] Add get_veg_type_from_crop_n(). --- src/cpl/share_esmf/cropcalStreamMod.F90 | 4 ++-- src/main/pftconMod.F90 | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/cpl/share_esmf/cropcalStreamMod.F90 b/src/cpl/share_esmf/cropcalStreamMod.F90 index 1578bd023d..792aafec5f 100644 --- a/src/cpl/share_esmf/cropcalStreamMod.F90 +++ b/src/cpl/share_esmf/cropcalStreamMod.F90 @@ -22,7 +22,7 @@ module cropcalStreamMod use clm_varpar , only : mxsowings use perf_mod , only : t_startf, t_stopf use spmdMod , only : masterproc, mpicom, iam - use pftconMod , only : npcropmin, is_prognostic_crop, get_crop_n_from_veg_type + use pftconMod , only : npcropmin, is_prognostic_crop, get_crop_n_from_veg_type, get_veg_type_from_crop_n use CNPhenologyMod , only : generate_crop_gdds ! ! !PUBLIC TYPES: @@ -144,7 +144,7 @@ subroutine cropcal_init(bounds) allocate(stream_varnames_gdd20_baseline(ncft)) allocate(stream_varnames_gdd20_season_enddate(ncft)) do n = 1,ncft - ivt = npcropmin + n - 1 + ivt = get_veg_type_from_crop_n(n) write(stream_varnames_sdate(n),'(a,i0)') "sdate1_",ivt write(stream_varnames_cultivar_gdds(n),'(a,i0)') "gdd1_",ivt write(stream_varnames_gdd20_baseline(n),'(a,i0)') "gdd20bl_",ivt diff --git a/src/main/pftconMod.F90 b/src/main/pftconMod.F90 index 8de10b355b..9511148be7 100644 --- a/src/main/pftconMod.F90 +++ b/src/main/pftconMod.F90 @@ -319,6 +319,7 @@ module pftconMod public :: is_prognostic_crop public :: get_crop_n_from_veg_type + public :: get_veg_type_from_crop_n !----------------------------------------------------------------------- contains @@ -1644,5 +1645,19 @@ elemental integer function get_crop_n_from_veg_type(veg_type) result(crop_n) end function get_crop_n_from_veg_type + !----------------------------------------------------------------------- + elemental integer function get_veg_type_from_crop_n(crop_n) result(veg_type) + ! + ! !DESCRIPTION: + ! Given a return a 1-indexed number indicating where a PFT would be in a list of all simulated + ! crops, return vegetation type (ivt) + ! + ! !ARGUMENTS + integer, intent(in) :: crop_n + + veg_type = npcropmin + crop_n - 1 + + end function get_veg_type_from_crop_n + end module pftconMod From c77a405e904d0c343f42b9e5e1e8b1d3cefa523c Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 6 Aug 2025 16:02:43 -0600 Subject: [PATCH 181/196] Add pftconMod public var num_cfts_possible. --- src/cpl/share_esmf/cropcalStreamMod.F90 | 45 ++++++++++++------------- src/main/pftconMod.F90 | 26 ++++++++++++++ 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/src/cpl/share_esmf/cropcalStreamMod.F90 b/src/cpl/share_esmf/cropcalStreamMod.F90 index 792aafec5f..591476e59e 100644 --- a/src/cpl/share_esmf/cropcalStreamMod.F90 +++ b/src/cpl/share_esmf/cropcalStreamMod.F90 @@ -22,7 +22,8 @@ module cropcalStreamMod use clm_varpar , only : mxsowings use perf_mod , only : t_startf, t_stopf use spmdMod , only : masterproc, mpicom, iam - use pftconMod , only : npcropmin, is_prognostic_crop, get_crop_n_from_veg_type, get_veg_type_from_crop_n + use pftconMod , only : is_prognostic_crop, get_crop_n_from_veg_type, get_veg_type_from_crop_n + use pftconMod , only : num_cfts_possible use CNPhenologyMod , only : generate_crop_gdds ! ! !PUBLIC TYPES: @@ -46,7 +47,6 @@ module cropcalStreamMod character(len=CS), allocatable :: stream_varnames_cultivar_gdds(:) character(len=CS), allocatable :: stream_varnames_gdd20_baseline(:) character(len=CS), allocatable :: stream_varnames_gdd20_season_enddate(:) ! start uses stream_varnames_sdate - integer :: ncft ! Number of crop functional types (excl. generic crops) logical :: allow_invalid_swindow_inputs ! Fall back on paramfile sowing windows in cases of invalid values in stream_fldFileName_swindow_start and _end? character(len=FL) :: stream_fldFileName_swindow_start ! sowing window start stream filename to read character(len=FL) :: stream_fldFileName_swindow_end ! sowing window end stream filename to read @@ -138,12 +138,11 @@ subroutine cropcal_init(bounds) stream_fldFileName_gdd20_season_start = '' stream_fldFileName_gdd20_season_end = '' ! Will need modification to work with mxsowings > 1 - ncft = mxpft - npcropmin + 1 ! Ignores generic crops - allocate(stream_varnames_sdate(ncft)) - allocate(stream_varnames_cultivar_gdds(ncft)) - allocate(stream_varnames_gdd20_baseline(ncft)) - allocate(stream_varnames_gdd20_season_enddate(ncft)) - do n = 1,ncft + allocate(stream_varnames_sdate(num_cfts_possible)) + allocate(stream_varnames_cultivar_gdds(num_cfts_possible)) + allocate(stream_varnames_gdd20_baseline(num_cfts_possible)) + allocate(stream_varnames_gdd20_season_enddate(num_cfts_possible)) + do n = 1,num_cfts_possible ivt = get_veg_type_from_crop_n(n) write(stream_varnames_sdate(n),'(a,i0)') "sdate1_",ivt write(stream_varnames_cultivar_gdds(n),'(a,i0)') "gdd1_",ivt @@ -201,7 +200,7 @@ subroutine cropcal_init(bounds) write(iulog,'(a,l1)') ' allow_invalid_gdd20_season_inputs = ',allow_invalid_gdd20_season_inputs write(iulog,'(a,a)' ) ' stream_fldFileName_gdd20_season_start = ',stream_fldFileName_gdd20_season_start write(iulog,'(a,a)' ) ' stream_fldFileName_gdd20_season_end = ',stream_fldFileName_gdd20_season_end - do n = 1,ncft + do n = 1,num_cfts_possible write(iulog,'(a,a)' ) ' stream_varnames_sdate = ',trim(stream_varnames_sdate(n)) write(iulog,'(a,a)' ) ' stream_varnames_cultivar_gdds = ',trim(stream_varnames_cultivar_gdds(n)) write(iulog,'(a,a)' ) ' stream_varnames_gdd20_season_enddate = ',trim(stream_varnames_gdd20_season_enddate(n)) @@ -550,13 +549,13 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) dayspyr = get_curr_days_per_year() ! Read prescribed sowing window start dates from input files - allocate(dataptr2d_swindow_start(begg:endg, ncft)) + allocate(dataptr2d_swindow_start(begg:endg, num_cfts_possible)) dataptr2d_swindow_start(begg:endg,:) = -1._r8 - allocate(dataptr2d_swindow_end (begg:endg, ncft)) + allocate(dataptr2d_swindow_end (begg:endg, num_cfts_possible)) dataptr2d_swindow_end(begg:endg,:) = -1._r8 if (use_cropcal_rx_swindows) then ! Starting with npcropmin will skip generic crops - do n = 1, ncft + do n = 1, num_cfts_possible call dshr_fldbun_getFldPtr(sdat_cropcal_swindow_start%pstrm(1)%fldbun_model, trim(stream_varnames_sdate(n)), & fldptr1=dataptr1d_swindow_start, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) then @@ -640,11 +639,11 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) deallocate(dataptr2d_swindow_start) deallocate(dataptr2d_swindow_end) - allocate(dataptr2d_cultivar_gdds(begg:endg, ncft)) + allocate(dataptr2d_cultivar_gdds(begg:endg, num_cfts_possible)) if (use_cropcal_rx_cultivar_gdds) then ! Read prescribed cultivar GDDs from input files ! Starting with npcropmin will skip generic crops - do n = 1, ncft + do n = 1, num_cfts_possible call dshr_fldbun_getFldPtr(sdat_cropcal_cultivar_gdds%pstrm(1)%fldbun_model, trim(stream_varnames_cultivar_gdds(n)), & fldptr1=dataptr1d_cultivar_gdds, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) then @@ -673,8 +672,8 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) if (is_prognostic_crop(ivt)) then n = get_crop_n_from_veg_type(ivt) - if (n > ncft) then - write(iulog,'(a,i0,a,i0,a)') 'n (',n,') > ncft (',ncft,')' + if (n > num_cfts_possible) then + write(iulog,'(a,i0,a,i0,a)') 'n (',n,') > ncft (',num_cfts_possible,')' call ESMF_Finalize(endflag=ESMF_END_ABORT) end if @@ -697,11 +696,11 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) deallocate(dataptr2d_cultivar_gdds) - allocate(dataptr2d_gdd20_baseline(begg:endg, ncft)) + allocate(dataptr2d_gdd20_baseline(begg:endg, num_cfts_possible)) if (adapt_cropcal_rx_cultivar_gdds) then ! Read GDD20 baselines from input files ! Starting with npcropmin will skip generic crops - do n = 1, ncft + do n = 1, num_cfts_possible call dshr_fldbun_getFldPtr(sdat_cropcal_gdd20_baseline%pstrm(1)%fldbun_model, trim(stream_varnames_gdd20_baseline(n)), & fldptr1=dataptr1d_gdd20_baseline, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) then @@ -724,8 +723,8 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) if (is_prognostic_crop(ivt)) then n = get_crop_n_from_veg_type(ivt) - if (n > ncft) then - write(iulog,'(a,i0,a,i0,a)') 'n (',n,') > ncft (',ncft,')' + if (n > num_cfts_possible) then + write(iulog,'(a,i0,a,i0,a)') 'n (',n,') > ncft (',num_cfts_possible,')' call ESMF_Finalize(endflag=ESMF_END_ABORT) end if @@ -750,13 +749,13 @@ subroutine cropcal_interp(bounds, num_pcropp, filter_pcropp, init, crop_inst) ! Read prescribed gdd20 season start dates from input files - allocate(dataptr2d_gdd20_season_start(begg:endg, ncft)) + allocate(dataptr2d_gdd20_season_start(begg:endg, num_cfts_possible)) dataptr2d_gdd20_season_start(begg:endg,:) = -1._r8 - allocate(dataptr2d_gdd20_season_end (begg:endg, ncft)) + allocate(dataptr2d_gdd20_season_end (begg:endg, num_cfts_possible)) dataptr2d_gdd20_season_end(begg:endg,:) = -1._r8 if (stream_gdd20_seasons) then ! Starting with npcropmin will skip generic crops - do n = 1, ncft + do n = 1, num_cfts_possible call dshr_fldbun_getFldPtr(sdat_cropcal_gdd20_season_start%pstrm(1)%fldbun_model, trim(stream_varnames_sdate(n)), & fldptr1=dataptr1d_gdd20_season_start, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) then diff --git a/src/main/pftconMod.F90 b/src/main/pftconMod.F90 index 9511148be7..a64edc4488 100644 --- a/src/main/pftconMod.F90 +++ b/src/main/pftconMod.F90 @@ -107,6 +107,9 @@ module pftconMod ! at all, as given by the mergetoclmpft list. integer, public :: num_cfts_known_to_model + ! Number of prognostic crop functional types on the parameter file, even if not actually used + integer, public :: num_cfts_possible + ! !PUBLIC TYPES: type, public :: pftcon_type @@ -295,6 +298,7 @@ module pftconMod procedure, private :: InitRead procedure, private :: set_is_pft_known_to_model ! Set is_pft_known_to_model based on mergetoclmpft procedure, private :: set_num_cfts_known_to_model ! Set the module-level variable, num_cfts_known_to_model + procedure, private :: set_num_cfts_possible ! Set the module-level variable, num_cfts_possible end type pftcon_type @@ -1251,6 +1255,7 @@ subroutine InitRead(this) call this%set_is_pft_known_to_model() call this%set_num_cfts_known_to_model() + call this%set_num_cfts_possible() ! Set vegetation family identifier (tree/shrub/grass) do m = 0,mxpft @@ -1443,6 +1448,27 @@ subroutine set_num_cfts_known_to_model(this) end subroutine set_num_cfts_known_to_model + !----------------------------------------------------------------------- + subroutine set_num_cfts_possible(this) + ! + ! !DESCRIPTION: + ! Set the module-level variable, num_cfts_possible + ! + ! !USES: + ! + ! !ARGUMENTS: + class(pftcon_type), intent(in) :: this + ! + ! !LOCAL VARIABLES: + integer :: m + + character(len=*), parameter :: subname = 'set_num_cfts_possible' + !----------------------------------------------------------------------- + + num_cfts_possible = npcropmax - npcropmin + 1 + + end subroutine set_num_cfts_possible + !----------------------------------------------------------------------- subroutine Clean(this) ! From ff4ead6d68b3b256c545c08f9f16abeb6e6eddaf Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 6 Aug 2025 16:03:58 -0600 Subject: [PATCH 182/196] npcropmin/max are now private to pftconMod. --- src/main/pftconMod.F90 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/pftconMod.F90 b/src/main/pftconMod.F90 index a64edc4488..a39a30b9e2 100644 --- a/src/main/pftconMod.F90 +++ b/src/main/pftconMod.F90 @@ -34,7 +34,6 @@ module pftconMod integer, public :: nc3_arctic_grass ! value for C3 arctic grass integer, public :: nc3_nonarctic_grass ! value for C3 non-arctic grass integer, public :: nc4_grass ! value for C4 grass - integer, public :: npcropmin ! value for first crop integer, public :: ntmp_corn ! value for temperate corn, rain fed (rf) integer, public :: nirrig_tmp_corn ! value for temperate corn, irrigated (ir) integer, public :: nswheat ! value for spring temperate cereal (rf) @@ -97,10 +96,13 @@ module pftconMod integer, public :: nirrig_trp_corn !value for tropical corn (ir) integer, public :: ntrp_soybean !value for tropical soybean (rf) integer, public :: nirrig_trp_soybean !value for tropical soybean (ir) - integer, public :: npcropmax ! value for last prognostic crop in list integer, public :: nc3crop ! value for generic crop (rf) integer, public :: nc3irrig ! value for irrigated generic crop (ir) + ! First and last prognostic crops + integer :: npcropmin ! value for first crop + integer :: npcropmax ! value for last prognostic crop in list + ! Number of crop functional types actually used in the model. This includes each CFT for ! which is_pft_known_to_model is true. Note that this includes irrigated crops even if ! irrigation is turned off in this run: it just excludes crop types that aren't handled From 8920ed74ad3cdaddb6a5a06180120cdc4086f20f Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 18 Aug 2025 12:38:51 -0600 Subject: [PATCH 183/196] CNFUNMod: Improve readability of a line. Co-authored-by: Samuel Levis --- src/biogeochem/CNFUNMod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/biogeochem/CNFUNMod.F90 b/src/biogeochem/CNFUNMod.F90 index e264786c0b..b8214eac22 100644 --- a/src/biogeochem/CNFUNMod.F90 +++ b/src/biogeochem/CNFUNMod.F90 @@ -1271,7 +1271,7 @@ subroutine CNFUN(bounds,num_soilc, filter_soilc,num_soilp& ! Calculate appropriate degree of retranslocation !------------------------------------------------------------------------------- - if(leafc(p).gt.0.0_r8.and.litterfall_n_step(p,istp)* fixerfrac>0.0_r8.and. (.not. is_prognostic_crop(ivt(p))))then + if (leafc(p) > 0.0_r8 .and. (litterfall_n_step(p,istp) * fixerfrac) > 0.0_r8 .and. (.not. is_prognostic_crop(ivt(p)))) then call fun_retranslocation(p,dt,npp_to_spend,& litterfall_c_step(p,istp)* fixerfrac,& litterfall_n_step(p,istp)* fixerfrac,& From c20859f839d717e8c78a2bbf0fc5ce98fc89a989 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Tue, 19 Aug 2025 10:44:15 -0600 Subject: [PATCH 184/196] is_prognostic_crop(): Improve note. --- src/main/pftconMod.F90 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/pftconMod.F90 b/src/main/pftconMod.F90 index a39a30b9e2..0ed2028fcb 100644 --- a/src/main/pftconMod.F90 +++ b/src/main/pftconMod.F90 @@ -1647,10 +1647,9 @@ elemental logical function is_prognostic_crop(veg_type) ! Given a vegetation type (pft, integer), return whether it's a prognostic crop. Does not ! include generic crops (those and natural PFTs will return .false.). ! - ! NOTE(wjs, 2017-02-02) This isn't a completely robust way to check if this is a - ! prognostic crop patch (at the very least it should also check if <= npcropmax; - ! ideally it should use a prognostic_crop flag that doesn't seem to exist - ! currently). + ! NOTE: This isn't a completely robust way to check if this is a prognostic crop patch. At the + ! very least, it should also check if <= npcropmax. Ideally it would use a new prognostic_crop + ! flag on the parameter file iteself. ! ! !ARGUMENTS integer, intent(in) :: veg_type From d685e6f70ec453494f2b04891ef26b38d115c706 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 19 Aug 2025 17:19:03 -0600 Subject: [PATCH 185/196] Change the directory name for the new endrun tests --- src/main/test/{endrun_test => abortutils_test}/CMakeLists.txt | 0 src/main/test/{endrun_test => abortutils_test}/test_endrun.pf | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/main/test/{endrun_test => abortutils_test}/CMakeLists.txt (100%) rename src/main/test/{endrun_test => abortutils_test}/test_endrun.pf (100%) diff --git a/src/main/test/endrun_test/CMakeLists.txt b/src/main/test/abortutils_test/CMakeLists.txt similarity index 100% rename from src/main/test/endrun_test/CMakeLists.txt rename to src/main/test/abortutils_test/CMakeLists.txt diff --git a/src/main/test/endrun_test/test_endrun.pf b/src/main/test/abortutils_test/test_endrun.pf similarity index 100% rename from src/main/test/endrun_test/test_endrun.pf rename to src/main/test/abortutils_test/test_endrun.pf From 5ac7e3b07b571d64559a35248a29e0167d5461bc Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 19 Aug 2025 17:22:04 -0600 Subject: [PATCH 186/196] Change the names from the specific subroutine endrun to abortutils the module name --- src/main/test/CMakeLists.txt | 2 +- src/main/test/abortutils_test/CMakeLists.txt | 2 +- .../abortutils_test/{test_endrun.pf => test_abortutils.pf} | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename src/main/test/abortutils_test/{test_endrun.pf => test_abortutils.pf} (99%) diff --git a/src/main/test/CMakeLists.txt b/src/main/test/CMakeLists.txt index 2ade2f6e87..bf8c164260 100644 --- a/src/main/test/CMakeLists.txt +++ b/src/main/test/CMakeLists.txt @@ -8,4 +8,4 @@ add_subdirectory(filter_test) add_subdirectory(initVertical_test) add_subdirectory(ncdio_utils_test) add_subdirectory(topo_test) -add_subdirectory(endrun_test) +add_subdirectory(abortutils_test) diff --git a/src/main/test/abortutils_test/CMakeLists.txt b/src/main/test/abortutils_test/CMakeLists.txt index 45b42a15e7..4b49e744f2 100644 --- a/src/main/test/abortutils_test/CMakeLists.txt +++ b/src/main/test/abortutils_test/CMakeLists.txt @@ -1,5 +1,5 @@ set(pfunit_sources - test_endrun.pf) + test_abortutils.pf) add_pfunit_ctest(endrun TEST_SOURCES "${pfunit_sources}" diff --git a/src/main/test/abortutils_test/test_endrun.pf b/src/main/test/abortutils_test/test_abortutils.pf similarity index 99% rename from src/main/test/abortutils_test/test_endrun.pf rename to src/main/test/abortutils_test/test_abortutils.pf index 6dfec9b799..2b01d67d9c 100644 --- a/src/main/test/abortutils_test/test_endrun.pf +++ b/src/main/test/abortutils_test/test_abortutils.pf @@ -1,4 +1,4 @@ -module test_endrun +module test_abortutils ! Tests of abortutils @@ -197,4 +197,4 @@ contains end subroutine endrun_nomsg_pt_context_badlvl_aborts -end module test_endrun +end module test_abortutils From 854a364f2ffab0889b07aa47ccbe66ba2440d674 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 19 Aug 2025 17:30:16 -0600 Subject: [PATCH 187/196] Move the descriptions of test subroutines to just after the subroutine statement, for notes that are specific to a line in the subroutine, add NOTE: to most of them in response to review --- src/main/test/abortutils_test/test_abortutils.pf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/test/abortutils_test/test_abortutils.pf b/src/main/test/abortutils_test/test_abortutils.pf index 2b01d67d9c..bf1f7babd2 100644 --- a/src/main/test/abortutils_test/test_abortutils.pf +++ b/src/main/test/abortutils_test/test_abortutils.pf @@ -28,11 +28,11 @@ contains use GridcellType , only : grc class(TestAbortUtils), intent(inout) :: this - ! Setup a single gridcell with one vegetated patch + ! NOTE: Setup a single gridcell with one vegetated patch ! So there's only one: gridcell, landunit, column, patch ! This isn't needed for some tests, but doesn't hurt to do it call setup_single_veg_patch(pft_type=1) - ! Set lat and lon for this gridcell, so something is printed in the log + ! NOTE: Set lat and lon for this gridcell, so something is printed in the log grc%londeg(1) = 255.0 grc%latdeg(1) = 30.0 end subroutine setUp @@ -104,10 +104,10 @@ contains @Test subroutine endrun_addmsg_pt_context_aborts(this) + ! Test pt_context operation of endrun with an additional message sent in use decompMod, only : subgrid_level_lndgrid, subgrid_level_gridcell use decompMod, only : subgrid_level_landunit, subgrid_level_column, subgrid_level_patch use decompMod, only : subgrid_level_cohort - ! Test pt_context operation of endrun with an additional message sent in class(TestAbortUtils), intent(inout) :: this character(len=CL) :: msg = "test_message" character(len=CL) :: add_msg = "additional_test_message" @@ -128,10 +128,10 @@ contains @Test subroutine endrun_nomsg_pt_context_bad_pt_aborts(this) + ! Test pt_context operation of endrun with an additional message sent in use decompMod, only : subgrid_level_lndgrid, subgrid_level_gridcell use decompMod, only : subgrid_level_landunit, subgrid_level_column, subgrid_level_patch use decompMod, only : subgrid_level_cohort - ! Test pt_context operation of endrun with an additional message sent in class(TestAbortUtils), intent(inout) :: this integer :: p = 2, l integer, parameter :: nlevel = 6 @@ -155,7 +155,7 @@ contains character(len=CL) :: msg = "test_message" integer :: p = 1 - ! Also test without an additional msg + ! NOTE: Also test without an additional msg call endrun(subgrid_index=p, subgrid_level=subgrid_level_lndgrid, msg=msg) @assertExceptionRaised(endrun_msg(msg)) @@ -167,7 +167,7 @@ contains class(TestAbortUtils), intent(inout) :: this integer :: p = 1 - ! Also test without either msg or additional msg + ! NOTE: Also test without either msg or additional msg call endrun(subgrid_index=p, subgrid_level=subgrid_level_cohort) @assertExceptionRaised(endrun_msg('')) @@ -180,7 +180,7 @@ contains integer :: p = 1 character(len=CL) :: add_msg = "additional_test_message" - ! Don't use msg but do use additional_msg + ! NOTE: Don't use msg but do use additional_msg call endrun(subgrid_index=p, subgrid_level=subgrid_level_unspecified, additional_msg=add_msg) @assertExceptionRaised(endrun_msg('')) From 5b40496935e86e3a1a8237fb493cd2369cb26f94 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 20 Aug 2025 17:28:27 -0600 Subject: [PATCH 188/196] Reformat with black. --- python/ctsm/crop_calendars/generate_gdds.py | 4 +--- python/ctsm/crop_calendars/import_ds.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/python/ctsm/crop_calendars/generate_gdds.py b/python/ctsm/crop_calendars/generate_gdds.py index cb69d9e230..308431d003 100644 --- a/python/ctsm/crop_calendars/generate_gdds.py +++ b/python/ctsm/crop_calendars/generate_gdds.py @@ -280,9 +280,7 @@ def make_dummy(this_crop_gridded, addend): for var_index, this_var in enumerate(dummy_vars): if this_var in gdd_maps_ds: - error( - logger, f"{this_var} is already in gdd_maps_ds. Why overwrite it with dummy?" - ) + error(logger, f"{this_var} is already in gdd_maps_ds. Why overwrite it with dummy?") dummy_gridded.name = this_var dummy_gridded.attrs["long_name"] = dummy_longnames[var_index] gdd_maps_ds[this_var] = dummy_gridded diff --git a/python/ctsm/crop_calendars/import_ds.py b/python/ctsm/crop_calendars/import_ds.py index d2ace51ef0..66a0ec9746 100644 --- a/python/ctsm/crop_calendars/import_ds.py +++ b/python/ctsm/crop_calendars/import_ds.py @@ -254,9 +254,7 @@ def import_ds( filetime_sel = utils.safer_timeslice(filetime, time_slice) include_this_file = filetime_sel.size if include_this_file: - log( - logger, f"Including filetime : {filetime_sel['time'].values}" - ) + log(logger, f"Including filetime : {filetime_sel['time'].values}") new_filelist.append(file) # If you found some matching files, but then you find one that doesn't, stop going From a6e2309ed85da74d6d9111446761eea317509c38 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 20 Aug 2025 17:28:45 -0600 Subject: [PATCH 189/196] Add previous commit to .git-blame-ignore-revs. --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index ae6a568078..e56311635e 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -74,3 +74,4 @@ cdf40d265cc82775607a1bf25f5f527bacc97405 62d7711506a0fb9a3ad138ceceffbac1b79a6caa 49ad0f7ebe0b07459abc00a5c33c55a646f1e7e0 ac03492012837799b7111607188acff9f739044a +d858665d799690d73b56bcb961684382551193f4 From 22a56e532cfe45ee14a406a7bd603817a480cabd Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 20 Aug 2025 17:29:32 -0600 Subject: [PATCH 190/196] Small change for pylint. --- python/ctsm/test/test_unit_ctsm_logging.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ctsm/test/test_unit_ctsm_logging.py b/python/ctsm/test/test_unit_ctsm_logging.py index edfd817a46..d1b4891b17 100755 --- a/python/ctsm/test/test_unit_ctsm_logging.py +++ b/python/ctsm/test/test_unit_ctsm_logging.py @@ -5,7 +5,6 @@ import unittest import io from contextlib import redirect_stdout -from datetime import datetime from ctsm import unit_testing from ctsm.ctsm_logging import log, error From 1a879d9802882e49e0ad97bd74158fd0551e6c57 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 20 Aug 2025 21:11:22 -0600 Subject: [PATCH 191/196] set_paramfile: Test & document changing 1-d non-pft param. --- .../using-clm-tools/paramfile-tools.md | 7 ++- python/ctsm/test/test_sys_set_paramfile.py | 46 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/doc/source/users_guide/using-clm-tools/paramfile-tools.md b/doc/source/users_guide/using-clm-tools/paramfile-tools.md index d63763a110..71d881e362 100644 --- a/doc/source/users_guide/using-clm-tools/paramfile-tools.md +++ b/doc/source/users_guide/using-clm-tools/paramfile-tools.md @@ -47,11 +47,16 @@ For more information, do `tools/param_utils/set_paramfile --help`. ### Example usage -Change a parameter for all PFTs: +Change a scalar parameter: ```bash tools/param_utils/set_paramfile -i paramfile.nc -o output.nc jmaxha=51000 ``` +Change a one-dimensional parameter (`mimics_fmet` has the `segment` dimension, length 4): +```bash +tools/param_utils/set_paramfile -i paramfile.nc -o output.nc mimics_fmet=0.1,0.2,0.3,0.4 +``` + Change a parameter for specific PFTs: ```bash tools/param_utils/set_paramfile -i paramfile.nc -o output.nc -p needleleaf_evergreen_temperate_tree,c4_grass medlynintercept=99.9,100.1 medlynslope=2.99,1.99 diff --git a/python/ctsm/test/test_sys_set_paramfile.py b/python/ctsm/test/test_sys_set_paramfile.py index 0579655ef4..7d82a32302 100755 --- a/python/ctsm/test/test_sys_set_paramfile.py +++ b/python/ctsm/test/test_sys_set_paramfile.py @@ -179,6 +179,52 @@ def test_set_paramfile_changeparams_scalar_double(self): if not np.isnan(fv_in): self.assertEqual(fv_in, fv_out) + def test_set_paramfile_changeparams_1d_double(self): + """ + Test that set_paramfile can copy to a new file with a 1-d double param changed (not PFT- + dimensioned) + """ + output_path = os.path.join(self.tempdir, "output.nc") + this_var = "mimics_fmet" + sys.argv = [ + "set_paramfile", + "-i", + PARAMFILE, + "-o", + output_path, + f"{this_var}=0.1,0.2,0.3,0.4", + ] + sp.main() + self.assertTrue(os.path.exists(output_path)) + ds_in = open_paramfile(PARAMFILE) + ds_out = open_paramfile(output_path) + + for var in ds_in.variables: + # Check that all variables/coords are equal except the ones we changed, which should be + # set to what we asked + if var == this_var: + self.assertTrue( + np.array_equal(ds_in[var].values, np.array([0.75, 0.85, 0.013, 40])) + ) + self.assertTrue(np.array_equal(ds_out[var].values, np.array([0.1, 0.2, 0.3, 0.4]))) + else: + self.assertTrue(are_paramfile_dataarrays_identical(ds_in[var], ds_out[var])) + + # Check that data type hasn't changed + self.assertTrue(ds_in[var].dtype == ds_out[var].dtype) + + # Check that fill value hasn't changed + if "_FillValue" in ds_in[var].encoding: + fv_in = ds_in[var].encoding["_FillValue"] + fv_out = ds_out[var].encoding["_FillValue"] + if isinstance(fv_in, bytes): + self.assertTrue(isinstance(fv_out, bytes)) + self.assertEqual(fv_in, fv_out) + else: + self.assertEqual(np.isnan(fv_in), np.isnan(fv_out)) + if not np.isnan(fv_in): + self.assertEqual(fv_in, fv_out) + def test_set_paramfile_changeparams_scalar_int(self): """Test that set_paramfile can copy to a new file with a scalar integer param changed""" output_path = os.path.join(self.tempdir, "output.nc") From 11147fe9caad81f9238b6a9a27c2d3f4a9bb9ef1 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 21 Aug 2025 01:09:34 -0600 Subject: [PATCH 192/196] Correct name of RTM_MODE --- .../clm/for_testing_fastsetup_bypassrun/shell_commands | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/shell_commands b/cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/shell_commands index f3144c219d..0d8a5d36e1 100755 --- a/cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/shell_commands +++ b/cime_config/testdefs/testmods_dirs/clm/for_testing_fastsetup_bypassrun/shell_commands @@ -5,7 +5,7 @@ ./xmlchange ROF_NCPL='$ATM_NCPL' # Turn off ROF model when used with compsets that have them -./xmlchange ROF_MODE='NULL' +./xmlchange RTM_MODE='NULL' # Turn MEGAN off to run faster ./xmlchange CLM_BLDNML_OPTS='--no-megan' --append From b39e405bc93eec5c38d99a9781f6a59b9617dedc Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 21 Aug 2025 01:12:52 -0600 Subject: [PATCH 193/196] DATM seems to need more tasks for it to run, so change the mpasa2p75 layout so it's sequential all on the same 80 tasks --- cime_config/config_pes.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cime_config/config_pes.xml b/cime_config/config_pes.xml index d0794339db..76372a0a76 100644 --- a/cime_config/config_pes.xml +++ b/cime_config/config_pes.xml @@ -2098,7 +2098,7 @@ none - -1 + -80 -80 -80 -80 @@ -2120,13 +2120,13 @@ 0 - -1 - -1 - -1 - -1 - -1 - -1 - -1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 From 5b0d5a2cdcdc9fda357eb500b733343154875a15 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 21 Aug 2025 01:32:40 -0600 Subject: [PATCH 194/196] Correct the path variable to DIN_LOC_ROOT --- cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm b/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm index 69a3a55836..d9023871b5 100644 --- a/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm +++ b/cime_config/testdefs/testmods_dirs/clm/mpasa3p75/user_nl_clm @@ -1,6 +1,6 @@ ! Settings currently required to run at the mpasa3p75 grid ! urbantv files at that resolution and use a redistribution mapping -stream_fldfilename_urbantv = '$CSMDATA/lnd/clm2/urbandata/CTSM52_tbuildmax_OlesonFeddema_2020_mpasa3p75_fromf09_simyr1849-2106_c20240502.nc' +stream_fldfilename_urbantv = '$DIN_LOC_ROOT/lnd/clm2/urbandata/CTSM52_tbuildmax_OlesonFeddema_2020_mpasa3p75_fromf09_simyr1849-2106_c20240502.nc' stream_meshfile_urbantv = '$DIN_LOC_ROOT/lnd/clm2/urbandata/CTSM52_tbuildmax_Oleson_2020_mpasa3p75_ESMFmesh_cdf5_c202405021.nc' urbantvmapalgo = 'redist' From 30a7f35bda9fcd11578cf6f35d031aba7389a189 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 22 Aug 2025 14:05:57 -0600 Subject: [PATCH 195/196] Update ChangeLog and ChangeSum. --- doc/ChangeLog | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 80 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 850c5fd259..733ade6ad2 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,83 @@ =============================================================== +Tag name: ctsm5.3.071 +Originator(s): samrabin (Sam Rabin, UCAR/TSS) +Date: Fri Aug 22 13:49:25 MDT 2025 +One-line Summary: Merge b4b-dev to master + +Purpose and description of changes +---------------------------------- + +Merge b4b-dev to master. + + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm6_0 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed +---------- + +List of CTSM issues fixed (include CTSM Issue # and description): +- [Issue #3375: Enabling running an initialization test at mpasa3p75](https://github.com/ESCOMP/CTSM/issues/3375) +- [Issue #3402: py_env_create and a test broken after Derecho updates to conda/mamba](https://github.com/ESCOMP/CTSM/issues/3402) +- [Issue #3417: Use shr_abort_mod for endrun, and add optional arguments for file and line](https://github.com/ESCOMP/CTSM/issues/3417) +- [Issue #3420: Don't have endrun abort on bad subgrid_level](https://github.com/ESCOMP/CTSM/issues/3420) + + +Notes of particular relevance for users +--------------------------------------- + +Changes to documentation: +- Docs added for new query_paramfile and set_paramfile tools. + + +Notes of particular relevance for developers: +--------------------------------------------- + +Changes to tests or testing: +- New SETPARAMFILE SystemTest, one of which has been added to aux_clm and clm_pymods +- Adds a test with resolution mpasa3p75_mpasa3p75_mt13 to new uhr_decomp_init suite (of which it's the only member) +- run_self_tests testmod changed; replaced in some tests by new for_testing_fastsetup_bypassrun testmod. + + +Testing summary: +---------------- + + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + derecho ----- + izumi ------- + + +Other details +------------- + +Pull Requests that document the changes (include PR ids): +- [Pull Request #3403: Fix py_env_create and tests by samsrabin](https://github.com/ESCOMP/CTSM/pull/3403) +- [Pull Request #3390: Make npcropmin/max private to pftconMod by samsrabin](https://github.com/ESCOMP/CTSM/pull/3390) +- [Pull Request #3418: Endrun work by ekluzek](https://github.com/ESCOMP/CTSM/pull/3418) +- [Pull Request #3408: Logging improvements for GDD-generation workflow by samsrabin](https://github.com/ESCOMP/CTSM/pull/3408) +- [Pull Request #3397: New tools: query_paramfile and set_paramfile by samsrabin](https://github.com/ESCOMP/CTSM/pull/3397) +- [Pull Request #3413: Add a test for Mpasa3p75 by ekluzek](https://github.com/ESCOMP/CTSM/pull/3413) +- [Pull Request #3431: b4b-dev merge 2025-08-22 by samsrabin](https://github.com/ESCOMP/CTSM/pull/3431) + +=============================================================== +=============================================================== Tag name: ctsm5.3.070 Originator(s): glemieux (Gregory Lemieux, LBNL, glemieux@lbl.gov) Date: Fri 22 Aug 2025 02:29:15 AM MDT diff --git a/doc/ChangeSum b/doc/ChangeSum index b501ec82be..f01334b0e4 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.3.071 samrabin 08/22/2025 Merge b4b-dev to master ctsm5.3.070 glemieux 08/22/2025 Update default FATES parameter file and add FATES managed fire namelist option ctsm5.3.069 samrabin 08/12/2025 Add SystemTests to run subset_data and then CTSM ctsm5.3.068 slevis 08/11/2025 Change megan_use_gamma_sm to default false From 0f90c77c89316f6498a21bdb73665194560e5981 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Fri, 22 Aug 2025 22:19:33 -0600 Subject: [PATCH 196/196] Finalize ctsm5.3.071 ChangeLog. --- doc/ChangeLog | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/doc/ChangeLog b/doc/ChangeLog index 733ade6ad2..428ab62539 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -16,8 +16,6 @@ Significant changes to scientifically-supported configurations Does this tag change answers significantly for any of the following physics configurations? (Details of any changes will be given in the "Answer changes" section below.) - [Put an [X] in the box for any configuration with significant answer changes.] - [ ] clm6_0 [ ] clm5_0 @@ -60,8 +58,15 @@ Testing summary: regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): - derecho ----- - izumi ------- + derecho ----- DIFF + izumi ------- OK + + +Answer changes +-------------- + +Changes answers relative to baseline: +- SMS_D_Ln1.f10_f10_mg37.I2000Clm50BgcCropQianRs.derecho_intel.clm-run_self_tests changes answers due to differences in user_nl_clm and shell_commands. Other details