From 6bf022ddf8b8fea057e5da011a78807169ea254c Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Sat, 12 Dec 2020 15:45:56 +0000 Subject: [PATCH 01/10] Point to the feature/hafs_couplehycom branch for hafs_forecast.fd. --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 1e9966cb0..b59c5332e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "ufs-weather-model"] path = sorc/hafs_forecast.fd url = https://github.com/hafs-community/ufs-weather-model.git - branch = support/HAFS + branch = feature/hafs_couplehycom [submodule "EMC_post"] path = sorc/hafs_post.fd url = https://github.com/hafs-community/EMC_post.git From ae48fbb7d9a5a961229525b069de6bef152ae94e Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Thu, 21 Jan 2021 14:56:31 -0600 Subject: [PATCH 02/10] Update hafs_forecast.fd to point the latest commit of its feature/hafs_couplehycom branch. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 730d968cc..883303404 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 730d968cc4caa6d0fcc280a85c20a913245b88b9 +Subproject commit 88330340413564c6f9a64318d55e24a5cbd7529b From 49db75f0a1c8c43bdd9548b248b07328650021c4 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Tue, 26 Jan 2021 08:49:10 -0600 Subject: [PATCH 03/10] A. Update hafs_forecast.fd to point the latest commit of the feature/hafs_couplehycom branch, which has the CMEPS based HYCOM coupling capabilities. B. Update the workflow to support the CMEPS-based HYCOM coupling --- parm/forecast/globnest/nems.configure.atmonly | 13 +++ .../regional/nems.configure.atm_ocn.tmp | 13 ++- .../regional/nems.configure.atm_ocn_cmeps.tmp | 98 +++++++++++++++++++ parm/forecast/regional/nems.configure.atmonly | 13 +++ parm/hafs.conf | 21 ++-- parm/hafs_holdvars.conf | 7 -- parm/hafs_holdvars.txt | 7 -- parm/hafs_hycom.conf | 35 ++----- rocoto/cronjob_hafs_hycom.sh | 15 ++- rocoto/cronjob_hafs_rt.sh | 26 +++-- scripts/exhafs_forecast.sh | 78 ++++++++++++--- sorc/build_forecast.sh | 2 +- ush/hafs/launcher.py | 59 ----------- 13 files changed, 246 insertions(+), 141 deletions(-) create mode 100644 parm/forecast/regional/nems.configure.atm_ocn_cmeps.tmp diff --git a/parm/forecast/globnest/nems.configure.atmonly b/parm/forecast/globnest/nems.configure.atmonly index 88633a425..fe59cc645 100644 --- a/parm/forecast/globnest/nems.configure.atmonly +++ b/parm/forecast/globnest/nems.configure.atmonly @@ -1,5 +1,18 @@ EARTH_component_list: ATM + ATM_model: fv3 + runSeq:: ATM :: + +ATM_attributes:: +:: + +DRIVER_attributes:: + start_type = startup +:: + +ALLCOMP_attributes:: + mediator_read_restart = false +:: diff --git a/parm/forecast/regional/nems.configure.atm_ocn.tmp b/parm/forecast/regional/nems.configure.atm_ocn.tmp index cf5070020..c9f0d56bf 100644 --- a/parm/forecast/regional/nems.configure.atm_ocn.tmp +++ b/parm/forecast/regional/nems.configure.atm_ocn.tmp @@ -12,10 +12,7 @@ _OCN_petlist_bounds_ # run sequence runSeq:: @_cpl_dt_ - _runSeq_OCN2ATM_ - _runSeq_ATM2OCN_ - _runSeq_ATM_ - _runSeq_OCN_ + _runSeq_ALL_ @ :: @@ -49,3 +46,11 @@ OCN_attributes:: end_min = 0 end_sec = 0 :: + +DRIVER_attributes:: + start_type = startup +:: + +ALLCOMP_attributes:: + mediator_read_restart = false +:: diff --git a/parm/forecast/regional/nems.configure.atm_ocn_cmeps.tmp b/parm/forecast/regional/nems.configure.atm_ocn_cmeps.tmp new file mode 100644 index 000000000..9b3d4a7d9 --- /dev/null +++ b/parm/forecast/regional/nems.configure.atm_ocn_cmeps.tmp @@ -0,0 +1,98 @@ +# component list +EARTH_component_list: ATM OCN MED + +# component models +ATM_model: fv3 +OCN_model: hycom +MED_model: nems + +# component petLists +_ATM_petlist_bounds_ +_OCN_petlist_bounds_ +_MED_petlist_bounds_ + +# run sequence +runSeq:: + @_cpl_dt_ +_runSeq_ALL_ + @ +:: + +# component attributes + +ATM_attributes:: + Verbosity = 1 + Diagnostic = 0 +:: + +OCN_attributes:: + Verbosity = 1 + Diagnostic = 0 + cdf_impexp_freq = 3 + cpl_hour = 0 + cpl_min = 0 + cpl_sec = _cpl_dt_ + base_dtg = _base_dtg_ + merge_import = _merge_import_ + skip_first_import = .true. + hycom_arche_output = .false. + hyc_esmf_exp_output = .true. + hyc_esmf_imp_output = .true. + hyc_impexp_file = nems.configure + espc_show_impexp_minmax = .true. + ocean_start_dtg = _ocean_start_dtg_ + start_hour = 0 + start_min = 0 + start_sec = 0 + end_hour = _end_hour_ + end_min = 0 + end_sec = 0 +:: + +MED_attributes:: + Verbosity = 1 + Diagnostic = 0 + coupling_mode = hafs + system_type = ufs + normalization = none + merge_type = copy + ATM_model = fv3 + OCN_model = hycom + history_ymd = -999 + ScalarFieldCount = 0 + ScalarFieldIdxGridNX = 0 + ScalarFieldIdxGridNY = 0 + ScalarFieldName = cpl_scalars +:: + +DRIVER_attributes:: + start_type = startup +:: + +ALLCOMP_attributes:: + mediator_read_restart = false +# ScalarFieldCount = 3 +# ScalarFieldIdxGridNX = 1 +# ScalarFieldIdxGridNY = 2 +# ScalarFieldIdxNextSwCday = 3 +# ScalarFieldName = cpl_scalars +:: + +ocn_export_fields:: + 'sst' 'sea_surface_temperature' 'K' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'mask' 'So_omask' '1' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' +:: + +ocn_import_fields:: + 'u10' 'inst_zonal_wind_height10m' 'm_s-1' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'v10' 'inst_merid_wind_height10m' 'm_s-1' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'taux10' 'mean_zonal_moment_flx_atm' 'N_m-2' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'tauy10' 'mean_merid_moment_flx_atm' 'N_m-2' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'airtmp' 'inst_temp_height2m' 'K' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'airhum' 'inst_spec_humid_height2m' 'kg_kg-1' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'prcp' 'mean_prec_rate' 'kg_m-2_s-1' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'swflxd' 'mean_net_sw_flx' 'W_m-2' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'lwflxd' 'mean_net_lw_flx' 'W_m-2' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'mslprs' 'inst_pres_height_surface' 'Pa' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'gt' 'inst_temp_height_surface' 'K' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' +:: diff --git a/parm/forecast/regional/nems.configure.atmonly b/parm/forecast/regional/nems.configure.atmonly index 88633a425..fe59cc645 100644 --- a/parm/forecast/regional/nems.configure.atmonly +++ b/parm/forecast/regional/nems.configure.atmonly @@ -1,5 +1,18 @@ EARTH_component_list: ATM + ATM_model: fv3 + runSeq:: ATM :: + +ATM_attributes:: +:: + +DRIVER_attributes:: + start_type = startup +:: + +ALLCOMP_attributes:: + mediator_read_restart = false +:: diff --git a/parm/hafs.conf b/parm/hafs.conf index 66bbdb0e2..c2996e66b 100644 --- a/parm/hafs.conf +++ b/parm/hafs.conf @@ -239,19 +239,18 @@ output_grid_dlat=0.03 ;; output grid spacing dlat . . . . #output_grid_dlon=0.03 ;; output grid spacing dlon . . . . #output_grid_dlat=0.03 ;; output grid spacing dlat . . . . -cplflx=.false. # Ocean coupling related options. Only useful when run_ocean=yes +# When run_ocean=yes, then the following options are available for cpl_ocean +# 0, run ocean model side by side (no coupling) +# 1, direct coupling through the nearest point regridding method +# 2, direct coupling through the bilinear regridding method +# 3, CMEPS based coupling through the bilinear regridding method cpl_ocean=0 -cpl_dt=360 -ocean_tasks=60 -runSeq_OCN2ATM=auto -runSeq_ATM2OCN=auto -runSeq_ATM=auto -runSeq_OCN=auto -ATM_petlist_bounds=auto -OCN_petlist_bounds=auto -ocean_start_dtg=auto -merge_import=.false. +cpl_dt=360 ;; coupling time step in seconds +#ocean_tasks=120 ;; Number of PEs for the OCN component +ocean_tasks=60 ;; Number of PEs for the OCN component +ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31,0,0,0), e.g., 43340.00000 +merge_import=.true. [post] # Grid definition for post and tracker, used by wgrib2 diff --git a/parm/hafs_holdvars.conf b/parm/hafs_holdvars.conf index 29ee47ea3..fdeea06ec 100644 --- a/parm/hafs_holdvars.conf +++ b/parm/hafs_holdvars.conf @@ -85,14 +85,7 @@ synop_gridspecs={synop_gridspecs} trker_gridspecs={trker_gridspecs} cpl_dt={forecast/cpl_dt} -cplflx={forecast/cplflx} cpl_ocean={forecast/cpl_ocean} -runSeq_OCN2ATM={forecast/runSeq_OCN2ATM} -runSeq_ATM2OCN={forecast/runSeq_ATM2OCN} -runSeq_ATM={forecast/runSeq_ATM} -runSeq_OCN={forecast/runSeq_OCN} ocean_tasks={forecast/ocean_tasks} -ATM_petlist_bounds={forecast/ATM_petlist_bounds} -OCN_petlist_bounds={forecast/OCN_petlist_bounds} ocean_start_dtg={forecast/ocean_start_dtg} merge_import={forecast/merge_import} diff --git a/parm/hafs_holdvars.txt b/parm/hafs_holdvars.txt index 81f99540a..935d57ca4 100644 --- a/parm/hafs_holdvars.txt +++ b/parm/hafs_holdvars.txt @@ -116,14 +116,7 @@ export run_ocean={run_ocean} export ocean_model={ocean_model} export cpl_dt={cpl_dt} export cpl_ocean={cpl_ocean} -export cplflx={cplflx} -export runSeq_OCN2ATM={runSeq_OCN2ATM} -export runSeq_ATM2OCN={runSeq_ATM2OCN} -export runSeq_ATM={runSeq_ATM} -export runSeq_OCN={runSeq_OCN} export ocean_tasks={ocean_tasks} -export ATM_petlist_bounds={ATM_petlist_bounds} -export OCN_petlist_bounds={OCN_petlist_bounds} export ocean_start_dtg={ocean_start_dtg} export merge_import={merge_import} diff --git a/parm/hafs_hycom.conf b/parm/hafs_hycom.conf index a2fae72b2..70cd29e8f 100644 --- a/parm/hafs_hycom.conf +++ b/parm/hafs_hycom.conf @@ -24,40 +24,19 @@ nstf_n4=0 nstf_n5=0 write_groups=1 -write_tasks_per_group=48 - -#ocean_tasks=120 ;; Number of PEs for the OCN component -ocean_tasks=60 ;; Number of PEs for the OCN component -cpl_dt=360 ;; coupling time step in seconds -cplflx=auto +write_tasks_per_group=24 +# Ocean coupling related options. Only useful when run_ocean=yes # When run_ocean=yes, then the following options are available for cpl_ocean # 0, run ocean model side by side (no coupling) # 1, direct coupling through the nearest point regridding method # 2, direct coupling through the bilinear regridding method +# 3, CMEPS based coupling through the bilinear regridding method cpl_ocean=2 - -## By default, the following itmes will be generated automatically -## However, they can be overriden by the users if values (other than auto) are given. -## if run_ocean=yes and cpl_ocean=0 -#cplflx=.false. -#runSeq_OCN2ATM="" -#runSeq_ATM2OCN="" -## if run_ocean=yes and cpl_ocean=1 -#cplflx=.true. -#runSeq_OCN2ATM="OCN -> ATM :remapMethod=nearest_stod:srcmaskvalues=0" -#runSeq_ATM2OCN="ATM -> OCN :remapMethod=nearest_stod:srcmaskvalues=1:dstmaskvalues=0" -## if run_ocean=yes and cpl_ocean=2 (auto option not implemented yet) -#cplflx=.true. -#runSeq_OCN2ATM="OCN -> ATM :remapMethod=bilinear:unmappedaction=ignore:zeroregion=select:srcmaskvalues=0" -#runSeq_ATM2OCN="ATM -> OCN :remapMethod=bilinear:unmappedaction=ignore:zeroregion=select:srcmaskvalues=1:dstmaskvalues=0" -# -#runSeq_ATM=auto -#runSeq_OCN=auto -#ATM_petlist_bounds=auto ;; e.g., "ATM_petlist_bounds: 0000 1319" -#OCN_petlist_bounds=auto ;; e.g., "ATM_petlist_bounds: 1320 1439" -#ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31,0,0,0), e.g., 43340.00000 - +cpl_dt=360 ;; coupling time step in seconds +#ocean_tasks=120 ;; Number of PEs for the OCN component +ocean_tasks=60 ;; Number of PEs for the OCN component +ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31,0,0,0), e.g., 43340.00000 merge_import=.true. [rocotostr] diff --git a/rocoto/cronjob_hafs_hycom.sh b/rocoto/cronjob_hafs_hycom.sh index c21907fb5..41212bbb9 100755 --- a/rocoto/cronjob_hafs_hycom.sh +++ b/rocoto/cronjob_hafs_hycom.sh @@ -35,9 +35,18 @@ scrubopt="config.scrub_work=no config.scrub_com=no" #=============================================================================== +# Run regional hafs-hycom coupled configuration with the cmeps-based coupling + ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY \ + config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_cplocean3 \ + config.ictype=gfsnemsio config.bctype=gfsgrib2_0p25 \ + ${scrubopt} \ + ../parm/hafs_regional_static.conf \ + ../parm/hafs_hycom.conf \ + forecast.cpl_ocean=3 + # Run regional hafs-hycom coupled configuration with the bilinear regridding method ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY \ - config.EXPT=${EXPT} config.SUBEXPT=${EXPT} \ + config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_cplocean2 \ config.ictype=gfsnemsio config.bctype=gfsgrib2_0p25 \ ${scrubopt} \ ../parm/hafs_regional_static.conf \ @@ -46,7 +55,7 @@ scrubopt="config.scrub_work=no config.scrub_com=no" # Run regional hafs-hycom coupled configuration with the nearest point regridding method #${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY \ -# config.EXPT=${EXPT} config.SUBEXPT=${EXPT} \ +# config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_cplocean1 \ # config.ictype=gfsnemsio config.bctype=gfsgrib2_0p25 \ # ${scrubopt} \ # ../parm/hafs_regional_static.conf \ @@ -55,7 +64,7 @@ scrubopt="config.scrub_work=no config.scrub_com=no" # Run regional hafs-hycom coupled configuration in side-by-side model (no coupling) #${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY \ -# config.EXPT=${EXPT} config.SUBEXPT=${EXPT} \ +# config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_cplocean0 \ # config.ictype=gfsnemsio config.bctype=gfsgrib2_0p25 \ # ${scrubopt} \ # ../parm/hafs_regional_static.conf \ diff --git a/rocoto/cronjob_hafs_rt.sh b/rocoto/cronjob_hafs_rt.sh index c93a019da..143edb649 100755 --- a/rocoto/cronjob_hafs_rt.sh +++ b/rocoto/cronjob_hafs_rt.sh @@ -18,14 +18,14 @@ date #PYTHON3=/apps/intel/intelpython3/bin/python3 # MSU Orion -#HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS -#dev="-s sites/orion.ent -f" -#PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 + HOMEhafs=/work/noaa/hwrf/save/${USER}/HAFS + dev="-s sites/orion.ent -f" + PYTHON3=/apps/intel-2020/intel-2020/intelpython3/bin/python3 # NOAA RDHPCS Hera - HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS - dev="-s sites/hera.ent -f" - PYTHON3=/apps/intel/intelpython3/bin/python3 +#HOMEhafs=/scratch1/NCEPDEV/hwrf/save/${USER}/HAFS +#dev="-s sites/hera.ent -f" +#PYTHON3=/apps/intel/intelpython3/bin/python3 cd ${HOMEhafs}/rocoto @@ -40,11 +40,23 @@ scrubopt="config.scrub_work=no config.scrub_com=no" config.ictype=gfsnetcdf config.bctype=gfsgrib2ab_0p25 \ config.halo_blend=10 \ config.GFSVER=PROD2021 \ - dir.COMgfs=/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFSv16 \ + dir.COMgfs=/work/noaa/hwrf/noscrub/hafs-input/COMGFSv16 \ config.NHRS=12 ${scrubopt} \ ../parm/hafs_regional_static.conf \ ../parm/hafs_hycom.conf +# Regional static NATL basin-focused and cmeps-based ocean-coupled configuration with GFSv16 netcdf format IC and grib2ab format BC + ${PYTHON3} ./run_hafs.py -t ${dev} 2019082900 00L HISTORY \ + config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_rt_regional_coupled_netcdf_cmeps \ + config.ictype=gfsnetcdf config.bctype=gfsgrib2ab_0p25 \ + config.halo_blend=10 \ + config.GFSVER=PROD2021 \ + dir.COMgfs=/work/noaa/hwrf/noscrub/hafs-input/COMGFSv16 \ + config.NHRS=12 ${scrubopt} \ + ../parm/hafs_regional_static.conf \ + ../parm/hafs_hycom.conf \ + forecast.cpl_ocean=3 + # Regional static NATL basin-focused configuration with GFS nemsio format IC/BC #${PYTHON3} ./run_hafs.py -t ${dev} 2019091600 09L HISTORY \ # config.EXPT=${EXPT} config.SUBEXPT=${EXPT}_rt_regional_static \ diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh index eba5e7d04..99d5ff889 100755 --- a/scripts/exhafs_forecast.sh +++ b/scripts/exhafs_forecast.sh @@ -59,14 +59,57 @@ export run_ocean=${run_ocean:-no} export ocean_model=${ocean_model:-hycom} export cpl_ocean=${cpl_ocean:-0} export ocean_tasks=${ocean_tasks:-120} -export cpl_dt=${cpl_dt:-360} export cplflx=${cplflx:-.false.} -export ATM_petlist_bounds=${ATM_petlist_bounds:-'ATM_petlist_bounds: 0000 1319'} -export OCN_petlist_bounds=${OCN_petlist_bounds:-'OCN_petlist_bounds: 1320 1439'} -export runSeq_OCN2ATM=${runSeq_OCN2ATM:-''} -export runSeq_ATM2OCN=${runSeq_ATM2OCN:-''} -export runSeq_ATM=${runSeq_ATM:-'ATM'} -export runSeq_OCN=${runSeq_OCN:-'OCN'} +export cpl_dt=${cpl_dt:-360} + +if [ $gtype = regional ]; then + if [ $quilting = .true. ]; then + ATM_tasks=$(($layoutx*$layouty+$write_groups*$write_tasks_per_group)) + else + ATM_tasks=$(($layoutx*$layouty)) + fi +elif [ $gtype = nest ]; then + if [ $quilting = .true. ]; then + ATM_tasks=$((6*$glob_layoutx*$glob_layouty+$layoutx*$layouty+$write_groups*$write_tasks_per_group)) + else + ATM_tasks=$((6*$glob_layoutx*$glob_layouty+$layoutx*$layouty)) + fi +else + echo "Error: please specify grid type with 'gtype' as uniform, stretch, nest, or regional" + exit 9 +fi + +export ATM_petlist_bounds=$(printf "ATM_petlist_bounds: %04d %04d" 0 $(($ATM_tasks-1))) + +# run ocean model side by side (no coupling) +if [ ${run_ocean} = yes ] && [ $cpl_ocean -eq 0 ]; then + export cplflx=.false. + export OCN_petlist_bounds=$(printf "OCN_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) + export MED_petlist_bounds="" + export runSeq_ALL="ATM\n OCN" +fi +# direct coupling through the nearest point regridding method +if [ ${run_ocean} = yes ] && [ $cpl_ocean -eq 1 ]; then + export cplflx=.true. + export runSeq_ALL="OCN -> ATM :remapMethod=nearest_stod:srcmaskvalues=0\n ATM -> OCN :remapMethod=nearest_stod:srcmaskvalues=1:dstmaskvalues=0\n ATM\n OCN" + export OCN_petlist_bounds=$(printf "OCN_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) + export MED_petlist_bounds="" +fi +# direct coupling through the bilinear regridding method +if [ ${run_ocean} = yes ] && [ $cpl_ocean -eq 2 ]; then + export cplflx=.true. + export OCN_petlist_bounds=$(printf "OCN_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) + export MED_petlist_bounds="" + export runSeq_ALL="OCN -> ATM :remapMethod=bilinear:unmappedaction=ignore:zeroregion=select:srcmaskvalues=0\n ATM -> OCN :remapMethod=bilinear:unmappedaction=ignore:zeroregion=select:srcmaskvalues=1:dstmaskvalues=0\n ATM\n OCN" +fi +# CMEPS based coupling through the bilinear regridding method +if [ ${run_ocean} = yes ] && [ $cpl_ocean -eq 3 ]; then + export cplflx=.true. + export OCN_petlist_bounds=$(printf "OCN_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) + export MED_petlist_bounds=$(printf "MED_petlist_bounds: %04d %04d" $ATM_tasks $(($ATM_tasks+$ocean_tasks-1))) + export runSeq_ALL="ATM -> MED :remapMethod=redist\n MED med_phases_post_atm\n OCN -> MED :remapMethod=redist\n MED med_phases_post_ocn\n MED med_phases_prep_atm\n MED med_phases_prep_ocn_accum\n MED med_phases_prep_ocn_avg\n MED -> ATM :remapMethod=redist\n MED -> OCN :remapMethod=redist\n ATM\n OCN" +fi + export ocean_start_dtg=${ocean_start_dtg:-43340.00000} #export base_dtg=${CDATE:-2019082900} #export end_hour=${NHRS:-126} @@ -269,20 +312,22 @@ cp ${PARMforecast}/field_table . cp ${PARMforecast}/input.nml.tmp . cp ${PARMforecast}/model_configure.tmp . -if [ ${run_ocean} = yes ]; then - cp ${PARMforecast}/nems.configure.atm_ocn.tmp ./ +if [ ${run_ocean} = yes ]; then + if [[ ${cpl_ocean} -eq 3 ]]; then + cp ${PARMforecast}/nems.configure.atm_ocn_cmeps.tmp ./nems.configure.tmp + else + cp ${PARMforecast}/nems.configure.atm_ocn.tmp ./nems.configure.tmp + fi sed -e "s/_ATM_petlist_bounds_/${ATM_petlist_bounds}/g" \ -e "s/_OCN_petlist_bounds_/${OCN_petlist_bounds}/g" \ + -e "s/_MED_petlist_bounds_/${MED_petlist_bounds}/g" \ -e "s/_cpl_dt_/${cpl_dt}/g" \ - -e "s/_runSeq_OCN2ATM_/${runSeq_OCN2ATM}/g" \ - -e "s/_runSeq_ATM2OCN_/${runSeq_ATM2OCN}/g" \ - -e "s/_runSeq_ATM_/${runSeq_ATM}/g" \ - -e "s/_runSeq_OCN_/${runSeq_OCN}/g" \ + -e "s/_runSeq_ALL_/${runSeq_ALL}/g" \ -e "s/_base_dtg_/${CDATE}/g" \ -e "s/_ocean_start_dtg_/${ocean_start_dtg}/g" \ -e "s/_end_hour_/${NHRS}/g" \ -e "s/_merge_import_/${merge_import:-.false.}/g" \ - nems.configure.atm_ocn.tmp > nems.configure + nems.configure.tmp > nems.configure elif [ ${run_ocean} = no ]; then cp ${PARMforecast}/nems.configure.atmonly ./nems.configure else @@ -399,6 +444,11 @@ cat model_configure.tmp | sed s/NTASKS/$TOTAL_TASKS/ | sed s/YR/$yr/ | \ sed s/_cpl_/${cplflx:-.false.}/ \ > model_configure +#------------------------------------------------------------------- +# Copy the fd_nems.yaml file +#------------------------------------------------------------------- +cp ${HOMEhafs}/sorc/hafs_forecast.fd/CMEPS-interface/CMEPS/mediator/fd_nems.yaml ./ + #------------------------------------------------------------------- # Link the executable and run the forecast #------------------------------------------------------------------- diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index d6ec7b46a..daeb148e8 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -23,5 +23,5 @@ if [ $target = jet ]; then target=jet.intel ; fi cd hafs_forecast.fd/ cd tests/ -./compile.sh "$target" "CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_nonsstugwd,HAFS_v0_gfdlmp_nocpnsstugwd,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp_nougwd,HAFS_v0_gfdlmp_nocpugwd,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf_nougwd,HAFS_v0_hwrf 32BIT=Y HYCOM=Y" 32bit YES NO +./compile.sh "$target" "CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_nonsstugwd,HAFS_v0_gfdlmp_nocpnsstugwd,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp_nougwd,HAFS_v0_gfdlmp_nocpugwd,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf_nougwd,HAFS_v0_hwrf 32BIT=Y HYCOM=Y CMEPS=Y" 32bit YES NO diff --git a/ush/hafs/launcher.py b/ush/hafs/launcher.py index bececfe33..ee2ffaa7a 100644 --- a/ush/hafs/launcher.py +++ b/ush/hafs/launcher.py @@ -1394,65 +1394,6 @@ def make_holdvars(self,part1='{PARMhafs}/hafs_holdvars.txt',part2=None): self.set('holdvars','trker_gridspecs',trker_gridspecs) run_ocean=self.getbool('config','run_ocean') - cpl_ocean=self.getint('forecast','cpl_ocean',0) - ocean_tasks=self.getint('forecast','ocean_tasks',120) - self.set('holdvars','ocean_tasks',ocean_tasks) - OCN_tasks=ocean_tasks - - # Set ATM/OCN_petlist_bounds if needed - runSeq_ATM=self.getstr('forecast','runSeq_ATM','auto') - runSeq_OCN=self.getstr('forecast','runSeq_OCN','auto') - ATM_petlist_bounds=self.getstr('forecast','ATM_petlist_bounds','auto') - OCN_petlist_bounds=self.getstr('forecast','OCN_petlist_bounds','auto') - gtype=self.getstr('grid','gtype','regional') - glob_layoutx=self.getint('forecast','glob_layoutx',8) - glob_layouty=self.getint('forecast','glob_layouty',8) - layoutx=self.getint('forecast','layoutx',40) - layouty=self.getint('forecast','layouty',30) - quilting=self.getbool('forecast','quilting',True) - write_groups=self.getint('forecast','write_groups',3) - write_tasks_per_group=self.getint('forecast','write_tasks_per_group',48) - runSeq_OCN2ATM=self.getstr('forecast','runSeq_OCN2ATM','auto') - runSeq_ATM2OCN=self.getstr('forecast','runSeq_ATM2OCN','auto') - cplflx=self.getstr('forecast','cplflx','auto') - - if run_ocean and cpl_ocean==0: - if cplflx=='auto': self.set('holdvars','cplflx','.false.') - if runSeq_OCN2ATM=='auto': self.set('holdvars','runSeq_OCN2ATM','') - if runSeq_ATM2OCN=='auto': self.set('holdvars','runSeq_ATM2OCN','') - if run_ocean and cpl_ocean==1: - if cplflx=='auto': self.set('holdvars','cplflx','.true.') - if runSeq_OCN2ATM=='auto': self.set('holdvars','runSeq_OCN2ATM','"OCN -> ATM :remapMethod=nearest_stod:srcmaskvalues=0"') - if runSeq_ATM2OCN=='auto': self.set('holdvars','runSeq_ATM2OCN','"ATM -> OCN :remapMethod=nearest_stod:srcmaskvalues=1:dstmaskvalues=0"') - if run_ocean and cpl_ocean==2: - if cplflx=='auto': self.set('holdvars','cplflx','.true.') - if runSeq_OCN2ATM=='auto': self.set('holdvars','runSeq_OCN2ATM','"OCN -> ATM :remapmethod=bilinear:unmappedaction=ignore:zeroregion=select:srcmaskvalues=0"') - if runSeq_ATM2OCN=='auto': self.set('holdvars','runSeq_ATM2OCN','"ATM -> OCN :remapmethod=bilinear:unmappedaction=ignore:zeroregion=select:srcmaskvalues=1:dstmaskvalues=0"') - - if runSeq_ATM=='auto': self.set('holdvars','runSeq_ATM','ATM') - if run_ocean and runSeq_OCN=='auto': - self.set('holdvars','runSeq_OCN','OCN') - - if gtype=='regional': - if quilting: - ATM_tasks=layoutx*layouty+write_groups*write_tasks_per_group - else: - ATM_tasks=layoutx*layouty - elif gtype=='nest': - if quilting: - ATM_tasks=6*glob_layoutx*glob_layouty+layoutx*layouty+write_groups*write_tasks_per_group - else: - ATM_tasks=6*glob_layoutx*glob_layouty+layoutx*layouty - else: - logger.warning('unsupported grid type') - - if ATM_petlist_bounds=='auto': - ATM_petlist_bounds='"ATM_petlist_bounds: %04d %04d"'%(0, ATM_tasks-1) - self.set('holdvars','ATM_petlist_bounds',ATM_petlist_bounds) - - if run_ocean and OCN_petlist_bounds=='auto': - OCN_petlist_bounds='"OCN_petlist_bounds: %04d %04d"'%(ATM_tasks, ATM_tasks+OCN_tasks-1) - self.set('holdvars','OCN_petlist_bounds',OCN_petlist_bounds) # Set ocean_start_dtg if needed ocean_start_dtg=self.getstr('forecast','ocean_start_dtg','auto') From 4ce85ec37770c7091e6604820849a66425ac0790 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Wed, 27 Jan 2021 10:13:08 -0600 Subject: [PATCH 04/10] A. Use correct number of nodes/PEs for one regional ocean coupled forecast_resource entity on Orion and Hera. B. Update the indentation in nems.configure*.tmp files --- parm/forecast/regional/nems.configure.atm_ocn.tmp | 2 +- parm/forecast/regional/nems.configure.atm_ocn_cmeps.tmp | 2 +- rocoto/sites/hera.ent | 4 ++-- rocoto/sites/orion.ent | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/parm/forecast/regional/nems.configure.atm_ocn.tmp b/parm/forecast/regional/nems.configure.atm_ocn.tmp index c9f0d56bf..d6eb554e6 100644 --- a/parm/forecast/regional/nems.configure.atm_ocn.tmp +++ b/parm/forecast/regional/nems.configure.atm_ocn.tmp @@ -12,7 +12,7 @@ _OCN_petlist_bounds_ # run sequence runSeq:: @_cpl_dt_ - _runSeq_ALL_ + _runSeq_ALL_ @ :: diff --git a/parm/forecast/regional/nems.configure.atm_ocn_cmeps.tmp b/parm/forecast/regional/nems.configure.atm_ocn_cmeps.tmp index 9b3d4a7d9..048d65a39 100644 --- a/parm/forecast/regional/nems.configure.atm_ocn_cmeps.tmp +++ b/parm/forecast/regional/nems.configure.atm_ocn_cmeps.tmp @@ -14,7 +14,7 @@ _MED_petlist_bounds_ # run sequence runSeq:: @_cpl_dt_ -_runSeq_ALL_ + _runSeq_ALL_ @ :: diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent index cc6eec563..5fba901f6 100644 --- a/rocoto/sites/hera.ent +++ b/rocoto/sites/hera.ent @@ -42,9 +42,9 @@ 72:ppn=20:tpp=2TOTAL_TASKS1440NCTSK20&FORECAST_EXTRA;"> - 69:ppn=20:tpp=2TOTAL_TASKS1380NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> 65:ppn=20:tpp=2TOTAL_TASKS1300NCTSK20&FORECAST_EXTRA;"> - 69:ppn=20:tpp=2TOTAL_TASKS1380NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> 101:ppn=20:tpp=2TOTAL_TASKS2020NCTSK20&FORECAST_EXTRA;"> 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent index 414b53ffd..9fd1bd995 100644 --- a/rocoto/sites/orion.ent +++ b/rocoto/sites/orion.ent @@ -42,9 +42,9 @@ 72:ppn=20:tpp=2TOTAL_TASKS1440NCTSK20&FORECAST_EXTRA;"> - 69:ppn=20:tpp=2TOTAL_TASKS1380NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> 65:ppn=20:tpp=2TOTAL_TASKS1300NCTSK20&FORECAST_EXTRA;"> - 69:ppn=20:tpp=2TOTAL_TASKS1380NCTSK20&FORECAST_EXTRA;"> + 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;"> 101:ppn=20:tpp=2TOTAL_TASKS2020NCTSK20&FORECAST_EXTRA;"> 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;"> From 7e86b6804a6170a89e68b5606fecb01923fb55f8 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Wed, 27 Jan 2021 11:48:06 -0600 Subject: [PATCH 05/10] Point hafs_forecast.fd to the support/HAFS branch to get this branch ready to be merged back into the develop branch. --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index b59c5332e..1e9966cb0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "ufs-weather-model"] path = sorc/hafs_forecast.fd url = https://github.com/hafs-community/ufs-weather-model.git - branch = feature/hafs_couplehycom + branch = support/HAFS [submodule "EMC_post"] path = sorc/hafs_post.fd url = https://github.com/hafs-community/EMC_post.git From 6c957e0516a5545aa3504de876410ab573cb4106 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Thu, 28 Jan 2021 13:41:41 -0600 Subject: [PATCH 06/10] Update submodule hafs_forecast.fd, which now points to the latest support/HAFS branch of CMEPS. And a new ccpp SDF suite_HAFS_v0_gfdlmp_nonsst.xml is added. --- sorc/build_forecast.sh | 2 +- sorc/hafs_forecast.fd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh index daeb148e8..dee5441a8 100755 --- a/sorc/build_forecast.sh +++ b/sorc/build_forecast.sh @@ -23,5 +23,5 @@ if [ $target = jet ]; then target=jet.intel ; fi cd hafs_forecast.fd/ cd tests/ -./compile.sh "$target" "CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_nonsstugwd,HAFS_v0_gfdlmp_nocpnsstugwd,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp_nougwd,HAFS_v0_gfdlmp_nocpugwd,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf_nougwd,HAFS_v0_hwrf 32BIT=Y HYCOM=Y CMEPS=Y" 32bit YES NO +./compile.sh "$target" "CCPP=Y STATIC=Y SUITES=HAFS_v0_gfdlmp_nonsstugwd,HAFS_v0_gfdlmp_nocpnsstugwd,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nonsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp_nougwd,HAFS_v0_gfdlmp_nocpugwd,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf_nougwd,HAFS_v0_hwrf 32BIT=Y HYCOM=Y CMEPS=Y" 32bit YES NO diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 7c41eef22..45a64f2e7 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 7c41eef22473ce02fda33f909efe8f016454bd36 +Subproject commit 45a64f2e7396cdaadea21a02838c0c5e7123e68c From b5caaa42b956b3c34dc904adae2b34afbbee9242 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Fri, 29 Jan 2021 21:33:23 -0600 Subject: [PATCH 07/10] Update nems.configure.atm_ocn.tmp by adding the options needed for the direct coupling when the forecast model is built with the CMEPS=Y option. --- .../regional/nems.configure.atm_ocn.tmp | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/parm/forecast/regional/nems.configure.atm_ocn.tmp b/parm/forecast/regional/nems.configure.atm_ocn.tmp index d6eb554e6..1e7fd22ba 100644 --- a/parm/forecast/regional/nems.configure.atm_ocn.tmp +++ b/parm/forecast/regional/nems.configure.atm_ocn.tmp @@ -36,7 +36,7 @@ OCN_attributes:: hycom_arche_output = .false. hyc_esmf_exp_output = .true. hyc_esmf_imp_output = .true. - hyc_impexp_file = dummy_file + hyc_impexp_file = nems.configure espc_show_impexp_minmax = .true. ocean_start_dtg = _ocean_start_dtg_ start_hour = 0 @@ -53,4 +53,28 @@ DRIVER_attributes:: ALLCOMP_attributes:: mediator_read_restart = false +# ScalarFieldCount = 3 +# ScalarFieldIdxGridNX = 1 +# ScalarFieldIdxGridNY = 2 +# ScalarFieldIdxNextSwCday = 3 +# ScalarFieldName = cpl_scalars +:: + +ocn_export_fields:: + 'sst' 'sea_surface_temperature' 'K' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' +# 'mask' 'So_omask' '1' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' +:: + +ocn_import_fields:: + 'u10' 'inst_zonal_wind_height10m' 'm_s-1' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'v10' 'inst_merid_wind_height10m' 'm_s-1' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'taux10' 'mean_zonal_moment_flx_atm' 'N_m-2' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'tauy10' 'mean_merid_moment_flx_atm' 'N_m-2' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'airtmp' 'inst_temp_height2m' 'K' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'airhum' 'inst_spec_humid_height2m' 'kg_kg-1' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'prcp' 'mean_prec_rate' 'kg_m-2_s-1' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'swflxd' 'mean_net_sw_flx' 'W_m-2' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'lwflxd' 'mean_net_lw_flx' 'W_m-2' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'mslprs' 'inst_pres_height_surface' 'Pa' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' + 'gt' 'inst_temp_height_surface' 'K' '.TRUE.' 'FLD_REMAP_BILINR' 'FLD_MASK_NNE' '1' :: From dc7b95dbfcc2f73e58ef5f7c19abd54145a7d617 Mon Sep 17 00:00:00 2001 From: Daniel Rosen Date: Sat, 30 Jan 2021 06:24:29 +0000 Subject: [PATCH 08/10] A. Update hafs_forecast.fd to point the latest commit of the feature/hafs_couplehycom branch, which the HYCOM set_impexp_fields fix. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 45a64f2e7..2ed56a038 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 45a64f2e7396cdaadea21a02838c0c5e7123e68c +Subproject commit 2ed56a0381dda3583031dab56bb6bd2c398e48cc From 5b3a8b14fc7f914d7ae19fc46505baf6c87dca3b Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Sat, 30 Jan 2021 09:20:22 -0600 Subject: [PATCH 09/10] * Update hafs_forecast.fd to point its latest support/HAFS branch * Update nems.configure.atm_ocn.tmp for the direct coupling configuration --- parm/forecast/regional/nems.configure.atm_ocn.tmp | 2 +- sorc/hafs_forecast.fd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/forecast/regional/nems.configure.atm_ocn.tmp b/parm/forecast/regional/nems.configure.atm_ocn.tmp index 1e7fd22ba..40cf8c3cd 100644 --- a/parm/forecast/regional/nems.configure.atm_ocn.tmp +++ b/parm/forecast/regional/nems.configure.atm_ocn.tmp @@ -36,7 +36,7 @@ OCN_attributes:: hycom_arche_output = .false. hyc_esmf_exp_output = .true. hyc_esmf_imp_output = .true. - hyc_impexp_file = nems.configure + hyc_impexp_file = dummy_file espc_show_impexp_minmax = .true. ocean_start_dtg = _ocean_start_dtg_ start_hour = 0 diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index 2ed56a038..a0b36cb9f 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit 2ed56a0381dda3583031dab56bb6bd2c398e48cc +Subproject commit a0b36cb9f1a809a936fe90eff3e458fb4c31aa4d From 153ef38760a812442a63d3a7435818869a0ddf77 Mon Sep 17 00:00:00 2001 From: "Bin.Liu" Date: Thu, 4 Feb 2021 14:43:07 -0600 Subject: [PATCH 10/10] Update hafs_forecast.fd after it being synced with its develop as of 02/04/2021. --- sorc/hafs_forecast.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd index a0b36cb9f..e1f164a46 160000 --- a/sorc/hafs_forecast.fd +++ b/sorc/hafs_forecast.fd @@ -1 +1 @@ -Subproject commit a0b36cb9f1a809a936fe90eff3e458fb4c31aa4d +Subproject commit e1f164a46e10ee757344d994cd3de41b07b829cb