diff --git a/env/CONTAINER.env b/env/CONTAINER.env index 378b046944e..4f85ae56de1 100755 --- a/env/CONTAINER.env +++ b/env/CONTAINER.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun" + echo "atmanlrun atmensanlrun aeroanlrun landanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" diff --git a/env/HERA.env b/env/HERA.env index f97af13d95b..d2d4223455d 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun" + echo "atmanlrun atmensanlrun aeroanlrun landanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -74,6 +74,14 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [[ "${step}" = "landanlrun" ]]; then + + nth_max=$((npe_node_max / npe_node_landanlrun)) + + export NTHREADS_LANDANL=${nth_landanlrun:-${nth_max}} + [[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max} + export APRUN_LANDANL="${launcher} -n ${npe_landanlrun}" + elif [[ "${step}" = "ocnanalbmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" diff --git a/env/JET.env b/env/JET.env index 1632e1cc0e0..11533cb7878 100755 --- a/env/JET.env +++ b/env/JET.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun" + echo "atmanlrun atmensanlrun aeroanlrun landanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -73,6 +73,14 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [[ "${step}" = "landanlrun" ]]; then + + nth_max=$((npe_node_max / npe_node_landanlrun)) + + export NTHREADS_LANDANL=${nth_landanlrun:-${nth_max}} + [[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max} + export APRUN_LANDANL="${launcher} -n ${npe_landanlrun}" + elif [[ "${step}" = "ocnanalbmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" diff --git a/env/ORION.env b/env/ORION.env index 43450bffcfa..0c541593c74 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun" + echo "atmanlrun atmensanlrun aeroanlrun landanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -74,6 +74,14 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [[ "${step}" = "landanlrun" ]]; then + + nth_max=$((npe_node_max / npe_node_landanlrun)) + + export NTHREADS_LANDANL=${nth_landanlrun:-${nth_max}} + [[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max} + export APRUN_LANDANL="${launcher} -n ${npe_landanlrun}" + elif [[ "${step}" = "ocnanalbmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" diff --git a/env/S4.env b/env/S4.env index c69a845cdf9..fef8ac562fb 100755 --- a/env/S4.env +++ b/env/S4.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun" + echo "atmanlrun atmensanlrun aeroanlrun landanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -72,6 +72,14 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [[ "${step}" = "landanlrun" ]]; then + + nth_max=$((npe_node_max / npe_node_landanlrun)) + + export NTHREADS_LANDANL=${nth_landanlrun:-${nth_max}} + [[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max} + export APRUN_LANDANL="${launcher} -n ${npe_landanlrun}" + elif [[ "${step}" = "ocnanalbmat" ]]; then echo "WARNING: ${step} is not enabled on S4!" diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 95c3f72fe47..6d4acf5d19c 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun" + echo "atmanlrun atmensanlrun aeroanlrun landanlrun" echo "anal sfcanl fcst post vrfy metp" echo "eobs eupd ecen esfc efcs epos" echo "postsnd awips gempak" @@ -61,6 +61,14 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" +elif [[ "${step}" = "landanlrun" ]]; then + + nth_max=$((npe_node_max / npe_node_landanlrun)) + + export NTHREADS_LANDANL=${nth_landanlrun:-${nth_max}} + [[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max} + export APRUN_LANDANL="${launcher} -n ${npe_landanlrun}" + elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export OMP_PLACES=cores diff --git a/jobs/JGLOBAL_LAND_ANALYSIS_FINALIZE b/jobs/JGLOBAL_LAND_ANALYSIS_FINALIZE new file mode 100755 index 00000000000..39b20ba6627 --- /dev/null +++ b/jobs/JGLOBAL_LAND_ANALYSIS_FINALIZE @@ -0,0 +1,50 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export WIPE_DATA="NO" +export DATA=${DATA:-${DATAROOT}/${RUN}landanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "landanlfinal" -c "base landanl landanlfinal" + +############################################## +# Set variables used in the script +############################################## +GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +GDUMP="gdas" + +############################################## +# Begin JOB SPECIFIC work +############################################## + +export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos} +mkdir -p "${COMOUT}" + +# COMIN_GES and COMIN_GES_ENS are used in script +export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos" +export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos" + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASLANDFINALPY:-${HOMEgfs}/scripts/exglobal_land_analysis_finalize.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || ( echo "FATAL ERROR: ${DATAROOT} does not exist, ABORT!"; exit 1 ) +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/jobs/JGLOBAL_LAND_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_LAND_ANALYSIS_INITIALIZE new file mode 100755 index 00000000000..5f71d58a00c --- /dev/null +++ b/jobs/JGLOBAL_LAND_ANALYSIS_INITIALIZE @@ -0,0 +1,43 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export DATA=${DATA:-${DATAROOT}/${RUN}landanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "landanlinit" -c "base landanl landanlinit" + +############################################## +# Set variables used in the script +############################################## +GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +GDUMP="gdas" + +############################################## +# Begin JOB SPECIFIC work +############################################## + +export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos} +mkdir -p "${COMOUT}" + +# COMIN_GES and COMIN_GES_ENS are used in script +export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos" +export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos" + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASLANDINITPY:-${HOMEgfs}/scripts/exglobal_land_analysis_initialize.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/JGLOBAL_LAND_ANALYSIS_RUN b/jobs/JGLOBAL_LAND_ANALYSIS_RUN new file mode 100755 index 00000000000..d489b603f43 --- /dev/null +++ b/jobs/JGLOBAL_LAND_ANALYSIS_RUN @@ -0,0 +1,44 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export WIPE_DATA="NO" +export DATA=${DATA:-${DATAROOT}/${RUN}landanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "landanlrun" -c "base landanl landanlrun" + +############################################## +# Set variables used in the script +############################################## +GDATE=$(date +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +GDUMP="gdas" + +############################################## +# Begin JOB SPECIFIC work +############################################## + +export COMOUT=${COMOUT:-${ROTDIR}/${RUN}.${PDY}/${cyc}/atmos} +mkdir -p "${COMOUT}" + +# COMIN_GES and COMIN_GES_ENS are used in script +export COMIN_GES="${ROTDIR}/${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos" +export COMIN_GES_ENS="${ROTDIR}/enkf${GDUMP}.${GDATE:0:8}/${GDATE:8:2}/atmos" + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASLANDRUNSH:-${HOMEgfs}/scripts/exglobal_land_analysis_run.sh} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/rocoto/landanlfinal.sh b/jobs/rocoto/landanlfinal.sh new file mode 100755 index 00000000000..a6fa48c679d --- /dev/null +++ b/jobs/rocoto/landanlfinal.sh @@ -0,0 +1,23 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="landanlfinal" +export jobid="${job}.$$" + +############################################################### +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_LAND_ANALYSIS_FINALIZE" +status=$? +exit "${status}" diff --git a/jobs/rocoto/landanlinit.sh b/jobs/rocoto/landanlinit.sh new file mode 100755 index 00000000000..e9c0b2d7a2e --- /dev/null +++ b/jobs/rocoto/landanlinit.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="landanlinit" +export jobid="${job}.$$" + +############################################################### +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_LAND_ANALYSIS_INITIALIZE" +status=$? +exit "${status}" diff --git a/jobs/rocoto/landanlrun.sh b/jobs/rocoto/landanlrun.sh new file mode 100755 index 00000000000..3f306a32be8 --- /dev/null +++ b/jobs/rocoto/landanlrun.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="landanlrun" +export jobid="${job}.$$" + +############################################################### +# setup python path for workflow utilities and tasks +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_LAND_ANALYSIS_RUN" +status=$? +exit "${status}" diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn index bfc68adf7f8..51f60bab131 100644 --- a/parm/config/config.base.emc.dyn +++ b/parm/config/config.base.emc.dyn @@ -311,6 +311,7 @@ export imp_physics=@IMP_PHYSICS@ export DO_JEDIVAR="NO" export DO_JEDIENS="NO" export DO_JEDIOCNVAR="NO" +export DO_JEDILANDDA="NO" export DO_MERGENSST="NO" # Hybrid related diff --git a/parm/config/config.landanl b/parm/config/config.landanl new file mode 100755 index 00000000000..89bb8a4b7bf --- /dev/null +++ b/parm/config/config.landanl @@ -0,0 +1,23 @@ +#! /usr/bin/env bash + +########## config.landanl ########## +# configuration common to all land analysis tasks + +echo "BEGIN: config.landanl" + +obs_list_name=gdas_land_adpsfc_only.yaml +if [[ "${cyc}" == "18" ]]; then + obs_list_name=gdas_land_prototype.yaml +fi + +export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/land/obs/config/ +export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/land/obs/lists/${obs_list_name} +export LANDVARYAML=${HOMEgfs}/sorc/gdas.cd/parm/land/letkfoi/letkfoi.yaml +export FV3JEDI_FIX=${HOMEgfs}/fix/gdas + +export io_layout_x=@IO_LAYOUT_X@ +export io_layout_y=@IO_LAYOUT_Y@ + +export JEDIEXE=${HOMEgfs}/exec/fv3jedi_letkf.x + +echo "END: config.landanl" diff --git a/parm/config/config.landanlfinal b/parm/config/config.landanlfinal new file mode 100755 index 00000000000..242089325a4 --- /dev/null +++ b/parm/config/config.landanlfinal @@ -0,0 +1,10 @@ +#! /usr/bin/env bash + +########## config.landanlfinal ########## +# Post Land Analysis specific + +echo "BEGIN: config.landanlfinal" + +# Get task specific resources +. "${EXPDIR}/config.resources" landanlfinal +echo "END: config.landanlfinal" diff --git a/parm/config/config.landanlinit b/parm/config/config.landanlinit new file mode 100755 index 00000000000..62054525c88 --- /dev/null +++ b/parm/config/config.landanlinit @@ -0,0 +1,10 @@ +#! /usr/bin/env bash + +########## config.landanlinit ########## +# Pre Land Analysis specific + +echo "BEGIN: config.landanlinit" + +# Get task specific resources +. "${EXPDIR}/config.resources" landanlinit +echo "END: config.landanlinit" diff --git a/parm/config/config.landanlrun b/parm/config/config.landanlrun new file mode 100755 index 00000000000..0f44011c1d7 --- /dev/null +++ b/parm/config/config.landanlrun @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +########## config.landanlrun ########## +# Land Analysis specific + +echo "BEGIN: config.landanlrun" + +# Get task specific resources +. "${EXPDIR}/config.resources" landanlrun + +echo "END: config.landanlrun" diff --git a/parm/config/config.resources b/parm/config/config.resources index 96666cc60d8..30eae816709 100644 --- a/parm/config/config.resources +++ b/parm/config/config.resources @@ -11,6 +11,7 @@ if [[ $# -ne 1 ]]; then echo "getic init coupled_ic aerosol_init" echo "atmanlinit atmanlrun atmanlfinal" echo "atmensanlinit atmensanlrun atmensanlfinal" + echo "landanlinit landanlrun landanlfinal" echo "aeroanlinit aeroanlrun aeroanlfinal" echo "anal sfcanl analcalc analdiag gldas fcst post vrfy fit2obs metp arch echgres" echo "eobs ediag eomg eupd ecen esfc efcs epos earc" @@ -204,6 +205,46 @@ elif [[ "${step}" = "atmanlfinal" ]]; then export npe_node_atmanlfinal export is_exclusive=True +elif [[ "${step}" = "landanlinit" || "${step}" = "landanlrun" || "${step}" = "landanlfinal" ]]; then + # below lines are for creating JEDI YAML + case ${CASE} in + C768) + layout_x=6 + layout_y=6 + ;; + C384) + layout_x=5 + layout_y=5 + ;; + C192 | C96 | C48) + layout_x=1 + layout_y=1 + ;; + *) + echo "FATAL ERROR: Resolution not supported for land analysis'" + exit 1 + esac + + export layout_x + export layout_y + + if [[ "${step}" = "landanlinit" || "${step}" = "landanlfinal" ]]; then + declare -x "wtime_${step}"="00:10:00" + declare -x "npe_${step}"=1 + declare -x "nth_${step}"=1 + temp_stepname="nth_${step}" + declare -x "npe_node_${step}"="$(echo "${npe_node_max} / ${!temp_stepname}" | bc)" + declare -x "memory_${step}"="3072M" + elif [[ "${step}" = "landanlrun" ]]; then + export wtime_landanlrun="00:30:00" + npe_landanlrun=$(echo "${layout_x} * ${layout_y} * 6" | bc) + export npe_landanlrun + export nth_landanlrun=1 + npe_node_landanlrun=$(echo "${npe_node_max} / ${nth_landanlrun}" | bc) + export npe_node_landanlrun + export is_exclusive=True + fi + elif [[ "${step}" = "aeroanlinit" ]]; then # below lines are for creating JEDI YAML diff --git a/parm/config/yaml/defaults.yaml b/parm/config/yaml/defaults.yaml index 82899746684..4c3817ef01d 100644 --- a/parm/config/yaml/defaults.yaml +++ b/parm/config/yaml/defaults.yaml @@ -2,6 +2,10 @@ aeroanl: IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 +landanl: + IO_LAYOUT_X: 1 + IO_LAYOUT_Y: 1 + ocnanal: SOCA_INPUT_FIX_DIR: '/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25' CASE_ANL: 'C48'