diff --git a/modulefiles/GDAS/gaeac6.intel.lua b/modulefiles/GDAS/gaeac6.intel.lua index b156f7409..f8a620e27 100644 --- a/modulefiles/GDAS/gaeac6.intel.lua +++ b/modulefiles/GDAS/gaeac6.intel.lua @@ -8,7 +8,6 @@ local pkgNameVer = myModuleFullName() prepend_path("MODULEPATH", '/ncrc/proj/epic/spack-stack/c6/spack-stack-1.9.2/envs/ue-intel-2023.2.0/install/modulefiles/Core') prepend_path("MODULEPATH", '/ncrc/proj/epic/spack-stack/c6/spack-stack-1.9.2/envs/ue-intel-2023.2.0/install/modulefiles/gcc/12.3.0') -prepend_path("MODULEPATH", '/ncrc/proj/epic/rocoto/modulefiles') -- below two lines get us access to the spack-stack modules load("stack-intel/2023.2.0") @@ -58,7 +57,6 @@ load("ncview/2.1.9") load("netcdf-cxx4/4.3.1") load("json/3.11.3") --load("crtm/2.4.0.1") -load("rocoto/1.3.6") load("prod_util/2.1.1") load("py-jinja2/3.1.4") @@ -73,6 +71,12 @@ load("py-pip/23.1.2") unload("cray-libsci") +prepend_path("MODULEPATH", "/autofs/ncrc-svm1_proj/hurr1/hafs/shared/modulefiles") +load("rocoto/1.3.7_fix") + +append_path("MODULEPATH", "/usw/hpss/modulefiles") +load("hsi/9.3") + setenv("CC","cc") setenv("CXX","CC") setenv("FC","ftn") diff --git a/modulefiles/GDAS/wcoss2.intel.lua b/modulefiles/GDAS/wcoss2.intel.lua index 23f7d11b9..598e32bc2 100644 --- a/modulefiles/GDAS/wcoss2.intel.lua +++ b/modulefiles/GDAS/wcoss2.intel.lua @@ -52,7 +52,6 @@ load("libxaw/1.0.13") load("udunits/2.2.28") load("ncview/2.1.9") load("netcdf-cxx4/4.3.1") -load("core/rocoto/1.3.5") load("prod_util/2.0.14") load("py-setuptools/63.4.3") @@ -76,6 +75,9 @@ load("py-markupsafe/2.1.3") load("py-cftime/1.0.3.4") load("py-packaging/23.1") +prepend_path("MODULEPATH", "/apps/ops/test/nco/modulefiles/core") +load("rocoto/1.3.5") + setenv("CC","cc") setenv("CXX","CC") setenv("FC","ftn") diff --git a/test/atm/global-workflow/jjob_ens_final.sh b/test/atm/global-workflow/jjob_ens_final.sh index 2bfa5273b..7f3ee1071 100755 --- a/test/atm/global-workflow/jjob_ens_final.sh +++ b/test/atm/global-workflow/jjob_ens_final.sh @@ -4,6 +4,8 @@ set -x bindir=$1 srcdir=$2 +type="jjob_ens_final" + # Set g-w HOMEgfs topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) export HOMEgfs=$topdir @@ -37,23 +39,37 @@ wxflowPATH="${HOMEgfs}/ush/python" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH -# Set NETCDF and UTILROOT variables (used in config.base) -if [[ $MACHINE_ID = 'hera' ]]; then - NETCDF=$( which ncdump ) - export NETCDF - export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [[ $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - ncdump=$( which ncdump ) - NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) - export NETCDF - export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 +# Export library path +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${HOMEgfs}/lib" + +# Create yaml with job configuration +memory="8Gb" +if [[ ${MACHINE_ID} == "gaeac6" ]]; then + memory=0 fi +config_yaml="./config_${type}.yaml" +cat < ${config_yaml} +machine: ${MACHINE_ID} +homegfs: ${HOMEgfs} +job_name: ${type} +walltime: "00:30:00" +nodes: 1 +ntasks_per_node: 1 +threads_per_task: 1 +memory: ${memory} +command: ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE +filename: submit_${type}.sh +EOF + +# Create script to execute j-job +$HOMEgfs/sorc/gdas.cd/test/workflow/generate_job_script.py ${config_yaml} +SCHEDULER=$(echo `grep SCHEDULER ${HOMEgfs}/sorc/gdas.cd/test/workflow/hosts/${MACHINE_ID}.yaml | cut -d":" -f2` | tr -d ' ') -# Execute j-job -if [[ $MACHINE_ID = 'hera' || $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmensanlfinal-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE -elif [[ $MACHINE_ID = 'ursa' ]]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:10:00 --export=ALL --wait --output=atmensanlfinal-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE +# Submit script to execute j-job +if [[ $SCHEDULER = 'slurm' ]]; then + sbatch --export=ALL --wait submit_${type}.sh +elif [[ $SCHEDULER = 'pbspro' ]]; then + qsub -V -W block=true submit_${type}.sh else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_inc.sh b/test/atm/global-workflow/jjob_ens_inc.sh index 402581f47..59f917617 100755 --- a/test/atm/global-workflow/jjob_ens_inc.sh +++ b/test/atm/global-workflow/jjob_ens_inc.sh @@ -4,6 +4,8 @@ set -x bindir=$1 srcdir=$2 +type="jjob_ens_inc" + # Set g-w HOMEgfs topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) export HOMEgfs=$topdir @@ -37,23 +39,37 @@ wxflowPATH="${HOMEgfs}/ush/python" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH -# Set NETCDF and UTILROOT variables (used in config.base) -if [[ $MACHINE_ID = 'hera' ]]; then - NETCDF=$( which ncdump ) - export NETCDF - export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [[ $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - ncdump=$( which ncdump ) - NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) - export NETCDF - export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 +# Export library path +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${HOMEgfs}/lib" + +# Create yaml with job configuration +memory="8Gb" +if [[ ${MACHINE_ID} == "gaeac6" ]]; then + memory=0 fi +config_yaml="./config_${type}.yaml" +cat < ${config_yaml} +machine: ${MACHINE_ID} +homegfs: ${HOMEgfs} +job_name: ${type} +walltime: "00:30:00" +nodes: 1 +ntasks_per_node: 6 +threads_per_task: 1 +memory: ${memory} +command: ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT +filename: submit_${type}.sh +EOF + +# Create script to execute j-job +$HOMEgfs/sorc/gdas.cd/test/workflow/generate_job_script.py ${config_yaml} +SCHEDULER=$(echo `grep SCHEDULER ${HOMEgfs}/sorc/gdas.cd/test/workflow/hosts/${MACHINE_ID}.yaml | cut -d":" -f2` | tr -d ' ') -# Execute j-job -if [[ $MACHINE_ID = 'hera' || $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlfv3inc-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT -elif [[ $MACHINE_ID = 'ursa' ]]; then - sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:30:00 --export=ALL --wait --output=atmensanlfv3inc-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT +# Submit script to execute j-job +if [[ $SCHEDULER = 'slurm' ]]; then + sbatch --export=ALL --wait submit_${type}.sh +elif [[ $SCHEDULER = 'pbspro' ]]; then + qsub -V -W block=true submit_${type}.sh else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT fi diff --git a/test/atm/global-workflow/jjob_ens_init.sh b/test/atm/global-workflow/jjob_ens_init.sh index 8c2922399..86acb2e54 100755 --- a/test/atm/global-workflow/jjob_ens_init.sh +++ b/test/atm/global-workflow/jjob_ens_init.sh @@ -4,6 +4,8 @@ set -x bindir=$1 srcdir=$2 +type="jjob_ens_init" + # Set g-w HOMEgfs topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) export HOMEgfs=$topdir @@ -41,17 +43,8 @@ wxflowPATH="${HOMEgfs}/ush/python" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH -# Set NETCDF and UTILROOT variables (used in config.base) -if [[ $MACHINE_ID = 'hera' ]]; then - NETCDF=$( which ncdump ) - export NETCDF - export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [[ $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - ncdump=$( which ncdump ) - NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) - export NETCDF - export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 -fi +# Export library path +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${HOMEgfs}/lib" # Set date variables for previous cycle gPDY=$(date +%Y%m%d -d "${PDY} ${cyc} - 6 hours") @@ -113,11 +106,34 @@ done # NOTE: atmensanlinit creates input yaml for atmensanlletkf job cp $EXPDIR/config.base_lobsdiag_forenkf_false $EXPDIR/config.base -# Execute j-job -if [[ $MACHINE_ID = 'hera' || $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmensanlinit-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE -elif [[ $MACHINE_ID = 'ursa' ]]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:10:00 --export=ALL --wait --output=atmensanlinit-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE +# Create yaml with job configuration +memory="8Gb" +if [[ ${MACHINE_ID} == "gaeac6" ]]; then + memory=0 +fi +config_yaml="./config_${type}.yaml" +cat < ${config_yaml} +machine: ${MACHINE_ID} +homegfs: ${HOMEgfs} +job_name: ${type} +walltime: "00:30:00" +nodes: 1 +ntasks_per_node: 1 +threads_per_task: 1 +memory: ${memory} +command: ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE +filename: submit_${type}.sh +EOF + +# Create script to execute j-job +$HOMEgfs/sorc/gdas.cd/test/workflow/generate_job_script.py ${config_yaml} +SCHEDULER=$(echo `grep SCHEDULER ${HOMEgfs}/sorc/gdas.cd/test/workflow/hosts/${MACHINE_ID}.yaml | cut -d":" -f2` | tr -d ' ') + +# Submit script to execute j-job +if [[ $SCHEDULER = 'slurm' ]]; then + sbatch --export=ALL --wait submit_${type}.sh +elif [[ $SCHEDULER = 'pbspro' ]]; then + qsub -V -W block=true submit_${type}.sh else - ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE + ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_init_split.sh b/test/atm/global-workflow/jjob_ens_init_split.sh index 01ac03eb0..df8217c5c 100755 --- a/test/atm/global-workflow/jjob_ens_init_split.sh +++ b/test/atm/global-workflow/jjob_ens_init_split.sh @@ -4,6 +4,8 @@ set -x bindir=$1 srcdir=$2 +type="jjob_ens_init_split" + # Set g-w HOMEgfs topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) export HOMEgfs=$topdir @@ -41,17 +43,8 @@ wxflowPATH="${HOMEgfs}/ush/python" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH -# Set NETCDF and UTILROOT variables (used in config.base) -if [[ $MACHINE_ID = 'hera' ]]; then - NETCDF=$( which ncdump ) - export NETCDF - export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [[ $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - ncdump=$( which ncdump ) - NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) - export NETCDF - export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 -fi +# Export library path +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${HOMEgfs}/lib" # Set date variables for previous cycle gPDY=$(date +%Y%m%d -d "${PDY} ${cyc} - 6 hours") @@ -116,11 +109,34 @@ done # NOTE: atmensanlinit creates input yaml for atmensanlobs and atmensanlsol jobs cp $EXPDIR/config.base_lobsdiag_forenkf_true $EXPDIR/config.base -# Execute j-job -if [[ $MACHINE_ID = 'hera' || $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmensanlinit_split-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE -elif [[ $MACHINE_ID = 'ursa' ]]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:10:00 --export=ALL --wait --output=atmensanlinit_split-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE +# Create yaml with job configuration +memory="8Gb" +if [[ ${MACHINE_ID} == "gaeac6" ]]; then + memory=0 +fi +config_yaml="./config_${type}.yaml" +cat < ${config_yaml} +machine: ${MACHINE_ID} +homegfs: ${HOMEgfs} +job_name: ${type} +walltime: "00:30:00" +nodes: 1 +ntasks_per_node: 1 +threads_per_task: 1 +memory: ${memory} +command: ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE +filename: submit_${type}.sh +EOF + +# Create script to execute j-job +$HOMEgfs/sorc/gdas.cd/test/workflow/generate_job_script.py ${config_yaml} +SCHEDULER=$(echo `grep SCHEDULER ${HOMEgfs}/sorc/gdas.cd/test/workflow/hosts/${MACHINE_ID}.yaml | cut -d":" -f2` | tr -d ' ') + +# Submit script to execute j-job +if [[ $SCHEDULER = 'slurm' ]]; then + sbatch --export=ALL --wait submit_${type}.sh +elif [[ $SCHEDULER = 'pbspro' ]]; then + qsub -V -W block=true submit_${type}.sh else - ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE + ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_letkf.sh b/test/atm/global-workflow/jjob_ens_letkf.sh index f79fc1841..7bc590670 100755 --- a/test/atm/global-workflow/jjob_ens_letkf.sh +++ b/test/atm/global-workflow/jjob_ens_letkf.sh @@ -4,6 +4,8 @@ set -x bindir=$1 srcdir=$2 +type="jjob_ens_letkf" + # Set g-w HOMEgfs topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) export HOMEgfs=$topdir @@ -39,26 +41,40 @@ wxflowPATH="${HOMEgfs}/ush/python" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH -# Set NETCDF and UTILROOT variables (used in config.base) -if [[ $MACHINE_ID = 'hera' ]]; then - NETCDF=$( which ncdump ) - export NETCDF - export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [[ $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - ncdump=$( which ncdump ) - NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) - export NETCDF - export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 -fi +# Export library path +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${HOMEgfs}/lib" # Set lobsdiag_forenkf=.false. to run letkf as combined observer and solver job cp $EXPDIR/config.base_lobsdiag_forenkf_false $EXPDIR/config.base -# Execute j-job -if [[ $MACHINE_ID = 'hera' || $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:30:00 --mem="32Gb" --export=ALL --wait --output=atmensanlletkf-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF -elif [[ $MACHINE_ID = 'ursa' ]]; then - sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --partition=u1-compute --mem="32Gb" --time=00:30:00 --export=ALL --wait --output=atmensanlletkf-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF +# Create yaml with job configuration +memory="32Gb" +if [[ ${MACHINE_ID} == "gaeac6" ]]; then + memory=0 +fi +config_yaml="./config_${type}.yaml" +cat < ${config_yaml} +machine: ${MACHINE_ID} +homegfs: ${HOMEgfs} +job_name: ${type} +walltime: "00:30:00" +nodes: 1 +ntasks_per_node: 6 +threads_per_task: 1 +memory: ${memory} +command: ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF +filename: submit_${type}.sh +EOF + +# Create script to execute j-job +$HOMEgfs/sorc/gdas.cd/test/workflow/generate_job_script.py ${config_yaml} +SCHEDULER=$(echo `grep SCHEDULER ${HOMEgfs}/sorc/gdas.cd/test/workflow/hosts/${MACHINE_ID}.yaml | cut -d":" -f2` | tr -d ' ') + +# Submit script to execute j-job +if [[ $SCHEDULER = 'slurm' ]]; then + sbatch --export=ALL --wait submit_${type}.sh +elif [[ $SCHEDULER = 'pbspro' ]]; then + qsub -V -W block=true submit_${type}.sh else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF fi diff --git a/test/atm/global-workflow/jjob_ens_obs.sh b/test/atm/global-workflow/jjob_ens_obs.sh index 814f0d2d0..6b192caf3 100755 --- a/test/atm/global-workflow/jjob_ens_obs.sh +++ b/test/atm/global-workflow/jjob_ens_obs.sh @@ -4,6 +4,8 @@ set -x bindir=$1 srcdir=$2 +type="jjob_ens_obs" + # Set g-w HOMEgfs topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) export HOMEgfs=$topdir @@ -39,26 +41,40 @@ wxflowPATH="${HOMEgfs}/ush/python" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH -# Set NETCDF and UTILROOT variables (used in config.base) -if [[ $MACHINE_ID = 'hera' ]]; then - NETCDF=$( which ncdump ) - export NETCDF - export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [[ $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - ncdump=$( which ncdump ) - NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) - export NETCDF - export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 -fi +# Export library path +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${HOMEgfs}/lib" # Set lobsdiag_forenkf=.true. to run letkf as stand-alone observer job cp $EXPDIR/config.base_lobsdiag_forenkf_true $EXPDIR/config.base -# Execute j-job -if [[ $MACHINE_ID = 'hera' || $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules`' ]]; then - sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlobs-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS -elif [[ $MACHINE_ID = 'ursa' ]]; then - sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:30:00 --export=ALL --wait --output=atmensanlobs-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS +# Create yaml with job configuration +memory="32Gb" +if [[ ${MACHINE_ID} == "gaeac6" ]]; then + memory=0 +fi +config_yaml="./config_${type}.yaml" +cat < ${config_yaml} +machine: ${MACHINE_ID} +homegfs: ${HOMEgfs} +job_name: ${type} +walltime: "00:30:00" +nodes: 1 +ntasks_per_node: 6 +threads_per_task: 1 +memory: ${memory} +command: ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS +filename: submit_${type}.sh +EOF + +# Create script to execute j-job +$HOMEgfs/sorc/gdas.cd/test/workflow/generate_job_script.py ${config_yaml} +SCHEDULER=$(echo `grep SCHEDULER ${HOMEgfs}/sorc/gdas.cd/test/workflow/hosts/${MACHINE_ID}.yaml | cut -d":" -f2` | tr -d ' ') + +# Submit script to execute j-job +if [[ $SCHEDULER = 'slurm' ]]; then + sbatch --export=ALL --wait submit_${type}.sh +elif [[ $SCHEDULER = 'pbspro' ]]; then + qsub -V -W block=true submit_${type}.sh else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS fi diff --git a/test/atm/global-workflow/jjob_ens_sol.sh b/test/atm/global-workflow/jjob_ens_sol.sh index 538745f42..d8ba91d8a 100755 --- a/test/atm/global-workflow/jjob_ens_sol.sh +++ b/test/atm/global-workflow/jjob_ens_sol.sh @@ -4,6 +4,8 @@ set -x bindir=$1 srcdir=$2 +type="jjob_ens_sol" + # Set g-w HOMEgfs topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) export HOMEgfs=$topdir @@ -39,26 +41,40 @@ wxflowPATH="${HOMEgfs}/ush/python" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH -# Set NETCDF and UTILROOT variables (used in config.base) -if [[ $MACHINE_ID = 'hera' ]]; then - NETCDF=$( which ncdump ) - export NETCDF - export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [[ $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - ncdump=$( which ncdump ) - NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) - export NETCDF - export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 -fi +# Export library path +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${HOMEgfs}/lib" # Set lobsdiag_forenkf=.true. to run letkf as stand-alone solver job cp $EXPDIR/config.base_lobsdiag_forenkf_true $EXPDIR/config.base -# Execute j-job -if [[ $MACHINE_ID = 'hera' || $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlsol-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL -elif [[ $MACHINE_ID = 'ursa' ]]; then - sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:30:00 --export=ALL --wait --output=atmensanlsol-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL +# Create yaml with job configuration +memory="8Gb" +if [[ ${MACHINE_ID} == "gaeac6" ]]; then + memory=0 +fi +config_yaml="./config_${type}.yaml" +cat < ${config_yaml} +machine: ${MACHINE_ID} +homegfs: ${HOMEgfs} +job_name: ${type} +walltime: "00:30:00" +nodes: 1 +ntasks_per_node: 6 +threads_per_task: 1 +memory: ${memory} +command: ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL +filename: submit_${type}.sh +EOF + +# Create script to execute j-job +$HOMEgfs/sorc/gdas.cd/test/workflow/generate_job_script.py ${config_yaml} +SCHEDULER=$(echo `grep SCHEDULER ${HOMEgfs}/sorc/gdas.cd/test/workflow/hosts/${MACHINE_ID}.yaml | cut -d":" -f2` | tr -d ' ') + +# Submit script to execute j-job +if [[ $SCHEDULER = 'slurm' ]]; then + sbatch --export=ALL --wait submit_${type}.sh +elif [[ $SCHEDULER = 'pbspro' ]]; then + qsub -V -W block=true submit_${type}.sh else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL fi diff --git a/test/atm/global-workflow/jjob_var_final.sh b/test/atm/global-workflow/jjob_var_final.sh index ea841d8f0..bafc98e67 100755 --- a/test/atm/global-workflow/jjob_var_final.sh +++ b/test/atm/global-workflow/jjob_var_final.sh @@ -4,6 +4,8 @@ set -x bindir=$1 srcdir=$2 +type="jjob_var_final" + # Set g-w HOMEgfs topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) export HOMEgfs=$topdir @@ -37,23 +39,37 @@ wxflowPATH="${HOMEgfs}/ush/python" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH -# Set NETCDF and UTILROOT variables (used in config.base) -if [[ $MACHINE_ID = 'hera' ]]; then - NETCDF=$( which ncdump ) - export NETCDF - export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [[ $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - ncdump=$( which ncdump ) - NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) - export NETCDF - export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 +# Export library path +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${HOMEgfs}/lib" + +# Create yaml with job configuration +memory="8Gb" +if [[ ${MACHINE_ID} == "gaeac6" ]]; then + memory=0 fi +config_yaml="./config_${type}.yaml" +cat < ${config_yaml} +machine: ${MACHINE_ID} +homegfs: ${HOMEgfs} +job_name: ${type} +walltime: "00:30:00" +nodes: 1 +ntasks_per_node: 1 +threads_per_task: 1 +memory: ${memory} +command: ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE +filename: submit_${type}.sh +EOF + +# Create script to execute j-job +$HOMEgfs/sorc/gdas.cd/test/workflow/generate_job_script.py ${config_yaml} +SCHEDULER=$(echo `grep SCHEDULER ${HOMEgfs}/sorc/gdas.cd/test/workflow/hosts/${MACHINE_ID}.yaml | cut -d":" -f2` | tr -d ' ') -# Execute j-job -if [[ $MACHINE_ID = 'hera' || $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmanlfinal-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE -elif [[ $MACHINE_ID = 'ursa' ]]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:10:00 --export=ALL --wait --output=atmanlfinal-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE +# Submit script to execute j-job +if [[ $SCHEDULER = 'slurm' ]]; then + sbatch --export=ALL --wait submit_${type}.sh +elif [[ $SCHEDULER = 'pbspro' ]]; then + qsub -V -W block=true submit_${type}.sh else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_var_inc.sh b/test/atm/global-workflow/jjob_var_inc.sh index ad3bbac41..449ebd676 100755 --- a/test/atm/global-workflow/jjob_var_inc.sh +++ b/test/atm/global-workflow/jjob_var_inc.sh @@ -4,6 +4,8 @@ set -x bindir=$1 srcdir=$2 +type="jjob_var_inc" + # Set g-w HOMEgfs topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) export HOMEgfs=$topdir @@ -37,23 +39,37 @@ wxflowPATH="${HOMEgfs}/ush/python" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH -# Set NETCDF and UTILROOT variables (used in config.base) -if [[ $MACHINE_ID = 'hera' ]]; then - NETCDF=$( which ncdump ) - export NETCDF - export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [[ $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - ncdump=$( which ncdump ) - NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) - export NETCDF - export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 +# Export library path +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${HOMEgfs}/lib" + +# Create yaml with job confiugration +memory="8Gb" +if [[ ${MACHINE_ID} == "gaeac6" ]]; then + memory=0 fi +config_yaml="./config_${type}.yaml" +cat < ${config_yaml} +machine: ${MACHINE_ID} +homegfs: ${HOMEgfs} +job_name: ${type} +walltime: "00:30:00" +nodes: 1 +ntasks_per_node: 6 +threads_per_task: 1 +memory: ${memory} +command: ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT +filename: submit_${type}.sh +EOF + +# Create script to execute j-job +$HOMEgfs/sorc/gdas.cd/test/workflow/generate_job_script.py ${config_yaml} +SCHEDULER=$(echo `grep SCHEDULER ${HOMEgfs}/sorc/gdas.cd/test/workflow/hosts/${MACHINE_ID}.yaml | cut -d":" -f2` | tr -d ' ') -# Execute j-job -if [[ $MACHINE_ID = 'hera' || $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmanlfv3inc-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT -elif [[ $MACHINE_ID = "ursa" ]]; then - sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:10:00 --export=ALL --wait --output=atmanlfv3inc-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT +# Submit script to execute j-job +if [[ $SCHEDULER = 'slurm' ]]; then + sbatch --export=ALL --wait submit_${type}.sh +elif [[ $SCHEDULER = 'pbspro' ]]; then + qsub -V -W block=true submit_${type}.sh else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT fi diff --git a/test/atm/global-workflow/jjob_var_init.sh b/test/atm/global-workflow/jjob_var_init.sh index 64ae4246d..b2d83f06f 100755 --- a/test/atm/global-workflow/jjob_var_init.sh +++ b/test/atm/global-workflow/jjob_var_init.sh @@ -4,6 +4,8 @@ set -x bindir=$1 srcdir=$2 +type="jjob_var_init" + # Set g-w HOMEgfs topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) export HOMEgfs=$topdir @@ -41,17 +43,8 @@ wxflowPATH="${HOMEgfs}/ush/python" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH -# Set NETCDF and UTILROOT variables (used in config.base) -if [[ $MACHINE_ID = 'hera' ]]; then - NETCDF=$( which ncdump ) - export NETCDF - export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [[ $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - ncdump=$( which ncdump ) - NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) - export NETCDF - export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 -fi +# Export library path +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${HOMEgfs}/lib" # Set date variables for previous cycle gPDY=$(date +%Y%m%d -d "${PDY} ${cyc} - 6 hours") @@ -85,7 +78,6 @@ for file in $flist; do ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.${file} $COMIN_ATMOS_ANALYSIS_PREV/$gprefix.${file} done - # Link atmospheric history on gaussian grid dpath=gdas.$gPDY/$gcyc/model/atmos/history mkdir -p $COMIN_ATMOS_HISTORY_PREV @@ -109,13 +101,13 @@ for imem in $(seq 1 $NMEM_ENS); do COMIN_ATMOS_HISTORY_PREV_ENS:COM_ATMOS_HISTORY_TMPL source=$GDASAPP_TESTDATA/lowres/$dpath/$memchar/model/atmos/history - target=$COMIN__ATMOS_HISTORY_PREV_ENS + target=$COMIN_ATMOS_HISTORY_PREV_ENS mkdir -p $target rm -rf $target/enkfgdas.t${gcyc}z.atmf006.nc ln -fs $source/enkfgdas.t${gcyc}z.atmf006.nc $target/ source=$GDASAPP_TESTDATA/lowres/$dpath/$memchar/model/atmos/history - target=$COMIN__ATMOS_HISTORY_PREV_ENS + target=$COMIN_ATMOS_HISTORY_PREV_ENS flist=("cubed_sphere_grid_atmf006.nc" "cubed_sphere_grid_sfcf006.nc") for file in "${flist[@]}"; do rm -rf $target/enkf${gprefix}.${file} @@ -123,11 +115,34 @@ for imem in $(seq 1 $NMEM_ENS); do done done -# Execute j-job -if [[ $MACHINE_ID = 'hera' || $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmanlinit-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE -elif [[ $MACHINE_ID = 'ursa' ]]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:10:00 --export=ALL --wait --output=atmanlinit-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE +# Create yaml with job configuration +memory="8Gb" +if [[ ${MACHINE_ID} == "gaeac6" ]]; then + memory=0 +fi +config_yaml="./config_${type}.yaml" +cat < ${config_yaml} +machine: ${MACHINE_ID} +homegfs: ${HOMEgfs} +job_name: ${type} +walltime: "00:30:00" +nodes: 1 +ntasks_per_node: 1 +threads_per_task: 1 +memory: ${memory} +command: ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE +filename: submit_${type}.sh +EOF + +# Create script to execute j-job +$HOMEgfs/sorc/gdas.cd/test/workflow/generate_job_script.py ${config_yaml} +SCHEDULER=$(echo `grep SCHEDULER ${HOMEgfs}/sorc/gdas.cd/test/workflow/hosts/${MACHINE_ID}.yaml | cut -d":" -f2` | tr -d ' ') + +# Submit script to execute j-job +if [[ $SCHEDULER = 'slurm' ]]; then + sbatch --export=ALL --wait submit_${type}.sh +elif [[ $SCHEDULER = 'pbspro' ]]; then + qsub -V -W block=true submit_${type}.sh else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_var_run.sh b/test/atm/global-workflow/jjob_var_run.sh index 60fed3078..45bb70fc3 100755 --- a/test/atm/global-workflow/jjob_var_run.sh +++ b/test/atm/global-workflow/jjob_var_run.sh @@ -1,9 +1,12 @@ #! /usr/bin/env bash set -x + bindir=$1 srcdir=$2 +type="jjob_var_run" + # Set g-w HOMEgfs topdir=$(cd "$(dirname "$(readlink -f -n "${bindir}" )" )/../../.." && pwd -P) export HOMEgfs=$topdir @@ -39,23 +42,37 @@ wxflowPATH="${HOMEgfs}/ush/python" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" export PYTHONPATH -# Set NETCDF and UTILROOT variables (used in config.base) -if [[ $MACHINE_ID = 'hera' ]]; then - NETCDF=$( which ncdump ) - export NETCDF - export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [[ $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - ncdump=$( which ncdump ) - NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) - export NETCDF - export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 +# Export library path +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${HOMEgfs}/lib" + +# Create yaml with job configuration +memory="96Gb" +if [[ ${MACHINE_ID} == "gaeac6" ]]; then + memory=0 fi +config_yaml="./config_${type}.yaml" +cat < ${config_yaml} +machine: ${MACHINE_ID} +homegfs: ${HOMEgfs} +job_name: ${type} +walltime: "00:30:00" +nodes: 1 +ntasks_per_node: 6 +threads_per_task: 1 +memory: ${memory} +command: ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL +filename: submit_${type}.sh +EOF + +# Create script to execute j-job. Set job scheduler +${HOMEgfs}/sorc/gdas.cd/test/workflow/generate_job_script.py ${config_yaml} +SCHEDULER=$(echo `grep SCHEDULER ${HOMEgfs}/sorc/gdas.cd/test/workflow/hosts/${MACHINE_ID}.yaml | cut -d":" -f2` | tr -d ' ') -# Execute j-job -if [[ $MACHINE_ID = 'hera' || $MACHINE_ID = 'orion' || $MACHINE_ID = 'hercules' ]]; then - sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:30:00 --mem=96Gb --export=ALL --wait --output=atmanlvar-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL -elif [[ $MACHINE_ID = 'ursa' ]]; then - sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:20:00 --mem=96Gb --export=ALL --wait --output=atmanlvar-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL +# Submit script to execute j-job +if [[ $SCHEDULER = 'slurm' ]]; then + sbatch --export=ALL --wait submit_${type}.sh +elif [[ $SCHEDULER = 'pbspro' ]]; then + qsub -V -W block=true submit_${type}.sh else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL fi diff --git a/test/gw-ci/create_exp.sh b/test/gw-ci/create_exp.sh index 891229b85..9093fbe40 100755 --- a/test/gw-ci/create_exp.sh +++ b/test/gw-ci/create_exp.sh @@ -15,6 +15,8 @@ export RUNTESTS=${exp_path}/${pslot} export HPC_ACCOUNT="da-cpu" if [[ $MACHINE_ID = wcoss2 ]]; then export HPC_ACCOUNT="GFS-DEV" +elif [[ $MACHINE_ID = gaeac6 ]]; then + export HPC_ACCOUNT="gfs-cpu" fi # Source the gw environement diff --git a/test/gw-ci/run_exp.sh b/test/gw-ci/run_exp.sh index 3c02dbb1a..8597ba63f 100755 --- a/test/gw-ci/run_exp.sh +++ b/test/gw-ci/run_exp.sh @@ -13,9 +13,9 @@ WORKFLOW_DB=${pslot}/EXPDIR/${pslot}/${pslot}.db # Boot the task echo "booting ${TASK_ARRAY[@]} for cycle $CYCLE" if [[ ! -e "$WORKFLOW_DB" ]]; then - rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task_args" -c "$CYCLE" + rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task_args" -c "$CYCLE" fi -rocotoboot -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task_args" -c "$CYCLE" +rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task_args" -c "$CYCLE" # Loop through tasks IFS=',' read -r -a TASK_ARRAY <<< "$task_args" @@ -23,7 +23,7 @@ num_tasks=${#TASK_ARRAY[@]} while true; do # Update the status of the task rocotorun -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task_args" -c "$CYCLE" - + num_succeeded=0 for task in "${TASK_ARRAY[@]}"; do @@ -44,8 +44,8 @@ while true; do echo "$pslot"_"$task"_"$CYCLE"" is in state: $STATUS" else echo "$pslot"_"$task"_"$CYCLE"" is in unrecognized state: $STATUS. Rewinding..." - rocotorewind -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task_args" -c "$CYCLE" - rocotoboot -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task_args" -c "$CYCLE" + rocotorewind -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task" -c "$CYCLE" + rocotoboot -w "$WORKFLOW_XML" -d "$WORKFLOW_DB" -t "$task" -c "$CYCLE" fi done if [[ "$num_succeeded" == "$num_tasks" ]]; then diff --git a/test/snow/apply_jedi_incr.sh b/test/snow/apply_jedi_incr.sh index 60413c58d..c32b991a1 100755 --- a/test/snow/apply_jedi_incr.sh +++ b/test/snow/apply_jedi_incr.sh @@ -18,6 +18,10 @@ EXECDIR=$project_source_dir/build/bin WORKDIR=$project_binary_dir/test/snow/apply_jedi_incr RSTDIR=$GDASAPP_TESTDATA/lowres/gdas.$GYMD/$GHR/model/atmos/restart INCDIR=$GDASAPP_TESTDATA/snow/C${RES} +HOMEgfs=$project_source_dir/../../ + +# Detect machine +source "${HOMEgfs}/ush/detect_machine.sh" export TPATH="$GDASAPP_TESTDATA/snow/C${RES}" export TSTUB="C${RES}_oro_data" @@ -77,9 +81,55 @@ done echo 'do_snowDA: calling apply snow increment' -# (n=6) -> this is fixed, at one task per tile (with minor code change, could run on a single proc). -srun '--export=ALL' -n 6 ${EXECDIR}/apply_incr.exe ${WORKDIR}/apply_incr.log +# Create script to run executable +runsh="./apply_incr.sh" +cat < $runsh +#!/bin/bash +set -ex +# Set APRUN for machine +APRUN="srun -n 6" +if [[ ${MACHINE_ID} == 'wcoss' ]]; then + APRUN="mpiexec -n 6" +fi + +# Run executable +\${APRUN} ${EXECDIR}/apply_incr.exe ${WORKDIR}/apply_incr.log +EOF +chmod 755 $runsh +# Create yaml with job configuration +memory="8Gb" +if [[ ${MACHINE_ID} == "gaeac6" ]]; then + memory=0 +fi +submitsh="./submit.sh" +config_yaml="./config.yaml" +cat < ${config_yaml} +machine: ${MACHINE_ID} +homegfs: ${HOMEgfs} +job_name: apply_jedi_incr +walltime: "00:30:00" +nodes: 1 +ntasks_per_node: 6 +threads_per_task: 1 +memory: ${memory} +command: ${runsh} +filename: ${submitsh} +EOF + + +# Create submission script +$HOMEgfs/sorc/gdas.cd/test/workflow/generate_job_script.py ${config_yaml} +SCHEDULER=$(echo `grep SCHEDULER ${HOMEgfs}/sorc/gdas.cd/test/workflow/hosts/${MACHINE_ID}.yaml | cut -d":" -f2` | tr -d ' ') + +# Submit script +if [[ $SCHEDULER = 'slurm' ]]; then + sbatch --export=ALL --wait ${submitsh} +elif [[ $SCHEDULER = 'pbspro' ]]; then + qsub -V -W block=true ${submitsh} +else + echo "UNKOWN SCHEDULER $SCHEDULER" +fi rc=$? exit $rc diff --git a/test/snow/test_imsproc.sh b/test/snow/test_imsproc.sh index 01cd6ec61..83d7f493d 100755 --- a/test/snow/test_imsproc.sh +++ b/test/snow/test_imsproc.sh @@ -18,6 +18,10 @@ DOY=$(date +%j -d "$YY$MM$DD + 1 day") EXECDIR=$project_source_dir/build/bin WORKDIR=$project_binary_dir/test/snow/ims_proc RSTDIR=$GDASAPP_TESTDATA/lowres/gdas.$GYMD/$GHR/model/atmos/restart +HOMEgfs=$project_source_dir/../.. + +# Detect machine +source "${HOMEgfs}/ush/detect_machine.sh" export OBSDIR=$GDASAPP_TESTDATA/snow/snow_ice_cover export TSTUB="oro_C${RES}.mx100" @@ -59,6 +63,11 @@ ${EXECDIR}/calcfIMS.exe export PYTHONPATH=$PYTHONPATH:${project_source_dir}/iodaconv/src/:${project_source_dir}/build/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR} IMS_IODA=${EXECDIR}/imsfv3_scf2ioda.py +# TODO: Remove LD_LIBRARY_PATH line as soon as permanent solution is available +if [[ ${MACHINE_ID} == 'wcoss2' ]]; then + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/cray/pe/mpich/8.1.19/ofi/intel/19.0/lib" +fi + echo 'do_snowDA: calling ioda converter' python ${IMS_IODA} -i IMSscf.${YY}${MM}${DD}.${TSTUB}.nc -o ioda.IMSscf.${YY}${MM}${DD}.${TSTUB}.nc diff --git a/test/workflow/generate_job_script.py b/test/workflow/generate_job_script.py new file mode 100755 index 000000000..b65351906 --- /dev/null +++ b/test/workflow/generate_job_script.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +import os +import yaml +import sys + + +def create_job_script(job_config, machine_config): + scheduler = machine_config.get('SCHEDULER', 'slurm') + account = machine_config.get('HPC_ACCOUNT') + queue = machine_config.get('QUEUE') + partition = machine_config.get('PARTITION_BATCH', 'none') + cluster = machine_config.get('CLUSTERS', 'none') + machine_id = machine_config.get('MACHINE_ID', 'none') + job_name = job_config.get('job_name', 'myjob') + walltime = job_config.get('walltime', '01:00:00') + nodes = job_config.get('nodes', 1) + ntasks_per_node = job_config.get('ntasks_per_node', 1) + threads_per_task = job_config.get('threads_per_task', 1) + ncpus = ntasks_per_node * threads_per_task + memory = job_config.get('memory', '4gb') + command = job_config.get('command', 'python script.py') + filename = job_config.get('filename', 'job_script.sh') + + if scheduler == 'pbspro': + script = f"""#!/bin/bash +#PBS -N {job_name} +#PBS -j oe +#PBS -A {account} +#PBS -q {queue} +#PBS -l walltime={walltime} +#PBS -l select={nodes}:mpiprocs={ntasks_per_node}:ompthreads={threads_per_task}:ncpus={ncpus}:mem={memory} +#PBS -l place=vscatter + +set -x +cd $PBS_O_WORKDIR +{command} +""" + elif scheduler == 'slurm': + if machine_id == 'gaeac6': + script = f"""#!/bin/bash +#SBATCH -J {job_name} +#SBATCH -o {job_name}.o%J +#SBATCH -e {job_name}.o%J +#SBATCH -A {account} +#SBATCH -q {queue} +#SBATCH -p {partition} +#SBATCH -M {cluster} +#SBATCH -t {walltime} +#SBATCH --nodes={nodes} +#SBATCH --ntasks-per-node={ntasks_per_node} +#SBATCH --cpus-per-task={threads_per_task} +#SBATCH --mem={memory} + +set -x +cd $SLURM_SUBMIT_DIR +{command} +""" + else: + script = f"""#!/bin/bash +#SBATCH -J {job_name} +#SBATCH -o {job_name}.o%J +#SBATCH -e {job_name}.o%J +#SBATCH -A {account} +#SBATCH -q {queue} +#SBATCH -p {partition} +#SBATCH -t {walltime} +#SBATCH --nodes={nodes} +#SBATCH --ntasks-per-node={ntasks_per_node} +#SBATCH --cpus-per-task={threads_per_task} +#SBATCH --mem={memory} + +set -x +cd $SLURM_SUBMIT_DIR +{command} +""" + else: + raise ValueError("Unsupported scheduler. Use 'pbspro' or 'slurm'.") + + with open(filename, 'w') as f: + f.write(script) + + os.chmod(filename, 0o755) + print(f"{scheduler.upper()} job script written to {filename}") + + +def main(): + if len(sys.argv) != 2: + print("Usage: python generate_job_script.py config.yaml") + sys.exit(1) + + job_config_file = sys.argv[1] + with open(job_config_file, 'r') as f: + job_config = yaml.safe_load(f) + + homegfs = job_config.get('homegfs') + machine = job_config.get('machine') + + machine_config_file = os.path.join(homegfs, "sorc/gdas.cd/test/workflow/hosts/") + machine.lower() + ".yaml" + + with open(machine_config_file, 'r') as f: + machine_config = yaml.safe_load(f) + + machine_config["MACHINE_ID"] = machine.lower() + print(f"machine_config {machine_config}") + + create_job_script(job_config, machine_config) + + +if __name__ == "__main__": + main() diff --git a/test/workflow/hosts/gaeac6.yaml b/test/workflow/hosts/gaeac6.yaml new file mode 100644 index 000000000..64452e7ea --- /dev/null +++ b/test/workflow/hosts/gaeac6.yaml @@ -0,0 +1,19 @@ +# Scheduler and queue information +SCHEDULER: slurm +QUEUE: normal +QUEUE_SERVICE: normal +QUEUE_DTN: 'hpss' +PARTITION_BUILD: eslogin_c6 +PARTITION_BATCH: batch +PARTITION_SERVICE: batch +PARTITION_DTN: 'dtn_f5_f6' +CLUSTERS: 'c6' +CLUSTERS_DTN: 'es' +CLUSTERS_BUILD: 'es' +CONSTRAINT_DTN: 'f6' +RESERVATION: '' +PARTITION_CRON: 'cron_c6' +USE_SCRONTAB: 'YES' + +# HPC account information +HPC_ACCOUNT: gfs-cpu diff --git a/test/workflow/hosts/hera.yaml b/test/workflow/hosts/hera.yaml new file mode 100644 index 000000000..383b234fe --- /dev/null +++ b/test/workflow/hosts/hera.yaml @@ -0,0 +1,8 @@ +# Scheduler and queue information +SCHEDULER: slurm +QUEUE: batch +PARTITION_BATCH: hera +PARTITION_SERVICE: service + +# HPC account information +HPC_ACCOUNT: da-cpu diff --git a/test/workflow/hosts/hercules.yaml b/test/workflow/hosts/hercules.yaml new file mode 100644 index 000000000..a08e5f4b7 --- /dev/null +++ b/test/workflow/hosts/hercules.yaml @@ -0,0 +1,8 @@ +# Scheduler and queue information +SCHEDULER: slurm +QUEUE: batch +PARTITION_BATCH: hercules +PARTITION_SERVICE: service + +# HPC account information +HPC_ACCOUNT: da-cpu diff --git a/test/workflow/hosts/orion.yaml b/test/workflow/hosts/orion.yaml new file mode 100644 index 000000000..c475ae542 --- /dev/null +++ b/test/workflow/hosts/orion.yaml @@ -0,0 +1,8 @@ +# Scheduler and queue information +SCHEDULER: slurm +QUEUE: batch +PARTITION_BATCH: orion +PARTITION_SERVICE: service + +# HPC account information +HPC_ACCOUNT: da-cpu diff --git a/test/workflow/hosts/ursa.yaml b/test/workflow/hosts/ursa.yaml new file mode 100644 index 000000000..e91e5593e --- /dev/null +++ b/test/workflow/hosts/ursa.yaml @@ -0,0 +1,8 @@ +# Scheduler and queue information +SCHEDULER: slurm +QUEUE: batch +PARTITION_BATCH: u1-compute +PARTITION_SERVICE: service + +# HPC account information +HPC_ACCOUNT: da-cpu diff --git a/test/workflow/hosts/wcoss2.yaml b/test/workflow/hosts/wcoss2.yaml new file mode 100644 index 000000000..98195cddb --- /dev/null +++ b/test/workflow/hosts/wcoss2.yaml @@ -0,0 +1,7 @@ +# Scheduler and queue information +SCHEDULER: pbspro +QUEUE: dev +QUEUE_SERVICE: dev_transfer + +# HPC account information +HPC_ACCOUNT: GFS-DEV diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt index e196c1c16..a74bea39c 100644 --- a/utils/test/CMakeLists.txt +++ b/utils/test/CMakeLists.txt @@ -2,7 +2,7 @@ list( APPEND utils_test_input testinput/gdas_meanioda.yaml testinput/gdas_rads2ioda.yaml - testinput/gdas_ghrsst2ioda.yaml +# testinput/gdas_ghrsst2ioda.yaml testinput/gdas_rtofstmp.yaml testinput/gdas_rtofssal.yaml testinput/gdas_smap2ioda.yaml @@ -12,12 +12,12 @@ list( APPEND utils_test_input testinput/gdas_icecmirs2ioda.yaml testinput/gdas_icecjpssrr2ioda.yaml testinput/gdas_insituall2ioda.yaml - testinput/gdas_viirsaod2ioda.yaml +# testinput/gdas_viirsaod2ioda.yaml ) set( gdas_utils_test_ref testref/rads2ioda.test - testref/ghrsst2ioda.test +# testref/ghrsst2ioda.test testref/rtofstmp.test testref/rtofssal.test testref/smap2ioda.test @@ -27,7 +27,7 @@ set( gdas_utils_test_ref testref/icecmirs2ioda.test testref/icecjpssrr2ioda.test testref/insituall2ioda.test - testref/viirsaod2ioda.test +# testref/viirsaod2ioda.test ) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testinput) @@ -74,12 +74,12 @@ ecbuild_add_test( TARGET test_gdasapp_util_rads2ioda TEST_DEPENDS test_gdasapp_util_prepdata) # Test the GHRSST to IODA converter -ecbuild_add_test( TARGET test_gdasapp_util_ghrsst2ioda - COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_ioda_obsprovider2ioda.x - ARGS "../testinput/gdas_ghrsst2ioda.yaml" - LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc - TEST_DEPENDS test_gdasapp_util_prepdata) +#ecbuild_add_test( TARGET test_gdasapp_util_ghrsst2ioda +# COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_ioda_obsprovider2ioda.x +# ARGS "../testinput/gdas_ghrsst2ioda.yaml" +# LIBS gdas-utils +# WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc +# TEST_DEPENDS test_gdasapp_util_prepdata) # copy rtofs binary input files to the testing area # and generate the tests @@ -146,12 +146,12 @@ ecbuild_add_test( TARGET test_gdasapp_util_smos2ioda TEST_DEPENDS test_gdasapp_util_prepdata) # Test the VIIRS AOD to IODA converter -ecbuild_add_test( TARGET test_gdasapp_util_viirsaod2ioda - COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_ioda_obsprovider2ioda.x - ARGS "../testinput/gdas_viirsaod2ioda.yaml" - LIBS gdas-utils - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc - TEST_DEPENDS test_gdasapp_util_prepdata) +#ecbuild_add_test( TARGET test_gdasapp_util_viirsaod2ioda +# COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_ioda_obsprovider2ioda.x +# ARGS "../testinput/gdas_viirsaod2ioda.yaml" +# LIBS gdas-utils +# WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc +# TEST_DEPENDS test_gdasapp_util_prepdata) # Test the ABI to IODA converter ecbuild_add_test( TARGET test_gdasapp_util_icecabi2ioda