diff --git a/build.sh b/build.sh index 04f150edf..fd1d45870 100755 --- a/build.sh +++ b/build.sh @@ -72,7 +72,7 @@ while getopts "p:t:c:hvdfa" opt; do done case ${BUILD_TARGET} in - hera | orion | hercules | wcoss2 | noaacloud | gaeac5 | gaeac6 ) + hera | orion | hercules | wcoss2 | noaacloud | gaeac5 | gaeac6 | ursa ) echo "Building GDASApp on $BUILD_TARGET" source $dir_root/ush/module-setup.sh module use $dir_root/modulefiles diff --git a/modulefiles/GDAS/ursa.intel.lua b/modulefiles/GDAS/ursa.intel.lua new file mode 100644 index 000000000..c9122c501 --- /dev/null +++ b/modulefiles/GDAS/ursa.intel.lua @@ -0,0 +1,93 @@ +help([[ +Load environment for running the GDAS application with Intel compilers and MPI. +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +prepend_path("MODULEPATH", '/contrib/spack-stack/spack-stack-1.9.1/envs/ue-oneapi-2024.2.1/install/modulefiles/Core') + +load("stack-oneapi/2024.2.1") +load("stack-intel-oneapi-mpi/2021.13") +load("stack-python/3.11.7") + +load("cmake/3.30.2") +load("gettext/0.22.5") +load("curl/8.10.1") +load("zlib/1.2.13") +load("git/2.43.5") +load("hdf5/1.14.3") +load("parallel-netcdf/1.12.3") +load("netcdf-c/4.9.2") +load("nccmp/1.9.1") +load("netcdf-fortran/4.6.1") +load("nco/5.2.4") +load("parallelio/2.6.2") +load("wget/1.21.1") +load("boost/1.84.0") +load("bufr/12.1.0") +load("git-lfs/3.5.1") +load("ecbuild/3.7.2") +load("openjpeg/2.3.1") +load("eccodes/2.33.0") +load("eigen/3.4.0") +load("openblas/0.3.24") +load("eckit/1.28.3") +load("fftw/3.3.10") +load("fckit/0.13.2") +load("fiat/1.4.1") +load("fms/2024.02") +load("esmf/8.8.0") +load("atlas/0.40.0") +load("sp/2.5.0") +load("gsl-lite/0.37.0") +load("libjpeg/2.1.0") +load("krb5/1.21.1-1") +load("libtirpc/1.3.3") +load("hdf/4.2.15") +load("jedi-cmake/1.4.0") +load("libpng/1.6.37") +load("libxt/1.3.0") +load("libxmu/1.2.1") +load("libxpm/3.5.17") +load("libxaw/1.0.16") +load("udunits/2.2.28") +load("ncview/2.1.9") +load("netcdf-cxx4/4.3.1") +load("json/3.11.3") +--load("crtm/v2.4_jedi") +load("rocoto/1.3.7") +load("prod_util/2.1.1") + +load("py-jinja2/3.1.4") +load("py-netcdf4/1.7.1.post2") +load("py-pybind11/2.13.5") +load("py-pycodestyle/2.11.0") +load("py-pyyaml/6.0.2") +load("py-scipy/1.14.1") +load("py-xarray/2024.7.0") +load("py-f90nml/1.4.3") +load("py-pip/23.1.2") +load("py-click/8.1.7") + +setenv("CC","mpiicx") +setenv("CXX","mpiicpx") +setenv("FC","mpiifort") +setenv("I_MPI_CC", "icx") +setenv("I_MPI_CXX", "icpx") +setenv("I_MPI_F90", "ifort") + +local mpiexec = '/apps/slurm/default/bin/srun' +local mpinproc = '-n' +setenv('MPIEXEC_EXEC', mpiexec) +setenv('MPIEXEC_NPROC', mpinproc) + +setenv("CRTM_FIX","/scratch3/NCEPDEV/da/role.jedipara/GDASApp/fix/crtm/2.4.0") +setenv("GDASAPP_TESTDATA","/scratch3/NCEPDEV/da/role.jedipara/GDASApp/testdata") +setenv("GDASAPP_UNIT_TEST_DATA_PATH", "/scratch3/NCEPDEV/da/role.jedipara/GDASApp/unittestdata") + +whatis("Name: ".. pkgName) +whatis("Version: ".. tostring(pkgVersion)) +whatis("Category: GDASApp") +whatis("Description: Load all libraries needed for GDASApp") diff --git a/test/atm/global-workflow/jjob_ens_final.sh b/test/atm/global-workflow/jjob_ens_final.sh index 8b93b6698..479ee5bd0 100755 --- a/test/atm/global-workflow/jjob_ens_final.sh +++ b/test/atm/global-workflow/jjob_ens_final.sh @@ -48,6 +48,8 @@ fi # Execute j-job if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmensanlfinal-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE +elif [[ $machine = 'URSA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:10:00 --export=ALL --wait --output=atmensanlfinal-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_inc.sh b/test/atm/global-workflow/jjob_ens_inc.sh index 194cbe3a3..2798eb5cb 100755 --- a/test/atm/global-workflow/jjob_ens_inc.sh +++ b/test/atm/global-workflow/jjob_ens_inc.sh @@ -47,7 +47,9 @@ fi # Execute j-job if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlfv3inc-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT + sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlfv3inc-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT +elif [[ $machine = 'URSA' ]]; then + sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:30:00 --export=ALL --wait --output=atmensanlfv3inc-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT fi diff --git a/test/atm/global-workflow/jjob_ens_init.sh b/test/atm/global-workflow/jjob_ens_init.sh index cfbced790..dce1d69cd 100755 --- a/test/atm/global-workflow/jjob_ens_init.sh +++ b/test/atm/global-workflow/jjob_ens_init.sh @@ -113,6 +113,8 @@ cp $EXPDIR/config.base_lobsdiag_forenkf_false $EXPDIR/config.base # Execute j-job if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmensanlinit-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE +elif [[ $machine = 'URSA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:10:00 --export=ALL --wait --output=atmensanlinit-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_init_split.sh b/test/atm/global-workflow/jjob_ens_init_split.sh index bd4d0b9e9..52595c10f 100755 --- a/test/atm/global-workflow/jjob_ens_init_split.sh +++ b/test/atm/global-workflow/jjob_ens_init_split.sh @@ -116,6 +116,8 @@ cp $EXPDIR/config.base_lobsdiag_forenkf_true $EXPDIR/config.base # Execute j-job if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmensanlinit_split-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE +elif [[ $machine = 'URSA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:10:00 --export=ALL --wait --output=atmensanlinit_split-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_letkf.sh b/test/atm/global-workflow/jjob_ens_letkf.sh index 93715541d..adfe4884a 100755 --- a/test/atm/global-workflow/jjob_ens_letkf.sh +++ b/test/atm/global-workflow/jjob_ens_letkf.sh @@ -53,7 +53,9 @@ cp $EXPDIR/config.base_lobsdiag_forenkf_false $EXPDIR/config.base # Execute j-job if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlletkf-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF + sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlletkf-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF +elif [[ $machine = 'URSA' ]]; then + sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --partition=u1-compute --mem="32Gb" --time=00:30:00 --export=ALL --wait --output=atmensanlletkf-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF fi diff --git a/test/atm/global-workflow/jjob_ens_obs.sh b/test/atm/global-workflow/jjob_ens_obs.sh index 126b68d7e..fac36ef9a 100755 --- a/test/atm/global-workflow/jjob_ens_obs.sh +++ b/test/atm/global-workflow/jjob_ens_obs.sh @@ -53,7 +53,9 @@ cp $EXPDIR/config.base_lobsdiag_forenkf_true $EXPDIR/config.base # Execute j-job if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlobs-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS + sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlobs-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS +elif [[ $machine = 'URSA' ]]; then + sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:30:00 --export=ALL --wait --output=atmensanlobs-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS fi diff --git a/test/atm/global-workflow/jjob_ens_sol.sh b/test/atm/global-workflow/jjob_ens_sol.sh index 331bc72f8..68dab69aa 100755 --- a/test/atm/global-workflow/jjob_ens_sol.sh +++ b/test/atm/global-workflow/jjob_ens_sol.sh @@ -53,7 +53,9 @@ cp $EXPDIR/config.base_lobsdiag_forenkf_true $EXPDIR/config.base # Execute j-job if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlsol-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL + sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait --output=atmensanlsol-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL +elif [[ $machine = 'URSA' ]]; then + sbatch --nodes=1 --ntasks=6 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:30:00 --export=ALL --wait --output=atmensanlsol-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL fi diff --git a/test/atm/global-workflow/jjob_var_final.sh b/test/atm/global-workflow/jjob_var_final.sh index 6d178f61c..694f52efe 100755 --- a/test/atm/global-workflow/jjob_var_final.sh +++ b/test/atm/global-workflow/jjob_var_final.sh @@ -48,6 +48,8 @@ fi # Execute j-job if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmanlfinal-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE +elif [[ $machine = 'URSA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:10:00 --export=ALL --wait --output=atmanlfinal-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_var_inc.sh b/test/atm/global-workflow/jjob_var_inc.sh index ebb8bc420..457f71299 100755 --- a/test/atm/global-workflow/jjob_var_inc.sh +++ b/test/atm/global-workflow/jjob_var_inc.sh @@ -48,6 +48,8 @@ fi # Execute j-job if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmanlfv3inc-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT +elif [[ $machine = "URSA" ]]; then + sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:10:00 --export=ALL --wait --output=atmanlfv3inc-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT fi diff --git a/test/atm/global-workflow/jjob_var_init.sh b/test/atm/global-workflow/jjob_var_init.sh index b69b3f56d..453a1448a 100755 --- a/test/atm/global-workflow/jjob_var_init.sh +++ b/test/atm/global-workflow/jjob_var_init.sh @@ -124,6 +124,8 @@ done # Execute j-job if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait --output=atmanlinit-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE +elif [[ $machine = 'URSA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:10:00 --export=ALL --wait --output=atmanlinit-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_var_run.sh b/test/atm/global-workflow/jjob_var_run.sh index 6f9543fc7..abf6e8d38 100755 --- a/test/atm/global-workflow/jjob_var_run.sh +++ b/test/atm/global-workflow/jjob_var_run.sh @@ -51,6 +51,8 @@ fi # Execute j-job if [[ $machine = 'HERA' || $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:20:00 --export=ALL --wait --output=atmanlvar-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL +elif [[ $machine = 'URSA' ]]; then + sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --partition=u1-compute --time=00:20:00 --mem=96Gb --export=ALL --wait --output=atmanlvar-%j.out ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL fi diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 9301c8492..f2018bd87 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -31,6 +31,10 @@ case $(hostname -f) in hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 hecflow01) MACHINE_ID=hera ;; ### heraecflow01 + ufe0[1-9]) MACHINE_ID=ursa ;; ### ursa01-09 + ufe1[0-2]) MACHINE_ID=ursa ;; ### ursa10-12 + uecflow01) MACHINE_ID=ursa ;; ### ursaecflow01 + s4-submit.ssec.wisc.edu) MACHINE_ID=s4 ;; ### s4 fe[1-8]) MACHINE_ID=jet ;; ### jet01-8 @@ -73,9 +77,14 @@ elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then elif [[ -d /mnt/lfs5 ]]; then # We are on NOAA Jet MACHINE_ID=jet -elif [[ -d /scratch1 ]]; then - # We are on NOAA Hera - MACHINE_ID=hera +elif [[ -d /scratch3 ]]; then + # We are on NOAA Hera or Ursa + mount=$(findmnt -n -o SOURCE /home) + if [[ ${mount} =~ "ursa" ]]; then + MACHINE_ID=ursa + else + MACHINE_ID=hera + fi elif [[ -d /work ]]; then # We are on MSU Orion or Hercules mount=$(findmnt -n -o SOURCE /home) diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 11536daf3..5d50660b4 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -15,6 +15,13 @@ elif [[ $MACHINE_ID = hera* ]] ; then fi module purge +elif [[ $MACHINE_ID = ursa* ]] ; then + # We are on NOAA Ursa + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/lmod/lmod/init/bash + fi + module purge + elif [[ $MACHINE_ID = orion* ]] ; then # We are on Orion if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/utils/test/prepdata.sh b/utils/test/prepdata.sh index cf28b99e0..a5b511ca5 100755 --- a/utils/test/prepdata.sh +++ b/utils/test/prepdata.sh @@ -6,15 +6,6 @@ # Get the machine hostname MACHINE_NAME=$(hostname) -# Check if the machine name is "hera" -if [[ "$MACHINE_NAME" =~ ^hfe0[1-9]$ || "$MACHINE_NAME" =~ ^hfe1[01]$ ]]; then - echo "Running on hera, loading anaconda modules." - module use -a /contrib/anaconda/modulefiles - module load anaconda/latest -else - echo "Not running on hera, skipping anaconda module loading." -fi - set -e cdl2nc4() { @@ -23,7 +14,7 @@ cdl2nc4() { local input_cdl="$2" echo "Generating ${output_nc4}" - ncgen -o "$output_nc4" "$input_cdl" + ncgen -4 -o "$output_nc4" "$input_cdl" } project_source_dir=$1