diff --git a/modulefiles/ufs_expanse.intel b/modulefiles/ufs_expanse.intel deleted file mode 100644 index c3a6e52ce6..0000000000 --- a/modulefiles/ufs_expanse.intel +++ /dev/null @@ -1,32 +0,0 @@ -#%Module - -proc ModulesHelp {} { - puts stderr "\tcit - loads modules required for building and running UFS Model on Expanse/Intel" -} - -module-whatis "loads UFS Model prerequisites for Expanse/Intel" - -module load slurm/expanse/20.02.3 -module load cpu/0.15.4 -module load intel/19.1.1.217 -module load intel-mpi/2019.8.254 -module load python/3.8.5 -module load cmake/3.18.2 - -module use module use /expanse/lustre/scratch/domh/temp_project/hpc-stack-20210929/modulefiles/stack - -module load hpc/1.2.0 - -module load hpc-intel/19.1.1.217 -module load hpc-intel-mpi/2019.8.254 - -module load jasper/2.0.25 -module load zlib/1.2.11 -module load png/1.6.35 - -module load ufs_common - -setenv CC mpiicc -setenv CXX mpiicpc -setenv FC mpiifort -setenv CMAKE_Platform expanse.intel diff --git a/modulefiles/ufs_odin b/modulefiles/ufs_odin deleted file mode 100644 index bca7e0680a..0000000000 --- a/modulefiles/ufs_odin +++ /dev/null @@ -1,20 +0,0 @@ -#%Module###################################################################### -## -## nems prerequisites -## - -proc ModulesHelp {} { - puts stderr "The prerequisites for compiling or running FV3 on Odin. " -} - -#module load newdefaults intel/2016.2.181 mvapich2/1.8 szip hdf5 netcdf4/4.2.1.1 - -module use /scratch/ywang/external/modulefiles -module load bacio/v2.0.2 -module load sp/v2.0.2 -module load ip/v3.0.0 -module load w3nco/v2.0.6 -module load w3emc/v2.3.0 -module load nemsio/v2.2.2 - -module load esmf/7.1.0r diff --git a/modulefiles/ufs_stampede.intel b/modulefiles/ufs_stampede.intel deleted file mode 100644 index 968cc9dd9f..0000000000 --- a/modulefiles/ufs_stampede.intel +++ /dev/null @@ -1,55 +0,0 @@ -#%Module###################################################################### -## -## NEMS FV3 Prerequisites: Stampede/Intel - -proc ModulesHelp {} { - puts stderr "\tcit - loads modules required for building and running FV3 on Stampede/Intel" -} - -module-whatis "loads NEMS FV3 prerequisites for Stampede/Intel" - -# NOTE: the "module purge" and loading of the module command are -# handled by the module-setup.sh (or .csh) script. - -## -## NCEP libraries -## -module use /work2/08048/hanglei/stampede2/NCEP/install/hpc-stack/modulefiles/stack -module load hpc/1.1.0 -module load hpc-intel/18.0.2 -module load hpc-impi/18.0.2 -module load python3/3.7.0 - -module load bacio/2.4.1 -module load crtm/2.3.0 -module load g2/3.4.3 -module load g2tmpl/1.10.2 -module load ip/3.3.3 -module load nemsio/2.5.2 -module load sp/2.3.3 -module load w3emc/2.7.3 -module load w3nco/2.4.1 -module load upp/10.0.8 - -module load gfsio/1.4.1 -module load sfcio/1.4.1 -module load sigio/2.3.2 - -module load jasper/2.0.25 -module load png/1.6.35 -module load zlib/1.2.11 - -module load hdf5/1.10.6 -module load netcdf/4.7.4 -module load pio/2.5.3 -module load fms/2020.04.03 -module load esmf/8_1_1 - -## -## load cmake -## -module load cmake/3.20.1 -setenv CC mpiicc -setenv CXX mpiicpc -setenv FC mpiifort -setenv CMAKE_Platform stampede.intel diff --git a/tests/default_vars.sh b/tests/default_vars.sh index 39a22ff5dd..b16efa108c 100644 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -363,20 +363,6 @@ elif [[ ${MACHINE_ID} = derecho ]]; then export WPG_cpl_atmw_gdas=24 export WAV_tasks_atmw_gdas=248 -elif [[ ${MACHINE_ID} = stampede ]]; then - - echo "Unknown MACHINE_ID ${MACHINE_ID}. Please update tasks configurations in default_vars.sh" - exit 1 - - # TPN_dflt=48 ; INPES_dflt=3 ; JNPES_dflt=8 - # TPN_thrd=24 ; INPES_thrd=3 ; JNPES_thrd=4 - # TPN_c384=20 ; INPES_c384=8 ; JNPES_c384=6 - # TPN_c768=20 ; INPES_c768=8 ; JNPES_c768=16 - # TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4 - - # TPN_cpl_atmw_gdas=12; INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8 - # THRD_cpl_atmw_gdas=4; WPG_cpl_atmw_gdas=24; APB_cpl_atmw_gdas="0 311"; WPB_cpl_atmw_gdas="312 559" - elif [[ ${MACHINE_ID} = noaacloud ]] ; then if [[ ${PW_CSP} == aws ]]; then @@ -415,18 +401,6 @@ elif [[ ${MACHINE_ID} = noaacloud ]] ; then export ICE_tasks_cpl_thrd=10 export WAV_tasks_cpl_thrd=12 -elif [[ ${MACHINE_ID} = expanse ]]; then - - echo "Unknown MACHINE_ID ${MACHINE_ID}. Please update tasks configurations in default_vars.sh" - exit 1 - - # TPN_dflt=64 ; INPES_dflt=3 ; JNPES_dflt=8 - # TPN_thrd=64 ; INPES_thrd=3 ; JNPES_thrd=4 - # TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4 - - # TPN_cpl_atmw_gdas=12; INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8 - # THRD_cpl_atmw_gdas=2; WPG_cpl_atmw_gdas=24; APB_cpl_atmw_gdas="0 311"; WPB_cpl_atmw_gdas="312 559" - else echo "Unknown MACHINE_ID ${MACHINE_ID}" diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index 99419f3555..5895e09f7e 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -48,13 +48,9 @@ case $(hostname -f) in derecho7.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho7 derecho8.hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho8 - login[1-4].stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1-4 - login[1-4].frontera.tacc.utexas.edu) MACHINE_ID=frontera ;; ### frontera1-4 c*.frontera.tacc.utexas.edu) MACHINE_ID=frontera ;; ### frontera compute - login0[1-2].expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse1-2 - discover3[1-5].prv.cube) MACHINE_ID=discover ;; ### discover31-35 *) MACHINE_ID=UNKNOWN ;; # Unknown platform esac diff --git a/tests/fv3_conf/fv3_slurm.IN_expanse b/tests/fv3_conf/fv3_slurm.IN_expanse deleted file mode 100644 index 6c4819806a..0000000000 --- a/tests/fv3_conf/fv3_slurm.IN_expanse +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -#SBATCH --job-name="@[JBNME]" -#SBATCH -o out -#SBATCH -e err -#SBATCH -A @[ACCNR] -#SBATCH -p @[QUEUE] -#SBATCH --nodes=@[NODES] -#SBATCH --ntasks-per-node=@[TPN] -#SBATCH -t @[WLCLK] - -set -eux -echo -n " $( date +%s )," > job_timestamp.txt - -set +x -MACHINE_ID=expanse -source ./module-setup.sh -module use $PWD/modulefiles -module load modules.fv3 -module list -set -x - -ulimit -s unlimited - -echo "Model started: "`date` - -export OMP_STACK_SIZE=512M -export OMP_NUM_THREADS=@[THRD] -export I_MPI_PMI_LIBRARY=/cm/shared/apps/slurm/current/lib64/libpmi.so - -# This "if" block is part of the rt.sh self-tests in error-test.conf. It emulates the model failing to run. -if [ "${JOB_SHOULD_FAIL:-NO}" = WHEN_RUNNING ] ; then - echo "The job should abort now, with exit status 1." 1>&2 - echo "If error checking is working, the metascheduler should mark the job as failed." 1>&2 - false -fi - -srun -n @[TASKS] ./fv3.exe - -echo "Model ended: " `date` -echo -n " $( date +%s )," >> job_timestamp.txt diff --git a/tests/fv3_conf/fv3_slurm.IN_stampede b/tests/fv3_conf/fv3_slurm.IN_stampede deleted file mode 100644 index d80a3e068c..0000000000 --- a/tests/fv3_conf/fv3_slurm.IN_stampede +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -l -#SBATCH -o out -#SBATCH -e err -#SBATCH -N @[NODES] -#SBATCH --ntasks-per-node=@[TPN] -#SBATCH -A @[ACCNR] -#SBATCH -p @[QUEUE] -#SBATCH -t 00:@[WLCLK]:00 - -set -eux -echo -n " $( date +%s )," > job_timestamp.txt - -set +x -MACHINE_ID=stampede -source ./module-setup.sh -module use $PWD/modulefiles -module load modules.fv3 -module list -set -x - -echo "Model started: " `date` - -export MPI_TYPE_DEPTH=20 -export OMP_STACKSIZE=512M -export OMP_NUM_THREADS=@[THRD] -export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4 -export LD_BIND_NOW=1 - -# Avoid job errors because of filesystem synchronization delays -#sync && sleep 1 - -# This "if" block is part of the rt.sh self-tests in error-test.conf. It emulates the model failing to run. -if [ "${JOB_SHOULD_FAIL:-NO}" = WHEN_RUNNING ] ; then - echo "The job should abort now, with exit status 1." 1>&2 - echo "If error checking is working, the metascheduler should mark the job as failed." 1>&2 - false -fi - -#mpirun -prepend-rank -np $SBATCH_NP ./fv3.exe -ibrun -n @[TASKS] ./fv3.exe - -echo "Model ended: " `date` -echo -n " $( date +%s )," >> job_timestamp.txt diff --git a/tests/module-setup.sh b/tests/module-setup.sh index d39bbeb95c..f7e563d314 100755 --- a/tests/module-setup.sh +++ b/tests/module-setup.sh @@ -18,7 +18,7 @@ elif [[ ${MACHINE_ID} = hera ]] ; then elif [[ ${MACHINE_ID} = orion ]] ; then # We are on Orion if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/init/bash + source /apps/other/lmod/lmod/init/bash fi module purge @@ -51,13 +51,6 @@ elif [[ ${MACHINE_ID} = derecho ]] ; then fi module purge -elif [[ ${MACHINE_ID} = stampede ]] ; then - # We are on TACC Stampede - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /opt/apps/lmod/lmod/init/bash - fi - module purge - elif [[ ${MACHINE_ID} = frontera ]] ; then # We are on TACC Frontera if ( ! eval module help > /dev/null 2>&1 ) ; then @@ -81,14 +74,6 @@ elif [[ ${MACHINE_ID} = gaeac6 ]]; then source /opt/cray/pe/lmod/lmod/init/bash fi module reset - -elif [[ ${MACHINE_ID} = expanse ]]; then - # We are on SDSC Expanse - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /etc/profile.d/modules.sh - fi - module purge - module load slurm/expanse/20.02.3 elif [[ ${MACHINE_ID} = noaacloud ]] ; then # We are on NOAA Cloud diff --git a/tests/rt.sh b/tests/rt.sh index 65c97132b9..df13d80cc1 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -947,39 +947,6 @@ case ${MACHINE_ID} in ROCOTO_SCHEDULER="pbspro" fi ;; - stampede) - echo "rt.sh: Setting up stampede..." - export PYTHONPATH= - if [[ "${ECFLOW:-false}" == true ]] ; then - ECFLOW_START= - fi - QUEUE=skx-normal - COMPILE_QUEUE=skx-dev - PARTITION= - dprefix="${SCRATCH}/ufs-weather-model/run" - DISKNM="/work2/07736/minsukji/stampede2/ufs-weather-model/RT" - STMP="${dprefix}" - PTMP="${dprefix}" - SCHEDULER="slurm" - export MPIEXEC="ibrun" - export MPIEXECOPTS= - ;; - expanse) - echo "rt.sh: Setting up expanse..." - export PYTHONPATH= - - if [[ "${ECFLOW:-false}" == true ]] ; then - export ECFLOW_START= - fi - QUEUE="compute" - COMPILE_QUEUE="shared" - PARTITION= - dprefix="/expanse/lustre/scratch/${USER}/temp_project/run" - DISKNM="/expanse/lustre/scratch/domh/temp_project/RT" - STMP="${dprefix}" - PTMP="${dprefix}" - SCHEDULER="slurm" - ;; noaacloud) echo "rt.sh: Setting up noaacloud..." export PATH="/contrib/EPIC/bin:${PATH}" @@ -1093,7 +1060,7 @@ if [[ ${ROCOTO} == true ]]; then echo "rt.sh: Verifying ROCOTO support..." case ${MACHINE_ID} in - wcoss2|acorn|expanse|stampede) + wcoss2|acorn) die "Rocoto not supported on this machine, please do not use '-r'." ;; *) @@ -1135,7 +1102,7 @@ fi if [[ ${ECFLOW} == true ]]; then echo "Verifying ECFLOW support..." case ${MACHINE_ID} in - expanse|stampede|noaacloud) + noaacloud) die "ECFLOW not supported on this machine, please do not use '-e'." ;; *) diff --git a/tests/run_test.sh b/tests/run_test.sh index cd46804607..59446fad2f 100755 --- a/tests/run_test.sh +++ b/tests/run_test.sh @@ -113,7 +113,7 @@ case ${MACHINE_ID} in module load miniconda/3.9.12 module load nccmp/1.9.0.1 ;; - stampede|expanse|noaacloud) + noaacloud) echo "No special nccmp load necessary" ;; gaeac5) @@ -483,7 +483,7 @@ if [[ ${skip_check_results} == false ]]; then else if [[ ${i##*.} == nc* ]] ; then - if [[ " orion hercules hera wcoss2 acorn derecho gaeac5 gaeac6 jet s4 noaacloud " =~ ${MACHINE_ID} ]]; then + if [[ " orion hercules hera wcoss2 acorn derecho gaeac5 gaeac6 jet s4 noaacloud frontera" =~ ${MACHINE_ID} ]]; then printf "USING NCCMP.." >> "${RT_LOG}" printf "USING NCCMP.." if [[ ${CMP_DATAONLY} == false ]]; then