Skip to content

Gaea intel 2022 update, reg. tests updates for Jet,Hera,Orion,Cheyenne,Gaea #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hu5970 merged 18 commits into from
Jun 29, 2023
Merged

Gaea intel 2022 update, reg. tests updates for Jet,Hera,Orion,Cheyenne,Gaea #11

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
f01cca6
Gaea modulefile update, fix files location update on Jet,Hera,Orion,G…
Jun 14, 2023
aace46e
Gaea modulefile update
Jun 14, 2023
3bf5e01
Modulefile for Gaea and regression tests configuration
Jun 23, 2023
1fe104e
Merge branch 'intel2022_gaea' into gaea_intel2022
natalie-perlin Jun 23, 2023
ff26a62
Merge pull request #1 from natalie-perlin/gaea_intel2022
natalie-perlin Jun 23, 2023
fe42c85
Intel 2022 for Gaea,update tests scripts for Gaea, Cheyenne, Orion, Jet
Jun 26, 2023
a7a6a2e
Merge branch 'intel2022_gaea' of github.com:natalie-perlin/GSI into i…
Jun 26, 2023
3baf6c7
update Cheyenne modulefile
Jun 26, 2023
d3fe9a2
updatesto scripts to address comments
Jun 26, 2023
7bd5f95
update regression/regression_driver.sh
Jun 26, 2023
e50cf96
update regression/regression_driver.sh
Jun 26, 2023
a0e48e2
update ush/build.sh
Jun 26, 2023
4864e07
Merge remote-tracking branch 'ming/intel2022' into intel2022_gaea
Jun 27, 2023
1d8beed
fixed typ inmodulefile for gaea
Jun 27, 2023
e2398c7
ncdiag/1.1.1 for spack build
Jun 28, 2023
164126b
update ncdiag/1.1.1
Jun 28, 2023
8d6009b
gaea modulefile update
Jun 28, 2023
29a1bb6
updated test scripts for gaea
Jun 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions modulefiles/gsi_cheyenne.intel.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ load("mkl/2022.1")

load("gsi_common")

local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
load(pathJoin("prod_util", prod_util_ver))
load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2"))
pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix")

pushenv("CFLAGS", "-xHOST")
pushenv("FFLAGS", "-xHOST")
Expand Down
2 changes: 1 addition & 1 deletion modulefiles/gsi_common.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0"
local ncio_ver=os.getenv("ncio_ver") or "1.1.2"
local crtm_ver=os.getenv("crtm_ver") or "2.4.0"
local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.0"
local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1"

load(pathJoin("netcdf", netcdf_ver))

Expand Down
62 changes: 0 additions & 62 deletions modulefiles/gsi_gaea
View file
Open in desktop

This file was deleted.

34 changes: 34 additions & 0 deletions modulefiles/gsi_gaea.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
help([[
]])

load("cmake/3.20.1")

prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack")
load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))

load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2"))
load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "7.7.20"))
load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2"))
load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20"))

load("gsi_common")

local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
load(pathJoin("prod_util", prod_util_ver))

-- Needed at runtime:
load("alps")

local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/"
prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64"))
pushenv("MKLROOT", MKLROOT)

pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601")

setenv("CC","cc")
setenv("FC","ftn")
setenv("CXX","CC")
pushenv("CRAYPE_LINK_TYPE","dynamic")

whatis("Description: GSI environment on Gaea with Intel Compilers")

2 changes: 1 addition & 1 deletion modulefiles/gsi_hera.intel.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,6 @@ load(pathJoin("prod_util", prod_util_ver))
pushenv("CFLAGS", "-xHOST")
pushenv("FFLAGS", "-xHOST")

pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20221128")
pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20230601")

whatis("Description: GSI environment on Hera with Intel Compilers")
2 changes: 1 addition & 1 deletion modulefiles/gsi_jet.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,6 @@ pushenv("CFLAGS", "-axSSE4.2,AVX,CORE-AVX2")
pushenv("FFLAGS", "-axSSE4.2,AVX,CORE-AVX2")


pushenv("GSI_BINARY_SOURCE_DIR", "/mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/gsi/20221128")
pushenv("GSI_BINARY_SOURCE_DIR", "/mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/gsi/20230601")

whatis("Description: GSI environment on Jet with Intel Compilers")
2 changes: 1 addition & 1 deletion modulefiles/gsi_orion.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,6 @@ load(pathJoin("prod_util", prod_util_ver))
pushenv("CFLAGS", "-xHOST")
pushenv("FFLAGS", "-xHOST")

pushenv("GSI_BINARY_SOURCE_DIR", "/work/noaa/global/glopara/fix/gsi/20221128")
pushenv("GSI_BINARY_SOURCE_DIR", "/work/noaa/global/glopara/fix/gsi/20230601")

whatis("Description: GSI environment on Orion with Intel Compilers")
3 changes: 2 additions & 1 deletion regression/regression_driver.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ if [ -d "$config_path" ]; then
source $config_path/local_vars.sh
fi


# source the necessary files to setup
if [ "$#" -eq 2 ]; then
export regdir=$2
Expand Down Expand Up @@ -41,7 +42,7 @@ for jn in `seq ${RSTART} ${REND}`; do
fi
rm -f ${job[$jn]}.out

/bin/sh $ush/$sub_cmd -q $queue -j ${job[$jn]} -t ${topts[$jn]} -p ${popts[$jn]} -r ${ropts[$jn]} $scripts/${regtest}.sh
/bin/sh $ush/$sub_cmd -q $queue -j ${job[$jn]} -t ${topts[$jn]} -p ${popts[$jn]} -r ${ropts[$jn]} $scripts/${regtest}.sh

if [ $debug == ".true." ]; then break; fi
$scripts/regression_wait.sh ${job[$jn]} ${rcname} $check_resource
Expand Down
73 changes: 59 additions & 14 deletions regression/regression_param.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ case $machine in
sub_cmd="sub_jet"
memnode=96
numcore=40
;;
Gaea)
sub_cmd="sub_gaea"
memnode=64
numcore=36
;;
wcoss2)
sub_cmd="sub_wcoss2"
Expand All @@ -28,7 +33,9 @@ case $machine in
sub_cmd="sub_discover"
;;
Cheyenne)
sub_cmd="sub_ncar -a p48503002 -q economy -d $PWD"
sub_cmd="sub_cheyenne"
memnode=128
numcore=36
;;
*) # EXIT out for unresolved machine
echo "unknown $machine"
Expand Down Expand Up @@ -56,8 +63,11 @@ case $regtest in
topts[1]="0:15:00" ; popts[1]="12/5/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:30:00" ; popts[1]="16/2/" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="16/4/" ; ropts[2]="/2"
topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:15:00" ; popts[1]="18/2/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="18/4/" ; ropts[2]="/2"
elif [[ "$machine" = "wcoss2" ]]; then
topts[1]="0:15:00" ; popts[1]="12/5/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2"
Expand Down Expand Up @@ -86,8 +96,11 @@ case $regtest in
topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:35:00" ; popts[1]="16/2/" ; ropts[1]="/1"
topts[2]="0:25:00" ; popts[2]="16/4/" ; ropts[2]="/2"
topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:15:00" ; popts[1]="18/2/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="18/4/" ; ropts[2]="/2"
elif [[ "$machine" = "wcoss2" ]]; then
topts[1]="0:15:00" ; popts[1]="28/2/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="28/4/" ; ropts[2]="/2"
Expand All @@ -104,6 +117,8 @@ case $regtest in
popts[1]="12/5/"
elif [[ "$machine" = "Jet" ]]; then
popts[1]="12/5/"
elif [[ "$machine" = "Gaea" ]]; then
popts[1]="18/5/"
elif [[ "$machine" = "wcoss2" ]]; then
popts[1]="28/4/"
topts[1]="3:00:00"
Expand All @@ -129,8 +144,11 @@ case $regtest in
topts[1]="0:30:00" ; popts[1]="48/2" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="60/3" ; ropts[2]="/2"
elif [[ "$machine" = "Cheyenne" ]]; then
Comment thread
natalie-perlin marked this conversation as resolved.
topts[1]="1:59:00" ; popts[1]="6/8/" ; ropts[1]="/1"
topts[2]="0:35:00" ; popts[2]="6/10/" ; ropts[2]="/2"
topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:10:00" ; popts[1]="18/8/" ; ropts[1]="/1"
topts[2]="0:10:00" ; popts[2]="18/10/" ; ropts[2]="/2"
elif [[ "$machine" = "wcoss2" ]]; then
topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2"
Expand All @@ -155,6 +173,12 @@ case $regtest in
elif [[ "$machine" = "Jet" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1"
elif [[ "$machine" = "wcoss2" ]]; then
topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
Expand All @@ -179,6 +203,12 @@ case $regtest in
elif [[ "$machine" = "Jet" ]]; then
topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/2"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1"
elif [[ "$machine" = "wcoss2" ]]; then
topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1"
Expand All @@ -204,8 +234,11 @@ case $regtest in
topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:15:00" ; popts[1]="8/6/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="8/8/" ; ropts[2]="/1"
topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:30:00" ; popts[1]="8/6/" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="8/8/" ; ropts[2]="/1"
elif [[ "$machine" = "wcoss2" ]]; then
topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1"
topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/2"
Expand Down Expand Up @@ -233,6 +266,9 @@ case $regtest in
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:20:00" ; popts[1]="6/6/" ; ropts[1]="/1"
topts[2]="0:20:00" ; popts[2]="8/8/" ; ropts[2]="/1"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:20:00" ; popts[1]="6/6/" ; ropts[1]="/1"
topts[2]="0:20:00" ; popts[2]="8/8/" ; ropts[2]="/1"
elif [[ "$machine" = "wcoss2" ]]; then
topts[1]="0:15:00" ; popts[1]="10/10/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="14/14/" ; ropts[2]="/2"
Expand All @@ -258,8 +294,11 @@ case $regtest in
topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
elif [[ "$machine" = "Cheyenne" ]]; then
topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2"
topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
elif [[ "$machine" = "Gaea" ]]; then
topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
elif [[ "$machine" = "wcoss2" ]]; then
topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1"
topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2"
Expand Down Expand Up @@ -310,19 +349,25 @@ if [[ "$machine" = "Hera" ]]; then
export APRUN="srun"
elif [[ "$machine" = "Orion" ]]; then
export OMP_STACKSIZE=2048M
export APRUN="srun -n \$ntasks"
export APRUN="srun --label -n \$ntasks"
elif [[ "$machine" = "Jet" ]]; then
export OMP_STACKSIZE=1024M
export MPI_BUFS_PER_PROC=256
export MPI_BUFS_PER_HOST=256
export MPI_GROUP_MAX=256
export APRUN="srun"
export APRUN="srun --label -n \$ntasks --cpus-per-task=\$threads"
elif [[ "$machine" = "Gaea" ]]; then
export OMP_STACKSIZE=1024M
export MPI_BUFS_PER_PROC=256
export MPI_BUFS_PER_HOST=256
export MPI_GROUP_MAX=256
export APRUN="srun --export=ALL --mpi=pmi2 -n \$ntasks"
elif [[ "$machine" = "Cheyenne" ]]; then
export OMP_STACKSIZE=1024M
export MPI_BUFS_PER_PROC=256
export MPI_BUFS_PER_HOST=256
export MPI_GROUP_MAX=256
export APRUN="mpirun -v -np \$NCPUS"
export APRUN="mpirun -v -np \$ntasks"
elif [[ "$machine" = "wcoss2" ]]; then
export OMP_PLACES=cores
export OMP_STACKSIZE=2G
Expand Down
34 changes: 24 additions & 10 deletions regression/regression_var.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,19 +49,33 @@ fi
echo "Running Regression Tests on '$machine'";

case $machine in
Gaea)
export queue="batch"
Comment thread
natalie-perlin marked this conversation as resolved.
Outdated
export noscrub="/lustre/f2/scratch/$LOGNAME/gsi_tmp/noscrub"
export ptmp="/lustre/f2/scratch/$LOGNAME/gsi_tmp/ptmp"
export casesdir="/lustre/f2/dev/role.epic/contrib/GSI_data/CASES/regtest"

export group="global"
if [[ "$cmaketest" = "false" ]]; then
export basedir="/lustre/f2/dev/$LOGNAME/sandbox/GSI"
fi

export check_resource="no"
export accnt="nggps_emc"
;;
Cheyenne)
export queue="economy"
export noscrub="/glade/scratch/$LOGNAME"
export queue="regular"
export noscrub="/glade/scratch/$LOGNAME/tmp_gsi/noscrub"
Comment thread
natalie-perlin marked this conversation as resolved.
Outdated
export group="global"
if [[ "$cmaketest" = "false" ]]; then
export basedir="/glade/scratch/$LOGNAME/gsi"
export basedir="/glade/scratch/$LOGNAME/tmp_gsi"
fi
export ptmp="/glade/scratch/$LOGNAME/$ptmpName"
export ptmp="/glade/scratch/$LOGNAME/tmp_gsi/ptmp"
Comment thread
natalie-perlin marked this conversation as resolved.
Outdated

export casesdir="/glade/p/ral/jntp/tools/CASES"
export casesdir="/glade/work/epicufsrt/contrib/GSI_data/CASES/regtest"

export check_resource="no"
export accnt="p48503002"
export accnt="NRAL0032"
;;
wcoss2)
export local_or_default="${local_or_default:-/lfs/h2/emc/da/noscrub/$LOGNAME}"
Expand All @@ -84,7 +98,7 @@ case $machine in
export accnt="${accnt:-GFS-DEV}"
;;
Orion)
export local_or_default="${local_or_default:-/work/noaa/da/$LOGNAME}"
export local_or_default="${local_or_default:-/work/noaa/epic-ps/$LOGNAME}"
Comment thread
natalie-perlin marked this conversation as resolved.
Outdated
if [ -d $local_or_default ]; then
export noscrub="$local_or_default/noscrub"
elif [ -d /work/noaa/global/$LOGNAME ]; then
Expand All @@ -94,14 +108,14 @@ case $machine in
export queue="${queue:-batch}"
export group="${group:-global}"
if [[ "$cmaketest" = "false" ]]; then
export basedir="/work/noaa/da/$LOGNAME/gsi"
export basedir="/work/noaa/epic-ps/$LOGNAME/gsi"
Comment thread
natalie-perlin marked this conversation as resolved.
Outdated
fi
export ptmp="${ptmp:-/work/noaa/stmp/$LOGNAME/$ptmpName}"
export ptmp="${ptmp:-/work/noaa/epic-ps/$LOGNAME/ptmp}"
Comment thread
natalie-perlin marked this conversation as resolved.
Outdated

export casesdir="/work/noaa/da/rtreadon/CASES/regtest"

export check_resource="no"
export accnt="${accnt:-da-cpu}"
export accnt="${accnt:-epic}"
Comment thread
natalie-perlin marked this conversation as resolved.
Outdated
;;
Hera)

Expand Down
Loading