From daa1689e38ddb1dc4e306bd732e0228be12df05b Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Mon, 26 Aug 2019 10:24:09 -0600 Subject: [PATCH 1/3] izumi initial port --- cice.setup | 88 ++++++++++++++----- cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 | 12 +-- configuration/scripts/cice.batch.csh | 11 +++ configuration/scripts/cice.launch.csh | 2 +- .../scripts/machines/Macros.cheyenne_intel | 0 .../scripts/machines/Macros.fram_intel | 0 .../scripts/machines/Macros.hobart_intel | 0 .../scripts/machines/Macros.hobart_nag | 0 .../scripts/machines/Macros.izumi_gnu | 52 +++++++++++ .../scripts/machines/Macros.izumi_intel | 52 +++++++++++ .../scripts/machines/Macros.izumi_nag | 53 +++++++++++ .../scripts/machines/Macros.izumi_pgi | 52 +++++++++++ .../scripts/machines/Macros.phase2_intel | 0 .../scripts/machines/Macros.phase3_intel | 0 .../scripts/machines/Macros.testmachine_intel | 0 .../scripts/machines/env.brooks_intel | 0 .../scripts/machines/env.cesium_intel | 0 .../scripts/machines/env.hobart_intel | 2 +- configuration/scripts/machines/env.izumi_gnu | 31 +++++++ .../scripts/machines/env.izumi_intel | 31 +++++++ configuration/scripts/machines/env.izumi_nag | 31 +++++++ configuration/scripts/machines/env.izumi_pgi | 31 +++++++ .../scripts/machines/env.millikan_intel | 0 .../scripts/options/set_nml.boxrestore | 2 - doc/source/user_guide/ug_running.rst | 1 + 25 files changed, 420 insertions(+), 31 deletions(-) mode change 100755 => 100644 configuration/scripts/machines/Macros.cheyenne_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.fram_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.hobart_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.hobart_nag create mode 100644 configuration/scripts/machines/Macros.izumi_gnu create mode 100644 configuration/scripts/machines/Macros.izumi_intel create mode 100644 configuration/scripts/machines/Macros.izumi_nag create mode 100644 configuration/scripts/machines/Macros.izumi_pgi mode change 100755 => 100644 configuration/scripts/machines/Macros.phase2_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.phase3_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.testmachine_intel mode change 100644 => 100755 configuration/scripts/machines/env.brooks_intel mode change 100644 => 100755 configuration/scripts/machines/env.cesium_intel create mode 100755 configuration/scripts/machines/env.izumi_gnu create mode 100755 configuration/scripts/machines/env.izumi_intel create mode 100755 configuration/scripts/machines/env.izumi_nag create mode 100755 configuration/scripts/machines/env.izumi_pgi mode change 100644 => 100755 configuration/scripts/machines/env.millikan_intel diff --git a/cice.setup b/cice.setup index 7380def71..c7ff2542c 100755 --- a/cice.setup +++ b/cice.setup @@ -493,14 +493,10 @@ EOF set blckx = `echo ${pesx} | cut -d x -f 3` set blcky = `echo ${pesx} | cut -d x -f 4` set mblck = `echo ${pesx} | cut -d x -f 5` - if ($?ICE_MACHINE_MAXPES) then - @ pesreq = ${task} * ${thrd} - if (${pesreq} > ${ICE_MACHINE_MAXPES}) then - @ task = ${ICE_MACHINE_MAXPES} / ${thrd} - @ mblck = ${mblck} * ((${pesreq} / ${ICE_MACHINE_MAXPES}) + 1) - endif + if (${task} == 0 || ${thrd} == 0 || ${blckx} == 0 || ${blcky} == 0 || ${mblck} == 0) then + echo "${0}: ERROR in -p argument, cannot have zeros" + exit -1 endif - set pesx = ${task}x${thrd}x${blckx}x${blcky}x${mblck} else set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*$/OK/'` if (${chck} == OK) then @@ -509,13 +505,10 @@ EOF set blckx = `echo ${pesx} | cut -d x -f 3` set blcky = `echo ${pesx} | cut -d x -f 4` set mblck = 0 - if ($?ICE_MACHINE_MAXPES) then - @ pesreq = ${task} * ${thrd} - if (${pesreq} > ${ICE_MACHINE_MAXPES}) then - @ task = ${ICE_MACHINE_MAXPES} / ${thrd} - endif + if (${task} == 0 || ${thrd} == 0 || ${blckx} == 0 || ${blcky} == 0) then + echo "${0}: ERROR in -p argument, cannot have zeros" + exit -1 endif - set pesx = ${task}x${thrd}x${blckx}x${blcky} else set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*$/OK/'` if (${chck} == OK) then @@ -524,20 +517,73 @@ EOF set blckx = 0 set blcky = 0 set mblck = 0 - if ($?ICE_MACHINE_MAXPES) then - @ pesreq = ${task} * ${thrd} - if (${pesreq} > ${ICE_MACHINE_MAXPES}) then - @ task = ${ICE_MACHINE_MAXPES} / ${thrd} - endif + if (${task} == 0 || ${thrd} == 0) then + echo "${0}: ERROR in -p argument, cannot have zeros" + exit -1 endif - set pesx = ${task}x${thrd} else - echo "${0}: ERROR in -p argument, ${pesx}, must be [m]x[n], [m]x[n]x[bx]x[by], or [m]x[n]x[bx]x[by]x[mb] " - exit -1 + set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*$/OK/'` + if (${chck} == OK) then + set task = `echo ${pesx} | cut -d x -f 1` + set thrd = 1 + set blckx = 0 + set blcky = 0 + set mblck = 0 + if (${task} == 0) then + echo "${0}: ERROR in -p argument, cannot have zeros" + exit -1 + endif + else + echo "${0}: ERROR in -p argument, ${pesx}, must be [m], [m]x[n], [m]x[n]x[bx]x[by], or [m]x[n]x[bx]x[by]x[mb] " + exit -1 + endif + endif + endif + endif + + @ blkreq = ${task} * ${mblck} + # check max threads, reduce threads, increase tasks + if ($?ICE_MACHINE_MAXTHREADS) then + if (${thrd} > ${ICE_MACHINE_MAXTHREADS}) then + @ pesreq = ${task} * ${thrd} + @ task = ${pesreq} / ${ICE_MACHINE_MAXTHREADS} + @ thrd = ${ICE_MACHINE_MAXTHREADS} + @ peschk = ${task} * ${thrd} + if (${pesreq} > ${peschk}) then + @ task = ${task} + 1 endif +# echo "tcx1 reset to $task $thrd $mblck" + endif + endif + + # check max pes, reduce tasks + if ($?ICE_MACHINE_MAXPES) then + @ pesreq = ${task} * ${thrd} + if (${pesreq} > ${ICE_MACHINE_MAXPES}) then + @ task = ${ICE_MACHINE_MAXPES} / ${thrd} +# echo "tcx2 reset to $task $thrd $mblck" endif endif + # check max blocks and adjust as needed + if ($mblck > 0) then + @ mblck = ${blkreq} / ${task} + @ blkchk = ${task} * ${mblck} + if (${blkreq} > ${blkchk}) then + @ mblck = $mblck + 1 + endif +# echo "tcx3 reset to $task $thrd $mblck" + endif + + # update pesx based on use defined settings and machine limits to reflect actual value + set pesx = ${task}x${thrd}x${blckx}x${blcky}x${mblck} + if (${mblck} == 0) then + set pesx = ${task}x${thrd}x${blckx}x${blcky} + endif + if (${blckx} == 0 || ${blcky} == 0) then + set pesx = ${task}x${thrd} + endif + set testname_noid = ${spval} # create case for test cases if (${docase} == 0) then diff --git a/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 b/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 index b1f162967..0aabbcd6e 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 @@ -47,7 +47,7 @@ module dmi_omp #if defined (_OPENMP) ! Please note, this constant will create a compiler info for a constant ! expression in IF statements: - real(kind=dbl_kind), private :: rdomp_iam, rdomp_nt + real(kind=dbl_kind) :: rdomp_iam, rdomp_nt !$OMP THREADPRIVATE(domp_iam,domp_nt,rdomp_iam,rdomp_nt) #endif @@ -70,9 +70,9 @@ subroutine domp_init(nt_out) !$OMP PARALLEL DEFAULT(none) #if defined (_OPENMP) domp_iam = omp_get_thread_num() - rdomp_iam = real(domp_iam,8) + rdomp_iam = real(domp_iam,dbl_kind) domp_nt = omp_get_num_threads() - rdomp_nt = real(domp_nt,8) + rdomp_nt = real(domp_nt,dbl_kind) #else domp_iam = 0 domp_nt = 1 @@ -139,7 +139,7 @@ subroutine domp_get_domain_rlu(lower,upper,d_lower,d_upper) #if defined (_OPENMP) if (omp_in_parallel()) then - dlen = real(upper-lower+1, 8) + dlen = real(upper-lower+1, dbl_kind) d_lower = lower + floor((rdomp_iam*dlen+p5)/rdomp_nt, 4) d_upper = lower -1 + floor((rdomp_iam*dlen+dlen+p5)/rdomp_nt, 4) endif @@ -843,7 +843,7 @@ subroutine stepu_last(NA_len, rhow, & real(kind=dbl_kind),dimension(:), intent(in), contiguous :: & uvel_init, vvel_init, aiu, forcex, forcey, umassdti, Tbu, & uocn, vocn, fm, uarear,Cw - real(kind=DBL_KIND),dimension(:), intent(in), contiguous :: & + real(kind=dbl_kind),dimension(:), intent(in), contiguous :: & str1,str2,str3,str4,str5,str6,str7,str8 real(kind=dbl_kind),dimension(:), intent(inout), contiguous :: & uvel,vvel, strintx,strinty, taubx,tauby @@ -1003,7 +1003,7 @@ module ice_dyn_evp_1d stressm_1, stressm_2, stressm_3, stressm_4, & stress12_1,stress12_2,stress12_3,stress12_4, & divu,rdg_conv,rdg_shear,shear,taubx,tauby - real (kind=DBL_KIND), dimension(:), allocatable :: & + real (kind=dbl_kind), dimension(:), allocatable :: & str1, str2, str3, str4, str5, str6, str7, str8 real (kind=dbl_kind), dimension(:), allocatable :: & HTE,HTN, & diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 0647c5813..852c59461 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -79,6 +79,17 @@ cat >> ${jobfile} << EOFB #PBS -l nodes=1:ppn=24 EOFB +else if (${ICE_MACHINE} =~ izumi*) then +cat >> ${jobfile} << EOFB +#PBS -j oe +###PBS -m ae +#PBS -V +#PBS -q ${queue} +#PBS -N ${ICE_CASENAME} +#PBS -l nodes=${nnodes}:ppn=${taskpernode} +#PBS -l walltime=${batchtime} +EOFB + else if (${ICE_MACHINE} =~ thunder* || ${ICE_MACHINE} =~ gordon* || ${ICE_MACHINE} =~ conrad* || ${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr*) then cat >> ${jobfile} << EOFB #PBS -N ${shortcase} diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index 77e237683..b9ea5c806 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -34,7 +34,7 @@ EOFR endif #======= -else if (${ICE_MACHINE} =~ hobart*) then +else if (${ICE_MACHINE} =~ hobart* || ${ICE_MACHINE} =~ izumi*) then if (${ICE_COMMDIR} =~ serial*) then cat >> ${jobfile} << EOFR ./cice >&! \$ICE_RUNLOG_FILE diff --git a/configuration/scripts/machines/Macros.cheyenne_intel b/configuration/scripts/machines/Macros.cheyenne_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.fram_intel b/configuration/scripts/machines/Macros.fram_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.hobart_intel b/configuration/scripts/machines/Macros.hobart_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.hobart_nag b/configuration/scripts/machines/Macros.hobart_nag old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.izumi_gnu b/configuration/scripts/machines/Macros.izumi_gnu new file mode 100644 index 000000000..0d48f1013 --- /dev/null +++ b/configuration/scripts/machines/Macros.izumi_gnu @@ -0,0 +1,52 @@ +#============================================================================== +# Makefile macros for NCAR izumi, gnu compiler +#============================================================================== + +CPP := /usr/bin/cpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 + +FIXEDFLAGS := -ffixed-line-length-132 +FREEFLAGS := -ffree-form +FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow +else + FFLAGS += -O2 +endif + +SCC := gcc +SFC := gfortran +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +INCLDIR := -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif + +## if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:= + INCLDIR += -I + SLIBS := $(SLIB) -L$(PIO_PATH) -lpiofS +endif + diff --git a/configuration/scripts/machines/Macros.izumi_intel b/configuration/scripts/machines/Macros.izumi_intel new file mode 100644 index 000000000..502f7b218 --- /dev/null +++ b/configuration/scripts/machines/Macros.izumi_intel @@ -0,0 +1,52 @@ +#============================================================================== +# Makefile macros for NCAR izumi, intel compiler +#============================================================================== + +CPP := /usr/bin/cpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -qno-opt-dynamic-align -fp-model precise + +FIXEDFLAGS := -fixed -132 +FREEFLAGS := -free +FFLAGS := -qno-opt-dynamic-align -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +INCLDIR := -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -mkl + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + +## if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:= + INCLDIR += -I + SLIBS := $(SLIB) -L$(PIO_PATH) -lpiofS +endif + diff --git a/configuration/scripts/machines/Macros.izumi_nag b/configuration/scripts/machines/Macros.izumi_nag new file mode 100644 index 000000000..8e42df4aa --- /dev/null +++ b/configuration/scripts/machines/Macros.izumi_nag @@ -0,0 +1,53 @@ +#============================================================================== +# Makefile macros for NCAR izumi, NAG compiler +#============================================================================== + +CPP := /usr/bin/cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 $(ICE_CPPDEFS) +CFLAGS := -c + +FIXEDFLAGS := -fixed +FREEFLAGS := -free +FFLAGS := -Wp,-macro=no_com -convert=BIG_ENDIAN -wmismatch=mpi_bcast,mpi_isend,mpi_irecv,mpi_send,mpi_recv,mpi_allreduce +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -C=all -g -time -f2003 -ieee=stop +else + FFLAGS += -O2 -ieee=full +endif + +SCC := cc +SFC := nagfor +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +INCLDIR := -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -openmp + CFLAGS += -fopenmp + FFLAGS += -openmp +else + FFLAGS += -gline +endif + +## if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:= + INCLDIR += -I + SLIBS := $(SLIB) -L$(PIO_PATH) -lpiof +endif diff --git a/configuration/scripts/machines/Macros.izumi_pgi b/configuration/scripts/machines/Macros.izumi_pgi new file mode 100644 index 000000000..8a8ca4b97 --- /dev/null +++ b/configuration/scripts/machines/Macros.izumi_pgi @@ -0,0 +1,52 @@ +#============================================================================== +# Makefile macros for NCAR izumi, pgi compiler +#============================================================================== + +CPP := pgcc -E +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 ${ICE_CPPDEFS} +CFLAGS := -c -O2 -Kieee + +FIXEDFLAGS := -Mextend +FREEFLAGS := -Mfree +FFLAGS := -Kieee -Mbyteswapio -traceback +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -Mbounds -Mchkptr +else + FFLAGS += -O2 +endif + +SCC := pgcc +SFC := pgf90 +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +INCLDIR := -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -mp + CFLAGS += -mp + FFLAGS += -mp +endif + +## if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:= + INCLDIR += -I + SLIBS := $(SLIB) -L$(PIO_PATH) -lpiofS +endif + diff --git a/configuration/scripts/machines/Macros.phase2_intel b/configuration/scripts/machines/Macros.phase2_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.phase3_intel b/configuration/scripts/machines/Macros.phase3_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.testmachine_intel b/configuration/scripts/machines/Macros.testmachine_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/env.brooks_intel b/configuration/scripts/machines/env.brooks_intel old mode 100644 new mode 100755 diff --git a/configuration/scripts/machines/env.cesium_intel b/configuration/scripts/machines/env.cesium_intel old mode 100644 new mode 100755 diff --git a/configuration/scripts/machines/env.hobart_intel b/configuration/scripts/machines/env.hobart_intel index 76f4e5e55..80092297a 100755 --- a/configuration/scripts/machines/env.hobart_intel +++ b/configuration/scripts/machines/env.hobart_intel @@ -15,7 +15,7 @@ module load compiler/intel/18.0.3 endif setenv ICE_MACHINE_ENVNAME hobart -setenv ICE_MACHINE_COMPILER ifort +setenv ICE_MACHINE_COMPILER intel setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata diff --git a/configuration/scripts/machines/env.izumi_gnu b/configuration/scripts/machines/env.izumi_gnu new file mode 100755 index 000000000..5413a33b1 --- /dev/null +++ b/configuration/scripts/machines/env.izumi_gnu @@ -0,0 +1,31 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /usr/share/Modules/init/csh + +module purge +module load compiler/gnu/8.2.0 + +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_ENVNAME izumi +setenv ICE_MACHINE_COMPILER gnu +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata +setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QUEUE "long" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_TPNODE 48 +setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.izumi_intel b/configuration/scripts/machines/env.izumi_intel new file mode 100755 index 000000000..bffd9bc01 --- /dev/null +++ b/configuration/scripts/machines/env.izumi_intel @@ -0,0 +1,31 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /usr/share/Modules/init/csh + +module purge +module load compiler/intel/19.0.1 + +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_ENVNAME izumi +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata +setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QUEUE "long" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_TPNODE 48 +setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.izumi_nag b/configuration/scripts/machines/env.izumi_nag new file mode 100755 index 000000000..755548b1f --- /dev/null +++ b/configuration/scripts/machines/env.izumi_nag @@ -0,0 +1,31 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /usr/share/Modules/init/csh + +module purge +module load compiler/nag/6.2 + +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_ENVNAME izumi +setenv ICE_MACHINE_COMPILER nag +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata +setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QUEUE "long" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_TPNODE 48 +setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.izumi_pgi b/configuration/scripts/machines/env.izumi_pgi new file mode 100755 index 000000000..45a0bd7b8 --- /dev/null +++ b/configuration/scripts/machines/env.izumi_pgi @@ -0,0 +1,31 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /usr/share/Modules/init/csh + +module purge +module load compiler/pgi/18.10 + +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_ENVNAME izumi +setenv ICE_MACHINE_COMPILER pgi +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata +setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QUEUE "long" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_TPNODE 48 +setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.millikan_intel b/configuration/scripts/machines/env.millikan_intel old mode 100644 new mode 100755 diff --git a/configuration/scripts/options/set_nml.boxrestore b/configuration/scripts/options/set_nml.boxrestore index 294d507aa..bc913a3dc 100644 --- a/configuration/scripts/options/set_nml.boxrestore +++ b/configuration/scripts/options/set_nml.boxrestore @@ -5,8 +5,6 @@ restart_ext = .true. use_leap_years = .true. ndtd = 2 kcatbound = 1 -distribution_type = 'cartesian' -processor_shape = 'slenderX1' ew_boundary_type = 'cyclic' ns_boundary_type = 'open' histfreq = 'd','x','x','x','x' diff --git a/doc/source/user_guide/ug_running.rst b/doc/source/user_guide/ug_running.rst index 5be0a8683..ea2533d2a 100644 --- a/doc/source/user_guide/ug_running.rst +++ b/doc/source/user_guide/ug_running.rst @@ -322,6 +322,7 @@ system. Some variables are optional. "ICE_MACHINE_SUBMIT", "string", "batch job submission command" "ICE_MACHINE_TPNODE", "integer", "machine maximum MPI tasks per node" "ICE_MACHINE_MAXPES", "integer", "machine maximum total processors per job (optional)" + "ICE_MACHINE_MAXTHREADS", "integer", "machine maximum threads per mpi task (optional)" "ICE_MACHINE_MAXRUNLENGTH", "integer", "batch wall time limit in hours (optional)" "ICE_MACHINE_ACCT", "string", "batch default account" "ICE_MACHINE_QUEUE", "string", "batch default queue" From e6fd180913d444875db5eeab1d883a1c5feba519 Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Tue, 27 Aug 2019 16:04:15 -0600 Subject: [PATCH 2/3] update izumi port and add nothread_suite --- configuration/scripts/cice.batch.csh | 1 + configuration/scripts/machines/env.izumi_gnu | 3 +- .../scripts/machines/env.izumi_intel | 3 +- configuration/scripts/machines/env.izumi_nag | 3 +- configuration/scripts/machines/env.izumi_pgi | 3 +- configuration/scripts/tests/nothread_suite.ts | 61 +++++++++++++++++++ 6 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 configuration/scripts/tests/nothread_suite.ts diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 852c59461..7345a3658 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -80,6 +80,7 @@ cat >> ${jobfile} << EOFB EOFB else if (${ICE_MACHINE} =~ izumi*) then +if (${runlength} > 2) set queue = "medium" cat >> ${jobfile} << EOFB #PBS -j oe ###PBS -m ae diff --git a/configuration/scripts/machines/env.izumi_gnu b/configuration/scripts/machines/env.izumi_gnu index 5413a33b1..9f9938d68 100755 --- a/configuration/scripts/machines/env.izumi_gnu +++ b/configuration/scripts/machines/env.izumi_gnu @@ -24,8 +24,9 @@ setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_QSTAT "qstat " -setenv ICE_MACHINE_QUEUE "long" +setenv ICE_MACHINE_QUEUE "short" setenv ICE_MACHINE_ACCT P00000000 setenv ICE_MACHINE_TPNODE 48 setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_MAXTHREADS 1 setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.izumi_intel b/configuration/scripts/machines/env.izumi_intel index bffd9bc01..63d175990 100755 --- a/configuration/scripts/machines/env.izumi_intel +++ b/configuration/scripts/machines/env.izumi_intel @@ -24,8 +24,9 @@ setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_QSTAT "qstat " -setenv ICE_MACHINE_QUEUE "long" +setenv ICE_MACHINE_QUEUE "short" setenv ICE_MACHINE_ACCT P00000000 setenv ICE_MACHINE_TPNODE 48 setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_MAXTHREADS 1 setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.izumi_nag b/configuration/scripts/machines/env.izumi_nag index 755548b1f..785cc410a 100755 --- a/configuration/scripts/machines/env.izumi_nag +++ b/configuration/scripts/machines/env.izumi_nag @@ -24,8 +24,9 @@ setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_QSTAT "qstat " -setenv ICE_MACHINE_QUEUE "long" +setenv ICE_MACHINE_QUEUE "short" setenv ICE_MACHINE_ACCT P00000000 setenv ICE_MACHINE_TPNODE 48 setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_MAXTHREADS 1 setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.izumi_pgi b/configuration/scripts/machines/env.izumi_pgi index 45a0bd7b8..b89eafeb8 100755 --- a/configuration/scripts/machines/env.izumi_pgi +++ b/configuration/scripts/machines/env.izumi_pgi @@ -24,8 +24,9 @@ setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_QSTAT "qstat " -setenv ICE_MACHINE_QUEUE "long" +setenv ICE_MACHINE_QUEUE "short" setenv ICE_MACHINE_ACCT P00000000 setenv ICE_MACHINE_TPNODE 48 setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_MAXTHREADS 1 setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/tests/nothread_suite.ts b/configuration/scripts/tests/nothread_suite.ts new file mode 100644 index 000000000..d9ab722c0 --- /dev/null +++ b/configuration/scripts/tests/nothread_suite.ts @@ -0,0 +1,61 @@ +# Test Grid PEs Sets BFB-compare + +restart gx3 8x1x25x29x2 dslenderX2 +logbfb gx3 8x1x25x29x2 dslenderX2,diag1,reprosum + +smoke gx3 16x1 diag1,run5day +smoke gx3 1x1 debug,diag1,run2day +smoke gx3 4x1 debug,diag1,run5day,thread +restart gx3 16x1 thread +smoke gx3 16x1 diag24,run1year,medium + +restart gx1 160x1 droundrobin,medium +restart tx1 160x1 dsectrobin,medium +restart gx3 16x1 none +restart gx3 16x1 iobinary + +restart gx3 12x1 alt01 +restart gx3 16x1 alt02 +restart gx3 8x1 alt03 +restart gx3 16x1 alt04 +restart gx3 16x1 alt05 +restart gx3 18x1 alt01,debug,short +restart gx3 20x1 alt02,debug,short +restart gx3 24x1 alt03,debug,short +smoke gx3 24x1 alt04,debug,short +smoke gx3 32x1 alt05,debug,short + +restart gbox128 8x1 short +restart gbox128 16x1 boxdyn,short +restart gbox128 24x1 boxdyn,short,debug +restart gbox128 12x1 boxadv,short +smoke gbox128 20x1 boxadv,short,debug +restart gbox128 32x1 boxrestore,short +smoke gbox128 24x1 boxrestore,short,debug +restart gbox80 1x1 box2001 +smoke gbox80 1x1 boxslotcyl + +smoke gx3 16x1 bgcz +smoke gx3 16x1 bgcz,debug +smoke gx3 24x1 bgcskl,debug +restart gx1 128x1 bgcsklclim,medium +restart gx1 256x1 bgczclim,medium + +decomp gx3 8x1x5x29x20 +restart gx3 1x1x50x58x4 droundrobin restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 4x1x25x116x1 dslenderX1 restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 12x1x4x29x9 dspacecurve restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 16x1x8x10x10 droundrobin restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 6x1x50x58x1 droundrobin restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 8x1x19x19x5 droundrobin restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 20x1x5x29x20 dsectrobin,short restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 32x1x5x10x10 drakeX2 restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 16x1x8x10x10 droundrobin,maskhalo restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 4x1x25x29x4 droundrobin restart_gx3_8x1x25x29x2_dslenderX2 + +logbfb gx3 1x1x50x58x4 droundrobin,diag1,maskhalo,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +logbfb gx3 4x1x25x116x1 dslenderX1,diag1,maskhalo,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +logbfb gx3 20x1x5x29x20 dsectrobin,diag1,short,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +logbfb gx3 16x1x8x10x10 droundrobin,diag1,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +logbfb gx3 6x1x50x58x1 droundrobin,diag1,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +logbfb gx3 12x1x4x29x9 dspacecurve,diag1,maskhalo,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum From c2022398bcbb5326558bf246c93078bcb2713b7e Mon Sep 17 00:00:00 2001 From: Anthony Craig Date: Wed, 28 Aug 2019 12:44:33 -0600 Subject: [PATCH 3/3] update kind conversions in ice_dyn_evp_1d.F90 --- cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 b/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 index 0aabbcd6e..b7092fc95 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 @@ -140,8 +140,8 @@ subroutine domp_get_domain_rlu(lower,upper,d_lower,d_upper) #if defined (_OPENMP) if (omp_in_parallel()) then dlen = real(upper-lower+1, dbl_kind) - d_lower = lower + floor((rdomp_iam*dlen+p5)/rdomp_nt, 4) - d_upper = lower -1 + floor((rdomp_iam*dlen+dlen+p5)/rdomp_nt, 4) + d_lower = lower + floor((rdomp_iam*dlen+p5)/rdomp_nt, JPIM) + d_upper = lower -1 + floor((rdomp_iam*dlen+dlen+p5)/rdomp_nt, JPIM) endif #endif @@ -843,7 +843,7 @@ subroutine stepu_last(NA_len, rhow, & real(kind=dbl_kind),dimension(:), intent(in), contiguous :: & uvel_init, vvel_init, aiu, forcex, forcey, umassdti, Tbu, & uocn, vocn, fm, uarear,Cw - real(kind=dbl_kind),dimension(:), intent(in), contiguous :: & + real(kind=DBL_KIND),dimension(:), intent(in), contiguous :: & str1,str2,str3,str4,str5,str6,str7,str8 real(kind=dbl_kind),dimension(:), intent(inout), contiguous :: & uvel,vvel, strintx,strinty, taubx,tauby @@ -1003,7 +1003,7 @@ module ice_dyn_evp_1d stressm_1, stressm_2, stressm_3, stressm_4, & stress12_1,stress12_2,stress12_3,stress12_4, & divu,rdg_conv,rdg_shear,shear,taubx,tauby - real (kind=dbl_kind), dimension(:), allocatable :: & + real (kind=DBL_KIND), dimension(:), allocatable :: & str1, str2, str3, str4, str5, str6, str7, str8 real (kind=dbl_kind), dimension(:), allocatable :: & HTE,HTN, &