Verif updates mar3

README.md

@@ -30,6 +30,8 @@ EMC_verif-global is supported on the following machines:
 * Orion
 * S4
 * Jet
+* Gaea-C5
+* Gaea-C6
 
 For questions or issues, please e-mail Mallory Row at mallory.row@noaa.gov.
 

modulefiles/emc_verif_global_gaeac5.lua

@@ -1,72 +1,38 @@
 help([[
-Load environment to run EMC_verif-global on Gaea C5 using Intel
+Load environment to run EMC_verif-global on Gaea-C5 using Intel
 ]])
 
-prepend_path("MODULEPATH", "/sw/rdtn/modulefiles")
-load("hsi")
-
-prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
-
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
-stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28"
-load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+Core_ver=os.getenv("Core_ver") or "24.10"
+load(pathJoin("Core", Core_ver))
+
+prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"
+load(pathJoin("prod_util", prod_util_ver))
 
-cmake_ver=os.getenv("cmake_ver") or "3.23.1"
-load(pathJoin("cmake", cmake_ver))
+grib_util_ver=os.getenv("grib_util_ver") or "1.3.0"
+load(pathJoin("grib-util", grib_util_ver))
 
+intel_oneapi_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
 
 netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
 load(pathJoin("netcdf-c", netcdf_c_ver))
 
-netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
-load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
-
-nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
-load(pathJoin("nccmp", nccmp_ver))
-
-esmf_ver=os.getenv("esmf_ver") or "8.6.0"
-load(pathJoin("esmf", esmf_ver))
-
-nco_ver=os.getenv("nco_ver") or "5.0.6"
+nco_ver=os.getenv("nco_ver") or "5.1.9"
 load(pathJoin("nco", nco_ver))
 
-grads_ver=os.getenv("grads_ver","2.2.3")
-load(pathJoin("grads",grads_ver))
-
-imagemagick_ver=os.getenv("imagemagick_ver","7.1.1-11")
-load(pathJoin("imagemagick",imagemagick_ver))
+grads_ver=os.getenv("grads_ver") or "2.2.3"
+load(pathJoin("grads", grads_ver))
 
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
+imagemagick_ver=os.getenv("imagemagick_ver") or "7.1.1-29"
+load(pathJoin("imagemagick", imagemagick_ver))
 
-hpc_ver=os.getenv("hpc_ver") or "1.1.0"
-load(pathJoin("hpc", hpc_ver))
-
-hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2"
-load(pathJoin("hpc-intel", hpc_intel_ver))
-
-hpc_impi_ver=os.getenv("hpc_impi_ver") or "2018.0.4"
-load(pathJoin("hpc-impi", impi_ver))
-
-prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
-load(pathJoin("prod_util", prod_util_ver))
-
-grib_util_ver=os.getenv("grib_util_ver") or "1.2.3"
-load(pathJoin("grib_util", grib_util_ver))
-
-prepend_path("MODULEPATH", "/contrib/anaconda/modulefiles")
-
-anaconda_ver=os.getenv("anaconda_ver") or "latest"
-load(pathJoin("anaconda", anaconda_ver))
-
-prepend_path("MODULEPATH", "/contrib/met/modulefiles")
-
-met_ver=os.getenv("met_ver") or "9.1"
+met_ver=os.getenv("met_ver") or "9.1.3"
 load(pathJoin("met", met_ver))
 
-prepend_path("MODULEPATH", "/contrib/METplus/modulefiles")
-
-metplus_ver=os.getenv("metplus_ver") or "3.1"
+metplus_ver=os.getenv("metplus_ver") or "3.1.1"
 load(pathJoin("metplus", metplus_ver))

modulefiles/emc_verif_global_gaeac6.lua

@@ -1,72 +1,38 @@
 help([[
-Load environment to run EMC_verif-global on Gaea C5 using Intel
+Load environment to run EMC_verif-global on Gaea-C6 using Intel
 ]])
 
-prepend_path("MODULEPATH", "/sw/rdtn/modulefiles")
-load("hsi")
-
-prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
-
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/gsi-addon/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
-stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28"
-load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+Core_ver=os.getenv("Core_ver") or "24.11"
+load(pathJoin("Core", Core_ver))
+
+prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"
+load(pathJoin("prod_util", prod_util_ver))
 
-cmake_ver=os.getenv("cmake_ver") or "3.23.1"
-load(pathJoin("cmake", cmake_ver))
+grib_util_ver=os.getenv("grib_util_ver") or "1.3.0"
+load(pathJoin("grib-util", grib_util_ver))
 
+intel_oneapi_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
 
 netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
 load(pathJoin("netcdf-c", netcdf_c_ver))
 
-netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
-load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
-
-nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
-load(pathJoin("nccmp", nccmp_ver))
-
-esmf_ver=os.getenv("esmf_ver") or "8.6.0"
-load(pathJoin("esmf", esmf_ver))
-
-nco_ver=os.getenv("nco_ver") or "5.0.6"
+nco_ver=os.getenv("nco_ver") or "5.1.9"
 load(pathJoin("nco", nco_ver))
 
-grads_ver=os.getenv("grads_ver","2.2.3")
-load(pathJoin("grads",grads_ver))
-
-imagemagick_ver=os.getenv("imagemagick_ver","7.1.1-11")
-load(pathJoin("imagemagick",imagemagick_ver))
+grads_ver=os.getenv("grads_ver") or "2.2.3"
+load(pathJoin("grads", grads_ver))
 
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
+imagemagick_ver=os.getenv("imagemagick_ver") or "7.1.1-29"
+load(pathJoin("imagemagick", imagemagick_ver))
 
-hpc_ver=os.getenv("hpc_ver") or "1.1.0"
-load(pathJoin("hpc", hpc_ver))
-
-hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2"
-load(pathJoin("hpc-intel", hpc_intel_ver))
-
-hpc_impi_ver=os.getenv("hpc_impi_ver") or "2018.0.4"
-load(pathJoin("hpc-impi", impi_ver))
-
-prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
-load(pathJoin("prod_util", prod_util_ver))
-
-grib_util_ver=os.getenv("grib_util_ver") or "1.2.3"
-load(pathJoin("grib_util", grib_util_ver))
-
-prepend_path("MODULEPATH", "/contrib/anaconda/modulefiles")
-
-anaconda_ver=os.getenv("anaconda_ver") or "latest"
-load(pathJoin("anaconda", anaconda_ver))
-
-prepend_path("MODULEPATH", "/contrib/met/modulefiles")
-
-met_ver=os.getenv("met_ver") or "9.1"
+met_ver=os.getenv("met_ver") or "9.1.3"
 load(pathJoin("met", met_ver))
 
-prepend_path("MODULEPATH", "/contrib/METplus/modulefiles")
-
-metplus_ver=os.getenv("metplus_ver") or "3.1"
-load(pathJoin("metplus", metplus_ver))
+metplus_ver=os.getenv("metplus_ver") or "3.1.1"
+load(pathJoin("metplus", metplus_ver))

parm/config/config.vrfy

@@ -31,9 +31,9 @@ echo "BEGIN: $(basename ${BASH_SOURCE[0]})"
 #RUN_MAPSDA:          run to make analysis maps of time-mean increments, ENKF ensemble mean and ensemble spread
 export RUN_GRID2GRID_STEP1="YES"
 export RUN_GRID2GRID_STEP2="NO"
-export RUN_GRID2OBS_STEP1="NO"
+export RUN_GRID2OBS_STEP1="YES"
 export RUN_GRID2OBS_STEP2="NO"
-export RUN_PRECIP_STEP1="NO"
+export RUN_PRECIP_STEP1="YES"
 export RUN_PRECIP_STEP2="NO"
 export RUN_SATELLITE_STEP1="NO"
 export RUN_SATELLITE_STEP2="NO"
@@ -53,25 +53,25 @@ export RUN_MAPSDA="NO"
 #model_hpss_dir_list:    HPSS directory of data
 #model_data_run_hpss:    get files from HPSS ("YES") if not online
 #hpss_walltime:          how long to run HPSS job to get data in minutes
-export model_list="C96_TEST_NEW"
-export model_dir_list="/gpfs/f5/epic/scratch/$USER/archive"
-export model_stat_dir_list="/gpfs/f5/epic/scratch/$USER/archive"
-export model_file_format_list="pgbf{lead?fmt=%2H}.gfs.{init?fmt=%Y%m%d%H}.grib2"
+export model_list="gfs"
+export model_dir_list="/lfs/h2/emc/vpppg/noscrub/emc.vpppg/verification/global/archive/model_data"
+export model_stat_dir_list="/lfs/h2/emc/vpppg/noscrub/$USER/archive"
+export model_file_format_list="pgbf{lead?fmt=%2H}.gfs.{init?fmt=%Y%m%d%H}"
 export model_data_run_hpss="NO"
 export model_hpss_dir_list="/NCEPPROD/hpssprod/runhistory"
 export hpss_walltime="10"
 ## OUTPUT DATA SETTINGS
 #OUTPUTROOT: base output directory
-export OUTPUTROOT="/gpfs/f5/epic/scratch/$USER/verif_global_standalone"
+export OUTPUTROOT="/lfs/h2/emc/stmp/$USER/verif_global_standalone"
 ## DATE SETTINGS
 #start_date:          verification start date, format YYYYMMDD
 #end_date:            verification end date, format YYYYMMDD
 #spinup_period_start: spinup period start, format YYYYMMDDHH, if none use "NA"
 #spinup_period_end:   spinup period end, format YYYYMMDDHH, if none use "NA"
 #make_met_data_by:    how to treat dates, "VALID" or "INIT"
 #plot_by:             how to plot data, "VALID" or "INIT"
-export start_date=20210323
-export end_date=20210323
+export start_date=$(date -d "24 hours ago" '+%Y%m%d')
+export end_date=$(date -d "24 hours ago" '+%Y%m%d')
 export spinup_period_start="NA"
 export spinup_period_end="NA"
 export make_met_data_by="VALID"

scripts/exgrid2obs_step1.sh

@@ -89,7 +89,7 @@ if [ $MPMD = YES ]; then
         if [ $machine = WCOSS2 ]; then
             export LD_LIBRARY_PATH=/apps/dev/pmi-fix:$LD_LIBRARY_PATH
             launcher="mpiexec -np ${nproc} -ppn ${nproc} --cpu-bind verbose,core cfp"
-        elif [ $machine = HERA -o $machine = ORION -o $machine = S4 -o $machine = JET -o $machine = HERCULES ]; then
+        elif [ $machine = HERA -o $machine = ORION -o $machine = S4 -o $machine = JET -o $machine = HERCULES -o $machine = GAEAC5 -o $machine = GAEAC6 ]; then
             launcher="srun --export=ALL --multi-prog"
         fi
         $launcher $MP_CMDFILE

ush/build_webpage.py

@@ -946,12 +946,12 @@ def tropcyc_write_template_body2(template_filename):
                   +'--job-name='+web_job_name+' '+web_job_filename)
 elif machine == 'GAEAC5':
     os.system('sbatch --ntasks=1 --time='+walltime.strftime('%H:%M:%S')+' '
-                  +'--partition='+QUEUESERV+' --account='+ACCOUNT+' '
+                  +'--clusters='+CLUSTERS+' --account='+ACCOUNT+' '
                   +'--output='+web_job_output+' '
                   +'--job-name='+web_job_name+' '+web_job_filename)
 elif machine == 'GAEAC6':
     os.system('sbatch --ntasks=1 --time='+walltime.strftime('%H:%M:%S')+' '
-                  +'--partition='+QUEUESERV+' --account='+ACCOUNT+' '
+                  +'--clusters='+CLUSTERS+' --account='+ACCOUNT+' '
                   +'--output='+web_job_output+' '
                   +'--job-name='+web_job_name+' '+web_job_filename)
 elif machine == 'JET':
@@ -963,7 +963,7 @@ def tropcyc_write_template_body2(template_filename):
                   +'--partition='+QUEUESERV+' --account='+ACCOUNT+' '
                   +'--output='+web_job_output+' '
                   +'--job-name='+web_job_name+' '+web_job_filename)
-elif machine in ["ORION", "HERCULES", "S4"]:
+elif machine in ["ORION", "HERCULES", "S4","GAEAC5","GAEAC6"]:
     if webhost == 'emcrzdm.ncep.noaa.gov':
         print("ERROR: Currently " + machine + " cannot connect to "+webhost)
     else:

ush/get_data_files.py

@@ -462,7 +462,7 @@ def get_hpss_data(hpss_job_filename, save_data_dir, save_data_file,
                          +hpss_job_name+' -t R,PD -h | wc -l')
     elif machine in ['ORION', 'S4', 'HERCULES', 'GAEAC5', 'GAEAC6']:
         print("ERROR: No HPSS access from "+machine)
-    if machine not in ['ORION', 'S4', 'HERCULES']:
+    if machine not in ['ORION', 'S4', 'HERCULES', 'GAEAC5', 'GAEAC6']:
         sleep_counter, sleep_checker = 1, 10
         while (sleep_counter*sleep_checker) <= walltime_seconds:
             sleep(sleep_checker)

ush/get_machine.py

@@ -58,7 +58,7 @@
     s4_match = re.match(re.compile(r"s4-submit.ssec.wisc.edu"), hostname)
     jet_match = re.match(re.compile(r"^fe[0-9]{1}"), hostname)
     gaeac5_match = re.match(re.compile(r"^gaea5[1-8]{1}"), hostname)
-    gaeac6_match = re.match(re.compile(r"^gaea6[1-8]{1}"), hostname) 
+    gaeac6_match = re.match(re.compile(r"^gaea6[1-8]{1}"), hostname)
     if cactus_match or dogwood_match or cactus_match2 or dogwood_match2:
         machine = 'WCOSS2'
     elif hera_match:
@@ -75,7 +75,7 @@
         machine = 'GAEAC5'
     elif gaeac6_match:
         machine = 'GAEAC6'
-else:
+    else:
         print("Cannot find match for "+hostname)
         sys.exit(1)