EMC-verif_global updates for C5 and C6

README.md

@@ -30,6 +30,8 @@ EMC_verif-global is supported on the following machines:
 * Orion
 * S4
 * Jet
+* Gaea-C5
+* Gaea-C6
 
 For questions or issues, please e-mail Mallory Row at mallory.row@noaa.gov.
 

modulefiles/emc_verif_global_gaeac5.lua

@@ -1,72 +1,38 @@
 help([[
-Load environment to run EMC_verif-global on Gaea C5 using Intel
+Load environment to run EMC_verif-global on Gaea-C5 using Intel
 ]])
 
-prepend_path("MODULEPATH", "/sw/rdtn/modulefiles")
-load("hsi")
-
-prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
-
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
-stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28"
-load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+Core_ver=os.getenv("Core_ver") or "24.10"
+load(pathJoin("Core", Core_ver))
+
+prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"
+load(pathJoin("prod_util", prod_util_ver))
 
-cmake_ver=os.getenv("cmake_ver") or "3.23.1"
-load(pathJoin("cmake", cmake_ver))
+grib_util_ver=os.getenv("grib_util_ver") or "1.3.0"
+load(pathJoin("grib-util", grib_util_ver))
 
+intel_oneapi_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
 
 netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
 load(pathJoin("netcdf-c", netcdf_c_ver))
 
-netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
-load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
-
-nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
-load(pathJoin("nccmp", nccmp_ver))
-
-esmf_ver=os.getenv("esmf_ver") or "8.6.0"
-load(pathJoin("esmf", esmf_ver))
-
-nco_ver=os.getenv("nco_ver") or "5.0.6"
+nco_ver=os.getenv("nco_ver") or "5.1.9"
 load(pathJoin("nco", nco_ver))
 
-grads_ver=os.getenv("grads_ver","2.2.3")
-load(pathJoin("grads",grads_ver))
-
-imagemagick_ver=os.getenv("imagemagick_ver","7.1.1-11")
-load(pathJoin("imagemagick",imagemagick_ver))
+grads_ver=os.getenv("grads_ver") or "2.2.3"
+load(pathJoin("grads", grads_ver))
 
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
+imagemagick_ver=os.getenv("imagemagick_ver") or "7.1.1-29"
+load(pathJoin("imagemagick", imagemagick_ver))
 
-hpc_ver=os.getenv("hpc_ver") or "1.1.0"
-load(pathJoin("hpc", hpc_ver))
-
-hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2"
-load(pathJoin("hpc-intel", hpc_intel_ver))
-
-hpc_impi_ver=os.getenv("hpc_impi_ver") or "2018.0.4"
-load(pathJoin("hpc-impi", impi_ver))
-
-prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
-load(pathJoin("prod_util", prod_util_ver))
-
-grib_util_ver=os.getenv("grib_util_ver") or "1.2.3"
-load(pathJoin("grib_util", grib_util_ver))
-
-prepend_path("MODULEPATH", "/contrib/anaconda/modulefiles")
-
-anaconda_ver=os.getenv("anaconda_ver") or "latest"
-load(pathJoin("anaconda", anaconda_ver))
-
-prepend_path("MODULEPATH", "/contrib/met/modulefiles")
-
-met_ver=os.getenv("met_ver") or "9.1"
+met_ver=os.getenv("met_ver") or "9.1.3"
 load(pathJoin("met", met_ver))
 
-prepend_path("MODULEPATH", "/contrib/METplus/modulefiles")
-
-metplus_ver=os.getenv("metplus_ver") or "3.1"
+metplus_ver=os.getenv("metplus_ver") or "3.1.1"
 load(pathJoin("metplus", metplus_ver))

modulefiles/emc_verif_global_gaeac6.lua

@@ -1,72 +1,38 @@
 help([[
-Load environment to run EMC_verif-global on Gaea C5 using Intel
+Load environment to run EMC_verif-global on Gaea-C6 using Intel
 ]])
 
-prepend_path("MODULEPATH", "/sw/rdtn/modulefiles")
-load("hsi")
-
-prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
-
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/gsi-addon/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
-stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28"
-load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+Core_ver=os.getenv("Core_ver") or "24.11"
+load(pathJoin("Core", Core_ver))
+
+prod_util_ver=os.getenv("prod_util_ver") or "2.1.1"
+load(pathJoin("prod_util", prod_util_ver))
 
-cmake_ver=os.getenv("cmake_ver") or "3.23.1"
-load(pathJoin("cmake", cmake_ver))
+grib_util_ver=os.getenv("grib_util_ver") or "1.3.0"
+load(pathJoin("grib-util", grib_util_ver))
 
+intel_oneapi_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
 
 netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
 load(pathJoin("netcdf-c", netcdf_c_ver))
 
-netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
-load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
-
-nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
-load(pathJoin("nccmp", nccmp_ver))
-
-esmf_ver=os.getenv("esmf_ver") or "8.6.0"
-load(pathJoin("esmf", esmf_ver))
-
-nco_ver=os.getenv("nco_ver") or "5.0.6"
+nco_ver=os.getenv("nco_ver") or "5.1.9"
 load(pathJoin("nco", nco_ver))
 
-grads_ver=os.getenv("grads_ver","2.2.3")
-load(pathJoin("grads",grads_ver))
-
-imagemagick_ver=os.getenv("imagemagick_ver","7.1.1-11")
-load(pathJoin("imagemagick",imagemagick_ver))
+grads_ver=os.getenv("grads_ver") or "2.2.3"
+load(pathJoin("grads", grads_ver))
 
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
+imagemagick_ver=os.getenv("imagemagick_ver") or "7.1.1-29"
+load(pathJoin("imagemagick", imagemagick_ver))
 
-hpc_ver=os.getenv("hpc_ver") or "1.1.0"
-load(pathJoin("hpc", hpc_ver))
-
-hpc_intel_ver=os.getenv("hpc_intel_ver") or "2022.1.2"
-load(pathJoin("hpc-intel", hpc_intel_ver))
-
-hpc_impi_ver=os.getenv("hpc_impi_ver") or "2018.0.4"
-load(pathJoin("hpc-impi", impi_ver))
-
-prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
-load(pathJoin("prod_util", prod_util_ver))
-
-grib_util_ver=os.getenv("grib_util_ver") or "1.2.3"
-load(pathJoin("grib_util", grib_util_ver))
-
-prepend_path("MODULEPATH", "/contrib/anaconda/modulefiles")
-
-anaconda_ver=os.getenv("anaconda_ver") or "latest"
-load(pathJoin("anaconda", anaconda_ver))
-
-prepend_path("MODULEPATH", "/contrib/met/modulefiles")
-
-met_ver=os.getenv("met_ver") or "9.1"
+met_ver=os.getenv("met_ver") or "9.1.3"
 load(pathJoin("met", met_ver))
 
-prepend_path("MODULEPATH", "/contrib/METplus/modulefiles")
-
-metplus_ver=os.getenv("metplus_ver") or "3.1"
-load(pathJoin("metplus", metplus_ver))
+metplus_ver=os.getenv("metplus_ver") or "3.1.1"
+load(pathJoin("metplus", metplus_ver))

scripts/exgrid2obs_step1.sh

@@ -89,7 +89,7 @@ if [ $MPMD = YES ]; then
         if [ $machine = WCOSS2 ]; then
             export LD_LIBRARY_PATH=/apps/dev/pmi-fix:$LD_LIBRARY_PATH
             launcher="mpiexec -np ${nproc} -ppn ${nproc} --cpu-bind verbose,core cfp"
-        elif [ $machine = HERA -o $machine = ORION -o $machine = S4 -o $machine = JET -o $machine = HERCULES ]; then
+        elif [ $machine = HERA -o $machine = ORION -o $machine = S4 -o $machine = JET -o $machine = HERCULES -o $machine = GAEAC5 -o $machine = GAEAC6 ]; then
             launcher="srun --export=ALL --multi-prog"
         fi
         $launcher $MP_CMDFILE

ush/build_webpage.py

@@ -963,7 +963,7 @@ def tropcyc_write_template_body2(template_filename):
                   +'--partition='+QUEUESERV+' --account='+ACCOUNT+' '
                   +'--output='+web_job_output+' '
                   +'--job-name='+web_job_name+' '+web_job_filename)
-elif machine in ["ORION", "HERCULES", "S4"]:
+elif machine in ["ORION", "HERCULES", "S4","GAEAC5","GAEAC6"]:
     if webhost == 'emcrzdm.ncep.noaa.gov':
         print("ERROR: Currently " + machine + " cannot connect to "+webhost)
     else:

ush/get_data_files.py

@@ -460,9 +460,9 @@ def get_hpss_data(hpss_job_filename, save_data_dir, save_data_file,
                   +'--job-name='+hpss_job_name+' '+hpss_job_filename)
         job_check_cmd = ('squeue -u '+os.environ['USER']+' -n '
                          +hpss_job_name+' -t R,PD -h | wc -l')
-    elif machine in ['ORION', 'S4', 'HERCULES', 'GAEAC5', 'GAEAC5']:
+    elif machine in ['ORION', 'S4', 'HERCULES', 'GAEAC5', 'GAEAC6']:
         print("ERROR: No HPSS access from "+machine)
-    if machine not in ['ORION', 'S4', 'HERCULES']:
+    if machine not in ['ORION', 'S4', 'HERCULES', 'GAEAC5', 'GAEAC6']:
         sleep_counter, sleep_checker = 1, 10
         while (sleep_counter*sleep_checker) <= walltime_seconds:
             sleep(sleep_checker)

ush/get_machine.py

@@ -58,7 +58,7 @@
     s4_match = re.match(re.compile(r"s4-submit.ssec.wisc.edu"), hostname)
     jet_match = re.match(re.compile(r"^fe[0-9]{1}"), hostname)
     gaeac5_match = re.match(re.compile(r"^gaea5[1-8]{1}"), hostname)
-    gaeac6_match = re.match(re.compile(r"^gaea6[1-8]{1}"), hostname) 
+    gaeac6_match = re.match(re.compile(r"^gaea6[1-8]{1}"), hostname)
     if cactus_match or dogwood_match or cactus_match2 or dogwood_match2:
         machine = 'WCOSS2'
     elif hera_match:
@@ -75,7 +75,7 @@
         machine = 'GAEAC5'
     elif gaeac6_match:
         machine = 'GAEAC6'
-else:
+    else:
         print("Cannot find match for "+hostname)
         sys.exit(1)
 

ush/load_modules.sh

@@ -156,6 +156,42 @@ elif [ $machine = JET ]; then
         "ERROR: $METplus_version is not supported on $machine"
         exit 1
     fi
+elif [ $machine = GAEAC5 ]; then
+    module reset
+    module use ${HOMEverif_global}/modulefiles
+    module load emc_verif_global_gaeac5
+    module swap python/3.11.6 python/3.9
+    if [[ "${MET_version}" =~ ^(9.1) ]]; then
+        export HOMEMET="${met_ROOT}"
+        export HOMEMET_bin_exec="bin"
+    else
+        "ERROR: $MET_version is not supported on $machine"
+        exit 1
+    fi
+    if [[ "${METplus_version}" =~ ^(3.1) ]]; then
+        export HOMEMETplus="${metplus_ROOT}"
+    else
+        "ERROR: $METplus_version is not supported on $machine"
+        exit 1
+    fi
+elif [ $machine = GAEAC6 ]; then
+    module reset
+    module use ${HOMEverif_global}/modulefiles
+    module load emc_verif_global_gaeac6
+    module swap python/3.11.6 python/3.11
+    if [ $MET_version = 9.1 ]; then
+        export HOMEMET="${met_ROOT}"
+        export HOMEMET_bin_exec="bin"
+    else
+        "ERROR: $MET_version is not supported on $machine"
+        exit 1
+    fi
+    if [ $METplus_version = 3.1 ]; then
+        export HOMEMETplus="${metplus_ROOT}"
+    else
+        "ERROR: $METplus_version is not supported on $machine"
+        exit 1
+    fi
 else
     echo "ERROR: $machine is not supported"
     exit 1

ush/plotting_scripts/plot_maps2d_lat_lon_errors.py

@@ -296,6 +296,10 @@ def plot_subplot_data(ax_tmp, map_ax_tmp, plot_data, plot_data_lat,
     config['data_dir']='/home/mrow/.local/share/cartopy'
 elif machine == 'HERCULES':
     config['data_dir']='/home/dhuber/.local/share/cartopy'
+elif machine == 'GAEAC5':
+    config['data_dir']='/gpfs/f5/ufs-ard/world-shared/global/metplus.data/cartopy'
+elif machine == 'GAEAC6':
+    config['data_dir']='/gpfs/f6/bil-fire8/world-shared/global/metplus.data/cartopy'
 
 # Set up information
 env_var_model_list = []

ush/plotting_scripts/plot_maps2d_model2obs_calc_vars_lat_lon_errors.py

@@ -290,6 +290,10 @@ def plot_subplot_data(ax_tmp, map_ax_tmp, plot_data, plot_data_lat,
     config['data_dir']='/home/mrow/.local/share/cartopy'
 elif machine == 'HERCULES':
     config['data_dir']='/home/dhuber/.local/share/cartopy'
+elif machine == 'GAEAC5':
+    config['data_dir']='/gpfs/f5/ufs-ard/world-shared/global/metplus.data/cartopy'
+elif machine == 'GAEAC6':
+    config['data_dir']='/gpfs/f6/bil-fire8/world-shared/global/metplus.data/cartopy'
 
 # Set up information
 RUN_type = 'model2obs'

ush/plotting_scripts/plot_mapsda_lat_lon_errors.py

@@ -342,6 +342,10 @@ def plot_subplot_data(ax_tmp, map_ax_tmp, plot_data, plot_data_lat,
     config['data_dir']='/home/dhuber/.local/share/cartopy'
 elif machine == 'ORION':
     config['data_dir']='/home/mrow/.local/share/cartopy'
+elif machine == 'GAEAC5':
+    config['data_dir']='/gpfs/f5/ufs-ard/world-shared/global/metplus.data/cartopy'
+elif machine == 'GAEAC6':
+    config['data_dir']='/gpfs/f6/bil-fire8/world-shared/global/metplus.data/cartopy'
 
 # Set up information
 env_var_model_list = []

ush/run_batch.py

@@ -70,6 +70,24 @@
         job_card.write('#SBATCH --nodes=1\n')
         job_card.write('#SBATCH --ntasks-per-node='+nproc+'\n')
         job_card.write('#SBATCH --time=6:00:00\n')
+    elif machine == 'GAEAC5':
+        job_card.write('#!/bin/sh\n')
+        job_card.write('#SBATCH --account='+ACCOUNT+'\n')
+        job_card.write('#SBATCH --job-name='+job_name+'\n')
+        job_card.write('#SBATCH --output='+job_output_filename+'\n')
+        job_card.write('#SBATCH --nodes=1\n')
+        job_card.write('#SBATCH --clusters=c5\n')
+        job_card.write('#SBATCH --ntasks-per-node='+nproc+'\n')
+        job_card.write('#SBATCH --time=6:00:00\n')
+    elif machine == 'GAEAC6':
+        job_card.write('#!/bin/sh\n')
+        job_card.write('#SBATCH --account='+ACCOUNT+'\n')
+        job_card.write('#SBATCH --job-name='+job_name+'\n')
+        job_card.write('#SBATCH --output='+job_output_filename+'\n')
+        job_card.write('#SBATCH --nodes=1\n')
+        job_card.write('#SBATCH --clusters=c6\n')
+        job_card.write('#SBATCH --ntasks-per-node='+nproc+'\n')
+        job_card.write('#SBATCH --time=6:00:00\n')
     job_card.write('\n')
     job_card.write('/bin/sh '+script)
 

ush/run_verif_global_in_global_workflow.sh

@@ -301,6 +301,14 @@ elif [ $machine = "WCOSS2" ]; then
     export global_archive="/lfs/h2/emc/vpppg/noscrub/emc.vpppg/verification/global/archive/model_data"
     export prepbufr_arch_dir="/lfs/h2/emc/vpppg/noscrub/emc.vpppg/verification/global/archive/obs_data/prepbufr"
     export ccpa_24hr_arch_dir="/lfs/h2/emc/vpppg/noscrub/emc.vpppg/verification/global/archive/obs_data/ccpa_accum24hr"
+elif [ $machine = "GAEAC5" ]; then
+    export global_archive="/gpfs/f5/ufs-ard/world-shared/global/metplus.data/archive"
+    export prepbufr_arch_dir="/gpfs/f5/ufs-ard/world-shared/global/metplus.data/prepbufr"
+    export ccpa_24hr_arch_dir="/gpfs/f5/ufs-ard/world-shared/global/metplus.data/obdata/ccpa_accum24hr"
+elif [ $machine = "GAEAC6" ]; then
+    export global_archive="/gpfs/f6/bil-fire8/world-shared/global/metplus.data/archive"
+    export prepbufr_arch_dir="/gpfs/f6/bil-fire8/world-shared/global/metplus.data/prepbufr"
+    export ccpa_24hr_arch_dir="/gpfs/f6/bil-fire8/world-shared/global/metplus.data/obdata/ccpa_accum24hr"
 fi
 
 ## Set operational directories