diff --git a/parm/fixed_files_mapping.yaml b/parm/fixed_files_mapping.yaml new file mode 100644 index 0000000000..2439e535a0 --- /dev/null +++ b/parm/fixed_files_mapping.yaml @@ -0,0 +1,208 @@ +fixed_files: + # + #----------------------------------------------------------------------- + # + # FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: + # This array is used to set some of the namelist variables in the forecast + # model's namelist file that represent the relative or absolute paths of + # various fixed files (the first column of the array, where columns are + # delineated by the pipe symbol "|") to the full paths to surface climatology + # files (on the native FV3-LAM grid) in the FIXlam directory derived from + # the corresponding surface climatology fields (the second column of the + # array). + # + #----------------------------------------------------------------------- + # + FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: [ + "FNALBC | snowfree_albedo", + "FNALBC2 | facsf", + "FNTG3C | substrate_temperature", + "FNVEGC | vegetation_greenness", + "FNVETC | vegetation_type", + "FNSOTC | soil_type", + "FNVMNC | vegetation_greenness", + "FNVMXC | vegetation_greenness", + "FNSLPC | slope_type", + "FNABSC | maximum_snow_albedo" + ] + + # + #----------------------------------------------------------------------- + # + # Set the array parameter containing the names of all the fields that the + # MAKE_SFC_CLIMO_TN task generates on the native FV3-LAM grid. + # + #----------------------------------------------------------------------- + # + SFC_CLIMO_FIELDS: [ + "facsf", + "maximum_snow_albedo", + "slope_type", + "snowfree_albedo", + "soil_type", + "substrate_temperature", + "vegetation_greenness", + "vegetation_type", + ] + + # + #----------------------------------------------------------------------- + # + # FNGLAC, ..., FNMSKH: + # Names of (some of the) global data files that are assumed to exist in + # a system directory specified (this directory is machine-dependent; + # the experiment generation scripts will set it and store it in the + # variable FIXgsm). These file names also appear directly in the forecast + # model's input namelist file. + # + #----------------------------------------------------------------------- + # + FNGLAC: &FNGLAC "global_glacier.2x2.grb" + FNMXIC: &FNMXIC "global_maxice.2x2.grb" + FNTSFC: &FNTSFC "RTGSST.1982.2012.monthly.clim.grb" + FNSNOC: &FNSNOC "global_snoclim.1.875.grb" + FNZORC: &FNZORC "igbp" + FNAISC: &FNAISC "CFSR.SEAICE.1982.2012.monthly.clim.grb" + FNSMCC: &FNSMCC "global_soilmgldas.t126.384.190.grb" + FNMSKH: &FNMSKH "seaice_newland.grb" + # + #----------------------------------------------------------------------- + # + # FIXgsm_FILES_TO_COPY_TO_FIXam: + # If not running in NCO mode, this array contains the names of the files + # to copy from the FIXgsm system directory to the FIXam directory under + # the experiment directory. Note that the last element has a dummy value. + # This last element will get reset by the workflow generation scripts to + # the name of the ozone production/loss file to copy from FIXgsm. The + # name of this file depends on the ozone parameterization being used, + # and that in turn depends on the CCPP physics suite specified for the + # experiment. Thus, the CCPP physics suite XML must first be read in to + # determine the ozone parameterizaton and then the name of the ozone + # production/loss file. These steps are carried out elsewhere (in one + # of the workflow generation scripts/functions). + # + #----------------------------------------------------------------------- + # + FIXgsm_FILES_TO_COPY_TO_FIXam: [ + *FNGLAC, + *FNMXIC, + *FNTSFC, + *FNSNOC, + *FNAISC, + *FNSMCC, + *FNMSKH, + "global_climaeropac_global.txt", + "fix_co2_proj/global_co2historicaldata_2010.txt", + "fix_co2_proj/global_co2historicaldata_2011.txt", + "fix_co2_proj/global_co2historicaldata_2012.txt", + "fix_co2_proj/global_co2historicaldata_2013.txt", + "fix_co2_proj/global_co2historicaldata_2014.txt", + "fix_co2_proj/global_co2historicaldata_2015.txt", + "fix_co2_proj/global_co2historicaldata_2016.txt", + "fix_co2_proj/global_co2historicaldata_2017.txt", + "fix_co2_proj/global_co2historicaldata_2018.txt", + "fix_co2_proj/global_co2historicaldata_2019.txt", + "fix_co2_proj/global_co2historicaldata_2020.txt", + "fix_co2_proj/global_co2historicaldata_2021.txt", + "global_co2historicaldata_glob.txt", + "co2monthlycyc.txt", + "global_h2o_pltc.f77", + "global_hyblev.l65.txt", + "global_zorclim.1x1.grb", + "global_sfc_emissivity_idx.txt", + "global_tg3clim.2.6x1.5.grb", + "global_solarconstant_noaa_an.txt", + "global_albedo4.1x1.grb", + "geo_em.d01.lat-lon.2.5m.HGT_M.nc", + "HGT.Beljaars_filtered.lat-lon.30s_res.nc", + "replace_with_FIXgsm_ozone_prodloss_filename" + ] + # + #----------------------------------------------------------------------- + # + # FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: + # This array is used to set some of the namelist variables in the forecast + # model's namelist file that represent the relative or absolute paths of + # various fixed files (the first column of the array, where columns are + # delineated by the pipe symbol "|") to the full paths to these files in + # the FIXam directory derived from the corresponding workflow variables + # containing file names (the second column of the array). + # + #----------------------------------------------------------------------- + # + FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: [ + !join_str ["FNGLAC | ",*FNGLAC], + !join_str ["FNMXIC | ",*FNMXIC], + !join_str ["FNTSFC | ",*FNTSFC], + !join_str ["FNSNOC | ",*FNSNOC], + !join_str ["FNAISC | ",*FNAISC], + !join_str ["FNSMCC | ",*FNSMCC], + !join_str ["FNMSKH | ",*FNMSKH] + ] + #"FNZORC | $FNZORC", + + # + #----------------------------------------------------------------------- + # + # FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: + # This array is used to set some of the namelist variables in the forecast + # model's namelist file that represent the relative or absolute paths of + # various fixed files (the first column of the array, where columns are + # delineated by the pipe symbol "|") to the full paths to surface climatology + # files (on the native FV3-LAM grid) in the FIXlam directory derived from + # the corresponding surface climatology fields (the second column of the + # array). + # + #----------------------------------------------------------------------- + # + FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: [ + "FNALBC | snowfree_albedo", + "FNALBC2 | facsf", + "FNTG3C | substrate_temperature", + "FNVEGC | vegetation_greenness", + "FNVETC | vegetation_type", + "FNSOTC | soil_type", + "FNVMNC | vegetation_greenness", + "FNVMXC | vegetation_greenness", + "FNSLPC | slope_type", + "FNABSC | maximum_snow_albedo" + ] + + + # + #----------------------------------------------------------------------- + # + # CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: + # This array specifies the mapping to use between the symlinks that need + # to be created in each cycle directory (these are the "files" that FV3 + # looks for) and their targets in the FIXam directory. The first column + # of the array specifies the symlink to be created, and the second column + # specifies its target file in FIXam (where columns are delineated by the + # pipe symbol "|"). + # + #----------------------------------------------------------------------- + # + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: [ + "aerosol.dat | global_climaeropac_global.txt", + "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", + "co2historicaldata_2011.txt | fix_co2_proj/global_co2historicaldata_2011.txt", + "co2historicaldata_2012.txt | fix_co2_proj/global_co2historicaldata_2012.txt", + "co2historicaldata_2013.txt | fix_co2_proj/global_co2historicaldata_2013.txt", + "co2historicaldata_2014.txt | fix_co2_proj/global_co2historicaldata_2014.txt", + "co2historicaldata_2015.txt | fix_co2_proj/global_co2historicaldata_2015.txt", + "co2historicaldata_2016.txt | fix_co2_proj/global_co2historicaldata_2016.txt", + "co2historicaldata_2017.txt | fix_co2_proj/global_co2historicaldata_2017.txt", + "co2historicaldata_2018.txt | fix_co2_proj/global_co2historicaldata_2018.txt", + "co2historicaldata_2019.txt | fix_co2_proj/global_co2historicaldata_2019.txt", + "co2historicaldata_2020.txt | fix_co2_proj/global_co2historicaldata_2020.txt", + "co2historicaldata_2021.txt | fix_co2_proj/global_co2historicaldata_2021.txt", + "co2historicaldata_glob.txt | global_co2historicaldata_glob.txt", + "co2monthlycyc.txt | co2monthlycyc.txt", + "global_h2oprdlos.f77 | global_h2o_pltc.f77", + "global_albedo4.1x1.grb | global_albedo4.1x1.grb", + "global_zorclim.1x1.grb | global_zorclim.1x1.grb", + "global_tg3clim.2.6x1.5.grb | global_tg3clim.2.6x1.5.grb", + "sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt", + "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", + "global_o3prdlos.f77 | " + ] diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 008cfbd063..626a855ba5 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -290,7 +290,7 @@ static) files in the FIXam directory: # isn't really an advantage to using relative symlinks, so we use symlinks # with absolute paths. # -if [ "${RUN_ENVIR}" != "nco" ]; then +if [ "${SYMLINK_FIX_FILES}" == "FALSE" ]; then relative_link_flag="TRUE" else relative_link_flag="FALSE" diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index ec06a88150..53edf57fde 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -69,56 +69,11 @@ WE2Edir="$TESTSdir/WE2E" # #----------------------------------------------------------------------- # - -# This line will return two numbers: the python major and minor versions -pyversion=($(/usr/bin/env python3 -c 'import platform; major, minor, patch = platform.python_version_tuple(); print(major); print(minor)')) - -#Now, set an error check variable so that we can print all python errors rather than just the first -pyerrors=0 - -# Check that the call to python3 returned no errors, then check if the -# python3 minor version is 6 or higher -if [[ -z "$pyversion" ]];then - print_info_msg "\ - - Error: python3 not found" - pyerrors=$((pyerrors+1)) -else - if [[ ${#pyversion[@]} -lt 2 ]]; then - print_info_msg "\ - - Error retrieving python3 version" - pyerrors=$((pyerrors+1)) - elif [[ ${pyversion[1]} -lt 6 ]]; then - print_info_msg "\ - - Error: python version must be 3.6 or higher - python version: ${pyversion[*]}" - pyerrors=$((pyerrors+1)) - fi +python3 $USHdir/check_python_version.py +if [[ $? -ne 0 ]]; then + exit 1 fi -#Next, check for the non-standard python packages: jinja2, yaml, and f90nml -pkgs=(jinja2 yaml f90nml) -for pkg in ${pkgs[@]} ; do - if ! /usr/bin/env python3 -c "import ${pkg}" &> /dev/null; then - print_info_msg "\ - - Error: python module ${pkg} not available" - pyerrors=$((pyerrors+1)) - fi -done - -#Finally, check if the number of errors is >0, and if so exit with helpful message -if [ $pyerrors -gt 0 ];then - print_err_msg_exit "\ - Errors found: check your python environment - - Instructions for setting up python environments can be found on the web: - https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started - -" -fi # #----------------------------------------------------------------------- # diff --git a/ush/UFS_plot_domains.py b/ush/UFS_plot_domains.py index d752c2696b..847cd96725 100755 --- a/ush/UFS_plot_domains.py +++ b/ush/UFS_plot_domains.py @@ -128,7 +128,6 @@ def get_lambert_points(gnomonic_map, lambert_map, pps): - #print("Hello from a function") # This function takes the lambert domain we have defined, lambert_map, as well as # pps (the number of points to interpolate and draw for each side of the lambert "rectangle"), @@ -187,9 +186,6 @@ def get_lambert_points(gnomonic_map, lambert_map, pps): # Need to replace final instruction with Path.CLOSEPOLY instructions[-1] = Path.CLOSEPOLY - #print("vertices=", vertices) - #print("instructions=", instructions) - return vertices, instructions diff --git a/ush/check_python_version.py b/ush/check_python_version.py new file mode 100755 index 0000000000..afc3dac62f --- /dev/null +++ b/ush/check_python_version.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +import sys +import logging +import platform +from textwrap import dedent + + +def check_python_version(): + """Check if python version >= 3.6 and presence of some + non-standard packages currently jinja2, yaml, f90nml""" + + # Check for non-standard python packages + try: + import jinja2 + import yaml + import f90nml + except ImportError as error: + logging.error( + dedent( + """ + Error: Missing python package required by the SRW app + """ + ) + ) + raise + + # check python version + major, minor, patch = platform.python_version_tuple() + if int(major) < 3 or int(minor) < 6: + logging.error( + dedent( + f""" + Error: python version must be 3.6 or higher + Your python version is: {major}.{minor}""" + ) + ) + raise Exception("Python version below 3.6") + + +if __name__ == "__main__": + try: + check_python_version() + except: + logging.exception( + dedent( + f""" + ************************************************************************* + FATAL ERROR: + The system does not meet minimum requirements for running the SRW app. + Instructions for setting up python environments can be found on the web: + https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started + *************************************************************************\n + """ + ) + ) + sys.exit(1) diff --git a/ush/check_ruc_lsm.py b/ush/check_ruc_lsm.py index 8d6e287df9..3afe1f8264 100644 --- a/ush/check_ruc_lsm.py +++ b/ush/check_ruc_lsm.py @@ -32,7 +32,9 @@ class Testing(unittest.TestCase): def test_check_ruc_lsm(self): USHdir = os.path.dirname(os.path.abspath(__file__)) self.assertTrue( - check_ruc_lsm(ccpp_phys_suite_fp=f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml") + check_ruc_lsm( + ccpp_phys_suite_fp=f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml" + ) ) def setUp(self): diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 2c5831a4df..43d29abe2f 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -38,7 +38,6 @@ user: # #----------------------------------------------------------------------- # - # mach_doc_start # Set machine and queue parameters. Definitions: # # MACHINE: @@ -46,7 +45,7 @@ user: # supported platform, and you want to use the Rocoto workflow manager, # you will need set MACHINE: "linux" and WORKFLOW_MANAGER: "rocoto". This # combination will assume a Slurm batch manager when generating the XML. - # Please see ush/valid_param_vals.sh for a full list of supported + # Please see ush/valid_param_vals.yaml for a full list of supported # platforms. # # ACCOUNT: @@ -72,13 +71,8 @@ platform: # the XML. Valid options: "rocoto" or "none" # # NCORES_PER_NODE: - # The number of cores available per node on the compute platform. Set - # for supported platforms in setup.sh, but is now also configurable for - # all platforms. - # - # LMOD_PATH: - # Path to the LMOD sh file on your Linux system. Is set automatically - # for supported machines. + # The number of cores available per node on the compute platform, now + # configurable for all platforms. # # BUILD_MOD_FN: # Name of alternative build module file to use if using an @@ -147,13 +141,10 @@ platform: # If this is not set or set to an empty string, it will be (re)set to a # machine-dependent value. # - # mach_doc_end - # #----------------------------------------------------------------------- # WORKFLOW_MANAGER: "" NCORES_PER_NODE: "" - LMOD_PATH: "" BUILD_MOD_FN: "" WFLOW_MOD_FN: "" BUILD_VER_FN: "" @@ -177,14 +168,15 @@ platform: # will run without MPI. # # RUN_CMD_FCST: - # The run command for the model forecast step. This will be appended to - # the end of the variable definitions file, so it can reference other - # variables. + # The run command for the model forecast step. # # RUN_CMD_POST: # The run command for post-processing (UPP). Can be left blank for smaller # domains, in which case UPP will run without MPI. # + # RUN_CMD_SERIAL: + # The run command for some serial jobs + # #----------------------------------------------------------------------- # RUN_CMD_SERIAL: "" @@ -211,6 +203,9 @@ platform: # MET_INSTALL_DIR: # Location to top-level directory of MET installation. # + # MET_BIN_EXEC: + # Subdirectory containing MET binaries e.g. "bin" + # # METPLUS_PATH: # Location to top-level directory of METplus installation. # @@ -328,7 +323,7 @@ platform: PRE_TASK_CMDS: "" # #----------------------------------------------------------------------- - # Test directories + # Test directories used in run_WE2E script #----------------------------------------------------------------------- # TEST_EXTRN_MDL_SOURCE_BASEDIR: "" @@ -340,14 +335,6 @@ platform: # WORKFLOW config parameters #----------------------------- workflow: - # - #----------------------------------------------------------------------- - # - # Unique ID for workflow run that will be set in setup.py - # - #----------------------------------------------------------------------- - # - WORKFLOW_ID: "" # #----------------------------------------------------------------------- # @@ -370,7 +357,6 @@ workflow: # #----------------------------------------------------------------------- # - # dir_doc_start # Set directories. Definitions: # # EXPT_BASEDIR: @@ -388,8 +374,6 @@ workflow: # This cannot be empty. If set to a null string here, it must be set to # a (non-empty) value in the user-defined experiment configuration file. # - # dir_doc_end - # # EXEC_SUBDIR: # The name of the subdirectory of ufs-srweather-app where executables are # installed. @@ -1042,7 +1026,7 @@ task_make_grid: # in generating the files with 0-cell-, 3-cell-, and 4-cell-wide halos; # they are not needed by the forecast model. # NOTE: Probably don't need to make ESGgrid_WIDE_HALO_WIDTH a user-specified - # variable. Just set it in the function set_gridparams_ESGgrid.sh. + # variable. Just set it in the function set_gridparams_ESGgrid.py. # # Note that: # @@ -1263,8 +1247,7 @@ task_get_extrn_lbcs: # should start. For example, the forecast should use lateral boundary # conditions from the GFS started 6 hours earlier, then # EXTRN_MDL_LBCS_OFFSET_HRS=6. - # Note: the default value is model-dependent and set in - # set_extrn_mdl_params.sh + # Note: the default value is model-dependent and set in setup.py # # FV3GFS_FILE_FMT_LBCS: # If using the FV3GFS model as the source of the LBCs (i.e. if @@ -1379,7 +1362,6 @@ task_make_lbcs: KMP_AFFINITY_MAKE_LBCS: "scatter" OMP_NUM_THREADS_MAKE_LBCS: 1 OMP_STACKSIZE_MAKE_LBCS: "1024m" - LBC_SPEC_FCST_HRS: [] #---------------------------- # FORECAST config parameters @@ -1387,7 +1369,7 @@ task_make_lbcs: task_run_fcst: RUN_FCST_TN: "run_fcst" NNODES_RUN_FCST: "" # This is calculated in the workflow generation scripts, so no need to set here. - PPN_RUN_FCST: "" # will be calculated from NCORES_PER_NODE and OMP_NUM_THREADS in setup.sh + PPN_RUN_FCST: "" # will be calculated from NCORES_PER_NODE and OMP_NUM_THREADS in setup.py WTIME_RUN_FCST: 04:30:00 MAXTRIES_RUN_FCST: 1 # @@ -1576,7 +1558,7 @@ task_run_fcst: # commonly used set of grid-dependent parameters. The predefined grid # parameters are specified in the script # - # $HOMEdir/ush/set_predef_grid_params.sh + # $HOMEdir/ush/set_predef_grid_params.py # #----------------------------------------------------------------------- # @@ -1613,6 +1595,9 @@ task_run_fcst: # # Set parameters associated with the fixed (i.e. static) files. Definitions: # + # SYMLINK_FIX_FILES: + # Symlink fix files to experiment directory if true, otherwise copy the files + # # FIXgsm: # System directory in which the majority of fixed (i.e. time-independent) # files that are needed to run the FV3-LAM model are located @@ -1623,158 +1608,12 @@ task_run_fcst: # FIXlut: # System directory where the lookup tables for optics properties are located # - # FNGLAC, ..., FNMSKH: - # Names of (some of the) global data files that are assumed to exist in - # a system directory specified (this directory is machine-dependent; - # the experiment generation scripts will set it and store it in the - # variable FIXgsm). These file names also appear directly in the forecast - # model's input namelist file. - # - # FIXgsm_FILES_TO_COPY_TO_FIXam: - # If not running in NCO mode, this array contains the names of the files - # to copy from the FIXgsm system directory to the FIXam directory under - # the experiment directory. Note that the last element has a dummy value. - # This last element will get reset by the workflow generation scripts to - # the name of the ozone production/loss file to copy from FIXgsm. The - # name of this file depends on the ozone parameterization being used, - # and that in turn depends on the CCPP physics suite specified for the - # experiment. Thus, the CCPP physics suite XML must first be read in to - # determine the ozone parameterizaton and then the name of the ozone - # production/loss file. These steps are carried out elsewhere (in one - # of the workflow generation scripts/functions). - # - # FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: - # This array is used to set some of the namelist variables in the forecast - # model's namelist file that represent the relative or absolute paths of - # various fixed files (the first column of the array, where columns are - # delineated by the pipe symbol "|") to the full paths to these files in - # the FIXam directory derived from the corresponding workflow variables - # containing file names (the second column of the array). - # - # FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: - # This array is used to set some of the namelist variables in the forecast - # model's namelist file that represent the relative or absolute paths of - # various fixed files (the first column of the array, where columns are - # delineated by the pipe symbol "|") to the full paths to surface climatology - # files (on the native FV3-LAM grid) in the FIXlam directory derived from - # the corresponding surface climatology fields (the second column of the - # array). - # - # CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: - # This array specifies the mapping to use between the symlinks that need - # to be created in each cycle directory (these are the "files" that FV3 - # looks for) and their targets in the FIXam directory. The first column - # of the array specifies the symlink to be created, and the second column - # specifies its target file in FIXam (where columns are delineated by the - # pipe symbol "|"). - # - #----------------------------------------------------------------------- - # - # Because the default values are dependent on the platform, we set these - # to a null string which will then be overwritten in setup.sh unless the - # user has specified a different value in config.sh + #----------------------------------------------------------------------- # SYMLINK_FIX_FILES: true FIXgsm: "" FIXaer: "" FIXlut: "" - - FNGLAC: &FNGLAC "global_glacier.2x2.grb" - FNMXIC: &FNMXIC "global_maxice.2x2.grb" - FNTSFC: &FNTSFC "RTGSST.1982.2012.monthly.clim.grb" - FNSNOC: &FNSNOC "global_snoclim.1.875.grb" - FNZORC: &FNZORC "igbp" - FNAISC: &FNAISC "CFSR.SEAICE.1982.2012.monthly.clim.grb" - FNSMCC: &FNSMCC "global_soilmgldas.t126.384.190.grb" - FNMSKH: &FNMSKH "seaice_newland.grb" - - FIXgsm_FILES_TO_COPY_TO_FIXam: [ - *FNGLAC, - *FNMXIC, - *FNTSFC, - *FNSNOC, - *FNAISC, - *FNSMCC, - *FNMSKH, - "global_climaeropac_global.txt", - "fix_co2_proj/global_co2historicaldata_2010.txt", - "fix_co2_proj/global_co2historicaldata_2011.txt", - "fix_co2_proj/global_co2historicaldata_2012.txt", - "fix_co2_proj/global_co2historicaldata_2013.txt", - "fix_co2_proj/global_co2historicaldata_2014.txt", - "fix_co2_proj/global_co2historicaldata_2015.txt", - "fix_co2_proj/global_co2historicaldata_2016.txt", - "fix_co2_proj/global_co2historicaldata_2017.txt", - "fix_co2_proj/global_co2historicaldata_2018.txt", - "fix_co2_proj/global_co2historicaldata_2019.txt", - "fix_co2_proj/global_co2historicaldata_2020.txt", - "fix_co2_proj/global_co2historicaldata_2021.txt", - "global_co2historicaldata_glob.txt", - "co2monthlycyc.txt", - "global_h2o_pltc.f77", - "global_hyblev.l65.txt", - "global_zorclim.1x1.grb", - "global_sfc_emissivity_idx.txt", - "global_tg3clim.2.6x1.5.grb", - "global_solarconstant_noaa_an.txt", - "global_albedo4.1x1.grb", - "geo_em.d01.lat-lon.2.5m.HGT_M.nc", - "HGT.Beljaars_filtered.lat-lon.30s_res.nc", - "replace_with_FIXgsm_ozone_prodloss_filename" - ] - - # - # It is possible to remove this as a workflow variable and make it only - # a local one since it is used in only one script. - # - FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING: [ - !join_str ["FNGLAC | ",*FNGLAC], - !join_str ["FNMXIC | ",*FNMXIC], - !join_str ["FNTSFC | ",*FNTSFC], - !join_str ["FNSNOC | ",*FNSNOC], - !join_str ["FNAISC | ",*FNAISC], - !join_str ["FNSMCC | ",*FNSMCC], - !join_str ["FNMSKH | ",*FNMSKH] - ] - #"FNZORC | $FNZORC", - - FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: [ - "FNALBC | snowfree_albedo", - "FNALBC2 | facsf", - "FNTG3C | substrate_temperature", - "FNVEGC | vegetation_greenness", - "FNVETC | vegetation_type", - "FNSOTC | soil_type", - "FNVMNC | vegetation_greenness", - "FNVMXC | vegetation_greenness", - "FNSLPC | slope_type", - "FNABSC | maximum_snow_albedo" - ] - - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING: [ - "aerosol.dat | global_climaeropac_global.txt", - "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", - "co2historicaldata_2011.txt | fix_co2_proj/global_co2historicaldata_2011.txt", - "co2historicaldata_2012.txt | fix_co2_proj/global_co2historicaldata_2012.txt", - "co2historicaldata_2013.txt | fix_co2_proj/global_co2historicaldata_2013.txt", - "co2historicaldata_2014.txt | fix_co2_proj/global_co2historicaldata_2014.txt", - "co2historicaldata_2015.txt | fix_co2_proj/global_co2historicaldata_2015.txt", - "co2historicaldata_2016.txt | fix_co2_proj/global_co2historicaldata_2016.txt", - "co2historicaldata_2017.txt | fix_co2_proj/global_co2historicaldata_2017.txt", - "co2historicaldata_2018.txt | fix_co2_proj/global_co2historicaldata_2018.txt", - "co2historicaldata_2019.txt | fix_co2_proj/global_co2historicaldata_2019.txt", - "co2historicaldata_2020.txt | fix_co2_proj/global_co2historicaldata_2020.txt", - "co2historicaldata_2021.txt | fix_co2_proj/global_co2historicaldata_2021.txt", - "co2historicaldata_glob.txt | global_co2historicaldata_glob.txt", - "co2monthlycyc.txt | co2monthlycyc.txt", - "global_h2oprdlos.f77 | global_h2o_pltc.f77", - "global_albedo4.1x1.grb | global_albedo4.1x1.grb", - "global_zorclim.1x1.grb | global_zorclim.1x1.grb", - "global_tg3clim.2.6x1.5.grb | global_tg3clim.2.6x1.5.grb", - "sfc_emissivity_idx.txt | global_sfc_emissivity_idx.txt", - "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", - "global_o3prdlos.f77 | " - ] #---------------------------- # POST config parameters @@ -2184,7 +2023,7 @@ global: # # Set default SPP stochastic physics options. Each SPP option is an array, # applicable (in order) to the scheme/parameter listed in SPP_VAR_LIST. - # Enter each value of the array in config.sh as shown below without commas + # Enter each value of the array in config.yaml as shown below without commas # or single quotes (e.g., SPP_VAR_LIST=( "pbl" "sfc" "mp" "rad" "gwd" ). # Both commas and single quotes will be added by Jinja when creating the # namelist. diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index cd1943ef77..15a42729f4 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -36,26 +36,25 @@ def create_diag_table_file(run_dir): # create a diagnostic table file within the specified run directory print_info_msg( - f''' - Creating a diagnostics table file (\"{DIAG_TABLE_FN}\") in the specified + f""" + Creating a diagnostics table file ('{DIAG_TABLE_FN}') in the specified run directory... - run_dir = \"{run_dir}\"''', + run_dir = '{run_dir}'""", verbose=VERBOSE, ) diag_table_fp = os.path.join(run_dir, DIAG_TABLE_FN) print_info_msg( - f''' - + f""" Using the template diagnostics table file: diag_table_tmpl_fp = {DIAG_TABLE_TMPL_FP} to create: - diag_table_fp = \"{diag_table_fp}\"''', + diag_table_fp = '{diag_table_fp}'""", verbose=VERBOSE, ) @@ -65,9 +64,9 @@ def create_diag_table_file(run_dir): print_info_msg( dedent( f""" - The variable \"settings\" specifying values to be used in the \"{DIAG_TABLE_FN}\" - file has been set as follows:\n - settings =\n\n""" + The variable 'settings' specifying values to be used in the '{DIAG_TABLE_FN}' + file has been set as follows:\n + settings =\n\n""" ) + settings_str, verbose=VERBOSE, @@ -82,14 +81,14 @@ def create_diag_table_file(run_dir): print_err_msg_exit( dedent( f""" - Call to python script fill_jinja_template.py to create a \"{DIAG_TABLE_FN}\" - file from a jinja2 template failed. Parameters passed to this script are: - Full path to template diag table file: - DIAG_TABLE_TMPL_FP = \"{DIAG_TABLE_TMPL_FP}\" - Full path to output diag table file: - diag_table_fp = \"{diag_table_fp}\" - Namelist settings specified on command line:\n - settings =\n\n""" + Call to python script fill_jinja_template.py to create a '{DIAG_TABLE_FN}' + file from a jinja2 template failed. Parameters passed to this script are: + Full path to template diag table file: + DIAG_TABLE_TMPL_FP = '{DIAG_TABLE_TMPL_FP}' + Full path to output diag table file: + diag_table_fp = '{diag_table_fp}' + Namelist settings specified on command line:\n + settings =\n\n""" ) + settings_str ) @@ -133,9 +132,7 @@ def setUp(self): USHdir = os.path.dirname(os.path.abspath(__file__)) PARMdir = os.path.join(USHdir, "..", "parm") DIAG_TABLE_FN = "diag_table" - DIAG_TABLE_TMPL_FP = os.path.join( - PARMdir, f"{DIAG_TABLE_FN}.FV3_GFS_v15p2" - ) + DIAG_TABLE_TMPL_FP = os.path.join(PARMdir, f"{DIAG_TABLE_FN}.FV3_GFS_v15p2") set_env_var("DEBUG", True) set_env_var("VERBOSE", True) set_env_var("USHdir", USHdir) diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 1cd445a3e9..7015e59729 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -52,10 +52,10 @@ def create_model_configure_file( # ----------------------------------------------------------------------- # print_info_msg( - f''' - Creating a model configuration file (\"{MODEL_CONFIG_FN}\") in the specified + f""" + Creating a model configuration file ('{MODEL_CONFIG_FN}') in the specified run directory (run_dir): - run_dir = \"{run_dir}\"''', + run_dir = '{run_dir}'""", verbose=VERBOSE, ) # @@ -190,7 +190,7 @@ def create_model_configure_file( print_info_msg( dedent( f""" - The variable \"settings\" specifying values to be used in the \"{MODEL_CONFIG_FN}\" + The variable 'settings' specifying values to be used in the '{MODEL_CONFIG_FN}' file has been set as follows:\n settings =\n\n""" ) @@ -223,14 +223,14 @@ def create_model_configure_file( print_err_msg_exit( dedent( f""" - Call to python script fill_jinja_template.py to create a \"{MODEL_CONFIG_FN}\" - file from a jinja2 template failed. Parameters passed to this script are: - Full path to template model config file: - MODEL_CONFIG_TMPL_FP = \"{MODEL_CONFIG_TMPL_FP}\" - Full path to output model config file: - model_config_fp = \"{model_config_fp}\" - Namelist settings specified on command line:\n - settings =\n\n""" + Call to python script fill_jinja_template.py to create a '{MODEL_CONFIG_FN}' + file from a jinja2 template failed. Parameters passed to this script are: + Full path to template model config file: + MODEL_CONFIG_TMPL_FP = '{MODEL_CONFIG_TMPL_FP}' + Full path to output model config file: + model_config_fp = '{model_config_fp}' + Namelist settings specified on command line:\n + settings =\n\n""" ) + settings_str ) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 11a86c9aaa..969485fce0 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -2,7 +2,6 @@ import os import sys -import platform import subprocess import unittest import logging @@ -37,33 +36,10 @@ from get_crontab_contents import add_crontab_line from fill_jinja_template import fill_jinja_template from set_namelist import set_namelist +from check_python_version import check_python_version -def python_error_handler(): - """Error handler for missing packages""" - - print_err_msg_exit( - """ - Errors found: check your python environment - - Instructions for setting up python environments can be found on the web: - https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started - """, - stack_trace=False, - ) - - -# Check for non-standard python packages -try: - import jinja2 - import yaml - import f90nml -except ImportError as error: - print_info_msg(error.__class__.__name__ + ": " + str(error)) - python_error_handler() - - -def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> None: +def generate_FV3LAM_wflow(USHdir, logfile: str = "log.generate_FV3LAM_wflow") -> None: """Function to setup a forecast experiment and create a workflow (according to the parameters specified in the config file) @@ -77,28 +53,21 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> # Set up logging to write to screen and logfile setup_logging(logfile) + # Check python version and presence of some non-standard packages + check_python_version() + + # Note start of workflow generation log_info( - """ + """ ======================================================================== Starting experiment generation... ========================================================================""" ) - # check python version - major, minor, patch = platform.python_version_tuple() - if int(major) < 3 or int(minor) < 6: - logging.error( - f""" - - Error: python version must be 3.6 or higher - python version: {major}.{minor}""" - ) - raise - - # define utilities + # define utilities define_macos_utilities() - # The setup function reads the user configuration file and fills in + # The setup function reads the user configuration file and fills in # non-user-specified values from config_defaults.yaml setup() @@ -132,11 +101,11 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> template_xml_fp = os.path.join(PARMdir, WFLOW_XML_FN) log_info( - f''' + f""" Creating rocoto workflow XML file (WFLOW_XML_FP) from jinja template XML file (template_xml_fp): - template_xml_fp = \"{template_xml_fp}\" - WFLOW_XML_FP = \"{WFLOW_XML_FP}\"''' + template_xml_fp = '{template_xml_fp}' + WFLOW_XML_FP = '{WFLOW_XML_FP}'""" ) ensmem_indx_name = "" @@ -427,13 +396,13 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> settings_str = cfg_to_yaml_str(settings) log_info( - f""" - The variable \"settings\" specifying values of the rococo XML variables - has been set as follows: - #----------------------------------------------------------------------- - settings =\n\n""", - verbose=VERBOSE, - ) + f""" + The variable 'settings' specifying values of the rococo XML variables + has been set as follows: + #----------------------------------------------------------------------- + settings =\n\n""", + verbose=VERBOSE, + ) log_info(settings_str, verbose=VERBOSE) # @@ -448,15 +417,15 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> logging.exception( dedent( f""" - Call to python script fill_jinja_template.py to create a rocoto workflow - XML file from a template file failed. Parameters passed to this script - are: - Full path to template rocoto XML file: - template_xml_fp = \"{template_xml_fp}\" - Full path to output rocoto XML file: - WFLOW_XML_FP = \"{WFLOW_XML_FP}\" - Namelist settings specified on command line:\n - settings =\n\n""" + Call to python script fill_jinja_template.py to create a rocoto workflow + XML file from a template file failed. Parameters passed to this script + are: + Full path to template rocoto XML file: + template_xml_fp = '{template_xml_fp}' + Full path to output rocoto XML file: + WFLOW_XML_FP = '{WFLOW_XML_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" ) + settings_str ) @@ -469,11 +438,11 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> # ----------------------------------------------------------------------- # log_info( - f''' + f""" Creating symlink in the experiment directory (EXPTDIR) that points to the workflow launch script (WFLOW_LAUNCH_SCRIPT_FP): - EXPTDIR = \"{EXPTDIR}\" - WFLOW_LAUNCH_SCRIPT_FP = \"{WFLOW_LAUNCH_SCRIPT_FP}\"''', + EXPTDIR = '{EXPTDIR}' + WFLOW_LAUNCH_SCRIPT_FP = '{WFLOW_LAUNCH_SCRIPT_FP}'""", verbose=VERBOSE, ) @@ -498,21 +467,21 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> if SYMLINK_FIX_FILES: log_info( - f''' + f""" Symlinking fixed files from system directory (FIXgsm) to a subdirectory (FIXam): - FIXgsm = \"{FIXgsm}\" - FIXam = \"{FIXam}\"''', + FIXgsm = '{FIXgsm}' + FIXam = '{FIXam}'""", verbose=VERBOSE, ) - ln_vrfy(f'''-fsn "{FIXgsm}" "{FIXam}"''') + ln_vrfy(f"""-fsn '{FIXgsm}' '{FIXam}'""") else: log_info( - f''' + f""" Copying fixed files from system directory (FIXgsm) to a subdirectory (FIXam): - FIXgsm = \"{FIXgsm}\" - FIXam = \"{FIXam}\"''', + FIXgsm = '{FIXgsm}' + FIXam = '{FIXam}'""", verbose=VERBOSE, ) @@ -533,12 +502,12 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> # if USE_MERRA_CLIMO: log_info( - f''' + f""" Copying MERRA2 aerosol climatology data files from system directory (FIXaer/FIXlut) to a subdirectory (FIXclim) in the experiment directory: - FIXaer = \"{FIXaer}\" - FIXlut = \"{FIXlut}\" - FIXclim = \"{FIXclim}\"''', + FIXaer = '{FIXaer}' + FIXlut = '{FIXlut}' + FIXclim = '{FIXclim}'""", verbose=VERBOSE, ) @@ -616,9 +585,9 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> # ----------------------------------------------------------------------- # log_info( - f''' + f""" Setting parameters in weather model's namelist file (FV3_NML_FP): - FV3_NML_FP = \"{FV3_NML_FP}\"''' + FV3_NML_FP = '{FV3_NML_FP}'""" ) # # Set npx and npy, which are just NX plus 1 and NY plus 1, respectively. @@ -848,9 +817,9 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> settings_str = cfg_to_yaml_str(settings) log_info( - f""" - The variable \"settings\" specifying values of the weather model's - namelist variables has been set as follows:\n""", + f""" + The variable 'settings' specifying values of the weather model's + namelist variables has been set as follows:\n""", verbose=VERBOSE, ) log_info("\nsettings =\n\n" + settings_str, verbose=VERBOSE) @@ -886,18 +855,18 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> logging.exception( dedent( f""" - Call to python script set_namelist.py to generate an FV3 namelist file - failed. Parameters passed to this script are: - Full path to base namelist file: - FV3_NML_BASE_SUITE_FP = \"{FV3_NML_BASE_SUITE_FP}\" - Full path to yaml configuration file for various physics suites: - FV3_NML_YAML_CONFIG_FP = \"{FV3_NML_YAML_CONFIG_FP}\" - Physics suite to extract from yaml configuration file: - CCPP_PHYS_SUITE = \"{CCPP_PHYS_SUITE}\" - Full path to output namelist file: - FV3_NML_FP = \"{FV3_NML_FP}\" - Namelist settings specified on command line:\n - settings =\n\n""" + Call to python script set_namelist.py to generate an FV3 namelist file + failed. Parameters passed to this script are: + Full path to base namelist file: + FV3_NML_BASE_SUITE_FP = '{FV3_NML_BASE_SUITE_FP}' + Full path to yaml configuration file for various physics suites: + FV3_NML_YAML_CONFIG_FP = '{FV3_NML_YAML_CONFIG_FP}' + Physics suite to extract from yaml configuration file: + CCPP_PHYS_SUITE = '{CCPP_PHYS_SUITE}' + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Namelist settings specified on command line:\n + settings =\n\n""" ) + settings_str ) @@ -921,7 +890,15 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> if NOMADS: raise Exception("Nomads script does not work!") - # get_nomads_data(NOMADS_file_type,EXPTDIR,USHdir,DATE_FIRST_CYCL,CYCL_HRS,FCST_LEN_HRS,LBC_SPEC_INTVL_HRS) + # get_nomads_data( + # NOMADS_file_type, + # EXPTDIR, + # USHdir, + # DATE_FIRST_CYCL, + # CYCL_HRS, + # FCST_LEN_HRS, + # LBC_SPEC_INTVL_HRS, + # ) # # ----------------------------------------------------------------------- @@ -933,21 +910,8 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> # ----------------------------------------------------------------------- # cp_vrfy(os.path.join(USHdir, EXPT_CONFIG_FN), EXPTDIR) - # - # ----------------------------------------------------------------------- - # - # For convenience, print out the commands that need to be issued on the - # command line in order to launch the workflow and to check its status. - # Also, print out the line that should be placed in the user's cron table - # in order for the workflow to be continually resubmitted. - # - # ----------------------------------------------------------------------- - # - if WORKFLOW_MANAGER == "rocoto": - wflow_db_fn = f"{os.path.splitext(WFLOW_XML_FN)[0]}.db" - rocotorun_cmd = f"rocotorun -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" - rocotostat_cmd = f"rocotostat -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" + # Note workflow generation completion log_info( f""" ======================================================================== @@ -955,19 +919,26 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> Experiment generation completed. The experiment directory is: - EXPTDIR=\"{EXPTDIR}\" + EXPTDIR='{EXPTDIR}' ======================================================================== ======================================================================== """ ) + # # ----------------------------------------------------------------------- # - # If rocoto is required, print instructions on how to use it + # For convenience, print out the commands that need to be issued on the + # command line in order to launch the workflow and to check its status. + # Also, print out the line that should be placed in the user's cron table + # in order for the workflow to be continually resubmitted. # # ----------------------------------------------------------------------- # if WORKFLOW_MANAGER == "rocoto": + wflow_db_fn = f"{os.path.splitext(WFLOW_XML_FN)[0]}.db" + rocotorun_cmd = f"rocotorun -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" + rocotostat_cmd = f"rocotostat -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" log_info( f""" @@ -992,67 +963,89 @@ def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> the rocotorun command must be issued immediately before issuing the rocotostat command. - For automatic resubmission of the workflow (say every 3 minutes), the - following line can be added to the user's crontab (use \"crontab -e\" to + For automatic resubmission of the workflow (say every {CRON_RELAUNCH_INTVL_MNTS} minutes), the + following line can be added to the user's crontab (use 'crontab -e' to edit the cron table): - */{CRON_RELAUNCH_INTVL_MNTS} * * * * cd {EXPTDIR} && ./launch_FV3LAM_wflow.sh called_from_cron=\"TRUE\" + */{CRON_RELAUNCH_INTVL_MNTS} * * * * cd {EXPTDIR} && ./launch_FV3LAM_wflow.sh called_from_cron="TRUE" """ ) # If we got to this point everything was successful: move the log file to the experiment directory. mv_vrfy(logfile, EXPTDIR) -def get_nomads_data(NOMADS_file_type,EXPTDIR,USHdir,DATE_FIRST_CYCL,CYCL_HRS,FCST_LEN_HRS,LBC_SPEC_INTVL_HRS): + +def get_nomads_data( + NOMADS_file_type, + EXPTDIR, + USHdir, + DATE_FIRST_CYCL, + CYCL_HRS, + FCST_LEN_HRS, + LBC_SPEC_INTVL_HRS, +): print("Getting NOMADS online data") print(f"NOMADS_file_type= {NOMADS_file_type}") cd_vrfy(EXPTDIR) NOMADS_script = os.path.join(USHdir, "NOMADS_get_extrn_mdl_files.sh") - run_command(f"""{NOMADS_script} {date_to_str(DATE_FIRST_CYCL,format="%Y%m%d")} \ - {date_to_str(DATE_FIRST_CYCL,format="%H")} {NOMADS_file_type} {FCST_LEN_HRS} {LBC_SPEC_INTVL_HRS}""") + run_command( + f"""{NOMADS_script} \ + {date_to_str(DATE_FIRST_CYCL,format='%Y%m%d')} \ + {date_to_str(DATE_FIRST_CYCL,format='%H')} \ + {NOMADS_file_type} \ + {FCST_LEN_HRS} \ + {LBC_SPEC_INTVL_HRS}""" + ) + -def setup_logging(logfile: str = 'log.generate_FV3LAM_wflow') -> None: +def setup_logging(logfile: str = "log.generate_FV3LAM_wflow") -> None: """ Sets up logging, printing high-priority (INFO and higher) messages to screen, and printing all messages with detailed timing and routine info in the specified text file. """ - logging.basicConfig(level=logging.DEBUG, - format='%(name)-22s %(levelname)-8s %(message)s', - filename=logfile, - filemode='w') - logging.debug(f'Finished setting up debug file logging in {logfile}') + logging.basicConfig( + level=logging.DEBUG, + format="%(name)-22s %(levelname)-8s %(message)s", + filename=logfile, + filemode="w", + ) + logging.debug(f"Finished setting up debug file logging in {logfile}") console = logging.StreamHandler() console.setLevel(logging.INFO) logging.getLogger().addHandler(console) - logging.debug('Logging set up successfully') + logging.debug("Logging set up successfully") + if __name__ == "__main__": USHdir = os.path.dirname(os.path.abspath(__file__)) - logfile=f'{USHdir}/log.generate_FV3LAM_wflow' + logfile = f"{USHdir}/log.generate_FV3LAM_wflow" # Call the generate_FV3LAM_wflow function defined above to generate the # experiment/workflow. try: generate_FV3LAM_wflow(USHdir, logfile) except: - logging.exception(dedent( - f""" - ********************************************************************* - FATAL ERROR: - Experiment generation failed. See the error message(s) printed below. - For more detailed information, check the log file from the workflow - generation script: {logfile} - *********************************************************************\n - """ - )) + logging.exception( + dedent( + f""" + ********************************************************************* + FATAL ERROR: + Experiment generation failed. See the error message(s) printed below. + For more detailed information, check the log file from the workflow + generation script: {logfile} + *********************************************************************\n + """ + ) + ) + class Testing(unittest.TestCase): def test_generate_FV3LAM_wflow(self): # run workflows in separate process to avoid conflict between community and nco settings - def run_workflow(USHdir,logfile): - p = Process(target=generate_FV3LAM_wflow,args=(USHdir,logfile)) + def run_workflow(USHdir, logfile): + p = Process(target=generate_FV3LAM_wflow, args=(USHdir, logfile)) p.start() p.join() exit_code = p.exitcode @@ -1060,18 +1053,22 @@ def run_workflow(USHdir,logfile): sys.exit(exit_code) USHdir = os.path.dirname(os.path.abspath(__file__)) - logfile='log.generate_FV3LAM_wflow' + logfile = "log.generate_FV3LAM_wflow" SED = get_env_var("SED") # community test case cp_vrfy(f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") - run_command(f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""") + run_command( + f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""" + ) run_workflow(USHdir, logfile) # nco test case set_env_var("OPSROOT", f"{USHdir}/../../nco_dirs") cp_vrfy(f"{USHdir}/config.nco.yaml", f"{USHdir}/config.yaml") - run_command(f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""") + run_command( + f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""" + ) run_workflow(USHdir, logfile) def setUp(self): diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index c6c8d41ee0..a1e7863a43 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -64,22 +64,22 @@ def get_crontab_contents(called_from_cron): __crontab_cmd__ = "/usr/bin/crontab" print_info_msg( - f''' + f""" Getting crontab content with command: ========================================================= {__crontab_cmd__} -l - =========================================================''', + =========================================================""", verbose=DEBUG, ) (_, __crontab_contents__, _) = run_command(f"""{__crontab_cmd__} -l""") print_info_msg( - f''' + f""" Crontab contents: ========================================================= {__crontab_contents__} - =========================================================''', + =========================================================""", verbose=DEBUG, ) @@ -102,9 +102,9 @@ def add_crontab_line(): time_stamp = datetime.now().strftime("%F_%T") crontab_backup_fp = os.path.join(EXPTDIR, f"crontab.bak.{time_stamp}") log_info( - f''' + f""" Copying contents of user cron table to backup file: - crontab_backup_fp = \"{crontab_backup_fp}\"''', + crontab_backup_fp = '{crontab_backup_fp}'""", verbose=VERBOSE, ) @@ -120,25 +120,25 @@ def add_crontab_line(): ) # Create backup - run_command(f'''printf "%s" '{crontab_contents}' > "{crontab_backup_fp}"''') + run_command(f"""printf "%s" '{crontab_contents}' > '{crontab_backup_fp}'""") # Add crontab line if CRONTAB_LINE in crontab_contents: log_info( - f''' + f""" The following line already exists in the cron table and thus will not be added: - CRONTAB_LINE = \"{CRONTAB_LINE}\"''' + CRONTAB_LINE = '{CRONTAB_LINE}'""" ) else: log_info( - f''' + f""" Adding the following line to the user's cron table in order to automatically resubmit SRW workflow: - CRONTAB_LINE = \"{CRONTAB_LINE}\"''', + CRONTAB_LINE = '{CRONTAB_LINE}'""", verbose=VERBOSE, ) @@ -173,11 +173,11 @@ def delete_crontab_line(called_from_cron): # Then record the results back into the user's cron table. # print_info_msg( - f''' + f""" Crontab contents before delete: ========================================================= {crontab_contents} - =========================================================''', + =========================================================""", verbose=True, ) @@ -189,11 +189,11 @@ def delete_crontab_line(called_from_cron): run_command(f"""echo '{crontab_contents}' | {crontab_cmd}""") print_info_msg( - f''' + f""" Crontab contents after delete: ========================================================= {crontab_contents} - =========================================================''', + =========================================================""", verbose=True, ) diff --git a/ush/link_fix.py b/ush/link_fix.py index 752fb7743f..5602d5170f 100755 --- a/ush/link_fix.py +++ b/ush/link_fix.py @@ -230,7 +230,7 @@ def link_fix(verbose, file_group): f""" The resolution could not be extracted from the current file's name. The full path to the file (fp) is: - fp = \"{fp}\" + fp = '{fp}' This may be because fp contains the * globbing character, which would imply that no files were found that match the globbing pattern specified in fp.""" @@ -243,8 +243,8 @@ def link_fix(verbose, file_group): f""" The resolutions (as obtained from the file names) of the previous and current file (fp_prev and fp, respectively) are different: - fp_prev = \"{fp_prev}\" - fp = \"{fp}\" + fp_prev = '{fp_prev}' + fp = '{fp}' Please ensure that all files have the same resolution.""" ) diff --git a/ush/machine/odin.yaml b/ush/machine/odin.yaml index 67a2fb49f1..38caa4bfc1 100644 --- a/ush/machine/odin.yaml +++ b/ush/machine/odin.yaml @@ -2,7 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 24 SCHED: slurm - DOMAIN_PREGEN_BASEDIR: /FV3LAM_pregen + DOMAIN_PREGEN_BASEDIR: /scratch/ywang/UFS_SRW_App/develop/FV3LAM_pregen PARTITION_DEFAULT: workq QUEUE_DEFAULT: workq PARTITION_FCST: workq diff --git a/ush/preamble.sh b/ush/preamble.sh index 0357ace76f..4c75665ed6 100644 --- a/ush/preamble.sh +++ b/ush/preamble.sh @@ -91,7 +91,7 @@ TRACE=${DEBUG:-"FALSE"} if [[ $STRICT == "TRUE" ]]; then # Exit on error and undefined variable - set -eu + set -euo pipefail fi if [[ $TRACE == "TRUE" ]]; then # Turn on debugging diff --git a/ush/python_utils/__init__.py b/ush/python_utils/__init__.py index add96ee981..02e2b83b77 100644 --- a/ush/python_utils/__init__.py +++ b/ush/python_utils/__init__.py @@ -1,7 +1,6 @@ from .misc import uppercase, lowercase, find_pattern_in_str, find_pattern_in_file from .check_for_preexist_dir_file import check_for_preexist_dir_file from .check_var_valid_value import check_var_valid_value -from .count_files import count_files from .create_symlink_to_file import create_symlink_to_file from .define_macos_utilities import define_macos_utilities from .environment import ( @@ -25,12 +24,9 @@ mkdir_vrfy, cd_vrfy, ) -from .get_elem_inds import get_elem_inds -from .interpol_to_arbit_CRES import interpol_to_arbit_CRES from .print_input_args import print_input_args from .print_msg import print_info_msg, print_err_msg_exit, log_info from .run_command import run_command -from .get_charvar_from_netcdf import get_charvar_from_netcdf from .xml_parser import load_xml_file, has_tag_with_value from .config_parser import ( load_json_config, diff --git a/ush/python_utils/check_for_preexist_dir_file.py b/ush/python_utils/check_for_preexist_dir_file.py index cfd4d50f43..39ad7f1706 100644 --- a/ush/python_utils/check_for_preexist_dir_file.py +++ b/ush/python_utils/check_for_preexist_dir_file.py @@ -7,6 +7,7 @@ from .filesys_cmds_vrfy import rm_vrfy, mv_vrfy from .print_msg import log_info + def check_for_preexist_dir_file(path, method): """Check for a preexisting directory or file and, if present, deal with it according to the specified method @@ -21,10 +22,12 @@ def check_for_preexist_dir_file(path, method): try: check_var_valid_value(method, ["delete", "rename", "quit"]) except ValueError: - errmsg = dedent(f''' - Invalid method for dealing with pre-existing directory specified - method = {method} - ''') + errmsg = dedent( + f""" + Invalid method for dealing with pre-existing directory specified + method = {method} + """ + ) raise ValueError(errmsg) from None if os.path.exists(path): @@ -43,8 +46,10 @@ def check_for_preexist_dir_file(path, method): ) mv_vrfy(path, new_path) else: - raise FileExistsError(dedent( - f""" + raise FileExistsError( + dedent( + f""" Specified directory or file already exists {path}""" - )) + ) + ) diff --git a/ush/python_utils/check_var_valid_value.py b/ush/python_utils/check_var_valid_value.py index a8b88e17a6..0c9bcc49c6 100644 --- a/ush/python_utils/check_var_valid_value.py +++ b/ush/python_utils/check_var_valid_value.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 + def check_var_valid_value(var, values): """Check if specified variable has a valid value diff --git a/ush/python_utils/config_parser.py b/ush/python_utils/config_parser.py index c1b69db5ff..e53b8cec1f 100644 --- a/ush/python_utils/config_parser.py +++ b/ush/python_utils/config_parser.py @@ -60,8 +60,8 @@ def increase_indent(self, flow=False, indentless=False): def str_presenter(dumper, data): if len(data.splitlines()) > 1: - return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') - return dumper.represent_scalar('tag:yaml.org,2002:str', data) + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + return dumper.represent_scalar("tag:yaml.org,2002:str", data) yaml.add_representer(str, str_presenter) @@ -214,10 +214,13 @@ def load_ini_config(config_file, return_string=0): """Load a config file with a format similar to Microsoft's INI files""" if not os.path.exists(config_file): - raise FileNotFoundError(dedent(f''' - The specified configuration file does not exist: - "{config_file}"''' - )) + raise FileNotFoundError( + dedent( + f""" + The specified configuration file does not exist: + '{config_file}'""" + ) + ) config = configparser.RawConfigParser() config.optionxform = str @@ -233,7 +236,7 @@ def get_ini_value(config, section, key): """Finds the value of a property in a given section""" if not section in config: - raise KeyError(f'Section not found: {section}') + raise KeyError(f"Section not found: {section}") else: return config[section][key] diff --git a/ush/python_utils/count_files.py b/ush/python_utils/count_files.py deleted file mode 100644 index 36862895b7..0000000000 --- a/ush/python_utils/count_files.py +++ /dev/null @@ -1,16 +0,0 @@ -import glob - - -def count_files(ext, dirct="."): - """Function that returns the number of files in the specified directory - ending with the specified file extension - - Args: - ext: File extension string - dir: Directory to parse (default is current directory) - Returns: - int: Number of files - """ - - files = glob.glob(dirct + "/*." + ext) - return len(files) diff --git a/ush/python_utils/create_symlink_to_file.py b/ush/python_utils/create_symlink_to_file.py index 3a3fddb08b..363a49fa40 100644 --- a/ush/python_utils/create_symlink_to_file.py +++ b/ush/python_utils/create_symlink_to_file.py @@ -22,26 +22,26 @@ def create_symlink_to_file(target, symlink, relative=True): if target is None: print_err_msg_exit( - f''' - The argument \"target\" specifying the target of the symbolic link that + f""" + The argument 'target' specifying the target of the symbolic link that this function will create was not specified in the call to this function: - target = \"{target}\"''' + target = '{target}'""" ) if symlink is None: print_err_msg_exit( - f''' - The argument \"symlink\" specifying the target of the symbolic link that + f""" + The argument 'symlink' specifying the target of the symbolic link that this function will create was not specified in the call to this function: - symlink = \"{symlink}\"''' + symlink = '{symlink}'""" ) if not os.path.exists(target): print_err_msg_exit( - f''' + f""" Cannot create symlink to specified target file because the latter does not exist or is not a file: - target = \"{target}\"''' + target = '{target}'""" ) relative_flag = "" diff --git a/ush/python_utils/filesys_cmds_vrfy.py b/ush/python_utils/filesys_cmds_vrfy.py index 7cae27b7d5..d99b97c7f2 100644 --- a/ush/python_utils/filesys_cmds_vrfy.py +++ b/ush/python_utils/filesys_cmds_vrfy.py @@ -17,7 +17,7 @@ def cmd_vrfy(cmd, *args): cmd += " " + " ".join([str(a) for a in args]) ret = os.system(cmd) if ret != 0: - print_err_msg_exit(f'System call "{cmd}" failed.') + print_err_msg_exit(f"System call '{cmd}' failed.") return ret diff --git a/ush/python_utils/get_charvar_from_netcdf.py b/ush/python_utils/get_charvar_from_netcdf.py deleted file mode 100644 index c5f642b03a..0000000000 --- a/ush/python_utils/get_charvar_from_netcdf.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 - -import os -from .print_msg import print_err_msg_exit -from .run_command import run_command - - -def get_charvar_from_netcdf(nc_file, nc_var_name): - """Searches NetCDF file and extract a scalar variable - - Args: - nc_file: Path to netCDF file - nc_var_name: name of the scalar variable - Returns: - value of the variable - """ - - SED = os.getenv("SED") - - cmd = f"ncdump -v {nc_var_name} {nc_file} | \ - {SED} -r -e '1,/data:/d' \ - -e '/^[ ]*'{nc_var_name}'/d' \ - -e '/^}}$/d' \ - -e 's/.*\"(.*)\".*/\\1/' \ - -e '/^$/d' \ - " - (ret, nc_var_value, _) = run_command(cmd) - - if ret != 0: - print_err_msg_exit( - f''' - Attempt to extract the value of the NetCDF variable spcecified by nc_var_name - from the file specified by nc_file failed: - nc_file = \"{nc_file}\" - nc_var_name = \"{nc_var_name}\"''' - ) - - if nc_var_value is None: - print_err_msg_exit( - f''' - In the specified NetCDF file (nc_file), the specified variable (nc_var_name) - was not found: - nc_file = \"{nc_file}\" - nc_var_name = \"{nc_var_name}\" - nc_var_value = \"{nc_var_value}\"''' - ) - - return nc_var_value diff --git a/ush/python_utils/get_elem_inds.py b/ush/python_utils/get_elem_inds.py deleted file mode 100644 index 20ac2e8967..0000000000 --- a/ush/python_utils/get_elem_inds.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 - -from .misc import lowercase -from .check_var_valid_value import check_var_valid_value - - -def get_elem_inds(arr, match, ret_type): - """Function that returns indices of elements of array - that match a given string - - Args: - arr: the list - match: element to match (case insenensitive) - ret_type: the return type can be any of [ 'first', 'last', 'all' ] - Returns: - A list of indices - """ - ret_type = lowercase(ret_type) - check_var_valid_value(ret_type, ["first", "last", "all"]) - - if ret_type == "first": - return arr.index(match) - if ret_type == "last": - for i in range(len(arr) - 1, -1, -1): - if arr[i] == match: - return i - return None - return [i for i, e in enumerate(arr) if e == match] diff --git a/ush/python_utils/interpol_to_arbit_CRES.py b/ush/python_utils/interpol_to_arbit_CRES.py deleted file mode 100644 index d87e258000..0000000000 --- a/ush/python_utils/interpol_to_arbit_CRES.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 - - -def interpol_to_arbit_CRES(RES, RES_array, prop_array): - """Function to interpolate (or extrapolate) a grid cell size-dependent property - to an arbitrary cubed-sphere resolution using arrays that specify a set of property - values for a corresponding set of resolutions - - Args: - RES: The cubed-sphere resolution at which to find the value of a property. - This is in units of number of cells (in either of the two horizontal - directions) on any one of the tiles of a cubed-sphere grid. - - RES_array: The name of the array containing the cubed-sphere resolutions for - which corresponding property values are given (in prop_array). These - are assumed to be given from smallest to largest. - - prop_array: The name of the array containing the values of the property corres- - ponding to the cubed-sphere resolutions in RES_array. - Returns: - Interpolated (extrapolated) property value - """ - - num_valid_RESes = len(RES_array) - i_min = 0 - i_max = num_valid_RESes - 1 - - if RES <= RES_array[i_min]: - prop = prop_array[i_min] - elif RES > RES_array[i_max]: - prop = prop_array[i_max] - else: - for i in range(0, num_valid_RESes - 1): - if RES_array[i] < RES <= RES_array[i + 1]: - RES1 = RES_array[i] - RES2 = RES_array[i + 1] - prop1 = prop_array[i] - prop2 = prop_array[i + 1] - m_slope = (prop2 - prop1) / (RES2 - RES1) - y_intcpt = (RES2 * prop1 - RES1 * prop2) / (RES2 - RES1) - prop = m_slope * RES + y_intcpt - - return prop diff --git a/ush/python_utils/print_input_args.py b/ush/python_utils/print_input_args.py index 3edcef3428..6d2a6ec2f1 100644 --- a/ush/python_utils/print_input_args.py +++ b/ush/python_utils/print_input_args.py @@ -33,23 +33,23 @@ def print_input_args(valid_args): if num_valid_args == 0: msg = dedent( - f''' + f""" No arguments have been passed to function {function} in script {filename_base} located - \"{filename}\"''' + '{filename}'""" ) else: msg = dedent( f""" The arguments to function {function} in script {filename_base} located - \"{filename}\" + '{filename}' have been set as follows:\n\n""" ) for k, v in valid_arg_names.items(): - msg = msg + f' {k}="{v}"\n' + msg = msg + f" {k}='{v}'\n" print_info_msg(msg, verbose=DEBUG) return num_valid_args diff --git a/ush/python_utils/print_msg.py b/ush/python_utils/print_msg.py index 8db0f06b03..078d708ba9 100644 --- a/ush/python_utils/print_msg.py +++ b/ush/python_utils/print_msg.py @@ -41,6 +41,7 @@ def print_info_msg(info_msg, verbose=True): return True return False + def log_info(info_msg, verbose=True, dedent_=True): """Function to print information message using the logging module. This function should not be used if python logging has not been initialized. @@ -54,11 +55,10 @@ def log_info(info_msg, verbose=True, dedent_=True): """ # "sys._getframe().f_back.f_code.co_name" returns the name of the calling function - logger=getLogger(sys._getframe().f_back.f_code.co_name) + logger = getLogger(sys._getframe().f_back.f_code.co_name) if verbose: if dedent_: - logger.info(indent(dedent(info_msg), ' ')) + logger.info(indent(dedent(info_msg), " ")) else: logger.info(info_msg) - diff --git a/ush/python_utils/test_python_utils.py b/ush/python_utils/test_python_utils.py index 7e25fdaf5b..69b6daf1a2 100644 --- a/ush/python_utils/test_python_utils.py +++ b/ush/python_utils/test_python_utils.py @@ -49,11 +49,6 @@ def test_check_for_preexist_dir_file(self): def test_check_var_valid_value(self): self.assertTrue(check_var_valid_value("rice", ["egg", "spam", "rice"])) - def test_count_files(self): - (_, target_cnt, _) = run_command("ls -l *.py | wc -l") - cnt = count_files("py") - self.assertEqual(cnt, int(target_cnt)) - def test_filesys_cmds(self): dPATH = f"{self.PATH}/test_data/dir" mkdir_vrfy(dPATH) @@ -63,27 +58,9 @@ def test_filesys_cmds(self): cmd_vrfy(f"rm -rf {dPATH}") self.assertFalse(os.path.exists("tt.py")) - def test_get_charvar_from_netcdf(self): - FILE = f"{self.PATH}/test_data/sample.nc" - val = get_charvar_from_netcdf(FILE, "pressure") - self.assertTrue(val and (val.split()[0], "955.5,")) - def test_run_command(self): self.assertEqual(run_command("echo hello"), (0, "hello", "")) - def test_get_elem_inds(self): - arr = ["egg", "spam", "egg", "rice", "egg"] - self.assertEqual(get_elem_inds(arr, "egg", "first"), 0) - self.assertEqual(get_elem_inds(arr, "egg", "last"), 4) - self.assertEqual(get_elem_inds(arr, "egg", "all"), [0, 2, 4]) - - def test_interpol_to_arbit_CRES(self): - RES = 800 - RES_array = [5, 25, 40, 60, 80, 100, 400, 700, 1000, 1500, 2800, 3000] - prop_array = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.65, 0.7, 1.0, 1.1, 1.2, 1.3] - prop = interpol_to_arbit_CRES(RES, RES_array, prop_array) - self.assertAlmostEqual(prop, 0.8) - def test_create_symlink_to_file(self): TARGET = f"{self.PATH}/test_python_utils.py" SYMLINK = f"{self.PATH}/test_data/test_python_utils.py" diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 52f217bb85..2d7bcefa17 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -47,7 +47,7 @@ def clean_up_output_dir(expected_subdir, local_archive, output_path, source_path unavailable = {} # Check to make sure the files exist on disk for file_path in source_paths: - local_file_path = os.path.join( os.getcwd(), file_path.lstrip("/") ) + local_file_path = os.path.join(os.getcwd(), file_path.lstrip("/")) if not os.path.exists(local_file_path): logging.info(f"File does not exist: {local_file_path}") unavailable["hpss"] = source_paths @@ -493,7 +493,6 @@ def hpss_requested_files(cla, file_names, store_specs, members=-1, ens_group=-1) if isinstance(archive_internal_dirs, dict): archive_internal_dirs = archive_internal_dirs.get(cla.anl_or_fcst, [""]) - # which_archive matters for choosing the correct file names within, # but we can safely just try all options for the # archive_internal_dir diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index e285fc92ed..098aadc37f 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -101,10 +101,10 @@ def set_FV3nml_ens_stoch_seeds(cdate): print_info_msg( dedent( f""" - The variable \"settings\" specifying seeds in \"{FV3_NML_FP}\" - has been set as follows: + The variable 'settings' specifying seeds in '{FV3_NML_FP}' + has been set as follows: - settings =\n\n""" + settings =\n\n""" ) + settings_str, verbose=VERBOSE, @@ -118,15 +118,15 @@ def set_FV3nml_ens_stoch_seeds(cdate): print_err_msg_exit( dedent( f""" - Call to python script set_namelist.py to set the variables in the FV3 - namelist file that specify the paths to the surface climatology files - failed. Parameters passed to this script are: - Full path to base namelist file: - FV3_NML_FP = \"{FV3_NML_FP}\" - Full path to output namelist file: - fv3_nml_ensmem_fp = \"{fv3_nml_ensmem_fp}\" - Namelist settings specified on command line (these have highest precedence):\n - settings =\n\n""" + Call to python script set_namelist.py to set the variables in the FV3 + namelist file that specify the paths to the surface climatology files + failed. Parameters passed to this script are: + Full path to base namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Full path to output namelist file: + fv3_nml_ensmem_fp = '{fv3_nml_ensmem_fp}' + Namelist settings specified on command line (these have highest precedence):\n + settings =\n\n""" ) + settings_str ) @@ -181,11 +181,13 @@ def setUp(self): "-p", os.path.join( EXPTDIR, - f'{date_to_str(self.cdate,format="%Y%m%d%H")}{os.sep}mem{i+1}', + f"{date_to_str(self.cdate,format='%Y%m%d%H')}{os.sep}mem{i+1}", ), ) - cd_vrfy(f'{EXPTDIR}{os.sep}{date_to_str(self.cdate,format="%Y%m%d%H")}{os.sep}mem2') + cd_vrfy( + f"{EXPTDIR}{os.sep}{date_to_str(self.cdate,format='%Y%m%d%H')}{os.sep}mem2" + ) set_env_var("USHdir", USHdir) set_env_var("ENSMEM_INDX", 2) diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index e94ad5b183..4cad4dfff9 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -17,6 +17,7 @@ rm_vrfy, import_vars, set_env_var, + load_config_file, load_shell_config, flatten_dict, define_macos_utilities, @@ -46,6 +47,11 @@ def set_FV3nml_sfc_climo_filenames(): # import all environment variables import_vars() + # fixed file mapping variables + fixed_cfg = load_config_file(os.path.join(PARMdir, "fixed_files_mapping.yaml")) + IMPORTS = ["SFC_CLIMO_FIELDS", "FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING"] + import_vars(dictionary=flatten_dict(fixed_cfg), env_vars=IMPORTS) + # The regular expression regex_search set below will be used to extract # from the elements of the array FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING # the name of the namelist variable to set and the corresponding surface @@ -82,9 +88,9 @@ def set_FV3nml_sfc_climo_filenames(): print_info_msg( dedent( f""" - The variable \"settings\" specifying values of the namelist variables - has been set as follows:\n - settings =\n\n""" + The variable 'settings' specifying values of the namelist variables + has been set as follows:\n + settings =\n\n""" ) + settings_str, verbose=VERBOSE, @@ -102,15 +108,15 @@ def set_FV3nml_sfc_climo_filenames(): print_err_msg_exit( dedent( f""" - Call to python script set_namelist.py to set the variables in the FV3 - namelist file that specify the paths to the surface climatology files - failed. Parameters passed to this script are: - Full path to base namelist file: - fv3_nml_base_fp = \"{fv3_nml_base_fp}\" - Full path to output namelist file: - FV3_NML_FP = \"{FV3_NML_FP}\" - Namelist settings specified on command line (these have highest precedence):\n - settings =\n\n""" + Call to python script set_namelist.py to set the variables in the FV3 + namelist file that specify the paths to the surface climatology files + failed. Parameters passed to this script are: + Full path to base namelist file: + fv3_nml_base_fp = '{fv3_nml_base_fp}' + Full path to output namelist file: + FV3_NML_FP = '{FV3_NML_FP}' + Namelist settings specified on command line (these have highest precedence):\n + settings =\n\n""" ) + settings_str ) @@ -159,38 +165,10 @@ def setUp(self): os.path.join(PARMdir, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), ) - set_env_var("USHdir", USHdir) + set_env_var("PARMdir", PARMdir) set_env_var("EXPTDIR", EXPTDIR) set_env_var("FIXlam", FIXlam) set_env_var("DO_ENSEMBLE", False) set_env_var("CRES", "C3357") set_env_var("RUN_ENVIR", "nco") set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) - - FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING = [ - "FNALBC | snowfree_albedo", - "FNALBC2 | facsf", - "FNTG3C | substrate_temperature", - "FNVEGC | vegetation_greenness", - "FNVETC | vegetation_type", - "FNSOTC | soil_type", - "FNVMNC | vegetation_greenness", - "FNVMXC | vegetation_greenness", - "FNSLPC | slope_type", - "FNABSC | maximum_snow_albedo", - ] - SFC_CLIMO_FIELDS = [ - "facsf", - "maximum_snow_albedo", - "slope_type", - "snowfree_albedo", - "soil_type", - "substrate_temperature", - "vegetation_greenness", - "vegetation_type", - ] - set_env_var( - "FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING", - FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING, - ) - set_env_var("SFC_CLIMO_FIELDS", SFC_CLIMO_FIELDS) diff --git a/ush/set_cycle_dates.py b/ush/set_cycle_dates.py index 09adf55e88..069a3891b1 100644 --- a/ush/set_cycle_dates.py +++ b/ush/set_cycle_dates.py @@ -43,6 +43,13 @@ def test_set_cycle_dates(self): incr_cycl_freq=6, ) self.assertEqual( - cdates, ["2022010106", "2022010112", "2022010118", - "2022010200", "2022010206", "2022010212"] + cdates, + [ + "2022010106", + "2022010112", + "2022010118", + "2022010200", + "2022010206", + "2022010212", + ], ) diff --git a/ush/set_extrn_mdl_params.py b/ush/set_extrn_mdl_params.py deleted file mode 100644 index 7d52055031..0000000000 --- a/ush/set_extrn_mdl_params.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 - -import unittest - -from python_utils import import_vars, export_vars, set_env_var, get_env_var - - -def set_extrn_mdl_params(): - """Sets parameters associated with the external model used for initial - conditions (ICs) and lateral boundary conditions (LBCs). - Args: - None - Returns: - None - """ - - # import all env variables - import_vars() - - global EXTRN_MDL_LBCS_OFFSET_HRS - - # - # ----------------------------------------------------------------------- - # - # Set EXTRN_MDL_LBCS_OFFSET_HRS, which is the number of hours to shift - # the starting time of the external model that provides lateral boundary - # conditions. - # - # ----------------------------------------------------------------------- - # - if EXTRN_MDL_NAME_LBCS == "RAP": - EXTRN_MDL_LBCS_OFFSET_HRS = EXTRN_MDL_LBCS_OFFSET_HRS or "3" - else: - EXTRN_MDL_LBCS_OFFSET_HRS = EXTRN_MDL_LBCS_OFFSET_HRS or "0" - - # export values we set above - env_vars = ["EXTRN_MDL_LBCS_OFFSET_HRS"] - export_vars(env_vars=env_vars) - - -class Testing(unittest.TestCase): - def test_extrn_mdl_params(self): - set_extrn_mdl_params() - EXTRN_MDL_LBCS_OFFSET_HRS = get_env_var("EXTRN_MDL_LBCS_OFFSET_HRS") - self.assertEqual(EXTRN_MDL_LBCS_OFFSET_HRS, 3) - - def setUp(self): - set_env_var("EXTRN_MDL_NAME_LBCS", "RAP") - set_env_var("EXTRN_MDL_LBCS_OFFSET_HRS", None) diff --git a/ush/set_gridparams_ESGgrid.py b/ush/set_gridparams_ESGgrid.py index ce8cd02ffe..dc2269b2ce 100644 --- a/ush/set_gridparams_ESGgrid.py +++ b/ush/set_gridparams_ESGgrid.py @@ -5,11 +5,11 @@ from datetime import datetime, timedelta from python_utils import ( - import_vars, - set_env_var, - print_input_args, - load_config_file, - flatten_dict, + import_vars, + set_env_var, + print_input_args, + load_config_file, + flatten_dict, ) @@ -35,7 +35,7 @@ def set_gridparams_ESGgrid(lon_ctr, lat_ctr, nx, ny, halo_width, delx, dely, paz # get constants IMPORTS = ["RADIUS_EARTH", "DEGS_PER_RADIAN"] USHdir = os.path.dirname(os.path.abspath(__file__)) - constants_cfg = load_config_file(os.path.join(USHdir,"constants.yaml")) + constants_cfg = load_config_file(os.path.join(USHdir, "constants.yaml")) import_vars(dictionary=flatten_dict(constants_cfg), env_vars=IMPORTS) # diff --git a/ush/set_gridparams_GFDLgrid.py b/ush/set_gridparams_GFDLgrid.py index 47253a1c73..70a309c3fd 100644 --- a/ush/set_gridparams_GFDLgrid.py +++ b/ush/set_gridparams_GFDLgrid.py @@ -63,7 +63,7 @@ def set_gridparams_GFDLgrid( import_vars(env_vars=IMPORTS) IMPORTS = ["NH4"] USHdir = os.path.dirname(os.path.abspath(__file__)) - constants_cfg = load_config_file(os.path.join(USHdir,"constants.yaml")) + constants_cfg = load_config_file(os.path.join(USHdir, "constants.yaml")) import_vars(dictionary=flatten_dict(constants_cfg), env_vars=IMPORTS) # diff --git a/ush/set_ozone_param.py b/ush/set_ozone_param.py index 5ed4449fe1..28f6e1ef68 100644 --- a/ush/set_ozone_param.py +++ b/ush/set_ozone_param.py @@ -17,7 +17,13 @@ find_pattern_in_str, ) -def set_ozone_param(ccpp_phys_suite_fp): + +def set_ozone_param( + ccpp_phys_suite_fp, + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, + VERBOSE, +): """Function that does the following: (1) Determines the ozone parameterization being used by checking in the CCPP physics suite XML. @@ -41,15 +47,14 @@ def set_ozone_param(ccpp_phys_suite_fp): Args: ccpp_phys_suite_fp: full path to CCPP physics suite + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING + FIXgsm_FILES_TO_COPY_TO_FIXam Returns: ozone_param: a string """ print_input_args(locals()) - # import all environment variables - import_vars() - # # ----------------------------------------------------------------------- # @@ -90,8 +95,10 @@ def set_ozone_param(ccpp_phys_suite_fp): fixgsm_ozone_fn = "global_o3prdlos.f77" ozone_param = "ozphys" else: - raise KeyError(f'Unknown or no ozone parameterization specified in the ' - 'CCPP physics suite file "{ccpp_phys_suite_fp}"') + raise KeyError( + f"Unknown or no ozone parameterization specified in the " + "CCPP physics suite file '{ccpp_phys_suite_fp}'" + ) # # ----------------------------------------------------------------------- # @@ -151,24 +158,27 @@ def set_ozone_param(ccpp_phys_suite_fp): CCPP suite definition file), the array specifying the mapping between the symlinks that need to be created in the cycle directories and the files in the FIXam directory is: - """, verbose=VERBOSE) - log_info(f""" + """, + verbose=VERBOSE, + ) + log_info( + f""" CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = {list_to_str(CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING)} - """, verbose=VERBOSE, dedent_=False) + """, + verbose=VERBOSE, + dedent_=False, + ) else: raise Exception( - f''' + f""" Unable to set name of the ozone production/loss file in the FIXgsm directory in the array that specifies the mapping between the symlinks that need to be created in the cycle directories and the files in the FIXgsm directory: - fixgsm_ozone_fn_is_set = \"{fixgsm_ozone_fn_is_set}\"''' + fixgsm_ozone_fn_is_set = '{fixgsm_ozone_fn_is_set}'""" ) - EXPORTS = ["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", "FIXgsm_FILES_TO_COPY_TO_FIXam"] - export_vars(env_vars=EXPORTS) - return ozone_param @@ -178,15 +188,15 @@ def test_set_ozone_param(self): self.assertEqual( "ozphys_2015", set_ozone_param( - ccpp_phys_suite_fp=f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml" + f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml", + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, + VERBOSE=True, ), ) def setUp(self): - define_macos_utilities() - set_env_var("DEBUG", True) - set_env_var("VERBOSE", True) - + global CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = [ "aerosol.dat | global_climaeropac_global.txt", "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", @@ -209,6 +219,7 @@ def setUp(self): "solarconstant_noaa_an.txt | global_solarconstant_noaa_an.txt", "global_o3prdlos.f77 | ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77", ] + global FIXgsm_FILES_TO_COPY_TO_FIXam FIXgsm_FILES_TO_COPY_TO_FIXam = [ "global_glacier.2x2.grb", "global_maxice.2x2.grb", @@ -241,9 +252,3 @@ def setUp(self): "HGT.Beljaars_filtered.lat-lon.30s_res.nc", "ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77", ] - - set_env_var( - "CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, - ) - set_env_var("FIXgsm_FILES_TO_COPY_TO_FIXam", FIXgsm_FILES_TO_COPY_TO_FIXam) diff --git a/ush/set_predef_grid_params.py b/ush/set_predef_grid_params.py index 6b432b8f03..faf8c75c58 100644 --- a/ush/set_predef_grid_params.py +++ b/ush/set_predef_grid_params.py @@ -41,12 +41,13 @@ def set_predef_grid_params(): try: params_dict = params_dict[PREDEF_GRID_NAME] except KeyError: - errmsg = dedent(f''' - PREDEF_GRID_NAME = {PREDEF_GRID_NAME} not found in predef_grid_params.yaml - Check your config file settings.''') + errmsg = dedent( + f""" + PREDEF_GRID_NAME = {PREDEF_GRID_NAME} not found in predef_grid_params.yaml + Check your config file settings.""" + ) raise Exception(errmsg) from None - # if QUILTING = False, remove key if not QUILTING: params_dict.pop("QUILTING") diff --git a/ush/set_thompson_mp_fix_files.py b/ush/set_thompson_mp_fix_files.py index 93dc3c5de6..df69c2b232 100644 --- a/ush/set_thompson_mp_fix_files.py +++ b/ush/set_thompson_mp_fix_files.py @@ -17,7 +17,12 @@ ) -def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn): +def set_thompson_mp_fix_files( + ccpp_phys_suite_fp, + thompson_mp_climo_fn, + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, +): """Function that first checks whether the Thompson microphysics parameterization is being called by the selected physics suite. If not, it sets the output variable whose name is specified by @@ -31,14 +36,17 @@ def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn): Args: ccpp_phys_suite_fp: full path to CCPP physics suite thompson_mp_climo_fn: netcdf file for thompson microphysics + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING + FIXgsm_FILES_TO_COPY_TO_FIXam Returns: boolean: sdf_uses_thompson_mp """ print_input_args(locals()) - # import all environment variables - import_vars() + # import some environment variables + IMPORTS = ["EXTRN_MDL_NAME_ICS", "EXTRN_MDL_NAME_LBCS", "CCPP_PHYS_SUITE"] + import_vars(env_vars=IMPORTS) # # ----------------------------------------------------------------------- @@ -99,7 +107,7 @@ def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn): CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING. After these modifications, the values of these parameters are as follows: - CCPP_PHYS_SUITE = \"{CCPP_PHYS_SUITE}\" + CCPP_PHYS_SUITE = '{CCPP_PHYS_SUITE}' """ ) log_info( @@ -113,12 +121,6 @@ def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn): """ ) - EXPORTS = [ - "CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", - "FIXgsm_FILES_TO_COPY_TO_FIXam", - ] - export_vars(env_vars=EXPORTS) - return sdf_uses_thompson_mp @@ -128,19 +130,19 @@ def test_set_thompson_mp_fix_files(self): self.assertEqual( True, set_thompson_mp_fix_files( - ccpp_phys_suite_fp=f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml", - thompson_mp_climo_fn="Thompson_MP_MONTHLY_CLIMO.nc", + f"{USHdir}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml", + "Thompson_MP_MONTHLY_CLIMO.nc", + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, ), ) def setUp(self): - define_macos_utilities() - set_env_var("DEBUG", True) - set_env_var("VERBOSE", True) set_env_var("EXTRN_MDL_NAME_ICS", "FV3GFS") set_env_var("EXTRN_MDL_NAME_LBCS", "FV3GFS") set_env_var("CCPP_PHYS_SUITE", "FV3_GSD_SAR") + global CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = [ "aerosol.dat | global_climaeropac_global.txt", "co2historicaldata_2010.txt | fix_co2_proj/global_co2historicaldata_2010.txt", @@ -164,6 +166,7 @@ def setUp(self): "global_o3prdlos.f77 | ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77", ] + global FIXgsm_FILES_TO_COPY_TO_FIXam FIXgsm_FILES_TO_COPY_TO_FIXam = [ "global_glacier.2x2.grb", "global_maxice.2x2.grb", @@ -196,9 +199,3 @@ def setUp(self): "HGT.Beljaars_filtered.lat-lon.30s_res.nc", "ozprdlos_2015_new_sbuvO3_tclm15_nuchem.f77", ] - - set_env_var( - "CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", - CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, - ) - set_env_var("FIXgsm_FILES_TO_COPY_TO_FIXam", FIXgsm_FILES_TO_COPY_TO_FIXam) diff --git a/ush/setup.py b/ush/setup.py index 3fa7313727..1bdf2453a8 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -32,7 +32,6 @@ from set_cycle_dates import set_cycle_dates from set_predef_grid_params import set_predef_grid_params from set_ozone_param import set_ozone_param -from set_extrn_mdl_params import set_extrn_mdl_params from set_gridparams_ESGgrid import set_gridparams_ESGgrid from set_gridparams_GFDLgrid import set_gridparams_GFDLgrid from link_fix import link_fix @@ -69,7 +68,7 @@ def setup(): log_info( f""" ======================================================================== - Starting function setup() in \"{os.path.basename(__file__)}\"... + Starting function setup() in '{os.path.basename(__file__)}'... ========================================================================""" ) # @@ -83,42 +82,64 @@ def setup(): # EXPT_DEFAULT_CONFIG_FN = "config_defaults.yaml" cfg_d = load_config_file(os.path.join(USHdir, EXPT_DEFAULT_CONFIG_FN)) - import_vars(dictionary=flatten_dict(cfg_d), - env_vars=["EXPT_CONFIG_FN", - "EXTRN_MDL_NAME_ICS", "EXTRN_MDL_NAME_LBCS", - "FV3GFS_FILE_FMT_ICS", "FV3GFS_FILE_FMT_LBCS"]) - + import_vars( + dictionary=flatten_dict(cfg_d), + env_vars=[ + "EXPT_CONFIG_FN", + "EXTRN_MDL_NAME_ICS", + "EXTRN_MDL_NAME_LBCS", + "FV3GFS_FILE_FMT_ICS", + "FV3GFS_FILE_FMT_LBCS", + ], + ) # Load the user config file, then ensure all user-specified - # variables correspond to a default value. + # variables correspond to a default value. if not os.path.exists(EXPT_CONFIG_FN): - raise FileNotFoundError(f'User config file not found: EXPT_CONFIG_FN = {EXPT_CONFIG_FN}') + raise FileNotFoundError( + f"User config file not found: EXPT_CONFIG_FN = {EXPT_CONFIG_FN}" + ) try: cfg_u = load_config_file(os.path.join(USHdir, EXPT_CONFIG_FN)) except: - errmsg = dedent(f'''\n - Could not load YAML config file: {EXPT_CONFIG_FN} - Reference the above traceback for more information. - ''') + errmsg = dedent( + f"""\n + Could not load YAML config file: {EXPT_CONFIG_FN} + Reference the above traceback for more information. + """ + ) raise Exception(errmsg) cfg_u = flatten_dict(cfg_u) for key in cfg_u: if key not in flatten_dict(cfg_d): - raise Exception(dedent(f''' - User-specified variable "{key}" in {EXPT_CONFIG_FN} is not valid. - Check {EXPT_DEFAULT_CONFIG_FN} for allowed user-specified variables.\n''')) + raise Exception( + dedent( + f""" + User-specified variable "{key}" in {EXPT_CONFIG_FN} is not valid. + Check {EXPT_DEFAULT_CONFIG_FN} for allowed user-specified variables.\n""" + ) + ) # Mandatory variables *must* be set in the user's config; the default value is invalid - mandatory = ['MACHINE'] + mandatory = ["MACHINE"] for val in mandatory: if val not in cfg_u: - raise Exception(f'Mandatory variable "{val}" not found in user config file {EXPT_CONFIG_FN}') + raise Exception( + f"Mandatory variable '{val}' not found in user config file {EXPT_CONFIG_FN}" + ) - import_vars(dictionary=cfg_u, env_vars=["MACHINE", - "EXTRN_MDL_NAME_ICS", "EXTRN_MDL_NAME_LBCS", - "FV3GFS_FILE_FMT_ICS", "FV3GFS_FILE_FMT_LBCS"]) + import_vars( + dictionary=cfg_u, + env_vars=[ + "MACHINE", + "EXTRN_MDL_NAME_ICS", + "EXTRN_MDL_NAME_LBCS", + "FV3GFS_FILE_FMT_ICS", + "FV3GFS_FILE_FMT_LBCS", + ], + ) # # ----------------------------------------------------------------------- # @@ -131,23 +152,25 @@ def setup(): global MACHINE, EXTRN_MDL_SYSBASEDIR_ICS, EXTRN_MDL_SYSBASEDIR_LBCS MACHINE_FILE = os.path.join(USHdir, "machine", f"{lowercase(MACHINE)}.yaml") if not os.path.exists(MACHINE_FILE): - raise FileNotFoundError(dedent( - f""" - The machine file {MACHINE_FILE} does not exist. - Check that you have specified the correct machine ({MACHINE}) in your config file {EXPT_CONFIG_FN}""" - )) + raise FileNotFoundError( + dedent( + f""" + The machine file {MACHINE_FILE} does not exist. + Check that you have specified the correct machine ({MACHINE}) in your config file {EXPT_CONFIG_FN}""" + ) + ) machine_cfg = load_config_file(MACHINE_FILE) # ics and lbcs - def get_location(xcs,fmt): - if ("data" in machine_cfg) and (xcs in machine_cfg["data"]): - v = machine_cfg["data"][xcs] - if not isinstance(v,dict): - return v - else: - return v[fmt] - else: - return "" + def get_location(xcs, fmt): + if ("data" in machine_cfg) and (xcs in machine_cfg["data"]): + v = machine_cfg["data"][xcs] + if not isinstance(v, dict): + return v + else: + return v[fmt] + else: + return "" EXTRN_MDL_SYSBASEDIR_ICS = get_location(EXTRN_MDL_NAME_ICS, FV3GFS_FILE_FMT_ICS) EXTRN_MDL_SYSBASEDIR_LBCS = get_location(EXTRN_MDL_NAME_LBCS, FV3GFS_FILE_FMT_LBCS) @@ -155,10 +178,12 @@ def get_location(xcs,fmt): # remove the data key and provide machine specific default values for cfg_d if "data" in machine_cfg: machine_cfg.pop("data") - machine_cfg.update({ - "EXTRN_MDL_SYSBASEDIR_ICS": EXTRN_MDL_SYSBASEDIR_ICS, - "EXTRN_MDL_SYSBASEDIR_LBCS": EXTRN_MDL_SYSBASEDIR_LBCS, - }) + machine_cfg.update( + { + "EXTRN_MDL_SYSBASEDIR_ICS": EXTRN_MDL_SYSBASEDIR_ICS, + "EXTRN_MDL_SYSBASEDIR_LBCS": EXTRN_MDL_SYSBASEDIR_LBCS, + } + ) machine_cfg = flatten_dict(machine_cfg) update_dict(machine_cfg, cfg_d) @@ -180,6 +205,12 @@ def get_location(xcs,fmt): # make machine name uppercase MACHINE = uppercase(MACHINE) + # Load fixed-files mapping file + cfg_f = load_config_file( + os.path.join(USHdir, os.pardir, "parm", "fixed_files_mapping.yaml") + ) + import_vars(dictionary=flatten_dict(cfg_f)) + # Load constants file and save its contents to a variable for later cfg_c = load_config_file(os.path.join(USHdir, CONSTANTS_FN)) import_vars(dictionary=flatten_dict(cfg_c)) @@ -194,6 +225,7 @@ def get_location(xcs,fmt): # global WORKFLOW_ID WORKFLOW_ID = "id_" + str(int(datetime.datetime.now().timestamp())) + cfg_d["workflow"]["WORKFLOW_ID"] = WORKFLOW_ID log_info(f"""WORKFLOW ID = {WORKFLOW_ID}""") # @@ -223,7 +255,7 @@ def get_location(xcs,fmt): if DEBUG and not VERBOSE: log_info( """ - Resetting VERBOSE to \"TRUE\" because DEBUG has been set to \"TRUE\"...""" + Resetting VERBOSE to 'TRUE' because DEBUG has been set to 'TRUE'...""" ) VERBOSE = True @@ -303,7 +335,7 @@ def get_location(xcs,fmt): or (len(ISEED_SPP) != N_VAR_SPP) ): raise Exception( - f''' + f""" All MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or RRTMG SPP-related namelist variables set in {EXPT_CONFIG_FN} must be equal in number of entries to what is found in SPP_VAR_LIST: @@ -315,7 +347,7 @@ def get_location(xcs,fmt): SPP_SIGTOP2 (length {len(SPP_SIGTOP2)}) SPP_STDDEV_CUTOFF (length {len(SPP_STDDEV_CUTOFF)}) ISEED_SPP (length {len(ISEED_SPP)}) - ''' + """ ) # # ----------------------------------------------------------------------- @@ -333,7 +365,7 @@ def get_location(xcs,fmt): or (len(LSM_SPP_TSCALE) != N_VAR_LNDP) ): raise Exception( - f''' + f""" All Noah or RUC-LSM SPP-related namelist variables (except ISEED_LSM_SPP) set in {EXPT_CONFIG_FN} must be equal in number of entries to what is found in SPP_VAR_LIST: @@ -341,16 +373,14 @@ def get_location(xcs,fmt): LSM_SPP_MAG_LIST (length {len(LSM_SPP_MAG_LIST)}) LSM_SPP_LSCALE (length {len(LSM_SPP_LSCALE)}) LSM_SPP_TSCALE (length {len(LSM_SPP_TSCALE)}) - ''' + """ ) # # The current script should be located in the ush subdirectory of the # workflow directory. Thus, the workflow directory is the one above the # directory of the current script. # - HOMEdir = os.path.abspath( - os.path.dirname(__file__) + os.sep + os.pardir - ) + HOMEdir = os.path.abspath(os.path.dirname(__file__) + os.sep + os.pardir) # # ----------------------------------------------------------------------- @@ -381,22 +411,25 @@ def get_location(xcs,fmt): try: UFS_WTHR_MDL_DIR = get_ini_value(cfg, external_name, property_name) except KeyError: - errmsg = dedent(f''' - Externals configuration file {mng_extrns_cfg_fn} - does not contain "{external_name}".''') + errmsg = dedent( + f""" + Externals configuration file {mng_extrns_cfg_fn} + does not contain '{external_name}'.""" + ) raise Exception(errmsg) from None - UFS_WTHR_MDL_DIR = os.path.join(HOMEdir, UFS_WTHR_MDL_DIR) if not os.path.exists(UFS_WTHR_MDL_DIR): - raise FileNotFoundError(dedent( - f""" - The base directory in which the FV3 source code should be located - (UFS_WTHR_MDL_DIR) does not exist: - UFS_WTHR_MDL_DIR = \"{UFS_WTHR_MDL_DIR}\" - Please clone the external repository containing the code in this directory, - build the executable, and then rerun the workflow.""" - )) + raise FileNotFoundError( + dedent( + f""" + The base directory in which the FV3 source code should be located + (UFS_WTHR_MDL_DIR) does not exist: + UFS_WTHR_MDL_DIR = '{UFS_WTHR_MDL_DIR}' + Please clone the external repository containing the code in this directory, + build the executable, and then rerun the workflow.""" + ) + ) # # Define some other useful paths # @@ -428,17 +461,28 @@ def get_location(xcs,fmt): RELATIVE_LINK_FLAG = "--relative" # Mandatory variables *must* be set in the user's config or the machine file; the default value is invalid - mandatory = ['NCORES_PER_NODE', 'FIXgsm', 'FIXaer', 'FIXlut', 'TOPO_DIR', 'SFC_CLIMO_INPUT_DIR'] + mandatory = [ + "NCORES_PER_NODE", + "FIXgsm", + "FIXaer", + "FIXlut", + "TOPO_DIR", + "SFC_CLIMO_INPUT_DIR", + ] globalvars = globals() for val in mandatory: # globals() returns dictionary of global variables if not globalvars[val]: - raise Exception(dedent(f''' - Mandatory variable "{val}" not found in: - user config file {EXPT_CONFIG_FN} - OR - machine file {MACHINE_FILE} - ''')) + raise Exception( + dedent( + f""" + Mandatory variable '{val}' not found in: + user config file {EXPT_CONFIG_FN} + OR + machine file {MACHINE_FILE} + """ + ) + ) # # ----------------------------------------------------------------------- @@ -478,10 +522,13 @@ def get_location(xcs,fmt): # if WORKFLOW_MANAGER is not None: if not ACCOUNT: - raise Exception(dedent(f''' - ACCOUNT must be specified in config or machine file if using a workflow manager. - WORKFLOW_MANAGER = {WORKFLOW_MANAGER}\n''' - )) + raise Exception( + dedent( + f""" + ACCOUNT must be specified in config or machine file if using a workflow manager. + WORKFLOW_MANAGER = {WORKFLOW_MANAGER}\n""" + ) + ) # # ----------------------------------------------------------------------- # @@ -516,57 +563,67 @@ def get_location(xcs,fmt): CPL = True else: raise Exception( - f''' + f""" The coupling flag CPL has not been specified for this value of FCST_MODEL: - FCST_MODEL = \"{FCST_MODEL}\"''' + FCST_MODEL = '{FCST_MODEL}'""" ) # Make sure RESTART_INTERVAL is set to an integer value if not isinstance(RESTART_INTERVAL, int): - raise Exception(f"\nRESTART_INTERVAL = {RESTART_INTERVAL}, must be an integer value\n") + raise Exception( + f"\nRESTART_INTERVAL = {RESTART_INTERVAL}, must be an integer value\n" + ) # Check that input dates are in a date format # get dictionary of all variables allvars = dict(globals()) allvars.update(locals()) - dates = ['DATE_FIRST_CYCL', 'DATE_LAST_CYCL'] + dates = ["DATE_FIRST_CYCL", "DATE_LAST_CYCL"] for val in dates: if not isinstance(allvars[val], datetime.date): - raise Exception(dedent(f''' - Date variable {val}={allvars[val]} is not in a valid date format + raise Exception( + dedent( + f""" + Date variable {val}={allvars[val]} is not in a valid date format - For examples of valid formats, see the users guide. - ''')) + For examples of valid formats, see the users guide. + """ + ) + ) # If using a custom post configuration file, make sure that it exists. if USE_CUSTOM_POST_CONFIG_FILE: try: - #os.path.exists returns exception if passed an empty string or None, so use "try/except" as a 2-for-1 error catch + # os.path.exists returns exception if passed an empty string or None, so use "try/except" as a 2-for-1 error catch if not os.path.exists(CUSTOM_POST_CONFIG_FP): raise except: - raise FileNotFoundError(dedent( - f''' - USE_CUSTOM_POST_CONFIG_FILE has been set, but the custom post configuration file - CUSTOM_POST_CONFIG_FP = {CUSTOM_POST_CONFIG_FP} - could not be found.''' - )) from None + raise FileNotFoundError( + dedent( + f""" + USE_CUSTOM_POST_CONFIG_FILE has been set, but the custom post configuration file + CUSTOM_POST_CONFIG_FP = {CUSTOM_POST_CONFIG_FP} + could not be found.""" + ) + ) from None # If using external CRTM fix files to allow post-processing of synthetic # satellite products from the UPP, make sure the CRTM fix file directory exists. if USE_CRTM: try: - #os.path.exists returns exception if passed an empty string or None, so use "try/except" as a 2-for-1 error catch + # os.path.exists returns exception if passed an empty string or None, so use "try/except" as a 2-for-1 error catch if not os.path.exists(CRTM_DIR): raise except: - raise FileNotFoundError(dedent( - f''' - USE_CRTM has been set, but the external CRTM fix file directory: - CRTM_DIR = {CRTM_DIR} - could not be found.''' - )) from None + raise FileNotFoundError( + dedent( + f""" + USE_CRTM has been set, but the external CRTM fix file directory: + CRTM_DIR = {CRTM_DIR} + could not be found.""" + ) + ) from None # The forecast length (in integer hours) cannot contain more than 3 characters. # Thus, its maximum value is 999. @@ -582,7 +639,7 @@ def get_location(xcs,fmt): # ----------------------------------------------------------------------- # # Check whether the forecast length (FCST_LEN_HRS) is evenly divisible - # by the BC update interval (LBC_SPEC_INTVL_HRS). If so, generate an + # by the BC update interval (LBC_SPEC_INTVL_HRS). If so, generate an # array of forecast hours at which the boundary values will be updated. # # ----------------------------------------------------------------------- @@ -614,6 +671,7 @@ def get_location(xcs,fmt): LBC_SPEC_INTVL_HRS, LBC_SPEC_INTVL_HRS + FCST_LEN_HRS, LBC_SPEC_INTVL_HRS ) ] + cfg_d["task_make_lbcs"]["LBC_SPEC_FCST_HRS"] = LBC_SPEC_FCST_HRS # # ----------------------------------------------------------------------- # @@ -627,11 +685,7 @@ def get_location(xcs,fmt): # get dictionary of all variables allvars = dict(globals()) allvars.update(locals()) - vlist = ['DT_ATMOS', - 'LAYOUT_X', - 'LAYOUT_Y', - 'BLOCKSIZE', - 'EXPT_SUBDIR'] + vlist = ["DT_ATMOS", "LAYOUT_X", "LAYOUT_Y", "BLOCKSIZE", "EXPT_SUBDIR"] for val in vlist: if not allvars[val]: raise Exception(f"\nMandatory variable '{val}' has not been set\n") @@ -653,12 +707,12 @@ def get_location(xcs,fmt): # if DT_SUBHOURLY_POST_MNTS < 0 or DT_SUBHOURLY_POST_MNTS > 59: raise ValueError( - f''' - When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), + f""" + When performing sub-hourly post (i.e. SUB_HOURLY_POST set to 'TRUE'), DT_SUBHOURLY_POST_MNTS must be set to an integer between 0 and 59, inclusive but in this case is not: - SUB_HOURLY_POST = \"{SUB_HOURLY_POST}\" - DT_SUBHOURLY_POST_MNTS = \"{DT_SUBHOURLY_POST_MNTS}\"''' + SUB_HOURLY_POST = '{SUB_HOURLY_POST}' + DT_SUBHOURLY_POST_MNTS = '{DT_SUBHOURLY_POST_MNTS}'""" ) # # Check that DT_SUBHOURLY_POST_MNTS (after converting to seconds) is @@ -668,14 +722,14 @@ def get_location(xcs,fmt): if rem != 0: raise ValueError( f""" - When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), + When performing sub-hourly post (i.e. SUB_HOURLY_POST set to 'TRUE'), the time interval specified by DT_SUBHOURLY_POST_MNTS (after converting to seconds) must be evenly divisible by the time step DT_ATMOS used in the forecast model, i.e. the remainder (rem) must be zero. In this case, it is not: - SUB_HOURLY_POST = \"{SUB_HOURLY_POST}\" - DT_SUBHOURLY_POST_MNTS = \"{DT_SUBHOURLY_POST_MNTS}\" - DT_ATMOS = \"{DT_ATMOS}\" + SUB_HOURLY_POST = '{SUB_HOURLY_POST}' + DT_SUBHOURLY_POST_MNTS = '{DT_SUBHOURLY_POST_MNTS}' + DT_ATMOS = '{DT_ATMOS}' rem = (DT_SUBHOURLY_POST_MNTS*60) %% DT_ATMOS = {rem} Please reset DT_SUBHOURLY_POST_MNTS and/or DT_ATMOS so that this remainder is zero.""" @@ -689,12 +743,12 @@ def get_location(xcs,fmt): if DT_SUBHOURLY_POST_MNTS == 0: logger.warning( f""" - When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), + When performing sub-hourly post (i.e. SUB_HOURLY_POST set to 'TRUE'), DT_SUBHOURLY_POST_MNTS must be set to a value greater than 0; otherwise, sub-hourly output is not really being performed: - SUB_HOURLY_POST = \"{SUB_HOURLY_POST}\" - DT_SUBHOURLY_POST_MNTS = \"{DT_SUBHOURLY_POST_MNTS}\" - Resetting SUB_HOURLY_POST to \"FALSE\". If you do not want this, you + SUB_HOURLY_POST = '{SUB_HOURLY_POST}' + DT_SUBHOURLY_POST_MNTS = '{DT_SUBHOURLY_POST_MNTS}' + Resetting SUB_HOURLY_POST to 'FALSE'. If you do not want this, you must set DT_SUBHOURLY_POST_MNTS to something other than zero.""" ) SUB_HOURLY_POST = False @@ -725,7 +779,7 @@ def get_location(xcs,fmt): pass EXPT_BASEDIR = os.path.abspath(EXPT_BASEDIR) - mkdir_vrfy(f' -p "{EXPT_BASEDIR}"') + mkdir_vrfy(f" -p '{EXPT_BASEDIR}'") # # ----------------------------------------------------------------------- @@ -740,21 +794,25 @@ def get_location(xcs,fmt): try: check_for_preexist_dir_file(EXPTDIR, PREEXISTING_DIR_METHOD) except ValueError: - logger.exception(f''' - Check that the following values are valid: - EXPTDIR {EXPTDIR} - PREEXISTING_DIR_METHOD {PREEXISTING_DIR_METHOD} - ''') + logger.exception( + f""" + Check that the following values are valid: + EXPTDIR {EXPTDIR} + PREEXISTING_DIR_METHOD {PREEXISTING_DIR_METHOD} + """ + ) raise except FileExistsError: - errmsg = dedent(f''' - EXPTDIR ({EXPTDIR}) already exists, and PREEXISTING_DIR_METHOD = {PREEXISTING_DIR_METHOD} - - To ignore this error, delete the directory, or set - PREEXISTING_DIR_METHOD = delete, or - PREEXISTING_DIR_METHOD = rename - in your config file. - ''') + errmsg = dedent( + f""" + EXPTDIR ({EXPTDIR}) already exists, and PREEXISTING_DIR_METHOD = {PREEXISTING_DIR_METHOD} + + To ignore this error, delete the directory, or set + PREEXISTING_DIR_METHOD = delete, or + PREEXISTING_DIR_METHOD = rename + in your config file. + """ + ) raise FileExistsError(errmsg) from None # # ----------------------------------------------------------------------- @@ -796,22 +854,35 @@ def get_location(xcs,fmt): # Main directory locations if RUN_ENVIR == "nco": - try: OPSROOT = os.path.abspath(f"{EXPT_BASEDIR}{os.sep}..{os.sep}nco_dirs") \ - if OPSROOT is None else OPSROOT - except NameError: OPSROOT = EXPTDIR - try: COMROOT - except NameError: COMROOT = os.path.join(OPSROOT, "com") - try: PACKAGEROOT - except NameError: PACKAGEROOT = os.path.join(OPSROOT, "packages") - try: DATAROOT - except NameError: DATAROOT = os.path.join(OPSROOT, "tmp") - try: DCOMROOT - except NameError: DCOMROOT = os.path.join(OPSROOT, "dcom") + try: + OPSROOT = ( + os.path.abspath(f"{EXPT_BASEDIR}{os.sep}..{os.sep}nco_dirs") + if OPSROOT is None + else OPSROOT + ) + except NameError: + OPSROOT = EXPTDIR + try: + COMROOT + except NameError: + COMROOT = os.path.join(OPSROOT, "com") + try: + PACKAGEROOT + except NameError: + PACKAGEROOT = os.path.join(OPSROOT, "packages") + try: + DATAROOT + except NameError: + DATAROOT = os.path.join(OPSROOT, "tmp") + try: + DCOMROOT + except NameError: + DCOMROOT = os.path.join(OPSROOT, "dcom") COMIN_BASEDIR = os.path.join(COMROOT, NET, model_ver) COMOUT_BASEDIR = os.path.join(COMROOT, NET, model_ver) - LOGDIR = os.path.join(OPSROOT,"output") + LOGDIR = os.path.join(OPSROOT, "output") else: @@ -825,34 +896,52 @@ def get_location(xcs,fmt): LOGDIR = os.path.join(EXPTDIR, "log") - try: DBNROOT - except NameError: DBNROOT = None - try: SENDECF - except NameError: SENDECF = False - try: SENDDBN - except NameError: SENDDBN = False - try: SENDDBN_NTC - except NameError: SENDDBN_NTC = False - try: SENDCOM - except NameError: SENDCOM = False - try: SENDWEB - except NameError: SENDWEB = False - try: KEEPDATA - except NameError: KEEPDATA = True - try: MAILTO - except NameError: MAILTO = None - try: MAILCC - except NameError: MAILCC = None + try: + DBNROOT + except NameError: + DBNROOT = None + try: + SENDECF + except NameError: + SENDECF = False + try: + SENDDBN + except NameError: + SENDDBN = False + try: + SENDDBN_NTC + except NameError: + SENDDBN_NTC = False + try: + SENDCOM + except NameError: + SENDCOM = False + try: + SENDWEB + except NameError: + SENDWEB = False + try: + KEEPDATA + except NameError: + KEEPDATA = True + try: + MAILTO + except NameError: + MAILTO = None + try: + MAILCC + except NameError: + MAILCC = None # create NCO directories if RUN_ENVIR == "nco": - mkdir_vrfy(f' -p "{OPSROOT}"') - mkdir_vrfy(f' -p "{COMROOT}"') - mkdir_vrfy(f' -p "{PACKAGEROOT}"') - mkdir_vrfy(f' -p "{DATAROOT}"') - mkdir_vrfy(f' -p "{DCOMROOT}"') + mkdir_vrfy(f" -p '{OPSROOT}'") + mkdir_vrfy(f" -p '{COMROOT}'") + mkdir_vrfy(f" -p '{PACKAGEROOT}'") + mkdir_vrfy(f" -p '{DATAROOT}'") + mkdir_vrfy(f" -p '{DCOMROOT}'") if DBNROOT is not None: - mkdir_vrfy(f' -p "{DBNROOT}"') + mkdir_vrfy(f" -p '{DBNROOT}'") # # ----------------------------------------------------------------------- @@ -868,7 +957,7 @@ def get_location(xcs,fmt): POST_OUTPUT_DOMAIN_NAME = POST_OUTPUT_DOMAIN_NAME or PREDEF_GRID_NAME if type(POST_OUTPUT_DOMAIN_NAME) != int: - POST_OUTPUT_DOMAIN_NAME = lowercase(POST_OUTPUT_DOMAIN_NAME) + POST_OUTPUT_DOMAIN_NAME = lowercase(POST_OUTPUT_DOMAIN_NAME) if POST_OUTPUT_DOMAIN_NAME is None: if PREDEF_GRID_NAME is None: @@ -876,10 +965,10 @@ def get_location(xcs,fmt): f""" The domain name used in naming the run_post output files (POST_OUTPUT_DOMAIN_NAME) has not been set: - POST_OUTPUT_DOMAIN_NAME = \"{POST_OUTPUT_DOMAIN_NAME}\" + POST_OUTPUT_DOMAIN_NAME = '{POST_OUTPUT_DOMAIN_NAME}' If this experiment is not using a predefined grid (i.e. if PREDEF_GRID_NAME is set to a null string), POST_OUTPUT_DOMAIN_NAME - must be set in the configuration file (\"{EXPT_CONFIG_FN}\"). """ + must be set in the configuration file ('{EXPT_CONFIG_FN}'). """ ) # # ----------------------------------------------------------------------- @@ -981,10 +1070,10 @@ def get_location(xcs,fmt): CCPP_PHYS_SUITE_FP = os.path.join(EXPTDIR, CCPP_PHYS_SUITE_FN) if not os.path.exists(CCPP_PHYS_SUITE_IN_CCPP_FP): raise FileNotFoundError( - f''' + f""" The CCPP suite definition file (CCPP_PHYS_SUITE_IN_CCPP_FP) does not exist in the local clone of the ufs-weather-model: - CCPP_PHYS_SUITE_IN_CCPP_FP = \"{CCPP_PHYS_SUITE_IN_CCPP_FP}\"''' + CCPP_PHYS_SUITE_IN_CCPP_FP = '{CCPP_PHYS_SUITE_IN_CCPP_FP}'""" ) # # ----------------------------------------------------------------------- @@ -1008,10 +1097,10 @@ def get_location(xcs,fmt): FIELD_DICT_FP = os.path.join(EXPTDIR, FIELD_DICT_FN) if not os.path.exists(FIELD_DICT_IN_UWM_FP): raise FileNotFoundError( - f''' + f""" The field dictionary file (FIELD_DICT_IN_UWM_FP) does not exist in the local clone of the ufs-weather-model: - FIELD_DICT_IN_UWM_FP = \"{FIELD_DICT_IN_UWM_FP}\"''' + FIELD_DICT_IN_UWM_FP = '{FIELD_DICT_IN_UWM_FP}'""" ) # # ----------------------------------------------------------------------- @@ -1022,13 +1111,13 @@ def get_location(xcs,fmt): # ----------------------------------------------------------------------- # - # export env vars before calling another module - export_vars() - - OZONE_PARAM = set_ozone_param(CCPP_PHYS_SUITE_IN_CCPP_FP) + OZONE_PARAM = set_ozone_param( + CCPP_PHYS_SUITE_IN_CCPP_FP, + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, + VERBOSE=VERBOSE, + ) - IMPORTS = ["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", "FIXgsm_FILES_TO_COPY_TO_FIXam"] - import_vars(env_vars=IMPORTS) # # ----------------------------------------------------------------------- # @@ -1084,10 +1173,10 @@ def get_location(xcs,fmt): if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_ICS[:idx]): raise FileNotFoundError( - f''' + f""" The directory (EXTRN_MDL_SOURCE_BASEDIR_ICS) in which the user-staged external model files for generating ICs should be located does not exist: - EXTRN_MDL_SOURCE_BASEDIR_ICS = \"{EXTRN_MDL_SOURCE_BASEDIR_ICS}\"''' + EXTRN_MDL_SOURCE_BASEDIR_ICS = '{EXTRN_MDL_SOURCE_BASEDIR_ICS}'""" ) idx = EXTRN_MDL_SOURCE_BASEDIR_LBCS.find("$") @@ -1096,10 +1185,10 @@ def get_location(xcs,fmt): if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_LBCS[:idx]): raise FileNotFoundError( - f''' + f""" The directory (EXTRN_MDL_SOURCE_BASEDIR_LBCS) in which the user-staged external model files for generating LBCs should be located does not exist: - EXTRN_MDL_SOURCE_BASEDIR_LBCS = \"{EXTRN_MDL_SOURCE_BASEDIR_LBCS}\"''' + EXTRN_MDL_SOURCE_BASEDIR_LBCS = '{EXTRN_MDL_SOURCE_BASEDIR_LBCS}'""" ) # # ----------------------------------------------------------------------- @@ -1174,11 +1263,11 @@ def get_location(xcs,fmt): # Ensemble verification can only be run in ensemble mode if (not DO_ENSEMBLE) and (RUN_TASK_VX_ENSGRID or RUN_TASK_VX_ENSPOINT): raise Exception( - f''' + f""" Ensemble verification can not be run unless running in ensemble mode: - DO_ENSEMBLE = \"{DO_ENSEMBLE}\" - RUN_TASK_VX_ENSGRID = \"{RUN_TASK_VX_ENSGRID}\" - RUN_TASK_VX_ENSPOINT = \"{RUN_TASK_VX_ENSPOINT}\"''' + DO_ENSEMBLE = '{DO_ENSEMBLE}' + RUN_TASK_VX_ENSGRID = '{RUN_TASK_VX_ENSGRID}' + RUN_TASK_VX_ENSPOINT = '{RUN_TASK_VX_ENSPOINT}'""" ) # @@ -1219,21 +1308,23 @@ def get_location(xcs,fmt): # experiment directory (EXPTDIR). # if not RUN_TASK_MAKE_GRID: - if (GRID_DIR is None): + if GRID_DIR is None: GRID_DIR = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) - msg = dedent(f""" - GRID_DIR not specified! - Setting GRID_DIR = {GRID_DIR} - """) + msg = dedent( + f""" + GRID_DIR not specified! + Setting GRID_DIR = {GRID_DIR} + """ + ) logger.warning(msg) - if not os.path.exists(GRID_DIR): + if not os.path.exists(GRID_DIR): raise FileNotFoundError( - f''' + f""" The directory (GRID_DIR) that should contain the pregenerated grid files does not exist: - GRID_DIR = \"{GRID_DIR}\"''' + GRID_DIR = '{GRID_DIR}'""" ) else: GRID_DIR = os.path.join(EXPTDIR, "grid") @@ -1244,21 +1335,23 @@ def get_location(xcs,fmt): # the experiment directory (EXPTDIR). # if not RUN_TASK_MAKE_OROG: - if (OROG_DIR is None): + if OROG_DIR is None: OROG_DIR = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) - msg = dedent(f""" - OROG_DIR not specified! - Setting OROG_DIR = {OROG_DIR} - """) + msg = dedent( + f""" + OROG_DIR not specified! + Setting OROG_DIR = {OROG_DIR} + """ + ) logger.warning(msg) if not os.path.exists(OROG_DIR): raise FileNotFoundError( - f''' + f""" The directory (OROG_DIR) that should contain the pregenerated orography files does not exist: - OROG_DIR = \"{OROG_DIR}\"''' + OROG_DIR = '{OROG_DIR}'""" ) else: OROG_DIR = os.path.join(EXPTDIR, "orog") @@ -1269,40 +1362,42 @@ def get_location(xcs,fmt): # a predefined location under the experiment directory (EXPTDIR). # if not RUN_TASK_MAKE_SFC_CLIMO: - if (SFC_CLIMO_DIR is None): + if SFC_CLIMO_DIR is None: SFC_CLIMO_DIR = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) - msg = dedent(f""" - SFC_CLIMO_DIR not specified! - Setting SFC_CLIMO_DIR ={SFC_CLIMO_DIR} - """) + msg = dedent( + f""" + SFC_CLIMO_DIR not specified! + Setting SFC_CLIMO_DIR ={SFC_CLIMO_DIR} + """ + ) logger.warning(msg) if not os.path.exists(SFC_CLIMO_DIR): raise FileNotFoundError( - f''' + f""" The directory (SFC_CLIMO_DIR) that should contain the pregenerated surface climatology files does not exist: - SFC_CLIMO_DIR = \"{SFC_CLIMO_DIR}\"''' + SFC_CLIMO_DIR = '{SFC_CLIMO_DIR}'""" ) else: SFC_CLIMO_DIR = os.path.join(EXPTDIR, "sfc_climo") + # # ----------------------------------------------------------------------- # - # Set cycle-independent parameters associated with the external models - # from which we will obtain the ICs and LBCs. + # Set EXTRN_MDL_LBCS_OFFSET_HRS, which is the number of hours to shift + # the starting time of the external model that provides lateral boundary + # conditions. # # ----------------------------------------------------------------------- # + global EXTRN_MDL_LBCS_OFFSET_HRS + if EXTRN_MDL_NAME_LBCS == "RAP": + EXTRN_MDL_LBCS_OFFSET_HRS = EXTRN_MDL_LBCS_OFFSET_HRS or "3" + else: + EXTRN_MDL_LBCS_OFFSET_HRS = EXTRN_MDL_LBCS_OFFSET_HRS or "0" - # export env vars before calling another module - export_vars() - - set_extrn_mdl_params() - - IMPORTS = ["EXTRN_MDL_LBCS_OFFSET_HRS"] - import_vars(env_vars=IMPORTS) # # ----------------------------------------------------------------------- # @@ -1361,7 +1456,6 @@ def get_location(xcs,fmt): "STRETCH_FAC": STRETCH_FAC, } - # Extract the basic grid params from the dictionary (LON_CTR, LAT_CTR, NX, NY, NHW, STRETCH_FAC) = ( grid_params[k] for k in ["LON_CTR", "LAT_CTR", "NX", "NY", "NHW", "STRETCH_FAC"] @@ -1370,14 +1464,14 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- # - # Create a new experiment directory. For platforms with no workflow + # Create a new experiment directory. For platforms with no workflow # manager we need to create LOGDIR as well, since it won't be created # later at runtime. # # ----------------------------------------------------------------------- # - mkdir_vrfy(f' -p "{EXPTDIR}"') - mkdir_vrfy(f' -p "{LOGDIR}"') + mkdir_vrfy(f" -p '{EXPTDIR}'") + mkdir_vrfy(f" -p '{LOGDIR}'") # # ----------------------------------------------------------------------- # NOTE: currently this is executed no matter what, should it be dependent on the logic described below?? @@ -1390,7 +1484,7 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- # - mkdir_vrfy(f' -p "{FIXlam}"') + mkdir_vrfy(f" -p '{FIXlam}'") RES_IN_FIXLAM_FILENAMES = "" # # ----------------------------------------------------------------------- @@ -1481,10 +1575,14 @@ def get_location(xcs,fmt): if WRITE_DOPOST: # Turn off run_post if RUN_TASK_RUN_POST: - logger.warning(dedent(f""" - Inline post is turned on, deactivating post-processing tasks: - RUN_TASK_RUN_POST = False - """)) + logger.warning( + dedent( + f""" + Inline post is turned on, deactivating post-processing tasks: + RUN_TASK_RUN_POST = False + """ + ) + ) RUN_TASK_RUN_POST = False # Check if SUB_HOURLY_POST is on @@ -1509,12 +1607,12 @@ def get_location(xcs,fmt): if VERBOSE: log_info( - f""" - The number of MPI tasks for the forecast (including those for the write - component if it is being used) are: - PE_MEMBER01 = {PE_MEMBER01}""", - verbose=VERBOSE, - ) + f""" + The number of MPI tasks for the forecast (including those for the write + component if it is being used) are: + PE_MEMBER01 = {PE_MEMBER01}""", + verbose=VERBOSE, + ) # # ----------------------------------------------------------------------- # @@ -1579,48 +1677,19 @@ def get_location(xcs,fmt): # ----------------------------------------------------------------------- # SDF_USES_THOMPSON_MP = set_thompson_mp_fix_files( - ccpp_phys_suite_fp=CCPP_PHYS_SUITE_IN_CCPP_FP, - thompson_mp_climo_fn=THOMPSON_MP_CLIMO_FN, + CCPP_PHYS_SUITE_IN_CCPP_FP, + THOMPSON_MP_CLIMO_FN, + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING, + FIXgsm_FILES_TO_COPY_TO_FIXam, ) - IMPORTS = ["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", "FIXgsm_FILES_TO_COPY_TO_FIXam"] - import_vars(env_vars=IMPORTS) - - # - # ----------------------------------------------------------------------- - # - # Generate the shell script that will appear in the experiment directory - # (EXPTDIR) and will contain definitions of variables needed by the va- - # rious scripts in the workflow. We refer to this as the experiment/ - # workflow global variable definitions file. We will create this file - # by: - # - # 1) Copying the default workflow/experiment configuration file (speci- - # fied by EXPT_DEFAULT_CONFIG_FN and located in the shell script di- - # rectory specified by USHdir) to the experiment directory and rena- - # ming it to the name specified by GLOBAL_VAR_DEFNS_FN. - # - # 2) Resetting the default variable values in this file to their current - # values. This is necessary because these variables may have been - # reset by the user-specified configuration file (if one exists in - # USHdir) and/or by this setup script, e.g. because predef_domain is - # set to a valid non-empty value. - # - # 3) Appending to the variable definitions file any new variables intro- - # duced in this setup script that may be needed by the scripts that - # perform the various tasks in the workflow (and which source the va- - # riable defintions file). - # - # First, set the full path to the variable definitions file and copy the - # default configuration script into it. - # - # ----------------------------------------------------------------------- - # - # global variable definition file path global GLOBAL_VAR_DEFNS_FP GLOBAL_VAR_DEFNS_FP = os.path.join(EXPTDIR, GLOBAL_VAR_DEFNS_FN) + # fixed files section + cfg_d.update(cfg_f) + # update dictionary with globals() values update_dict(globals(), cfg_d) @@ -1646,7 +1715,7 @@ def get_location(xcs,fmt): # # Full path to workflow (re)launch script, its log file, and the line # that gets added to the cron table to launch this script if the flag - # USE_CRON_TO_RELAUNCH is set to \"TRUE\". + # USE_CRON_TO_RELAUNCH is set to 'TRUE'. # # ----------------------------------------------------------------------- # @@ -1767,7 +1836,7 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- # - # Flag in the \"{MODEL_CONFIG_FN}\" file for coupling the ocean model to + # Flag in the '{MODEL_CONFIG_FN}' file for coupling the ocean model to # the weather model. # # ----------------------------------------------------------------------- @@ -1856,11 +1925,11 @@ def get_location(xcs,fmt): f""" Generating the global experiment variable definitions file specified by GLOBAL_VAR_DEFNS_FN: - GLOBAL_VAR_DEFNS_FN = \"{GLOBAL_VAR_DEFNS_FN}\" + GLOBAL_VAR_DEFNS_FN = '{GLOBAL_VAR_DEFNS_FN}' Full path to this file is: - GLOBAL_VAR_DEFNS_FP = \"{GLOBAL_VAR_DEFNS_FP}\" - For more detailed information, set DEBUG to \"TRUE\" in the experiment - configuration file (\"{EXPT_CONFIG_FN}\").""" + GLOBAL_VAR_DEFNS_FP = '{GLOBAL_VAR_DEFNS_FP}' + For more detailed information, set DEBUG to 'TRUE' in the experiment + configuration file ('{EXPT_CONFIG_FN}').""" ) with open(GLOBAL_VAR_DEFNS_FP, "a") as f: diff --git a/ush/test_retrieve_data.py b/ush/test_retrieve_data.py index 3ffdf4744c..dc7a775f05 100644 --- a/ush/test_retrieve_data.py +++ b/ush/test_retrieve_data.py @@ -1,4 +1,4 @@ -''' +""" Functional test suite for gathering data using retreve_data.py. The tests reflect some use cases of gathering various model input data @@ -14,7 +14,7 @@ To run a single test: python -m unittest -b test_retrieve_data.FunctionalTesting.test_rap_lbcs_from_aws -''' +""" import glob import os import tempfile @@ -22,22 +22,21 @@ import retrieve_data + class FunctionalTesting(unittest.TestCase): - ''' Test class for retrieve data ''' + """Test class for retrieve data""" def setUp(self): self.path = os.path.dirname(__file__) - self.config = f'{self.path}/../parm/data_locations.yml' + self.config = f"{self.path}/../parm/data_locations.yml" - @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") + @unittest.skipIf(os.environ.get("CI") == "true", "Skipping HPSS tests") def test_fv3gfs_grib2_lbcs_from_hpss(self): - ''' Get FV3GFS grib2 files from HPSS for LBCS, offset by 6 hours - - ''' + """Get FV3GFS grib2 files from HPSS for LBCS, offset by 6 hours""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -56,18 +55,18 @@ def test_fv3gfs_grib2_lbcs_from_hpss(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 3) - @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") + @unittest.skipIf(os.environ.get("CI") == "true", "Skipping HPSS tests") def test_fv3gfs_netcdf_lbcs_from_hpss(self): - ''' Get FV3GFS netcdf files from HPSS for LBCS. Tests fcst lead + """Get FV3GFS netcdf files from HPSS for LBCS. Tests fcst lead times > 40 hours, since they come from a different archive file. - ''' + """ - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -86,20 +85,19 @@ def test_fv3gfs_netcdf_lbcs_from_hpss(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 2) # GDAS Tests def test_gdas_ics_from_aws(self): - ''' In real time, GDAS is used for LBCS with a 6 hour offset. - ''' + """In real time, GDAS is used for LBCS with a 6 hour offset.""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) - out_path_tmpl = f'mem{{mem:03d}}' + out_path_tmpl = f"mem{{mem:03d}}" args = [ '--anl_or_fcst', 'anl', @@ -120,22 +118,19 @@ def test_gdas_ics_from_aws(self): for mem in [9, 10]: files_on_disk = glob.glob( - os.path.join(out_path_tmpl.format(mem=mem), '*') - ) + os.path.join(out_path_tmpl.format(mem=mem), "*") + ) self.assertEqual(len(files_on_disk), 2) - # GEFS Tests def test_gefs_grib2_ics_from_aws(self): - ''' Get GEFS grib2 a & b files for ICS offset by 6 hours. + """Get GEFS grib2 a & b files for ICS offset by 6 hours.""" - ''' - - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) - out_path_tmpl = f'mem{{mem:03d}}' + out_path_tmpl = f"mem{{mem:03d}}" args = [ '--anl_or_fcst', 'anl', @@ -155,19 +150,17 @@ def test_gefs_grib2_ics_from_aws(self): # Verify files exist in temp dir for mem in [1, 2]: files_on_disk = glob.glob( - os.path.join(out_path_tmpl.format(mem=mem), '*') - ) + os.path.join(out_path_tmpl.format(mem=mem), "*") + ) self.assertEqual(len(files_on_disk), 2) - - # HRRR Tests - @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") + @unittest.skipIf(os.environ.get("CI") == "true", "Skipping HPSS tests") def test_hrrr_ics_from_hpss(self): - ''' Get HRRR ICS from hpss ''' + """Get HRRR ICS from hpss""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -185,16 +178,16 @@ def test_hrrr_ics_from_hpss(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 1) - @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") + @unittest.skipIf(os.environ.get("CI") == "true", "Skipping HPSS tests") def test_hrrr_lbcs_from_hpss(self): - ''' Get HRRR LBCS from hpss for 3 hour boundary conditions ''' + """Get HRRR LBCS from hpss for 3 hour boundary conditions""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -212,15 +205,15 @@ def test_hrrr_lbcs_from_hpss(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 8) def test_hrrr_ics_from_aws(self): - ''' Get HRRR ICS from aws ''' + """Get HRRR ICS from aws""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -238,15 +231,15 @@ def test_hrrr_ics_from_aws(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 1) def test_hrrr_lbcs_from_aws(self): - ''' Get HRRR LBCS from aws for 3 hour boundary conditions ''' + """Get HRRR LBCS from aws for 3 hour boundary conditions""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -264,16 +257,16 @@ def test_hrrr_lbcs_from_aws(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 8) # RAP tests def test_rap_ics_from_aws(self): - ''' Get RAP ICS from aws offset by 3 hours ''' + """Get RAP ICS from aws offset by 3 hours""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -291,16 +284,16 @@ def test_rap_ics_from_aws(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 1) def test_rap_lbcs_from_aws(self): - ''' Get RAP LBCS from aws for 6 hour boundary conditions offset - by 3 hours. Use 09Z start time for longer LBCS.''' + """Get RAP LBCS from aws for 6 hour boundary conditions offset + by 3 hours. Use 09Z start time for longer LBCS.""" - with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + with tempfile.TemporaryDirectory(dir=".") as tmp_dir: os.chdir(tmp_dir) args = [ @@ -318,6 +311,6 @@ def test_rap_lbcs_from_aws(self): # Verify files exist in temp dir - path = os.path.join(tmp_dir, '*') + path = os.path.join(tmp_dir, "*") files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 5)