diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index bddf87ca83..ec06a88150 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -65,6 +65,63 @@ WE2Edir="$TESTSdir/WE2E" # #----------------------------------------------------------------------- # +# Run python checks +# +#----------------------------------------------------------------------- +# + +# This line will return two numbers: the python major and minor versions +pyversion=($(/usr/bin/env python3 -c 'import platform; major, minor, patch = platform.python_version_tuple(); print(major); print(minor)')) + +#Now, set an error check variable so that we can print all python errors rather than just the first +pyerrors=0 + +# Check that the call to python3 returned no errors, then check if the +# python3 minor version is 6 or higher +if [[ -z "$pyversion" ]];then + print_info_msg "\ + + Error: python3 not found" + pyerrors=$((pyerrors+1)) +else + if [[ ${#pyversion[@]} -lt 2 ]]; then + print_info_msg "\ + + Error retrieving python3 version" + pyerrors=$((pyerrors+1)) + elif [[ ${pyversion[1]} -lt 6 ]]; then + print_info_msg "\ + + Error: python version must be 3.6 or higher + python version: ${pyversion[*]}" + pyerrors=$((pyerrors+1)) + fi +fi + +#Next, check for the non-standard python packages: jinja2, yaml, and f90nml +pkgs=(jinja2 yaml f90nml) +for pkg in ${pkgs[@]} ; do + if ! /usr/bin/env python3 -c "import ${pkg}" &> /dev/null; then + print_info_msg "\ + + Error: python module ${pkg} not available" + pyerrors=$((pyerrors+1)) + fi +done + +#Finally, check if the number of errors is >0, and if so exit with helpful message +if [ $pyerrors -gt 0 ];then + print_err_msg_exit "\ + Errors found: check your python environment + + Instructions for setting up python environments can be found on the web: + https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started + +" +fi +# +#----------------------------------------------------------------------- +# # Save current shell options (in a global array). Then set new options # for this script or function. # diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 96bca2e66f..2c5831a4df 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -54,7 +54,7 @@ user: # #----------------------------------------------------------------------- MACHINE: "BIG_COMPUTER" - ACCOUNT: "project_name" + ACCOUNT: "" #---------------------------- # PLATFORM config parameters diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 63be16bd50..11a86c9aaa 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -5,6 +5,7 @@ import platform import subprocess import unittest +import logging from multiprocessing import Process from textwrap import dedent from datetime import datetime, timedelta @@ -12,6 +13,7 @@ from python_utils import ( print_info_msg, print_err_msg_exit, + log_info, import_vars, cp_vrfy, cd_vrfy, @@ -61,57 +63,43 @@ def python_error_handler(): python_error_handler() -def generate_FV3LAM_wflow(): +def generate_FV3LAM_wflow(USHdir, logfile: str = 'log.generate_FV3LAM_wflow') -> None: """Function to setup a forecast experiment and create a workflow - (according to the parameters specified in the config file + (according to the parameters specified in the config file) Args: - None + USHdir (str): The full path of the ush/ directory where this script is located + logfile (str): The name of the file where logging is written Returns: None """ - print( - dedent( + # Set up logging to write to screen and logfile + setup_logging(logfile) + + log_info( """ ======================================================================== - ======================================================================== - Starting experiment generation... - - ======================================================================== ========================================================================""" - ) ) - # set USHdir - USHdir = os.path.dirname(os.path.abspath(__file__)) - # check python version major, minor, patch = platform.python_version_tuple() if int(major) < 3 or int(minor) < 6: - print_info_msg( + logging.error( f""" Error: python version must be 3.6 or higher python version: {major}.{minor}""" ) + raise - # define macros + # define utilities define_macos_utilities() - # - # ----------------------------------------------------------------------- - # - # Source the file that defines and then calls the setup function. The - # setup function in turn first sources the default configuration file - # (which contains default values for the experiment/workflow parameters) - # and then sources the user-specified configuration file (which contains - # user-specified values for a subset of the experiment/workflow parame- - # ters that override their default values). - # - # ----------------------------------------------------------------------- - # + # The setup function reads the user configuration file and fills in + # non-user-specified values from config_defaults.yaml setup() # import all environment variables @@ -134,8 +122,8 @@ def generate_FV3LAM_wflow(): # Create a multiline variable that consists of a yaml-compliant string # specifying the values that the jinja variables in the template rocoto # XML should be set to. These values are set either in the user-specified - # workflow configuration file (EXPT_CONFIG_FN) or in the setup.sh script - # sourced above. Then call the python script that generates the XML. + # workflow configuration file (EXPT_CONFIG_FN) or in the setup() function + # called above. Then call the python script that generates the XML. # # ----------------------------------------------------------------------- # @@ -143,7 +131,7 @@ def generate_FV3LAM_wflow(): template_xml_fp = os.path.join(PARMdir, WFLOW_XML_FN) - print_info_msg( + log_info( f''' Creating rocoto workflow XML file (WFLOW_XML_FP) from jinja template XML file (template_xml_fp): @@ -438,17 +426,15 @@ def generate_FV3LAM_wflow(): # End of "settings" variable. settings_str = cfg_to_yaml_str(settings) - print_info_msg( - dedent( + log_info( f""" The variable \"settings\" specifying values of the rococo XML variables has been set as follows: #----------------------------------------------------------------------- - settings =\n\n""" + settings =\n\n""", + verbose=VERBOSE, ) - + settings_str, - verbose=VERBOSE, - ) + log_info(settings_str, verbose=VERBOSE) # # Call the python script to generate the experiment's actual XML file @@ -459,7 +445,7 @@ def generate_FV3LAM_wflow(): ["-q", "-u", settings_str, "-t", template_xml_fp, "-o", WFLOW_XML_FP] ) except: - print_err_msg_exit( + logging.exception( dedent( f""" Call to python script fill_jinja_template.py to create a rocoto workflow @@ -482,7 +468,7 @@ def generate_FV3LAM_wflow(): # # ----------------------------------------------------------------------- # - print_info_msg( + log_info( f''' Creating symlink in the experiment directory (EXPTDIR) that points to the workflow launch script (WFLOW_LAUNCH_SCRIPT_FP): @@ -505,23 +491,13 @@ def generate_FV3LAM_wflow(): # if USE_CRON_TO_RELAUNCH: add_crontab_line() - # - # ----------------------------------------------------------------------- - # - # Create the FIXam directory under the experiment directory. In NCO mode, - # this will be a symlink to the directory specified in FIXgsm, while in - # community mode, it will be an actual directory with files copied into - # it from FIXgsm. - # - # ----------------------------------------------------------------------- - # # - # Symlink fix files + # Copy or symlink fix files # if SYMLINK_FIX_FILES: - print_info_msg( + log_info( f''' Symlinking fixed files from system directory (FIXgsm) to a subdirectory (FIXam): FIXgsm = \"{FIXgsm}\" @@ -530,12 +506,9 @@ def generate_FV3LAM_wflow(): ) ln_vrfy(f'''-fsn "{FIXgsm}" "{FIXam}"''') - # - # Copy relevant fix files. - # else: - print_info_msg( + log_info( f''' Copying fixed files from system directory (FIXgsm) to a subdirectory (FIXam): FIXgsm = \"{FIXgsm}\" @@ -559,7 +532,7 @@ def generate_FV3LAM_wflow(): # ----------------------------------------------------------------------- # if USE_MERRA_CLIMO: - print_info_msg( + log_info( f''' Copying MERRA2 aerosol climatology data files from system directory (FIXaer/FIXlut) to a subdirectory (FIXclim) in the experiment directory: @@ -585,27 +558,27 @@ def generate_FV3LAM_wflow(): # # ----------------------------------------------------------------------- # - print_info_msg( + log_info( f""" Copying templates of various input files to the experiment directory...""", verbose=VERBOSE, ) - print_info_msg( + log_info( f""" Copying the template data table file to the experiment directory...""", verbose=VERBOSE, ) cp_vrfy(DATA_TABLE_TMPL_FP, DATA_TABLE_FP) - print_info_msg( + log_info( f""" Copying the template field table file to the experiment directory...""", verbose=VERBOSE, ) cp_vrfy(FIELD_TABLE_TMPL_FP, FIELD_TABLE_FP) - print_info_msg( + log_info( f""" Copying the template NEMS configuration file to the experiment directory...""", verbose=VERBOSE, @@ -616,7 +589,7 @@ def generate_FV3LAM_wflow(): # clone of the FV3 code repository to the experiment directory (EXPT- # DIR). # - print_info_msg( + log_info( f""" Copying the CCPP physics suite definition XML file from its location in the forecast model directory sturcture to the experiment directory...""", @@ -628,7 +601,7 @@ def generate_FV3LAM_wflow(): # clone of the FV3 code repository to the experiment directory (EXPT- # DIR). # - print_info_msg( + log_info( f""" Copying the field dictionary file from its location in the forecast model directory sturcture to the experiment directory...""", @@ -642,7 +615,7 @@ def generate_FV3LAM_wflow(): # # ----------------------------------------------------------------------- # - print_info_msg( + log_info( f''' Setting parameters in weather model's namelist file (FV3_NML_FP): FV3_NML_FP = \"{FV3_NML_FP}\"''' @@ -660,9 +633,6 @@ def generate_FV3LAM_wflow(): # Otherwise, leave it unspecified (which means it gets set to the default # value in the forecast model). # - # NOTE: - # May want to remove kice from FV3.input.yml (and maybe input.nml.FV3). - # kice = None if SDF_USES_RUC_LSM: kice = 9 @@ -877,17 +847,13 @@ def generate_FV3LAM_wflow(): settings_str = cfg_to_yaml_str(settings) - print_info_msg( - dedent( + log_info( f""" The variable \"settings\" specifying values of the weather model's - namelist variables has been set as follows: - - settings =\n\n""" - ) - + settings_str, + namelist variables has been set as follows:\n""", verbose=VERBOSE, ) + log_info("\nsettings =\n\n" + settings_str, verbose=VERBOSE) # # ----------------------------------------------------------------------- # @@ -917,7 +883,7 @@ def generate_FV3LAM_wflow(): ] ) except: - print_err_msg_exit( + logging.exception( dedent( f""" Call to python script set_namelist.py to generate an FV3 namelist file @@ -950,6 +916,13 @@ def generate_FV3LAM_wflow(): if not RUN_TASK_MAKE_GRID: set_FV3nml_sfc_climo_filenames() + + # Call function to get NOMADS data + if NOMADS: + raise Exception("Nomads script does not work!") + + # get_nomads_data(NOMADS_file_type,EXPTDIR,USHdir,DATE_FIRST_CYCL,CYCL_HRS,FCST_LEN_HRS,LBC_SPEC_INTVL_HRS) + # # ----------------------------------------------------------------------- # @@ -975,7 +948,7 @@ def generate_FV3LAM_wflow(): rocotorun_cmd = f"rocotorun -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" rocotostat_cmd = f"rocotostat -w {WFLOW_XML_FN} -d {wflow_db_fn} -v 10" - print_info_msg( + log_info( f""" ======================================================================== ======================================================================== @@ -988,33 +961,16 @@ def generate_FV3LAM_wflow(): ======================================================================== """ ) - # # ----------------------------------------------------------------------- # - # If rocoto is required, print instructions on how to load and use it + # If rocoto is required, print instructions on how to use it # # ----------------------------------------------------------------------- # if WORKFLOW_MANAGER == "rocoto": - print_info_msg( + log_info( f""" - To launch the workflow, first ensure that you have a compatible version - of rocoto available. For most pre-configured platforms, rocoto can be - loaded via a module: - - > module load rocoto - - For more details on rocoto, see the User's Guide. - - To launch the workflow, first ensure that you have a compatible version - of rocoto loaded. For example, to load version 1.3.1 of rocoto, use - - > module load rocoto/1.3.1 - - (This version has been tested on hera; later versions may also work but - have not been tested.) - To launch the workflow, change location to the experiment directory (EXPTDIR) and issue the rocotrun command, as follows: @@ -1043,127 +999,60 @@ def generate_FV3LAM_wflow(): */{CRON_RELAUNCH_INTVL_MNTS} * * * * cd {EXPTDIR} && ./launch_FV3LAM_wflow.sh called_from_cron=\"TRUE\" """ ) - # - # If necessary, run the NOMADS script to source external model data. - # - if NOMADS: - print("Getting NOMADS online data") - print(f"NOMADS_file_type= {NOMADS_file_type}") - cd_vrfy(EXPTDIR) - NOMADS_script = os.path.join(USHdir, "NOMADS_get_extrn_mdl_files.h") - run_command( - f"""{NOMADS_script} {date_to_str(DATE_FIRST_CYCL,format="%Y%m%d")} \ - {date_to_str(DATE_FIRST_CYCL,format="%H")} {NOMADS_file_type} {FCST_LEN_HRS} {LBC_SPEC_INTVL_HRS}""" - ) + # If we got to this point everything was successful: move the log file to the experiment directory. + mv_vrfy(logfile, EXPTDIR) + +def get_nomads_data(NOMADS_file_type,EXPTDIR,USHdir,DATE_FIRST_CYCL,CYCL_HRS,FCST_LEN_HRS,LBC_SPEC_INTVL_HRS): + print("Getting NOMADS online data") + print(f"NOMADS_file_type= {NOMADS_file_type}") + cd_vrfy(EXPTDIR) + NOMADS_script = os.path.join(USHdir, "NOMADS_get_extrn_mdl_files.sh") + run_command(f"""{NOMADS_script} {date_to_str(DATE_FIRST_CYCL,format="%Y%m%d")} \ + {date_to_str(DATE_FIRST_CYCL,format="%H")} {NOMADS_file_type} {FCST_LEN_HRS} {LBC_SPEC_INTVL_HRS}""") + +def setup_logging(logfile: str = 'log.generate_FV3LAM_wflow') -> None: + """ + Sets up logging, printing high-priority (INFO and higher) messages to screen, and printing all + messages with detailed timing and routine info in the specified text file. + """ + logging.basicConfig(level=logging.DEBUG, + format='%(name)-22s %(levelname)-8s %(message)s', + filename=logfile, + filemode='w') + logging.debug(f'Finished setting up debug file logging in {logfile}') + console = logging.StreamHandler() + console.setLevel(logging.INFO) + logging.getLogger().addHandler(console) + logging.debug('Logging set up successfully') -# -# ----------------------------------------------------------------------- -# -# Start of the script that will call the experiment/workflow generation -# function defined above. -# -# ----------------------------------------------------------------------- -# if __name__ == "__main__": - # - # ----------------------------------------------------------------------- - # - # Set directories. - # - # ----------------------------------------------------------------------- - # + USHdir = os.path.dirname(os.path.abspath(__file__)) - # - # Set the name of and full path to the temporary file in which we will - # save some experiment/workflow variables. The need for this temporary - # file is explained below. - # - tmp_fn = "tmp" - tmp_fp = os.path.join(USHdir, tmp_fn) - rm_vrfy("-f", tmp_fp) - # - # Set the name of and full path to the log file in which the output from - # the experiment/workflow generation function will be saved. - # - log_fn = "log.generate_FV3LAM_wflow" - log_fp = os.path.join(USHdir, log_fn) - rm_vrfy("-f", log_fp) - # + logfile=f'{USHdir}/log.generate_FV3LAM_wflow' + # Call the generate_FV3LAM_wflow function defined above to generate the - # experiment/workflow. Note that we pipe the output of the function - # (and possibly other commands) to the "tee" command in order to be able - # to both save it to a file and print it out to the screen (stdout). - # The piping causes the call to the function (and the other commands - # grouped with it using the curly braces, { ... }) to be executed in a - # subshell. As a result, the experiment/workflow variables that the - # function sets are not available outside of the grouping, i.e. they are - # not available at and after the call to "tee". Since some of these va- - # riables are needed after the call to "tee" below, we save them in a - # temporary file and read them in outside the subshell later below. - # - def workflow_func(): - retval = 1 - generate_FV3LAM_wflow() - retval = 0 - run_command(f'''echo "{EXPTDIR}" >> "{tmp_fp}"''') - run_command(f'''echo "{retval}" >> "{tmp_fp}"''') - - # create tee functionality - tee = subprocess.Popen(["tee", log_fp], stdin=subprocess.PIPE) - os.dup2(tee.stdin.fileno(), sys.stdout.fileno()) - os.dup2(tee.stdin.fileno(), sys.stderr.fileno()) - - # create workflow process - p = Process(target=workflow_func) - p.start() - p.join() - - # - # Read in experiment/workflow variables needed later below from the tem- - # porary file created in the subshell above containing the call to the - # generate_FV3LAM_wflow function. These variables are not directly - # available here because the call to generate_FV3LAM_wflow above takes - # place in a subshell (due to the fact that we are then piping its out- - # put to the "tee" command). Then remove the temporary file. - # - (_, exptdir, _) = run_command(f'''sed "1q;d" "{tmp_fp}"''') - (_, retval, _) = run_command(f''' sed "2q;d" "{tmp_fp}"''') - if retval: - retval = int(retval) - else: - retval = 1 - rm_vrfy(tmp_fp) - # - # If the call to the generate_FV3LAM_wflow function above was success- - # ful, move the log file in which the "tee" command saved the output of - # the function to the experiment directory. - # - if retval == 0: - mv_vrfy(log_fp, exptdir) - # - # If the call to the generate_FV3LAM_wflow function above was not suc- - # cessful, print out an error message and exit with a nonzero return - # code. - # - else: - print_err_msg_exit( + # experiment/workflow. + try: + generate_FV3LAM_wflow(USHdir, logfile) + except: + logging.exception(dedent( f""" - Experiment generation failed. Check the log file from the ex- - periment/workflow generation script in the file specified by log_fp: - log_fp = \"{log_fp}\" - Stopping.""" - ) + ********************************************************************* + FATAL ERROR: + Experiment generation failed. See the error message(s) printed below. + For more detailed information, check the log file from the workflow + generation script: {logfile} + *********************************************************************\n + """ + )) class Testing(unittest.TestCase): def test_generate_FV3LAM_wflow(self): - # run workflows in separate process to avoid conflict - def workflow_func(): - generate_FV3LAM_wflow() - - def run_workflow(): - p = Process(target=workflow_func) + # run workflows in separate process to avoid conflict between community and nco settings + def run_workflow(USHdir,logfile): + p = Process(target=generate_FV3LAM_wflow,args=(USHdir,logfile)) p.start() p.join() exit_code = p.exitcode @@ -1171,18 +1060,19 @@ def run_workflow(): sys.exit(exit_code) USHdir = os.path.dirname(os.path.abspath(__file__)) + logfile='log.generate_FV3LAM_wflow' SED = get_env_var("SED") # community test case cp_vrfy(f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") run_command(f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""") - run_workflow() + run_workflow(USHdir, logfile) # nco test case set_env_var("OPSROOT", f"{USHdir}/../../nco_dirs") cp_vrfy(f"{USHdir}/config.nco.yaml", f"{USHdir}/config.yaml") run_command(f"""{SED} -i 's/MACHINE: hera/MACHINE: linux/g' {USHdir}/config.yaml""") - run_workflow() + run_workflow(USHdir, logfile) def setUp(self): define_macos_utilities() diff --git a/ush/get_crontab_contents.py b/ush/get_crontab_contents.py index b0950f5c6f..c6c8d41ee0 100644 --- a/ush/get_crontab_contents.py +++ b/ush/get_crontab_contents.py @@ -5,8 +5,10 @@ import unittest import argparse from datetime import datetime +from textwrap import dedent from python_utils import ( + log_info, import_vars, set_env_var, print_input_args, @@ -99,7 +101,7 @@ def add_crontab_line(): # time_stamp = datetime.now().strftime("%F_%T") crontab_backup_fp = os.path.join(EXPTDIR, f"crontab.bak.{time_stamp}") - print_info_msg( + log_info( f''' Copying contents of user cron table to backup file: crontab_backup_fp = \"{crontab_backup_fp}\"''', @@ -123,7 +125,7 @@ def add_crontab_line(): # Add crontab line if CRONTAB_LINE in crontab_contents: - print_info_msg( + log_info( f''' The following line already exists in the cron table and thus will not be added: @@ -132,7 +134,7 @@ def add_crontab_line(): else: - print_info_msg( + log_info( f''' Adding the following line to the user's cron table in order to automatically resubmit SRW workflow: diff --git a/ush/python_utils/__init__.py b/ush/python_utils/__init__.py index 8a74a595d0..add96ee981 100644 --- a/ush/python_utils/__init__.py +++ b/ush/python_utils/__init__.py @@ -28,7 +28,7 @@ from .get_elem_inds import get_elem_inds from .interpol_to_arbit_CRES import interpol_to_arbit_CRES from .print_input_args import print_input_args -from .print_msg import print_info_msg, print_err_msg_exit +from .print_msg import print_info_msg, print_err_msg_exit, log_info from .run_command import run_command from .get_charvar_from_netcdf import get_charvar_from_netcdf from .xml_parser import load_xml_file, has_tag_with_value diff --git a/ush/python_utils/check_for_preexist_dir_file.py b/ush/python_utils/check_for_preexist_dir_file.py index fc67447fb5..cfd4d50f43 100644 --- a/ush/python_utils/check_for_preexist_dir_file.py +++ b/ush/python_utils/check_for_preexist_dir_file.py @@ -2,10 +2,10 @@ import os from datetime import datetime -from .print_msg import print_info_msg, print_err_msg_exit +from textwrap import dedent from .check_var_valid_value import check_var_valid_value from .filesys_cmds_vrfy import rm_vrfy, mv_vrfy - +from .print_msg import log_info def check_for_preexist_dir_file(path, method): """Check for a preexisting directory or file and, if present, deal with it @@ -18,7 +18,14 @@ def check_for_preexist_dir_file(path, method): None """ - check_var_valid_value(method, ["delete", "rename", "quit"]) + try: + check_var_valid_value(method, ["delete", "rename", "quit"]) + except ValueError: + errmsg = dedent(f''' + Invalid method for dealing with pre-existing directory specified + method = {method} + ''') + raise ValueError(errmsg) from None if os.path.exists(path): if method == "delete": @@ -27,7 +34,7 @@ def check_for_preexist_dir_file(path, method): now = datetime.now() d = now.strftime("_old_%Y%m%d_%H%M%S") new_path = path + d - print_info_msg( + log_info( f""" Specified directory or file already exists: {path} @@ -36,8 +43,8 @@ def check_for_preexist_dir_file(path, method): ) mv_vrfy(path, new_path) else: - print_err_msg_exit( + raise FileExistsError(dedent( f""" Specified directory or file already exists {path}""" - ) + )) diff --git a/ush/python_utils/check_var_valid_value.py b/ush/python_utils/check_var_valid_value.py index 10d8365cb6..a8b88e17a6 100644 --- a/ush/python_utils/check_var_valid_value.py +++ b/ush/python_utils/check_var_valid_value.py @@ -1,23 +1,17 @@ #!/usr/bin/env python3 -from .print_msg import print_err_msg_exit - - -def check_var_valid_value(var, values, err_msg=None): +def check_var_valid_value(var, values): """Check if specified variable has a valid value Args: var: the variable values: list of valid values - err_msg: additional error message to print Returns: True: if var has valid value, exit(1) otherwise """ + if not var: + var = "" if var not in values: - if err_msg is not None: - err_msg = f"The value specified in var = {var} is not supported." - print_err_msg_exit( - err_msg + f"{var} must be set to one of the following:\n {values}" - ) + raise ValueError(f"Got '{var}', expected one of the following:\n {values}") return True diff --git a/ush/python_utils/config_parser.py b/ush/python_utils/config_parser.py index b89ab2b5a7..c1b69db5ff 100644 --- a/ush/python_utils/config_parser.py +++ b/ush/python_utils/config_parser.py @@ -36,7 +36,6 @@ from xml.dom import minidom from .environment import list_to_str, str_to_list -from .print_msg import print_err_msg_exit from .run_command import run_command ########## @@ -45,11 +44,8 @@ def load_yaml_config(config_file): """Safe load a yaml file""" - try: - with open(config_file, "r") as f: - cfg = yaml.safe_load(f) - except yaml.YAMLError as e: - print_err_msg_exit(str(e)) + with open(config_file, "r") as f: + cfg = yaml.safe_load(f) return cfg @@ -102,7 +98,7 @@ def load_json_config(config_file): with open(config_file, "r") as f: cfg = json.load(f) except json.JSONDecodeError as e: - print_err_msg_exit(str(e)) + raise Exception(f"Unable to load json file {config_file}") return cfg @@ -218,11 +214,10 @@ def load_ini_config(config_file, return_string=0): """Load a config file with a format similar to Microsoft's INI files""" if not os.path.exists(config_file): - print_err_msg_exit( - f''' - The specified configuration file does not exist: - \"{config_file}\"''' - ) + raise FileNotFoundError(dedent(f''' + The specified configuration file does not exist: + "{config_file}"''' + )) config = configparser.RawConfigParser() config.optionxform = str @@ -238,12 +233,7 @@ def get_ini_value(config, section, key): """Finds the value of a property in a given section""" if not section in config: - print_err_msg_exit( - f''' - Section not found: - section = \"{section}\" - valid sections = \"{config.keys()}\"''' - ) + raise KeyError(f'Section not found: {section}') else: return config[section][key] diff --git a/ush/python_utils/print_msg.py b/ush/python_utils/print_msg.py index 606284e140..8db0f06b03 100644 --- a/ush/python_utils/print_msg.py +++ b/ush/python_utils/print_msg.py @@ -2,7 +2,8 @@ import traceback import sys -from textwrap import dedent +from textwrap import dedent, indent +from logging import getLogger def print_err_msg_exit(error_msg="", stack_trace=True): @@ -39,3 +40,25 @@ def print_info_msg(info_msg, verbose=True): print(dedent(info_msg)) return True return False + +def log_info(info_msg, verbose=True, dedent_=True): + """Function to print information message using the logging module. This function + should not be used if python logging has not been initialized. + + Args: + info_msg : info message to print + verbose : set to False to silence printing + dedent_ : set to False to disable "dedenting" (print string as-is) + Returns: + None + """ + + # "sys._getframe().f_back.f_code.co_name" returns the name of the calling function + logger=getLogger(sys._getframe().f_back.f_code.co_name) + + if verbose: + if dedent_: + logger.info(indent(dedent(info_msg), ' ')) + else: + logger.info(info_msg) + diff --git a/ush/set_ozone_param.py b/ush/set_ozone_param.py index e371791452..5ed4449fe1 100644 --- a/ush/set_ozone_param.py +++ b/ush/set_ozone_param.py @@ -5,20 +5,18 @@ from textwrap import dedent from python_utils import ( + log_info, import_vars, export_vars, set_env_var, list_to_str, print_input_args, - print_info_msg, - print_err_msg_exit, define_macos_utilities, load_xml_file, has_tag_with_value, find_pattern_in_str, ) - def set_ozone_param(ccpp_phys_suite_fp): """Function that does the following: (1) Determines the ozone parameterization being used by checking in the @@ -92,13 +90,8 @@ def set_ozone_param(ccpp_phys_suite_fp): fixgsm_ozone_fn = "global_o3prdlos.f77" ozone_param = "ozphys" else: - print_err_msg_exit( - f''' - Unknown or no ozone parameterization - specified in the CCPP physics suite file (ccpp_phys_suite_fp): - ccpp_phys_suite_fp = \"{ccpp_phys_suite_fp}\" - ozone_param = \"{ozone_param}\"''' - ) + raise KeyError(f'Unknown or no ozone parameterization specified in the ' + 'CCPP physics suite file "{ccpp_phys_suite_fp}"') # # ----------------------------------------------------------------------- # @@ -151,27 +144,21 @@ def set_ozone_param(ccpp_phys_suite_fp): # ----------------------------------------------------------------------- # if fixgsm_ozone_fn_is_set: - - msg = dedent( + log_info( f""" After setting the file name of the ozone production/loss file in the FIXgsm directory (based on the ozone parameterization specified in the CCPP suite definition file), the array specifying the mapping between the symlinks that need to be created in the cycle directories and the files in the FIXam directory is: - - """ - ) - msg += dedent( - f""" + """, verbose=VERBOSE) + log_info(f""" CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = {list_to_str(CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING)} - """ - ) - print_info_msg(msg, verbose=VERBOSE) + """, verbose=VERBOSE, dedent_=False) else: - print_err_msg_exit( + raise Exception( f''' Unable to set name of the ozone production/loss file in the FIXgsm directory in the array that specifies the mapping between the symlinks that need to diff --git a/ush/set_predef_grid_params.py b/ush/set_predef_grid_params.py index 2e48e5b4a8..6b432b8f03 100644 --- a/ush/set_predef_grid_params.py +++ b/ush/set_predef_grid_params.py @@ -2,6 +2,7 @@ import unittest import os +from textwrap import dedent from python_utils import ( import_vars, @@ -37,7 +38,14 @@ def set_predef_grid_params(): USHdir = os.path.dirname(os.path.abspath(__file__)) params_dict = load_config_file(os.path.join(USHdir, "predef_grid_params.yaml")) - params_dict = params_dict[PREDEF_GRID_NAME] + try: + params_dict = params_dict[PREDEF_GRID_NAME] + except KeyError: + errmsg = dedent(f''' + PREDEF_GRID_NAME = {PREDEF_GRID_NAME} not found in predef_grid_params.yaml + Check your config file settings.''') + raise Exception(errmsg) from None + # if QUILTING = False, remove key if not QUILTING: diff --git a/ush/set_thompson_mp_fix_files.py b/ush/set_thompson_mp_fix_files.py index 0d7a454ad4..93dc3c5de6 100644 --- a/ush/set_thompson_mp_fix_files.py +++ b/ush/set_thompson_mp_fix_files.py @@ -5,13 +5,12 @@ from textwrap import dedent from python_utils import ( + log_info, import_vars, export_vars, set_env_var, list_to_str, print_input_args, - print_info_msg, - print_err_msg_exit, define_macos_utilities, load_xml_file, has_tag_with_value, @@ -90,7 +89,7 @@ def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn): mapping = f"{fix_file} | {fix_file}" CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING.append(mapping) - msg = dedent( + log_info( f""" Since the Thompson microphysics parameterization is being used by this physics suite (CCPP_PHYS_SUITE), the names of the fixed files needed by @@ -100,21 +99,19 @@ def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn): CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING. After these modifications, the values of these parameters are as follows: + CCPP_PHYS_SUITE = \"{CCPP_PHYS_SUITE}\" """ ) - msg += dedent( + log_info( f""" - CCPP_PHYS_SUITE = \"{CCPP_PHYS_SUITE}\" - FIXgsm_FILES_TO_COPY_TO_FIXam = {list_to_str(FIXgsm_FILES_TO_COPY_TO_FIXam)} """ ) - msg += dedent( + log_info( f""" CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = {list_to_str(CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING)} """ ) - print_info_msg(msg) EXPORTS = [ "CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", diff --git a/ush/setup.py b/ush/setup.py index 4d6841be54..3fa7313727 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -3,9 +3,12 @@ import os import sys import datetime +import traceback from textwrap import dedent +from logging import getLogger from python_utils import ( + log_info, cd_vrfy, mkdir_vrfy, rm_vrfy, @@ -18,8 +21,6 @@ import_vars, export_vars, get_env_var, - print_info_msg, - print_err_msg_exit, load_config_file, cfg_to_shell_str, cfg_to_yaml_str, @@ -56,6 +57,7 @@ def setup(): None """ + logger = getLogger(__name__) global USHdir USHdir = os.path.dirname(os.path.abspath(__file__)) cd_vrfy(USHdir) @@ -64,7 +66,7 @@ def setup(): import_vars() # print message - print_info_msg( + log_info( f""" ======================================================================== Starting function setup() in \"{os.path.basename(__file__)}\"... @@ -86,26 +88,37 @@ def setup(): "EXTRN_MDL_NAME_ICS", "EXTRN_MDL_NAME_LBCS", "FV3GFS_FILE_FMT_ICS", "FV3GFS_FILE_FMT_LBCS"]) - # - # ----------------------------------------------------------------------- - # - # Load the user config file but don't source it yet. - # - # ----------------------------------------------------------------------- - # - if os.path.exists(EXPT_CONFIG_FN): + + # Load the user config file, then ensure all user-specified + # variables correspond to a default value. + if not os.path.exists(EXPT_CONFIG_FN): + raise FileNotFoundError(f'User config file not found: EXPT_CONFIG_FN = {EXPT_CONFIG_FN}') + + try: cfg_u = load_config_file(os.path.join(USHdir, EXPT_CONFIG_FN)) - cfg_u = flatten_dict(cfg_u) - import_vars(dictionary=cfg_u, - env_vars=["MACHINE", - "EXTRN_MDL_NAME_ICS", "EXTRN_MDL_NAME_LBCS", - "FV3GFS_FILE_FMT_ICS", "FV3GFS_FILE_FMT_LBCS"]) - else: - print_err_msg_exit( - f''' - User config file not found - EXPT_CONFIG_FN = \"{EXPT_CONFIG_FN}\"''' - ) + except: + errmsg = dedent(f'''\n + Could not load YAML config file: {EXPT_CONFIG_FN} + Reference the above traceback for more information. + ''') + raise Exception(errmsg) + + cfg_u = flatten_dict(cfg_u) + for key in cfg_u: + if key not in flatten_dict(cfg_d): + raise Exception(dedent(f''' + User-specified variable "{key}" in {EXPT_CONFIG_FN} is not valid. + Check {EXPT_DEFAULT_CONFIG_FN} for allowed user-specified variables.\n''')) + + # Mandatory variables *must* be set in the user's config; the default value is invalid + mandatory = ['MACHINE'] + for val in mandatory: + if val not in cfg_u: + raise Exception(f'Mandatory variable "{val}" not found in user config file {EXPT_CONFIG_FN}') + + import_vars(dictionary=cfg_u, env_vars=["MACHINE", + "EXTRN_MDL_NAME_ICS", "EXTRN_MDL_NAME_LBCS", + "FV3GFS_FILE_FMT_ICS", "FV3GFS_FILE_FMT_LBCS"]) # # ----------------------------------------------------------------------- # @@ -117,6 +130,12 @@ def setup(): # global MACHINE, EXTRN_MDL_SYSBASEDIR_ICS, EXTRN_MDL_SYSBASEDIR_LBCS MACHINE_FILE = os.path.join(USHdir, "machine", f"{lowercase(MACHINE)}.yaml") + if not os.path.exists(MACHINE_FILE): + raise FileNotFoundError(dedent( + f""" + The machine file {MACHINE_FILE} does not exist. + Check that you have specified the correct machine ({MACHINE}) in your config file {EXPT_CONFIG_FN}""" + )) machine_cfg = load_config_file(MACHINE_FILE) # ics and lbcs @@ -161,27 +180,21 @@ def get_location(xcs,fmt): # make machine name uppercase MACHINE = uppercase(MACHINE) - # - # ----------------------------------------------------------------------- - # - # Source constants.sh and save its contents to a variable for later - # - # ----------------------------------------------------------------------- - # + # Load constants file and save its contents to a variable for later cfg_c = load_config_file(os.path.join(USHdir, CONSTANTS_FN)) import_vars(dictionary=flatten_dict(cfg_c)) # # ----------------------------------------------------------------------- # - # Generate a unique number for this workflow run. This maybe used to + # Generate a unique number for this workflow run. This may be used to # get unique log file names for example # # ----------------------------------------------------------------------- # global WORKFLOW_ID WORKFLOW_ID = "id_" + str(int(datetime.datetime.now().timestamp())) - print_info_msg(f"""WORKFLOW ID = {WORKFLOW_ID}""") + log_info(f"""WORKFLOW ID = {WORKFLOW_ID}""") # # ----------------------------------------------------------------------- @@ -208,7 +221,7 @@ def get_location(xcs,fmt): # global VERBOSE if DEBUG and not VERBOSE: - print_info_msg( + log_info( """ Resetting VERBOSE to \"TRUE\" because DEBUG has been set to \"TRUE\"...""" ) @@ -289,12 +302,20 @@ def get_location(xcs,fmt): or (len(SPP_STDDEV_CUTOFF) != N_VAR_SPP) or (len(ISEED_SPP) != N_VAR_SPP) ): - print_err_msg_exit( + raise Exception( f''' All MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or RRTMG SPP-related namelist - variables set in {CONFIG_FN} must be equal in number of entries to what is + variables set in {EXPT_CONFIG_FN} must be equal in number of entries to what is found in SPP_VAR_LIST: - Number of entries in SPP_VAR_LIST = \"{len(SPP_VAR_LIST)}\"''' + SPP_VAR_LIST (length {len(SPP_VAR_LIST)}) + SPP_MAG_LIST (length {len(SPP_MAG_LIST)}) + SPP_LSCALE (length {len(SPP_LSCALE)}) + SPP_TSCALE (length {len(SPP_TSCALE)}) + SPP_SIGTOP1 (length {len(SPP_SIGTOP1)}) + SPP_SIGTOP2 (length {len(SPP_SIGTOP2)}) + SPP_STDDEV_CUTOFF (length {len(SPP_STDDEV_CUTOFF)}) + ISEED_SPP (length {len(ISEED_SPP)}) + ''' ) # # ----------------------------------------------------------------------- @@ -311,12 +332,16 @@ def get_location(xcs,fmt): or (len(LSM_SPP_LSCALE) != N_VAR_LNDP) or (len(LSM_SPP_TSCALE) != N_VAR_LNDP) ): - print_err_msg_exit( + raise Exception( f''' All Noah or RUC-LSM SPP-related namelist variables (except ISEED_LSM_SPP) - set in {CONFIG_FN} must be equal in number of entries to what is found in + set in {EXPT_CONFIG_FN} must be equal in number of entries to what is found in SPP_VAR_LIST: - Number of entries in SPP_VAR_LIST = \"{len(LSM_SPP_VAR_LIST)}\"''' + LSM_SPP_VAR_LIST (length {len(LSM_SPP_VAR_LIST)}) + LSM_SPP_MAG_LIST (length {len(LSM_SPP_MAG_LIST)}) + LSM_SPP_LSCALE (length {len(LSM_SPP_LSCALE)}) + LSM_SPP_TSCALE (length {len(LSM_SPP_TSCALE)}) + ''' ) # # The current script should be located in the ush subdirectory of the @@ -345,37 +370,39 @@ def get_location(xcs,fmt): mng_extrns_cfg_fn = os.readlink(mng_extrns_cfg_fn) except: pass - property_name = "local_path" cfg = load_ini_config(mng_extrns_cfg_fn) + # # Get the base directory of the FV3 forecast model code. # external_name = FCST_MODEL - UFS_WTHR_MDL_DIR = get_ini_value(cfg, external_name, property_name) + property_name = "local_path" - if not UFS_WTHR_MDL_DIR: - print_err_msg_exit( - f""" - Externals.cfg does not contain "{external_name}".""" - ) + try: + UFS_WTHR_MDL_DIR = get_ini_value(cfg, external_name, property_name) + except KeyError: + errmsg = dedent(f''' + Externals configuration file {mng_extrns_cfg_fn} + does not contain "{external_name}".''') + raise Exception(errmsg) from None + UFS_WTHR_MDL_DIR = os.path.join(HOMEdir, UFS_WTHR_MDL_DIR) if not os.path.exists(UFS_WTHR_MDL_DIR): - print_err_msg_exit( + raise FileNotFoundError(dedent( f""" The base directory in which the FV3 source code should be located (UFS_WTHR_MDL_DIR) does not exist: UFS_WTHR_MDL_DIR = \"{UFS_WTHR_MDL_DIR}\" Please clone the external repository containing the code in this directory, build the executable, and then rerun the workflow.""" - ) + )) # # Define some other useful paths # global SCRIPTSdir, JOBSdir, SORCdir, PARMdir, MODULESdir global EXECdir, PARMdir, FIXdir, VX_CONFIG_DIR, METPLUS_CONF, MET_CONFIG - USHdir = os.path.join(HOMEdir, "ush") SCRIPTSdir = os.path.join(HOMEdir, "scripts") JOBSdir = os.path.join(HOMEdir, "jobs") SORCdir = os.path.join(HOMEdir, "sorc") @@ -400,26 +427,18 @@ def get_location(xcs,fmt): RELATIVE_LINK_FLAG = "--relative" - if not NCORES_PER_NODE: - print_err_msg_exit( - f""" - NCORES_PER_NODE has not been specified in the file {MACHINE_FILE} - Please ensure this value has been set for your desired platform. """ - ) - - if not (FIXgsm and FIXaer and FIXlut and TOPO_DIR and SFC_CLIMO_INPUT_DIR): - print_err_msg_exit( - f""" - One or more fix file directories have not been specified for this machine: - MACHINE = \"{MACHINE}\" - FIXgsm = \"{FIXgsm or ""} - FIXaer = \"{FIXaer or ""} - FIXlut = \"{FIXlut or ""} - TOPO_DIR = \"{TOPO_DIR or ""} - SFC_CLIMO_INPUT_DIR = \"{SFC_CLIMO_INPUT_DIR or ""} - DOMAIN_PREGEN_BASEDIR = \"{DOMAIN_PREGEN_BASEDIR or ""} - You can specify the missing location(s) in config.sh""" - ) + # Mandatory variables *must* be set in the user's config or the machine file; the default value is invalid + mandatory = ['NCORES_PER_NODE', 'FIXgsm', 'FIXaer', 'FIXlut', 'TOPO_DIR', 'SFC_CLIMO_INPUT_DIR'] + globalvars = globals() + for val in mandatory: + # globals() returns dictionary of global variables + if not globalvars[val]: + raise Exception(dedent(f''' + Mandatory variable "{val}" not found in: + user config file {EXPT_CONFIG_FN} + OR + machine file {MACHINE_FILE} + ''')) # # ----------------------------------------------------------------------- @@ -459,12 +478,10 @@ def get_location(xcs,fmt): # if WORKFLOW_MANAGER is not None: if not ACCOUNT: - print_err_msg_exit( - f''' - The variable ACCOUNT cannot be empty if you are using a workflow manager: - ACCOUNT = \"ACCOUNT\" - WORKFLOW_MANAGER = \"WORKFLOW_MANAGER\"''' - ) + raise Exception(dedent(f''' + ACCOUNT must be specified in config or machine file if using a workflow manager. + WORKFLOW_MANAGER = {WORKFLOW_MANAGER}\n''' + )) # # ----------------------------------------------------------------------- # @@ -481,12 +498,7 @@ def get_location(xcs,fmt): GTYPE = "regional" TILE_RGNL = "7" - # ----------------------------------------------------------------------- - # - # Set USE_MERRA_CLIMO to either "TRUE" or "FALSE" so we don't - # have to consider other valid values later on. - # - # ----------------------------------------------------------------------- + # USE_MERRA_CLIMO must be True for the physics suite FV3_GFS_v15_thompson_mynn_lam3km" global USE_MERRA_CLIMO if CCPP_PHYS_SUITE == "FV3_GFS_v15_thompson_mynn_lam3km": USE_MERRA_CLIMO = True @@ -503,123 +515,64 @@ def get_location(xcs,fmt): elif FCST_MODEL == "fv3gfs_aqm": CPL = True else: - print_err_msg_exit( + raise Exception( f''' The coupling flag CPL has not been specified for this value of FCST_MODEL: FCST_MODEL = \"{FCST_MODEL}\"''' ) - # - # ----------------------------------------------------------------------- - # - # Make sure RESTART_INTERVAL is set to an integer value if present - # - # ----------------------------------------------------------------------- - # + + # Make sure RESTART_INTERVAL is set to an integer value if not isinstance(RESTART_INTERVAL, int): - print_err_msg_exit( - f''' - RESTART_INTERVAL must be set to an integer number of hours. - RESTART_INTERVAL = \"{RESTART_INTERVAL}\"''' - ) - # - # ----------------------------------------------------------------------- - # - # Check that DATE_FIRST_CYCL and DATE_LAST_CYCL are strings consisting - # of exactly 10 digits. - # - # ----------------------------------------------------------------------- - # - if not isinstance(DATE_FIRST_CYCL, datetime.date): - print_err_msg_exit( - f''' - DATE_FIRST_CYCL must be a string consisting of exactly 10 digits of the - form \"YYYYMMDDHH\", where YYYY is the 4-digit year, MM is the 2-digit - month, DD is the 2-digit day-of-month, and HH is the 2-digit - cycle hour. - DATE_FIRST_CYCL = \"{DATE_FIRST_CYCL}\"''' - ) + raise Exception(f"\nRESTART_INTERVAL = {RESTART_INTERVAL}, must be an integer value\n") - if not isinstance(DATE_LAST_CYCL, datetime.date): - print_err_msg_exit( - f''' - DATE_LAST_CYCL must be a string consisting of exactly 10 digits of the - form \"YYYYMMDDHH\", where YYYY is the 4-digit year, MM is the 2-digit - month, DD is the 2-digit day-of-month, and HH is the 2-digit - cycle hour. - DATE_LAST_CYCL = \"{DATE_LAST_CYCL}\"''' - ) - # - # ----------------------------------------------------------------------- - # - # Call a function to generate the array ALL_CDATES containing the cycle - # dates/hours for which to run forecasts. The elements of this array - # will have the form YYYYMMDDHH. They are the starting dates/times of - # the forecasts that will be run in the experiment. Then set NUM_CYCLES - # to the number of elements in this array. - # - # ----------------------------------------------------------------------- - # + # Check that input dates are in a date format - ALL_CDATES = set_cycle_dates( - date_start=DATE_FIRST_CYCL, - date_end=DATE_LAST_CYCL, - incr_cycl_freq=INCR_CYCL_FREQ, - ) + # get dictionary of all variables + allvars = dict(globals()) + allvars.update(locals()) + dates = ['DATE_FIRST_CYCL', 'DATE_LAST_CYCL'] + for val in dates: + if not isinstance(allvars[val], datetime.date): + raise Exception(dedent(f''' + Date variable {val}={allvars[val]} is not in a valid date format - NUM_CYCLES = len(ALL_CDATES) + For examples of valid formats, see the users guide. + ''')) - # Completely arbitrary cutoff of 90 cycles. - if NUM_CYCLES > 90: - ALL_CDATES = None - print_info_msg( - f""" - Too many cycles in ALL_CDATES to list, redefining in abbreviated form." - ALL_CDATES="{DATE_FIRST_CYCL}...{DATE_LAST_CYCL}""" - ) - # - # ----------------------------------------------------------------------- - # # If using a custom post configuration file, make sure that it exists. - # - # ----------------------------------------------------------------------- - # if USE_CUSTOM_POST_CONFIG_FILE: - if not os.path.exists(CUSTOM_POST_CONFIG_FP): - print_err_msg_exit( + try: + #os.path.exists returns exception if passed an empty string or None, so use "try/except" as a 2-for-1 error catch + if not os.path.exists(CUSTOM_POST_CONFIG_FP): + raise + except: + raise FileNotFoundError(dedent( f''' - The custom post configuration specified by CUSTOM_POST_CONFIG_FP does not - exist: - CUSTOM_POST_CONFIG_FP = \"{CUSTOM_POST_CONFIG_FP}\"''' - ) - # - # ----------------------------------------------------------------------- - # + USE_CUSTOM_POST_CONFIG_FILE has been set, but the custom post configuration file + CUSTOM_POST_CONFIG_FP = {CUSTOM_POST_CONFIG_FP} + could not be found.''' + )) from None + # If using external CRTM fix files to allow post-processing of synthetic - # satellite products from the UPP, then make sure the fix file directory - # exists. - # - # ----------------------------------------------------------------------- - # + # satellite products from the UPP, make sure the CRTM fix file directory exists. if USE_CRTM: - if not os.path.exists(CRTM_DIR): - print_err_msg_exit( + try: + #os.path.exists returns exception if passed an empty string or None, so use "try/except" as a 2-for-1 error catch + if not os.path.exists(CRTM_DIR): + raise + except: + raise FileNotFoundError(dedent( f''' - The external CRTM fix file directory specified by CRTM_DIR does not exist: - CRTM_DIR = \"{CRTM_DIR}\"''' - ) - # - # ----------------------------------------------------------------------- - # - # The forecast length (in integer hours) cannot contain more than 3 cha- - # racters. Thus, its maximum value is 999. Check whether the specified - # forecast length exceeds this maximum value. If so, print out a warn- - # ing and exit this script. - # - # ----------------------------------------------------------------------- - # + USE_CRTM has been set, but the external CRTM fix file directory: + CRTM_DIR = {CRTM_DIR} + could not be found.''' + )) from None + + # The forecast length (in integer hours) cannot contain more than 3 characters. + # Thus, its maximum value is 999. fcst_len_hrs_max = 999 if FCST_LEN_HRS > fcst_len_hrs_max: - print_err_msg_exit( + raise ValueError( f""" Forecast length is greater than maximum allowed length: FCST_LEN_HRS = {FCST_LEN_HRS} @@ -629,16 +582,15 @@ def get_location(xcs,fmt): # ----------------------------------------------------------------------- # # Check whether the forecast length (FCST_LEN_HRS) is evenly divisible - # by the BC update interval (LBC_SPEC_INTVL_HRS). If not, print out a - # warning and exit this script. If so, generate an array of forecast - # hours at which the boundary values will be updated. + # by the BC update interval (LBC_SPEC_INTVL_HRS). If so, generate an + # array of forecast hours at which the boundary values will be updated. # # ----------------------------------------------------------------------- # rem = FCST_LEN_HRS % LBC_SPEC_INTVL_HRS if rem != 0: - print_err_msg_exit( + raise Exception( f""" The forecast length (FCST_LEN_HRS) is not evenly divisible by the lateral boundary conditions update interval (LBC_SPEC_INTVL_HRS): @@ -672,48 +624,18 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- # - if not DT_ATMOS: - print_err_msg_exit( - f''' - The forecast model main time step (DT_ATMOS) is set to a null string: - DT_ATMOS = {DT_ATMOS} - Please set this to a valid numerical value in the user-specified experiment - configuration file (EXPT_CONFIG_FP) and rerun: - EXPT_CONFIG_FP = \"{EXPT_CONFIG_FP}\"''' - ) - - if not LAYOUT_X: - print_err_msg_exit( - f''' - The number of MPI processes to be used in the x direction (LAYOUT_X) by - the forecast job is set to a null string: - LAYOUT_X = {LAYOUT_X} - Please set this to a valid numerical value in the user-specified experiment - configuration file (EXPT_CONFIG_FP) and rerun: - EXPT_CONFIG_FP = \"{EXPT_CONFIG_FP}\"''' - ) + # get dictionary of all variables + allvars = dict(globals()) + allvars.update(locals()) + vlist = ['DT_ATMOS', + 'LAYOUT_X', + 'LAYOUT_Y', + 'BLOCKSIZE', + 'EXPT_SUBDIR'] + for val in vlist: + if not allvars[val]: + raise Exception(f"\nMandatory variable '{val}' has not been set\n") - if not LAYOUT_Y: - print_err_msg_exit( - f''' - The number of MPI processes to be used in the y direction (LAYOUT_Y) by - the forecast job is set to a null string: - LAYOUT_Y = {LAYOUT_Y} - Please set this to a valid numerical value in the user-specified experiment - configuration file (EXPT_CONFIG_FP) and rerun: - EXPT_CONFIG_FP = \"{EXPT_CONFIG_FP}\"''' - ) - - if not BLOCKSIZE: - print_err_msg_exit( - f''' - The cache size to use for each MPI task of the forecast (BLOCKSIZE) is - set to a null string: - BLOCKSIZE = {BLOCKSIZE} - Please set this to a valid numerical value in the user-specified experiment - configuration file (EXPT_CONFIG_FP) and rerun: - EXPT_CONFIG_FP = \"{EXPT_CONFIG_FP}\"''' - ) # # ----------------------------------------------------------------------- # @@ -730,7 +652,7 @@ def get_location(xcs,fmt): # Check that DT_SUBHOURLY_POST_MNTS is between 0 and 59, inclusive. # if DT_SUBHOURLY_POST_MNTS < 0 or DT_SUBHOURLY_POST_MNTS > 59: - print_err_msg_exit( + raise ValueError( f''' When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), DT_SUBHOURLY_POST_MNTS must be set to an integer between 0 and 59, @@ -744,7 +666,7 @@ def get_location(xcs,fmt): # rem = DT_SUBHOURLY_POST_MNTS * 60 % DT_ATMOS if rem != 0: - print_err_msg_exit( + raise ValueError( f""" When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), the time interval specified by DT_SUBHOURLY_POST_MNTS (after converting @@ -765,7 +687,7 @@ def get_location(xcs,fmt): # informational message that such a change was made. # if DT_SUBHOURLY_POST_MNTS == 0: - print_info_msg( + logger.warning( f""" When performing sub-hourly post (i.e. SUB_HOURLY_POST set to \"TRUE\"), DT_SUBHOURLY_POST_MNTS must be set to a value greater than 0; otherwise, @@ -804,20 +726,7 @@ def get_location(xcs,fmt): EXPT_BASEDIR = os.path.abspath(EXPT_BASEDIR) mkdir_vrfy(f' -p "{EXPT_BASEDIR}"') - # - # ----------------------------------------------------------------------- - # - # If the experiment subdirectory name (EXPT_SUBDIR) is set to an empty - # string, print out an error message and exit. - # - # ----------------------------------------------------------------------- - # - if not EXPT_SUBDIR: - print_err_msg_exit( - f''' - The name of the experiment subdirectory (EXPT_SUBDIR) cannot be empty: - EXPT_SUBDIR = \"{EXPT_SUBDIR}\"''' - ) + # # ----------------------------------------------------------------------- # @@ -828,7 +737,25 @@ def get_location(xcs,fmt): # global EXPTDIR EXPTDIR = os.path.join(EXPT_BASEDIR, EXPT_SUBDIR) - check_for_preexist_dir_file(EXPTDIR, PREEXISTING_DIR_METHOD) + try: + check_for_preexist_dir_file(EXPTDIR, PREEXISTING_DIR_METHOD) + except ValueError: + logger.exception(f''' + Check that the following values are valid: + EXPTDIR {EXPTDIR} + PREEXISTING_DIR_METHOD {PREEXISTING_DIR_METHOD} + ''') + raise + except FileExistsError: + errmsg = dedent(f''' + EXPTDIR ({EXPTDIR}) already exists, and PREEXISTING_DIR_METHOD = {PREEXISTING_DIR_METHOD} + + To ignore this error, delete the directory, or set + PREEXISTING_DIR_METHOD = delete, or + PREEXISTING_DIR_METHOD = rename + in your config file. + ''') + raise FileExistsError(errmsg) from None # # ----------------------------------------------------------------------- # @@ -945,7 +872,7 @@ def get_location(xcs,fmt): if POST_OUTPUT_DOMAIN_NAME is None: if PREDEF_GRID_NAME is None: - print_err_msg_exit( + raise Exception( f""" The domain name used in naming the run_post output files (POST_OUTPUT_DOMAIN_NAME) has not been set: @@ -966,9 +893,6 @@ def get_location(xcs,fmt): # (4) The FV3 namelist file # (5) The model configuration file # (6) The NEMS configuration file - # - # If using CCPP, it also needs: - # # (7) The CCPP physics suite definition file # # The workflow contains templates for the first six of these files. @@ -1056,7 +980,7 @@ def get_location(xcs,fmt): ) CCPP_PHYS_SUITE_FP = os.path.join(EXPTDIR, CCPP_PHYS_SUITE_FN) if not os.path.exists(CCPP_PHYS_SUITE_IN_CCPP_FP): - print_err_msg_exit( + raise FileNotFoundError( f''' The CCPP suite definition file (CCPP_PHYS_SUITE_IN_CCPP_FP) does not exist in the local clone of the ufs-weather-model: @@ -1083,7 +1007,7 @@ def get_location(xcs,fmt): ) FIELD_DICT_FP = os.path.join(EXPTDIR, FIELD_DICT_FN) if not os.path.exists(FIELD_DICT_IN_UWM_FP): - print_err_msg_exit( + raise FileNotFoundError( f''' The field dictionary file (FIELD_DICT_IN_UWM_FP) does not exist in the local clone of the ufs-weather-model: @@ -1101,7 +1025,7 @@ def get_location(xcs,fmt): # export env vars before calling another module export_vars() - OZONE_PARAM = set_ozone_param(ccpp_phys_suite_fp=CCPP_PHYS_SUITE_IN_CCPP_FP) + OZONE_PARAM = set_ozone_param(CCPP_PHYS_SUITE_IN_CCPP_FP) IMPORTS = ["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING", "FIXgsm_FILES_TO_COPY_TO_FIXam"] import_vars(env_vars=IMPORTS) @@ -1159,7 +1083,7 @@ def get_location(xcs,fmt): idx = len(EXTRN_MDL_SOURCE_BASEDIR_ICS) if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_ICS[:idx]): - print_err_msg_exit( + raise FileNotFoundError( f''' The directory (EXTRN_MDL_SOURCE_BASEDIR_ICS) in which the user-staged external model files for generating ICs should be located does not exist: @@ -1171,7 +1095,7 @@ def get_location(xcs,fmt): idx = len(EXTRN_MDL_SOURCE_BASEDIR_LBCS) if not os.path.exists(EXTRN_MDL_SOURCE_BASEDIR_LBCS[:idx]): - print_err_msg_exit( + raise FileNotFoundError( f''' The directory (EXTRN_MDL_SOURCE_BASEDIR_LBCS) in which the user-staged external model files for generating LBCs should be located does not exist: @@ -1180,10 +1104,9 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- # - # Make sure that DO_ENSEMBLE is set to a valid value. Then set the names - # of the ensemble members. These will be used to set the ensemble member - # directories. Also, set the full path to the FV3 namelist file corresponding - # to each ensemble member. + # If DO_ENSEMBLE, set the names of the ensemble members; these will be + # used to set the ensemble member directories. Also, set the full path + # to the FV3 namelist file corresponding to each ensemble member. # # ----------------------------------------------------------------------- # @@ -1199,13 +1122,8 @@ def get_location(xcs,fmt): FV3_NML_ENSMEM_FPS.append( os.path.join(EXPTDIR, f"{FV3_NML_FN}_{ENSMEM_NAMES[i]}") ) - # - # ----------------------------------------------------------------------- - # + # Set the full path to the forecast model executable. - # - # ----------------------------------------------------------------------- - # global FV3_EXEC_FP FV3_EXEC_FP = os.path.join(EXECdir, FV3_EXEC_FN) # @@ -1240,13 +1158,6 @@ def get_location(xcs,fmt): global LOAD_MODULES_RUN_TASK_FP LOAD_MODULES_RUN_TASK_FP = os.path.join(USHdir, "load_modules_run_task.sh") - # - # ----------------------------------------------------------------------- - # - # Turn off some tasks that can not be run in NCO mode - # - # ----------------------------------------------------------------------- - # global RUN_TASK_MAKE_GRID, RUN_TASK_MAKE_OROG, RUN_TASK_MAKE_SFC_CLIMO global RUN_TASK_VX_GRIDSTAT, RUN_TASK_VX_POINTSTAT, RUN_TASK_VX_ENSGRID, RUN_TASK_VX_ENSPOINT @@ -1260,15 +1171,9 @@ def get_location(xcs,fmt): FIXclim = os.path.join(FIXdir, "fix_clim") FIXlam = os.path.join(FIXdir, "fix_lam") - # - # ----------------------------------------------------------------------- - # - # Make sure that DO_ENSEMBLE is set to TRUE when running ensemble vx. - # - # ----------------------------------------------------------------------- - # + # Ensemble verification can only be run in ensemble mode if (not DO_ENSEMBLE) and (RUN_TASK_VX_ENSGRID or RUN_TASK_VX_ENSPOINT): - print_err_msg_exit( + raise Exception( f''' Ensemble verification can not be run unless running in ensemble mode: DO_ENSEMBLE = \"{DO_ENSEMBLE}\" @@ -1314,21 +1219,22 @@ def get_location(xcs,fmt): # experiment directory (EXPTDIR). # if not RUN_TASK_MAKE_GRID: - if (GRID_DIR is None) or (not os.path.exists(GRID_DIR)): + if (GRID_DIR is None): GRID_DIR = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) - msg = f"""Setting GRID_DIR to: - GRID_DIR = \"{GRID_DIR}\" - """ - print_info_msg(msg) - - if not os.path.exists(GRID_DIR): - print_err_msg_exit( - f''' - The directory (GRID_DIR) that should contain the pregenerated grid files - does not exist: - GRID_DIR = \"{GRID_DIR}\"''' - ) + msg = dedent(f""" + GRID_DIR not specified! + Setting GRID_DIR = {GRID_DIR} + """) + logger.warning(msg) + + if not os.path.exists(GRID_DIR): + raise FileNotFoundError( + f''' + The directory (GRID_DIR) that should contain the pregenerated grid files + does not exist: + GRID_DIR = \"{GRID_DIR}\"''' + ) else: GRID_DIR = os.path.join(EXPTDIR, "grid") # @@ -1338,21 +1244,22 @@ def get_location(xcs,fmt): # the experiment directory (EXPTDIR). # if not RUN_TASK_MAKE_OROG: - if (OROG_DIR is None) or (not os.path.exists(OROG_DIR)): + if (OROG_DIR is None): OROG_DIR = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) - msg = f"""Setting OROG_DIR to: - OROG_DIR = \"{OROG_DIR}\" - """ - print_info_msg(msg) - - if not os.path.exists(OROG_DIR): - print_err_msg_exit( - f''' - The directory (OROG_DIR) that should contain the pregenerated orography - files does not exist: - OROG_DIR = \"{OROG_DIR}\"''' - ) + msg = dedent(f""" + OROG_DIR not specified! + Setting OROG_DIR = {OROG_DIR} + """) + logger.warning(msg) + + if not os.path.exists(OROG_DIR): + raise FileNotFoundError( + f''' + The directory (OROG_DIR) that should contain the pregenerated orography + files does not exist: + OROG_DIR = \"{OROG_DIR}\"''' + ) else: OROG_DIR = os.path.join(EXPTDIR, "orog") # @@ -1362,21 +1269,22 @@ def get_location(xcs,fmt): # a predefined location under the experiment directory (EXPTDIR). # if not RUN_TASK_MAKE_SFC_CLIMO: - if (SFC_CLIMO_DIR is None) or (not os.path.exists(SFC_CLIMO_DIR)): + if (SFC_CLIMO_DIR is None): SFC_CLIMO_DIR = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) - msg = f"""Setting SFC_CLIMO_DIR to: - SFC_CLIMO_DIR = \"{SFC_CLIMO_DIR}\" - """ - print_info_msg(msg) - - if not os.path.exists(SFC_CLIMO_DIR): - print_err_msg_exit( - f''' - The directory (SFC_CLIMO_DIR) that should contain the pregenerated surface - climatology files does not exist: - SFC_CLIMO_DIR = \"{SFC_CLIMO_DIR}\"''' - ) + msg = dedent(f""" + SFC_CLIMO_DIR not specified! + Setting SFC_CLIMO_DIR ={SFC_CLIMO_DIR} + """) + logger.warning(msg) + + if not os.path.exists(SFC_CLIMO_DIR): + raise FileNotFoundError( + f''' + The directory (SFC_CLIMO_DIR) that should contain the pregenerated surface + climatology files does not exist: + SFC_CLIMO_DIR = \"{SFC_CLIMO_DIR}\"''' + ) else: SFC_CLIMO_DIR = os.path.join(EXPTDIR, "sfc_climo") @@ -1462,10 +1370,9 @@ def get_location(xcs,fmt): # # ----------------------------------------------------------------------- # - # Create a new experiment directory. Note that at this point we are - # guaranteed that there is no preexisting experiment directory. For - # platforms with no workflow manager, we need to create LOGDIR as well, - # since it won't be created later at runtime. + # Create a new experiment directory. For platforms with no workflow + # manager we need to create LOGDIR as well, since it won't be created + # later at runtime. # # ----------------------------------------------------------------------- # @@ -1473,7 +1380,7 @@ def get_location(xcs,fmt): mkdir_vrfy(f' -p "{LOGDIR}"') # # ----------------------------------------------------------------------- - # + # NOTE: currently this is executed no matter what, should it be dependent on the logic described below?? # If not running the MAKE_GRID_TN, MAKE_OROG_TN, and/or MAKE_SFC_CLIMO # tasks, create symlinks under the FIXlam directory to pregenerated grid, # orography, and surface climatology files. In the process, also set @@ -1520,7 +1427,7 @@ def get_location(xcs,fmt): res_in_orog_fns = link_fix(verbose=VERBOSE, file_group="orog") if not RES_IN_FIXLAM_FILENAMES and (res_in_orog_fns != RES_IN_FIXLAM_FILENAMES): - print_err_msg_exit( + raise Exception( f""" The resolution extracted from the orography file names (res_in_orog_fns) does not match the resolution in other groups of files already consi- @@ -1546,7 +1453,7 @@ def get_location(xcs,fmt): res_in_sfc_climo_fns = link_fix(verbose=VERBOSE, file_group="sfc_climo") if RES_IN_FIXLAM_FILENAMES and res_in_sfc_climo_fns != RES_IN_FIXLAM_FILENAMES: - print_err_msg_exit( + raise Exception( f""" The resolution extracted from the surface climatology file names (res_- in_sfc_climo_fns) does not match the resolution in other groups of files @@ -1569,21 +1476,20 @@ def get_location(xcs,fmt): CRES = "" if not RUN_TASK_MAKE_GRID: CRES = f"C{RES_IN_FIXLAM_FILENAMES}" - # - # ----------------------------------------------------------------------- - # - # Make sure that WRITE_DOPOST is set to a valid value. - # - # ----------------------------------------------------------------------- - # + global RUN_TASK_RUN_POST if WRITE_DOPOST: # Turn off run_post - RUN_TASK_RUN_POST = False + if RUN_TASK_RUN_POST: + logger.warning(dedent(f""" + Inline post is turned on, deactivating post-processing tasks: + RUN_TASK_RUN_POST = False + """)) + RUN_TASK_RUN_POST = False # Check if SUB_HOURLY_POST is on if SUB_HOURLY_POST: - print_err_msg_exit( + raise Exception( f""" SUB_HOURLY_POST is NOT available with Inline Post yet.""" ) @@ -1601,7 +1507,8 @@ def get_location(xcs,fmt): if QUILTING: PE_MEMBER01 = PE_MEMBER01 + WRTCMP_write_groups * WRTCMP_write_tasks_per_group - print_info_msg( + if VERBOSE: + log_info( f""" The number of MPI tasks for the forecast (including those for the write component if it is being used) are: @@ -1857,72 +1764,50 @@ def get_location(xcs,fmt): # the make_grid task is complete. # "CRES": CRES, + # + # ----------------------------------------------------------------------- + # + # Flag in the \"{MODEL_CONFIG_FN}\" file for coupling the ocean model to + # the weather model. + # + # ----------------------------------------------------------------------- + # + "CPL": CPL, + # + # ----------------------------------------------------------------------- + # + # Name of the ozone parameterization. The value this gets set to depends + # on the CCPP physics suite being used. + # + # ----------------------------------------------------------------------- + # + "OZONE_PARAM": OZONE_PARAM, + # + # ----------------------------------------------------------------------- + # + # Computational parameters. + # + # ----------------------------------------------------------------------- + # + "PE_MEMBER01": PE_MEMBER01, + # + # ----------------------------------------------------------------------- + # + # IF DO_SPP is set to "TRUE", N_VAR_SPP specifies the number of physics + # parameterizations that are perturbed with SPP. If DO_LSM_SPP is set to + # "TRUE", N_VAR_LNDP specifies the number of LSM parameters that are + # perturbed. LNDP_TYPE determines the way LSM perturbations are employed + # and FHCYC_LSM_SPP_OR_NOT sets FHCYC based on whether LSM perturbations + # are turned on or not. + # + # ----------------------------------------------------------------------- + # + "N_VAR_SPP": N_VAR_SPP, + "N_VAR_LNDP": N_VAR_LNDP, + "LNDP_TYPE": LNDP_TYPE, + "LNDP_MODEL_TYPE": LNDP_MODEL_TYPE, + "FHCYC_LSM_SPP_OR_NOT": FHCYC_LSM_SPP_OR_NOT, } - # - # ----------------------------------------------------------------------- - # - # Continue appending variable definitions to the variable definitions - # file. - # - # ----------------------------------------------------------------------- - # - settings.update( - { - # - # ----------------------------------------------------------------------- - # - # Flag in the \"{MODEL_CONFIG_FN}\" file for coupling the ocean model to - # the weather model. - # - # ----------------------------------------------------------------------- - # - "CPL": CPL, - # - # ----------------------------------------------------------------------- - # - # Name of the ozone parameterization. The value this gets set to depends - # on the CCPP physics suite being used. - # - # ----------------------------------------------------------------------- - # - "OZONE_PARAM": OZONE_PARAM, - # - # ----------------------------------------------------------------------- - # - # The number of cycles for which to make forecasts and the list of - # starting dates/hours of these cycles. - # - # ----------------------------------------------------------------------- - # - "NUM_CYCLES": NUM_CYCLES, - "ALL_CDATES": ALL_CDATES, - # - # ----------------------------------------------------------------------- - # - # Computational parameters. - # - # ----------------------------------------------------------------------- - # - "PE_MEMBER01": PE_MEMBER01, - # - # ----------------------------------------------------------------------- - # - # IF DO_SPP is set to "TRUE", N_VAR_SPP specifies the number of physics - # parameterizations that are perturbed with SPP. If DO_LSM_SPP is set to - # "TRUE", N_VAR_LNDP specifies the number of LSM parameters that are - # perturbed. LNDP_TYPE determines the way LSM perturbations are employed - # and FHCYC_LSM_SPP_OR_NOT sets FHCYC based on whether LSM perturbations - # are turned on or not. - # - # ----------------------------------------------------------------------- - # - "N_VAR_SPP": N_VAR_SPP, - "N_VAR_LNDP": N_VAR_LNDP, - "LNDP_TYPE": LNDP_TYPE, - "LNDP_MODEL_TYPE": LNDP_MODEL_TYPE, - "FHCYC_LSM_SPP_OR_NOT": FHCYC_LSM_SPP_OR_NOT, - } - ) # write derived settings cfg_d["derived"] = settings @@ -1964,10 +1849,10 @@ def get_location(xcs,fmt): # print content of var_defns if DEBUG=True all_lines = cfg_to_yaml_str(cfg_d) - print_info_msg(all_lines, verbose=DEBUG) + log_info(all_lines, verbose=DEBUG) # print info message - print_info_msg( + log_info( f""" Generating the global experiment variable definitions file specified by GLOBAL_VAR_DEFNS_FN: @@ -2000,27 +1885,13 @@ def get_location(xcs,fmt): continue vkey = "valid_vals_" + k if (vkey in cfg_v) and not (v in cfg_v[vkey]): - print_err_msg_exit( + raise Exception( f""" - The variable {k}={v} in {EXPT_DEFAULT_CONFIG_FN} or {EXPT_CONFIG_FN} does not have - a valid value. Possible values are: + The variable {k}={v} in {EXPT_DEFAULT_CONFIG_FN} or {EXPT_CONFIG_FN} + does not have a valid value. Possible values are: {k} = {cfg_v[vkey]}""" ) - # - # ----------------------------------------------------------------------- - # - # Print message indicating successful completion of script. - # - # ----------------------------------------------------------------------- - # - print_info_msg( - f""" - ======================================================================== - Function setup() in \"{os.path.basename(__file__)}\" completed successfully!!! - ========================================================================""" - ) - # # -----------------------------------------------------------------------