diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 890e92b82b..92d01de481 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -15,7 +15,7 @@ pipeline { // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') // Use the line below to enable hera // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use') - choice(name: 'SRW_PLATFORM_FILTER', choices: ['cheyenne', 'gaea', 'jet', 'orion'], description: 'Specify the platform(s) to use') + choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'jet', 'orion'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') // Uncomment the following line to re-enable comprehensive tests @@ -78,7 +78,9 @@ pipeline { axes { axis { name 'SRW_PLATFORM' - values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + // Uncomment line below to re-add use of Hera + // values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'cheyenne', 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { diff --git a/tests/WE2E/monitor_jobs.py b/tests/WE2E/monitor_jobs.py index 9e34a87264..8fbd4f2afb 100755 --- a/tests/WE2E/monitor_jobs.py +++ b/tests/WE2E/monitor_jobs.py @@ -46,7 +46,7 @@ def monitor_jobs(expt_dict: dict, monitor_file: str = '', debug: bool = False) - logging.info("Checking tests available for monitoring...") for expt in expt_dict: logging.info(f"Starting experiment {expt} running") - expt_dict[expt] = update_expt_status(expt_dict[expt], expt) + expt_dict[expt] = update_expt_status(expt_dict[expt], expt, True) write_monitor_file(monitor_file,expt_dict) @@ -81,11 +81,11 @@ def monitor_jobs(expt_dict: dict, monitor_file: str = '', debug: bool = False) - endtime = datetime.now() total_walltime = endtime - starttime - logging.info(f'All {num_expts} experiments finished in {str(total_walltime)}') + logging.info(f'All {len(expt_dict)} experiments finished in {str(total_walltime)}') return monitor_file -def update_expt_status(expt: dict, name: str) -> dict: +def update_expt_status(expt: dict, name: str, refresh: bool = False) -> dict: """ This function reads the dictionary showing the location of a given experiment, runs a `rocotorun` command to update the experiment (running new jobs and updating the status of @@ -121,15 +121,18 @@ def update_expt_status(expt: dict, name: str) -> dict: to ensure there are no un-submitted jobs. We will no longer monitor this experiment. Args: - expt (dict): A dictionary containing the information for an individual experiment, as - described in the main monitor_jobs() function. - name (str): [optional] + expt (dict): A dictionary containing the information for an individual experiment, as + described in the main monitor_jobs() function. + name (str): Name of the experiment; used for logging only + refresh (bool): If true, this flag will check an experiment status even if it is listed + as DEAD, ERROR, or COMPLETE. Used for initial checks for experiments + that may have been restarted. Returns: dict: The updated experiment dictionary. """ #If we are no longer tracking this experiment, return unchanged - if expt["status"] in ['DEAD','ERROR','COMPLETE']: + if (expt["status"] in ['DEAD','ERROR','COMPLETE']) and not refresh: return expt # Update experiment, read rocoto database diff --git a/tests/WE2E/run_WE2E_tests.py b/tests/WE2E/run_WE2E_tests.py index dc472f8333..d12691da10 100755 --- a/tests/WE2E/run_WE2E_tests.py +++ b/tests/WE2E/run_WE2E_tests.py @@ -40,6 +40,11 @@ def run_we2e_tests(homedir, args) -> None: run_envir = args.run_envir machine = args.machine.lower() + # Check for invalid input + if run_envir: + if run_envir not in ['nco', 'community']: + raise KeyError(f"Invalid 'run_envir' provided: {run_envir}") + # If args.tests is a list of length more than one, we assume it is a list of test names if len(args.tests) > 1: tests_to_check=args.tests @@ -134,6 +139,10 @@ def run_we2e_tests(homedir, args) -> None: test_cfg['user'].update({"ACCOUNT": args.account}) if run_envir: test_cfg['user'].update({"RUN_ENVIR": run_envir}) + if run_envir == "nco": + if 'nco' not in test_cfg: + test_cfg['nco'] = dict() + test_cfg['nco'].update({"model_ver": "we2e"}) # if platform section was not in input config, initialize as empty dict if 'platform' not in test_cfg: test_cfg['platform'] = dict() @@ -170,7 +179,7 @@ def run_we2e_tests(homedir, args) -> None: with open(ushdir + "/config.yaml","w") as f: f.writelines(cfg_to_yaml_str(test_cfg)) - logging.debug(f"Calling workflow generation function for test {test_name}\n") + logging.info(f"Calling workflow generation function for test {test_name}\n") if args.quiet: console_handler = logging.getLogger().handlers[1] console_handler.setLevel(logging.WARNING) diff --git a/ush/bash_utils/change_case.sh b/ush/bash_utils/change_case.sh index 752ead9018..b8cc1c7db4 100644 --- a/ush/bash_utils/change_case.sh +++ b/ush/bash_utils/change_case.sh @@ -46,30 +46,6 @@ function echo_uppercase() { # local func_name="${FUNCNAME[0]}" # -#----------------------------------------------------------------------- -# -# Get information about the script or function that calls this function. -# Note that caller_name will be set as follows: -# -# 1) If the caller is a function, caller_name will be set to the name of -# that function. -# 2) If the caller is a sourced script, caller_name will be set to -# "script". Note that a sourced script cannot be the top level -# script since by defintion, it is sourced by another script or func- -# tion. -# 3) If the caller is the top-level script, caller_name will be set to -# "main". -# -# Thus, if caller_name is set to "script" or "main", the caller is a -# script, and if it is set to anything else, the caller is a function. -# -#----------------------------------------------------------------------- -# - local caller_fp=$( $READLINK -f "${BASH_SOURCE[1]}" ) - local caller_fn=$( basename "${caller_fp}" ) - local caller_dir=$( dirname "${caller_fp}" ) - local caller_name="${FUNCNAME[1]}" -# # Get input string local input @@ -161,30 +137,6 @@ function echo_lowercase() { # local func_name="${FUNCNAME[0]}" # -#----------------------------------------------------------------------- -# -# Get information about the script or function that calls this function. -# Note that caller_name will be set as follows: -# -# 1) If the caller is a function, caller_name will be set to the name of -# that function. -# 2) If the caller is a sourced script, caller_name will be set to -# "script". Note that a sourced script cannot be the top level -# script since by defintion, it is sourced by another script or func- -# tion. -# 3) If the caller is the top-level script, caller_name will be set to -# "main". -# -# Thus, if caller_name is set to "script" or "main", the caller is a -# script, and if it is set to anything else, the caller is a function. -# -#----------------------------------------------------------------------- -# - local caller_fp=$( $READLINK -f "${BASH_SOURCE[1]}" ) - local caller_fn=$( basename "${caller_fp}" ) - local caller_dir=$( dirname "${caller_fp}" ) - local caller_name="${FUNCNAME[1]}" -# # Get input string local input diff --git a/ush/bash_utils/filesys_cmds_vrfy.sh b/ush/bash_utils/filesys_cmds_vrfy.sh index 0f7d842467..b355d293ad 100644 --- a/ush/bash_utils/filesys_cmds_vrfy.sh +++ b/ush/bash_utils/filesys_cmds_vrfy.sh @@ -65,10 +65,14 @@ function filesys_cmd_vrfy() { # #----------------------------------------------------------------------- # - local caller_fp=$( $READLINK -f "${BASH_SOURCE[2]}" ) - local caller_fn=$( basename "${caller_fp}" ) - local caller_dir=$( dirname "${caller_fp}" ) - local caller_name="${FUNCNAME[2]}" + local caller_name="main" + local caller_fp="" + if [ -z "${BASH_SOURCE[2]-x}" ]; then + caller_fp=$( $READLINK -f "${BASH_SOURCE[2]}" ) + local caller_fn=$( basename "${caller_fp}" ) + local caller_dir=$( dirname "${caller_fp}" ) + caller_name="${FUNCNAME[2]}" + fi # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/print_msg.sh b/ush/bash_utils/print_msg.sh index aa66a9bd90..28a70d1431 100644 --- a/ush/bash_utils/print_msg.sh +++ b/ush/bash_utils/print_msg.sh @@ -47,30 +47,6 @@ function print_info_msg() { # #----------------------------------------------------------------------- # -# Get information about the script or function that calls this function. -# Note that caller_name will be set as follows: -# -# 1) If the caller is a function, caller_name will be set to the name of -# that function. -# 2) If the caller is a sourced script, caller_name will be set to -# "script". Note that a sourced script cannot be the top level -# script since by defintion, it is sourced by another script or func- -# tion. -# 3) If the caller is the top-level script, caller_name will be set to -# "main". -# -# Thus, if caller_name is set to "script" or "main", the caller is a -# script, and if it is set to anything else, the caller is a function. -# -#----------------------------------------------------------------------- -# - local caller_fp=$( $READLINK -f "${BASH_SOURCE[1]}" ) - local caller_fn=$( basename "${caller_fp}" ) - local caller_dir=$( dirname "${caller_fp}" ) - local caller_name="${FUNCNAME[1]}" -# -#----------------------------------------------------------------------- -# # Declare local variables. # #----------------------------------------------------------------------- diff --git a/ush/preamble.sh b/ush/preamble.sh index 4c75665ed6..0572905d55 100644 --- a/ush/preamble.sh +++ b/ush/preamble.sh @@ -104,6 +104,6 @@ if [[ $TRACE == "TRUE" ]]; then id="" fi # Print the script name and line number of each command as it is executed - export PS4='+ $(basename $BASH_SOURCE)[$LINENO]'"$id: " + export PS4='+ $(basename ${BASH_SOURCE:-$0})[$LINENO]'"$id: " fi diff --git a/ush/setup.py b/ush/setup.py index 993a63d47b..0550a16dfe 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -201,7 +201,7 @@ def load_config_for_setup(ushdir, default_config, user_config): raise Exception( dedent( f""" - Date variable {val}={cfg_d['user'][val]} is not in a valid date format. + Date variable {val}={cfg_d['workflow'][val]} is not in a valid date format. For examples of valid formats, see the Users' Guide. """