From c5b58573596a4a757f2557d7e2dc42f28b75fb23 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 9 Dec 2022 16:46:11 -0500 Subject: [PATCH 1/8] add python coding norms checking with pycodestyle --- .github/workflows/pynorms.yaml | 24 ++++++++++++++++++++++++ .pycodestyle | 6 ++++++ 2 files changed, 30 insertions(+) create mode 100644 .github/workflows/pynorms.yaml create mode 100644 .pycodestyle diff --git a/.github/workflows/pynorms.yaml b/.github/workflows/pynorms.yaml new file mode 100644 index 00000000000..90e71a5f918 --- /dev/null +++ b/.github/workflows/pynorms.yaml @@ -0,0 +1,24 @@ +name: Check Norms +on: [push, pull_request] + +jobs: + check_norms: + runs-on: ubuntu-latest + name: Check Python coding norms with pycodestyle + + steps: + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install pycodestyle + + - name: Checkout + uses: actions/checkout@v2 + with: + path: global-workflow + + - name: Run pycodestyle + run: | + cd $GITHUB_WORKSPACE/global-workflow + pycodestyle -v --config ./.pycodestyle . diff --git a/.pycodestyle b/.pycodestyle new file mode 100644 index 00000000000..162186cb7c2 --- /dev/null +++ b/.pycodestyle @@ -0,0 +1,6 @@ +[pycodestyle] +count = False +ignore = E226,E401,E402,W504 +max-line-length = 160 +statistics = True +exclude = Experimental From a0129ee54b92b509441e6a219b3b62a3183fc337 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 9 Dec 2022 22:00:35 -0500 Subject: [PATCH 2/8] fix scripts/exgfs_aero_init_aerosol.py --- scripts/exgfs_aero_init_aerosol.py | 406 +++++++++++++++-------------- 1 file changed, 206 insertions(+), 200 deletions(-) diff --git a/scripts/exgfs_aero_init_aerosol.py b/scripts/exgfs_aero_init_aerosol.py index ecd968ce2f3..db5e462f649 100755 --- a/scripts/exgfs_aero_init_aerosol.py +++ b/scripts/exgfs_aero_init_aerosol.py @@ -4,7 +4,7 @@ 'script'-level control of the aerosol init job. Reads environment variables, determines the atmospheric IC files and most recent available -restart files, then calls the script that merges the tracers from the restart files into +restart files, then calls the script that merges the tracers from the restart files into the IC files. INPUTS @@ -22,7 +22,7 @@ Additionally, the following data files are used: - Tiled atmospheric initial conditions that follow the naming pattern determined by `atm_base_pattern` and `atm_file_pattern` -- Restart files from a previous cycle that fit the pattern determined by restart_base_pattern and restart_file_pattern, +- Restart files from a previous cycle that fit the pattern determined by restart_base_pattern and restart_file_pattern, tracer_file_pattern, and dycore_file_pattern - A static file containing a list of tracers from the restart files to be added to the IC files, determined by `tracer_list_file_pattern` @@ -64,223 +64,229 @@ def main() -> None: - # Read in environment variables and make sure they exist - cdate = get_env_var("CDATE") - incr = int(get_env_var('STEP_GFS')) - fcst_length = int(get_env_var('FHMAX_GFS')) - cdump = get_env_var("CDUMP") - rot_dir = get_env_var("ROTDIR") - ush_gfs = get_env_var("USHgfs") - parm_gfs = get_env_var("PARMgfs") + # Read in environment variables and make sure they exist + cdate = get_env_var("CDATE") + incr = int(get_env_var('STEP_GFS')) + fcst_length = int(get_env_var('FHMAX_GFS')) + cdump = get_env_var("CDUMP") + rot_dir = get_env_var("ROTDIR") + ush_gfs = get_env_var("USHgfs") + parm_gfs = get_env_var("PARMgfs") - # os.chdir(data) + # os.chdir(data) - merge_script = merge_script_pattern.format(ush_gfs=ush_gfs) - tracer_list_file = tracer_list_file_pattern.format(parm_gfs=parm_gfs) + merge_script = merge_script_pattern.format(ush_gfs=ush_gfs) + tracer_list_file = tracer_list_file_pattern.format(parm_gfs=parm_gfs) - time = datetime.strptime(cdate, "%Y%m%d%H") - atm_source_path = time.strftime(atm_base_pattern.format(**locals())) + time = datetime.strptime(cdate, "%Y%m%d%H") + atm_source_path = time.strftime(atm_base_pattern.format(**locals())) - if(debug): - for var in ['merge_script', 'tracer_list_file', 'atm_source_path']: - print(f'{var} = {f"{var}"}') + if (debug): + for var in ['merge_script', 'tracer_list_file', 'atm_source_path']: + print(f'{var} = {f"{var}"}') - atm_files, ctrl_files = get_atm_files(atm_source_path) - tracer_files, rest_files, core_files = get_restart_files(time, incr, max_lookback, fcst_length, rot_dir, cdump) + atm_files, ctrl_files = get_atm_files(atm_source_path) + tracer_files, rest_files, core_files = get_restart_files(time, incr, max_lookback, fcst_length, rot_dir, cdump) - if (tracer_files is not None): - merge_tracers(merge_script, atm_files, tracer_files, rest_files, core_files[0], ctrl_files[0], tracer_list_file) + if (tracer_files is not None): + merge_tracers(merge_script, atm_files, tracer_files, rest_files, core_files[0], ctrl_files[0], tracer_list_file) - return + return def get_env_var(varname: str, fail_on_missing: bool = True) -> str: - ''' - Retrieve environment variable and exit or print warning if not defined - - Parameters - ---------- - varname : str - Environment variable to read - fail_on_missing : bool, optional - Whether to fail (if True) or print warning (False) if environment variable is not defined (default: True) - - Returns - ---------- - str - Value of the named variable - - Raises - ---------- - RuntimeError - If fail_on_missing is True and environment variable is not defined - - ''' - if(debug): - print(f'Trying to read envvar {varname}') - - var = os.environ.get(varname) - if(var is None): - if(fail_on_missing is True): - raise RuntimeError(f'Environment variable {varname} not set') - else: - print(f"WARNING: Environment variable {varname} not set, continuing using None") - if(debug): - print(f'\tValue: {var}') - return(var) + ''' + Retrieve environment variable and exit or print warning if not defined + + Parameters + ---------- + varname : str + Environment variable to read + fail_on_missing : bool, optional + Whether to fail (if True) or print warning (False) if environment variable is not defined (default: True) + + Returns + ---------- + str + Value of the named variable + + Raises + ---------- + RuntimeError + If fail_on_missing is True and environment variable is not defined + + ''' + if (debug): + print(f'Trying to read envvar {varname}') + + var = os.environ.get(varname) + if (var is None): + if (fail_on_missing is True): + raise RuntimeError(f'Environment variable {varname} not set') + else: + print(f"WARNING: Environment variable {varname} not set, continuing using None") + if (debug): + print(f'\tValue: {var}') + return (var) def get_atm_files(path: str) -> typing.List[typing.List[str]]: - ''' - Checks whether all atmospheric IC files exist in the given location and returns a list - of the filenames. - - Parameters - ---------- - path : str - Location where atmospheric IC files should exist - - Returns - ---------- - list of str - List of the full paths to each of the atmospheric files - - Raises - ---------- - IOError - If fail_on_missing is True and environment variable is not defined - - ''' - print(f'Checking for atm files in {path}') - - file_list = [] - for file_pattern in atm_file_pattern, atm_ctrl_pattern: - files = list(map(lambda tile: file_pattern.format(tile=tile, path=path), tiles)) - for file_name in files: - if(debug): - print(f"\tChecking for {file_name}") - if(not os.path.isfile(file_name)): - raise IOError(f"Atmosphere file {file_name} not found") - elif(debug): - print(f"\t\tFound {file_name}") - file_list = file_list + [files] - return file_list + ''' + Checks whether all atmospheric IC files exist in the given location and returns a list + of the filenames. + + Parameters + ---------- + path : str + Location where atmospheric IC files should exist + + Returns + ---------- + list of str + List of the full paths to each of the atmospheric files + + Raises + ---------- + IOError + If fail_on_missing is True and environment variable is not defined + + ''' + print(f'Checking for atm files in {path}') + + file_list = [] + for file_pattern in atm_file_pattern, atm_ctrl_pattern: + files = list(map(lambda tile: file_pattern.format(tile=tile, path=path), tiles)) + for file_name in files: + if (debug): + print(f"\tChecking for {file_name}") + if (not os.path.isfile(file_name)): + raise IOError(f"Atmosphere file {file_name} not found") + elif (debug): + print(f"\t\tFound {file_name}") + file_list = file_list + [files] + return file_list def get_restart_files(time: datetime, incr: int, max_lookback: int, fcst_length: int, rot_dir: str, cdump: str) -> typing.List[typing.List[str]]: - ''' - Determines the last cycle where all the necessary restart files are available. Ideally the immediate previous cycle - - Parameters - ---------- - time : datetime - Initial time for the current forecast - incr : int - Forecast cadence in hours - max_lookback : int - Maximum number of cycles to look back before failing - fcst_length : int - Length of forecast in hours - rot_dir : str - Path to the ROTDIR (COM) directory - cdump : str - CDUMP of current forecast portion (currently should always be 'gfs') - - Returns - ---------- - list of str - Full pathnames of all restart files needed from previous cycle (fv_core and fv_tracer files) - If all needed files aren't found within lookback period, An array of three None is returned instead. - - ''' - print(f"Looking for restart tracer files in {rot_dir}") - for lookback in map(lambda i: incr * (i + 1), range(max_lookback)): - if(lookback > fcst_length): - # Trying to look back farther than the length of a forecast - break - elif(lookback == fcst_length): - # Restart files at the end of the cycle don't have a timestamp - timestamp = "" - else: - timestamp = time.strftime("%Y%m%d.%H0000.") - - last_time = time - timedelta(hours=lookback) - - if(debug): - print(f"\tChecking {last_time}") - file_list = [] - file_base = last_time.strftime(restart_base_pattern.format(**locals())) - - for file_pattern in tracer_file_pattern, restart_file_pattern, dycore_file_pattern: - files = list(map(lambda tile: file_pattern.format(timestamp=timestamp, file_base=file_base, tile=tile), tiles)) - if(debug): - print(f"\t\tLooking for files {files} in directory {file_base}") - file_list = file_list + [files] - - found = all([os.path.isfile(file) for file in files for files in file_list]) - - if(found): - break - else: - print(last_time.strftime("Restart files not found for %Y%m%d_%H")) - - if(found): - return file_list - else: - print("WARNING: Unable to find restart files, will use zero fields") - return [ None, None, None ] + ''' + Determines the last cycle where all the necessary restart files are available. Ideally the immediate previous cycle + + Parameters + ---------- + time : datetime + Initial time for the current forecast + incr : int + Forecast cadence in hours + max_lookback : int + Maximum number of cycles to look back before failing + fcst_length : int + Length of forecast in hours + rot_dir : str + Path to the ROTDIR (COM) directory + cdump : str + CDUMP of current forecast portion (currently should always be 'gfs') + + Returns + ---------- + list of str + Full pathnames of all restart files needed from previous cycle (fv_core and fv_tracer files) + If all needed files aren't found within lookback period, An array of three None is returned instead. + + ''' + print(f"Looking for restart tracer files in {rot_dir}") + for lookback in map(lambda i: incr * (i + 1), range(max_lookback)): + if (lookback > fcst_length): + # Trying to look back farther than the length of a forecast + break + elif (lookback == fcst_length): + # Restart files at the end of the cycle don't have a timestamp + timestamp = "" + else: + timestamp = time.strftime("%Y%m%d.%H0000.") + + last_time = time - timedelta(hours=lookback) + + if (debug): + print(f"\tChecking {last_time}") + file_list = [] + file_base = last_time.strftime(restart_base_pattern.format(**locals())) + + for file_pattern in tracer_file_pattern, restart_file_pattern, dycore_file_pattern: + files = list(map(lambda tile: file_pattern.format(timestamp=timestamp, file_base=file_base, tile=tile), tiles)) + if (debug): + print(f"\t\tLooking for files {files} in directory {file_base}") + file_list = file_list + [files] + + found = all([os.path.isfile(file) for file in files for files in file_list]) + + if (found): + break + else: + print(last_time.strftime("Restart files not found for %Y%m%d_%H")) + + if (found): + return file_list + else: + print("WARNING: Unable to find restart files, will use zero fields") + return [None, None, None] # Merge tracer data into atmospheric data -def merge_tracers(merge_script: str, atm_files: typing.List[str], tracer_files: typing.List[str], rest_files: typing.List[str], core_file: str, ctrl_file: str, tracer_list_file: str) -> None: - ''' - Call the merger script to merge the tracers into the atmospheric IC files. Merged file is written to a temp file - which then overwrites the original upon successful completion of the script. - - Parameters - ---------- - merge_script : str - Full path to the merge script - atm_files : list of str - List of paths to atmospheric IC files - tracer_files : list of str - List of paths to tracer restart files - rest_files : list of str - List of paths to dycore tile restart files - core_file : str - Path of dycore restart file - ctrl_file : str - Path of control file - tracer_list_file : str - Full path to the file listing the tracer variables to add - - Returns - ---------- - None - - Raises - ---------- - ValueError - If `atm_files`, `tracer_files`, and `rest_files` are not all the same length - CalledProcessError - If merge script exits with a non-zero error - - ''' - print("Merging tracers") - if(len(atm_files) != len(tracer_files)): - raise ValueError("Atmosphere file list and tracer file list are not the same length") - - if(len(atm_files) != len(rest_files)): - raise ValueError("Atmosphere file list and dycore file list are not the same length") - - for atm_file, tracer_file, rest_file in zip(atm_files, tracer_files, rest_files): - if debug: - print(f"\tMerging tracers from {tracer_file} into {atm_file}") - temp_file = f'{atm_file}.tmp' - subprocess.run([merge_script, atm_file, tracer_file, core_file, ctrl_file, rest_file, tracer_list_file, temp_file], check=True) - os.replace(temp_file, atm_file) +def merge_tracers(merge_script: str, + atm_files: typing.List[str], + tracer_files: typing.List[str], + rest_files: typing.List[str], + core_file: str, + ctrl_file: str, + tracer_list_file: str) -> None: + ''' + Call the merger script to merge the tracers into the atmospheric IC files. Merged file is written to a temp file + which then overwrites the original upon successful completion of the script. + + Parameters + ---------- + merge_script : str + Full path to the merge script + atm_files : list of str + List of paths to atmospheric IC files + tracer_files : list of str + List of paths to tracer restart files + rest_files : list of str + List of paths to dycore tile restart files + core_file : str + Path of dycore restart file + ctrl_file : str + Path of control file + tracer_list_file : str + Full path to the file listing the tracer variables to add + + Returns + ---------- + None + + Raises + ---------- + ValueError + If `atm_files`, `tracer_files`, and `rest_files` are not all the same length + CalledProcessError + If merge script exits with a non-zero error + + ''' + print("Merging tracers") + if (len(atm_files) != len(tracer_files)): + raise ValueError("Atmosphere file list and tracer file list are not the same length") + + if (len(atm_files) != len(rest_files)): + raise ValueError("Atmosphere file list and dycore file list are not the same length") + + for atm_file, tracer_file, rest_file in zip(atm_files, tracer_files, rest_files): + if debug: + print(f"\tMerging tracers from {tracer_file} into {atm_file}") + temp_file = f'{atm_file}.tmp' + subprocess.run([merge_script, atm_file, tracer_file, core_file, ctrl_file, rest_file, tracer_list_file, temp_file], check=True) + os.replace(temp_file, atm_file) if __name__ == "__main__": - main() - exit(0) + main() + exit(0) From af3e3b3851f59fd4270a5ef2a65a8b2b06dbcb18 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 9 Dec 2022 22:15:36 -0500 Subject: [PATCH 3/8] fix norms in ush/python/pygw --- ush/python/pygw/src/pygw/__init__.py | 1 - ush/python/pygw/src/pygw/logger.py | 7 +++--- ush/python/pygw/src/pygw/template.py | 10 ++++---- ush/python/pygw/src/pygw/yaml_file.py | 3 +-- ush/python/pygw/src/tests/test_template.py | 27 +++++++++++----------- 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/ush/python/pygw/src/pygw/__init__.py b/ush/python/pygw/src/pygw/__init__.py index 17e8573021a..d44158004c6 100644 --- a/ush/python/pygw/src/pygw/__init__.py +++ b/ush/python/pygw/src/pygw/__init__.py @@ -6,4 +6,3 @@ import os pygw_directory = os.path.dirname(__file__) - diff --git a/ush/python/pygw/src/pygw/logger.py b/ush/python/pygw/src/pygw/logger.py index 5807d5cd946..15b0e74c72d 100644 --- a/ush/python/pygw/src/pygw/logger.py +++ b/ush/python/pygw/src/pygw/logger.py @@ -45,6 +45,7 @@ class Logger: LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] DEFAULT_LEVEL = 'INFO' DEFAULT_FORMAT = '%(asctime)s - %(levelname)-8s - %(name)-12s: %(message)s' + def __init__(self, name: str = None, level: str = DEFAULT_LEVEL, _format: str = DEFAULT_FORMAT, @@ -93,7 +94,7 @@ def __init__(self, name: str = None, level=self.level, _format=self.format, colored_log=self.colored_log, - ) + ) _handlers.append(_handler) self._logger.addHandler(_handler) @@ -151,8 +152,8 @@ def add_handlers(cls, logger: logging.Logger, handlers: List[logging.Handler]): @classmethod def add_stream_handler(cls, level: str = DEFAULT_LEVEL, - _format: str = DEFAULT_FORMAT, - colored_log: bool = False): + _format: str = DEFAULT_FORMAT, + colored_log: bool = False): """ Create stream handler This classmethod will allow setting a custom stream handler on children diff --git a/ush/python/pygw/src/pygw/template.py b/ush/python/pygw/src/pygw/template.py index ed7878b6a58..1cdb4d39ab2 100644 --- a/ush/python/pygw/src/pygw/template.py +++ b/ush/python/pygw/src/pygw/template.py @@ -41,12 +41,12 @@ class Template: """ substitutions = { - TemplateConstants.DOLLAR_CURLY_BRACE: TemplateConstants.SubPair(re.compile('\${.*?}+'), slice(2, -1)), - TemplateConstants.DOLLAR_PARENTHESES: TemplateConstants.SubPair(re.compile('\$\(.*?\)+'), slice(2, -1)), - TemplateConstants.DOUBLE_CURLY_BRACES: TemplateConstants.SubPair(re.compile('{{.*?}}+'), slice(2, -2)), - TemplateConstants.AT_SQUARE_BRACES: TemplateConstants.SubPair(re.compile('@\[.*?\]+'), slice(2, -1)), + TemplateConstants.DOLLAR_CURLY_BRACE: TemplateConstants.SubPair(re.compile(r'\${.*?}+'), slice(2, -1)), + TemplateConstants.DOLLAR_PARENTHESES: TemplateConstants.SubPair(re.compile(r'\$\(.*?\)+'), slice(2, -1)), + TemplateConstants.DOUBLE_CURLY_BRACES: TemplateConstants.SubPair(re.compile(r'{{.*?}}+'), slice(2, -2)), + TemplateConstants.AT_SQUARE_BRACES: TemplateConstants.SubPair(re.compile(r'@\[.*?\]+'), slice(2, -1)), TemplateConstants.AT_ANGLE_BRACKETS: TemplateConstants.SubPair( - re.compile('@\<.*?\>+'), slice(2, -1)) + re.compile(r'@\<.*?\>+'), slice(2, -1)) } @classmethod diff --git a/ush/python/pygw/src/pygw/yaml_file.py b/ush/python/pygw/src/pygw/yaml_file.py index e25c18619f6..58032711055 100644 --- a/ush/python/pygw/src/pygw/yaml_file.py +++ b/ush/python/pygw/src/pygw/yaml_file.py @@ -76,7 +76,7 @@ def parse_yaml(path=None, data=None, envtag = '!ENV' inctag = '!INC' # pattern for global vars: look for ${word} - pattern = re.compile('.*?\${(\w+)}.*?') + pattern = re.compile(r'.*?\${(\w+)}.*?') loader = loader or yaml.SafeLoader # the envtag will be used to mark where to start searching for the pattern @@ -91,7 +91,6 @@ def expand_env_variables(line): for g in match: full_value = full_value.replace( f'${{{g}}}', os.environ.get(g, f'${{{g}}}') - #f'${{{g}}}', os.environ.get(g, g) ) return full_value return line diff --git a/ush/python/pygw/src/tests/test_template.py b/ush/python/pygw/src/tests/test_template.py index 5d7bb378b9f..0404c6a2fd7 100644 --- a/ush/python/pygw/src/tests/test_template.py +++ b/ush/python/pygw/src/tests/test_template.py @@ -114,9 +114,9 @@ def test_substitute_with_dependencies(): 'a': 1, 'b': 2 }, - 'dd': { '2': 'a', '1': 'b' }, - 'ee': { '3': 'a', '1': 'b' }, - 'ff': { '4': 'a', '1': 'b $(greeting)' }, + 'dd': {'2': 'a', '1': 'b'}, + 'ee': {'3': 'a', '1': 'b'}, + 'ff': {'4': 'a', '1': 'b $(greeting)'}, 'host': { 'name': 'xenon', 'config': '$(root)/hosts', @@ -128,21 +128,20 @@ def test_substitute_with_dependencies(): } } output = {'complex': {'a': 1, 'b': 2}, - 'config': '/home/user/config/config.yaml', - 'config_file': 'config.yaml', - 'dd': {'1': 'b', '2': 'a'}, - 'dictionary': {'a': 1, 'b': 2}, - 'ee': {'1': 'b', '3': 'a'}, - 'ff': {'1': 'b hello world', '4': 'a'}, - 'greeting': 'hello world', - 'host': {'config': '/home/user/hosts', + 'config': '/home/user/config/config.yaml', + 'config_file': 'config.yaml', + 'dd': {'1': 'b', '2': 'a'}, + 'dictionary': {'a': 1, 'b': 2}, + 'ee': {'1': 'b', '3': 'a'}, + 'ff': {'1': 'b hello world', '4': 'a'}, + 'greeting': 'hello world', + 'host': {'config': '/home/user/hosts', 'config_file': '/home/user/config/config.yaml/xenon.config.yaml', 'name': 'xenon', 'proxy2': {'config': '/home/user/xenon.hello world.yaml', 'list': [['/home/user/xenon', 'toto.xenon.hello world'], 'config.yaml']}}, - 'root': '/home/user', - 'world': 'world'} - + 'root': '/home/user', + 'world': 'world'} assert Template.substitute_with_dependencies(input, input, TemplateConstants.DOLLAR_PARENTHESES) == output From d8922b335df707e4231e575c861b2de85fd766dc Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 9 Dec 2022 22:24:48 -0500 Subject: [PATCH 4/8] fix norms in ush --- ush/calcanl_gfs.py | 111 ++++++++++++++-------------- ush/calcinc_gfs.py | 133 +++++++++++++++++----------------- ush/gsi_utils.py | 26 ++++--- ush/merge_fv3_aerosol_tile.py | 4 +- 4 files changed, 141 insertions(+), 133 deletions(-) diff --git a/ush/calcanl_gfs.py b/ush/calcanl_gfs.py index b718dfeda54..4f8a4eb3e99 100755 --- a/ush/calcanl_gfs.py +++ b/ush/calcanl_gfs.py @@ -18,11 +18,11 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, ComIn_Ges, GPrefix, GSuffix, FixDir, atmges_ens_mean, RunDir, NThreads, NEMSGet, IAUHrs, ExecCMD, ExecCMDMPI, ExecAnl, ExecChgresInc, Cdump): - print('calcanl_gfs beginning at: ',datetime.datetime.utcnow()) + print('calcanl_gfs beginning at: ', datetime.datetime.utcnow()) IAUHH = IAUHrs - ######## copy and link files + # copy and link files if DoIAU and l4DEnsVar and Write4Danl: for fh in IAUHH: if fh == 6: @@ -81,7 +81,6 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, gsi_utils.link_file(ComIn_Ges+'/'+GPrefix+'atmf'+format(fh, '03')+'.ensres'+GSuffix, CalcAnlDir6+'/ges.ensres.'+format(fh, '02')) - else: # for full res analysis CalcAnlDir = RunDir+'/calcanl_'+format(6, '02') @@ -101,12 +100,12 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, gsi_utils.link_file(ComOut+'/'+APrefix+'atmanl.ensres'+ASuffix, CalcAnlDir+'/anl.ensres.06') gsi_utils.link_file(ComIn_Ges+'/'+GPrefix+'atmf006.ensres'+GSuffix, CalcAnlDir+'/ges.ensres.06') - ######## get dimension information from background and increment files + # get dimension information from background and increment files AnlDims = gsi_utils.get_ncdims('siginc.nc') if ASuffix == ".nc": GesDims = gsi_utils.get_ncdims('sigf06') else: - GesDims = gsi_utils.get_nemsdims('sigf06',NEMSGet) + GesDims = gsi_utils.get_nemsdims('sigf06', NEMSGet) levs = AnlDims['lev'] LonA = AnlDims['lon'] @@ -118,31 +117,31 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, levs2 = levs + 1 siglevel = FixDir+'/global_hyblev.l'+str(levs2)+'.txt' - ####### determine how many forecast hours to process - nFH=0 + # determine how many forecast hours to process + nFH = 0 for fh in IAUHH: # first check to see if increment file exists CalcAnlDir = RunDir+'/calcanl_'+format(fh, '02') if (os.path.isfile(CalcAnlDir+'/siginc.nc.'+format(fh, '02'))): print('will process increment file: '+CalcAnlDir+'/siginc.nc.'+format(fh, '02')) - nFH+=1 + nFH += 1 else: print('Increment file: '+CalcAnlDir+'/siginc.nc.'+format(fh, '02')+' does not exist. Skipping.') sys.stdout.flush() - ######## need to gather information about runtime environment - ExecCMD = ExecCMD.replace("$ncmd","1") + # need to gather information about runtime environment + ExecCMD = ExecCMD.replace("$ncmd", "1") os.environ['OMP_NUM_THREADS'] = str(NThreads) os.environ['ncmd'] = str(nFH) - ExecCMDMPI1 = ExecCMDMPI.replace("$ncmd",str(1)) - ExecCMDMPI = ExecCMDMPI.replace("$ncmd",str(nFH)) - ExecCMDLevs = ExecCMDMPI.replace("$ncmd",str(levs)) - ExecCMDMPI10 = ExecCMDMPI.replace("$ncmd",str(10)) + ExecCMDMPI1 = ExecCMDMPI.replace("$ncmd", str(1)) + ExecCMDMPI = ExecCMDMPI.replace("$ncmd", str(nFH)) + ExecCMDLevs = ExecCMDMPI.replace("$ncmd", str(levs)) + ExecCMDMPI10 = ExecCMDMPI.replace("$ncmd", str(10)) # are we using mpirun with lsf, srun, or aprun with Cray? launcher = ExecCMDMPI.split(' ')[0] if launcher == 'mpirun': - hostfile = os.getenv('LSB_DJOB_HOSTFILE','') + hostfile = os.getenv('LSB_DJOB_HOSTFILE', '') with open(hostfile) as f: hosts_tmp = f.readlines() hosts_tmp = [x.strip() for x in hosts_tmp] @@ -150,7 +149,7 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, [hosts.append(x) for x in hosts_tmp if x not in hosts] nhosts = len(hosts) ExecCMDMPI_host = 'mpirun -np '+str(nFH)+' --hostfile hosts' - tasks = int(os.getenv('LSB_DJOB_NUMPROC',1)) + tasks = int(os.getenv('LSB_DJOB_NUMPROC', 1)) if levs > tasks: ExecCMDMPILevs_host = 'mpirun -np '+str(tasks)+' --hostfile hosts' ExecCMDMPILevs_nohost = 'mpirun -np '+str(tasks) @@ -160,7 +159,7 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, ExecCMDMPI1_host = 'mpirun -np 1 --hostfile hosts' ExecCMDMPI10_host = 'mpirun -np 10 --hostfile hosts' elif launcher == 'mpiexec': - hostfile = os.getenv('PBS_NODEFILE','') + hostfile = os.getenv('PBS_NODEFILE', '') with open(hostfile) as f: hosts_tmp = f.readlines() hosts_tmp = [x.strip() for x in hosts_tmp] @@ -168,7 +167,7 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, [hosts.append(x) for x in hosts_tmp if x not in hosts] nhosts = len(hosts) ExecCMDMPI_host = 'mpiexec -l -n '+str(nFH) - tasks = int(os.getenv('ntasks',1)) + tasks = int(os.getenv('ntasks', 1)) print('nhosts,tasks=', nhosts, tasks) if levs > tasks: ExecCMDMPILevs_host = 'mpiexec -l -n '+str(tasks) @@ -179,7 +178,7 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, ExecCMDMPI1_host = 'mpiexec -l -n 1 --cpu-bind depth --depth '+str(NThreads) ExecCMDMPI10_host = 'mpiexec -l -n 10 --cpu-bind depth --depth '+str(NThreads) elif launcher == 'srun': - nodes = os.getenv('SLURM_JOB_NODELIST','') + nodes = os.getenv('SLURM_JOB_NODELIST', '') hosts_tmp = subprocess.check_output('scontrol show hostnames '+nodes, shell=True) if (sys.version_info > (3, 0)): hosts_tmp = hosts_tmp.decode('utf-8') @@ -194,7 +193,7 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, nhosts = len(hosts) ExecCMDMPI_host = 'srun -n '+str(nFH)+' --verbose --export=ALL -c 1 --distribution=arbitrary --cpu-bind=cores' # need to account for when fewer than LEVS tasks are available - tasks = int(os.getenv('SLURM_NPROCS',1)) + tasks = int(os.getenv('SLURM_NPROCS', 1)) if levs > tasks: ExecCMDMPILevs_host = 'srun -n '+str(tasks)+' --verbose --export=ALL -c 1 --distribution=arbitrary --cpu-bind=cores' ExecCMDMPILevs_nohost = 'srun -n '+str(tasks)+' --verbose --export=ALL' @@ -204,7 +203,7 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, ExecCMDMPI1_host = 'srun -n 1 --verbose --export=ALL -c 1 --distribution=arbitrary --cpu-bind=cores' ExecCMDMPI10_host = 'srun -n 10 --verbose --export=ALL -c 1 --distribution=arbitrary --cpu-bind=cores' elif launcher == 'aprun': - hostfile = os.getenv('LSB_DJOB_HOSTFILE','') + hostfile = os.getenv('LSB_DJOB_HOSTFILE', '') with open(hostfile) as f: hosts_tmp = f.readlines() hosts_tmp = [x.strip() for x in hosts_tmp] @@ -220,9 +219,9 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, print('unknown MPI launcher. Failure.') sys.exit(1) - ####### generate the full resolution analysis + # generate the full resolution analysis ihost = 0 - ### interpolate increment to full background resolution + # interpolate increment to full background resolution for fh in IAUHH: # first check to see if increment file exists CalcAnlDir = RunDir+'/calcanl_'+format(fh, '02') @@ -231,26 +230,26 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, # set up the namelist namelist = OrderedDict() namelist["setup"] = {"lon_out": LonB, - "lat_out": LatB, - "lev": levs, - "infile": "'siginc.nc."+format(fh, '02')+"'", - "outfile": "'inc.fullres."+format(fh, '02')+"'", - } + "lat_out": LatB, + "lev": levs, + "infile": "'siginc.nc."+format(fh, '02')+"'", + "outfile": "'inc.fullres."+format(fh, '02')+"'", + } gsi_utils.write_nml(namelist, CalcAnlDir+'/fort.43') if ihost >= nhosts: ihost = 0 with open(CalcAnlDir+'/hosts', 'w') as hostfile: hostfile.write(hosts[ihost]+'\n') - if launcher == 'srun': # need to write host per task not per node for slurm + if launcher == 'srun': # need to write host per task not per node for slurm # For xjet, each instance of chgres_inc must run on two nodes each - if os.getenv('SLURM_JOB_PARTITION','') == 'xjet': - for a in range(0,4): + if os.getenv('SLURM_JOB_PARTITION', '') == 'xjet': + for a in range(0, 4): hostfile.write(hosts[ihost]+'\n') - ihost+=1 - for a in range(0,5): + ihost += 1 + for a in range(0, 5): hostfile.write(hosts[ihost]+'\n') - for a in range(0,9): # need 9 more of the same host for the 10 tasks for chgres_inc + for a in range(0, 9): # need 9 more of the same host for the 10 tasks for chgres_inc hostfile.write(hosts[ihost]+'\n') if launcher == 'srun': os.environ['SLURM_HOSTFILE'] = CalcAnlDir+'/hosts' @@ -264,20 +263,20 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, print('Error with chgres_inc.x, exit code='+str(ec)) print(locals()) sys.exit(ec) - ihost+=1 + ihost += 1 else: print('f'+format(fh, '03')+' is in $IAUFHRS but increment file is missing. Skipping.') - #### generate analysis from interpolated increment + # generate analysis from interpolated increment CalcAnlDir6 = RunDir+'/calcanl_'+format(6, '02') # set up the namelist namelist = OrderedDict() - namelist["setup"] = {"datapath": "'./'", - "analysis_filename": "'anl'", - "firstguess_filename": "'ges'", - "increment_filename": "'inc.fullres'", - "fhr": 6, - } + namelist["setup"] = {"datapath": "'./'", + "analysis_filename": "'anl'", + "firstguess_filename": "'ges'", + "increment_filename": "'inc.fullres'", + "fhr": 6, + } gsi_utils.write_nml(namelist, CalcAnlDir6+'/calc_analysis.nml') @@ -298,8 +297,7 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, print(locals()) sys.exit(exit_fullres) - - ######## compute determinstic analysis on ensemble resolution + # compute determinstic analysis on ensemble resolution if Cdump in ["gdas", "gfs"]: chgres_jobs = [] for fh in IAUHH: @@ -308,15 +306,15 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, print(CalcAnlDir6+'/ges.ensres.'+format(fh, '02')) if (os.path.isfile(CalcAnlDir6+'/ges.ensres.'+format(fh, '02'))): print('Calculating analysis on ensemble resolution for f'+format(fh, '03')) - ######## generate ensres analysis from interpolated background + # generate ensres analysis from interpolated background # set up the namelist namelist = OrderedDict() - namelist["setup"] = {"datapath": "'./'", - "analysis_filename": "'anl.ensres'", - "firstguess_filename": "'ges.ensres'", - "increment_filename": "'siginc.nc'", - "fhr": fh, - } + namelist["setup"] = {"datapath": "'./'", + "analysis_filename": "'anl.ensres'", + "firstguess_filename": "'ges.ensres'", + "increment_filename": "'siginc.nc'", + "fhr": fh, + } gsi_utils.write_nml(namelist, CalcAnlDir6+'/calc_analysis.nml') @@ -328,7 +326,7 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, print(ExecCMDMPILevs_nohost+' '+CalcAnlDir6+'/calc_anl.x submitted') sys.stdout.flush() - ####### check on analysis steps + # check on analysis steps exit_ensres = ensres_anl_job.wait() if exit_ensres != 0: print('Error with calc_analysis.x for ensemble resolution, exit code='+str(exit_ensres)) @@ -337,9 +335,10 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, else: print('f'+format(fh, '03')+' is in $IAUFHRS but ensemble resolution guess file is missing. Skipping.') - print('calcanl_gfs successfully completed at: ',datetime.datetime.utcnow()) + print('calcanl_gfs successfully completed at: ', datetime.datetime.utcnow()) print(locals()) + # run the function if this script is called from the command line if __name__ == '__main__': DoIAU = gsi_utils.isTrue(os.getenv('DOIAU', 'NO')) @@ -350,7 +349,7 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, GSuffix = os.getenv('GSUFFIX', './') ComOut = os.getenv('COMOUT', './') APrefix = os.getenv('APREFIX', '') - ASuffix= os.getenv('ASUFFIX', '') + ASuffix = os.getenv('ASUFFIX', '') NThreads = os.getenv('NTHREADS_CHGRES', 1) FixDir = os.getenv('FIXgsm', './') atmges_ens_mean = os.getenv('ATMGES_ENSMEAN', './atmges_ensmean') @@ -359,8 +358,8 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, ExecCMDMPI = os.getenv('APRUN_CALCINC', '') ExecAnl = os.getenv('CALCANLEXEC', './calc_analysis.x') ExecChgresInc = os.getenv('CHGRESINCEXEC', './interp_inc.x') - NEMSGet = os.getenv('NEMSIOGET','nemsio_get') - IAUHrs = list(map(int,os.getenv('IAUFHRS','6').split(','))) + NEMSGet = os.getenv('NEMSIOGET', 'nemsio_get') + IAUHrs = list(map(int, os.getenv('IAUFHRS', '6').split(','))) Cdump = os.getenv('CDUMP', 'gdas') print(locals()) diff --git a/ush/calcinc_gfs.py b/ush/calcinc_gfs.py index 0306d9f39f1..648f0f14f3f 100755 --- a/ush/calcinc_gfs.py +++ b/ush/calcinc_gfs.py @@ -12,79 +12,82 @@ from collections import OrderedDict # main function + + def calcinc_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, IAUHrs, NThreads, IMP_Physics, Inc2Zero, RunDir, Exec, ExecCMD): - # run the calc_increment_ens executable + # run the calc_increment_ens executable - # copy and link files - if DoIAU and l4DEnsVar and Write4Danl: - nFH=0 - for fh in IAUHrs: - nFH+=1 - if fh == 6: - gsi_utils.link_file('sigf06', 'atmges_mem'+format(nFH, '03')) - gsi_utils.link_file('siganl', 'atmanl_mem'+format(nFH, '03')) - gsi_utils.link_file(ComOut+'/'+APrefix+'atminc.nc', 'atminc_mem'+format(nFH, '03')) - else: - gsi_utils.link_file('sigf'+format(fh, '02'), 'atmges_mem'+format(nFH, '03')) - gsi_utils.link_file('siga'+format(fh, '02'), 'atmanl_mem'+format(nFH, '03')) - gsi_utils.link_file(ComOut+'/'+APrefix+'atmi'+format(fh, '03')+'.nc', 'atminc_mem'+format(nFH, '03')) - else: - nFH=1 - gsi_utils.link_file('sigf06', 'atmges_mem001') - gsi_utils.link_file('siganl', 'atmanl_mem001') - gsi_utils.link_file(ComOut+'/'+APrefix+'atminc', 'atminc_mem001') - os.environ['OMP_NUM_THREADS'] = str(NThreads) - os.environ['ncmd'] = str(nFH) - shutil.copy(Exec,RunDir+'/calc_inc.x') - ExecCMD = ExecCMD.replace("$ncmd",str(nFH)) + # copy and link files + if DoIAU and l4DEnsVar and Write4Danl: + nFH = 0 + for fh in IAUHrs: + nFH += 1 + if fh == 6: + gsi_utils.link_file('sigf06', 'atmges_mem'+format(nFH, '03')) + gsi_utils.link_file('siganl', 'atmanl_mem'+format(nFH, '03')) + gsi_utils.link_file(ComOut+'/'+APrefix+'atminc.nc', 'atminc_mem'+format(nFH, '03')) + else: + gsi_utils.link_file('sigf'+format(fh, '02'), 'atmges_mem'+format(nFH, '03')) + gsi_utils.link_file('siga'+format(fh, '02'), 'atmanl_mem'+format(nFH, '03')) + gsi_utils.link_file(ComOut+'/'+APrefix+'atmi'+format(fh, '03')+'.nc', 'atminc_mem'+format(nFH, '03')) + else: + nFH = 1 + gsi_utils.link_file('sigf06', 'atmges_mem001') + gsi_utils.link_file('siganl', 'atmanl_mem001') + gsi_utils.link_file(ComOut+'/'+APrefix+'atminc', 'atminc_mem001') + os.environ['OMP_NUM_THREADS'] = str(NThreads) + os.environ['ncmd'] = str(nFH) + shutil.copy(Exec, RunDir+'/calc_inc.x') + ExecCMD = ExecCMD.replace("$ncmd", str(nFH)) - # set up the namelist - namelist = OrderedDict() - namelist["setup"] = {"datapath": "'./'", - "analysis_filename": "'atmanl'", - "firstguess_filename": "'atmges'", - "increment_filename": "'atminc'", - "debug": ".false.", - "nens": str(nFH), - "imp_physics": str(IMP_Physics)} + # set up the namelist + namelist = OrderedDict() + namelist["setup"] = {"datapath": "'./'", + "analysis_filename": "'atmanl'", + "firstguess_filename": "'atmges'", + "increment_filename": "'atminc'", + "debug": ".false.", + "nens": str(nFH), + "imp_physics": str(IMP_Physics)} - namelist["zeroinc"] = {"incvars_to_zero": Inc2Zero} - - gsi_utils.write_nml(namelist, RunDir+'/calc_increment.nml') + namelist["zeroinc"] = {"incvars_to_zero": Inc2Zero} + + gsi_utils.write_nml(namelist, RunDir+'/calc_increment.nml') + + # run the executable + try: + err = subprocess.check_call(ExecCMD+' '+RunDir+'/calc_inc.x', shell=True) + print(locals()) + except subprocess.CalledProcessError as e: + print('Error with calc_inc.x, exit code='+str(e.returncode)) + print(locals()) + sys.exit(e.returncode) - # run the executable - try: - err = subprocess.check_call(ExecCMD+' '+RunDir+'/calc_inc.x', shell=True) - print(locals()) - except subprocess.CalledProcessError as e: - print('Error with calc_inc.x, exit code='+str(e.returncode)) - print(locals()) - sys.exit(e.returncode) # run the function if this script is called from the command line if __name__ == '__main__': - DoIAU = gsi_utils.isTrue(os.getenv('DOIAU', 'NO')) - l4DEnsVar = gsi_utils.isTrue(os.getenv('l4densvar', 'NO')) - Write4Danl = gsi_utils.isTrue(os.getenv('lwrite4danl', 'NO')) - ComOut = os.getenv('COMOUT', './') - APrefix = os.getenv('APREFIX', '') - ASuffix= os.getenv('ASUFFIX', '') - NThreads = os.getenv('NTHREADS_CALCINC', 1) - IMP_Physics = os.getenv('imp_physics', 11) - RunDir = os.getenv('DATA', './') - ExecNC = os.getenv('CALCINCNCEXEC', './calc_increment_ens_ncio.x') - ExecNEMS = os.getenv('CALCINCEXEC', './calc_increment_ens.x') - Inc2Zero = os.getenv('INCREMENTS_TO_ZERO', '"NONE"') - ExecCMD = os.getenv('APRUN_CALCINC', '') - IAUHrs = list(map(int,os.getenv('IAUFHRS','6').split(','))) + DoIAU = gsi_utils.isTrue(os.getenv('DOIAU', 'NO')) + l4DEnsVar = gsi_utils.isTrue(os.getenv('l4densvar', 'NO')) + Write4Danl = gsi_utils.isTrue(os.getenv('lwrite4danl', 'NO')) + ComOut = os.getenv('COMOUT', './') + APrefix = os.getenv('APREFIX', '') + ASuffix = os.getenv('ASUFFIX', '') + NThreads = os.getenv('NTHREADS_CALCINC', 1) + IMP_Physics = os.getenv('imp_physics', 11) + RunDir = os.getenv('DATA', './') + ExecNC = os.getenv('CALCINCNCEXEC', './calc_increment_ens_ncio.x') + ExecNEMS = os.getenv('CALCINCEXEC', './calc_increment_ens.x') + Inc2Zero = os.getenv('INCREMENTS_TO_ZERO', '"NONE"') + ExecCMD = os.getenv('APRUN_CALCINC', '') + IAUHrs = list(map(int, os.getenv('IAUFHRS', '6').split(','))) - # determine if the analysis is in netCDF or NEMSIO - if ASuffix == ".nc": - Exec = ExecNC - else: - Exec = ExecNEMS + # determine if the analysis is in netCDF or NEMSIO + if ASuffix == ".nc": + Exec = ExecNC + else: + Exec = ExecNEMS - print(locals()) - calcinc_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, IAUHrs, - NThreads, IMP_Physics, Inc2Zero, RunDir, Exec, ExecCMD) + print(locals()) + calcinc_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, IAUHrs, + NThreads, IMP_Physics, Inc2Zero, RunDir, Exec, ExecCMD) diff --git a/ush/gsi_utils.py b/ush/gsi_utils.py index b33be51adb0..fbc85e1fd5e 100644 --- a/ush/gsi_utils.py +++ b/ush/gsi_utils.py @@ -1,6 +1,6 @@ -### gsi_utils.py -### a collection of functions, classes, etc. -### used for the GSI global analysis +# gsi_utils.py +# a collection of functions, classes, etc. +# used for the GSI global analysis def isTrue(str_in): """ isTrue(str_in) @@ -11,12 +11,13 @@ def isTrue(str_in): """ str_in = str_in.upper() - if str_in in ['YES','.TRUE.']: + if str_in in ['YES', '.TRUE.']: status = True else: status = False return status + def link_file(from_file, to_file): """ link_file(from_file, to_file) - function to check if a path exists, and if not, make a symlink @@ -33,16 +34,19 @@ def link_file(from_file, to_file): os.symlink(from_file, to_file) print("ln -s "+from_file+" "+to_file) + def copy_file(from_file, to_file): import shutil shutil.copy(from_file, to_file) print("cp "+from_file+" "+to_file) + def make_dir(directory): import os os.makedirs(directory) print("mkdir -p "+directory) + def write_nml(nml_dict, nml_file): """ write_nml(nml_dict, nml_file) - function to write out namelist dictionary nml_dict to file nml_file @@ -82,7 +86,8 @@ def get_ncdims(ncfile): return ncdims -def get_nemsdims(nemsfile,nemsexe): + +def get_nemsdims(nemsfile, nemsexe): """ get_nemsdims(nemsfile,nemsexe) - function to return dictionary of NEMSIO file dimensions for use input: nemsfile - string to path nemsio file @@ -93,17 +98,18 @@ def get_nemsdims(nemsfile,nemsexe): """ import subprocess ncdims = { - 'dimx': 'grid_xt', + 'dimx': 'grid_xt', 'dimy': 'grid_yt', 'dimz': 'pfull', - } + } nemsdims = {} - for dim in ['dimx','dimy','dimz']: - out = subprocess.Popen([nemsexe,nemsfile,dim],stdout=subprocess.PIPE,stderr=subprocess.STDOUT) + for dim in ['dimx', 'dimy', 'dimz']: + out = subprocess.Popen([nemsexe, nemsfile, dim], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = out.communicate() nemsdims[ncdims[dim]] = int(stdout.split(' ')[-1].rstrip()) return nemsdims + def get_timeinfo(ncfile): """ get_timeinfo(ncfile) - function to return datetime objects of initialized time and valid time @@ -122,7 +128,7 @@ def get_timeinfo(ncfile): date_str = time_units.split('since ')[1] date_str = re.sub("[^0-9]", "", date_str) initstr = date_str[0:10] - inittime = dt.datetime.strptime(initstr,"%Y%m%d%H") + inittime = dt.datetime.strptime(initstr, "%Y%m%d%H") nfhour = int(ncf['time'][0]) validtime = inittime + dt.timedelta(hours=nfhour) ncf.close() diff --git a/ush/merge_fv3_aerosol_tile.py b/ush/merge_fv3_aerosol_tile.py index 7538bc7a767..decf6e9cbab 100755 --- a/ush/merge_fv3_aerosol_tile.py +++ b/ush/merge_fv3_aerosol_tile.py @@ -155,8 +155,8 @@ def main() -> None: parser.add_argument('core_file', type=str, help="File containing the dycore sigma level coefficients") parser.add_argument('ctrl_file', type=str, help="File containing the sigma level coefficients for atmospheric IC data") parser.add_argument('rest_file', type=str, help="File containing the pressure level thickness for the restart state") - parser.add_argument('variable_file', type=str, help="File containing list of tracer variable_names in the chem_file to add to the atm_file, one tracer per line") - parser.add_argument('out_file', type=str, nargs="?", help="Name of file to create. If none is specified, the atm_file will be edited in place. New file will be a copy of atm_file with the specificed tracers listed in variable_file appended from chem_file and ntracers updated.") + parser.add_argument('variable_file', type=str, help="File with list of tracer variable_names in the chem_file to add to the atm_file, one tracer per line") + parser.add_argument('out_file', type=str, nargs="?", help="Name of file to create") args = parser.parse_args() From 0721b4c96721d09961e9fd5ea58888140e451da8 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 9 Dec 2022 22:26:08 -0500 Subject: [PATCH 5/8] fix norms in test --- test/diff_grib_files.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/diff_grib_files.py b/test/diff_grib_files.py index 43619f143d1..38af74fd958 100755 --- a/test/diff_grib_files.py +++ b/test/diff_grib_files.py @@ -1,6 +1,6 @@ #! /bin/env python3 ''' -Compares two grib2 files and print any variables that have a +Compares two grib2 files and print any variables that have a non-identity correlation. Syntax @@ -22,9 +22,10 @@ # TODO - Update to also check the min just in case the grib files have a constant offset + def count_nonid_corr(test_string: str, quiet=False): ''' - Scan a wgrib2 print of the correlation between two values and count + Scan a wgrib2 print of the correlation between two values and count how many variables have a non-identity correlation. Any such variables are printed. @@ -64,6 +65,7 @@ def count_nonid_corr(test_string: str, quiet=False): return count + if __name__ == '__main__': fileA = sys.argv[0] fileB = sys.argv[1] From 34d0069f364769f370f368dc554880b9e4d87a72 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 9 Dec 2022 22:55:26 -0500 Subject: [PATCH 6/8] fix norms in workflow --- workflow/applications.py | 1 - workflow/ecFlow/ecflow_definitions.py | 38 +++++++++++++++-------- workflow/ecFlow/ecflow_setup.py | 6 ++-- workflow/rocoto/workflow_tasks.py | 7 +++-- workflow/rocoto/workflow_xml.py | 4 +-- workflow/rocoto_viewer.py | 44 +++++++++++++++++++-------- workflow/setup_expt.py | 10 ++++-- 7 files changed, 73 insertions(+), 37 deletions(-) diff --git a/workflow/applications.py b/workflow/applications.py index 67474112e1e..73c139adb13 100644 --- a/workflow/applications.py +++ b/workflow/applications.py @@ -183,7 +183,6 @@ def _cycled_configs(self): configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'arch'] - if self.do_gldas: configs += ['gldas'] diff --git a/workflow/ecFlow/ecflow_definitions.py b/workflow/ecFlow/ecflow_definitions.py index 7193790732b..0aea65710c0 100644 --- a/workflow/ecFlow/ecflow_definitions.py +++ b/workflow/ecFlow/ecflow_definitions.py @@ -370,12 +370,12 @@ def add_repeat(self, repeat, parent=None): """ repeat_token = re.search( - "(\d{8,10})( | to )(\d{10})( | by )(\d{1,2}:)?(\d{1,2}:\d{1,2})", + r"(\d{8,10})( | to )(\d{10})( | by )(\d{1,2}:)?(\d{1,2}:\d{1,2})", repeat) start = repeat_token.group(1).strip() end = repeat_token.group(3).strip() byday = repeat_token.group(5).strip() if repeat_token.group(5) is not \ - None else repeat_token.group(5) + None else repeat_token.group(5) bytime = repeat_token.group(6).strip() startdate = datetime.strptime(start, "%Y%m%d%H") if len(start) == 10 \ @@ -633,8 +633,8 @@ def add_task(self, task, parents, scriptrepo, template=None, self.ecf_nodes[task_name].setup_script(scriptrepo, template) if self.build_tree: self.ecf_nodes[task_name].generate_ecflow_task(self.ecfhome, - self.get_suite_name(), - parents) + self.get_suite_name(), + parents) self.ecf_nodes[parents] += self.ecf_nodes[task_name] def add_task_edits(self, task, edit_dict, parent_node=None, index=None): @@ -1002,7 +1002,7 @@ def __init__(self, ecfitem, ecfparent=None): self.__populate_full_name_items() if (ecfparent and self.__max_value is None and (ecfparent.is_list or ecfparent.is_range) and - len(self.__items) == len(ecfparent.get_full_name_items())): + len(self.__items) == len(ecfparent.get_full_name_items())): self.use_parent_counter = True def get_name(self): @@ -1162,7 +1162,6 @@ def __set_max_value(self, range_token): None """ - if not range_token[0]: self.__max_value = None self.use_parent_counter = True @@ -1301,8 +1300,8 @@ def __populate_full_name_items(self): for item in self.__items: if isinstance(item, int): self.__full_name_items.append(f"{self.__base}" - f"{item:03}" - f"{self.__suffix}") + f"{item:03}" + f"{self.__suffix}") elif isinstance(item, str): self.__full_name_items.append(f"{self.__base}" f"{item}" @@ -1438,6 +1437,7 @@ def get_type(self): return 'task' + class EcfFamilyNode(EcfNode): """ Extension class for the EcfNodes to identify tasks. @@ -1455,6 +1455,7 @@ def get_type(self): return 'family' + class EcfEventNode(EcfNode): """ Extension class for the EcfNodes to identify events. @@ -1472,6 +1473,7 @@ def get_type(self): return 'event' + class ecfTriggerNode(EcfNode): """ Extension class for the EcfNodes to identify triggers. Overloads the @@ -1719,7 +1721,7 @@ def has_event(self): """ if 'event' in self.task_setup.keys(): if self.is_list or self.is_range: - self.event = EcfEventNode(self.task_setup['event'],self) + self.event = EcfEventNode(self.task_setup['event'], self) else: self.event = EcfEventNode(self.task_setup['event'], self.ecfparent) @@ -1744,6 +1746,7 @@ def invalid_event_range(self): "Please review the configuration.") sys.exit(1) + class EcfEventNode(EcfNode): """ Extension class for the EcfNodes to identify events. @@ -1769,6 +1772,7 @@ def get_type(self): """ return 'event' + class EcfEditNode(EcfNode): """ Extension class for the EcfNodes to identify edits. @@ -1795,7 +1799,8 @@ def get_type(self): return 'edit' -class EcfRoot( ): + +class EcfRoot(): """ A root level class that is not an EcfNode object from above but an object that will extend a class from the ecflow module. @@ -1821,7 +1826,8 @@ def get_base_name(): The name of the node if it has a prefix, this strips out the surrounding range and just returns the beginning. """ - return re.search("(.*)\{.*\}",self.name()).group(1).strip() + return re.search(r"(.*)\{.*\}", self.name()).group(1).strip() + class EcfSuite(ecflow.Suite, EcfRoot): """ @@ -1855,6 +1861,7 @@ def generate_folders(self, ecfhome): if not os.path.exists(folder_path): os.makedirs(folder_path) + class EcfFamily(ecflow.Family, EcfRoot): """ Extends the ecflow.Family and EcfRoot classes to provide the folder @@ -1895,6 +1902,7 @@ def generate_folders(self, ecfhome, suite, parents): if not os.path.exists(folder_path): os.makedirs(folder_path) + class EcfTask(ecflow.Task, EcfRoot): """ Extends the ecflow.Task and EcfRoot classes to allow the task scripts to @@ -1964,12 +1972,12 @@ def generate_ecflow_task(self, ecfhome, suite, parents): script_name = f"{self.name()}.ecf" ecfscript = None search_script = f"{self.template}.ecf" if self.template is not \ - None else script_name + None else script_name if parents: script_path = f"{ecfhome}/{suite}/{parents.replace('>','/')}/{script_name}" else: script_path = f"{ecfhome}/{suite}/{script_name}" - for root,dirs,files in os.walk(self.scriptrepo): + for root, dirs, files in os.walk(self.scriptrepo): if search_script in files and ecfscript is None: ecfscript = os.path.join(root, search_script) elif script_name in files: @@ -1985,14 +1993,18 @@ def generate_ecflow_task(self, ecfhome, suite, parents): sys.exit(1) # define Python user-defined exceptions + + class Error(Exception): """Base class for other exceptions""" pass + class RangeError(Error): """Raised when the range in the configuration file is incorrect""" pass + class ConfigurationError(Error): """Raised when there is an error in the configuration file.""" pass diff --git a/workflow/ecFlow/ecflow_setup.py b/workflow/ecFlow/ecflow_setup.py index ca7e5162ca3..1170ebc4794 100644 --- a/workflow/ecFlow/ecflow_setup.py +++ b/workflow/ecFlow/ecflow_setup.py @@ -187,7 +187,7 @@ def get_suite_names(suitename): # If the name does have a list, break apart the prefix and suffix # from the list and then run it through a for loop to get all # possible values. - name_token = re.search("(.*)\[(.*)\](.*)", suitename) + name_token = re.search(r"(.*)\[(.*)\](.*)", suitename) base = name_token.group(1).strip() list_items = name_token.group(2).strip().split(',') suffix = name_token.group(3).strip() @@ -647,7 +647,7 @@ def find_env_param(node, value, envconfig): new_node = node if value in node: - variable_lookup = re.search(f".*{value}([\dA-Za-z_]*)", node).group(1).strip() + variable_lookup = re.search(fr".*{value}([\dA-Za-z_]*)", node).group(1).strip() if variable_lookup in os.environ: if isinstance(os.environ[variable_lookup], datetime.datetime): new_variable = os.environ[variable_lookup].strftime("%Y%m%d%H") @@ -705,7 +705,7 @@ def runupdate(nested_dict, value): for k, v in nested_dict.items(): if isinstance(v, str) and value in v: lookup = v.split('.') - variable_lookup = re.findall("[\dA-Za-z_]*", lookup[1])[0] + variable_lookup = re.findall(r"[\dA-Za-z_]*", lookup[1])[0] if variable_lookup in os.environ: if isinstance(os.environ[variable_lookup], datetime.datetime): nested_dict[k] = os.environ[variable_lookup].strftime("%Y%m%d%H") diff --git a/workflow/rocoto/workflow_tasks.py b/workflow/rocoto/workflow_tasks.py index e953ede3c91..b79cffe4bef 100644 --- a/workflow/rocoto/workflow_tasks.py +++ b/workflow/rocoto/workflow_tasks.py @@ -153,8 +153,8 @@ def coupled_ic(self): prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/{self.cdump}" for file in ['gfs_ctrl.nc'] + \ [f'{datatype}_data.tile{tile}.nc' - for datatype in ['gfs', 'sfc'] - for tile in range(1, self.n_tiles + 1)]: + for datatype in ['gfs', 'sfc'] + for tile in range(1, self.n_tiles + 1)]: data = f"{prefix}/{atm_res}/INPUT/{file}" dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) @@ -549,6 +549,7 @@ def ocnanalpost(self): dependency=dependencies) return task + def gldas(self): deps = [] @@ -1189,7 +1190,7 @@ def _get_ecengroups(): def esfc(self): - #eupd_cdump = 'gdas' if 'gdas' in self.app_config.eupd_cdumps else 'gfs' + # eupd_cdump = 'gdas' if 'gdas' in self.app_config.eupd_cdumps else 'gfs' deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}analcalc'} diff --git a/workflow/rocoto/workflow_xml.py b/workflow/rocoto/workflow_xml.py index 52ff86db2c3..f5eed3c9aa6 100644 --- a/workflow/rocoto/workflow_xml.py +++ b/workflow/rocoto/workflow_xml.py @@ -167,8 +167,8 @@ def _assemble_xml(self) -> str: return ''.join(strings) def write(self, xml_file: str = None, crontab_file: str = None): - self._write_xml(xml_file = xml_file) - self._write_crontab(crontab_file = crontab_file) + self._write_xml(xml_file=xml_file) + self._write_crontab(crontab_file=crontab_file) def _write_xml(self, xml_file: str = None) -> None: diff --git a/workflow/rocoto_viewer.py b/workflow/rocoto_viewer.py index 63db6f25383..84c337af694 100755 --- a/workflow/rocoto_viewer.py +++ b/workflow/rocoto_viewer.py @@ -12,9 +12,14 @@ # rocoto_viewer.py -w my_gfs-workflow.xml -d my_database.db # # The script is located in the directory para/exp/rocoto/rocotoviewers/rocotoviewer_curses/rocoto_viewer.py -# The view will continuously update every four minutes and reflect the current status of your workflow. You may use your mouse or arrow keys to select a particular task and view its status details by pressing the key \p c as indicated as \b \ (which runs \b rocotocheck) or perform a \b rocotorewind by pressing \b \ to restart the workflow at that point. Running \b rocotorewind causes the state information of that task to be cleared from the database and resubmits the job to the scheduler. +# The view will continuously update every four minutes and reflect the current status of your workflow. +# You may use your mouse or arrow keys to select a particular task and view its status details by pressing the key \p c as indicated as \b \ +# (which runs \b rocotocheck) or perform a \b rocotorewind by pressing \b \ to restart the workflow at that point. +# Running \b rocotorewind causes the state information of that task to be cleared from the database and resubmits the job to the scheduler. # -# Tasks marked with the \b \< symbol are \b metatasks and can be expanded by highlight that task with the mouse, and then clicking on the \b \< symbol which then changes to \b \> . You can then click on the \b \> symbol to collapse it again. Alternatively, you can select the 'x' to expand and collapse metatasks when selected. +# Tasks marked with the \b \< symbol are \b metatasks and can be expanded by highlight that task with the mouse, +# and then clicking on the \b \< symbol which then changes to \b \>. +# You can then click on the \b \> symbol to collapse it again. Alternatively, you can select the 'x' to expand and collapse metatasks when selected. # # @cond ROCOTO_VIEWER_CURSES @@ -125,6 +130,7 @@ mlines = 0 mcols = 0 + def eprint(message: str) -> None: """ Print to stderr instead of stdout @@ -208,7 +214,7 @@ def string_to_timedelta(td_string: str) -> timedelta: and mdict['negative'] == '-': return -dt return dt - except(TypeError, ValueError, AttributeError): + except (TypeError, ValueError, AttributeError): raise @@ -1113,9 +1119,14 @@ def get_rocoto_stat(params, queue_stat): (theid, jobid, task_order, taskname, cycle, state, exit_status, duration, tries) = row if jobid != '-': if use_performance_metrics: - line = f"{datetime.fromtimestamp(cycle).strftime('%Y%m%d%H%M')} {taskname} {str(jobid)} {str(state)} {str(exit_status)} {str(tries)} {str(duration).split('.')[0]} {str(slots)} {str(qtime)} {str(cputime).split('.')[0]} {str(runtime)}" + line = (f"{datetime.fromtimestamp(cycle).strftime('%Y%m%d%H%M')} " + f"{taskname} {str(jobid)} {str(state)} {str(exit_status)} " + f"{str(tries)} {str(duration).split('.')[0]} {str(slots)} " + f"{str(qtime)} {str(cputime).split('.')[0]} {str(runtime)}") else: - line = f"{datetime.fromtimestamp(cycle).strftime('%Y%m%d%H%M')} {taskname} {str(jobid)} {str(state)} {str(exit_status)} {str(tries)} {str(duration).split('.')[0]}" + line = (f"{datetime.fromtimestamp(cycle).strftime('%Y%m%d%H%M')} " + f"{taskname} {str(jobid)} {str(state)} {str(exit_status)} " + f"{str(tries)} {str(duration).split('.')[0]}") info[cycle].append(line) for every_cycle in cycles: @@ -1279,7 +1290,8 @@ def main(screen): use_multiprocessing = False # header_string = ' '*18+'CYCLE'+' '*17+'TASK'+' '*39+'JOBID'+' '*6+'STATE'+' '*9+'EXIT'+' '*2+'TRIES'+' '*2+'DURATION' - header_string = ' ' * 7 + 'CYCLE' + ' ' * (int(job_name_length_max / 2) + 3) + 'TASK' + ' ' * (int(job_name_length_max / 2) + 3) + 'JOBID' + ' ' * 6 + 'STATE' + ' ' * 9 + 'EXIT' + ' ' * 1 + 'TRIES' + ' ' * 1 + 'DURATION' + header_string = ' ' * 7 + 'CYCLE' + ' ' * (int(job_name_length_max / 2) + 3) + 'TASK' + ' ' * (int(job_name_length_max / 2) + 3) + \ + 'JOBID' + ' ' * 6 + 'STATE' + ' ' * 9 + 'EXIT' + ' ' * 1 + 'TRIES' + ' ' * 1 + 'DURATION' header_string_under = '=== (updated:tttttttttttttttt) =================== PSLOT: pslot ' + '=' * 44 global use_performance_metrics @@ -1336,8 +1348,10 @@ def main(screen): html_ptr = open(html_output_file, 'w') html_ptr.write(ccs_html) stat_update_time = str(datetime.now()).rsplit(':', 1)[0] - html_discribe_line = f'\n\n\n\n' - html_discribe_line += f'\n\n
ExpandRefreshed: {stat_update_time}PSLOT: {PSLOT}
ROTDIR: {workflow_name}Turn Around Times
\n
\n' + html_discribe_line = f'\n\n\n\n' + html_discribe_line += f'' + html_discribe_line += f'\n\n
' + html_discribe_line += f'ExpandRefreshed: {stat_update_time}PSLOT: {PSLOT}
ROTDIR: {workflow_name}Turn Around Times
\n
\n' html_discribe_line += html_header_line html_ptr.write(html_discribe_line) else: @@ -1701,7 +1715,9 @@ def main(screen): column = column[:7] html_line += f'{column}' elif i == 3: - if meta_tasks[cycle][line_num][1] and len(metatasks_state_string_cycle[cycle][columns[1]].split()) != 1 and metatasks_state_cycle[cycle][columns[1]]: + if meta_tasks[cycle][line_num][1] \ + and len(metatasks_state_string_cycle[cycle][columns[1]].split()) != 1 \ + and metatasks_state_cycle[cycle][columns[1]]: column = metatasks_state_string_cycle[cycle][columns[1]] if len(column) > 15: if column.split()[1] == 'SUCCEEDED': @@ -1780,8 +1796,10 @@ def main(screen): html_ptr = open(html_output_file, 'w') html_ptr.write(ccs_html) stat_update_time = str(datetime.now()).rsplit(':', 1)[0] - html_discribe_line = f'\n\n\n\n' - html_discribe_line += f'\n\n
CollapseRefreshed: {stat_update_time}PSLOT: {PSLOT}
ROTDIR: {workflow_name}Turn Around Times
\n
\n' + html_discribe_line = f'\n\n\n\n' + html_discribe_line += f'\n\n
' + html_discribe_line += f'CollapseRefreshed: {stat_update_time}PSLOT: {PSLOT}
ROTDIR: {workflow_name}' + html_discribe_line += f'Turn Around Times
\n
\n' html_discribe_line += html_header_line html_ptr.write(html_discribe_line) html_output_firstpass = False @@ -1947,7 +1965,9 @@ def main(screen): else: pad.addstr(job_id + ' ' * (11 - len(job_id))) elif i == 3: - if meta_tasks[cycle][line_num][1] and len(metatasks_state_string_cycle[cycle][columns[1]].split()) != 1 and metatasks_state_cycle[cycle][columns[1]]: + if meta_tasks[cycle][line_num][1] \ + and len(metatasks_state_string_cycle[cycle][columns[1]].split()) != 1 \ + and metatasks_state_cycle[cycle][columns[1]]: column = metatasks_state_string_cycle[cycle][columns[1]] if red_override: the_text_color = 2 diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 02a8a6f9593..482f6bc66ae 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -18,6 +18,7 @@ _here = os.path.dirname(__file__) _top = os.path.abspath(os.path.join(os.path.abspath(_here), '..')) + def makedirs_if_missing(dirname): """ Creates a directory if not already present @@ -177,7 +178,6 @@ def edit_baseconfig(host, inputs): } tmpl_dict = dict(tmpl_dict, **extend_dict) - base_input = f'{inputs.configdir}/config.base.emc.dyn' base_output = f'{inputs.expdir}/{inputs.pslot}/config.base' edit_config(base_input, base_output, tmpl_dict) @@ -207,6 +207,7 @@ def edit_config(input_config, output_config, config_dict): return + def get_template_dict(input_dict): output_dict = dict() for key, value in input_dict.items(): @@ -214,6 +215,7 @@ def get_template_dict(input_dict): return output_dict + def input_args(): """ Method to collect user arguments for `setup_expt.py` @@ -246,7 +248,8 @@ def input_args(): type=str, required=False, default=os.getenv('HOME')) subp.add_argument('--expdir', help='full path to EXPDIR', type=str, required=False, default=os.getenv('HOME')) - subp.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', required=True, type=lambda dd: datetime.strptime(dd, '%Y%m%d%H')) + subp.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', + required=True, type=lambda dd: datetime.strptime(dd, '%Y%m%d%H')) subp.add_argument('--edate', help='end date experiment', required=True, type=lambda dd: datetime.strptime(dd, '%Y%m%d%H')) subp.add_argument('--icsdir', help='full path to initial condition directory', type=str, required=False, default=None) subp.add_argument('--configdir', help='full path to directory containing the config files', @@ -259,7 +262,7 @@ def input_args(): choices=['warm', 'cold'], required=False, default='cold') subp.add_argument('--yaml', help='Defaults to substitute from', type=str, - required=False, default=os.path.join(_top, 'parm/config/yaml/defaults.yaml')) + required=False, default=os.path.join(_top, 'parm/config/yaml/defaults.yaml')) # cycled mode additional arguments cycled.add_argument('--resens', help='resolution of the ensemble model forecast', @@ -304,6 +307,7 @@ def query_and_clean(dirname): return create_dir + def validate_user_request(host, inputs): expt_res = f'C{inputs.resdet}' supp_res = host.info['SUPPORTED_RESOLUTIONS'] From b56a5236244e41ed40fd1d6ea7d7e93f85c63109 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 9 Dec 2022 22:59:20 -0500 Subject: [PATCH 7/8] use checkout@v3 as per warning message from GHA --- .github/workflows/pynorms.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pynorms.yaml b/.github/workflows/pynorms.yaml index 90e71a5f918..6f9046c7340 100644 --- a/.github/workflows/pynorms.yaml +++ b/.github/workflows/pynorms.yaml @@ -14,7 +14,7 @@ jobs: pip install pycodestyle - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: path: global-workflow From 5a831cb44002d30b2798be006e758e6b5c67aa90 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Mon, 12 Dec 2022 09:29:25 -0500 Subject: [PATCH 8/8] only retain E402 (load modules at top), W504 (line continuation) --- .pycodestyle | 2 +- ush/calcanl_gfs.py | 216 +++++++++++----------- ush/calcinc_gfs.py | 22 +-- ush/gsi_utils.py | 12 +- ush/python/pygw/src/pygw/configuration.py | 6 +- workflow/test_configuration.py | 6 +- 6 files changed, 132 insertions(+), 132 deletions(-) diff --git a/.pycodestyle b/.pycodestyle index 162186cb7c2..8bd18fa9d71 100644 --- a/.pycodestyle +++ b/.pycodestyle @@ -1,6 +1,6 @@ [pycodestyle] count = False -ignore = E226,E401,E402,W504 +ignore = E402,W504 max-line-length = 160 statistics = True exclude = Experimental diff --git a/ush/calcanl_gfs.py b/ush/calcanl_gfs.py index 4f8a4eb3e99..0243366241f 100755 --- a/ush/calcanl_gfs.py +++ b/ush/calcanl_gfs.py @@ -27,78 +27,78 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, for fh in IAUHH: if fh == 6: # for full res analysis - CalcAnlDir = RunDir+'/calcanl_'+format(fh, '02') + CalcAnlDir = RunDir + '/calcanl_' + format(fh, '02') if not os.path.exists(CalcAnlDir): gsi_utils.make_dir(CalcAnlDir) - gsi_utils.copy_file(ExecAnl, CalcAnlDir+'/calc_anl.x') - gsi_utils.link_file(RunDir+'/siginc.nc', CalcAnlDir+'/siginc.nc.06') - gsi_utils.link_file(RunDir+'/sigf06', CalcAnlDir+'/ges.06') - gsi_utils.link_file(RunDir+'/siganl', CalcAnlDir+'/anl.06') - gsi_utils.copy_file(ExecChgresInc, CalcAnlDir+'/chgres_inc.x') + gsi_utils.copy_file(ExecAnl, CalcAnlDir + '/calc_anl.x') + gsi_utils.link_file(RunDir + '/siginc.nc', CalcAnlDir + '/siginc.nc.06') + gsi_utils.link_file(RunDir + '/sigf06', CalcAnlDir + '/ges.06') + gsi_utils.link_file(RunDir + '/siganl', CalcAnlDir + '/anl.06') + gsi_utils.copy_file(ExecChgresInc, CalcAnlDir + '/chgres_inc.x') # for ensemble res analysis if Cdump in ["gdas", "gfs"]: - CalcAnlDir = RunDir+'/calcanl_ensres_'+format(fh, '02') + CalcAnlDir = RunDir + '/calcanl_ensres_' + format(fh, '02') if not os.path.exists(CalcAnlDir): gsi_utils.make_dir(CalcAnlDir) - gsi_utils.copy_file(ExecAnl, CalcAnlDir+'/calc_anl.x') - gsi_utils.link_file(RunDir+'/siginc.nc', CalcAnlDir+'/siginc.nc.06') - gsi_utils.link_file(ComOut+'/'+APrefix+'atmanl.ensres'+ASuffix, CalcAnlDir+'/anl.ensres.06') - gsi_utils.link_file(ComIn_Ges+'/'+GPrefix+'atmf006.ensres'+GSuffix, CalcAnlDir+'/ges.ensres.06') - gsi_utils.link_file(RunDir+'/sigf06', CalcAnlDir+'/ges.06') + gsi_utils.copy_file(ExecAnl, CalcAnlDir + '/calc_anl.x') + gsi_utils.link_file(RunDir + '/siginc.nc', CalcAnlDir + '/siginc.nc.06') + gsi_utils.link_file(ComOut + '/' + APrefix + 'atmanl.ensres' + ASuffix, CalcAnlDir + '/anl.ensres.06') + gsi_utils.link_file(ComIn_Ges + '/' + GPrefix + 'atmf006.ensres' + GSuffix, CalcAnlDir + '/ges.ensres.06') + gsi_utils.link_file(RunDir + '/sigf06', CalcAnlDir + '/ges.06') else: - if os.path.isfile('sigi'+format(fh, '02')+'.nc'): + if os.path.isfile('sigi' + format(fh, '02') + '.nc'): # for full res analysis - CalcAnlDir = RunDir+'/calcanl_'+format(fh, '02') - CalcAnlDir6 = RunDir+'/calcanl_'+format(6, '02') + CalcAnlDir = RunDir + '/calcanl_' + format(fh, '02') + CalcAnlDir6 = RunDir + '/calcanl_' + format(6, '02') if not os.path.exists(CalcAnlDir): gsi_utils.make_dir(CalcAnlDir) if not os.path.exists(CalcAnlDir6): gsi_utils.make_dir(CalcAnlDir6) - gsi_utils.link_file(ComOut+'/'+APrefix+'atma'+format(fh, '03')+ASuffix, - CalcAnlDir6+'/anl.'+format(fh, '02')) - gsi_utils.link_file(RunDir+'/siga'+format(fh, '02'), - CalcAnlDir6+'/anl.'+format(fh, '02')) - gsi_utils.link_file(RunDir+'/sigi'+format(fh, '02')+'.nc', - CalcAnlDir+'/siginc.nc.'+format(fh, '02')) - gsi_utils.link_file(CalcAnlDir6+'/inc.fullres.'+format(fh, '02'), - CalcAnlDir+'/inc.fullres.'+format(fh, '02')) - gsi_utils.link_file(RunDir+'/sigf'+format(fh, '02'), - CalcAnlDir6+'/ges.'+format(fh, '02')) - gsi_utils.link_file(RunDir+'/sigf'+format(fh, '02'), - CalcAnlDir+'/ges.'+format(fh, '02')) - gsi_utils.copy_file(ExecChgresInc, CalcAnlDir+'/chgres_inc.x') + gsi_utils.link_file(ComOut + '/' + APrefix + 'atma' + format(fh, '03') + ASuffix, + CalcAnlDir6 + '/anl.' + format(fh, '02')) + gsi_utils.link_file(RunDir + '/siga' + format(fh, '02'), + CalcAnlDir6 + '/anl.' + format(fh, '02')) + gsi_utils.link_file(RunDir + '/sigi' + format(fh, '02') + '.nc', + CalcAnlDir + '/siginc.nc.' + format(fh, '02')) + gsi_utils.link_file(CalcAnlDir6 + '/inc.fullres.' + format(fh, '02'), + CalcAnlDir + '/inc.fullres.' + format(fh, '02')) + gsi_utils.link_file(RunDir + '/sigf' + format(fh, '02'), + CalcAnlDir6 + '/ges.' + format(fh, '02')) + gsi_utils.link_file(RunDir + '/sigf' + format(fh, '02'), + CalcAnlDir + '/ges.' + format(fh, '02')) + gsi_utils.copy_file(ExecChgresInc, CalcAnlDir + '/chgres_inc.x') # for ensemble res analysis - CalcAnlDir = RunDir+'/calcanl_ensres_'+format(fh, '02') - CalcAnlDir6 = RunDir+'/calcanl_ensres_'+format(6, '02') + CalcAnlDir = RunDir + '/calcanl_ensres_' + format(fh, '02') + CalcAnlDir6 = RunDir + '/calcanl_ensres_' + format(6, '02') if not os.path.exists(CalcAnlDir): gsi_utils.make_dir(CalcAnlDir) if not os.path.exists(CalcAnlDir6): gsi_utils.make_dir(CalcAnlDir6) - gsi_utils.link_file(ComOut+'/'+APrefix+'atma'+format(fh, '03')+'.ensres'+ASuffix, - CalcAnlDir6+'/anl.ensres.'+format(fh, '02')) - gsi_utils.link_file(RunDir+'/sigi'+format(fh, '02')+'.nc', - CalcAnlDir6+'/siginc.nc.'+format(fh, '02')) - gsi_utils.link_file(ComIn_Ges+'/'+GPrefix+'atmf'+format(fh, '03')+'.ensres'+GSuffix, - CalcAnlDir6+'/ges.ensres.'+format(fh, '02')) + gsi_utils.link_file(ComOut + '/' + APrefix + 'atma' + format(fh, '03') + '.ensres' + ASuffix, + CalcAnlDir6 + '/anl.ensres.' + format(fh, '02')) + gsi_utils.link_file(RunDir + '/sigi' + format(fh, '02') + '.nc', + CalcAnlDir6 + '/siginc.nc.' + format(fh, '02')) + gsi_utils.link_file(ComIn_Ges + '/' + GPrefix + 'atmf' + format(fh, '03') + '.ensres' + GSuffix, + CalcAnlDir6 + '/ges.ensres.' + format(fh, '02')) else: # for full res analysis - CalcAnlDir = RunDir+'/calcanl_'+format(6, '02') + CalcAnlDir = RunDir + '/calcanl_' + format(6, '02') if not os.path.exists(CalcAnlDir): gsi_utils.make_dir(CalcAnlDir) - gsi_utils.copy_file(ExecAnl, CalcAnlDir+'/calc_anl.x') - gsi_utils.link_file(RunDir+'/siginc.nc', CalcAnlDir+'/siginc.nc.06') - gsi_utils.link_file(RunDir+'/sigf06', CalcAnlDir+'/ges.06') - gsi_utils.link_file(RunDir+'/siganl', CalcAnlDir+'/anl.06') - gsi_utils.copy_file(ExecChgresInc, CalcAnlDir+'/chgres_inc.x') + gsi_utils.copy_file(ExecAnl, CalcAnlDir + '/calc_anl.x') + gsi_utils.link_file(RunDir + '/siginc.nc', CalcAnlDir + '/siginc.nc.06') + gsi_utils.link_file(RunDir + '/sigf06', CalcAnlDir + '/ges.06') + gsi_utils.link_file(RunDir + '/siganl', CalcAnlDir + '/anl.06') + gsi_utils.copy_file(ExecChgresInc, CalcAnlDir + '/chgres_inc.x') # for ensemble res analysis - CalcAnlDir = RunDir+'/calcanl_ensres_'+format(6, '02') + CalcAnlDir = RunDir + '/calcanl_ensres_' + format(6, '02') if not os.path.exists(CalcAnlDir): gsi_utils.make_dir(CalcAnlDir) - gsi_utils.copy_file(ExecAnl, CalcAnlDir+'/calc_anl.x') - gsi_utils.link_file(RunDir+'/siginc.nc', CalcAnlDir+'/siginc.nc.06') - gsi_utils.link_file(ComOut+'/'+APrefix+'atmanl.ensres'+ASuffix, CalcAnlDir+'/anl.ensres.06') - gsi_utils.link_file(ComIn_Ges+'/'+GPrefix+'atmf006.ensres'+GSuffix, CalcAnlDir+'/ges.ensres.06') + gsi_utils.copy_file(ExecAnl, CalcAnlDir + '/calc_anl.x') + gsi_utils.link_file(RunDir + '/siginc.nc', CalcAnlDir + '/siginc.nc.06') + gsi_utils.link_file(ComOut + '/' + APrefix + 'atmanl.ensres' + ASuffix, CalcAnlDir + '/anl.ensres.06') + gsi_utils.link_file(ComIn_Ges + '/' + GPrefix + 'atmf006.ensres' + GSuffix, CalcAnlDir + '/ges.ensres.06') # get dimension information from background and increment files AnlDims = gsi_utils.get_ncdims('siginc.nc') @@ -115,18 +115,18 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, # vertical coordinate info levs2 = levs + 1 - siglevel = FixDir+'/global_hyblev.l'+str(levs2)+'.txt' + siglevel = FixDir + '/global_hyblev.l' + str(levs2) + '.txt' # determine how many forecast hours to process nFH = 0 for fh in IAUHH: # first check to see if increment file exists - CalcAnlDir = RunDir+'/calcanl_'+format(fh, '02') - if (os.path.isfile(CalcAnlDir+'/siginc.nc.'+format(fh, '02'))): - print('will process increment file: '+CalcAnlDir+'/siginc.nc.'+format(fh, '02')) + CalcAnlDir = RunDir + '/calcanl_' + format(fh, '02') + if (os.path.isfile(CalcAnlDir + '/siginc.nc.' + format(fh, '02'))): + print('will process increment file: ' + CalcAnlDir + '/siginc.nc.' + format(fh, '02')) nFH += 1 else: - print('Increment file: '+CalcAnlDir+'/siginc.nc.'+format(fh, '02')+' does not exist. Skipping.') + print('Increment file: ' + CalcAnlDir + '/siginc.nc.' + format(fh, '02') + ' does not exist. Skipping.') sys.stdout.flush() # need to gather information about runtime environment @@ -148,14 +148,14 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, hosts = [] [hosts.append(x) for x in hosts_tmp if x not in hosts] nhosts = len(hosts) - ExecCMDMPI_host = 'mpirun -np '+str(nFH)+' --hostfile hosts' + ExecCMDMPI_host = 'mpirun -np ' + str(nFH) + ' --hostfile hosts' tasks = int(os.getenv('LSB_DJOB_NUMPROC', 1)) if levs > tasks: - ExecCMDMPILevs_host = 'mpirun -np '+str(tasks)+' --hostfile hosts' - ExecCMDMPILevs_nohost = 'mpirun -np '+str(tasks) + ExecCMDMPILevs_host = 'mpirun -np ' + str(tasks) + ' --hostfile hosts' + ExecCMDMPILevs_nohost = 'mpirun -np ' + str(tasks) else: - ExecCMDMPILevs_host = 'mpirun -np '+str(levs)+' --hostfile hosts' - ExecCMDMPILevs_nohost = 'mpirun -np '+str(levs) + ExecCMDMPILevs_host = 'mpirun -np ' + str(levs) + ' --hostfile hosts' + ExecCMDMPILevs_nohost = 'mpirun -np ' + str(levs) ExecCMDMPI1_host = 'mpirun -np 1 --hostfile hosts' ExecCMDMPI10_host = 'mpirun -np 10 --hostfile hosts' elif launcher == 'mpiexec': @@ -166,20 +166,20 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, hosts = [] [hosts.append(x) for x in hosts_tmp if x not in hosts] nhosts = len(hosts) - ExecCMDMPI_host = 'mpiexec -l -n '+str(nFH) + ExecCMDMPI_host = 'mpiexec -l -n ' + str(nFH) tasks = int(os.getenv('ntasks', 1)) print('nhosts,tasks=', nhosts, tasks) if levs > tasks: - ExecCMDMPILevs_host = 'mpiexec -l -n '+str(tasks) - ExecCMDMPILevs_nohost = 'mpiexec -l -n '+str(tasks) + ExecCMDMPILevs_host = 'mpiexec -l -n ' + str(tasks) + ExecCMDMPILevs_nohost = 'mpiexec -l -n ' + str(tasks) else: - ExecCMDMPILevs_host = 'mpiexec -l -n '+str(levs) - ExecCMDMPILevs_nohost = 'mpiexec -l -n '+str(levs) - ExecCMDMPI1_host = 'mpiexec -l -n 1 --cpu-bind depth --depth '+str(NThreads) - ExecCMDMPI10_host = 'mpiexec -l -n 10 --cpu-bind depth --depth '+str(NThreads) + ExecCMDMPILevs_host = 'mpiexec -l -n ' + str(levs) + ExecCMDMPILevs_nohost = 'mpiexec -l -n ' + str(levs) + ExecCMDMPI1_host = 'mpiexec -l -n 1 --cpu-bind depth --depth ' + str(NThreads) + ExecCMDMPI10_host = 'mpiexec -l -n 10 --cpu-bind depth --depth ' + str(NThreads) elif launcher == 'srun': nodes = os.getenv('SLURM_JOB_NODELIST', '') - hosts_tmp = subprocess.check_output('scontrol show hostnames '+nodes, shell=True) + hosts_tmp = subprocess.check_output('scontrol show hostnames ' + nodes, shell=True) if (sys.version_info > (3, 0)): hosts_tmp = hosts_tmp.decode('utf-8') hosts_tmp = str(hosts_tmp).splitlines() @@ -191,15 +191,15 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, hosts = [] [hosts.append(x) for x in hosts_tmp if x not in hosts] nhosts = len(hosts) - ExecCMDMPI_host = 'srun -n '+str(nFH)+' --verbose --export=ALL -c 1 --distribution=arbitrary --cpu-bind=cores' + ExecCMDMPI_host = 'srun -n ' + str(nFH) + ' --verbose --export=ALL -c 1 --distribution=arbitrary --cpu-bind=cores' # need to account for when fewer than LEVS tasks are available tasks = int(os.getenv('SLURM_NPROCS', 1)) if levs > tasks: - ExecCMDMPILevs_host = 'srun -n '+str(tasks)+' --verbose --export=ALL -c 1 --distribution=arbitrary --cpu-bind=cores' - ExecCMDMPILevs_nohost = 'srun -n '+str(tasks)+' --verbose --export=ALL' + ExecCMDMPILevs_host = 'srun -n ' + str(tasks) + ' --verbose --export=ALL -c 1 --distribution=arbitrary --cpu-bind=cores' + ExecCMDMPILevs_nohost = 'srun -n ' + str(tasks) + ' --verbose --export=ALL' else: - ExecCMDMPILevs_host = 'srun -n '+str(levs)+' --verbose --export=ALL -c 1 --distribution=arbitrary --cpu-bind=cores' - ExecCMDMPILevs_nohost = 'srun -n '+str(levs)+' --verbose --export=ALL' + ExecCMDMPILevs_host = 'srun -n ' + str(levs) + ' --verbose --export=ALL -c 1 --distribution=arbitrary --cpu-bind=cores' + ExecCMDMPILevs_nohost = 'srun -n ' + str(levs) + ' --verbose --export=ALL' ExecCMDMPI1_host = 'srun -n 1 --verbose --export=ALL -c 1 --distribution=arbitrary --cpu-bind=cores' ExecCMDMPI10_host = 'srun -n 10 --verbose --export=ALL -c 1 --distribution=arbitrary --cpu-bind=cores' elif launcher == 'aprun': @@ -210,11 +210,11 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, hosts = [] [hosts.append(x) for x in hosts_tmp if x not in hosts] nhosts = len(hosts) - ExecCMDMPI_host = 'aprun -l hosts -d '+str(NThreads)+' -n '+str(nFH) - ExecCMDMPILevs_host = 'aprun -l hosts -d '+str(NThreads)+' -n '+str(levs) - ExecCMDMPILevs_nohost = 'aprun -d '+str(NThreads)+' -n '+str(levs) - ExecCMDMPI1_host = 'aprun -l hosts -d '+str(NThreads)+' -n 1' - ExecCMDMPI10_host = 'aprun -l hosts -d '+str(NThreads)+' -n 10' + ExecCMDMPI_host = 'aprun -l hosts -d ' + str(NThreads) + ' -n ' + str(nFH) + ExecCMDMPILevs_host = 'aprun -l hosts -d ' + str(NThreads) + ' -n ' + str(levs) + ExecCMDMPILevs_nohost = 'aprun -d ' + str(NThreads) + ' -n ' + str(levs) + ExecCMDMPI1_host = 'aprun -l hosts -d ' + str(NThreads) + ' -n 1' + ExecCMDMPI10_host = 'aprun -l hosts -d ' + str(NThreads) + ' -n 10' else: print('unknown MPI launcher. Failure.') sys.exit(1) @@ -224,51 +224,51 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, # interpolate increment to full background resolution for fh in IAUHH: # first check to see if increment file exists - CalcAnlDir = RunDir+'/calcanl_'+format(fh, '02') - if (os.path.isfile(CalcAnlDir+'/siginc.nc.'+format(fh, '02'))): - print('Interpolating increment for f'+format(fh, '03')) + CalcAnlDir = RunDir + '/calcanl_' + format(fh, '02') + if (os.path.isfile(CalcAnlDir + '/siginc.nc.' + format(fh, '02'))): + print('Interpolating increment for f' + format(fh, '03')) # set up the namelist namelist = OrderedDict() namelist["setup"] = {"lon_out": LonB, "lat_out": LatB, "lev": levs, - "infile": "'siginc.nc."+format(fh, '02')+"'", - "outfile": "'inc.fullres."+format(fh, '02')+"'", + "infile": "'siginc.nc." + format(fh, '02') + "'", + "outfile": "'inc.fullres." + format(fh, '02') + "'", } - gsi_utils.write_nml(namelist, CalcAnlDir+'/fort.43') + gsi_utils.write_nml(namelist, CalcAnlDir + '/fort.43') if ihost >= nhosts: ihost = 0 - with open(CalcAnlDir+'/hosts', 'w') as hostfile: - hostfile.write(hosts[ihost]+'\n') + with open(CalcAnlDir + '/hosts', 'w') as hostfile: + hostfile.write(hosts[ihost] + '\n') if launcher == 'srun': # need to write host per task not per node for slurm # For xjet, each instance of chgres_inc must run on two nodes each if os.getenv('SLURM_JOB_PARTITION', '') == 'xjet': for a in range(0, 4): - hostfile.write(hosts[ihost]+'\n') + hostfile.write(hosts[ihost] + '\n') ihost += 1 for a in range(0, 5): - hostfile.write(hosts[ihost]+'\n') + hostfile.write(hosts[ihost] + '\n') for a in range(0, 9): # need 9 more of the same host for the 10 tasks for chgres_inc - hostfile.write(hosts[ihost]+'\n') + hostfile.write(hosts[ihost] + '\n') if launcher == 'srun': - os.environ['SLURM_HOSTFILE'] = CalcAnlDir+'/hosts' + os.environ['SLURM_HOSTFILE'] = CalcAnlDir + '/hosts' print('interp_inc', fh, namelist) - job = subprocess.Popen(ExecCMDMPI10_host+' '+CalcAnlDir+'/chgres_inc.x', shell=True, cwd=CalcAnlDir) - print(ExecCMDMPI10_host+' '+CalcAnlDir+'/chgres_inc.x submitted on '+hosts[ihost]) + job = subprocess.Popen(ExecCMDMPI10_host + ' ' + CalcAnlDir + '/chgres_inc.x', shell=True, cwd=CalcAnlDir) + print(ExecCMDMPI10_host + ' ' + CalcAnlDir + '/chgres_inc.x submitted on ' + hosts[ihost]) sys.stdout.flush() ec = job.wait() if ec != 0: - print('Error with chgres_inc.x at forecast hour: f'+format(fh, '03')) - print('Error with chgres_inc.x, exit code='+str(ec)) + print('Error with chgres_inc.x at forecast hour: f' + format(fh, '03')) + print('Error with chgres_inc.x, exit code=' + str(ec)) print(locals()) sys.exit(ec) ihost += 1 else: - print('f'+format(fh, '03')+' is in $IAUFHRS but increment file is missing. Skipping.') + print('f' + format(fh, '03') + ' is in $IAUFHRS but increment file is missing. Skipping.') # generate analysis from interpolated increment - CalcAnlDir6 = RunDir+'/calcanl_'+format(6, '02') + CalcAnlDir6 = RunDir + '/calcanl_' + format(6, '02') # set up the namelist namelist = OrderedDict() namelist["setup"] = {"datapath": "'./'", @@ -278,22 +278,22 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, "fhr": 6, } - gsi_utils.write_nml(namelist, CalcAnlDir6+'/calc_analysis.nml') + gsi_utils.write_nml(namelist, CalcAnlDir6 + '/calc_analysis.nml') # run the executable - if ihost >= nhosts-1: + if ihost >= nhosts - 1: ihost = 0 if launcher == 'srun': del os.environ['SLURM_HOSTFILE'] print('fullres_calc_anl', namelist) - fullres_anl_job = subprocess.Popen(ExecCMDMPILevs_nohost+' '+CalcAnlDir6+'/calc_anl.x', shell=True, cwd=CalcAnlDir6) - print(ExecCMDMPILevs_nohost+' '+CalcAnlDir6+'/calc_anl.x submitted') + fullres_anl_job = subprocess.Popen(ExecCMDMPILevs_nohost + ' ' + CalcAnlDir6 + '/calc_anl.x', shell=True, cwd=CalcAnlDir6) + print(ExecCMDMPILevs_nohost + ' ' + CalcAnlDir6 + '/calc_anl.x submitted') sys.stdout.flush() exit_fullres = fullres_anl_job.wait() sys.stdout.flush() if exit_fullres != 0: - print('Error with calc_analysis.x for deterministic resolution, exit code='+str(exit_fullres)) + print('Error with calc_analysis.x for deterministic resolution, exit code=' + str(exit_fullres)) print(locals()) sys.exit(exit_fullres) @@ -302,10 +302,10 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, chgres_jobs = [] for fh in IAUHH: # first check to see if guess file exists - CalcAnlDir6 = RunDir+'/calcanl_ensres_06' - print(CalcAnlDir6+'/ges.ensres.'+format(fh, '02')) - if (os.path.isfile(CalcAnlDir6+'/ges.ensres.'+format(fh, '02'))): - print('Calculating analysis on ensemble resolution for f'+format(fh, '03')) + CalcAnlDir6 = RunDir + '/calcanl_ensres_06' + print(CalcAnlDir6 + '/ges.ensres.' + format(fh, '02')) + if (os.path.isfile(CalcAnlDir6 + '/ges.ensres.' + format(fh, '02'))): + print('Calculating analysis on ensemble resolution for f' + format(fh, '03')) # generate ensres analysis from interpolated background # set up the namelist namelist = OrderedDict() @@ -316,24 +316,24 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, "fhr": fh, } - gsi_utils.write_nml(namelist, CalcAnlDir6+'/calc_analysis.nml') + gsi_utils.write_nml(namelist, CalcAnlDir6 + '/calc_analysis.nml') # run the executable - if ihost > nhosts-1: + if ihost > nhosts - 1: ihost = 0 print('ensres_calc_anl', namelist) - ensres_anl_job = subprocess.Popen(ExecCMDMPILevs_nohost+' '+CalcAnlDir6+'/calc_anl.x', shell=True, cwd=CalcAnlDir6) - print(ExecCMDMPILevs_nohost+' '+CalcAnlDir6+'/calc_anl.x submitted') + ensres_anl_job = subprocess.Popen(ExecCMDMPILevs_nohost + ' ' + CalcAnlDir6 + '/calc_anl.x', shell=True, cwd=CalcAnlDir6) + print(ExecCMDMPILevs_nohost + ' ' + CalcAnlDir6 + '/calc_anl.x submitted') sys.stdout.flush() # check on analysis steps exit_ensres = ensres_anl_job.wait() if exit_ensres != 0: - print('Error with calc_analysis.x for ensemble resolution, exit code='+str(exit_ensres)) + print('Error with calc_analysis.x for ensemble resolution, exit code=' + str(exit_ensres)) print(locals()) sys.exit(exit_ensres) else: - print('f'+format(fh, '03')+' is in $IAUFHRS but ensemble resolution guess file is missing. Skipping.') + print('f' + format(fh, '03') + ' is in $IAUFHRS but ensemble resolution guess file is missing. Skipping.') print('calcanl_gfs successfully completed at: ', datetime.datetime.utcnow()) print(locals()) diff --git a/ush/calcinc_gfs.py b/ush/calcinc_gfs.py index 648f0f14f3f..b3562d38805 100755 --- a/ush/calcinc_gfs.py +++ b/ush/calcinc_gfs.py @@ -24,21 +24,21 @@ def calcinc_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, IAUHrs, for fh in IAUHrs: nFH += 1 if fh == 6: - gsi_utils.link_file('sigf06', 'atmges_mem'+format(nFH, '03')) - gsi_utils.link_file('siganl', 'atmanl_mem'+format(nFH, '03')) - gsi_utils.link_file(ComOut+'/'+APrefix+'atminc.nc', 'atminc_mem'+format(nFH, '03')) + gsi_utils.link_file('sigf06', 'atmges_mem' + format(nFH, '03')) + gsi_utils.link_file('siganl', 'atmanl_mem' + format(nFH, '03')) + gsi_utils.link_file(ComOut + '/' + APrefix + 'atminc.nc', 'atminc_mem' + format(nFH, '03')) else: - gsi_utils.link_file('sigf'+format(fh, '02'), 'atmges_mem'+format(nFH, '03')) - gsi_utils.link_file('siga'+format(fh, '02'), 'atmanl_mem'+format(nFH, '03')) - gsi_utils.link_file(ComOut+'/'+APrefix+'atmi'+format(fh, '03')+'.nc', 'atminc_mem'+format(nFH, '03')) + gsi_utils.link_file('sigf' + format(fh, '02'), 'atmges_mem' + format(nFH, '03')) + gsi_utils.link_file('siga' + format(fh, '02'), 'atmanl_mem' + format(nFH, '03')) + gsi_utils.link_file(ComOut + '/' + APrefix + 'atmi' + format(fh, '03') + '.nc', 'atminc_mem' + format(nFH, '03')) else: nFH = 1 gsi_utils.link_file('sigf06', 'atmges_mem001') gsi_utils.link_file('siganl', 'atmanl_mem001') - gsi_utils.link_file(ComOut+'/'+APrefix+'atminc', 'atminc_mem001') + gsi_utils.link_file(ComOut + '/' + APrefix + 'atminc', 'atminc_mem001') os.environ['OMP_NUM_THREADS'] = str(NThreads) os.environ['ncmd'] = str(nFH) - shutil.copy(Exec, RunDir+'/calc_inc.x') + shutil.copy(Exec, RunDir + '/calc_inc.x') ExecCMD = ExecCMD.replace("$ncmd", str(nFH)) # set up the namelist @@ -53,14 +53,14 @@ def calcinc_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ASuffix, IAUHrs, namelist["zeroinc"] = {"incvars_to_zero": Inc2Zero} - gsi_utils.write_nml(namelist, RunDir+'/calc_increment.nml') + gsi_utils.write_nml(namelist, RunDir + '/calc_increment.nml') # run the executable try: - err = subprocess.check_call(ExecCMD+' '+RunDir+'/calc_inc.x', shell=True) + err = subprocess.check_call(ExecCMD + ' ' + RunDir + '/calc_inc.x', shell=True) print(locals()) except subprocess.CalledProcessError as e: - print('Error with calc_inc.x, exit code='+str(e.returncode)) + print('Error with calc_inc.x, exit code=' + str(e.returncode)) print(locals()) sys.exit(e.returncode) diff --git a/ush/gsi_utils.py b/ush/gsi_utils.py index fbc85e1fd5e..97d66e8ace5 100644 --- a/ush/gsi_utils.py +++ b/ush/gsi_utils.py @@ -29,22 +29,22 @@ def link_file(from_file, to_file): if not os.path.islink(to_file): os.symlink(from_file, to_file) else: - print(to_file+" exists, unlinking.") + print(to_file + " exists, unlinking.") os.unlink(to_file) os.symlink(from_file, to_file) - print("ln -s "+from_file+" "+to_file) + print("ln -s " + from_file + " " + to_file) def copy_file(from_file, to_file): import shutil shutil.copy(from_file, to_file) - print("cp "+from_file+" "+to_file) + print("cp " + from_file + " " + to_file) def make_dir(directory): import os os.makedirs(directory) - print("mkdir -p "+directory) + print("mkdir -p " + directory) def write_nml(nml_dict, nml_file): @@ -58,9 +58,9 @@ def write_nml(nml_dict, nml_file): nfile = open(nml_file, 'w') for nml, nmlvars in nml_dict.items(): - nfile.write('&'+nml+'\n') + nfile.write('&' + nml + '\n') for var, val in nmlvars.items(): - nfile.write(' '+str(var)+' = '+str(val)+'\n') + nfile.write(' ' + str(var) + ' = ' + str(val) + '\n') nfile.write('/\n\n') nfile.close() diff --git a/ush/python/pygw/src/pygw/configuration.py b/ush/python/pygw/src/pygw/configuration.py index cc6d5890284..f00adcf5a85 100644 --- a/ush/python/pygw/src/pygw/configuration.py +++ b/ush/python/pygw/src/pygw/configuration.py @@ -131,10 +131,10 @@ def _get_shell_env(scripts: List) -> Dict[str, Any]: begin = out.find(magic) if begin < 0: raise ShellScriptException(scripts, 'Cannot find magic string; ' - 'at least one script failed: '+repr(out)) - for entry in out[begin+len(magic):].split('\x00'): + 'at least one script failed: ' + repr(out)) + for entry in out[begin + len(magic):].split('\x00'): iequal = entry.find('=') - varbls[entry[0:iequal]] = entry[iequal+1:] + varbls[entry[0:iequal]] = entry[iequal + 1:] return varbls @staticmethod diff --git a/workflow/test_configuration.py b/workflow/test_configuration.py index d129da5819c..5c59fd35bf7 100644 --- a/workflow/test_configuration.py +++ b/workflow/test_configuration.py @@ -14,19 +14,19 @@ print(f'config.base: {cfg.find_config("config.base")}') -print('*'*80) +print('*' * 80) print('config.base ...') base = cfg.parse_config('config.base') cfg.print_config('config.base') print(type(base)) print(base.HOMEgfs) -print('*'*80) +print('*' * 80) print('config.anal...') cfg.print_config(['config.base', 'config.anal']) -print('*'*80) +print('*' * 80) print('config.efcs ...') configs = ['config.base', 'config.fcst', 'config.efcs'] cfg.print_config(configs)