From a33a0fc08bfc2648c2d06ff2bf7c527ee7016633 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 14 Apr 2023 11:34:04 -0400 Subject: [PATCH 1/4] initial blocks in place --- parm/config/config.fcst | 1 + scripts/exglobal_forecast.py | 27 +++++++++++++++ ush/python/pygfs/task/gfs_forecast.py | 35 +++++++++++++++++++ ush/python/pygfs/ufswm/__init__.py | 0 ush/python/pygfs/ufswm/gfs.py | 24 +++++++++++++ ush/python/pygfs/ufswm/ufs.py | 49 +++++++++++++++++++++++++++ 6 files changed, 136 insertions(+) create mode 100755 scripts/exglobal_forecast.py create mode 100644 ush/python/pygfs/task/gfs_forecast.py create mode 100644 ush/python/pygfs/ufswm/__init__.py create mode 100644 ush/python/pygfs/ufswm/gfs.py create mode 100644 ush/python/pygfs/ufswm/ufs.py diff --git a/parm/config/config.fcst b/parm/config/config.fcst index 357b68512c0..2a57647644d 100644 --- a/parm/config/config.fcst +++ b/parm/config/config.fcst @@ -71,6 +71,7 @@ fi ####################################################################### export FORECASTSH="$HOMEgfs/scripts/exglobal_forecast.sh" +#export FORECASTSH="$HOMEgfs/scripts/exglobal_forecast.py" # Temp. while this is worked on export FCSTEXECDIR="$HOMEgfs/exec" export FCSTEXEC="ufs_model.x" diff --git a/scripts/exglobal_forecast.py b/scripts/exglobal_forecast.py new file mode 100755 index 00000000000..2b21934bfac --- /dev/null +++ b/scripts/exglobal_forecast.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +import os + +from pygw.logger import Logger, logit +from pygw.yaml_file import save_as_yaml +from pygw.configuration import cast_strdict_as_dtypedict +from pygfs.task.gfs_forecast import GFSForecast + +# initialize root logger +logger = Logger(level=os.environ.get("LOGGING_LEVEL"), colored_log=True) + + +@logit(logger) +def main(): + + # instantiate the forecast + config = cast_strdict_as_dtypedict(os.environ) + save_as_yaml(config, f'{config.EXPDIR}/fcst.yaml') # Temporarily save the input to the Forecast + + fcst = GFSForecast(config) + fcst.initialize() + fcst.configure() + + +if __name__ == '__main__': + main() diff --git a/ush/python/pygfs/task/gfs_forecast.py b/ush/python/pygfs/task/gfs_forecast.py new file mode 100644 index 00000000000..9ef7a44525f --- /dev/null +++ b/ush/python/pygfs/task/gfs_forecast.py @@ -0,0 +1,35 @@ +import os +import logging +from typing import Dict + +from pygw.logger import logit +from pygfs.ufswm.ufs import UFS + +logger = logging.getLogger(__name__.split('.')[-1]) + + +class GFSForecast(Task): + """ + UFS-weather-model forecast task for the GFS + """ + + @logit(logger, name="GFSForecast") + def __init__(self, config, *args, **kwargs): + """ + Parameters + ---------- + config : Dict + dictionary object containing configuration from environment + + *args : tuple + Additional arguments to `Task` + + **kwargs : dict, optional + Extra keyword arguments to `Task` + """ + + super().__init__(config, *args, **kwargs) + + # Create and initialize the GFS variant of the UFS + self.gfs = UFS.create(model_name:str = "GFS", config: Dict) + diff --git a/ush/python/pygfs/ufswm/__init__.py b/ush/python/pygfs/ufswm/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ush/python/pygfs/ufswm/gfs.py b/ush/python/pygfs/ufswm/gfs.py new file mode 100644 index 00000000000..e5460fc1124 --- /dev/null +++ b/ush/python/pygfs/ufswm/gfs.py @@ -0,0 +1,24 @@ +import copy +import logging + +from pygw.logger import logit +from pygfs.ufswm.ufs import UFS + +logger = logging.getLogger(__name__.split('.')[-1]) + + +class GFS(UFS): + + @logit(logger, name="GFS") + def __init__(self, config): + + # Note there is no super() here needed since UFS does not initialize for all configurations equally. If this need comes along, we can adjust. + + # Make a deep copy of incoming config for caching purposes. _config should not be updated + self._config = copy.deepcopy(config) + + # Start putting fixed properties of the GFS + self.ntiles = 6 + + # Determine coupled/uncoupled from config and define as appropriate + diff --git a/ush/python/pygfs/ufswm/ufs.py b/ush/python/pygfs/ufswm/ufs.py new file mode 100644 index 00000000000..d22b6976e0e --- /dev/null +++ b/ush/python/pygfs/ufswm/ufs.py @@ -0,0 +1,49 @@ +import logging +from typing import Dict + +from pygw.template import Template, TemplateConstants +from pygw.logger import logit + +logger = logging.getLogger(__name__.split('.')[-1]) + + +class UFS: + + @classmethod + @logit(logger) + def create(cls, model_name: str, config: Dict, *args, **kwargs): + """ + Call the constructor of the appropriate variant of the UFS. + The variant is defined via 'model_name'. + E.g. + GFS for all global variants + """ + + return next(cc for cc in cls.__subclasses__() if cc.__name__ == model_name)(config, *args, **kwargs) + + + @logit(logger) + def parse_ufs_templates(input_template, output_file, ctx: Dict) -> None: + """ + This method parses UFS-weather-model templates of the pattern @[VARIABLE] + drawing the value from ctx['VARIABLE'] + """ + + with open(input_template, 'r') as fhi: + file_in = fhi.read() + file_out = Template.substitute_structure( + file_in, TemplateConstants.AT_SQUARE_BRACES, ctx.get) + + # If there are unrendered bits, find out what they are + pattern = r"@\[.*?\]+" + matches = re.findall(pattern, file_out) + if matches: + logger.warn(f"{input_template} was rendered incompletely") + logger.warn(f"The following variables were not substituted") + print(matches) # TODO: improve the formatting of this message + # TODO: Should we abort here? or continue to write output_file? + + with open(output_file, 'w') as fho: + fho.write(file_out) + + From 604be8085bc67f7dd1e28a5cfd704282589bcc26 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 14 Apr 2023 18:28:51 -0400 Subject: [PATCH 2/4] this works start to finish cleanly --- jobs/rocoto/fcst.sh | 18 +++++++++++++++ ush/python/pygfs/task/gfs_forecast.py | 10 ++++----- ush/python/pygfs/ufswm/gfs.py | 6 +---- ush/python/pygfs/ufswm/ufs.py | 32 +++++++++++++++++---------- ush/python/pygw/src/pygw/logger.py | 14 +++++------- 5 files changed, 50 insertions(+), 30 deletions(-) diff --git a/jobs/rocoto/fcst.sh b/jobs/rocoto/fcst.sh index c00d30814b1..f39ddcb34ea 100755 --- a/jobs/rocoto/fcst.sh +++ b/jobs/rocoto/fcst.sh @@ -18,10 +18,28 @@ module load prod_util if [[ "${MACHINE_ID}" = "wcoss2" ]]; then module load cray-pals fi +if [[ "${MACHINE_ID}" = "hera" ]]; then + module use "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/modulefiles/core" + module load "miniconda3/4.6.14" + module load "gfs_workflow/1.0.0" +#elif [[ "${MACHINE_ID}" = "orion" ]]; then +# module use "/home/rmahajan/opt/global-workflow/modulefiles/core" +# module load "python/3.7.5" +# module load "gfs_workflow/1.0.0" +#elif [[ "${MACHINE_ID}" = "wcoss2" ]]; then +# module load "python/3.7.5" +fi module list unset MACHINE_ID set_trace +############################################################### +# exglobal_forecast.py requires the following in PYTHONPATH +# This will be moved to a module load when ready +pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src:${HOMEgfs}/ush/python/pygfs" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}" +export PYTHONPATH + export job="fcst" export jobid="${job}.$$" diff --git a/ush/python/pygfs/task/gfs_forecast.py b/ush/python/pygfs/task/gfs_forecast.py index 9ef7a44525f..12e5f101448 100644 --- a/ush/python/pygfs/task/gfs_forecast.py +++ b/ush/python/pygfs/task/gfs_forecast.py @@ -1,9 +1,10 @@ import os import logging -from typing import Dict +from typing import Dict, Any from pygw.logger import logit -from pygfs.ufswm.ufs import UFS +from pygw.task import Task +from pygfs.ufswm.gfs import GFS logger = logging.getLogger(__name__.split('.')[-1]) @@ -14,7 +15,7 @@ class GFSForecast(Task): """ @logit(logger, name="GFSForecast") - def __init__(self, config, *args, **kwargs): + def __init__(self, config: Dict[str, Any], *args, **kwargs): """ Parameters ---------- @@ -31,5 +32,4 @@ def __init__(self, config, *args, **kwargs): super().__init__(config, *args, **kwargs) # Create and initialize the GFS variant of the UFS - self.gfs = UFS.create(model_name:str = "GFS", config: Dict) - + self.gfs = GFS(config) \ No newline at end of file diff --git a/ush/python/pygfs/ufswm/gfs.py b/ush/python/pygfs/ufswm/gfs.py index e5460fc1124..f86164d706f 100644 --- a/ush/python/pygfs/ufswm/gfs.py +++ b/ush/python/pygfs/ufswm/gfs.py @@ -12,13 +12,9 @@ class GFS(UFS): @logit(logger, name="GFS") def __init__(self, config): - # Note there is no super() here needed since UFS does not initialize for all configurations equally. If this need comes along, we can adjust. - - # Make a deep copy of incoming config for caching purposes. _config should not be updated - self._config = copy.deepcopy(config) + super().__init__("GFS", config) # Start putting fixed properties of the GFS self.ntiles = 6 # Determine coupled/uncoupled from config and define as appropriate - diff --git a/ush/python/pygfs/ufswm/ufs.py b/ush/python/pygfs/ufswm/ufs.py index d22b6976e0e..b63f91e5ab7 100644 --- a/ush/python/pygfs/ufswm/ufs.py +++ b/ush/python/pygfs/ufswm/ufs.py @@ -1,26 +1,36 @@ +import re +import copy import logging -from typing import Dict +from typing import Dict, Any from pygw.template import Template, TemplateConstants from pygw.logger import logit logger = logging.getLogger(__name__.split('.')[-1]) +UFS_VARIANTS = ['GFS'] class UFS: - @classmethod - @logit(logger) - def create(cls, model_name: str, config: Dict, *args, **kwargs): - """ - Call the constructor of the appropriate variant of the UFS. - The variant is defined via 'model_name'. - E.g. - GFS for all global variants + @logit(logger, name="UFS") + def __init__(self, model_name: str, config: Dict[str, Any]): + """Initialize the UFS-weather-model generic class and check if the model_name is a valid variant + + Parameters + ---------- + model_name: str + UFS variant + config : Dict + Incoming configuration dictionary """ - return next(cc for cc in cls.__subclasses__() if cc.__name__ == model_name)(config, *args, **kwargs) + # First check if this is a valid variant + if model_name not in UFS_VARIANTS: + logger.warn(f"{model_name} is not a valid UFS variant") + raise NotImplementedError(f"{model_name} is not yet implemented") + # Make a deep copy of incoming config for caching purposes. _config should not be updated + self._config = copy.deepcopy(config) @logit(logger) def parse_ufs_templates(input_template, output_file, ctx: Dict) -> None: @@ -45,5 +55,3 @@ def parse_ufs_templates(input_template, output_file, ctx: Dict) -> None: with open(output_file, 'w') as fho: fho.write(file_out) - - diff --git a/ush/python/pygw/src/pygw/logger.py b/ush/python/pygw/src/pygw/logger.py index 71782bfece2..1bf2ed29858 100644 --- a/ush/python/pygw/src/pygw/logger.py +++ b/ush/python/pygw/src/pygw/logger.py @@ -2,6 +2,7 @@ Logger """ +import os import sys from functools import wraps from pathlib import Path @@ -48,7 +49,7 @@ class Logger: DEFAULT_FORMAT = '%(asctime)s - %(levelname)-8s - %(name)-12s: %(message)s' def __init__(self, name: str = None, - level: str = None, + level: str = os.environ.get("LOGGING_LEVEL"), _format: str = DEFAULT_FORMAT, colored_log: bool = False, logfile_path: Union[str, Path] = None): @@ -74,18 +75,15 @@ def __init__(self, name: str = None, default : None """ - if level is None: - level = os.environ.get("LOGGING_LEVEL", Logger.DEFAULT_LEVEL) - self.name = name - self.level = level.upper() + self.level = level.upper() if level else Logger.DEFAULT_LEVEL self.format = _format self.colored_log = colored_log if self.level not in Logger.LOG_LEVELS: - raise LookupError('{self.level} is unknown logging level\n' + - 'Currently supported log levels are:\n' + - f'{" | ".join(Logger.LOG_LEVELS)}') + raise LookupError(f"{self.level} is unknown logging level\n" + + f"Currently supported log levels are:\n" + + f"{' | '.join(Logger.LOG_LEVELS)}") # Initialize the root logger if no name is present self._logger = logging.getLogger(name) if name else logging.getLogger() From 0a56b8d7c124000101ee078a0e4cc59b5a73eee8 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 14 Apr 2023 18:31:00 -0400 Subject: [PATCH 3/4] fix norms --- ush/python/pygfs/task/gfs_forecast.py | 2 +- ush/python/pygfs/ufswm/ufs.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/gfs_forecast.py b/ush/python/pygfs/task/gfs_forecast.py index 12e5f101448..3527c623e0d 100644 --- a/ush/python/pygfs/task/gfs_forecast.py +++ b/ush/python/pygfs/task/gfs_forecast.py @@ -32,4 +32,4 @@ def __init__(self, config: Dict[str, Any], *args, **kwargs): super().__init__(config, *args, **kwargs) # Create and initialize the GFS variant of the UFS - self.gfs = GFS(config) \ No newline at end of file + self.gfs = GFS(config) diff --git a/ush/python/pygfs/ufswm/ufs.py b/ush/python/pygfs/ufswm/ufs.py index b63f91e5ab7..a9118801b92 100644 --- a/ush/python/pygfs/ufswm/ufs.py +++ b/ush/python/pygfs/ufswm/ufs.py @@ -10,6 +10,7 @@ UFS_VARIANTS = ['GFS'] + class UFS: @logit(logger, name="UFS") From e9bfb0f0daacd383d84f615e9feaba258610a6a0 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 14 Apr 2023 18:34:04 -0400 Subject: [PATCH 4/4] put a TODO in the comment block for fcst.sh --- jobs/rocoto/fcst.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/rocoto/fcst.sh b/jobs/rocoto/fcst.sh index f39ddcb34ea..512bee127f3 100755 --- a/jobs/rocoto/fcst.sh +++ b/jobs/rocoto/fcst.sh @@ -22,6 +22,7 @@ if [[ "${MACHINE_ID}" = "hera" ]]; then module use "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/modulefiles/core" module load "miniconda3/4.6.14" module load "gfs_workflow/1.0.0" +# TODO: orion and wcoss2 will be uncommented when they are ready. This comment block will be removed in the next PR #elif [[ "${MACHINE_ID}" = "orion" ]]; then # module use "/home/rmahajan/opt/global-workflow/modulefiles/core" # module load "python/3.7.5"