initial commit for incoming yaml work

ush/python/pygw/README.md

@@ -0,0 +1,21 @@
+# global workflow specific tools
+
+Python tools specifically for global applications
+
+## Installation
+Simple installation instructions
+```sh
+$> git clone https://github.com/noaa-emc/global-workflow
+$> cd global-workflow/ush/python
+$> pip install .
+```
+
+It is not required to install this package.  Instead, 
+```sh
+$> cd global-workflow/ush/python
+$> export PYTHONPATH=$PWD/src/pygw
+```
+would put this package in the `PYTHONPATH`
+
+### Note:
+These instructions will be updated and the tools are under development.

ush/python/pygw/setup.cfg

@@ -0,0 +1,62 @@
+[metadata]
+name = pygw
+version = 0.0.1
+description = Global applications specific workflow related tools
+long_description = file: README.md
+long_description_content_type = text/markdown
+author = "NOAA/NWS/NCEP/EMC"
+#author_email = first.last@domain.tld
+keywords = NOAA, NWS, NCEP, EMC, GFS, GEFS
+home_page = https://github.com/noaa-emc/global-workflow
+license = GNU Lesser General Public License
+classifiers =
+  Development Status :: 1 - Beta
+  Intended Audience :: Developers
+  Intended Audience :: Science/Research
+  License :: OSI Approved :: GNU Lesser General Public License
+  Natural Language :: English
+  Operating System :: OS Independent
+  Programming Language :: Python
+  Programming Language :: Python :: 3
+  Programming Language :: Python :: 3.6
+  Programming Language :: Python :: 3.7
+  Programming Language :: Python :: 3.8
+  Programming Language :: Python :: 3.9
+  Topic :: Software Development :: Libraries :: Python Modules
+  Operating System :: OS Independent
+  Typing :: Typed
+project_urls =
+  Bug Tracker = https://github.com/noaa-emc/global-workflow/issues
+  CI = https://github.com/noaa-emc/global-workflow/actions
+
+[options]
+zip_safe = False
+include_package_data = True
+package_dir =
+  =src
+packages = find_namespace:
+python_requires = >= 3.6
+setup_requires =
+  setuptools
+install_requires =
+  numpy==1.21.6
+  PyYAML==6.0
+  Jinja2==3.1.2
+tests_require =
+  pytest
+
+[options.packages.find]
+where=src
+
+[options.package_data]
+* = *.txt, *.md
+
+[options.extras_require]
+dev = pytest-cov>=3
+
+[green]
+file-pattern = test_*.py
+verbose = 2
+no-skip-report = true
+quiet-stdout = true
+run-coverage = true

ush/python/pygw/setup.py

@@ -0,0 +1,4 @@
+''' Standard file for building the package with Distutils. '''
+
+import setuptools
+setuptools.setup()

ush/python/pygw/src/pygw/__init__.py

@@ -0,0 +1,9 @@
+"""
+Commonly used toolset for the global applications and beyond.
+"""
+__docformat__ = "restructuredtext"
+
+import os
+
+pygw_directory = os.path.dirname(__file__)
+

ush/python/pygw/src/pygw/attrdict.py

@@ -0,0 +1,169 @@
+# attrdict is a Python module that gives you dictionaries whose values are both
+# gettable and settable using attributes, in addition to standard item-syntax.
+# https://github.com/mewwts/addict
+# addict/addict.py -> attrdict.py
+# hash: 7e8d23d
+# License: MIT
+# class Dict -> class AttrDict to prevent name collisions w/ typing.Dict
+
+import copy
+
+__all__ = ['AttrDict']
+
+
+class AttrDict(dict):
+
+    def __init__(__self, *args, **kwargs):
+        object.__setattr__(__self, '__parent', kwargs.pop('__parent', None))
+        object.__setattr__(__self, '__key', kwargs.pop('__key', None))
+        object.__setattr__(__self, '__frozen', False)
+        for arg in args:
+            if not arg:
+                continue
+            elif isinstance(arg, dict):
+                for key, val in arg.items():
+                    __self[key] = __self._hook(val)
+            elif isinstance(arg, tuple) and (not isinstance(arg[0], tuple)):
+                __self[arg[0]] = __self._hook(arg[1])
+            else:
+                for key, val in iter(arg):
+                    __self[key] = __self._hook(val)
+
+        for key, val in kwargs.items():
+            __self[key] = __self._hook(val)
+
+    def __setattr__(self, name, value):
+        if hasattr(self.__class__, name):
+            raise AttributeError("'AttrDict' object attribute "
+                                 "'{0}' is read-only".format(name))
+        else:
+            self[name] = value
+
+    def __setitem__(self, name, value):
+        isFrozen = (hasattr(self, '__frozen') and
+                    object.__getattribute__(self, '__frozen'))
+        if isFrozen and name not in super(AttrDict, self).keys():
+            raise KeyError(name)
+        super(AttrDict, self).__setitem__(name, value)
+        try:
+            p = object.__getattribute__(self, '__parent')
+            key = object.__getattribute__(self, '__key')
+        except AttributeError:
+            p = None
+            key = None
+        if p is not None:
+            p[key] = self
+            object.__delattr__(self, '__parent')
+            object.__delattr__(self, '__key')
+
+    def __add__(self, other):
+        if not self.keys():
+            return other
+        else:
+            self_type = type(self).__name__
+            other_type = type(other).__name__
+            msg = "unsupported operand type(s) for +: '{}' and '{}'"
+            raise TypeError(msg.format(self_type, other_type))
+
+    @classmethod
+    def _hook(cls, item):
+        if isinstance(item, dict):
+            return cls(item)
+        elif isinstance(item, (list, tuple)):
+            return type(item)(cls._hook(elem) for elem in item)
+        return item
+
+    def __getattr__(self, item):
+        return self.__getitem__(item)
+
+    def __missing__(self, name):
+        if object.__getattribute__(self, '__frozen'):
+            raise KeyError(name)
+        return self.__class__(__parent=self, __key=name)
+
+    def __delattr__(self, name):
+        del self[name]
+
+    def to_dict(self):
+        base = {}
+        for key, value in self.items():
+            if isinstance(value, type(self)):
+                base[key] = value.to_dict()
+            elif isinstance(value, (list, tuple)):
+                base[key] = type(value)(
+                    item.to_dict() if isinstance(item, type(self)) else
+                    item for item in value)
+            else:
+                base[key] = value
+        return base
+
+    def copy(self):
+        return copy.copy(self)
+
+    def deepcopy(self):
+        return copy.deepcopy(self)
+
+    def __deepcopy__(self, memo):
+        other = self.__class__()
+        memo[id(self)] = other
+        for key, value in self.items():
+            other[copy.deepcopy(key, memo)] = copy.deepcopy(value, memo)
+        return other
+
+    def update(self, *args, **kwargs):
+        other = {}
+        if args:
+            if len(args) > 1:
+                raise TypeError()
+            other.update(args[0])
+        other.update(kwargs)
+        for k, v in other.items():
+            if ((k not in self) or
+                (not isinstance(self[k], dict)) or
+                    (not isinstance(v, dict))):
+                self[k] = v
+            else:
+                self[k].update(v)
+
+    def __getnewargs__(self):
+        return tuple(self.items())
+
+    def __getstate__(self):
+        return self
+
+    def __setstate__(self, state):
+        self.update(state)
+
+    def __or__(self, other):
+        if not isinstance(other, (AttrDict, dict)):
+            return NotImplemented
+        new = AttrDict(self)
+        new.update(other)
+        return new
+
+    def __ror__(self, other):
+        if not isinstance(other, (AttrDict, dict)):
+            return NotImplemented
+        new = AttrDict(other)
+        new.update(self)
+        return new
+
+    def __ior__(self, other):
+        self.update(other)
+        return self
+
+    def setdefault(self, key, default=None):
+        if key in self:
+            return self[key]
+        else:
+            self[key] = default
+            return default
+
+    def freeze(self, shouldFreeze=True):
+        object.__setattr__(self, '__frozen', shouldFreeze)
+        for key, val in self.items():
+            if isinstance(val, AttrDict):
+                val.freeze(shouldFreeze)
+
+    def unfreeze(self):
+        self.freeze(False)

ush/python/pygw/src/pygw/fsutils.py

@@ -0,0 +1,47 @@
+import os
+import errno
+import shutil
+import contextlib
+
+__all__ = ['mkdir', 'mkdir_p', 'rmdir', 'chdir', 'rm_p']
+
+
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise OSError(f"unable to create directory at {path}")
+
+
+mkdir = mkdir_p
+
+
+def rmdir(dir_path):
+    try:
+        shutil.rmtree(dir_path)
+    except OSError as exc:
+        raise OSError(f"unable to remove {dir_path}")
+
+
+@contextlib.contextmanager
+def chdir(path):
+    cwd = os.getcwd()
+    try:
+        os.chdir(path)
+        yield
+    finally:
+        print(f"WARNING: Unable to chdir({path})")  # TODO: use logging
+        os.chdir(cwd)
+
+
+def rm_p(path):
+    try:
+        os.unlink(path)
+    except OSError as exc:
+        if exc.errno == errno.ENOENT:
+            pass
+        else:
+            raise OSError(f"unable to remove {path}")

ush/python/pygw/src/pygw/jinja.py

@@ -0,0 +1,57 @@
+import io
+import os
+import sys
+import jinja2
+from pathlib import Path
+
+
+class Jinja:
+
+    def __init__(self, template_path, data, allow_missing=True):
+        """
+        Given a path to a (jinja2) template and a data object, substitute the
+        template file with data.
+        Allow for retaining missing or undefined variables.
+        """
+
+        self.data = data
+        self.undefined = jinja2.Undefined if allow_missing else jinja2.StrictUndefined
+
+        if Path(template_path).is_file():
+            self.template_path = Path(template_path)
+            self.output = self._render_file()
+        else:
+            self.output = self._render_stream()
+
+    def _render_stream(self):
+        raise NotImplementedError("Unable to handle templates other than files")
+
+    def _render_file(self):
+        template_dir = self.template_path.parent
+        template_file = self.template_path.relative_to(template_dir)
+
+        dirname = os.path.dirname(str(self.template_path))
+        relpath = os.path.relpath(str(self.template_path), dirname)
+
+        loader = jinja2.FileSystemLoader(template_dir)
+        output = self._render(str(template_file), loader)
+
+        return output
+
+    def _render(self, template_name, loader):
+        env = jinja2.Environment(loader=loader, undefined=self.undefined)
+        template = env.get_template(template_name)
+        try:
+            rendered = template.render(**self.data)
+        except jinja2.UndefinedError as ee:
+            raise Exception(f"Undefined variable in Jinja2 template\n{ee}")
+
+        return rendered
+
+    def save(self, output_file):
+        with open(output_file, 'wb') as fh:
+            fh.write(self.output.encode("utf-8"))
+
+    def dump(self):
+        io.TextIOWrapper(sys.stdout.buffer,
+                         encoding="utf-8").write(self.output)