tooling: Async/pathlib/mypy cleanups and utils

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
-addopts = -raq --ignore=tools/testing/external/*,__init__.py,testing/conf --color=yes --cov-append -p  tools.testing.plugin --cov-config=.coveragerc -vv tools
+addopts = -raq --ignore=tools/testing/external/*,__init__.py,testing/conf --color=yes --cov-append -p  tools.testing.plugin --cov-config=.coveragerc -Werror -vv tools
 testpaths =
 	  tests

tools/base/BUILD

@@ -16,10 +16,13 @@ envoy_py_library(
     ],
 )
 
+envoy_py_library("tools.base.functional")
+
 envoy_py_library(
     "tools.base.utils",
     deps = [
         requirement("pyyaml"),
+        requirement("setuptools"),
     ],
 )
 

tools/base/checker.py

@@ -1,20 +1,20 @@
 import argparse
 import asyncio
 import logging
-import os
+import pathlib
 from functools import cached_property
-from typing import Optional, Sequence, Tuple, Type
+from typing import Any, Iterable, Optional, Sequence, Tuple, Type
 
 from tools.base import runner
 
 
-class Checker(runner.Runner):
+class BaseChecker(runner.Runner):
     """Runs check methods prefixed with `check_` and named in `self.checks`
 
     Check methods should call the `self.warn`, `self.error` or `self.succeed`
     depending upon the outcome of the checks.
     """
-    _active_check: Optional[str] = None
+    _active_check = ""
     checks: Tuple[str, ...] = ()
 
     def __init__(self, *args):
@@ -24,7 +24,7 @@ def __init__(self, *args):
         self.warnings = {}
 
     @property
-    def active_check(self) -> Optional[str]:
+    def active_check(self) -> str:
         return self._active_check
 
     @property
@@ -58,14 +58,14 @@ def has_failed(self) -> bool:
         return bool(self.failed or self.warned)
 
     @cached_property
-    def path(self) -> str:
+    def path(self) -> pathlib.Path:
         """The "path" - usually Envoy src dir. This is used for finding configs for the tooling and should be a dir"""
         try:
-            path = self.args.path or self.args.paths[0]
+            path = pathlib.Path(self.args.path or self.args.paths[0])
         except IndexError:
             raise self.parser.error(
                 "Missing path: `path` must be set either as an arg or with --path")
-        if not os.path.isdir(path):
+        if not path.is_dir():
             raise self.parser.error(
                 "Incorrect path: `path` must be a directory, set either as first arg or with --path"
             )
@@ -174,7 +174,12 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None:
             "Paths to check. At least one path must be specified, or the `path` argument should be provided"
         )
 
-    def error(self, name: str, errors: list, log: bool = True, log_type: str = "error") -> int:
+    def error(
+            self,
+            name: str,
+            errors: Optional[Iterable[str]],
+            log: bool = True,
+            log_type: str = "error") -> int:
         """Record (and log) errors for a check type"""
         if not errors:
             return 0
@@ -197,13 +202,13 @@ def get_checks(self) -> Sequence[str]:
             self.checks if not self.args.check else
             [check for check in self.args.check if check in self.checks])
 
-    def on_check_begin(self, check: str) -> None:
+    def on_check_begin(self, check: str) -> Any:
         self._active_check = check
         self.log.notice(f"[{check}] Running check")
 
-    def on_check_run(self, check: str) -> None:
+    def on_check_run(self, check: str) -> Any:
         """Callback hook called after each check run"""
-        self._active_check = None
+        self._active_check = ""
         if self.exiting:
             return
         elif check in self.errors:
@@ -213,11 +218,11 @@ def on_check_run(self, check: str) -> None:
         else:
             self.log.success(f"[{check}] Check completed successfully")
 
-    def on_checks_begin(self) -> None:
+    def on_checks_begin(self) -> Any:
         """Callback hook called before all checks"""
         pass
 
-    def on_checks_complete(self) -> int:
+    def on_checks_complete(self) -> Any:
         """Callback hook called after all checks have run, and returning the final outcome of a checks_run"""
         if self.show_summary:
             self.summary.print_summary()
@@ -257,6 +262,21 @@ def warn(self, name: str, warnings: list, log: bool = True) -> None:
             self.log.warning(f"[{name}] {message}")
 
 
+class Checker(BaseChecker):
+
+    def on_check_begin(self, check: str) -> None:
+        super().on_check_begin(check)
+
+    def on_check_run(self, check: str) -> None:
+        super().on_check_run(check)
+
+    def on_checks_begin(self) -> None:
+        super().on_checks_complete()
+
+    def on_checks_complete(self) -> int:
+        return super().on_checks_complete()
+
+
 class ForkingChecker(runner.ForkingRunner, Checker):
     pass
 
@@ -267,7 +287,7 @@ class BazelChecker(runner.BazelRunner, Checker):
 
 class CheckerSummary(object):
 
-    def __init__(self, checker: Checker):
+    def __init__(self, checker: BaseChecker):
         self.checker = checker
 
     @property
@@ -319,7 +339,7 @@ def _section(self, message: str, lines: list = None) -> list:
         return section
 
 
-class AsyncChecker(Checker):
+class AsyncChecker(BaseChecker):
     """Async version of the Checker class for use with asyncio"""
 
     async def _run(self) -> int:

tools/base/functional.py

@@ -0,0 +1,68 @@
+#
+# Functional utilities
+#
+
+from typing import Any, Callable, Optional
+
+
+class NoCache(Exception):
+    pass
+
+
+class async_property:  # noqa: N801
+    name = None
+    cache_name = "__async_prop_cache__"
+    _instance = None
+
+    # If the decorator is called with `kwargs` then `fun` is `None`
+    # and instead `__call__` is triggered with `fun`
+    def __init__(self, fun: Optional[Callable] = None, cache: bool = False):
+        self.cache = cache
+        self._fun = fun
+        self.name = getattr(fun, "__name__", None)
+        self.__doc__ = getattr(fun, '__doc__')
+
+    def __call__(self, fun: Callable) -> 'async_property':
+        self._fun = fun
+        self.name = self.name or fun.__name__
+        self.__doc__ = getattr(fun, '__doc__')
+        return self
+
+    def __get__(self, instance: Any, cls=None) -> Any:
+        if instance is None:
+            return self
+        self._instance = instance
+        return self.async_result()
+
+    def fun(self, *args, **kwargs):
+        if self._fun:
+            return self._fun(*args, **kwargs)
+
+    @property
+    def prop_cache(self) -> dict:
+        return getattr(self._instance, self.cache_name, {})
+
+    # An async wrapper function to return the result
+    # This is returned when the prop is called
+    async def async_result(self) -> Any:
+        # retrieve the value from cache if available
+        try:
+            return self.get_cached_prop()
+        except (NoCache, KeyError):
+            pass
+
+        # derive the result, set the cache if required, and return the result
+        return self.set_prop_cache(await self.fun(self._instance))
+
+    def get_cached_prop(self) -> Any:
+        if not self.cache:
+            raise NoCache
+        return self.prop_cache[self.name]
+
+    def set_prop_cache(self, result: Any) -> Any:
+        if not self.cache:
+            return result
+        cache = self.prop_cache
+        cache[self.name] = result
+        setattr(self._instance, self.cache_name, cache)
+        return result

tools/base/requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --generate-hashes tools/base/requirements.txt
+#    pip-compile --allow-unsafe --generate-hashes tools/base/requirements.txt
 #
 colorama==0.4.4 \
     --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \
@@ -52,8 +52,14 @@ pyyaml==5.4.1 \
     --hash=sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247 \
     --hash=sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6 \
     --hash=sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0
-    # via -r tools/distribution/requirements.txt
+    # via -r tools/base/requirements.txt
 verboselogs==1.7 \
     --hash=sha256:d63f23bf568295b95d3530c6864a0b580cec70e7ff974177dead1e4ffbc6ff49 \
     --hash=sha256:e33ddedcdfdafcb3a174701150430b11b46ceb64c2a9a26198c76a156568e427
     # via -r tools/base/requirements.txt
+
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==57.4.0 \
+    --hash=sha256:6bac238ffdf24e8806c61440e755192470352850f3419a52f26ffe0a1a64f465 \
+    --hash=sha256:a49230977aa6cfb9d933614d2f7b79036e9945c4cdd7583163f4e920b83418d6
+    # via -r tools/base/requirements.txt

tools/base/runner.py

@@ -3,24 +3,25 @@
 #
 
 import argparse
+import inspect
 import logging
-import os
+import pathlib
 import subprocess
 import sys
 from functools import cached_property, wraps
-from typing import Callable, Tuple, Optional, Union
+from typing import Callable, Optional, Tuple, Type, Union
 
 from frozendict import frozendict
 
-import coloredlogs
-import verboselogs
+import coloredlogs  # type:ignore
+import verboselogs  # type:ignore
 
 LOG_LEVELS = (("debug", logging.DEBUG), ("info", logging.INFO), ("warn", logging.WARN),
               ("error", logging.ERROR))
-LOG_FIELD_STYLES = frozendict(
+LOG_FIELD_STYLES: frozendict = frozendict(
     name=frozendict(color="blue"), levelname=frozendict(color="cyan", bold=True))
 LOG_FMT = "%(name)s %(levelname)s %(message)s"
-LOG_LEVEL_STYLES = frozendict(
+LOG_LEVEL_STYLES: frozendict = frozendict(
     critical=frozendict(bold=True, color="red"),
     debug=frozendict(color="green"),
     error=frozendict(color="red", bold=True),
@@ -32,7 +33,7 @@
     warning=frozendict(color="yellow", bold=True))
 
 
-def catches(errors: Union[Tuple[Exception], Exception]) -> Callable:
+def catches(errors: Union[Type[Exception], Tuple[Type[Exception], ...]]) -> Callable:
     """Method decorator to catch specified errors
 
     logs and returns 1 for sys.exit if error/s are caught
@@ -48,6 +49,7 @@ def run(self):
             self.myrun()
     ```
 
+    Can work with `async` methods too.
     """
 
     def wrapper(fun: Callable) -> Callable:
@@ -60,7 +62,15 @@ def wrapped(self, *args, **kwargs) -> Optional[int]:
                 self.log.error(str(e) or repr(e))
                 return 1
 
-        return wrapped
+        @wraps(fun)
+        async def async_wrapped(self, *args, **kwargs) -> Optional[int]:
+            try:
+                return await fun(self, *args, **kwargs)
+            except errors as e:
+                self.log.error(str(e) or repr(e))
+                return 1
+
+        return async_wrapped if inspect.iscoroutinefunction(fun) else wrapped
 
     return wrapper
 
@@ -103,7 +113,7 @@ def log_level_styles(self):
         return LOG_LEVEL_STYLES
 
     @cached_property
-    def log(self) -> logging.Logger:
+    def log(self) -> verboselogs.VerboseLogger:
         """Instantiated logger"""
         verboselogs.install()
         logger = logging.getLogger(self.name)
@@ -135,8 +145,8 @@ def parser(self) -> argparse.ArgumentParser:
         return parser
 
     @cached_property
-    def path(self) -> str:
-        return os.getcwd()
+    def path(self) -> pathlib.Path:
+        return pathlib.Path(".")
 
     @cached_property
     def stdout(self) -> logging.Logger: