tooling: Add misc contextmanagers/decorators

.github/dependabot.yml

@@ -26,6 +26,11 @@ updates:
   schedule:
     interval: "daily"
 
+- package-ecosystem: "pip"
+  directory: "/tools/git"
+  schedule:
+    interval: "daily"
+
 - package-ecosystem: "pip"
   directory: "/tools/github"
   schedule:

bazel/repositories_extra.bzl

@@ -52,6 +52,11 @@ def _python_deps():
         requirements = "@envoy//tools/extensions:requirements.txt",
         extra_pip_args = ["--require-hashes"],
     )
+    pip_install(
+        name = "git_pip3",
+        requirements = "@envoy//tools/git:requirements.txt",
+        extra_pip_args = ["--require-hashes"],
+    )
     pip_install(
         name = "kafka_pip3",
         requirements = "@envoy//source/extensions/filters/network/kafka:requirements.txt",

tools/base/runner.py

@@ -7,12 +7,46 @@
 import os
 import subprocess
 import sys
-from functools import cached_property
+from functools import cached_property, wraps
+from typing import Callable, Tuple, Optional, Union
 
 LOG_LEVELS = (("debug", logging.DEBUG), ("info", logging.INFO), ("warn", logging.WARN),
               ("error", logging.ERROR))
 
 
+def catches(errors: Union[Tuple[Exception], Exception]) -> Callable:
+    """Method decorator to catch specified errors
+
+    logs and returns 1 for sys.exit if error/s are caught
+
+    can be used as so:
+
+    ```python
+
+    class MyRunner(runner.Runner):
+
+        @runner.catches((MyError, MyOtherError))
+        def run(self):
+            self.myrun()
+    ```
+
+    """
+
+    def wrapper(fun: Callable) -> Callable:
+
+        @wraps(fun)
+        def wrapped(self, *args, **kwargs) -> Optional[int]:
+            try:
+                return fun(self, *args, **kwargs)
+            except errors as e:
+                self.log.error(str(e) or repr(e))
+                return 1
+
+        return wrapped
+
+    return wrapper
+
+
 class BazelRunError(Exception):
     pass
 

tools/base/tests/test_runner.py

@@ -25,6 +25,89 @@ def __init__(self):
         self.args = PropertyMock()
 
 
+class Error1(Exception):
+
+    def __str__(self):
+        return ""
+
+    pass
+
+
+class Error2(Exception):
+    pass
+
+
+def _failing_runner(errors):
+
+    class DummyFailingRunner(object):
+
+        log = PropertyMock()
+        _runner = MagicMock()
+
+        def __init__(self, raises=None):
+            self.raises = raises
+
+        @runner.catches(errors)
+        def run(self, *args, **kwargs):
+            result = self._runner(*args, **kwargs)
+            if self.raises:
+                raise self.raises("AN ERROR OCCURRED")
+            return result
+
+    return DummyFailingRunner
+
+
+@pytest.mark.parametrize(
+    "errors",
+    [Error1, (Error1, Error2)])
+@pytest.mark.parametrize(
+    "raises",
+    [None, Error1, Error2])
+@pytest.mark.parametrize(
+    "args",
+    [(), ("ARG1", "ARG2")])
+@pytest.mark.parametrize(
+    "kwargs",
+    [{}, dict(key1="VAL1", key2="VAL2")])
+def test_catches(errors, raises, args, kwargs):
+    run = _failing_runner(errors)(raises)
+    should_fail = (
+        raises
+        and not (
+            raises == errors
+            or (isinstance(errors, tuple)
+                and raises in errors)))
+
+    if should_fail:
+        result = 1
+        with pytest.raises(raises):
+            run.run(*args, **kwargs)
+    else:
+        result = run.run(*args, **kwargs)
+
+    assert (
+        list(run._runner.call_args)
+        == [args, kwargs])
+
+    if not should_fail and raises:
+        assert result == 1
+        error = run.log.error.call_args[0][0]
+        _error = raises("AN ERROR OCCURRED")
+        assert (
+            error
+            == (str(_error) or repr(_error)))
+        assert (
+            list(run.log.error.call_args)
+            == [(error,), {}])
+    else:
+        assert not run.log.error.called
+
+    if raises:
+        assert result == 1
+    else:
+        assert result == run._runner.return_value
+
+
 def test_runner_constructor():
     run = runner.Runner("path1", "path2", "path3")
     assert run._args == ("path1", "path2", "path3")

tools/base/tests/test_utils.py

@@ -117,3 +117,24 @@ def test_util_coverage_with_data_file(patches):
     assert (
         list(m_config.return_value.write.call_args)
         == [(m_open.return_value.__enter__.return_value,), {}])
+
+
+def test_util_untar(patches):
+    patched = patches(
+        "tempfile.TemporaryDirectory",
+        "tarfile.open",
+        prefix="tools.base.utils")
+
+    with patched as (m_tmp, m_open):
+        with utils.untar("PATH") as tmpdir:
+            assert tmpdir == m_tmp.return_value.__enter__.return_value
+
+    assert (
+        list(m_tmp.call_args)
+        == [(), {}])
+    assert (
+        list(m_open.call_args)
+        == [('PATH',), {}])
+    assert (
+        list(m_open.return_value.__enter__.return_value.extractall.call_args)
+        == [(), {'path': tmpdir}])

tools/base/utils.py

@@ -4,6 +4,7 @@
 
 import io
 import os
+import tarfile
 import tempfile
 from configparser import ConfigParser
 from contextlib import ExitStack, contextmanager, redirect_stderr, redirect_stdout
@@ -69,3 +70,30 @@ def buffered(
     if stderr is not None:
         _stderr.seek(0)
         stderr.extend(mangle(_stderr.read().strip().split("\n")))
+
+
+@contextmanager
+def untar(tarball: str) -> Iterator[str]:
+    """Untar a tarball into a temporary directory
+
+    for example to list the contents of a tarball:
+
+    ```
+    import os
+
+    from tooling.base.utils import untar
+
+
+    with untar("path/to.tar") as tmpdir:
+        print(os.listdir(tmpdir))
+
+    ```
+
+    the created temp directory will be cleaned up on
+    exiting the contextmanager
+
+    """
+    with tempfile.TemporaryDirectory() as tmpdir:
+        with tarfile.open(tarball) as tarfiles:
+            tarfiles.extractall(path=tmpdir)
+            yield tmpdir

tools/git/BUILD

@@ -0,0 +1,14 @@
+load("@git_pip3//:requirements.bzl", "requirement")
+load("//bazel:envoy_build_system.bzl", "envoy_package")
+load("//tools/base:envoy_python.bzl", "envoy_py_library")
+
+licenses(["notice"])  # Apache 2
+
+envoy_package()
+
+envoy_py_library(
+    "tools.git.utils",
+    deps = [
+        requirement("gitpython"),
+    ],
+)

tools/git/requirements.txt

@@ -0,0 +1,18 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes tools/git/requirements.txt
+#
+gitdb==4.0.7 \
+    --hash=sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0 \
+    --hash=sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005
+    # via gitpython
+gitpython==3.1.18 \
+    --hash=sha256:b838a895977b45ab6f0cc926a9045c8d1c44e2b653c1fcc39fe91f42c6e8f05b \
+    --hash=sha256:fce760879cd2aebd2991b3542876dc5c4a909b30c9d69dfc488e504a8db37ee8
+    # via -r tools/git/requirements.txt
+smmap==4.0.0 \
+    --hash=sha256:7e65386bd122d45405ddf795637b7f7d2b532e7e401d46bbe3fb49b9986d5182 \
+    --hash=sha256:a9a7479e4c572e2e775c404dcd3080c8dc49f39918c2cf74913d30c4c478e3c2
+    # via gitdb

tools/git/tests/test_utils.py

@@ -0,0 +1,20 @@
+
+from tools.git import utils
+
+
+def test_util_git_repo(patches):
+    patched = patches(
+        "tempfile.TemporaryDirectory",
+        "Repo",
+        prefix="tools.git.utils")
+
+    with patched as (m_tmp, m_repo):
+        with utils.git_repo("URI") as tmpdir:
+            assert tmpdir == m_repo.clone_from.return_value
+
+    assert (
+        list(m_tmp.call_args)
+        == [(), {}])
+    assert (
+        list(m_repo.clone_from.call_args)
+        == [('URI', m_tmp.return_value.__enter__.return_value), {}])

tools/git/utils.py

@@ -0,0 +1,35 @@
+import tempfile
+from contextlib import contextmanager
+from typing import Iterator
+
+from git import Repo
+
+
+@contextmanager
+def git_repo(uri: str) -> Iterator[Repo]:
+    """Check out a git repository to a temporary directory
+
+    for example to add a file, commit and push it, it can be used as so:
+
+    ```python
+    import os
+
+    from tooling.git.utils import git_repo
+
+
+    with git_repo("git@github.com/envoyproxy/envoy") as repo:
+        filename = "foo.txt"
+
+        with open(os.path.join(repo.working_dir, filename), "w") as f:
+            f.write("bar")
+        repo.index.add([filename])
+        repo.index.commit(f"Added {filename}")
+        repo.remotes.origin.push()
+    ```
+
+    the temporary directory used to checkout the repo will be cleaned
+    up on in exiting the contextmanager.
+
+    """
+    with tempfile.TemporaryDirectory() as tmpdir:
+        yield Repo.clone_from(uri, tmpdir)