diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ee4c89391face..d750b52010f8d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -26,6 +26,11 @@ updates: schedule: interval: "daily" +- package-ecosystem: "pip" + directory: "/tools/git" + schedule: + interval: "daily" + - package-ecosystem: "pip" directory: "/tools/github" schedule: diff --git a/bazel/repositories_extra.bzl b/bazel/repositories_extra.bzl index 34a4c59d50f36..e7c34c4951cb1 100644 --- a/bazel/repositories_extra.bzl +++ b/bazel/repositories_extra.bzl @@ -52,6 +52,11 @@ def _python_deps(): requirements = "@envoy//tools/extensions:requirements.txt", extra_pip_args = ["--require-hashes"], ) + pip_install( + name = "git_pip3", + requirements = "@envoy//tools/git:requirements.txt", + extra_pip_args = ["--require-hashes"], + ) pip_install( name = "kafka_pip3", requirements = "@envoy//source/extensions/filters/network/kafka:requirements.txt", diff --git a/tools/base/runner.py b/tools/base/runner.py index b0749388bbd7e..58d542e02b5b1 100644 --- a/tools/base/runner.py +++ b/tools/base/runner.py @@ -7,12 +7,46 @@ import os import subprocess import sys -from functools import cached_property +from functools import cached_property, wraps +from typing import Callable, Tuple, Optional, Union LOG_LEVELS = (("debug", logging.DEBUG), ("info", logging.INFO), ("warn", logging.WARN), ("error", logging.ERROR)) +def catches(errors: Union[Tuple[Exception], Exception]) -> Callable: + """Method decorator to catch specified errors + + logs and returns 1 for sys.exit if error/s are caught + + can be used as so: + + ```python + + class MyRunner(runner.Runner): + + @runner.catches((MyError, MyOtherError)) + def run(self): + self.myrun() + ``` + + """ + + def wrapper(fun: Callable) -> Callable: + + @wraps(fun) + def wrapped(self, *args, **kwargs) -> Optional[int]: + try: + return fun(self, *args, **kwargs) + except errors as e: + self.log.error(str(e) or repr(e)) + return 1 + + return wrapped + + return wrapper + + class BazelRunError(Exception): pass diff --git a/tools/base/tests/test_runner.py b/tools/base/tests/test_runner.py index a187336a75690..c9d3018f1e6a8 100644 --- a/tools/base/tests/test_runner.py +++ b/tools/base/tests/test_runner.py @@ -25,6 +25,89 @@ def __init__(self): self.args = PropertyMock() +class Error1(Exception): + + def __str__(self): + return "" + + pass + + +class Error2(Exception): + pass + + +def _failing_runner(errors): + + class DummyFailingRunner(object): + + log = PropertyMock() + _runner = MagicMock() + + def __init__(self, raises=None): + self.raises = raises + + @runner.catches(errors) + def run(self, *args, **kwargs): + result = self._runner(*args, **kwargs) + if self.raises: + raise self.raises("AN ERROR OCCURRED") + return result + + return DummyFailingRunner + + +@pytest.mark.parametrize( + "errors", + [Error1, (Error1, Error2)]) +@pytest.mark.parametrize( + "raises", + [None, Error1, Error2]) +@pytest.mark.parametrize( + "args", + [(), ("ARG1", "ARG2")]) +@pytest.mark.parametrize( + "kwargs", + [{}, dict(key1="VAL1", key2="VAL2")]) +def test_catches(errors, raises, args, kwargs): + run = _failing_runner(errors)(raises) + should_fail = ( + raises + and not ( + raises == errors + or (isinstance(errors, tuple) + and raises in errors))) + + if should_fail: + result = 1 + with pytest.raises(raises): + run.run(*args, **kwargs) + else: + result = run.run(*args, **kwargs) + + assert ( + list(run._runner.call_args) + == [args, kwargs]) + + if not should_fail and raises: + assert result == 1 + error = run.log.error.call_args[0][0] + _error = raises("AN ERROR OCCURRED") + assert ( + error + == (str(_error) or repr(_error))) + assert ( + list(run.log.error.call_args) + == [(error,), {}]) + else: + assert not run.log.error.called + + if raises: + assert result == 1 + else: + assert result == run._runner.return_value + + def test_runner_constructor(): run = runner.Runner("path1", "path2", "path3") assert run._args == ("path1", "path2", "path3") diff --git a/tools/base/tests/test_utils.py b/tools/base/tests/test_utils.py index 4d48a9ac5d41f..43b14f33a59de 100644 --- a/tools/base/tests/test_utils.py +++ b/tools/base/tests/test_utils.py @@ -117,3 +117,24 @@ def test_util_coverage_with_data_file(patches): assert ( list(m_config.return_value.write.call_args) == [(m_open.return_value.__enter__.return_value,), {}]) + + +def test_util_untar(patches): + patched = patches( + "tempfile.TemporaryDirectory", + "tarfile.open", + prefix="tools.base.utils") + + with patched as (m_tmp, m_open): + with utils.untar("PATH") as tmpdir: + assert tmpdir == m_tmp.return_value.__enter__.return_value + + assert ( + list(m_tmp.call_args) + == [(), {}]) + assert ( + list(m_open.call_args) + == [('PATH',), {}]) + assert ( + list(m_open.return_value.__enter__.return_value.extractall.call_args) + == [(), {'path': tmpdir}]) diff --git a/tools/base/utils.py b/tools/base/utils.py index 68c3c57a927c6..3cf7409615896 100644 --- a/tools/base/utils.py +++ b/tools/base/utils.py @@ -4,6 +4,7 @@ import io import os +import tarfile import tempfile from configparser import ConfigParser from contextlib import ExitStack, contextmanager, redirect_stderr, redirect_stdout @@ -69,3 +70,30 @@ def buffered( if stderr is not None: _stderr.seek(0) stderr.extend(mangle(_stderr.read().strip().split("\n"))) + + +@contextmanager +def untar(tarball: str) -> Iterator[str]: + """Untar a tarball into a temporary directory + + for example to list the contents of a tarball: + + ``` + import os + + from tooling.base.utils import untar + + + with untar("path/to.tar") as tmpdir: + print(os.listdir(tmpdir)) + + ``` + + the created temp directory will be cleaned up on + exiting the contextmanager + + """ + with tempfile.TemporaryDirectory() as tmpdir: + with tarfile.open(tarball) as tarfiles: + tarfiles.extractall(path=tmpdir) + yield tmpdir diff --git a/tools/git/BUILD b/tools/git/BUILD new file mode 100644 index 0000000000000..d7baad83ad3f3 --- /dev/null +++ b/tools/git/BUILD @@ -0,0 +1,14 @@ +load("@git_pip3//:requirements.bzl", "requirement") +load("//bazel:envoy_build_system.bzl", "envoy_package") +load("//tools/base:envoy_python.bzl", "envoy_py_library") + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_py_library( + "tools.git.utils", + deps = [ + requirement("gitpython"), + ], +) diff --git a/tools/git/requirements.txt b/tools/git/requirements.txt new file mode 100644 index 0000000000000..886a35eec12ab --- /dev/null +++ b/tools/git/requirements.txt @@ -0,0 +1,18 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes tools/git/requirements.txt +# +gitdb==4.0.7 \ + --hash=sha256:6c4cc71933456991da20917998acbe6cf4fb41eeaab7d6d67fbc05ecd4c865b0 \ + --hash=sha256:96bf5c08b157a666fec41129e6d327235284cca4c81e92109260f353ba138005 + # via gitpython +gitpython==3.1.18 \ + --hash=sha256:b838a895977b45ab6f0cc926a9045c8d1c44e2b653c1fcc39fe91f42c6e8f05b \ + --hash=sha256:fce760879cd2aebd2991b3542876dc5c4a909b30c9d69dfc488e504a8db37ee8 + # via -r tools/git/requirements.txt +smmap==4.0.0 \ + --hash=sha256:7e65386bd122d45405ddf795637b7f7d2b532e7e401d46bbe3fb49b9986d5182 \ + --hash=sha256:a9a7479e4c572e2e775c404dcd3080c8dc49f39918c2cf74913d30c4c478e3c2 + # via gitdb diff --git a/tools/git/tests/test_utils.py b/tools/git/tests/test_utils.py new file mode 100644 index 0000000000000..51b4c2a2895bc --- /dev/null +++ b/tools/git/tests/test_utils.py @@ -0,0 +1,20 @@ + +from tools.git import utils + + +def test_util_git_repo(patches): + patched = patches( + "tempfile.TemporaryDirectory", + "Repo", + prefix="tools.git.utils") + + with patched as (m_tmp, m_repo): + with utils.git_repo("URI") as tmpdir: + assert tmpdir == m_repo.clone_from.return_value + + assert ( + list(m_tmp.call_args) + == [(), {}]) + assert ( + list(m_repo.clone_from.call_args) + == [('URI', m_tmp.return_value.__enter__.return_value), {}]) diff --git a/tools/git/utils.py b/tools/git/utils.py new file mode 100644 index 0000000000000..3ad1834192ff1 --- /dev/null +++ b/tools/git/utils.py @@ -0,0 +1,35 @@ +import tempfile +from contextlib import contextmanager +from typing import Iterator + +from git import Repo + + +@contextmanager +def git_repo(uri: str) -> Iterator[Repo]: + """Check out a git repository to a temporary directory + + for example to add a file, commit and push it, it can be used as so: + + ```python + import os + + from tooling.git.utils import git_repo + + + with git_repo("git@github.com/envoyproxy/envoy") as repo: + filename = "foo.txt" + + with open(os.path.join(repo.working_dir, filename), "w") as f: + f.write("bar") + repo.index.add([filename]) + repo.index.commit(f"Added {filename}") + repo.remotes.origin.push() + ``` + + the temporary directory used to checkout the repo will be cleaned + up on in exiting the contextmanager. + + """ + with tempfile.TemporaryDirectory() as tmpdir: + yield Repo.clone_from(uri, tmpdir)